wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
f8f071210378ad865963912d20edb1f7174466c3
llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073002.json
T
wuyang 26fc824409 research: audit AttnRes gradient scale study
2026-07-30 09:49:53 +08:00

8615 lines
250 KiB
JSON
Raw Blame History

{
"architecture": "baseline",
"batch_size": 32,
"canonical_sha256_without_self": "890c06c98d3b195e1aa79852477608280a7bcd106a01b463c2d793e8ec58d94f",
"depth": 32,
"diagnostics": [
{
"activation_grad_rms_by_block": [
0.00042041894630528986,
0.000374794501112774,
0.0003388339828234166,
0.000304876797599718,
0.0002806047268677503,
0.0002645689237397164,
0.00024906310136429965,
0.00023202023294288665,
0.00021741738601122051,
0.00020566635066643357,
0.0001967942516785115,
0.0001872276043286547,
0.00017897134239319712,
0.00017287296941503882,
0.00016580727242399007,
0.00016062518989201635,
0.0001559370430186391,
0.00015203763905446976,
0.00014800428471062332,
0.00014486612053588033,
0.00014247088984120637,
0.00013994764594826847,
0.00013763502647634596,
0.00013544136891141534,
0.00013363662583287805,
0.0001319239236181602,
0.00013088916603010148,
0.00012932090612594038,
0.00012785944272764027,
0.00012667682312894613,
0.0001251104404218495,
0.00012353518104646355
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00030814765159448143,
"first_to_last_ratio": 2.395816319369912,
"imbalance_abs_log_ratio": 0.8737240159497589,
"last_quartile_mean": 0.00012861906361649744,
"mean": 0.00019174550334355445,
"normalized": [
2.1925882952885543,
1.9546455826984732,
1.7671026277801187,
1.5900075479395406,
1.4634227242606306,
1.3797920635754504,
1.2989253829752037,
1.2100426288859099,
1.1338851875012121,
1.0726006455438846,
1.0263304653664347,
0.9764380445115058,
0.9333796061570758,
0.9015750899007978,
0.8647257408008662,
0.8376999047754503,
0.8132500647967916,
0.7929137132465669,
0.7718787774931073,
0.7555124788314888,
0.7430207611488979,
0.7298614231256383,
0.7178005433052705,
0.7063600791135226,
0.6969478997034861,
0.6880157360550423,
0.6826192205174408,
0.6744403590744623,
0.6668184676985713,
0.660650815377801,
0.6524817439796045,
0.644266378571199
],
"population_cv": 0.4034880502878489
},
"activation_output_rms_by_block": [
0.02817646972835064,
0.028377581387758255,
0.028678538277745247,
0.028959807008504868,
0.029224006459116936,
0.029430421069264412,
0.02957010455429554,
0.029960129410028458,
0.030406322330236435,
0.03066711500287056,
0.030792171135544777,
0.031063809990882874,
0.03129913657903671,
0.03160789608955383,
0.03180097043514252,
0.032092027366161346,
0.03248445317149162,
0.032903194427490234,
0.03316524624824524,
0.033593542873859406,
0.034045662730932236,
0.03455357626080513,
0.03491735830903053,
0.03521261364221573,
0.03560211881995201,
0.0359802283346653,
0.03664350509643555,
0.03706779330968857,
0.0375242717564106,
0.03805854171514511,
0.038408540189266205,
0.039021097123622894
],
"activation_output_statistics": {
"first_quartile_mean": 0.029047132236883044,
"first_to_last_ratio": 0.7789886319526242,
"imbalance_abs_log_ratio": 0.24975882634672159,
"last_quartile_mean": 0.03728826204314828,
"mean": 0.03285275783855468,
"normalized": [
0.8576591915605899,
0.863780798166523,
0.8729415782588962,
0.8815030735264118,
0.8895449996231614,
0.8958280219241153,
0.9000798258584322,
0.9119517319446604,
0.9255333290331198,
0.93347155674343,
0.9372781209682287,
0.945546493951495,
0.9527095634663979,
0.96210784631481,
0.9679848063720901,
0.9768442431490312,
0.9887892313676379,
1.0015352315073034,
1.0095117862319563,
1.0225486407851998,
1.0363106469855512,
1.051770948138011,
1.0628440534770849,
1.0718312847663467,
1.0836873724546434,
1.095196589323784,
1.1153859677933096,
1.128300810904444,
1.1421954875390588,
1.1584580479414466,
1.1691115972063533,
1.187757122716477
],
"population_cv": 0.09694557255182189
},
"bits_per_byte": 8.04236897991864,
"branch_output_rms_by_sublayer": [
0.0023512106854468584,
0.0027520405128598213,
0.0023248849902302027,
0.0026936428621411324,
0.0027838374953716993,
0.002722234698012471,
0.002502240240573883,
0.002735150745138526,
0.0026338943280279636,
0.002727407729253173,
0.0029348398093134165,
0.002726956969127059,
0.0029466808773577213,
0.002693725284188986,
0.0032394269946962595,
0.0027635409496724606,
0.003579996759071946,
0.0027838009409606457,
0.0033850702457129955,
0.0026989516336470842,
0.0034471398685127497,
0.002707457635551691,
0.0036693045403808355,
0.0027280766516923904,
0.0035536750219762325,
0.002806172939017415,
0.0038618813268840313,
0.0027198586612939835,
0.003969449549913406,
0.002747237915173173,
0.003774659475311637,
0.0027643220964819193,
0.004222253803163767,
0.002741035772487521,
0.004245356190949678,
0.00275977305136621,
0.004526515491306782,
0.0027434700168669224,
0.004249728750437498,
0.0026767924427986145,
0.004827029537409544,
0.002736816881224513,
0.004658437799662352,
0.0027651384007185698,
0.0048249042592942715,
0.0027371037285774946,
0.005304279271513224,
0.0027228749822825193,
0.005711541976779699,
0.002716122195124626,
0.005045780446380377,
0.002763758646324277,
0.005378549452871084,
0.0027021593414247036,
0.005505796521902084,
0.0027199271135032177,
0.005381250753998756,
0.0027382206171751022,
0.005084864795207977,
0.002702005673199892,
0.005992465652525425,
0.002712319139391184,
0.00573702622205019,
0.002734891138970852
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.010411069784541356,
0.009271735997530202,
0.009345779773643629,
0.008099773113238128,
0.008225801589217078,
0.008647498843153682,
0.007992995665779355,
0.0076659621960069,
0.007406648616870928,
0.006601475371863238,
0.006568536350729326,
0.006644454140273569,
0.005847021988988907,
0.006236102346072616,
0.006448411665711761,
0.005769055287848533,
0.005701484582581849,
0.005836584711958408,
0.005498613632501928,
0.004978853123018873,
0.005187902378445178,
0.005078997897134883,
0.005158829428828408,
0.005043411661395455,
0.00572819320620205,
0.004439465860302503,
0.004811331873512697,
0.004686191415610403,
0.004548827932369003,
0.00413026790671143,
0.004686908144290253,
0.004087931241888514
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00870757712038879,
"first_to_last_ratio": 1.8766776125890325,
"imbalance_abs_log_ratio": 0.6295029861071718,
"last_quartile_mean": 0.004639889697610857,
"mean": 0.006274566179006908,
"normalized": [
1.6592493389222875,
1.4776696480708191,
1.489470268862924,
1.2908897415629936,
1.3109753494573861,
1.3781827454599171,
1.2738722387727572,
1.2217517478188766,
1.1804240174646141,
1.0521006844983298,
1.0468510751716935,
1.0589503641708666,
0.9318607569319367,
0.9938698817038568,
1.027706375507919,
0.9194349255810409,
0.9086659411861104,
0.930197330850717,
0.8763336740154054,
0.7934975870804967,
0.8268145128188418,
0.8094580170542961,
0.8221810531042816,
0.8037865116905484,
0.9129225898305317,
0.707533514453289,
0.766799127820212,
0.7468550465352011,
0.7249629380893642,
0.6582555333514927,
0.7469692741422425,
0.6515081880187487
],
"population_cv": 0.26225105130829346
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.027978211641311646,
0.028061864897608757,
0.02817646972835064,
0.028305010870099068,
0.028377581387758255,
0.02853298746049404,
0.028678538277745247,
0.028806988149881363,
0.028959807008504868,
0.029115403071045876,
0.029224006459116936,
0.02931177243590355,
0.029430421069264412,
0.02947254478931427,
0.02957010455429554,
0.02984699048101902,
0.029960129410028458,
0.0302706528455019,
0.030406322330236435,
0.030570803210139275,
0.03066711500287056,
0.030674004927277565,
0.030792171135544777,
0.03094458393752575,
0.031063809990882874,
0.031205367296934128,
0.03129913657903671,
0.03147619590163231,
0.03160789608955383,
0.03167128562927246,
0.03180097043514252,
0.03195846453309059,
0.032092027366161346,
0.032371606677770615,
0.03248445317149162,
0.032751601189374924,
0.032903194427490234,
0.03311120718717575,
0.03316524624824524,
0.03350966051220894,
0.033593542873859406,
0.03394605591893196,
0.034045662730932236,
0.03444153815507889,
0.03455357626080513,
0.03483813256025314,
0.03491735830903053,
0.03510763868689537,
0.03521261364221573,
0.03553535044193268,
0.03560211881995201,
0.03590916842222214,
0.0359802283346653,
0.036534491926431656,
0.03664350509643555,
0.036974843591451645,
0.03706779330968857,
0.037423327565193176,
0.0375242717564106,
0.037971291691064835,
0.03805854171514511,
0.03832196444272995,
0.038408540189266205,
0.03889527916908264
],
"loss_nats": 5.574545383453369,
"loss_scale": 1.0,
"output_weights": null,
"step": 0,
"stream_state_rms_by_sublayer": [
0.028061864897608757,
0.02817646972835064,
0.028305010870099068,
0.028377581387758255,
0.02853298746049404,
0.028678538277745247,
0.028806988149881363,
0.028959807008504868,
0.029115403071045876,
0.029224006459116936,
0.02931177243590355,
0.029430421069264412,
0.02947254478931427,
0.02957010455429554,
0.02984699048101902,
0.029960129410028458,
0.0302706528455019,
0.030406322330236435,
0.030570803210139275,
0.03066711500287056,
0.030674004927277565,
0.030792171135544777,
0.03094458393752575,
0.031063809990882874,
0.031205367296934128,
0.03129913657903671,
0.03147619590163231,
0.03160789608955383,
0.03167128562927246,
0.03180097043514252,
0.03195846453309059,
0.032092027366161346,
0.032371606677770615,
0.03248445317149162,
0.032751601189374924,
0.032903194427490234,
0.03311120718717575,
0.03316524624824524,
0.03350966051220894,
0.033593542873859406,
0.03394605591893196,
0.034045662730932236,
0.03444153815507889,
0.03455357626080513,
0.03483813256025314,
0.03491735830903053,
0.03510763868689537,
0.03521261364221573,
0.03553535044193268,
0.03560211881995201,
0.03590916842222214,
0.0359802283346653,
0.036534491926431656,
0.03664350509643555,
0.036974843591451645,
0.03706779330968857,
0.037423327565193176,
0.0375242717564106,
0.037971291691064835,
0.03805854171514511,
0.03832196444272995,
0.038408540189266205,
0.03889527916908264,
0.039021097123622894
]
},
{
"activation_grad_rms_by_block": [
7.223192369565368e-05,
6.862939335405827e-05,
6.46814878564328e-05,
6.090753959142603e-05,
5.703999340767041e-05,
5.302598947309889e-05,
4.9269601731793955e-05,
4.545194315141998e-05,
4.1738676372915506e-05,
3.8814047002233565e-05,
3.584829028113745e-05,
3.317802838864736e-05,
3.07806913042441e-05,
2.884516288759187e-05,
2.6981158953276463e-05,
2.530650635890197e-05,
2.3913053155411035e-05,
2.265583498228807e-05,
2.164491161238402e-05,
2.069839865725953e-05,
1.9972700101789087e-05,
1.928487472468987e-05,
1.8700015061767772e-05,
1.8183285646955483e-05,
1.7775813830667175e-05,
1.7374133676639758e-05,
1.7024083717842586e-05,
1.673021324677393e-05,
1.646951204747893e-05,
1.624442847969476e-05,
1.604618228157051e-05,
1.58860220835777e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.890473403269425e-05,
"first_to_last_ratio": 3.528539860533838,
"imbalance_abs_log_ratio": 1.2608841481311845,
"last_quartile_mean": 1.669379867053067e-05,
"mean": 3.222918428491539e-05,
"normalized": [
2.2411961487173366,
2.1294176342582674,
2.0069228958644927,
1.8898256640002307,
1.7698242966194933,
1.64527867054697,
1.528726302727252,
1.4102728368677142,
1.2950584167422121,
1.2043136636380887,
1.1122928202038285,
1.029440524939878,
0.9550564802426835,
0.8950013327235407,
0.8371654310191393,
0.7852046808006393,
0.7419689230733446,
0.7029602357293278,
0.6715935290523237,
0.6422253344757239,
0.6197085202413003,
0.5983668266067783,
0.5802199303728628,
0.5641869644049918,
0.5515440190333021,
0.5390807760769664,
0.5282195033961989,
0.5191013554322059,
0.5110123762948401,
0.5040285331483811,
0.49787739397055714,
0.49290797877912856
],
"population_cv": 0.5370990514498413
},
"activation_output_rms_by_block": [
0.029305610805749893,
0.03049159236252308,
0.0324031226336956,
0.03488599509000778,
0.037802305072546005,
0.04105908423662186,
0.04460485652089119,
0.04878050833940506,
0.053197599947452545,
0.057943083345890045,
0.06266378611326218,
0.06765592098236084,
0.07297883927822113,
0.07823853194713593,
0.08444638550281525,
0.09052036702632904,
0.09696295112371445,
0.10418187081813812,
0.11156722903251648,
0.1197449341416359,
0.12838369607925415,
0.13761533796787262,
0.14720149338245392,
0.15857642889022827,
0.16927987337112427,
0.1816006898880005,
0.19378916919231415,
0.20805081725120544,
0.22281049191951752,
0.2388531118631363,
0.25510889291763306,
0.2763713300228119
],
"activation_output_statistics": {
"first_quartile_mean": 0.03741663438268006,
"first_to_last_ratio": 0.17145265067740043,
"imbalance_abs_log_ratio": 1.763448139840426,
"last_quartile_mean": 0.2182330470532179,
"mean": 0.11303362209582701,
"normalized": [
0.2592645467992289,
0.2697568369230271,
0.286668002253569,
0.30863378916082446,
0.3344341654423688,
0.36324664710658405,
0.39461582928906175,
0.4315575085973109,
0.47063518766436574,
0.5126181243379725,
0.554381828622084,
0.5985468724076087,
0.6456383324277757,
0.6921704400555022,
0.747090856127956,
0.8008269163451932,
0.8578239759627604,
0.9216892185390237,
0.9870269302520681,
1.0593744756769734,
1.135800956377508,
1.2174726016583441,
1.3022806015864932,
1.4029138052020595,
1.497606377928977,
1.6066077200821192,
1.714438284814271,
1.8406100184494245,
1.9711877562467606,
2.1131156149330748,
2.256929294023315,
2.445036484706394
],
"population_cv": 0.6334590447760624
},
"bits_per_byte": 5.11848367743279,
"branch_output_rms_by_sublayer": [
0.003032666863873601,
0.004902590066194534,
0.002639081096276641,
0.004850045777857304,
0.0032534960191696882,
0.005442064255475998,
0.0027798404917120934,
0.005710980389267206,
0.0031097980681806803,
0.006003146059811115,
0.0030368315055966377,
0.0059938253834843636,
0.0034559990745037794,
0.0061976369470357895,
0.0037613045424222946,
0.0062723769806325436,
0.004607585724443197,
0.0063410610891878605,
0.003909118939191103,
0.006816320586949587,
0.004674337804317474,
0.006481682416051626,
0.004910735413432121,
0.0065277558751404285,
0.00493645528331399,
0.006758649833500385,
0.004751654341816902,
0.006488988175988197,
0.006120393518358469,
0.006713794078677893,
0.005455858074128628,
0.00696563022211194,
0.005903614219278097,
0.007100075017660856,
0.006780208554118872,
0.007398899644613266,
0.007146476302295923,
0.007193166296929121,
0.007645847275853157,
0.00761415995657444,
0.00879710540175438,
0.007752604782581329,
0.00870877131819725,
0.007873632945120335,
0.008980454877018929,
0.008168715983629227,
0.011330032721161842,
0.008200661279261112,
0.010545967146754265,
0.007979212328791618,
0.011436101980507374,
0.008451862260699272,
0.01130830217152834,
0.008476903662085533,
0.013203900307416916,
0.008634857833385468,
0.01393965259194374,
0.009264753200113773,
0.014520598575472832,
0.009397287853062153,
0.015960348770022392,
0.00835501030087471,
0.02132205106317997,
0.009183173067867756
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00028524855645157025,
0.00022469127596424853,
0.00022670239393809648,
0.0001993403227560908,
0.0002100203582478305,
0.00019183315773228503,
0.00020704106188439305,
0.00020121077118127333,
0.00020796371696885172,
0.00017825031863398375,
0.00018469615204762195,
0.0001848551977445252,
0.00018214824531137669,
0.0001649843875278976,
0.0001966104199015976,
0.00016919176207710993,
0.0001681445657707544,
0.00017333427923610245,
0.0001624042419733422,
0.00015992743704390023,
0.00016485453322609178,
0.00015449118786750607,
0.00015016403044324942,
0.00015560032515501767,
0.00015156764536984935,
0.00014581309890539414,
0.000137484036700603,
0.0001500871171184727,
0.00014447415143049674,
0.00014218802742214665,
0.00014221421491139387,
0.00016697872359292516
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0002182609872694735,
"first_to_last_ratio": 1.4787241905811908,
"imbalance_abs_log_ratio": 0.3911796826204884,
"last_quartile_mean": 0.0001476008769314102,
"mean": 0.00017764111607924995,
"normalized": [
1.6057575112527105,
1.2648607536557492,
1.2761819951466593,
1.122151938446279,
1.1822733547459552,
1.0798916487706802,
1.1655019201299495,
1.1326813049942133,
1.170695847666675,
1.0034294006262718,
1.0397151072008834,
1.0406104274984227,
1.0253721060281789,
0.9287511313219521,
1.1067844215405909,
0.9524358201038854,
0.9465408092557868,
0.9757554054027329,
0.9142266472863736,
0.9002839014620511,
0.9280201389443322,
0.869681475084765,
0.8453224892837178,
0.8759251719945322,
0.8532238972323919,
0.8208296712138615,
0.7739426532271284,
0.8448895189980412,
0.8132922975221759,
0.8004229570293466,
0.8005703749797919,
0.9399779019539145
],
"population_cv": 0.17622384112301237
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.028563687577843666,
0.028535494580864906,
0.029305610805749893,
0.029406530782580376,
0.03049159236252308,
0.030669530853629112,
0.0324031226336956,
0.03261565789580345,
0.03488599509000778,
0.035065844655036926,
0.037802305072546005,
0.03803478553891182,
0.04105908423662186,
0.041278183460235596,
0.04460485652089119,
0.04512108862400055,
0.04878050833940506,
0.04953810200095177,
0.053197599947452545,
0.05369982123374939,
0.057943083345890045,
0.0585285909473896,
0.06266378611326218,
0.06350461393594742,
0.06765592098236084,
0.06854932755231857,
0.07297883927822113,
0.07389408349990845,
0.07823853194713593,
0.07984820008277893,
0.08444638550281525,
0.08570227771997452,
0.09052036702632904,
0.09210614860057831,
0.09696295112371445,
0.09899988770484924,
0.10418187081813812,
0.10663887858390808,
0.11156722903251648,
0.11457803845405579,
0.1197449341416359,
0.12300471216440201,
0.12838369607925415,
0.13199855387210846,
0.13761533796787262,
0.14150823652744293,
0.14720149338245392,
0.15258266031742096,
0.15857642889022827,
0.16335271298885345,
0.16927987337112427,
0.17534348368644714,
0.1816006898880005,
0.1876170039176941,
0.19378916919231415,
0.2015211284160614,
0.20805081725120544,
0.21581925451755524,
0.22281049191951752,
0.23164939880371094,
0.2388531118631363,
0.24887791275978088,
0.25510889291763306,
0.26933422684669495
],
"loss_nats": 3.5478625297546387,
"loss_scale": 1.0,
"output_weights": null,
"step": 100,
"stream_state_rms_by_sublayer": [
0.028535494580864906,
0.029305610805749893,
0.029406530782580376,
0.03049159236252308,
0.030669530853629112,
0.0324031226336956,
0.03261565789580345,
0.03488599509000778,
0.035065844655036926,
0.037802305072546005,
0.03803478553891182,
0.04105908423662186,
0.041278183460235596,
0.04460485652089119,
0.04512108862400055,
0.04878050833940506,
0.04953810200095177,
0.053197599947452545,
0.05369982123374939,
0.057943083345890045,
0.0585285909473896,
0.06266378611326218,
0.06350461393594742,
0.06765592098236084,
0.06854932755231857,
0.07297883927822113,
0.07389408349990845,
0.07823853194713593,
0.07984820008277893,
0.08444638550281525,
0.08570227771997452,
0.09052036702632904,
0.09210614860057831,
0.09696295112371445,
0.09899988770484924,
0.10418187081813812,
0.10663887858390808,
0.11156722903251648,
0.11457803845405579,
0.1197449341416359,
0.12300471216440201,
0.12838369607925415,
0.13199855387210846,
0.13761533796787262,
0.14150823652744293,
0.14720149338245392,
0.15258266031742096,
0.15857642889022827,
0.16335271298885345,
0.16927987337112427,
0.17534348368644714,
0.1816006898880005,
0.1876170039176941,
0.19378916919231415,
0.2015211284160614,
0.20805081725120544,
0.21581925451755524,
0.22281049191951752,
0.23164939880371094,
0.2388531118631363,
0.24887791275978088,
0.25510889291763306,
0.26933422684669495,
0.2763713300228119
]
},
{
"activation_grad_rms_by_block": [
0.00014093925710767508,
0.00014282441406976432,
0.00014458753867074847,
0.0001413008722011,
0.00013349838263820857,
0.00012304198753554374,
0.00011454652121756226,
0.00010784466576296836,
0.0001000517513602972,
9.189492993755266e-05,
8.563220035284758e-05,
8.04013543529436e-05,
7.555109914392233e-05,
7.068048580549657e-05,
6.581390334758908e-05,
6.151734851300716e-05,
5.7112174545181915e-05,
5.2947088988730684e-05,
4.8879119276534766e-05,
4.527245255303569e-05,
4.1345181671204045e-05,
3.832591755781323e-05,
3.484322223812342e-05,
3.232451854273677e-05,
3.010137334058527e-05,
2.8228545488673262e-05,
2.6716850697994232e-05,
2.5478970201220363e-05,
2.4533494070055895e-05,
2.4085495169856586e-05,
2.3973674615263008e-05,
2.415898779872805e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00013107295490044635,
"first_to_last_ratio": 5.058842318548806,
"imbalance_abs_log_ratio": 1.621137666322135,
"last_quartile_mean": 2.5909673922797083e-05,
"mean": 6.995168058665513e-05,
"normalized": [
2.01480873548251,
2.041758151797876,
2.06696304446375,
2.019978233776078,
1.9084371028489115,
1.7589568471213655,
1.6375092100276802,
1.5417022844700914,
1.430298036032952,
1.3136915248764403,
1.2241621592889058,
1.14938417031087,
1.0800469482692516,
1.010418694915251,
0.9408480653450457,
0.8794263125215463,
0.8164517859500485,
0.756909462999114,
0.6987554675828556,
0.6471960669615769,
0.591053443240527,
0.5478913049177658,
0.49810414769034944,
0.46209781197027316,
0.4303166569857621,
0.4035434924784137,
0.38193293533380923,
0.3642367129358298,
0.3507205811826659,
0.34431617607842635,
0.34271763614835193,
0.34536679599570513
],
"population_cv": 0.5909500348424317
},
"activation_output_rms_by_block": [
0.04959565028548241,
0.05870900675654411,
0.06652694195508957,
0.07286322116851807,
0.07801168411970139,
0.08261121064424515,
0.08867742121219635,
0.09177010506391525,
0.09494514763355255,
0.0981190949678421,
0.10163456201553345,
0.10214314609766006,
0.10418473184108734,
0.10623370856046677,
0.10849270969629288,
0.11100933700799942,
0.11405495554208755,
0.11736668646335602,
0.12177617102861404,
0.12738163769245148,
0.13297238945960999,
0.14011575281620026,
0.15017743408679962,
0.16121913492679596,
0.17427310347557068,
0.189283087849617,
0.2044389545917511,
0.22237628698349,
0.24115440249443054,
0.26191622018814087,
0.2883519232273102,
0.3199674189090729
],
"activation_output_statistics": {
"first_quartile_mean": 0.07359565515071154,
"first_to_last_ratio": 0.309589437408787,
"imbalance_abs_log_ratio": 1.1725082547452423,
"last_quartile_mean": 0.2377201747149229,
"mean": 0.13694853871129453,
"normalized": [
0.3621480795062482,
0.4286939262660582,
0.48578059014807806,
0.5320481828734462,
0.5696423258970309,
0.6032281280372068,
0.6475236760221296,
0.6701064934864231,
0.6932906953736058,
0.7164668998381211,
0.7421368856647125,
0.745850573206854,
0.7607582586968847,
0.775719913188861,
0.7922151686847111,
0.8105916138471667,
0.8328307597537082,
0.8570130617514684,
0.8892111750482725,
0.9301423650893323,
0.9709661067646236,
1.0231270383365143,
1.0965975650415178,
1.177224207310945,
1.2725444544024032,
1.3821475543353592,
1.4928158892059098,
1.6237945222058072,
1.7609125644108963,
1.9125156255978355,
2.105549470923482,
2.336406229084387
],
"population_cv": 0.48939845088217687
},
"bits_per_byte": 3.103317553853433,
"branch_output_rms_by_sublayer": [
0.011270907707512379,
0.01699911244213581,
0.012035840190947056,
0.01337367296218872,
0.010826056823134422,
0.010907532647252083,
0.01394603680819273,
0.008780849166214466,
0.015218054875731468,
0.009724137373268604,
0.015542514622211456,
0.009259181097149849,
0.025958625599741936,
0.009840037673711777,
0.03120059333741665,
0.009063811041414738,
0.014079542830586433,
0.008434661664068699,
0.01716585084795952,
0.009029604494571686,
0.021518073976039886,
0.01014146488159895,
0.030795104801654816,
0.00910214427858591,
0.018123170360922813,
0.009148034267127514,
0.016974516212940216,
0.00891866721212864,
0.01868187077343464,
0.00946173444390297,
0.017949005588889122,
0.00928061455488205,
0.021634839475154877,
0.01075725443661213,
0.02261096052825451,
0.010938199236989021,
0.017853887751698494,
0.012016859836876392,
0.016204316169023514,
0.012549139559268951,
0.019116709008812904,
0.014130656607449055,
0.021943919360637665,
0.015834644436836243,
0.015276721678674221,
0.01847725547850132,
0.01698230765759945,
0.019000399857759476,
0.017433995380997658,
0.019877459853887558,
0.017341874539852142,
0.022254064679145813,
0.018439622595906258,
0.023488705977797508,
0.013915229588747025,
0.02547476813197136,
0.008195439353585243,
0.027127297595143318,
0.007334497291594744,
0.02754027582705021,
0.014406879432499409,
0.029805582016706467,
0.02094510942697525,
0.03139834478497505
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0005239687790313407,
0.0004396407496139251,
0.0004479555677531567,
0.000484081397596248,
0.0004298002646761996,
0.0003778969963663625,
0.0003311985651517251,
0.0003381159607123189,
0.00027120323619484496,
0.00025249768611013433,
0.0002492173720055997,
0.0002512649978189958,
0.00021872457711399188,
0.0001974354513181695,
0.0001883229322305711,
0.00018548264077727127,
0.00019625685908145198,
0.00017137558371316787,
0.00015305597823747372,
0.00014260825992090635,
0.00013921721401447933,
0.00013266258711253505,
0.00012717821667718618,
0.00011256530678442586,
0.0001087245205863228,
0.00010436458537579927,
0.00010507936295752309,
9.703474953173527e-05,
9.385369360633854e-05,
9.122967598255955e-05,
0.00010232602359400015,
0.0001188300325189452
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0004215822851126596,
"first_to_last_ratio": 4.105774523524949,
"imbalance_abs_log_ratio": 1.4123944031885163,
"last_quartile_mean": 0.00010268033051915298,
"mean": 0.00022447405700517828,
"normalized": [
2.3342063934775927,
1.9585370152764836,
1.995578347580821,
2.156513781835738,
1.9146990543601428,
1.6834773755509973,
1.475442505786247,
1.506258519239571,
1.2081718476206305,
1.1248412822347198,
1.1102279494144423,
1.11934983120655,
0.9743868847567816,
0.8795468570054622,
0.8389518804225415,
0.8262987859349487,
0.874296396206376,
0.7634538529733728,
0.6818426159328645,
0.6352995166725061,
0.6201928894227087,
0.5909929587519093,
0.5665608684314569,
0.5014624330589309,
0.48435227677029374,
0.4649293854629791,
0.4681136179362547,
0.4322760092026885,
0.41810485745430026,
0.4064152321194739,
0.45584788264257925,
0.5293708952576376
],
"population_cv": 0.5698198347315865
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.044032808393239975,
0.044310975819826126,
0.04959565028548241,
0.052685707807540894,
0.05870900675654411,
0.061390094459056854,
0.06652694195508957,
0.0693279355764389,
0.07286322116851807,
0.07403621077537537,
0.07801168411970139,
0.07902701944112778,
0.08261121064424515,
0.08570802956819534,
0.08867742121219635,
0.08839071542024612,
0.09177010506391525,
0.0920921340584755,
0.09494514763355255,
0.09543459117412567,
0.0981190949678421,
0.09975367039442062,
0.10163456201553345,
0.09967570006847382,
0.10214314609766006,
0.10169660300016403,
0.10418473184108734,
0.1038530021905899,
0.10623370856046677,
0.10576826333999634,
0.10849270969629288,
0.10825389623641968,
0.11100933700799942,
0.11050844192504883,
0.11405495554208755,
0.11365467309951782,
0.11736668646335602,
0.117034412920475,
0.12177617102861404,
0.12205220758914948,
0.12738163769245148,
0.12672542035579681,
0.13297238945960999,
0.13195016980171204,
0.14011575281620026,
0.13969174027442932,
0.15017743408679962,
0.15003392100334167,
0.16121913492679596,
0.16189023852348328,
0.17427310347557068,
0.1747252345085144,
0.189283087849617,
0.18889831006526947,
0.2044389545917511,
0.20492415130138397,
0.22237628698349,
0.22264817357063293,
0.24115440249443054,
0.2428991198539734,
0.26191622018814087,
0.267524778842926,
0.2883519232273102,
0.2990366220474243
],
"loss_nats": 2.1510558128356934,
"loss_scale": 1.0,
"output_weights": null,
"step": 500,
"stream_state_rms_by_sublayer": [
0.044310975819826126,
0.04959565028548241,
0.052685707807540894,
0.05870900675654411,
0.061390094459056854,
0.06652694195508957,
0.0693279355764389,
0.07286322116851807,
0.07403621077537537,
0.07801168411970139,
0.07902701944112778,
0.08261121064424515,
0.08570802956819534,
0.08867742121219635,
0.08839071542024612,
0.09177010506391525,
0.0920921340584755,
0.09494514763355255,
0.09543459117412567,
0.0981190949678421,
0.09975367039442062,
0.10163456201553345,
0.09967570006847382,
0.10214314609766006,
0.10169660300016403,
0.10418473184108734,
0.1038530021905899,
0.10623370856046677,
0.10576826333999634,
0.10849270969629288,
0.10825389623641968,
0.11100933700799942,
0.11050844192504883,
0.11405495554208755,
0.11365467309951782,
0.11736668646335602,
0.117034412920475,
0.12177617102861404,
0.12205220758914948,
0.12738163769245148,
0.12672542035579681,
0.13297238945960999,
0.13195016980171204,
0.14011575281620026,
0.13969174027442932,
0.15017743408679962,
0.15003392100334167,
0.16121913492679596,
0.16189023852348328,
0.17427310347557068,
0.1747252345085144,
0.189283087849617,
0.18889831006526947,
0.2044389545917511,
0.20492415130138397,
0.22237628698349,
0.22264817357063293,
0.24115440249443054,
0.2428991198539734,
0.26191622018814087,
0.267524778842926,
0.2883519232273102,
0.2990366220474243,
0.3199674189090729
]
},
{
"activation_grad_rms_by_block": [
0.00010639085667207837,
0.00010674773511709645,
0.000114928770926781,
0.00011333057773299515,
0.00011114918743260205,
0.00010892991122091189,
0.00010597817890811712,
0.00010435423610033467,
0.00010266973549732938,
0.0001015385496430099,
0.00010008511890191585,
9.485893679084256e-05,
9.186805982608348e-05,
8.913136116461828e-05,
8.655320561956614e-05,
8.490345499012619e-05,
8.653065015096217e-05,
8.836571214487776e-05,
8.356331090908498e-05,
8.248459926107898e-05,
7.508730777772143e-05,
7.080488285282627e-05,
6.500465678982437e-05,
6.0319798649288714e-05,
5.538603363675065e-05,
5.071960185887292e-05,
4.5680528273805976e-05,
4.0155850001610816e-05,
3.449051291681826e-05,
2.892977681767661e-05,
2.4221473722718656e-05,
2.1527655917452648e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00010897618176386459,
"first_to_last_ratio": 2.8953050536910427,
"imbalance_abs_log_ratio": 1.0630904780785062,
"last_quartile_mean": 3.763892914321332e-05,
"mean": 7.927156963205562e-05,
"normalized": [
1.3421060938480058,
1.3466080665814153,
1.4498107134781197,
1.429649725103549,
1.4021317850587363,
1.3741359194288378,
1.3369002203440912,
1.3164144041136308,
1.29516465958575,
1.2808949048733107,
1.262560075023968,
1.1966325030668217,
1.1589029995557718,
1.1243799205481557,
1.0918568412522767,
1.0710454628842516,
1.0915723071032915,
1.1147213629682524,
1.054139728744489,
1.0405319289618822,
0.9472161094607345,
0.8931939052231708,
0.8200248473891447,
0.760926003222431,
0.6986872329364575,
0.6398208348124227,
0.5762536112989216,
0.5065605511281904,
0.4350930992903045,
0.3649451745683369,
0.305550575510795,
0.2715684326344833
],
"population_cv": 0.35248901334980626
},
"activation_output_rms_by_block": [
0.09771591424942017,
0.13586144149303436,
0.1991395801305771,
0.2241763025522232,
0.22696681320667267,
0.22763192653656006,
0.22912801802158356,
0.2290782928466797,
0.22818906605243683,
0.22590669989585876,
0.22214065492153168,
0.21565376222133636,
0.20698806643486023,
0.20169880986213684,
0.1979832947254181,
0.1976672261953354,
0.2011667788028717,
0.20515663921833038,
0.205980584025383,
0.20982402563095093,
0.2095191776752472,
0.2096652388572693,
0.2105090171098709,
0.2148621529340744,
0.2245095670223236,
0.2386746108531952,
0.25724881887435913,
0.2833212912082672,
0.3200184106826782,
0.3660464584827423,
0.43435168266296387,
0.5434449911117554
],
"activation_output_statistics": {
"first_quartile_mean": 0.19621228612959385,
"first_to_last_ratio": 0.5884274155428802,
"imbalance_abs_log_ratio": 0.5303016979635413,
"last_quartile_mean": 0.3334519788622856,
"mean": 0.23750704107806087,
"normalized": [
0.4114232311003534,
0.5720312159014314,
0.8384575852011324,
0.9438722386278381,
0.9556214088494162,
0.9584218030055994,
0.9647209488255828,
0.9645115858750019,
0.960767584054228,
0.951157906184392,
0.9353013448073788,
0.9079889221071891,
0.8715028636428086,
0.8492329698800171,
0.8335891594066359,
0.8322583839961553,
0.8469929055145556,
0.8637918197587332,
0.8672609582032722,
0.8834433904719,
0.8821598581845203,
0.8827748344031583,
0.8863274796164187,
0.9046559291834054,
0.9452754158498172,
1.0049159375226715,
1.0831208106786603,
1.1928963870807885,
1.3474059936500937,
1.5412025547589332,
1.8287949725254948,
2.288121601132417
],
"population_cv": 0.33570692978684213
},
"bits_per_byte": 2.0753701983577404,
"branch_output_rms_by_sublayer": [
0.017155639827251434,
0.059614092111587524,
0.05783598870038986,
0.017162656411528587,
0.08590122312307358,
0.003484122920781374,
0.037979111075401306,
0.0025395466946065426,
0.02590934932231903,
0.004171412438154221,
0.027761045843362808,
0.005880473647266626,
0.021641533821821213,
0.0071118478663265705,
0.01924360916018486,
0.00746425986289978,
0.022704657167196274,
0.00924856960773468,
0.018269814550876617,
0.010196910239756107,
0.022024810314178467,
0.012587273493409157,
0.03360125049948692,
0.015558412298560143,
0.026457328349351883,
0.018742306157946587,
0.024426763877272606,
0.01840486004948616,
0.02595876343548298,
0.019045589491724968,
0.023261789232492447,
0.01923760399222374,
0.0244763121008873,
0.01806296780705452,
0.0326010137796402,
0.017410961911082268,
0.037907715886831284,
0.019881773740053177,
0.034735601395368576,
0.020177578553557396,
0.04502493515610695,
0.02574281580746174,
0.034673795104026794,
0.027684781700372696,
0.0429018959403038,
0.03708433732390404,
0.03208737075328827,
0.0411839634180069,
0.027529660612344742,
0.04565756395459175,
0.02247321978211403,
0.048749130219221115,
0.01957971602678299,
0.05322333425283432,
0.02326148748397827,
0.06035253033041954,
0.014175337739288807,
0.07119441032409668,
0.011726614087820053,
0.07984230667352676,
0.013617267832159996,
0.10812961310148239,
0.01758233644068241,
0.16684016585350037
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00037331716215226093,
0.0002052857320094383,
0.00022573696371192266,
0.00015002318346068796,
0.00011232796492243865,
0.00011469567077281986,
0.0001328622706326584,
0.00011987742098506258,
0.00012268315232766492,
0.00013855849890850403,
0.00016581239426236876,
0.00020970093409220108,
0.00019415593885306704,
0.0001873084275669645,
0.00019997003245353254,
0.00019081336552079618,
0.00019544152058884818,
0.0002331889851483562,
0.00028258169221688257,
0.00026717547757069314,
0.00031847213451059053,
0.00028331189819324093,
0.0003090653888381074,
0.000275741819401602,
0.000255221776165695,
0.00023469865916502512,
0.00021558263955965886,
0.0002009260839676739,
0.00018496297858040776,
0.0001671377375581853,
0.00015254274522576858,
0.00014770686125523138
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00017926579608091117,
"first_to_last_ratio": 0.9200315924660546,
"imbalance_abs_log_ratio": 0.08334726989162704,
"last_quartile_mean": 0.00019484743518470574,
"mean": 0.0002052152347055736,
"normalized": [
1.8191493564689118,
1.0003435286077362,
1.1000010015620527,
0.7310528561679606,
0.5473665982138111,
0.5589042691561181,
0.6474288852057137,
0.584154588508271,
0.5978267281358565,
0.6751862214678978,
0.8079926156567426,
1.021858510617222,
0.946108797096017,
0.9127413363617942,
0.9744404831368078,
0.9298206626547972,
0.9523733502010805,
1.1363141994936055,
1.377001530233894,
1.301928085183418,
1.5518932352537513,
1.3805597747151384,
1.5060547979370522,
1.3436712912528852,
1.2436785043365166,
1.1436707391717382,
1.0505196647264496,
0.9790992576937408,
0.9013121216160088,
0.8144509241625318,
0.7433305107421713,
0.7197655742623075
],
"population_cv": 0.31225216605899336
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.05658925697207451,
0.0616849884390831,
0.09771591424942017,
0.1262788027524948,
0.13586144149303436,
0.19808362424373627,
0.1991395801305771,
0.22383953630924225,
0.2241763025522232,
0.22702015936374664,
0.22696681320667267,
0.227730393409729,
0.22763192653656006,
0.22866052389144897,
0.22912801802158356,
0.22953395545482635,
0.2290782928466797,
0.22848571836948395,
0.22818906605243683,
0.22744755446910858,
0.22590669989585876,
0.22389201819896698,
0.22214065492153168,
0.21693368256092072,
0.21565376222133636,
0.209661602973938,
0.20698806643486023,
0.20248256623744965,
0.20169880986213684,
0.19645103812217712,
0.1979832947254181,
0.1954326629638672,
0.1976672261953354,
0.19818583130836487,
0.2011667788028717,
0.20395968854427338,
0.20515663921833038,
0.2042165845632553,
0.205980584025383,
0.20741206407546997,
0.20982402563095093,
0.20696893334388733,
0.2095191776752472,
0.20577766001224518,
0.2096652388572693,
0.20420841872692108,
0.2105090171098709,
0.20596569776535034,
0.2148621529340744,
0.21317537128925323,
0.2245095670223236,
0.22439339756965637,
0.2386746108531952,
0.23865793645381927,
0.25724881887435913,
0.25777244567871094,
0.2833212912082672,
0.28324905037879944,
0.3200184106826782,
0.3202260434627533,
0.3660464584827423,
0.366729199886322,
0.43435168266296387,
0.4362008571624756
],
"loss_nats": 1.4385370016098022,
"loss_scale": 1.0,
"output_weights": null,
"step": 2000,
"stream_state_rms_by_sublayer": [
0.0616849884390831,
0.09771591424942017,
0.1262788027524948,
0.13586144149303436,
0.19808362424373627,
0.1991395801305771,
0.22383953630924225,
0.2241763025522232,
0.22702015936374664,
0.22696681320667267,
0.227730393409729,
0.22763192653656006,
0.22866052389144897,
0.22912801802158356,
0.22953395545482635,
0.2290782928466797,
0.22848571836948395,
0.22818906605243683,
0.22744755446910858,
0.22590669989585876,
0.22389201819896698,
0.22214065492153168,
0.21693368256092072,
0.21565376222133636,
0.209661602973938,
0.20698806643486023,
0.20248256623744965,
0.20169880986213684,
0.19645103812217712,
0.1979832947254181,
0.1954326629638672,
0.1976672261953354,
0.19818583130836487,
0.2011667788028717,
0.20395968854427338,
0.20515663921833038,
0.2042165845632553,
0.205980584025383,
0.20741206407546997,
0.20982402563095093,
0.20696893334388733,
0.2095191776752472,
0.20577766001224518,
0.2096652388572693,
0.20420841872692108,
0.2105090171098709,
0.20596569776535034,
0.2148621529340744,
0.21317537128925323,
0.2245095670223236,
0.22439339756965637,
0.2386746108531952,
0.23865793645381927,
0.25724881887435913,
0.25777244567871094,
0.2833212912082672,
0.28324905037879944,
0.3200184106826782,
0.3202260434627533,
0.3660464584827423,
0.366729199886322,
0.43435168266296387,
0.4362008571624756,
0.5434449911117554
]
},
{
"activation_grad_rms_by_block": [
0.0001095466359402053,
0.00011157916014781222,
0.0001209566617035307,
0.00011857408389914781,
0.00011687575170071796,
0.00011450610327301547,
0.00011203528265468776,
0.00011082263517891988,
0.00010936566832242534,
0.00010819115414051339,
0.00010705165914259851,
0.00010176753858104348,
9.876617696136236e-05,
9.541869076201692e-05,
9.238059283234179e-05,
9.015990508487448e-05,
9.102027252083644e-05,
9.192643483402207e-05,
8.731557318242267e-05,
8.749608969083056e-05,
7.95004380051978e-05,
7.466241368092597e-05,
6.735928764101118e-05,
6.187916733324528e-05,
5.64747242606245e-05,
5.17304906679783e-05,
4.670304406317882e-05,
4.1035571484826505e-05,
3.507353176246397e-05,
2.8758000553352758e-05,
2.3154751033871435e-05,
1.9675369912874885e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00011436203931225464,
"first_to_last_ratio": 3.02339634825199,
"imbalance_abs_log_ratio": 1.1063808181034647,
"last_quartile_mean": 3.78256854673964e-05,
"mean": 8.318008940477739e-05,
"normalized": [
1.3169814642434559,
1.341416689333394,
1.45415402374626,
1.4255104165869963,
1.4050928838507029,
1.376604715051423,
1.3469002432720767,
1.3323216646188754,
1.3148058520377592,
1.300685715953312,
1.286986584273256,
1.223460317358093,
1.1873776244786027,
1.1471337845969738,
1.1106094438392846,
1.0839120963928204,
1.0942555264386233,
1.1051495074341955,
1.0497172316985723,
1.0518874206187772,
0.955762834280288,
0.8975995844101339,
0.8098006160250947,
0.743918019036071,
0.6789452219244779,
0.6219095343387212,
0.5614690293960716,
0.49333406321717244,
0.4216577790844446,
0.34573178219860223,
0.2783689125554313,
0.23653941771003728
],
"population_cv": 0.36398106608216985
},
"activation_output_rms_by_block": [
0.11107808351516724,
0.1828140765428543,
0.2578570544719696,
0.27254435420036316,
0.2737041115760803,
0.2742544412612915,
0.27536308765411377,
0.2744285464286804,
0.2742423117160797,
0.27115195989608765,
0.26583269238471985,
0.255132257938385,
0.23856717348098755,
0.22743643820285797,
0.22118839621543884,
0.22011704742908478,
0.22253327071666718,
0.2256278693675995,
0.22584785521030426,
0.22925184667110443,
0.23080435395240784,
0.23398378491401672,
0.24050307273864746,
0.24922524392604828,
0.2630935311317444,
0.27895617485046387,
0.2995731830596924,
0.3278428912162781,
0.36820048093795776,
0.42407870292663574,
0.5123529434204102,
0.6704596281051636
],
"activation_output_statistics": {
"first_quartile_mean": 0.24025546945631504,
"first_to_last_ratio": 0.6112286812568156,
"imbalance_abs_log_ratio": 0.4922841160907453,
"last_quartile_mean": 0.39306969195604324,
"mean": 0.2780639645643532,
"normalized": [
0.39946953820209985,
0.6574533195240588,
0.9273299935716515,
0.9801498537480837,
0.9843206832100536,
0.9862998310153919,
0.9902868503134846,
0.9869259645298935,
0.9862562095945767,
0.975142393301143,
0.9560127390156568,
0.9175308218672072,
0.8579578941656613,
0.817928488357655,
0.7954586872196038,
0.7916058011110692,
0.8002952524442108,
0.8114243415938267,
0.8122154755440653,
0.824457232458282,
0.8300405063777765,
0.8414746775282531,
0.8649199586701105,
0.896287457875072,
0.9461619075450383,
1.0032086512450777,
1.077353491413524,
1.179019696888499,
1.3241574884211362,
1.5251120496359385,
1.842572244926166,
2.411170498685733
],
"population_cv": 0.35267348500337414
},
"bits_per_byte": 1.9134731260894795,
"branch_output_rms_by_sublayer": [
0.024391189217567444,
0.06930837035179138,
0.10445628315210342,
0.019052622839808464,
0.09584712982177734,
0.0026203703600913286,
0.031347230076789856,
0.0018832145724445581,
0.020275620743632317,
0.002480348339304328,
0.017658807337284088,
0.005030750762671232,
0.020567260682582855,
0.0064065041951835155,
0.016946211457252502,
0.006859246641397476,
0.017460251227021217,
0.009737920947372913,
0.022223282605409622,
0.009925507940351963,
0.02365109510719776,
0.012353206984698772,
0.04038562253117561,
0.01845649816095829,
0.03250120207667351,
0.023668669164180756,
0.029116366058588028,
0.023308098316192627,
0.029388509690761566,
0.023618459701538086,
0.026225870475172997,
0.024062534794211388,
0.02648484893143177,
0.02197168581187725,
0.03580358996987343,
0.021145176142454147,
0.04329143092036247,
0.025955181568861008,
0.04141230136156082,
0.02680986560881138,
0.05275771766901016,
0.035374872386455536,
0.044423554092645645,
0.038993529975414276,
0.05167620629072189,
0.05052104964852333,
0.03969942033290863,
0.05720135569572449,
0.0389031358063221,
0.06300201267004013,
0.026857485994696617,
0.06565802544355392,
0.030672859400510788,
0.0703272894024849,
0.02773989364504814,
0.07888873666524887,
0.019402364268898964,
0.09176591783761978,
0.018159963190555573,
0.10585808008909225,
0.02488313429057598,
0.1433597356081009,
0.024982891976833344,
0.2408120483160019
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00037295757895409754,
0.00022454763424927058,
0.00021936409959100382,
0.00010566073392567048,
9.515815728211845e-05,
0.00010551160558417538,
0.00011265453132986426,
0.00010566938037237213,
0.00012588745811984293,
0.0001281493774985002,
0.0001672472035341268,
0.0002455532725992925,
0.00022493633083964273,
0.0002220131528127045,
0.0002449305149794571,
0.0002379224969510024,
0.00024418235837095776,
0.00027770634611442576,
0.000335912973129177,
0.00033241348484404164,
0.00039774019686300875,
0.00036753844348542204,
0.0003869475308616486,
0.00034605021914329437,
0.00032032390763334626,
0.00028824702486555996,
0.00027123288540817104,
0.0002526057589974623,
0.00022790421762734515,
0.00020876185354851858,
0.00018495058171630387,
0.00016871013519521616
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00016769046516107157,
"first_to_last_ratio": 0.6977158937201501,
"imbalance_abs_log_ratio": 0.359943288132801,
"last_quartile_mean": 0.00024034204562399042,
"mean": 0.00023591848270084503,
"normalized": [
1.5808747779504164,
0.9518017905108639,
0.9298300713012261,
0.4478696739485771,
0.40335185354164543,
0.44723755585512437,
0.47751464845047825,
0.44790632409401143,
0.5336057466912151,
0.5431934625529073,
0.7089194607367985,
1.040839487386263,
0.9534493790589178,
0.9410587516122122,
1.0381997721223044,
1.0084945199172823,
1.035028521612661,
1.1771284001795213,
1.4238518715599293,
1.4090184077080408,
1.6859221554394308,
1.5579044052749225,
1.640174718113608,
1.466821145938367,
1.357773685072106,
1.2218077259807991,
1.1496890040281678,
1.0707332299935883,
0.9660295158660277,
0.8848897770050418,
0.7839596948867686,
0.7151204656107762
],
"population_cv": 0.3768493091252366
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.060260046273469925,
0.06848448514938354,
0.11107808351516724,
0.17387914657592773,
0.1828140765428543,
0.25734034180641174,
0.2578570544719696,
0.2722952663898468,
0.27254435420036316,
0.27370211482048035,
0.2737041115760803,
0.27443060278892517,
0.2742544412612915,
0.2751515209674835,
0.27536308765411377,
0.2749122083187103,
0.2744285464286804,
0.2747279405593872,
0.2742423117160797,
0.2733822464942932,
0.27115195989608765,
0.26904159784317017,
0.26583269238471985,
0.2588832378387451,
0.255132257938385,
0.24577246606349945,
0.23856717348098755,
0.2310730367898941,
0.22743643820285797,
0.22001685202121735,
0.22118839621543884,
0.21798942983150482,
0.22011704742908478,
0.21957813203334808,
0.22253327071666718,
0.22450163960456848,
0.2256278693675995,
0.22364993393421173,
0.22584785521030426,
0.22638610005378723,
0.22925184667110443,
0.22674965858459473,
0.23080435395240784,
0.2277289777994156,
0.23398378491401672,
0.2319381833076477,
0.24050307273864746,
0.23799650371074677,
0.24922524392604828,
0.24935781955718994,
0.2630935311317444,
0.2632073760032654,
0.27895617485046387,
0.2799976170063019,
0.2995731830596924,
0.30053073167800903,
0.3278428912162781,
0.32797256112098694,
0.36820048093795776,
0.3684364855289459,
0.42407870292663574,
0.4252319037914276,
0.5123529434204102,
0.5140777826309204
],
"loss_nats": 1.3263185024261475,
"loss_scale": 1.0,
"output_weights": null,
"step": 4000,
"stream_state_rms_by_sublayer": [
0.06848448514938354,
0.11107808351516724,
0.17387914657592773,
0.1828140765428543,
0.25734034180641174,
0.2578570544719696,
0.2722952663898468,
0.27254435420036316,
0.27370211482048035,
0.2737041115760803,
0.27443060278892517,
0.2742544412612915,
0.2751515209674835,
0.27536308765411377,
0.2749122083187103,
0.2744285464286804,
0.2747279405593872,
0.2742423117160797,
0.2733822464942932,
0.27115195989608765,
0.26904159784317017,
0.26583269238471985,
0.2588832378387451,
0.255132257938385,
0.24577246606349945,
0.23856717348098755,
0.2310730367898941,
0.22743643820285797,
0.22001685202121735,
0.22118839621543884,
0.21798942983150482,
0.22011704742908478,
0.21957813203334808,
0.22253327071666718,
0.22450163960456848,
0.2256278693675995,
0.22364993393421173,
0.22584785521030426,
0.22638610005378723,
0.22925184667110443,
0.22674965858459473,
0.23080435395240784,
0.2277289777994156,
0.23398378491401672,
0.2319381833076477,
0.24050307273864746,
0.23799650371074677,
0.24922524392604828,
0.24935781955718994,
0.2630935311317444,
0.2632073760032654,
0.27895617485046387,
0.2799976170063019,
0.2995731830596924,
0.30053073167800903,
0.3278428912162781,
0.32797256112098694,
0.36820048093795776,
0.3684364855289459,
0.42407870292663574,
0.4252319037914276,
0.5123529434204102,
0.5140777826309204,
0.6704596281051636
]
},
{
"activation_grad_rms_by_block": [
0.00018927091150544584,
0.0001941375230671838,
0.00020724961359519511,
0.00020481550018303096,
0.00020258830045349896,
0.00020023793331347406,
0.00019735096429940313,
0.0001957487838808447,
0.00019380047160666436,
0.00019302338478155434,
0.0001922047813422978,
0.00018312338215764612,
0.00017868881695903838,
0.00017335887241642922,
0.00016814567788969725,
0.0001643431605771184,
0.00016659151879139245,
0.0001678116386756301,
0.0001584039127919823,
0.0001587578735779971,
0.00014523210120387375,
0.00013619904348161072,
0.00012270425213500857,
0.00011189868382643908,
0.00010116446355823427,
9.155859152087942e-05,
8.147361222654581e-05,
7.049509440548718e-05,
5.894774221815169e-05,
4.627152156899683e-05,
3.385082527529448e-05,
2.4755347112659365e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00019892494128725957,
"first_to_last_ratio": 3.129490087873211,
"imbalance_abs_log_ratio": 1.140870080057971,
"last_quartile_mean": 6.356464973578113e-05,
"mean": 0.00014731888438745955,
"normalized": [
1.2847701928535473,
1.3178047327360134,
1.4068095509745602,
1.3902867988353147,
1.3751685759490058,
1.3592142931712237,
1.3396175590113453,
1.3287419647165597,
1.3155168287655152,
1.310241966494184,
1.304685289611514,
1.2430407881451104,
1.212938977253409,
1.1767593349436627,
1.141372191276318,
1.1155607189155992,
1.1308225655119981,
1.1391047344227307,
1.0752451286242999,
1.077647813028859,
0.9858349240678649,
0.9245185642554635,
0.8329159743857911,
0.759567819778877,
0.6867039753855606,
0.6214993542855888,
0.553042555034996,
0.4785204198266929,
0.40013703920751104,
0.314090903969238,
0.2297792670372409,
0.16803919752440544
],
"population_cv": 0.3716245052237534
},
"activation_output_rms_by_block": [
0.10094190388917923,
0.17939583957195282,
0.23316815495491028,
0.2387540638446808,
0.23957562446594238,
0.2399372160434723,
0.24013783037662506,
0.23953619599342346,
0.2386971116065979,
0.235844686627388,
0.23058047890663147,
0.2186841070652008,
0.2011982500553131,
0.1896955370903015,
0.18346232175827026,
0.1819538027048111,
0.18335868418216705,
0.18572981655597687,
0.1859624832868576,
0.18881060183048248,
0.18956796824932098,
0.19203881919384003,
0.19704653322696686,
0.20452122390270233,
0.21571911871433258,
0.22943545877933502,
0.2464856505393982,
0.26912161707878113,
0.30166757106781006,
0.3485357463359833,
0.4315659701824188,
0.6013737916946411
],
"activation_output_statistics": {
"first_quartile_mean": 0.2139308536425233,
"first_to_last_ratio": 0.6473178416327895,
"imbalance_abs_log_ratio": 0.4349178506515776,
"last_quartile_mean": 0.3304881155490875,
"mean": 0.2363282556179911,
"normalized": [
0.42712583658360814,
0.7590960255803447,
0.9866283417748029,
1.0102645712860119,
1.0137409250513,
1.015270964599827,
1.0161198446147375,
1.0135740873093813,
1.0100235834365736,
0.9979538249062154,
0.9756788425644265,
0.9253405035862031,
0.8513508024217668,
0.8026781926446058,
0.7763029489576774,
0.7699197974825631,
0.7758644166466246,
0.7858976323855102,
0.7868821390001439,
0.7989336752676848,
0.8021383975166516,
0.8125935626768981,
0.8337832169568393,
0.8654116426657729,
0.9127944441100949,
0.9708338013978413,
1.0429800274827308,
1.138761915603517,
1.2764769505828182,
1.4747950702067896,
1.8261293769290816,
2.5446546377709565
],
"population_cv": 0.36483989965797836
},
"bits_per_byte": 1.8402207275901343,
"branch_output_rms_by_sublayer": [
0.033478349447250366,
0.052373696118593216,
0.10464497655630112,
0.018782487139105797,
0.07005563378334045,
0.002347440691664815,
0.020065777003765106,
0.0017286264337599277,
0.017071161419153214,
0.002082002116367221,
0.015527697280049324,
0.004463158082216978,
0.01603785529732704,
0.005855648312717676,
0.013071421533823013,
0.006003717426210642,
0.014386727474629879,
0.008227710612118244,
0.014956407248973846,
0.008421593345701694,
0.015932755544781685,
0.010948682203888893,
0.037222135812044144,
0.016641829162836075,
0.029781410470604897,
0.021082323044538498,
0.025465231388807297,
0.02131747640669346,
0.024801449850201607,
0.021334858611226082,
0.024093298241496086,
0.021644918248057365,
0.023488737642765045,
0.020262837409973145,
0.031102469190955162,
0.020060395821928978,
0.03728024289011955,
0.024732086807489395,
0.03451274335384369,
0.025743624195456505,
0.0439009852707386,
0.03343674913048744,
0.0384296216070652,
0.037262238562107086,
0.044400904327631,
0.047452766448259354,
0.03565425053238869,
0.054450251162052155,
0.032797131687402725,
0.05981304869055748,
0.022211335599422455,
0.0634065493941307,
0.02717488259077072,
0.068015918135643,
0.023526398465037346,
0.07457657158374786,
0.01840120367705822,
0.08500036597251892,
0.017178243026137352,
0.09797899425029755,
0.022575926035642624,
0.13843585550785065,
0.029057970270514488,
0.24571193754673004
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0004888726102119165,
0.00034371112095624357,
0.00027329239576199496,
0.00013109360072049493,
0.0001309003833444521,
0.00015153657998531148,
0.00016751171475037753,
0.00015920301567702626,
0.00019282855273444342,
0.00018189099839941241,
0.00022228035276414997,
0.00039117902483509027,
0.00035836801505216564,
0.00037533451853890784,
0.0004202645380768536,
0.0004297743688669142,
0.0004147505595850994,
0.0004900318658719625,
0.0005824864404877667,
0.0005699482810090311,
0.0006776225188364896,
0.0006405760434101032,
0.0006565350519436726,
0.0006182198906110644,
0.0005596621809681081,
0.0005013865499612654,
0.000471541721969693,
0.0004314515507572245,
0.0003811454676239625,
0.0003426357383233983,
0.00029149764497300414,
0.0002412795793653285
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00023076517767597717,
"first_to_last_ratio": 0.5732227450358324,
"imbalance_abs_log_ratio": 0.5564809030131627,
"last_quartile_mean": 0.00040257505424274806,
"mean": 0.000384025402386654,
"normalized": [
1.2730215427772602,
0.8950218366288691,
0.7116518700677824,
0.3413670030830513,
0.34086386611648084,
0.3946004067531388,
0.4361995683340739,
0.4145637624167724,
0.5021244728500928,
0.4736431425342962,
0.5788167954065396,
1.0186279928462485,
0.9331883068801387,
0.9773689870676945,
1.0943665066554955,
1.1191300528453012,
1.0800081375020862,
1.2760402380324216,
1.5167914332429844,
1.4841421360850018,
1.7645252491766907,
1.6680564343635333,
1.7096136033278435,
1.6098411375105153,
1.4573571891075452,
1.305607771895378,
1.227892006724398,
1.1234974251073624,
0.9925006659851322,
0.8922215462674452,
0.7590582371931511,
0.6282906752152749
],
"population_cv": 0.4280332727708177
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.061826109886169434,
0.07441755384206772,
0.10094190388917923,
0.17219661176204681,
0.17939583957195282,
0.2327817976474762,
0.23316815495491028,
0.2385595589876175,
0.2387540638446808,
0.23968438804149628,
0.23957562446594238,
0.24012914299964905,
0.2399372160434723,
0.2401275336742401,
0.24013783037662506,
0.24013836681842804,
0.23953619599342346,
0.239403635263443,
0.2386971116065979,
0.23807503283023834,
0.235844686627388,
0.23385408520698547,
0.23058047890663147,
0.2232486605644226,
0.2186841070652008,
0.20881249010562897,
0.2011982500553131,
0.19461779296398163,
0.1896955370903015,
0.18348202109336853,
0.18346232175827026,
0.18089424073696136,
0.1819538027048111,
0.1813284158706665,
0.18335868418216705,
0.18508672714233398,
0.18572981655597687,
0.1841643899679184,
0.1859624832868576,
0.18672171235084534,
0.18881060183048248,
0.18648506700992584,
0.18956796824932098,
0.1869000494480133,
0.19203881919384003,
0.18984925746917725,
0.19704653322696686,
0.19547933340072632,
0.20452122390270233,
0.20447643101215363,
0.21571911871433258,
0.21594053506851196,
0.22943545877933502,
0.23029562830924988,
0.2464856505393982,
0.24702563881874084,
0.26912161707878113,
0.2691715657711029,
0.30166757106781006,
0.30165502429008484,
0.3485357463359833,
0.3492375612258911,
0.4315659701824188,
0.4323083460330963
],
"loss_nats": 1.2755438089370728,
"loss_scale": 1.0,
"output_weights": null,
"step": 8000,
"stream_state_rms_by_sublayer": [
0.07441755384206772,
0.10094190388917923,
0.17219661176204681,
0.17939583957195282,
0.2327817976474762,
0.23316815495491028,
0.2385595589876175,
0.2387540638446808,
0.23968438804149628,
0.23957562446594238,
0.24012914299964905,
0.2399372160434723,
0.2401275336742401,
0.24013783037662506,
0.24013836681842804,
0.23953619599342346,
0.239403635263443,
0.2386971116065979,
0.23807503283023834,
0.235844686627388,
0.23385408520698547,
0.23058047890663147,
0.2232486605644226,
0.2186841070652008,
0.20881249010562897,
0.2011982500553131,
0.19461779296398163,
0.1896955370903015,
0.18348202109336853,
0.18346232175827026,
0.18089424073696136,
0.1819538027048111,
0.1813284158706665,
0.18335868418216705,
0.18508672714233398,
0.18572981655597687,
0.1841643899679184,
0.1859624832868576,
0.18672171235084534,
0.18881060183048248,
0.18648506700992584,
0.18956796824932098,
0.1869000494480133,
0.19203881919384003,
0.18984925746917725,
0.19704653322696686,
0.19547933340072632,
0.20452122390270233,
0.20447643101215363,
0.21571911871433258,
0.21594053506851196,
0.22943545877933502,
0.23029562830924988,
0.2464856505393982,
0.24702563881874084,
0.26912161707878113,
0.2691715657711029,
0.30166757106781006,
0.30165502429008484,
0.3485357463359833,
0.3492375612258911,
0.4315659701824188,
0.4323083460330963,
0.6013737916946411
]
}
],
"environment": {
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
"compile": false,
"compute_capability": [
12,
0
],
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
"deterministic_algorithms": true,
"gpu": "NVIDIA GeForce RTX 5090",
"python": "3.10.14",
"torch": "2.11.0+cu128"
},
"evaluations": [
{
"bits_per_byte": 8.037726566232575,
"cross_entropy_nats": 5.57132750749588,
"step": 0
},
{
"bits_per_byte": 5.132850850111892,
"cross_entropy_nats": 3.5578210949897766,
"step": 100
},
{
"bits_per_byte": 3.1258965111502706,
"cross_entropy_nats": 2.1667063534259796,
"step": 500
},
{
"bits_per_byte": 1.9712387018569424,
"cross_entropy_nats": 1.3663585484027863,
"step": 2000
},
{
"bits_per_byte": 1.7961544728728112,
"cross_entropy_nats": 1.2449994087219238,
"step": 4000
},
{
"bits_per_byte": 1.7269767303333703,
"cross_entropy_nats": 1.1970490515232086,
"step": 8000
}
],
"gradient_gate": null,
"hashes": {
"final_mixer_parameters": null,
"final_model_state": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d",
"final_optimizer_state": "4b960a87623e39ce5348146d7169b13bd4e5894a99361177367c7518bf6f603e",
"final_public_parameters": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d",
"initial_mixer_parameters": null,
"initial_public_parameter_elements": 18985152,
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
"initial_public_parameter_tensors": 227,
"initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2"
},
"manifest": {
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"input_gate_tensor_hashes": {
"0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
"1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
"7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
},
"path": "experiments/k3/attnres_gradient/manifest.json",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
},
"model": {
"attnres_aggregation_groups": 8,
"context": 256,
"d_ff": 768,
"d_head": 32,
"d_model": 192,
"heads": 6,
"layers": 32,
"parameters": {
"core": 18985152,
"embedding": 98304,
"mixer": 0,
"total": 18985152
},
"sublayers": 64,
"sublayers_per_attnres_group": 8,
"transformer_blocks_per_attnres_group": 4,
"vocabulary": 256
},
"optimizer": {
"betas": [
0.9,
0.95
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"min_lr": 3e-05,
"name": "AdamW",
"peak_lr": 0.0003,
"warmup_steps": 400,
"weight_decay_ndim_ge_2": 0.1
},
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
"run_kind": "formal",
"schema_version": 1,
"seed": 2026073002,
"steps": 8000,
"target_bytes_seen": 65536000,
"timing": {
"mean_ms": 41.92772524377444,
"measured_steps": 7980,
"median_ms": 41.88774450449273,
"p95_ms": 42.070325661916286,
"peak_allocated_bytes": 5938562048,
"peak_reserved_bytes": 6325010432,
"warmup_steps_excluded": 20
},
"training_history": [
{
"bits_per_byte": 8.02169460149312,
"learning_rate": 7.499999999999999e-07,
"loss_nats": 5.560214996337891,
"step": 1,
"unclipped_grad_norm": 27.98912239074707
},
{
"bits_per_byte": 7.204295458472364,
"learning_rate": 7.499999999999999e-06,
"loss_nats": 4.9936370849609375,
"step": 10,
"unclipped_grad_norm": 12.742182731628418
},
{
"bits_per_byte": 6.681083646642096,
"learning_rate": 1.4999999999999999e-05,
"loss_nats": 4.630974292755127,
"step": 20,
"unclipped_grad_norm": 3.4196884632110596
},
{
"bits_per_byte": 6.539581824957005,
"learning_rate": 2.2499999999999998e-05,
"loss_nats": 4.53289270401001,
"step": 30,
"unclipped_grad_norm": 3.0530965328216553
},
{
"bits_per_byte": 6.404133792582285,
"learning_rate": 2.9999999999999997e-05,
"loss_nats": 4.43900728225708,
"step": 40,
"unclipped_grad_norm": 3.002453088760376
},
{
"bits_per_byte": 6.077271133589068,
"learning_rate": 3.75e-05,
"loss_nats": 4.2124433517456055,
"step": 50,
"unclipped_grad_norm": 2.940347909927368
},
{
"bits_per_byte": 5.8763623767123425,
"learning_rate": 4.4999999999999996e-05,
"loss_nats": 4.073184013366699,
"step": 60,
"unclipped_grad_norm": 2.7971668243408203
},
{
"bits_per_byte": 5.7093882046079605,
"learning_rate": 5.2499999999999995e-05,
"loss_nats": 3.957446336746216,
"step": 70,
"unclipped_grad_norm": 2.5766468048095703
},
{
"bits_per_byte": 5.512541333979312,
"learning_rate": 5.9999999999999995e-05,
"loss_nats": 3.82100248336792,
"step": 80,
"unclipped_grad_norm": 2.5652287006378174
},
{
"bits_per_byte": 5.341233887003791,
"learning_rate": 6.749999999999999e-05,
"loss_nats": 3.702261209487915,
"step": 90,
"unclipped_grad_norm": 2.3708243370056152
},
{
"bits_per_byte": 5.171287854692499,
"learning_rate": 7.5e-05,
"loss_nats": 3.584463596343994,
"step": 100,
"unclipped_grad_norm": 2.273510217666626
},
{
"bits_per_byte": 5.001223154352113,
"learning_rate": 8.249999999999999e-05,
"loss_nats": 3.466583728790283,
"step": 110,
"unclipped_grad_norm": 2.179899215698242
},
{
"bits_per_byte": 4.73609126012899,
"learning_rate": 8.999999999999999e-05,
"loss_nats": 3.282808303833008,
"step": 120,
"unclipped_grad_norm": 1.9098186492919922
},
{
"bits_per_byte": 4.5869558165027025,
"learning_rate": 9.75e-05,
"loss_nats": 3.1794354915618896,
"step": 130,
"unclipped_grad_norm": 1.6915351152420044
},
{
"bits_per_byte": 4.392154507523416,
"learning_rate": 0.00010499999999999999,
"loss_nats": 3.0444095134735107,
"step": 140,
"unclipped_grad_norm": 1.478805422782898
},
{
"bits_per_byte": 4.261904134823071,
"learning_rate": 0.0001125,
"loss_nats": 2.9541268348693848,
"step": 150,
"unclipped_grad_norm": 1.2380529642105103
},
{
"bits_per_byte": 4.155084334510974,
"learning_rate": 0.00011999999999999999,
"loss_nats": 2.880084991455078,
"step": 160,
"unclipped_grad_norm": 1.0523717403411865
},
{
"bits_per_byte": 4.00081108117281,
"learning_rate": 0.00012749999999999998,
"loss_nats": 2.77315092086792,
"step": 170,
"unclipped_grad_norm": 0.9717193841934204
},
{
"bits_per_byte": 3.86890313968345,
"learning_rate": 0.00013499999999999997,
"loss_nats": 2.6817193031311035,
"step": 180,
"unclipped_grad_norm": 1.2074095010757446
},
{
"bits_per_byte": 3.870022058809963,
"learning_rate": 0.0001425,
"loss_nats": 2.682494878768921,
"step": 190,
"unclipped_grad_norm": 0.712932288646698
},
{
"bits_per_byte": 3.7502069054333704,
"learning_rate": 0.00015,
"loss_nats": 2.599445343017578,
"step": 200,
"unclipped_grad_norm": 0.6916731595993042
},
{
"bits_per_byte": 3.7020534830182927,
"learning_rate": 0.0001575,
"loss_nats": 2.566067934036255,
"step": 210,
"unclipped_grad_norm": 0.8496056795120239
},
{
"bits_per_byte": 3.664806168425029,
"learning_rate": 0.00016499999999999997,
"loss_nats": 2.540250062942505,
"step": 220,
"unclipped_grad_norm": 0.7601449489593506
},
{
"bits_per_byte": 3.680203087226173,
"learning_rate": 0.0001725,
"loss_nats": 2.550922393798828,
"step": 230,
"unclipped_grad_norm": 0.7172863483428955
},
{
"bits_per_byte": 3.6667052008558563,
"learning_rate": 0.00017999999999999998,
"loss_nats": 2.5415663719177246,
"step": 240,
"unclipped_grad_norm": 0.8026229739189148
},
{
"bits_per_byte": 3.532286312663899,
"learning_rate": 0.0001875,
"loss_nats": 2.448394298553467,
"step": 250,
"unclipped_grad_norm": 0.9514944553375244
},
{
"bits_per_byte": 3.530420644867339,
"learning_rate": 0.000195,
"loss_nats": 2.44710111618042,
"step": 260,
"unclipped_grad_norm": 0.7874091267585754
},
{
"bits_per_byte": 3.4939940314839926,
"learning_rate": 0.00020249999999999996,
"loss_nats": 2.4218521118164062,
"step": 270,
"unclipped_grad_norm": 0.9165833592414856
},
{
"bits_per_byte": 3.5060328170443893,
"learning_rate": 0.00020999999999999998,
"loss_nats": 2.430196762084961,
"step": 280,
"unclipped_grad_norm": 0.971961259841919
},
{
"bits_per_byte": 3.3964946589527543,
"learning_rate": 0.00021749999999999997,
"loss_nats": 2.3542706966400146,
"step": 290,
"unclipped_grad_norm": 0.7937634587287903
},
{
"bits_per_byte": 3.488020042123622,
"learning_rate": 0.000225,
"loss_nats": 2.4177112579345703,
"step": 300,
"unclipped_grad_norm": 1.0019733905792236
},
{
"bits_per_byte": 3.3886120061331084,
"learning_rate": 0.00023249999999999996,
"loss_nats": 2.348806858062744,
"step": 310,
"unclipped_grad_norm": 0.8179588913917542
},
{
"bits_per_byte": 3.3666969448642643,
"learning_rate": 0.00023999999999999998,
"loss_nats": 2.3336164951324463,
"step": 320,
"unclipped_grad_norm": 0.8202242255210876
},
{
"bits_per_byte": 3.351965942887261,
"learning_rate": 0.0002475,
"loss_nats": 2.3234057426452637,
"step": 330,
"unclipped_grad_norm": 0.8768095374107361
},
{
"bits_per_byte": 3.357094121570688,
"learning_rate": 0.00025499999999999996,
"loss_nats": 2.326960325241089,
"step": 340,
"unclipped_grad_norm": 0.9328333139419556
},
{
"bits_per_byte": 3.3293237389705688,
"learning_rate": 0.0002625,
"loss_nats": 2.307711362838745,
"step": 350,
"unclipped_grad_norm": 0.7516238689422607
},
{
"bits_per_byte": 3.3205963073698848,
"learning_rate": 0.00026999999999999995,
"loss_nats": 2.301661968231201,
"step": 360,
"unclipped_grad_norm": 0.9922879338264465
},
{
"bits_per_byte": 3.3203087523776422,
"learning_rate": 0.00027749999999999997,
"loss_nats": 2.3014626502990723,
"step": 370,
"unclipped_grad_norm": 0.7593456506729126
},
{
"bits_per_byte": 3.3612334000116544,
"learning_rate": 0.000285,
"loss_nats": 2.329829454421997,
"step": 380,
"unclipped_grad_norm": 0.8890619874000549
},
{
"bits_per_byte": 3.310503677486652,
"learning_rate": 0.0002925,
"loss_nats": 2.294666290283203,
"step": 390,
"unclipped_grad_norm": 0.8439806699752808
},
{
"bits_per_byte": 3.2443041155165395,
"learning_rate": 0.0003,
"loss_nats": 2.2487802505493164,
"step": 400,
"unclipped_grad_norm": 1.1809823513031006
},
{
"bits_per_byte": 3.208035726171987,
"learning_rate": 0.0002999988466112388,
"loss_nats": 2.2236409187316895,
"step": 410,
"unclipped_grad_norm": 0.9897300601005554
},
{
"bits_per_byte": 3.232116049015194,
"learning_rate": 0.00029999538646466355,
"loss_nats": 2.2403321266174316,
"step": 420,
"unclipped_grad_norm": 0.898377537727356
},
{
"bits_per_byte": 3.2113598068478635,
"learning_rate": 0.0002999896196193985,
"loss_nats": 2.225944995880127,
"step": 430,
"unclipped_grad_norm": 1.0354490280151367
},
{
"bits_per_byte": 3.2315632967753207,
"learning_rate": 0.00029998154617398326,
"loss_nats": 2.2399489879608154,
"step": 440,
"unclipped_grad_norm": 0.9574596285820007
},
{
"bits_per_byte": 3.1852710706078806,
"learning_rate": 0.0002999711662663707,
"loss_nats": 2.2078616619110107,
"step": 450,
"unclipped_grad_norm": 0.916765570640564
},
{
"bits_per_byte": 3.155730642565081,
"learning_rate": 0.0002999584800739247,
"loss_nats": 2.1873857975006104,
"step": 460,
"unclipped_grad_norm": 0.9241403937339783
},
{
"bits_per_byte": 3.18185446326584,
"learning_rate": 0.0002999434878134173,
"loss_nats": 2.205493450164795,
"step": 470,
"unclipped_grad_norm": 0.9457137584686279
},
{
"bits_per_byte": 3.13667255709237,
"learning_rate": 0.000299926189741025,
"loss_nats": 2.17417573928833,
"step": 480,
"unclipped_grad_norm": 0.7014725208282471
},
{
"bits_per_byte": 3.105025341576501,
"learning_rate": 0.000299906586152324,
"loss_nats": 2.1522395610809326,
"step": 490,
"unclipped_grad_norm": 0.840235710144043
},
{
"bits_per_byte": 3.1575567543519423,
"learning_rate": 0.0002998846773822856,
"loss_nats": 2.1886515617370605,
"step": 500,
"unclipped_grad_norm": 1.1171972751617432
},
{
"bits_per_byte": 3.1160109053830136,
"learning_rate": 0.0002998604638052702,
"loss_nats": 2.1598541736602783,
"step": 510,
"unclipped_grad_norm": 0.8018220067024231
},
{
"bits_per_byte": 3.0148496782496044,
"learning_rate": 0.0002998339458350211,
"loss_nats": 2.0897345542907715,
"step": 520,
"unclipped_grad_norm": 1.302595853805542
},
{
"bits_per_byte": 3.0793170309602265,
"learning_rate": 0.0002998051239246572,
"loss_nats": 2.1344199180603027,
"step": 530,
"unclipped_grad_norm": 0.6869481205940247
},
{
"bits_per_byte": 3.040265962324714,
"learning_rate": 0.00029977399856666537,
"loss_nats": 2.107351779937744,
"step": 540,
"unclipped_grad_norm": 1.3514107465744019
},
{
"bits_per_byte": 2.996858229177544,
"learning_rate": 0.0002997405702928922,
"loss_nats": 2.077263832092285,
"step": 550,
"unclipped_grad_norm": 0.9677014350891113
},
{
"bits_per_byte": 2.9987245849047075,
"learning_rate": 0.0002997048396745345,
"loss_nats": 2.0785574913024902,
"step": 560,
"unclipped_grad_norm": 1.1580114364624023
},
{
"bits_per_byte": 2.9750642436949075,
"learning_rate": 0.0002996668073221302,
"loss_nats": 2.062157392501831,
"step": 570,
"unclipped_grad_norm": 1.1581705808639526
},
{
"bits_per_byte": 2.870633970333898,
"learning_rate": 0.000299626473885547,
"loss_nats": 1.989771842956543,
"step": 580,
"unclipped_grad_norm": 0.9525179862976074
},
{
"bits_per_byte": 2.9300787417087224,
"learning_rate": 0.00029958384005397226,
"loss_nats": 2.030975818634033,
"step": 590,
"unclipped_grad_norm": 0.8967422842979431
},
{
"bits_per_byte": 2.847721925638002,
"learning_rate": 0.0002995389065559004,
"loss_nats": 1.9738904237747192,
"step": 600,
"unclipped_grad_norm": 0.7550538182258606
},
{
"bits_per_byte": 2.905717055248704,
"learning_rate": 0.000299491674159121,
"loss_nats": 2.014089584350586,
"step": 610,
"unclipped_grad_norm": 0.6918516755104065
},
{
"bits_per_byte": 2.79248952533076,
"learning_rate": 0.00029944214367070535,
"loss_nats": 1.9356062412261963,
"step": 620,
"unclipped_grad_norm": 0.7874937653541565
},
{
"bits_per_byte": 2.8091388217954867,
"learning_rate": 0.00029939031593699284,
"loss_nats": 1.9471466541290283,
"step": 630,
"unclipped_grad_norm": 0.9318027496337891
},
{
"bits_per_byte": 2.8177121569409977,
"learning_rate": 0.00029933619184357654,
"loss_nats": 1.9530892372131348,
"step": 640,
"unclipped_grad_norm": 0.8821531534194946
},
{
"bits_per_byte": 2.7350290897815905,
"learning_rate": 0.00029927977231528774,
"loss_nats": 1.895777702331543,
"step": 650,
"unclipped_grad_norm": 1.1635478734970093
},
{
"bits_per_byte": 2.7242683072997553,
"learning_rate": 0.00029922105831618057,
"loss_nats": 1.8883188962936401,
"step": 660,
"unclipped_grad_norm": 0.8677663207054138
},
{
"bits_per_byte": 2.7265658235326318,
"learning_rate": 0.0002991600508495153,
"loss_nats": 1.889911413192749,
"step": 670,
"unclipped_grad_norm": 0.8130120635032654
},
{
"bits_per_byte": 2.7610239234942022,
"learning_rate": 0.00029909675095774114,
"loss_nats": 1.9137959480285645,
"step": 680,
"unclipped_grad_norm": 0.9226085543632507
},
{
"bits_per_byte": 2.640049951016096,
"learning_rate": 0.0002990311597224787,
"loss_nats": 1.8299431800842285,
"step": 690,
"unclipped_grad_norm": 0.8291181921958923
},
{
"bits_per_byte": 2.583675413924474,
"learning_rate": 0.00029896327826450103,
"loss_nats": 1.7908673286437988,
"step": 700,
"unclipped_grad_norm": 0.9929485321044922
},
{
"bits_per_byte": 2.6777724536736933,
"learning_rate": 0.0002988931077437152,
"loss_nats": 1.8560904264450073,
"step": 710,
"unclipped_grad_norm": 1.0362995862960815
},
{
"bits_per_byte": 2.7123457820113424,
"learning_rate": 0.00029882064935914164,
"loss_nats": 1.8800548315048218,
"step": 720,
"unclipped_grad_norm": 0.9176086187362671
},
{
"bits_per_byte": 2.545722970462673,
"learning_rate": 0.0002987459043488943,
"loss_nats": 1.7645606994628906,
"step": 730,
"unclipped_grad_norm": 1.4210379123687744
},
{
"bits_per_byte": 2.557563288043974,
"learning_rate": 0.0002986688739901591,
"loss_nats": 1.7727677822113037,
"step": 740,
"unclipped_grad_norm": 1.1384987831115723
},
{
"bits_per_byte": 2.56630310239552,
"learning_rate": 0.0002985895595991723,
"loss_nats": 1.7788257598876953,
"step": 750,
"unclipped_grad_norm": 0.9141139984130859
},
{
"bits_per_byte": 2.5326694872621855,
"learning_rate": 0.00029850796253119824,
"loss_nats": 1.7555127143859863,
"step": 760,
"unclipped_grad_norm": 0.9497804045677185
},
{
"bits_per_byte": 2.5270525338850063,
"learning_rate": 0.00029842408418050544,
"loss_nats": 1.7516193389892578,
"step": 770,
"unclipped_grad_norm": 1.0209811925888062
},
{
"bits_per_byte": 2.4536180057583974,
"learning_rate": 0.0002983379259803436,
"loss_nats": 1.7007184028625488,
"step": 780,
"unclipped_grad_norm": 0.8395339250564575
},
{
"bits_per_byte": 2.5403350979764427,
"learning_rate": 0.0002982494894029186,
"loss_nats": 1.7608261108398438,
"step": 790,
"unclipped_grad_norm": 1.0305956602096558
},
{
"bits_per_byte": 2.476898265309791,
"learning_rate": 0.0002981587759593675,
"loss_nats": 1.7168550491333008,
"step": 800,
"unclipped_grad_norm": 0.7494933605194092
},
{
"bits_per_byte": 2.5328731147208075,
"learning_rate": 0.00029806578719973283,
"loss_nats": 1.7556538581848145,
"step": 810,
"unclipped_grad_norm": 2.0184309482574463
},
{
"bits_per_byte": 2.4645244575455134,
"learning_rate": 0.00029797052471293596,
"loss_nats": 1.7082781791687012,
"step": 820,
"unclipped_grad_norm": 1.0523004531860352
},
{
"bits_per_byte": 2.4679443325579205,
"learning_rate": 0.0002978729901267499,
"loss_nats": 1.7106486558914185,
"step": 830,
"unclipped_grad_norm": 0.9168965816497803
},
{
"bits_per_byte": 2.457752984633092,
"learning_rate": 0.0002977731851077716,
"loss_nats": 1.7035845518112183,
"step": 840,
"unclipped_grad_norm": 1.0069817304611206
},
{
"bits_per_byte": 2.4003085434704525,
"learning_rate": 0.0002976711113613935,
"loss_nats": 1.6637670993804932,
"step": 850,
"unclipped_grad_norm": 0.9418690204620361
},
{
"bits_per_byte": 2.440524794565561,
"learning_rate": 0.0002975667706317743,
"loss_nats": 1.6916428804397583,
"step": 860,
"unclipped_grad_norm": 1.019945502281189
},
{
"bits_per_byte": 2.404373525406246,
"learning_rate": 0.00029746016470180915,
"loss_nats": 1.6665847301483154,
"step": 870,
"unclipped_grad_norm": 1.4541757106781006
},
{
"bits_per_byte": 2.4247352393724975,
"learning_rate": 0.00029735129539309926,
"loss_nats": 1.6806983947753906,
"step": 880,
"unclipped_grad_norm": 0.9109940528869629
},
{
"bits_per_byte": 2.4066381929528076,
"learning_rate": 0.0002972401645659207,
"loss_nats": 1.6681544780731201,
"step": 890,
"unclipped_grad_norm": 0.9647485613822937
},
{
"bits_per_byte": 2.3372799976169487,
"learning_rate": 0.00029712677411919283,
"loss_nats": 1.6200790405273438,
"step": 900,
"unclipped_grad_norm": 0.8496110439300537
},
{
"bits_per_byte": 2.3871222896634996,
"learning_rate": 0.0002970111259904454,
"loss_nats": 1.6546270847320557,
"step": 910,
"unclipped_grad_norm": 1.1169326305389404
},
{
"bits_per_byte": 2.3333503660273847,
"learning_rate": 0.0002968932221557859,
"loss_nats": 1.617355227470398,
"step": 920,
"unclipped_grad_norm": 0.8540288805961609
},
{
"bits_per_byte": 2.3443775496354062,
"learning_rate": 0.0002967730646298655,
"loss_nats": 1.624998688697815,
"step": 930,
"unclipped_grad_norm": 1.0937522649765015
},
{
"bits_per_byte": 2.285418285283967,
"learning_rate": 0.0002966506554658449,
"loss_nats": 1.5841312408447266,
"step": 940,
"unclipped_grad_norm": 0.9093461632728577
},
{
"bits_per_byte": 2.261280348252721,
"learning_rate": 0.00029652599675535876,
"loss_nats": 1.5674000978469849,
"step": 950,
"unclipped_grad_norm": 0.8109434843063354
},
{
"bits_per_byte": 2.2762792991997682,
"learning_rate": 0.0002963990906284808,
"loss_nats": 1.5777965784072876,
"step": 960,
"unclipped_grad_norm": 0.8615142107009888
},
{
"bits_per_byte": 2.320710157136921,
"learning_rate": 0.0002962699392536863,
"loss_nats": 1.6085937023162842,
"step": 970,
"unclipped_grad_norm": 1.2632875442504883
},
{
"bits_per_byte": 2.270013111315582,
"learning_rate": 0.0002961385448378161,
"loss_nats": 1.5734531879425049,
"step": 980,
"unclipped_grad_norm": 0.9684879183769226
},
{
"bits_per_byte": 2.2007698003905,
"learning_rate": 0.00029600490962603817,
"loss_nats": 1.5254573822021484,
"step": 990,
"unclipped_grad_norm": 0.8230604529380798
},
{
"bits_per_byte": 2.245051377386703,
"learning_rate": 0.00029586903590180956,
"loss_nats": 1.556151032447815,
"step": 1000,
"unclipped_grad_norm": 0.7837699055671692
},
{
"bits_per_byte": 2.2494322914521314,
"learning_rate": 0.00029573092598683735,
"loss_nats": 1.559187650680542,
"step": 1010,
"unclipped_grad_norm": 0.8822826743125916
},
{
"bits_per_byte": 2.262538057378481,
"learning_rate": 0.00029559058224103895,
"loss_nats": 1.5682718753814697,
"step": 1020,
"unclipped_grad_norm": 0.9820390939712524
},
{
"bits_per_byte": 2.1631948588473424,
"learning_rate": 0.0002954480070625017,
"loss_nats": 1.4994124174118042,
"step": 1030,
"unclipped_grad_norm": 0.7792394161224365
},
{
"bits_per_byte": 2.258897528625017,
"learning_rate": 0.00029530320288744205,
"loss_nats": 1.5657484531402588,
"step": 1040,
"unclipped_grad_norm": 0.8651536703109741
},
{
"bits_per_byte": 2.2449380408197843,
"learning_rate": 0.0002951561721901638,
"loss_nats": 1.556072473526001,
"step": 1050,
"unclipped_grad_norm": 0.8636346459388733
},
{
"bits_per_byte": 2.190915194496061,
"learning_rate": 0.00029500691748301594,
"loss_nats": 1.5186266899108887,
"step": 1060,
"unclipped_grad_norm": 0.9602811932563782
},
{
"bits_per_byte": 2.2937991718431627,
"learning_rate": 0.00029485544131634963,
"loss_nats": 1.5899404287338257,
"step": 1070,
"unclipped_grad_norm": 0.82843017578125
},
{
"bits_per_byte": 2.212529630091197,
"learning_rate": 0.0002947017462784745,
"loss_nats": 1.5336086750030518,
"step": 1080,
"unclipped_grad_norm": 0.9238869547843933
},
{
"bits_per_byte": 2.140786207370522,
"learning_rate": 0.00029454583499561476,
"loss_nats": 1.4838799238204956,
"step": 1090,
"unclipped_grad_norm": 0.8506197333335876
},
{
"bits_per_byte": 2.2365177702335415,
"learning_rate": 0.00029438771013186413,
"loss_nats": 1.5502359867095947,
"step": 1100,
"unclipped_grad_norm": 0.905985951423645
},
{
"bits_per_byte": 2.2504714106286445,
"learning_rate": 0.0002942273743891401,
"loss_nats": 1.5599079132080078,
"step": 1110,
"unclipped_grad_norm": 0.8354071378707886
},
{
"bits_per_byte": 2.1399247462723503,
"learning_rate": 0.00029406483050713824,
"loss_nats": 1.4832828044891357,
"step": 1120,
"unclipped_grad_norm": 0.8565837144851685
},
{
"bits_per_byte": 2.1407069233684743,
"learning_rate": 0.00029390008126328504,
"loss_nats": 1.4838249683380127,
"step": 1130,
"unclipped_grad_norm": 0.7768547534942627
},
{
"bits_per_byte": 2.1434902905900377,
"learning_rate": 0.00029373312947269045,
"loss_nats": 1.4857542514801025,
"step": 1140,
"unclipped_grad_norm": 0.8102613687515259
},
{
"bits_per_byte": 2.1476347285105297,
"learning_rate": 0.00029356397798809995,
"loss_nats": 1.4886269569396973,
"step": 1150,
"unclipped_grad_norm": 0.7917956113815308
},
{
"bits_per_byte": 2.0881920209275155,
"learning_rate": 0.0002933926296998457,
"loss_nats": 1.4474244117736816,
"step": 1160,
"unclipped_grad_norm": 0.8217911720275879
},
{
"bits_per_byte": 2.1987029128924305,
"learning_rate": 0.0002932190875357972,
"loss_nats": 1.5240247249603271,
"step": 1170,
"unclipped_grad_norm": 0.7990338802337646
},
{
"bits_per_byte": 2.091903062567833,
"learning_rate": 0.00029304335446131126,
"loss_nats": 1.4499967098236084,
"step": 1180,
"unclipped_grad_norm": 0.8429117202758789
},
{
"bits_per_byte": 2.129434492500322,
"learning_rate": 0.00029286543347918116,
"loss_nats": 1.4760115146636963,
"step": 1190,
"unclipped_grad_norm": 0.878530740737915
},
{
"bits_per_byte": 2.113836010032367,
"learning_rate": 0.0002926853276295856,
"loss_nats": 1.4651994705200195,
"step": 1200,
"unclipped_grad_norm": 0.7968210577964783
},
{
"bits_per_byte": 2.2311123555169234,
"learning_rate": 0.0002925030399900368,
"loss_nats": 1.5464892387390137,
"step": 1210,
"unclipped_grad_norm": 0.8802653551101685
},
{
"bits_per_byte": 2.085374257175779,
"learning_rate": 0.00029231857367532726,
"loss_nats": 1.4454712867736816,
"step": 1220,
"unclipped_grad_norm": 0.8048909306526184
},
{
"bits_per_byte": 2.0275156817898625,
"learning_rate": 0.0002921319318374776,
"loss_nats": 1.4053667783737183,
"step": 1230,
"unclipped_grad_norm": 0.7767927646636963
},
{
"bits_per_byte": 2.034634215691725,
"learning_rate": 0.00029194311766568174,
"loss_nats": 1.4103009700775146,
"step": 1240,
"unclipped_grad_norm": 0.769820511341095
},
{
"bits_per_byte": 2.2082001388383765,
"learning_rate": 0.000291752134386253,
"loss_nats": 1.5306077003479004,
"step": 1250,
"unclipped_grad_norm": 0.8053375482559204
},
{
"bits_per_byte": 2.1275602255712194,
"learning_rate": 0.00029155898526256856,
"loss_nats": 1.4747123718261719,
"step": 1260,
"unclipped_grad_norm": 0.7535898685455322
},
{
"bits_per_byte": 1.9931237951488738,
"learning_rate": 0.0002913636735950141,
"loss_nats": 1.3815281391143799,
"step": 1270,
"unclipped_grad_norm": 0.7098326683044434
},
{
"bits_per_byte": 2.1454299109264685,
"learning_rate": 0.0002911662027209272,
"loss_nats": 1.4870986938476562,
"step": 1280,
"unclipped_grad_norm": 0.7225908637046814
},
{
"bits_per_byte": 2.0451564582368134,
"learning_rate": 0.00029096657601454013,
"loss_nats": 1.4175944328308105,
"step": 1290,
"unclipped_grad_norm": 0.7847483158111572
},
{
"bits_per_byte": 2.0739233083160316,
"learning_rate": 0.00029076479688692266,
"loss_nats": 1.4375340938568115,
"step": 1300,
"unclipped_grad_norm": 0.7665325403213501
},
{
"bits_per_byte": 2.153918458625104,
"learning_rate": 0.00029056086878592323,
"loss_nats": 1.4929825067520142,
"step": 1310,
"unclipped_grad_norm": 0.8358681797981262
},
{
"bits_per_byte": 2.0810513012636918,
"learning_rate": 0.00029035479519611045,
"loss_nats": 1.4424748420715332,
"step": 1320,
"unclipped_grad_norm": 0.7432150840759277
},
{
"bits_per_byte": 1.9944121171864873,
"learning_rate": 0.0002901465796387134,
"loss_nats": 1.3824211359024048,
"step": 1330,
"unclipped_grad_norm": 0.9147225022315979
},
{
"bits_per_byte": 2.0106892431947467,
"learning_rate": 0.00028993622567156154,
"loss_nats": 1.393703579902649,
"step": 1340,
"unclipped_grad_norm": 0.7953330278396606
},
{
"bits_per_byte": 2.178856458948164,
"learning_rate": 0.0002897237368890237,
"loss_nats": 1.510268211364746,
"step": 1350,
"unclipped_grad_norm": 0.9230644702911377
},
{
"bits_per_byte": 2.000966204028218,
"learning_rate": 0.0002895091169219469,
"loss_nats": 1.3869640827178955,
"step": 1360,
"unclipped_grad_norm": 0.766094446182251
},
{
"bits_per_byte": 2.099968876910648,
"learning_rate": 0.00028929236943759425,
"loss_nats": 1.4555875062942505,
"step": 1370,
"unclipped_grad_norm": 0.802431583404541
},
{
"bits_per_byte": 2.0735674762113967,
"learning_rate": 0.00028907349813958225,
"loss_nats": 1.437287449836731,
"step": 1380,
"unclipped_grad_norm": 0.7674750685691833
},
{
"bits_per_byte": 2.0708314042188203,
"learning_rate": 0.0002888525067678175,
"loss_nats": 1.4353909492492676,
"step": 1390,
"unclipped_grad_norm": 0.7390276789665222
},
{
"bits_per_byte": 2.082916281129648,
"learning_rate": 0.00028862939909843273,
"loss_nats": 1.4437675476074219,
"step": 1400,
"unclipped_grad_norm": 0.8315935730934143
},
{
"bits_per_byte": 2.026164414102033,
"learning_rate": 0.0002884041789437225,
"loss_nats": 1.4044301509857178,
"step": 1410,
"unclipped_grad_norm": 0.8864192366600037
},
{
"bits_per_byte": 2.0547052789780182,
"learning_rate": 0.0002881768501520778,
"loss_nats": 1.424213171005249,
"step": 1420,
"unclipped_grad_norm": 0.6849789023399353
},
{
"bits_per_byte": 2.0500001776157135,
"learning_rate": 0.0002879474166079205,
"loss_nats": 1.4209518432617188,
"step": 1430,
"unclipped_grad_norm": 0.6987867951393127
},
{
"bits_per_byte": 2.022067959341132,
"learning_rate": 0.00028771588223163694,
"loss_nats": 1.4015907049179077,
"step": 1440,
"unclipped_grad_norm": 0.7956437468528748
},
{
"bits_per_byte": 2.068799257216225,
"learning_rate": 0.00028748225097951054,
"loss_nats": 1.4339823722839355,
"step": 1450,
"unclipped_grad_norm": 0.9165704250335693
},
{
"bits_per_byte": 2.0134160281241766,
"learning_rate": 0.000287246526843655,
"loss_nats": 1.3955936431884766,
"step": 1460,
"unclipped_grad_norm": 0.7257574200630188
},
{
"bits_per_byte": 2.0111035494006755,
"learning_rate": 0.00028700871385194533,
"loss_nats": 1.3939907550811768,
"step": 1470,
"unclipped_grad_norm": 0.77181476354599
},
{
"bits_per_byte": 1.915849582359102,
"learning_rate": 0.00028676881606794945,
"loss_nats": 1.3279657363891602,
"step": 1480,
"unclipped_grad_norm": 0.6826913356781006
},
{
"bits_per_byte": 2.0189380470780796,
"learning_rate": 0.00028652683759085854,
"loss_nats": 1.399421215057373,
"step": 1490,
"unclipped_grad_norm": 0.850294291973114
},
{
"bits_per_byte": 1.925917102775309,
"learning_rate": 0.0002862827825554171,
"loss_nats": 1.3349440097808838,
"step": 1500,
"unclipped_grad_norm": 0.6999683380126953
},
{
"bits_per_byte": 1.9985412486510523,
"learning_rate": 0.00028603665513185245,
"loss_nats": 1.3852832317352295,
"step": 1510,
"unclipped_grad_norm": 0.7689449787139893
},
{
"bits_per_byte": 1.9951115706275462,
"learning_rate": 0.000285788459525803,
"loss_nats": 1.3829059600830078,
"step": 1520,
"unclipped_grad_norm": 0.785367488861084
},
{
"bits_per_byte": 1.9521630311581806,
"learning_rate": 0.00028553819997824707,
"loss_nats": 1.3531363010406494,
"step": 1530,
"unclipped_grad_norm": 0.6442116498947144
},
{
"bits_per_byte": 2.0327179849958115,
"learning_rate": 0.00028528588076542964,
"loss_nats": 1.4089727401733398,
"step": 1540,
"unclipped_grad_norm": 0.7405364513397217
},
{
"bits_per_byte": 1.996995812550399,
"learning_rate": 0.00028503150619878996,
"loss_nats": 1.3842120170593262,
"step": 1550,
"unclipped_grad_norm": 0.7386215329170227
},
{
"bits_per_byte": 2.0870359535484155,
"learning_rate": 0.0002847750806248876,
"loss_nats": 1.4466230869293213,
"step": 1560,
"unclipped_grad_norm": 0.7484163641929626
},
{
"bits_per_byte": 2.0235489019477115,
"learning_rate": 0.000284516608425328,
"loss_nats": 1.4026172161102295,
"step": 1570,
"unclipped_grad_norm": 0.7196536064147949
},
{
"bits_per_byte": 2.00750653225788,
"learning_rate": 0.000284256094016688,
"loss_nats": 1.3914974927902222,
"step": 1580,
"unclipped_grad_norm": 0.7144435048103333
},
{
"bits_per_byte": 2.0128109931584413,
"learning_rate": 0.0002839935418504399,
"loss_nats": 1.395174264907837,
"step": 1590,
"unclipped_grad_norm": 0.8628523349761963
},
{
"bits_per_byte": 2.0153315708894866,
"learning_rate": 0.000283728956412876,
"loss_nats": 1.3969213962554932,
"step": 1600,
"unclipped_grad_norm": 0.715620756149292
},
{
"bits_per_byte": 2.048095125792106,
"learning_rate": 0.00028346234222503136,
"loss_nats": 1.4196313619613647,
"step": 1610,
"unclipped_grad_norm": 0.7313905358314514
},
{
"bits_per_byte": 2.0567931483594926,
"learning_rate": 0.00028319370384260675,
"loss_nats": 1.4256603717803955,
"step": 1620,
"unclipped_grad_norm": 0.761407732963562
},
{
"bits_per_byte": 1.907960738164025,
"learning_rate": 0.00028292304585589105,
"loss_nats": 1.3224976062774658,
"step": 1630,
"unclipped_grad_norm": 0.7100295424461365
},
{
"bits_per_byte": 2.0356341228238413,
"learning_rate": 0.0002826503728896824,
"loss_nats": 1.410994052886963,
"step": 1640,
"unclipped_grad_norm": 0.7265889048576355
},
{
"bits_per_byte": 1.9881094689803178,
"learning_rate": 0.0002823756896032095,
"loss_nats": 1.3780524730682373,
"step": 1650,
"unclipped_grad_norm": 0.7565467953681946
},
{
"bits_per_byte": 1.9576370669524927,
"learning_rate": 0.0002820990006900518,
"loss_nats": 1.3569306135177612,
"step": 1660,
"unclipped_grad_norm": 0.6934499144554138
},
{
"bits_per_byte": 2.105078653450433,
"learning_rate": 0.00028182031087805947,
"loss_nats": 1.4591293334960938,
"step": 1670,
"unclipped_grad_norm": 0.8928083181381226
},
{
"bits_per_byte": 2.0211131116635417,
"learning_rate": 0.0002815396249292725,
"loss_nats": 1.4009288549423218,
"step": 1680,
"unclipped_grad_norm": 0.6971679925918579
},
{
"bits_per_byte": 1.9058530907777012,
"learning_rate": 0.0002812569476398391,
"loss_nats": 1.321036696434021,
"step": 1690,
"unclipped_grad_norm": 0.6719903349876404
},
{
"bits_per_byte": 1.9394077658742894,
"learning_rate": 0.00028097228383993446,
"loss_nats": 1.3442950248718262,
"step": 1700,
"unclipped_grad_norm": 0.6790670156478882
},
{
"bits_per_byte": 2.021125322431753,
"learning_rate": 0.0002806856383936774,
"loss_nats": 1.4009373188018799,
"step": 1710,
"unclipped_grad_norm": 0.6948477029800415
},
{
"bits_per_byte": 2.0066268409987162,
"learning_rate": 0.00028039701619904774,
"loss_nats": 1.39088773727417,
"step": 1720,
"unclipped_grad_norm": 0.6923770904541016
},
{
"bits_per_byte": 1.9082544845316984,
"learning_rate": 0.0002801064221878024,
"loss_nats": 1.3227012157440186,
"step": 1730,
"unclipped_grad_norm": 0.875411868095398
},
{
"bits_per_byte": 1.950284636645457,
"learning_rate": 0.00027981386132539134,
"loss_nats": 1.3518342971801758,
"step": 1740,
"unclipped_grad_norm": 0.7188331484794617
},
{
"bits_per_byte": 2.0503372636114046,
"learning_rate": 0.0002795193386108724,
"loss_nats": 1.4211854934692383,
"step": 1750,
"unclipped_grad_norm": 0.6843723058700562
},
{
"bits_per_byte": 1.9245816574913592,
"learning_rate": 0.00027922285907682625,
"loss_nats": 1.334018349647522,
"step": 1760,
"unclipped_grad_norm": 0.7317860722541809
},
{
"bits_per_byte": 1.9854330749675908,
"learning_rate": 0.00027892442778927005,
"loss_nats": 1.376197338104248,
"step": 1770,
"unclipped_grad_norm": 0.7310082912445068
},
{
"bits_per_byte": 1.9060752923626159,
"learning_rate": 0.00027862404984757116,
"loss_nats": 1.3211907148361206,
"step": 1780,
"unclipped_grad_norm": 0.6923527717590332
},
{
"bits_per_byte": 1.8947768921141837,
"learning_rate": 0.0002783217303843597,
"loss_nats": 1.313359260559082,
"step": 1790,
"unclipped_grad_norm": 0.7431268692016602
},
{
"bits_per_byte": 1.9836379200578849,
"learning_rate": 0.00027801747456544134,
"loss_nats": 1.374953031539917,
"step": 1800,
"unclipped_grad_norm": 0.800605058670044
},
{
"bits_per_byte": 1.9736849400871523,
"learning_rate": 0.00027771128758970843,
"loss_nats": 1.3680541515350342,
"step": 1810,
"unclipped_grad_norm": 0.6639943718910217
},
{
"bits_per_byte": 2.0327128255162856,
"learning_rate": 0.00027740317468905165,
"loss_nats": 1.4089691638946533,
"step": 1820,
"unclipped_grad_norm": 0.7219430804252625
},
{
"bits_per_byte": 1.8641528013882975,
"learning_rate": 0.0002770931411282704,
"loss_nats": 1.2921322584152222,
"step": 1830,
"unclipped_grad_norm": 0.6714999079704285
},
{
"bits_per_byte": 1.9760018903595762,
"learning_rate": 0.0002767811922049827,
"loss_nats": 1.3696601390838623,
"step": 1840,
"unclipped_grad_norm": 0.6850439310073853
},
{
"bits_per_byte": 1.8495459708852175,
"learning_rate": 0.00027646733324953503,
"loss_nats": 1.2820075750350952,
"step": 1850,
"unclipped_grad_norm": 0.7066381573677063
},
{
"bits_per_byte": 2.002332434206672,
"learning_rate": 0.00027615156962491097,
"loss_nats": 1.387911081314087,
"step": 1860,
"unclipped_grad_norm": 0.7545463442802429
},
{
"bits_per_byte": 2.025226076758926,
"learning_rate": 0.0002758339067266396,
"loss_nats": 1.4037797451019287,
"step": 1870,
"unclipped_grad_norm": 0.7131425142288208
},
{
"bits_per_byte": 1.8761874593650736,
"learning_rate": 0.0002755143499827035,
"loss_nats": 1.3004740476608276,
"step": 1880,
"unclipped_grad_norm": 0.627357006072998
},
{
"bits_per_byte": 1.9627021280030534,
"learning_rate": 0.0002751929048534456,
"loss_nats": 1.3604414463043213,
"step": 1890,
"unclipped_grad_norm": 0.6769031882286072
},
{
"bits_per_byte": 1.87336023656754,
"learning_rate": 0.0002748695768314763,
"loss_nats": 1.2985143661499023,
"step": 1900,
"unclipped_grad_norm": 0.6752462983131409
},
{
"bits_per_byte": 1.8594791728511009,
"learning_rate": 0.00027454437144157947,
"loss_nats": 1.2888927459716797,
"step": 1910,
"unclipped_grad_norm": 0.6314391493797302
},
{
"bits_per_byte": 1.9380384400081196,
"learning_rate": 0.00027421729424061787,
"loss_nats": 1.3433458805084229,
"step": 1920,
"unclipped_grad_norm": 0.6901462078094482
},
{
"bits_per_byte": 1.8913296718602894,
"learning_rate": 0.0002738883508174385,
"loss_nats": 1.3109698295593262,
"step": 1930,
"unclipped_grad_norm": 0.6903179287910461
},
{
"bits_per_byte": 1.8624152606666307,
"learning_rate": 0.0002735575467927768,
"loss_nats": 1.2909278869628906,
"step": 1940,
"unclipped_grad_norm": 0.6572144627571106
},
{
"bits_per_byte": 1.930714558821127,
"learning_rate": 0.0002732248878191609,
"loss_nats": 1.3382693529129028,
"step": 1950,
"unclipped_grad_norm": 0.7911640405654907
},
{
"bits_per_byte": 1.901616298173696,
"learning_rate": 0.0002728903795808148,
"loss_nats": 1.3180999755859375,
"step": 1960,
"unclipped_grad_norm": 0.7476807236671448
},
{
"bits_per_byte": 1.8545080143279113,
"learning_rate": 0.0002725540277935614,
"loss_nats": 1.2854470014572144,
"step": 1970,
"unclipped_grad_norm": 0.6583976745605469
},
{
"bits_per_byte": 1.8296589290353475,
"learning_rate": 0.00027221583820472456,
"loss_nats": 1.2682229280471802,
"step": 1980,
"unclipped_grad_norm": 0.6357024908065796
},
{
"bits_per_byte": 1.9398576724889465,
"learning_rate": 0.0002718758165930314,
"loss_nats": 1.344606876373291,
"step": 1990,
"unclipped_grad_norm": 0.7643471956253052
},
{
"bits_per_byte": 1.866766077768158,
"learning_rate": 0.0002715339687685131,
"loss_nats": 1.2939436435699463,
"step": 2000,
"unclipped_grad_norm": 0.682447612285614
},
{
"bits_per_byte": 1.9452981716663407,
"learning_rate": 0.00027119030057240576,
"loss_nats": 1.3483779430389404,
"step": 2010,
"unclipped_grad_norm": 0.6716283559799194
},
{
"bits_per_byte": 1.922642553102881,
"learning_rate": 0.0002708448178770506,
"loss_nats": 1.332674264907837,
"step": 2020,
"unclipped_grad_norm": 0.7301247119903564
},
{
"bits_per_byte": 1.9056240098867518,
"learning_rate": 0.00027049752658579376,
"loss_nats": 1.3208779096603394,
"step": 2030,
"unclipped_grad_norm": 0.7755060791969299
},
{
"bits_per_byte": 1.9283054258478407,
"learning_rate": 0.00027014843263288514,
"loss_nats": 1.3365994691848755,
"step": 2040,
"unclipped_grad_norm": 0.648145854473114
},
{
"bits_per_byte": 1.8707481640662356,
"learning_rate": 0.00026979754198337724,
"loss_nats": 1.296703815460205,
"step": 2050,
"unclipped_grad_norm": 0.6962859630584717
},
{
"bits_per_byte": 1.774047478965604,
"learning_rate": 0.00026944486063302305,
"loss_nats": 1.2296760082244873,
"step": 2060,
"unclipped_grad_norm": 0.605278730392456
},
{
"bits_per_byte": 1.8770535639948187,
"learning_rate": 0.0002690903946081738,
"loss_nats": 1.3010743856430054,
"step": 2070,
"unclipped_grad_norm": 0.6994346976280212
},
{
"bits_per_byte": 1.9817110264376179,
"learning_rate": 0.00026873414996567596,
"loss_nats": 1.37361741065979,
"step": 2080,
"unclipped_grad_norm": 0.7077776789665222
},
{
"bits_per_byte": 1.8423689628820616,
"learning_rate": 0.00026837613279276736,
"loss_nats": 1.2770328521728516,
"step": 2090,
"unclipped_grad_norm": 0.6188594698905945
},
{
"bits_per_byte": 1.9159211271418608,
"learning_rate": 0.0002680163492069739,
"loss_nats": 1.3280153274536133,
"step": 2100,
"unclipped_grad_norm": 0.6360442638397217
},
{
"bits_per_byte": 1.8300450300865345,
"learning_rate": 0.00026765480535600414,
"loss_nats": 1.2684905529022217,
"step": 2110,
"unclipped_grad_norm": 0.6518478393554688
},
{
"bits_per_byte": 1.8775524856649717,
"learning_rate": 0.0002672915074176452,
"loss_nats": 1.3014202117919922,
"step": 2120,
"unclipped_grad_norm": 0.677630603313446
},
{
"bits_per_byte": 1.88719968048247,
"learning_rate": 0.00026692646159965624,
"loss_nats": 1.3081071376800537,
"step": 2130,
"unclipped_grad_norm": 0.6904116868972778
},
{
"bits_per_byte": 1.7861939256654409,
"learning_rate": 0.0002665596741396633,
"loss_nats": 1.2380952835083008,
"step": 2140,
"unclipped_grad_norm": 0.5910601615905762
},
{
"bits_per_byte": 1.8339614190119824,
"learning_rate": 0.000266191151305052,
"loss_nats": 1.271205186843872,
"step": 2150,
"unclipped_grad_norm": 0.7349988222122192
},
{
"bits_per_byte": 1.8980882460738961,
"learning_rate": 0.00026582089939286075,
"loss_nats": 1.3156545162200928,
"step": 2160,
"unclipped_grad_norm": 0.6920630931854248
},
{
"bits_per_byte": 1.8727663804741106,
"learning_rate": 0.0002654489247296732,
"loss_nats": 1.2981027364730835,
"step": 2170,
"unclipped_grad_norm": 0.7049499750137329
},
{
"bits_per_byte": 1.7856067768953952,
"learning_rate": 0.00026507523367151007,
"loss_nats": 1.2376883029937744,
"step": 2180,
"unclipped_grad_norm": 0.652675211429596
},
{
"bits_per_byte": 1.9184699100276477,
"learning_rate": 0.00026469983260372037,
"loss_nats": 1.3297820091247559,
"step": 2190,
"unclipped_grad_norm": 0.6759907007217407
},
{
"bits_per_byte": 1.9474407315307896,
"learning_rate": 0.0002643227279408727,
"loss_nats": 1.349863052368164,
"step": 2200,
"unclipped_grad_norm": 0.6637232899665833
},
{
"bits_per_byte": 1.9672947527116937,
"learning_rate": 0.0002639439261266453,
"loss_nats": 1.3636248111724854,
"step": 2210,
"unclipped_grad_norm": 0.7057329416275024
},
{
"bits_per_byte": 1.8580620358079913,
"learning_rate": 0.000263563433633716,
"loss_nats": 1.2879104614257812,
"step": 2220,
"unclipped_grad_norm": 0.6423102021217346
},
{
"bits_per_byte": 1.8639622446111417,
"learning_rate": 0.00026318125696365156,
"loss_nats": 1.2920001745224,
"step": 2230,
"unclipped_grad_norm": 0.664786159992218
},
{
"bits_per_byte": 1.8281711071227333,
"learning_rate": 0.000262797402646797,
"loss_nats": 1.2671916484832764,
"step": 2240,
"unclipped_grad_norm": 0.6536247730255127
},
{
"bits_per_byte": 1.8379201156695426,
"learning_rate": 0.00026241187724216334,
"loss_nats": 1.273949146270752,
"step": 2250,
"unclipped_grad_norm": 0.6435070633888245
},
{
"bits_per_byte": 1.8619839281782669,
"learning_rate": 0.0002620246873373162,
"loss_nats": 1.2906289100646973,
"step": 2260,
"unclipped_grad_norm": 0.6457217335700989
},
{
"bits_per_byte": 1.9082623957336382,
"learning_rate": 0.00026163583954826264,
"loss_nats": 1.322706699371338,
"step": 2270,
"unclipped_grad_norm": 0.6787397861480713
},
{
"bits_per_byte": 1.7849188462919439,
"learning_rate": 0.0002612453405193386,
"loss_nats": 1.2372114658355713,
"step": 2280,
"unclipped_grad_norm": 0.6086795330047607
},
{
"bits_per_byte": 1.8290736720744614,
"learning_rate": 0.000260853196923095,
"loss_nats": 1.2678172588348389,
"step": 2290,
"unclipped_grad_norm": 0.630083441734314
},
{
"bits_per_byte": 1.8820873242029221,
"learning_rate": 0.0002604594154601839,
"loss_nats": 1.3045635223388672,
"step": 2300,
"unclipped_grad_norm": 0.6462850570678711
},
{
"bits_per_byte": 1.834706619838171,
"learning_rate": 0.0002600640028592441,
"loss_nats": 1.2717217206954956,
"step": 2310,
"unclipped_grad_norm": 0.6394227743148804
},
{
"bits_per_byte": 1.8546760413778043,
"learning_rate": 0.0002596669658767859,
"loss_nats": 1.2855634689331055,
"step": 2320,
"unclipped_grad_norm": 0.6594879031181335
},
{
"bits_per_byte": 1.7978681080060082,
"learning_rate": 0.000259268311297076,
"loss_nats": 1.2461872100830078,
"step": 2330,
"unclipped_grad_norm": 0.6738982796669006
},
{
"bits_per_byte": 1.8462244699491042,
"learning_rate": 0.00025886804593202114,
"loss_nats": 1.279705286026001,
"step": 2340,
"unclipped_grad_norm": 0.687351405620575
},
{
"bits_per_byte": 1.7549572327371814,
"learning_rate": 0.00025846617662105193,
"loss_nats": 1.216443657875061,
"step": 2350,
"unclipped_grad_norm": 0.6065738201141357
},
{
"bits_per_byte": 1.8467686230564342,
"learning_rate": 0.00025806271023100637,
"loss_nats": 1.2800824642181396,
"step": 2360,
"unclipped_grad_norm": 0.676578938961029
},
{
"bits_per_byte": 1.8782005162934228,
"learning_rate": 0.00025765765365601165,
"loss_nats": 1.3018693923950195,
"step": 2370,
"unclipped_grad_norm": 0.6810930967330933
},
{
"bits_per_byte": 1.7780921669485956,
"learning_rate": 0.00025725101381736707,
"loss_nats": 1.2324795722961426,
"step": 2380,
"unclipped_grad_norm": 0.5974810719490051
},
{
"bits_per_byte": 1.7797094917973095,
"learning_rate": 0.00025684279766342567,
"loss_nats": 1.2336006164550781,
"step": 2390,
"unclipped_grad_norm": 0.6242655515670776
},
{
"bits_per_byte": 1.8773909939558115,
"learning_rate": 0.00025643301216947504,
"loss_nats": 1.301308274269104,
"step": 2400,
"unclipped_grad_norm": 0.7230958342552185
},
{
"bits_per_byte": 1.843124310684651,
"learning_rate": 0.00025602166433761866,
"loss_nats": 1.2775564193725586,
"step": 2410,
"unclipped_grad_norm": 0.6579370498657227
},
{
"bits_per_byte": 1.9371601246101633,
"learning_rate": 0.0002556087611966561,
"loss_nats": 1.342737078666687,
"step": 2420,
"unclipped_grad_norm": 0.6557210087776184
},
{
"bits_per_byte": 1.8373321069862427,
"learning_rate": 0.00025519430980196277,
"loss_nats": 1.2735415697097778,
"step": 2430,
"unclipped_grad_norm": 0.6670473217964172
},
{
"bits_per_byte": 1.7749123797167932,
"learning_rate": 0.0002547783172353695,
"loss_nats": 1.2302755117416382,
"step": 2440,
"unclipped_grad_norm": 0.6538338661193848
},
{
"bits_per_byte": 1.8761522029216466,
"learning_rate": 0.0002543607906050415,
"loss_nats": 1.3004496097564697,
"step": 2450,
"unclipped_grad_norm": 0.6531840562820435
},
{
"bits_per_byte": 1.8790119304401935,
"learning_rate": 0.0002539417370453568,
"loss_nats": 1.3024318218231201,
"step": 2460,
"unclipped_grad_norm": 0.6944200992584229
},
{
"bits_per_byte": 1.8171404838616947,
"learning_rate": 0.00025352116371678464,
"loss_nats": 1.2595458030700684,
"step": 2470,
"unclipped_grad_norm": 0.6366217136383057
},
{
"bits_per_byte": 1.8241991678010567,
"learning_rate": 0.0002530990778057627,
"loss_nats": 1.264438509941101,
"step": 2480,
"unclipped_grad_norm": 0.6316180229187012
},
{
"bits_per_byte": 1.7491533342185142,
"learning_rate": 0.00025267548652457475,
"loss_nats": 1.2124207019805908,
"step": 2490,
"unclipped_grad_norm": 0.6618465185165405
},
{
"bits_per_byte": 1.827854315079844,
"learning_rate": 0.00025225039711122696,
"loss_nats": 1.2669720649719238,
"step": 2500,
"unclipped_grad_norm": 0.6852855682373047
},
{
"bits_per_byte": 1.8052024801347035,
"learning_rate": 0.00025182381682932464,
"loss_nats": 1.2512710094451904,
"step": 2510,
"unclipped_grad_norm": 0.663596510887146
},
{
"bits_per_byte": 1.7913451500240836,
"learning_rate": 0.0002513957529679477,
"loss_nats": 1.2416658401489258,
"step": 2520,
"unclipped_grad_norm": 0.6331848502159119
},
{
"bits_per_byte": 1.7932974970766782,
"learning_rate": 0.00025096621284152656,
"loss_nats": 1.2430191040039062,
"step": 2530,
"unclipped_grad_norm": 0.6864507794380188
},
{
"bits_per_byte": 1.8207522915124645,
"learning_rate": 0.0002505352037897168,
"loss_nats": 1.2620493173599243,
"step": 2540,
"unclipped_grad_norm": 0.6478230953216553
},
{
"bits_per_byte": 1.8469901367107453,
"learning_rate": 0.0002501027331772739,
"loss_nats": 1.280236005783081,
"step": 2550,
"unclipped_grad_norm": 0.6545907258987427
},
{
"bits_per_byte": 1.7922375679994107,
"learning_rate": 0.00024966880839392746,
"loss_nats": 1.2422844171524048,
"step": 2560,
"unclipped_grad_norm": 0.6228907704353333
},
{
"bits_per_byte": 1.783868376260474,
"learning_rate": 0.0002492334368542548,
"loss_nats": 1.2364833354949951,
"step": 2570,
"unclipped_grad_norm": 0.6730257868766785
},
{
"bits_per_byte": 1.8186578867902572,
"learning_rate": 0.00024879662599755416,
"loss_nats": 1.260597586631775,
"step": 2580,
"unclipped_grad_norm": 0.6890478730201721
},
{
"bits_per_byte": 1.7885676302126492,
"learning_rate": 0.00024835838328771803,
"loss_nats": 1.2397406101226807,
"step": 2590,
"unclipped_grad_norm": 0.6227992177009583
},
{
"bits_per_byte": 1.8280013602463319,
"learning_rate": 0.0002479187162131051,
"loss_nats": 1.2670739889144897,
"step": 2600,
"unclipped_grad_norm": 0.6276137828826904
},
{
"bits_per_byte": 1.7336515060003441,
"learning_rate": 0.00024747763228641264,
"loss_nats": 1.2016756534576416,
"step": 2610,
"unclipped_grad_norm": 0.6445866823196411
},
{
"bits_per_byte": 1.7935037042750628,
"learning_rate": 0.00024703513904454795,
"loss_nats": 1.2431620359420776,
"step": 2620,
"unclipped_grad_norm": 0.6606776714324951
},
{
"bits_per_byte": 1.8032969123631437,
"learning_rate": 0.00024659124404849956,
"loss_nats": 1.2499501705169678,
"step": 2630,
"unclipped_grad_norm": 0.6334376931190491
},
{
"bits_per_byte": 1.810602563389145,
"learning_rate": 0.00024614595488320833,
"loss_nats": 1.2550140619277954,
"step": 2640,
"unclipped_grad_norm": 0.6691069006919861
},
{
"bits_per_byte": 1.7587798911179091,
"learning_rate": 0.00024569927915743745,
"loss_nats": 1.2190933227539062,
"step": 2650,
"unclipped_grad_norm": 0.6500447988510132
},
{
"bits_per_byte": 1.7476075541525593,
"learning_rate": 0.0002452512245036427,
"loss_nats": 1.2113492488861084,
"step": 2660,
"unclipped_grad_norm": 0.5952810049057007
},
{
"bits_per_byte": 1.7865554331975544,
"learning_rate": 0.00024480179857784177,
"loss_nats": 1.2383458614349365,
"step": 2670,
"unclipped_grad_norm": 0.6339060664176941
},
{
"bits_per_byte": 1.7817725956770598,
"learning_rate": 0.00024435100905948387,
"loss_nats": 1.2350306510925293,
"step": 2680,
"unclipped_grad_norm": 0.6211492419242859
},
{
"bits_per_byte": 1.8492028654967463,
"learning_rate": 0.00024389886365131793,
"loss_nats": 1.2817697525024414,
"step": 2690,
"unclipped_grad_norm": 0.6305774450302124
},
{
"bits_per_byte": 1.8342178451444189,
"learning_rate": 0.00024344537007926163,
"loss_nats": 1.2713829278945923,
"step": 2700,
"unclipped_grad_norm": 0.6566506624221802
},
{
"bits_per_byte": 1.7120219359319322,
"learning_rate": 0.00024299053609226878,
"loss_nats": 1.186683177947998,
"step": 2710,
"unclipped_grad_norm": 0.6094632744789124
},
{
"bits_per_byte": 1.8346758349436665,
"learning_rate": 0.00024253436946219733,
"loss_nats": 1.271700382232666,
"step": 2720,
"unclipped_grad_norm": 0.6520996689796448
},
{
"bits_per_byte": 1.7481449999365055,
"learning_rate": 0.0002420768779836764,
"loss_nats": 1.2117217779159546,
"step": 2730,
"unclipped_grad_norm": 0.598540186882019
},
{
"bits_per_byte": 1.7678683143027543,
"learning_rate": 0.00024161806947397305,
"loss_nats": 1.2253929376602173,
"step": 2740,
"unclipped_grad_norm": 0.6463729739189148
},
{
"bits_per_byte": 1.7756850977671197,
"learning_rate": 0.00024115795177285886,
"loss_nats": 1.2308111190795898,
"step": 2750,
"unclipped_grad_norm": 0.6158115863800049
},
{
"bits_per_byte": 1.8688916113501715,
"learning_rate": 0.0002406965327424758,
"loss_nats": 1.2954169511795044,
"step": 2760,
"unclipped_grad_norm": 0.6701047420501709
},
{
"bits_per_byte": 1.7889463360098492,
"learning_rate": 0.00024023382026720197,
"loss_nats": 1.2400031089782715,
"step": 2770,
"unclipped_grad_norm": 0.6446090936660767
},
{
"bits_per_byte": 1.7765037351852266,
"learning_rate": 0.00023976982225351698,
"loss_nats": 1.2313785552978516,
"step": 2780,
"unclipped_grad_norm": 0.637730598449707
},
{
"bits_per_byte": 1.7761525466121648,
"learning_rate": 0.0002393045466298666,
"loss_nats": 1.2311351299285889,
"step": 2790,
"unclipped_grad_norm": 0.7148120403289795
},
{
"bits_per_byte": 1.7209247898364963,
"learning_rate": 0.00023883800134652763,
"loss_nats": 1.1928541660308838,
"step": 2800,
"unclipped_grad_norm": 0.6417195200920105
},
{
"bits_per_byte": 1.792574482012451,
"learning_rate": 0.0002383701943754715,
"loss_nats": 1.2425179481506348,
"step": 2810,
"unclipped_grad_norm": 0.6152215003967285
},
{
"bits_per_byte": 1.7960922151531988,
"learning_rate": 0.00023790113371022884,
"loss_nats": 1.2449562549591064,
"step": 2820,
"unclipped_grad_norm": 0.6487079858779907
},
{
"bits_per_byte": 1.72477857707703,
"learning_rate": 0.0002374308273657521,
"loss_nats": 1.1955254077911377,
"step": 2830,
"unclipped_grad_norm": 0.6001560091972351
},
{
"bits_per_byte": 1.807270399528678,
"learning_rate": 0.00023695928337827911,
"loss_nats": 1.252704381942749,
"step": 2840,
"unclipped_grad_norm": 0.6889218688011169
},
{
"bits_per_byte": 1.7224509638802528,
"learning_rate": 0.00023648650980519564,
"loss_nats": 1.1939120292663574,
"step": 2850,
"unclipped_grad_norm": 0.6565119028091431
},
{
"bits_per_byte": 1.7068036383394527,
"learning_rate": 0.00023601251472489757,
"loss_nats": 1.1830661296844482,
"step": 2860,
"unclipped_grad_norm": 0.637535035610199
},
{
"bits_per_byte": 1.8267986855688483,
"learning_rate": 0.00023553730623665305,
"loss_nats": 1.2662403583526611,
"step": 2870,
"unclipped_grad_norm": 0.635680615901947
},
{
"bits_per_byte": 1.7536737262137922,
"learning_rate": 0.00023506089246046403,
"loss_nats": 1.2155539989471436,
"step": 2880,
"unclipped_grad_norm": 0.6707039475440979
},
{
"bits_per_byte": 1.7805597740231751,
"learning_rate": 0.00023458328153692736,
"loss_nats": 1.2341899871826172,
"step": 2890,
"unclipped_grad_norm": 0.6349734663963318
},
{
"bits_per_byte": 1.713862494261466,
"learning_rate": 0.0002341044816270961,
"loss_nats": 1.1879589557647705,
"step": 2900,
"unclipped_grad_norm": 0.6683149337768555
},
{
"bits_per_byte": 1.7497669683167927,
"learning_rate": 0.00023362450091233953,
"loss_nats": 1.212846040725708,
"step": 2910,
"unclipped_grad_norm": 0.6716704368591309
},
{
"bits_per_byte": 1.8262012178397509,
"learning_rate": 0.00023314334759420388,
"loss_nats": 1.2658262252807617,
"step": 2920,
"unclipped_grad_norm": 0.6724333763122559
},
{
"bits_per_byte": 1.761767401746047,
"learning_rate": 0.0002326610298942717,
"loss_nats": 1.2211641073226929,
"step": 2930,
"unclipped_grad_norm": 0.6612268686294556
},
{
"bits_per_byte": 1.8132794733498245,
"learning_rate": 0.00023217755605402175,
"loss_nats": 1.2568695545196533,
"step": 2940,
"unclipped_grad_norm": 0.6505576968193054
},
{
"bits_per_byte": 1.7731756989083804,
"learning_rate": 0.00023169293433468802,
"loss_nats": 1.2290717363357544,
"step": 2950,
"unclipped_grad_norm": 0.6778621077537537
},
{
"bits_per_byte": 1.7513619354208945,
"learning_rate": 0.00023120717301711857,
"loss_nats": 1.213951587677002,
"step": 2960,
"unclipped_grad_norm": 0.6551554203033447
},
{
"bits_per_byte": 1.8079858473562673,
"learning_rate": 0.00023072028040163408,
"loss_nats": 1.2532002925872803,
"step": 2970,
"unclipped_grad_norm": 0.6640533208847046
},
{
"bits_per_byte": 1.7386847502604952,
"learning_rate": 0.00023023226480788593,
"loss_nats": 1.2051644325256348,
"step": 2980,
"unclipped_grad_norm": 0.643166184425354
},
{
"bits_per_byte": 1.8672367942835695,
"learning_rate": 0.00022974313457471416,
"loss_nats": 1.2942699193954468,
"step": 2990,
"unclipped_grad_norm": 0.6637417078018188
},
{
"bits_per_byte": 1.7982774267150616,
"learning_rate": 0.0002292528980600049,
"loss_nats": 1.2464709281921387,
"step": 3000,
"unclipped_grad_norm": 0.6690325736999512
},
{
"bits_per_byte": 1.762204753627191,
"learning_rate": 0.00022876156364054758,
"loss_nats": 1.2214672565460205,
"step": 3010,
"unclipped_grad_norm": 0.6516304612159729
},
{
"bits_per_byte": 1.7294503138050674,
"learning_rate": 0.0002282691397118918,
"loss_nats": 1.1987636089324951,
"step": 3020,
"unclipped_grad_norm": 0.6919088363647461
},
{
"bits_per_byte": 1.7609028449601596,
"learning_rate": 0.00022777563468820376,
"loss_nats": 1.220564842224121,
"step": 3030,
"unclipped_grad_norm": 0.6205207109451294
},
{
"bits_per_byte": 1.7350170482481948,
"learning_rate": 0.00022728105700212282,
"loss_nats": 1.2026221752166748,
"step": 3040,
"unclipped_grad_norm": 0.7004191279411316
},
{
"bits_per_byte": 1.7472788953067604,
"learning_rate": 0.00022678541510461698,
"loss_nats": 1.2111214399337769,
"step": 3050,
"unclipped_grad_norm": 0.6576176881790161
},
{
"bits_per_byte": 1.768669753455775,
"learning_rate": 0.00022628871746483876,
"loss_nats": 1.225948452949524,
"step": 3060,
"unclipped_grad_norm": 0.7110167145729065
},
{
"bits_per_byte": 1.7354397816040157,
"learning_rate": 0.0002257909725699804,
"loss_nats": 1.2029151916503906,
"step": 3070,
"unclipped_grad_norm": 0.6717135906219482
},
{
"bits_per_byte": 1.748681069859245,
"learning_rate": 0.00022529218892512886,
"loss_nats": 1.2120933532714844,
"step": 3080,
"unclipped_grad_norm": 0.6585274934768677
},
{
"bits_per_byte": 1.7073046238014162,
"learning_rate": 0.00022479237505312034,
"loss_nats": 1.1834133863449097,
"step": 3090,
"unclipped_grad_norm": 0.6292386651039124
},
{
"bits_per_byte": 1.8085337840819162,
"learning_rate": 0.00022429153949439496,
"loss_nats": 1.253580093383789,
"step": 3100,
"unclipped_grad_norm": 0.6514226198196411
},
{
"bits_per_byte": 1.7739243393875863,
"learning_rate": 0.00022378969080685053,
"loss_nats": 1.229590654373169,
"step": 3110,
"unclipped_grad_norm": 0.6451593041419983
},
{
"bits_per_byte": 1.6671654209338944,
"learning_rate": 0.0002232868375656965,
"loss_nats": 1.1555910110473633,
"step": 3120,
"unclipped_grad_norm": 0.6327124834060669
},
{
"bits_per_byte": 1.7555804978639082,
"learning_rate": 0.00022278298836330733,
"loss_nats": 1.216875672340393,
"step": 3130,
"unclipped_grad_norm": 0.6219494342803955
},
{
"bits_per_byte": 1.7189500850392896,
"learning_rate": 0.00022227815180907572,
"loss_nats": 1.1914854049682617,
"step": 3140,
"unclipped_grad_norm": 0.6722466349601746
},
{
"bits_per_byte": 1.6998178750440565,
"learning_rate": 0.00022177233652926546,
"loss_nats": 1.178223967552185,
"step": 3150,
"unclipped_grad_norm": 0.6060081720352173
},
{
"bits_per_byte": 1.7570095017099274,
"learning_rate": 0.00022126555116686416,
"loss_nats": 1.2178661823272705,
"step": 3160,
"unclipped_grad_norm": 0.6486148834228516
},
{
"bits_per_byte": 1.8187225522669817,
"learning_rate": 0.00022075780438143532,
"loss_nats": 1.260642409324646,
"step": 3170,
"unclipped_grad_norm": 0.6782575845718384
},
{
"bits_per_byte": 1.680190870962291,
"learning_rate": 0.00022024910484897065,
"loss_nats": 1.1646195650100708,
"step": 3180,
"unclipped_grad_norm": 0.6483905911445618
},
{
"bits_per_byte": 1.7650935462137336,
"learning_rate": 0.00021973946126174158,
"loss_nats": 1.223469614982605,
"step": 3190,
"unclipped_grad_norm": 0.7115707993507385
},
{
"bits_per_byte": 1.6804184040093824,
"learning_rate": 0.00021922888232815086,
"loss_nats": 1.1647772789001465,
"step": 3200,
"unclipped_grad_norm": 0.6299659609794617
},
{
"bits_per_byte": 1.7133916057634035,
"learning_rate": 0.0002187173767725837,
"loss_nats": 1.1876325607299805,
"step": 3210,
"unclipped_grad_norm": 0.6491534113883972
},
{
"bits_per_byte": 1.710321027514899,
"learning_rate": 0.0002182049533352588,
"loss_nats": 1.1855041980743408,
"step": 3220,
"unclipped_grad_norm": 0.6104377508163452
},
{
"bits_per_byte": 1.6818119794293238,
"learning_rate": 0.00021769162077207884,
"loss_nats": 1.1657432317733765,
"step": 3230,
"unclipped_grad_norm": 0.6708391308784485
},
{
"bits_per_byte": 1.7427318460005987,
"learning_rate": 0.00021717738785448102,
"loss_nats": 1.2079696655273438,
"step": 3240,
"unclipped_grad_norm": 0.6418254375457764
},
{
"bits_per_byte": 1.733884714474914,
"learning_rate": 0.00021666226336928708,
"loss_nats": 1.2018373012542725,
"step": 3250,
"unclipped_grad_norm": 0.6041762828826904
},
{
"bits_per_byte": 1.83323737205185,
"learning_rate": 0.0002161462561185532,
"loss_nats": 1.2707033157348633,
"step": 3260,
"unclipped_grad_norm": 0.7252490520477295
},
{
"bits_per_byte": 1.6948699341787334,
"learning_rate": 0.00021562937491941957,
"loss_nats": 1.174794316291809,
"step": 3270,
"unclipped_grad_norm": 0.6451477408409119
},
{
"bits_per_byte": 1.6338448142251303,
"learning_rate": 0.00021511162860395975,
"loss_nats": 1.1324949264526367,
"step": 3280,
"unclipped_grad_norm": 0.6251513361930847
},
{
"bits_per_byte": 1.6883934115125416,
"learning_rate": 0.0002145930260190298,
"loss_nats": 1.1703051328659058,
"step": 3290,
"unclipped_grad_norm": 0.6296114921569824
},
{
"bits_per_byte": 1.744128001160303,
"learning_rate": 0.00021407357602611698,
"loss_nats": 1.208937406539917,
"step": 3300,
"unclipped_grad_norm": 0.6343309283256531
},
{
"bits_per_byte": 1.735473146238283,
"learning_rate": 0.0002135532875011885,
"loss_nats": 1.2029383182525635,
"step": 3310,
"unclipped_grad_norm": 0.6353676915168762
},
{
"bits_per_byte": 1.7632710460625405,
"learning_rate": 0.0002130321693345397,
"loss_nats": 1.2222063541412354,
"step": 3320,
"unclipped_grad_norm": 0.6778877973556519
},
{
"bits_per_byte": 1.6989030993241172,
"learning_rate": 0.00021251023043064224,
"loss_nats": 1.1775898933410645,
"step": 3330,
"unclipped_grad_norm": 0.629153847694397
},
{
"bits_per_byte": 1.7071973066272779,
"learning_rate": 0.00021198747970799188,
"loss_nats": 1.18333899974823,
"step": 3340,
"unclipped_grad_norm": 0.6583864688873291
},
{
"bits_per_byte": 1.70429750715108,
"learning_rate": 0.00021146392609895618,
"loss_nats": 1.1813290119171143,
"step": 3350,
"unclipped_grad_norm": 0.6497669816017151
},
{
"bits_per_byte": 1.7138317093669615,
"learning_rate": 0.00021093957854962178,
"loss_nats": 1.187937617301941,
"step": 3360,
"unclipped_grad_norm": 0.6299119591712952
},
{
"bits_per_byte": 1.6462306607749682,
"learning_rate": 0.00021041444601964157,
"loss_nats": 1.1410801410675049,
"step": 3370,
"unclipped_grad_norm": 0.6380488872528076
},
{
"bits_per_byte": 1.7443316286189245,
"learning_rate": 0.00020988853748208164,
"loss_nats": 1.2090785503387451,
"step": 3380,
"unclipped_grad_norm": 0.702843189239502
},
{
"bits_per_byte": 1.7228774808543925,
"learning_rate": 0.00020936186192326782,
"loss_nats": 1.1942076683044434,
"step": 3390,
"unclipped_grad_norm": 0.6957465410232544
},
{
"bits_per_byte": 1.7111405248462603,
"learning_rate": 0.00020883442834263225,
"loss_nats": 1.1860722303390503,
"step": 3400,
"unclipped_grad_norm": 0.6491814255714417
},
{
"bits_per_byte": 1.7448699343161251,
"learning_rate": 0.0002083062457525596,
"loss_nats": 1.209451675415039,
"step": 3410,
"unclipped_grad_norm": 0.6583186388015747
},
{
"bits_per_byte": 1.7129637129280568,
"learning_rate": 0.00020777732317823306,
"loss_nats": 1.1873359680175781,
"step": 3420,
"unclipped_grad_norm": 0.6641048192977905
},
{
"bits_per_byte": 1.670370317632723,
"learning_rate": 0.00020724766965748005,
"loss_nats": 1.157812476158142,
"step": 3430,
"unclipped_grad_norm": 0.629267692565918
},
{
"bits_per_byte": 1.6804600238108913,
"learning_rate": 0.00020671729424061788,
"loss_nats": 1.1648061275482178,
"step": 3440,
"unclipped_grad_norm": 0.6077277064323425
},
{
"bits_per_byte": 1.6713492428814338,
"learning_rate": 0.00020618620599029906,
"loss_nats": 1.1584910154342651,
"step": 3450,
"unclipped_grad_norm": 0.6522519588470459
},
{
"bits_per_byte": 1.669551508231965,
"learning_rate": 0.00020565441398135649,
"loss_nats": 1.1572449207305908,
"step": 3460,
"unclipped_grad_norm": 0.627102255821228
},
{
"bits_per_byte": 1.7251744811393164,
"learning_rate": 0.00020512192730064827,
"loss_nats": 1.1957998275756836,
"step": 3470,
"unclipped_grad_norm": 0.6381968855857849
},
{
"bits_per_byte": 1.7823232841251224,
"learning_rate": 0.00020458875504690273,
"loss_nats": 1.235412359237671,
"step": 3480,
"unclipped_grad_norm": 0.6999886631965637
},
{
"bits_per_byte": 1.718624865846508,
"learning_rate": 0.00020405490633056245,
"loss_nats": 1.1912599802017212,
"step": 3490,
"unclipped_grad_norm": 0.6294557452201843
},
{
"bits_per_byte": 1.6614911973339777,
"learning_rate": 0.00020352039027362917,
"loss_nats": 1.1516579389572144,
"step": 3500,
"unclipped_grad_norm": 0.6351874470710754
},
{
"bits_per_byte": 1.6797325371977416,
"learning_rate": 0.0002029852160095075,
"loss_nats": 1.164301872253418,
"step": 3510,
"unclipped_grad_norm": 0.6315440535545349
},
{
"bits_per_byte": 1.717181759423118,
"learning_rate": 0.000202449392682849,
"loss_nats": 1.1902596950531006,
"step": 3520,
"unclipped_grad_norm": 0.6176374554634094
},
{
"bits_per_byte": 1.6772098956748858,
"learning_rate": 0.0002019129294493959,
"loss_nats": 1.162553310394287,
"step": 3530,
"unclipped_grad_norm": 0.6610027551651001
},
{
"bits_per_byte": 1.6855345439072493,
"learning_rate": 0.00020137583547582478,
"loss_nats": 1.1683235168457031,
"step": 3540,
"unclipped_grad_norm": 0.658635139465332
},
{
"bits_per_byte": 1.747215777673894,
"learning_rate": 0.00020083811993958973,
"loss_nats": 1.2110776901245117,
"step": 3550,
"unclipped_grad_norm": 0.6718185544013977
},
{
"bits_per_byte": 1.666718266041651,
"learning_rate": 0.00020029979202876563,
"loss_nats": 1.1552810668945312,
"step": 3560,
"unclipped_grad_norm": 0.6717336773872375
},
{
"bits_per_byte": 1.7121595220526225,
"learning_rate": 0.00019976086094189118,
"loss_nats": 1.1867785453796387,
"step": 3570,
"unclipped_grad_norm": 0.6859359741210938
},
{
"bits_per_byte": 1.739672790589702,
"learning_rate": 0.00019922133588781175,
"loss_nats": 1.205849289894104,
"step": 3580,
"unclipped_grad_norm": 0.6660984754562378
},
{
"bits_per_byte": 1.6956691375572928,
"learning_rate": 0.0001986812260855219,
"loss_nats": 1.1753482818603516,
"step": 3590,
"unclipped_grad_norm": 0.6079815030097961
},
{
"bits_per_byte": 1.7521809168043032,
"learning_rate": 0.00019814054076400787,
"loss_nats": 1.2145192623138428,
"step": 3600,
"unclipped_grad_norm": 0.7036373615264893
},
{
"bits_per_byte": 1.7346027420422663,
"learning_rate": 0.00019759928916209004,
"loss_nats": 1.202335000038147,
"step": 3610,
"unclipped_grad_norm": 0.6451415419578552
},
{
"bits_per_byte": 1.6701776970637565,
"learning_rate": 0.00019705748052826493,
"loss_nats": 1.1576789617538452,
"step": 3620,
"unclipped_grad_norm": 0.620152473449707
},
{
"bits_per_byte": 1.6714409096343439,
"learning_rate": 0.00019651512412054723,
"loss_nats": 1.1585545539855957,
"step": 3630,
"unclipped_grad_norm": 0.6908113956451416
},
{
"bits_per_byte": 1.6125346159393705,
"learning_rate": 0.00019597222920631144,
"loss_nats": 1.117723822593689,
"step": 3640,
"unclipped_grad_norm": 0.604874312877655
},
{
"bits_per_byte": 1.6950919637809974,
"learning_rate": 0.0001954288050621338,
"loss_nats": 1.1749482154846191,
"step": 3650,
"unclipped_grad_norm": 0.6512246131896973
},
{
"bits_per_byte": 1.6897813115050047,
"learning_rate": 0.00019488486097363363,
"loss_nats": 1.1712671518325806,
"step": 3660,
"unclipped_grad_norm": 0.647068440914154
},
{
"bits_per_byte": 1.7204222565306753,
"learning_rate": 0.00019434040623531452,
"loss_nats": 1.1925058364868164,
"step": 3670,
"unclipped_grad_norm": 0.7147674560546875
},
{
"bits_per_byte": 1.6652036148355023,
"learning_rate": 0.0001937954501504058,
"loss_nats": 1.1542311906814575,
"step": 3680,
"unclipped_grad_norm": 0.6392054557800293
},
{
"bits_per_byte": 1.7297106955384738,
"learning_rate": 0.00019325000203070333,
"loss_nats": 1.198944091796875,
"step": 3690,
"unclipped_grad_norm": 0.7120746374130249
},
{
"bits_per_byte": 1.6313792709423611,
"learning_rate": 0.0001927040711964106,
"loss_nats": 1.1307859420776367,
"step": 3700,
"unclipped_grad_norm": 0.6207358241081238
},
{
"bits_per_byte": 1.6932649920808815,
"learning_rate": 0.00019215766697597923,
"loss_nats": 1.1736818552017212,
"step": 3710,
"unclipped_grad_norm": 0.6575222611427307
},
{
"bits_per_byte": 1.7191832935138596,
"learning_rate": 0.00019161079870594978,
"loss_nats": 1.1916470527648926,
"step": 3720,
"unclipped_grad_norm": 0.6928348541259766
},
{
"bits_per_byte": 1.6966805675270171,
"learning_rate": 0.0001910634757307922,
"loss_nats": 1.1760493516921997,
"step": 3730,
"unclipped_grad_norm": 0.6457412242889404
},
{
"bits_per_byte": 1.6651184834233252,
"learning_rate": 0.0001905157074027459,
"loss_nats": 1.1541721820831299,
"step": 3740,
"unclipped_grad_norm": 0.6149929761886597
},
{
"bits_per_byte": 1.690470445987012,
"learning_rate": 0.00018996750308166037,
"loss_nats": 1.1717448234558105,
"step": 3750,
"unclipped_grad_norm": 0.6539504528045654
},
{
"bits_per_byte": 1.7257394441474005,
"learning_rate": 0.00018941887213483482,
"loss_nats": 1.196191430091858,
"step": 3760,
"unclipped_grad_norm": 0.6810935735702515
},
{
"bits_per_byte": 1.709021182639678,
"learning_rate": 0.00018886982393685845,
"loss_nats": 1.184603214263916,
"step": 3770,
"unclipped_grad_norm": 0.6034973859786987
},
{
"bits_per_byte": 1.7700676284419878,
"learning_rate": 0.00018832036786945008,
"loss_nats": 1.2269173860549927,
"step": 3780,
"unclipped_grad_norm": 0.6868588328361511
},
{
"bits_per_byte": 1.6607432447853754,
"learning_rate": 0.0001877705133212979,
"loss_nats": 1.151139497756958,
"step": 3790,
"unclipped_grad_norm": 0.6561638712882996
},
{
"bits_per_byte": 1.6881032767805362,
"learning_rate": 0.00018722026968789906,
"loss_nats": 1.1701040267944336,
"step": 3800,
"unclipped_grad_norm": 0.6706013679504395
},
{
"bits_per_byte": 1.6338372469884925,
"learning_rate": 0.00018666964637139907,
"loss_nats": 1.1324896812438965,
"step": 3810,
"unclipped_grad_norm": 0.6373327374458313
},
{
"bits_per_byte": 1.6310115720348164,
"learning_rate": 0.00018611865278043115,
"loss_nats": 1.1305310726165771,
"step": 3820,
"unclipped_grad_norm": 0.6431099772453308
},
{
"bits_per_byte": 1.699418359346102,
"learning_rate": 0.00018556729832995555,
"loss_nats": 1.1779470443725586,
"step": 3830,
"unclipped_grad_norm": 0.6568827629089355
},
{
"bits_per_byte": 1.6534394855685337,
"learning_rate": 0.00018501559244109864,
"loss_nats": 1.1460769176483154,
"step": 3840,
"unclipped_grad_norm": 0.6673737168312073
},
{
"bits_per_byte": 1.6227488375567638,
"learning_rate": 0.00018446354454099178,
"loss_nats": 1.1248037815093994,
"step": 3850,
"unclipped_grad_norm": 0.6300650835037231
},
{
"bits_per_byte": 1.6095687751252414,
"learning_rate": 0.00018391116406261048,
"loss_nats": 1.1156680583953857,
"step": 3860,
"unclipped_grad_norm": 0.6291298270225525
},
{
"bits_per_byte": 1.5641092889532788,
"learning_rate": 0.00018335846044461305,
"loss_nats": 1.084157943725586,
"step": 3870,
"unclipped_grad_norm": 0.619691789150238
},
{
"bits_per_byte": 1.670964173726152,
"learning_rate": 0.0001828054431311793,
"loss_nats": 1.158224105834961,
"step": 3880,
"unclipped_grad_norm": 0.6558656692504883
},
{
"bits_per_byte": 1.6390438497607132,
"learning_rate": 0.00018225212157184934,
"loss_nats": 1.1360986232757568,
"step": 3890,
"unclipped_grad_norm": 0.6659069061279297
},
{
"bits_per_byte": 1.6436039697483404,
"learning_rate": 0.00018169850522136192,
"loss_nats": 1.1392594575881958,
"step": 3900,
"unclipped_grad_norm": 0.6627041101455688
},
{
"bits_per_byte": 1.6594528589559516,
"learning_rate": 0.000181144603539493,
"loss_nats": 1.1502450704574585,
"step": 3910,
"unclipped_grad_norm": 0.6734365224838257
},
{
"bits_per_byte": 1.6941402117911224,
"learning_rate": 0.00018059042599089412,
"loss_nats": 1.1742885112762451,
"step": 3920,
"unclipped_grad_norm": 0.6930717825889587
},
{
"bits_per_byte": 1.6772669939149722,
"learning_rate": 0.00018003598204493048,
"loss_nats": 1.162592887878418,
"step": 3930,
"unclipped_grad_norm": 0.7069801688194275
},
{
"bits_per_byte": 1.6121323485190024,
"learning_rate": 0.00017948128117551936,
"loss_nats": 1.1174449920654297,
"step": 3940,
"unclipped_grad_norm": 0.6367635726928711
},
{
"bits_per_byte": 1.7182599186613772,
"learning_rate": 0.00017892633286096816,
"loss_nats": 1.1910070180892944,
"step": 3950,
"unclipped_grad_norm": 0.690847635269165
},
{
"bits_per_byte": 1.6993077745015972,
"learning_rate": 0.00017837114658381247,
"loss_nats": 1.1778703927993774,
"step": 3960,
"unclipped_grad_norm": 0.6660118699073792
},
{
"bits_per_byte": 1.6431734971732308,
"learning_rate": 0.00017781573183065386,
"loss_nats": 1.1389610767364502,
"step": 3970,
"unclipped_grad_norm": 0.669096827507019
},
{
"bits_per_byte": 1.6275112091418058,
"learning_rate": 0.0001772600980919981,
"loss_nats": 1.12810480594635,
"step": 3980,
"unclipped_grad_norm": 0.6583772301673889
},
{
"bits_per_byte": 1.669720739160414,
"learning_rate": 0.0001767042548620927,
"loss_nats": 1.1573622226715088,
"step": 3990,
"unclipped_grad_norm": 0.6523779630661011
},
{
"bits_per_byte": 1.5952216384249642,
"learning_rate": 0.00017614821163876486,
"loss_nats": 1.1057233810424805,
"step": 4000,
"unclipped_grad_norm": 0.6264373064041138
},
{
"bits_per_byte": 1.6774286576067834,
"learning_rate": 0.00017559197792325906,
"loss_nats": 1.1627049446105957,
"step": 4010,
"unclipped_grad_norm": 0.6995967030525208
},
{
"bits_per_byte": 1.6794896976947233,
"learning_rate": 0.0001750355632200748,
"loss_nats": 1.1641335487365723,
"step": 4020,
"unclipped_grad_norm": 0.6873654127120972
},
{
"bits_per_byte": 1.6416081110850775,
"learning_rate": 0.00017447897703680418,
"loss_nats": 1.137876033782959,
"step": 4030,
"unclipped_grad_norm": 0.7148170471191406
},
{
"bits_per_byte": 1.62684030482079,
"learning_rate": 0.0001739222288839694,
"loss_nats": 1.1276397705078125,
"step": 4040,
"unclipped_grad_norm": 0.6498348712921143
},
{
"bits_per_byte": 1.644293964143602,
"learning_rate": 0.00017336532827486026,
"loss_nats": 1.1397377252578735,
"step": 4050,
"unclipped_grad_norm": 0.6837151646614075
},
{
"bits_per_byte": 1.6349320885438852,
"learning_rate": 0.00017280828472537163,
"loss_nats": 1.1332485675811768,
"step": 4060,
"unclipped_grad_norm": 0.6708939671516418
},
{
"bits_per_byte": 1.769896849669681,
"learning_rate": 0.0001722511077538408,
"loss_nats": 1.2267990112304688,
"step": 4070,
"unclipped_grad_norm": 0.7203540802001953
},
{
"bits_per_byte": 1.5942710903136454,
"learning_rate": 0.00017169380688088497,
"loss_nats": 1.1050645112991333,
"step": 4080,
"unclipped_grad_norm": 0.6567085385322571
},
{
"bits_per_byte": 1.6276448396615262,
"learning_rate": 0.0001711363916292384,
"loss_nats": 1.128197431564331,
"step": 4090,
"unclipped_grad_norm": 0.6684355139732361
},
{
"bits_per_byte": 1.7285615074654084,
"learning_rate": 0.00017057887152358974,
"loss_nats": 1.1981475353240967,
"step": 4100,
"unclipped_grad_norm": 0.7356466054916382
},
{
"bits_per_byte": 1.6342331510507786,
"learning_rate": 0.0001700212560904194,
"loss_nats": 1.1327641010284424,
"step": 4110,
"unclipped_grad_norm": 0.655012309551239
},
{
"bits_per_byte": 1.6424126459258137,
"learning_rate": 0.00016946355485783656,
"loss_nats": 1.1384336948394775,
"step": 4120,
"unclipped_grad_norm": 0.7107924222946167
},
{
"bits_per_byte": 1.6944599275390764,
"learning_rate": 0.0001689057773554166,
"loss_nats": 1.17451012134552,
"step": 4130,
"unclipped_grad_norm": 0.6998885869979858
},
{
"bits_per_byte": 1.7017107160994525,
"learning_rate": 0.00016834793311403806,
"loss_nats": 1.179535984992981,
"step": 4140,
"unclipped_grad_norm": 0.6857012510299683
},
{
"bits_per_byte": 1.6170882005862652,
"learning_rate": 0.00016779003166571981,
"loss_nats": 1.120880126953125,
"step": 4150,
"unclipped_grad_norm": 0.6785045266151428
},
{
"bits_per_byte": 1.6430269679546958,
"learning_rate": 0.00016723208254345838,
"loss_nats": 1.138859510421753,
"step": 4160,
"unclipped_grad_norm": 0.6962723135948181
},
{
"bits_per_byte": 1.6682418603456448,
"learning_rate": 0.00016667409528106481,
"loss_nats": 1.1563371419906616,
"step": 4170,
"unclipped_grad_norm": 0.7071301937103271
},
{
"bits_per_byte": 1.6402630347726797,
"learning_rate": 0.00016611607941300184,
"loss_nats": 1.1369436979293823,
"step": 4180,
"unclipped_grad_norm": 0.6712477207183838
},
{
"bits_per_byte": 1.67100166594404,
"learning_rate": 0.0001655580444742211,
"loss_nats": 1.158250093460083,
"step": 4190,
"unclipped_grad_norm": 0.6872346997261047
},
{
"bits_per_byte": 1.6579426792987255,
"learning_rate": 0.00016499999999999997,
"loss_nats": 1.149198293685913,
"step": 4200,
"unclipped_grad_norm": 0.7030718326568604
},
{
"bits_per_byte": 1.6808303024581988,
"learning_rate": 0.0001644419555257789,
"loss_nats": 1.1650627851486206,
"step": 4210,
"unclipped_grad_norm": 0.658928394317627
},
{
"bits_per_byte": 1.659455094730413,
"learning_rate": 0.0001638839205869981,
"loss_nats": 1.1502466201782227,
"step": 4220,
"unclipped_grad_norm": 0.6767720580101013
},
{
"bits_per_byte": 1.6554740403282406,
"learning_rate": 0.0001633259047189352,
"loss_nats": 1.1474871635437012,
"step": 4230,
"unclipped_grad_norm": 0.7309125065803528
},
{
"bits_per_byte": 1.6771517655388943,
"learning_rate": 0.0001627679174565416,
"loss_nats": 1.162513017654419,
"step": 4240,
"unclipped_grad_norm": 0.7190505862236023
},
{
"bits_per_byte": 1.6138194983239667,
"learning_rate": 0.00016220996833428016,
"loss_nats": 1.1186144351959229,
"step": 4250,
"unclipped_grad_norm": 0.6701101660728455
},
{
"bits_per_byte": 1.6526860295751036,
"learning_rate": 0.00016165206688596191,
"loss_nats": 1.1455546617507935,
"step": 4260,
"unclipped_grad_norm": 0.6757012009620667
},
{
"bits_per_byte": 1.5671267245626668,
"learning_rate": 0.00016109422264458335,
"loss_nats": 1.0862494707107544,
"step": 4270,
"unclipped_grad_norm": 0.647323727607727
},
{
"bits_per_byte": 1.640895586962553,
"learning_rate": 0.00016053644514216336,
"loss_nats": 1.13738214969635,
"step": 4280,
"unclipped_grad_norm": 0.666140079498291
},
{
"bits_per_byte": 1.6259669769197085,
"learning_rate": 0.00015997874390958058,
"loss_nats": 1.1270344257354736,
"step": 4290,
"unclipped_grad_norm": 0.6681356430053711
},
{
"bits_per_byte": 1.5856241465935652,
"learning_rate": 0.00015942112847641018,
"loss_nats": 1.0990709066390991,
"step": 4300,
"unclipped_grad_norm": 0.6779436469078064
},
{
"bits_per_byte": 1.5644216094472456,
"learning_rate": 0.00015886360837076156,
"loss_nats": 1.0843744277954102,
"step": 4310,
"unclipped_grad_norm": 0.6528535485267639
},
{
"bits_per_byte": 1.6990217673532124,
"learning_rate": 0.00015830619311911495,
"loss_nats": 1.1776721477508545,
"step": 4320,
"unclipped_grad_norm": 0.6877329349517822
},
{
"bits_per_byte": 1.6415279671697756,
"learning_rate": 0.00015774889224615915,
"loss_nats": 1.1378204822540283,
"step": 4330,
"unclipped_grad_norm": 0.7055521011352539
},
{
"bits_per_byte": 1.604049851859054,
"learning_rate": 0.00015719171527462834,
"loss_nats": 1.1118426322937012,
"step": 4340,
"unclipped_grad_norm": 0.6865817308425903
},
{
"bits_per_byte": 1.5633534252027368,
"learning_rate": 0.0001566346717251397,
"loss_nats": 1.0836340188980103,
"step": 4350,
"unclipped_grad_norm": 0.6733583211898804
},
{
"bits_per_byte": 1.6505902489916895,
"learning_rate": 0.00015607777111603056,
"loss_nats": 1.1441019773483276,
"step": 4360,
"unclipped_grad_norm": 0.6889402270317078
},
{
"bits_per_byte": 1.6184719729951074,
"learning_rate": 0.00015552102296319576,
"loss_nats": 1.1218392848968506,
"step": 4370,
"unclipped_grad_norm": 0.6762850284576416
},
{
"bits_per_byte": 1.556092145700658,
"learning_rate": 0.00015496443677992516,
"loss_nats": 1.0786008834838867,
"step": 4380,
"unclipped_grad_norm": 0.6198620200157166
},
{
"bits_per_byte": 1.7130244228038114,
"learning_rate": 0.0001544080220767409,
"loss_nats": 1.1873780488967896,
"step": 4390,
"unclipped_grad_norm": 0.6756254434585571
},
{
"bits_per_byte": 1.6522954569749941,
"learning_rate": 0.00015385178836123511,
"loss_nats": 1.1452839374542236,
"step": 4400,
"unclipped_grad_norm": 0.6876093745231628
},
{
"bits_per_byte": 1.661213789318136,
"learning_rate": 0.00015329574513790726,
"loss_nats": 1.151465654373169,
"step": 4410,
"unclipped_grad_norm": 0.6821660399436951
},
{
"bits_per_byte": 1.6753433679650718,
"learning_rate": 0.00015273990190800186,
"loss_nats": 1.1612595319747925,
"step": 4420,
"unclipped_grad_norm": 0.7177299857139587
},
{
"bits_per_byte": 1.6206171125993192,
"learning_rate": 0.0001521842681693461,
"loss_nats": 1.1233261823654175,
"step": 4430,
"unclipped_grad_norm": 0.6967712640762329
},
{
"bits_per_byte": 1.6227820302083802,
"learning_rate": 0.00015162885341618752,
"loss_nats": 1.1248267889022827,
"step": 4440,
"unclipped_grad_norm": 0.6610331535339355
},
{
"bits_per_byte": 1.6554300127696198,
"learning_rate": 0.00015107366713903178,
"loss_nats": 1.1474566459655762,
"step": 4450,
"unclipped_grad_norm": 0.6704992055892944
},
{
"bits_per_byte": 1.6458450756717338,
"learning_rate": 0.0001505187188244806,
"loss_nats": 1.140812873840332,
"step": 4460,
"unclipped_grad_norm": 0.7582810521125793
},
{
"bits_per_byte": 1.6144303806998315,
"learning_rate": 0.00014996401795506952,
"loss_nats": 1.1190378665924072,
"step": 4470,
"unclipped_grad_norm": 0.7301307916641235
},
{
"bits_per_byte": 1.6560006512051826,
"learning_rate": 0.00014940957400910585,
"loss_nats": 1.1478521823883057,
"step": 4480,
"unclipped_grad_norm": 0.7387185096740723
},
{
"bits_per_byte": 1.6943782357799166,
"learning_rate": 0.00014885539646050699,
"loss_nats": 1.1744534969329834,
"step": 4490,
"unclipped_grad_norm": 0.7154380083084106
},
{
"bits_per_byte": 1.6097710267226562,
"learning_rate": 0.00014830149477863805,
"loss_nats": 1.1158082485198975,
"step": 4500,
"unclipped_grad_norm": 0.6710496544837952
},
{
"bits_per_byte": 1.6061318738303991,
"learning_rate": 0.00014774787842815066,
"loss_nats": 1.113285779953003,
"step": 4510,
"unclipped_grad_norm": 0.7097434401512146
},
{
"bits_per_byte": 1.676021495557424,
"learning_rate": 0.00014719455686882066,
"loss_nats": 1.1617295742034912,
"step": 4520,
"unclipped_grad_norm": 0.7576807141304016
},
{
"bits_per_byte": 1.635137779794317,
"learning_rate": 0.00014664153955538695,
"loss_nats": 1.1333911418914795,
"step": 4530,
"unclipped_grad_norm": 0.6911670565605164
},
{
"bits_per_byte": 1.64362185594403,
"learning_rate": 0.00014608883593738946,
"loss_nats": 1.139271855354309,
"step": 4540,
"unclipped_grad_norm": 0.6786331534385681
},
{
"bits_per_byte": 1.650524379636409,
"learning_rate": 0.00014553645545900817,
"loss_nats": 1.1440563201904297,
"step": 4550,
"unclipped_grad_norm": 0.6853471994400024
},
{
"bits_per_byte": 1.6363368428361325,
"learning_rate": 0.0001449844075589013,
"loss_nats": 1.1342222690582275,
"step": 4560,
"unclipped_grad_norm": 0.7256327867507935
},
{
"bits_per_byte": 1.6365012582503573,
"learning_rate": 0.0001444327016700444,
"loss_nats": 1.134336233139038,
"step": 4570,
"unclipped_grad_norm": 0.697358250617981
},
{
"bits_per_byte": 1.6112284077060677,
"learning_rate": 0.0001438813472195688,
"loss_nats": 1.1168184280395508,
"step": 4580,
"unclipped_grad_norm": 0.6777223348617554
},
{
"bits_per_byte": 1.5962301446896239,
"learning_rate": 0.00014333035362860093,
"loss_nats": 1.1064224243164062,
"step": 4590,
"unclipped_grad_norm": 0.7330402731895447
},
{
"bits_per_byte": 1.5998985346325274,
"learning_rate": 0.00014277973031210088,
"loss_nats": 1.1089651584625244,
"step": 4600,
"unclipped_grad_norm": 0.6789212822914124
},
{
"bits_per_byte": 1.596745060746307,
"learning_rate": 0.00014222948667870207,
"loss_nats": 1.1067793369293213,
"step": 4610,
"unclipped_grad_norm": 0.6819480061531067
},
{
"bits_per_byte": 1.6577524664868712,
"learning_rate": 0.00014167963213054991,
"loss_nats": 1.14906644821167,
"step": 4620,
"unclipped_grad_norm": 0.7283186912536621
},
{
"bits_per_byte": 1.646706020821953,
"learning_rate": 0.00014113017606314152,
"loss_nats": 1.1414096355438232,
"step": 4630,
"unclipped_grad_norm": 0.7197591662406921
},
{
"bits_per_byte": 1.6542598428131492,
"learning_rate": 0.00014058112786516518,
"loss_nats": 1.1466455459594727,
"step": 4640,
"unclipped_grad_norm": 0.7083833813667297
},
{
"bits_per_byte": 1.6790158854915962,
"learning_rate": 0.0001400324969183396,
"loss_nats": 1.1638051271438599,
"step": 4650,
"unclipped_grad_norm": 0.684671938419342
},
{
"bits_per_byte": 1.6168221434253804,
"learning_rate": 0.00013948429259725406,
"loss_nats": 1.12069571018219,
"step": 4660,
"unclipped_grad_norm": 0.7103760838508606
},
{
"bits_per_byte": 1.6034386255178876,
"learning_rate": 0.00013893652426920776,
"loss_nats": 1.1114189624786377,
"step": 4670,
"unclipped_grad_norm": 0.6704027652740479
},
{
"bits_per_byte": 1.591610346722147,
"learning_rate": 0.00013838920129405016,
"loss_nats": 1.1032202243804932,
"step": 4680,
"unclipped_grad_norm": 0.7572834491729736
},
{
"bits_per_byte": 1.6094411639983013,
"learning_rate": 0.0001378423330240207,
"loss_nats": 1.115579605102539,
"step": 4690,
"unclipped_grad_norm": 0.6773369312286377
},
{
"bits_per_byte": 1.6221410508686145,
"learning_rate": 0.00013729592880358936,
"loss_nats": 1.124382495880127,
"step": 4700,
"unclipped_grad_norm": 0.7124287486076355
},
{
"bits_per_byte": 1.6214163159778787,
"learning_rate": 0.00013674999796929659,
"loss_nats": 1.12388014793396,
"step": 4710,
"unclipped_grad_norm": 0.7075444459915161
},
{
"bits_per_byte": 1.6415025137374477,
"learning_rate": 0.00013620454984959417,
"loss_nats": 1.1378028392791748,
"step": 4720,
"unclipped_grad_norm": 0.7044992446899414
},
{
"bits_per_byte": 1.654741394235565,
"learning_rate": 0.00013565959376468542,
"loss_nats": 1.1469793319702148,
"step": 4730,
"unclipped_grad_norm": 0.7124809622764587
},
{
"bits_per_byte": 1.6107767812649019,
"learning_rate": 0.00013511513902636634,
"loss_nats": 1.1165053844451904,
"step": 4740,
"unclipped_grad_norm": 0.7306708693504333
},
{
"bits_per_byte": 1.5756537963750956,
"learning_rate": 0.0001345711949378662,
"loss_nats": 1.0921599864959717,
"step": 4750,
"unclipped_grad_norm": 0.6872034668922424
},
{
"bits_per_byte": 1.550611402582962,
"learning_rate": 0.00013402777079368853,
"loss_nats": 1.0748019218444824,
"step": 4760,
"unclipped_grad_norm": 0.6966266632080078
},
{
"bits_per_byte": 1.5558193812163896,
"learning_rate": 0.0001334848758794528,
"loss_nats": 1.0784118175506592,
"step": 4770,
"unclipped_grad_norm": 0.6661347150802612
},
{
"bits_per_byte": 1.5972586009417833,
"learning_rate": 0.00013294251947173504,
"loss_nats": 1.10713529586792,
"step": 4780,
"unclipped_grad_norm": 0.7209707498550415
},
{
"bits_per_byte": 1.6013201432245594,
"learning_rate": 0.00013240071083790996,
"loss_nats": 1.1099505424499512,
"step": 4790,
"unclipped_grad_norm": 0.7224242091178894
},
{
"bits_per_byte": 1.6214809814546032,
"learning_rate": 0.0001318594592359921,
"loss_nats": 1.123924970626831,
"step": 4800,
"unclipped_grad_norm": 0.7595614790916443
},
{
"bits_per_byte": 1.598943858937588,
"learning_rate": 0.0001313187739144781,
"loss_nats": 1.108303427696228,
"step": 4810,
"unclipped_grad_norm": 0.6932185292243958
},
{
"bits_per_byte": 1.5882840302718095,
"learning_rate": 0.0001307786641121882,
"loss_nats": 1.1009145975112915,
"step": 4820,
"unclipped_grad_norm": 0.7010871171951294
},
{
"bits_per_byte": 1.5957891811728115,
"learning_rate": 0.00013023913905810876,
"loss_nats": 1.106116771697998,
"step": 4830,
"unclipped_grad_norm": 0.6711499691009521
},
{
"bits_per_byte": 1.5337598545208206,
"learning_rate": 0.00012970020797123432,
"loss_nats": 1.0631213188171387,
"step": 4840,
"unclipped_grad_norm": 0.7005295157432556
},
{
"bits_per_byte": 1.6063714456630511,
"learning_rate": 0.00012916188006041027,
"loss_nats": 1.1134518384933472,
"step": 4850,
"unclipped_grad_norm": 0.6924180388450623
},
{
"bits_per_byte": 1.58919708616524,
"learning_rate": 0.00012862416452417516,
"loss_nats": 1.1015474796295166,
"step": 4860,
"unclipped_grad_norm": 0.6728304028511047
},
{
"bits_per_byte": 1.5949746713383253,
"learning_rate": 0.00012808707055060405,
"loss_nats": 1.1055521965026855,
"step": 4870,
"unclipped_grad_norm": 0.6908511519432068
},
{
"bits_per_byte": 1.562132004416309,
"learning_rate": 0.000127550607317151,
"loss_nats": 1.0827873945236206,
"step": 4880,
"unclipped_grad_norm": 0.6754211187362671
},
{
"bits_per_byte": 1.5204766024987808,
"learning_rate": 0.00012701478399049248,
"loss_nats": 1.0539140701293945,
"step": 4890,
"unclipped_grad_norm": 0.686815619468689
},
{
"bits_per_byte": 1.522231341485534,
"learning_rate": 0.0001264796097263708,
"loss_nats": 1.0551303625106812,
"step": 4900,
"unclipped_grad_norm": 0.6894630789756775
},
{
"bits_per_byte": 1.5875051208460518,
"learning_rate": 0.0001259450936694375,
"loss_nats": 1.100374698638916,
"step": 4910,
"unclipped_grad_norm": 0.7035151124000549
},
{
"bits_per_byte": 1.5873929881576891,
"learning_rate": 0.00012541124495309727,
"loss_nats": 1.100296974182129,
"step": 4920,
"unclipped_grad_norm": 0.707631528377533
},
{
"bits_per_byte": 1.5818637459324498,
"learning_rate": 0.00012487807269935167,
"loss_nats": 1.0964643955230713,
"step": 4930,
"unclipped_grad_norm": 0.7146247029304504
},
{
"bits_per_byte": 1.6602286726939939,
"learning_rate": 0.00012434558601864349,
"loss_nats": 1.150782823562622,
"step": 4940,
"unclipped_grad_norm": 0.7488285899162292
},
{
"bits_per_byte": 1.5109480757103777,
"learning_rate": 0.0001238137940097009,
"loss_nats": 1.047309398651123,
"step": 4950,
"unclipped_grad_norm": 0.6982936263084412
},
{
"bits_per_byte": 1.6013034609074257,
"learning_rate": 0.0001232827057593821,
"loss_nats": 1.1099389791488647,
"step": 4960,
"unclipped_grad_norm": 0.7697318196296692
},
{
"bits_per_byte": 1.673659485830474,
"learning_rate": 0.00012275233034251992,
"loss_nats": 1.1600923538208008,
"step": 4970,
"unclipped_grad_norm": 0.7759159207344055
},
{
"bits_per_byte": 1.5500858236019253,
"learning_rate": 0.0001222226768217669,
"loss_nats": 1.0744376182556152,
"step": 4980,
"unclipped_grad_norm": 0.7765741944313049
},
{
"bits_per_byte": 1.6026017579387892,
"learning_rate": 0.00012169375424744034,
"loss_nats": 1.1108388900756836,
"step": 4990,
"unclipped_grad_norm": 0.7224560976028442
},
{
"bits_per_byte": 1.6136506113608193,
"learning_rate": 0.00012116557165736772,
"loss_nats": 1.118497371673584,
"step": 5000,
"unclipped_grad_norm": 0.7415750622749329
},
{
"bits_per_byte": 1.5910434919049032,
"learning_rate": 0.00012063813807673214,
"loss_nats": 1.1028273105621338,
"step": 5010,
"unclipped_grad_norm": 0.6908667087554932
},
{
"bits_per_byte": 1.598975331762696,
"learning_rate": 0.00012011146251791832,
"loss_nats": 1.1083252429962158,
"step": 5020,
"unclipped_grad_norm": 0.730786919593811
},
{
"bits_per_byte": 1.5824023955949522,
"learning_rate": 0.0001195855539803584,
"loss_nats": 1.0968377590179443,
"step": 5030,
"unclipped_grad_norm": 0.7392095923423767
},
{
"bits_per_byte": 1.5712477728426413,
"learning_rate": 0.00011906042145037817,
"loss_nats": 1.0891059637069702,
"step": 5040,
"unclipped_grad_norm": 0.7138375639915466
},
{
"bits_per_byte": 1.600523519585763,
"learning_rate": 0.0001185360739010438,
"loss_nats": 1.109398365020752,
"step": 5050,
"unclipped_grad_norm": 0.7417458295822144
},
{
"bits_per_byte": 1.556087158203783,
"learning_rate": 0.00011801252029200808,
"loss_nats": 1.0785974264144897,
"step": 5060,
"unclipped_grad_norm": 0.766182541847229
},
{
"bits_per_byte": 1.6239489324944845,
"learning_rate": 0.00011748976956935777,
"loss_nats": 1.1256356239318848,
"step": 5070,
"unclipped_grad_norm": 0.7354997992515564
},
{
"bits_per_byte": 1.6499730032577429,
"learning_rate": 0.00011696783066546024,
"loss_nats": 1.1436741352081299,
"step": 5080,
"unclipped_grad_norm": 0.8061900734901428
},
{
"bits_per_byte": 1.621231606610852,
"learning_rate": 0.00011644671249881147,
"loss_nats": 1.1237521171569824,
"step": 5090,
"unclipped_grad_norm": 0.789067804813385
},
{
"bits_per_byte": 1.5443199332490989,
"learning_rate": 0.00011592642397388293,
"loss_nats": 1.0704410076141357,
"step": 5100,
"unclipped_grad_norm": 0.7130411267280579
},
{
"bits_per_byte": 1.5718306220464153,
"learning_rate": 0.00011540697398097015,
"loss_nats": 1.0895099639892578,
"step": 5110,
"unclipped_grad_norm": 0.6817747950553894
},
{
"bits_per_byte": 1.5553593276253317,
"learning_rate": 0.00011488837139604016,
"loss_nats": 1.0780929327011108,
"step": 5120,
"unclipped_grad_norm": 0.7432519793510437
},
{
"bits_per_byte": 1.611734380664906,
"learning_rate": 0.00011437062508058039,
"loss_nats": 1.1171691417694092,
"step": 5130,
"unclipped_grad_norm": 0.7314459681510925
},
{
"bits_per_byte": 1.5509013653323167,
"learning_rate": 0.00011385374388144673,
"loss_nats": 1.075002908706665,
"step": 5140,
"unclipped_grad_norm": 0.7262561321258545
},
{
"bits_per_byte": 1.5519597465657264,
"learning_rate": 0.00011333773663071286,
"loss_nats": 1.0757365226745605,
"step": 5150,
"unclipped_grad_norm": 0.7442956566810608
},
{
"bits_per_byte": 1.582139434121783,
"learning_rate": 0.00011282261214551897,
"loss_nats": 1.0966554880142212,
"step": 5160,
"unclipped_grad_norm": 0.7529576420783997
},
{
"bits_per_byte": 1.5941560339202183,
"learning_rate": 0.00011230837922792112,
"loss_nats": 1.1049847602844238,
"step": 5170,
"unclipped_grad_norm": 0.7270960211753845
},
{
"bits_per_byte": 1.6180967068509247,
"learning_rate": 0.0001117950466647412,
"loss_nats": 1.1215791702270508,
"step": 5180,
"unclipped_grad_norm": 0.759490966796875
},
{
"bits_per_byte": 1.5759898504748815,
"learning_rate": 0.00011128262322741627,
"loss_nats": 1.092392921447754,
"step": 5190,
"unclipped_grad_norm": 0.7248204946517944
},
{
"bits_per_byte": 1.5225916451390915,
"learning_rate": 0.00011077111767184913,
"loss_nats": 1.05538010597229,
"step": 5200,
"unclipped_grad_norm": 0.7128092646598816
},
{
"bits_per_byte": 1.601221769148266,
"learning_rate": 0.00011026053873825837,
"loss_nats": 1.1098823547363281,
"step": 5210,
"unclipped_grad_norm": 0.7404454946517944
},
{
"bits_per_byte": 1.5843179383602621,
"learning_rate": 0.0001097508951510293,
"loss_nats": 1.098165512084961,
"step": 5220,
"unclipped_grad_norm": 0.7370045781135559
},
{
"bits_per_byte": 1.614152456736037,
"learning_rate": 0.0001092421956185646,
"loss_nats": 1.1188452243804932,
"step": 5230,
"unclipped_grad_norm": 0.8054904937744141
},
{
"bits_per_byte": 1.5701584347320763,
"learning_rate": 0.00010873444883313581,
"loss_nats": 1.0883508920669556,
"step": 5240,
"unclipped_grad_norm": 0.7422505617141724
},
{
"bits_per_byte": 1.5364362485335477,
"learning_rate": 0.00010822766347073447,
"loss_nats": 1.064976453781128,
"step": 5250,
"unclipped_grad_norm": 0.7392460107803345
},
{
"bits_per_byte": 1.510300904995181,
"learning_rate": 0.00010772184819092424,
"loss_nats": 1.0468608140945435,
"step": 5260,
"unclipped_grad_norm": 0.7152841091156006
},
{
"bits_per_byte": 1.6197508359869233,
"learning_rate": 0.0001072170116366926,
"loss_nats": 1.1227257251739502,
"step": 5270,
"unclipped_grad_norm": 0.762697160243988
},
{
"bits_per_byte": 1.6105141637570344,
"learning_rate": 0.00010671316243430345,
"loss_nats": 1.1163233518600464,
"step": 5280,
"unclipped_grad_norm": 0.7536966800689697
},
{
"bits_per_byte": 1.5885342650288148,
"learning_rate": 0.00010621030919314944,
"loss_nats": 1.101088047027588,
"step": 5290,
"unclipped_grad_norm": 0.7395375967025757
},
{
"bits_per_byte": 1.5242677880544004,
"learning_rate": 0.000105708460505605,
"loss_nats": 1.056541919708252,
"step": 5300,
"unclipped_grad_norm": 0.6971617937088013
},
{
"bits_per_byte": 1.589467786857698,
"learning_rate": 0.00010520762494687964,
"loss_nats": 1.1017351150512695,
"step": 5310,
"unclipped_grad_norm": 0.7348577380180359
},
{
"bits_per_byte": 1.5472704676073012,
"learning_rate": 0.00010470781107487111,
"loss_nats": 1.072486162185669,
"step": 5320,
"unclipped_grad_norm": 0.7314221858978271
},
{
"bits_per_byte": 1.6553672391020549,
"learning_rate": 0.00010420902743001955,
"loss_nats": 1.1474131345748901,
"step": 5330,
"unclipped_grad_norm": 0.7674097418785095
},
{
"bits_per_byte": 1.5389176142201964,
"learning_rate": 0.00010371128253516117,
"loss_nats": 1.0666964054107666,
"step": 5340,
"unclipped_grad_norm": 0.7431036829948425
},
{
"bits_per_byte": 1.593487021408362,
"learning_rate": 0.000103214584895383,
"loss_nats": 1.1045210361480713,
"step": 5350,
"unclipped_grad_norm": 0.7701666951179504
},
{
"bits_per_byte": 1.5047200679746828,
"learning_rate": 0.00010271894299787715,
"loss_nats": 1.0429924726486206,
"step": 5360,
"unclipped_grad_norm": 0.7093538045883179
},
{
"bits_per_byte": 1.5323434054083145,
"learning_rate": 0.00010222436531179622,
"loss_nats": 1.0621395111083984,
"step": 5370,
"unclipped_grad_norm": 0.8088909983634949
},
{
"bits_per_byte": 1.5949007187984543,
"learning_rate": 0.0001017308602881082,
"loss_nats": 1.1055009365081787,
"step": 5380,
"unclipped_grad_norm": 0.7591733336448669
},
{
"bits_per_byte": 1.5063042001717801,
"learning_rate": 0.00010123843635945242,
"loss_nats": 1.0440905094146729,
"step": 5390,
"unclipped_grad_norm": 0.7258779406547546
},
{
"bits_per_byte": 1.5763400071520381,
"learning_rate": 0.00010074710193999505,
"loss_nats": 1.0926356315612793,
"step": 5400,
"unclipped_grad_norm": 0.7020314335823059
},
{
"bits_per_byte": 1.540318240928823,
"learning_rate": 0.0001002568654252858,
"loss_nats": 1.0676672458648682,
"step": 5410,
"unclipped_grad_norm": 0.73995441198349
},
{
"bits_per_byte": 1.5024278832039832,
"learning_rate": 9.976773519211404e-05,
"loss_nats": 1.0414036512374878,
"step": 5420,
"unclipped_grad_norm": 0.7208534479141235
},
{
"bits_per_byte": 1.556975964543442,
"learning_rate": 9.92797195983659e-05,
"loss_nats": 1.0792135000228882,
"step": 5430,
"unclipped_grad_norm": 0.7300911545753479
},
{
"bits_per_byte": 1.5377587951186826,
"learning_rate": 9.879282698288142e-05,
"loss_nats": 1.0658931732177734,
"step": 5440,
"unclipped_grad_norm": 0.7053164839744568
},
{
"bits_per_byte": 1.5519281017579678,
"learning_rate": 9.830706566531197e-05,
"loss_nats": 1.0757145881652832,
"step": 5450,
"unclipped_grad_norm": 0.7570145726203918
},
{
"bits_per_byte": 1.5295854916190785,
"learning_rate": 9.782244394597826e-05,
"loss_nats": 1.060227870941162,
"step": 5460,
"unclipped_grad_norm": 0.7293761968612671
},
{
"bits_per_byte": 1.5453655877663446,
"learning_rate": 9.733897010572829e-05,
"loss_nats": 1.0711658000946045,
"step": 5470,
"unclipped_grad_norm": 0.7067918181419373
},
{
"bits_per_byte": 1.5185650153344405,
"learning_rate": 9.685665240579612e-05,
"loss_nats": 1.0525890588760376,
"step": 5480,
"unclipped_grad_norm": 0.7616989612579346
},
{
"bits_per_byte": 1.5811864782533522,
"learning_rate": 9.637549908766041e-05,
"loss_nats": 1.0959949493408203,
"step": 5490,
"unclipped_grad_norm": 0.8041682243347168
},
{
"bits_per_byte": 1.5613092394145813,
"learning_rate": 9.589551837290389e-05,
"loss_nats": 1.0822170972824097,
"step": 5500,
"unclipped_grad_norm": 0.7645291686058044
},
{
"bits_per_byte": 1.5218208188979245,
"learning_rate": 9.541671846307257e-05,
"loss_nats": 1.0548458099365234,
"step": 5510,
"unclipped_grad_norm": 0.7549375891685486
},
{
"bits_per_byte": 1.4947696677437132,
"learning_rate": 9.493910753953597e-05,
"loss_nats": 1.036095380783081,
"step": 5520,
"unclipped_grad_norm": 0.7179425358772278
},
{
"bits_per_byte": 1.5059726176209167,
"learning_rate": 9.446269376334689e-05,
"loss_nats": 1.043860673904419,
"step": 5530,
"unclipped_grad_norm": 0.7216454744338989
},
{
"bits_per_byte": 1.625827155024557,
"learning_rate": 9.39874852751024e-05,
"loss_nats": 1.1269375085830688,
"step": 5540,
"unclipped_grad_norm": 0.7688449621200562
},
{
"bits_per_byte": 1.5739950237075238,
"learning_rate": 9.351349019480433e-05,
"loss_nats": 1.0910102128982544,
"step": 5550,
"unclipped_grad_norm": 0.7435398697853088
},
{
"bits_per_byte": 1.5977501273579493,
"learning_rate": 9.304071662172086e-05,
"loss_nats": 1.107475996017456,
"step": 5560,
"unclipped_grad_norm": 0.8213235139846802
},
{
"bits_per_byte": 1.537488782356828,
"learning_rate": 9.256917263424787e-05,
"loss_nats": 1.0657060146331787,
"step": 5570,
"unclipped_grad_norm": 0.7458736896514893
},
{
"bits_per_byte": 1.569628556184768,
"learning_rate": 9.209886628977114e-05,
"loss_nats": 1.0879836082458496,
"step": 5580,
"unclipped_grad_norm": 0.745820939540863
},
{
"bits_per_byte": 1.5662062734152489,
"learning_rate": 9.162980562452843e-05,
"loss_nats": 1.0856114625930786,
"step": 5590,
"unclipped_grad_norm": 0.8023808002471924
},
{
"bits_per_byte": 1.5614218880508965,
"learning_rate": 9.116199865347238e-05,
"loss_nats": 1.0822951793670654,
"step": 5600,
"unclipped_grad_norm": 0.7724120020866394
},
{
"bits_per_byte": 1.5000096351502012,
"learning_rate": 9.069545337013333e-05,
"loss_nats": 1.0397274494171143,
"step": 5610,
"unclipped_grad_norm": 0.7330023050308228
},
{
"bits_per_byte": 1.564252034553495,
"learning_rate": 9.0230177746483e-05,
"loss_nats": 1.084256887435913,
"step": 5620,
"unclipped_grad_norm": 0.7773064374923706
},
{
"bits_per_byte": 1.4778873347870674,
"learning_rate": 8.976617973279797e-05,
"loss_nats": 1.0243934392929077,
"step": 5630,
"unclipped_grad_norm": 0.7191233038902283
},
{
"bits_per_byte": 1.4838417181252395,
"learning_rate": 8.930346725752419e-05,
"loss_nats": 1.0285207033157349,
"step": 5640,
"unclipped_grad_norm": 0.7019065022468567
},
{
"bits_per_byte": 1.5419510442161144,
"learning_rate": 8.88420482271411e-05,
"loss_nats": 1.0687990188598633,
"step": 5650,
"unclipped_grad_norm": 0.7909054160118103
},
{
"bits_per_byte": 1.4908660055344292,
"learning_rate": 8.838193052602691e-05,
"loss_nats": 1.0333895683288574,
"step": 5660,
"unclipped_grad_norm": 0.7980607151985168
},
{
"bits_per_byte": 1.5729701790910324,
"learning_rate": 8.792312201632352e-05,
"loss_nats": 1.0902998447418213,
"step": 5670,
"unclipped_grad_norm": 0.796278178691864
},
{
"bits_per_byte": 1.5668854329035062,
"learning_rate": 8.746563053780261e-05,
"loss_nats": 1.0860822200775146,
"step": 5680,
"unclipped_grad_norm": 0.8280957341194153
},
{
"bits_per_byte": 1.592162067066115,
"learning_rate": 8.700946390773114e-05,
"loss_nats": 1.103602647781372,
"step": 5690,
"unclipped_grad_norm": 0.7912419438362122
},
{
"bits_per_byte": 1.5918954939572776,
"learning_rate": 8.655462992073833e-05,
"loss_nats": 1.1034178733825684,
"step": 5700,
"unclipped_grad_norm": 0.7910536527633667
},
{
"bits_per_byte": 1.5647875885282816,
"learning_rate": 8.610113634868199e-05,
"loss_nats": 1.0846281051635742,
"step": 5710,
"unclipped_grad_norm": 0.8025376200675964
},
{
"bits_per_byte": 1.576262442976499,
"learning_rate": 8.564899094051613e-05,
"loss_nats": 1.092581868171692,
"step": 5720,
"unclipped_grad_norm": 0.7657404541969299
},
{
"bits_per_byte": 1.5180726290050204,
"learning_rate": 8.519820142215816e-05,
"loss_nats": 1.0522477626800537,
"step": 5730,
"unclipped_grad_norm": 0.7721706032752991
},
{
"bits_per_byte": 1.6017487239905095,
"learning_rate": 8.474877549635728e-05,
"loss_nats": 1.1102476119995117,
"step": 5740,
"unclipped_grad_norm": 0.7985641956329346
},
{
"bits_per_byte": 1.4880221004197618,
"learning_rate": 8.430072084256247e-05,
"loss_nats": 1.0314183235168457,
"step": 5750,
"unclipped_grad_norm": 0.7275866866111755
},
{
"bits_per_byte": 1.5012069783655082,
"learning_rate": 8.385404511679161e-05,
"loss_nats": 1.0405573844909668,
"step": 5760,
"unclipped_grad_norm": 0.7711101174354553
},
{
"bits_per_byte": 1.5255566260399664,
"learning_rate": 8.340875595150034e-05,
"loss_nats": 1.0574352741241455,
"step": 5770,
"unclipped_grad_norm": 0.8030062913894653
},
{
"bits_per_byte": 1.6161017081009161,
"learning_rate": 8.296486095545203e-05,
"loss_nats": 1.1201963424682617,
"step": 5780,
"unclipped_grad_norm": 0.7891528606414795
},
{
"bits_per_byte": 1.436315000489906,
"learning_rate": 8.252236771358729e-05,
"loss_nats": 0.9955776929855347,
"step": 5790,
"unclipped_grad_norm": 0.7522260546684265
},
{
"bits_per_byte": 1.5639025658069416,
"learning_rate": 8.208128378689484e-05,
"loss_nats": 1.084014654159546,
"step": 5800,
"unclipped_grad_norm": 0.77913498878479
},
{
"bits_per_byte": 1.4787818165542048,
"learning_rate": 8.16416167122819e-05,
"loss_nats": 1.0250134468078613,
"step": 5810,
"unclipped_grad_norm": 0.744920551776886
},
{
"bits_per_byte": 1.485352929678371,
"learning_rate": 8.120337400244578e-05,
"loss_nats": 1.0295681953430176,
"step": 5820,
"unclipped_grad_norm": 0.8005070090293884
},
{
"bits_per_byte": 1.551830071646976,
"learning_rate": 8.076656314574521e-05,
"loss_nats": 1.0756466388702393,
"step": 5830,
"unclipped_grad_norm": 0.7964350581169128
},
{
"bits_per_byte": 1.5252578921754176,
"learning_rate": 8.03311916060725e-05,
"loss_nats": 1.0572282075881958,
"step": 5840,
"unclipped_grad_norm": 0.8099321722984314
},
{
"bits_per_byte": 1.4836750669365535,
"learning_rate": 7.989726682272606e-05,
"loss_nats": 1.0284051895141602,
"step": 5850,
"unclipped_grad_norm": 0.7372169494628906
},
{
"bits_per_byte": 1.511822779472666,
"learning_rate": 7.946479621028318e-05,
"loss_nats": 1.0479156970977783,
"step": 5860,
"unclipped_grad_norm": 0.7115837931632996
},
{
"bits_per_byte": 1.5625698722454058,
"learning_rate": 7.903378715847343e-05,
"loss_nats": 1.083090901374817,
"step": 5870,
"unclipped_grad_norm": 0.7878124713897705
},
{
"bits_per_byte": 1.503004884997628,
"learning_rate": 7.860424703205224e-05,
"loss_nats": 1.0418035984039307,
"step": 5880,
"unclipped_grad_norm": 0.7451943159103394
},
{
"bits_per_byte": 1.5713764158654868,
"learning_rate": 7.817618317067534e-05,
"loss_nats": 1.0891951322555542,
"step": 5890,
"unclipped_grad_norm": 0.8092991709709167
},
{
"bits_per_byte": 1.5881084359852786,
"learning_rate": 7.774960288877295e-05,
"loss_nats": 1.1007928848266602,
"step": 5900,
"unclipped_grad_norm": 0.7848427891731262
},
{
"bits_per_byte": 1.5133455138634053,
"learning_rate": 7.732451347542522e-05,
"loss_nats": 1.048971176147461,
"step": 5910,
"unclipped_grad_norm": 0.7443398237228394
},
{
"bits_per_byte": 1.4825475486774968,
"learning_rate": 7.690092219423722e-05,
"loss_nats": 1.0276236534118652,
"step": 5920,
"unclipped_grad_norm": 0.7709684371948242
},
{
"bits_per_byte": 1.5970936695796059,
"learning_rate": 7.647883628321536e-05,
"loss_nats": 1.1070209741592407,
"step": 5930,
"unclipped_grad_norm": 0.8097158074378967
},
{
"bits_per_byte": 1.5296348506398763,
"learning_rate": 7.605826295464316e-05,
"loss_nats": 1.0602620840072632,
"step": 5940,
"unclipped_grad_norm": 0.8011659383773804
},
{
"bits_per_byte": 1.5315722352018457,
"learning_rate": 7.563920939495851e-05,
"loss_nats": 1.0616049766540527,
"step": 5950,
"unclipped_grad_norm": 0.7703563570976257
},
{
"bits_per_byte": 1.4753576419755263,
"learning_rate": 7.522168276463047e-05,
"loss_nats": 1.0226399898529053,
"step": 5960,
"unclipped_grad_norm": 0.7793571949005127
},
{
"bits_per_byte": 1.514912963743369,
"learning_rate": 7.480569019803719e-05,
"loss_nats": 1.0500576496124268,
"step": 5970,
"unclipped_grad_norm": 0.7870919108390808
},
{
"bits_per_byte": 1.4934732625215095,
"learning_rate": 7.439123880334388e-05,
"loss_nats": 1.0351967811584473,
"step": 5980,
"unclipped_grad_norm": 0.8144509196281433
},
{
"bits_per_byte": 1.5080035607449556,
"learning_rate": 7.39783356623813e-05,
"loss_nats": 1.0452684164047241,
"step": 5990,
"unclipped_grad_norm": 0.7810203433036804
},
{
"bits_per_byte": 1.5782032671914858,
"learning_rate": 7.356698783052497e-05,
"loss_nats": 1.0939271450042725,
"step": 6000,
"unclipped_grad_norm": 0.790114164352417
},
{
"bits_per_byte": 1.5490276143511665,
"learning_rate": 7.315720233657431e-05,
"loss_nats": 1.0737041234970093,
"step": 6010,
"unclipped_grad_norm": 0.8144342303276062
},
{
"bits_per_byte": 1.515940732064925,
"learning_rate": 7.274898618263286e-05,
"loss_nats": 1.0507700443267822,
"step": 6020,
"unclipped_grad_norm": 0.7434252500534058
},
{
"bits_per_byte": 1.493858503659442,
"learning_rate": 7.234234634398831e-05,
"loss_nats": 1.035463809967041,
"step": 6030,
"unclipped_grad_norm": 0.8062746524810791
},
{
"bits_per_byte": 1.5237423810560147,
"learning_rate": 7.193728976899362e-05,
"loss_nats": 1.0561777353286743,
"step": 6040,
"unclipped_grad_norm": 0.7702685594558716
},
{
"bits_per_byte": 1.5275683071071087,
"learning_rate": 7.153382337894797e-05,
"loss_nats": 1.058829665184021,
"step": 6050,
"unclipped_grad_norm": 0.7716134786605835
},
{
"bits_per_byte": 1.5215145177967377,
"learning_rate": 7.113195406797887e-05,
"loss_nats": 1.0546334981918335,
"step": 6060,
"unclipped_grad_norm": 0.7933171987533569
},
{
"bits_per_byte": 1.415498478403448,
"learning_rate": 7.073168870292397e-05,
"loss_nats": 0.9811487793922424,
"step": 6070,
"unclipped_grad_norm": 0.7639470100402832
},
{
"bits_per_byte": 1.4966260484771265,
"learning_rate": 7.033303412321404e-05,
"loss_nats": 1.0373821258544922,
"step": 6080,
"unclipped_grad_norm": 0.7900221943855286
},
{
"bits_per_byte": 1.4925770609278632,
"learning_rate": 6.993599714075586e-05,
"loss_nats": 1.0345755815505981,
"step": 6090,
"unclipped_grad_norm": 0.809319019317627
},
{
"bits_per_byte": 1.5936792980120267,
"learning_rate": 6.954058453981609e-05,
"loss_nats": 1.104654312133789,
"step": 6100,
"unclipped_grad_norm": 0.824783205986023
},
{
"bits_per_byte": 1.5281914002511847,
"learning_rate": 6.914680307690504e-05,
"loss_nats": 1.0592615604400635,
"step": 6110,
"unclipped_grad_norm": 0.8245336413383484
},
{
"bits_per_byte": 1.48945093228313,
"learning_rate": 6.875465948066139e-05,
"loss_nats": 1.0324087142944336,
"step": 6120,
"unclipped_grad_norm": 0.8355178833007812
},
{
"bits_per_byte": 1.5475705773330568,
"learning_rate": 6.836416045173733e-05,
"loss_nats": 1.072694182395935,
"step": 6130,
"unclipped_grad_norm": 0.7615814208984375
},
{
"bits_per_byte": 1.4980500648262705,
"learning_rate": 6.797531266268377e-05,
"loss_nats": 1.0383691787719727,
"step": 6140,
"unclipped_grad_norm": 0.7743073105812073
},
{
"bits_per_byte": 1.483834838819205,
"learning_rate": 6.758812275783663e-05,
"loss_nats": 1.0285159349441528,
"step": 6150,
"unclipped_grad_norm": 0.7909993529319763
},
{
"bits_per_byte": 1.5470265962083778,
"learning_rate": 6.720259735320298e-05,
"loss_nats": 1.072317123413086,
"step": 6160,
"unclipped_grad_norm": 0.7707679867744446
},
{
"bits_per_byte": 1.478142213075646,
"learning_rate": 6.681874303634842e-05,
"loss_nats": 1.024570107460022,
"step": 6170,
"unclipped_grad_norm": 0.7621827721595764
},
{
"bits_per_byte": 1.4841139666615555,
"learning_rate": 6.643656636628402e-05,
"loss_nats": 1.0287094116210938,
"step": 6180,
"unclipped_grad_norm": 0.7684755921363831
},
{
"bits_per_byte": 1.493858503659442,
"learning_rate": 6.605607387335466e-05,
"loss_nats": 1.035463809967041,
"step": 6190,
"unclipped_grad_norm": 0.8198496699333191
},
{
"bits_per_byte": 1.4883674415826944,
"learning_rate": 6.567727205912724e-05,
"loss_nats": 1.0316576957702637,
"step": 6200,
"unclipped_grad_norm": 0.7631723284721375
},
{
"bits_per_byte": 1.5053304344025948,
"learning_rate": 6.530016739627959e-05,
"loss_nats": 1.0434155464172363,
"step": 6210,
"unclipped_grad_norm": 0.7891849875450134
},
{
"bits_per_byte": 1.5365181122753584,
"learning_rate": 6.492476632848993e-05,
"loss_nats": 1.065033197402954,
"step": 6220,
"unclipped_grad_norm": 0.7962750196456909
},
{
"bits_per_byte": 1.4799394317771624,
"learning_rate": 6.455107527032678e-05,
"loss_nats": 1.0258158445358276,
"step": 6230,
"unclipped_grad_norm": 0.7991603016853333
},
{
"bits_per_byte": 1.5356217386990614,
"learning_rate": 6.417910060713926e-05,
"loss_nats": 1.0644118785858154,
"step": 6240,
"unclipped_grad_norm": 0.8105412721633911
},
{
"bits_per_byte": 1.4819163723488304,
"learning_rate": 6.380884869494799e-05,
"loss_nats": 1.0271861553192139,
"step": 6250,
"unclipped_grad_norm": 0.7951217293739319
},
{
"bits_per_byte": 1.4447946050906966,
"learning_rate": 6.34403258603367e-05,
"loss_nats": 1.001455307006836,
"step": 6260,
"unclipped_grad_norm": 0.7625995874404907
},
{
"bits_per_byte": 1.5022859975170215,
"learning_rate": 6.30735384003437e-05,
"loss_nats": 1.0413053035736084,
"step": 6270,
"unclipped_grad_norm": 0.8279880881309509
},
{
"bits_per_byte": 1.5067209141348208,
"learning_rate": 6.270849258235484e-05,
"loss_nats": 1.0443793535232544,
"step": 6280,
"unclipped_grad_norm": 0.7734115719795227
},
{
"bits_per_byte": 1.5122324421470212,
"learning_rate": 6.234519464399581e-05,
"loss_nats": 1.0481996536254883,
"step": 6290,
"unclipped_grad_norm": 0.7805989980697632
},
{
"bits_per_byte": 1.476189006109797,
"learning_rate": 6.198365079302614e-05,
"loss_nats": 1.0232162475585938,
"step": 6300,
"unclipped_grad_norm": 0.8065475225448608
},
{
"bits_per_byte": 1.5409463215697738,
"learning_rate": 6.16238672072326e-05,
"loss_nats": 1.0681025981903076,
"step": 6310,
"unclipped_grad_norm": 0.8388196229934692
},
{
"bits_per_byte": 1.506346851869194,
"learning_rate": 6.126585003432406e-05,
"loss_nats": 1.0441200733184814,
"step": 6320,
"unclipped_grad_norm": 0.8094848990440369
},
{
"bits_per_byte": 1.5104558613636083,
"learning_rate": 6.090960539182614e-05,
"loss_nats": 1.0469682216644287,
"step": 6330,
"unclipped_grad_norm": 0.7866588234901428
},
{
"bits_per_byte": 1.427382135612766,
"learning_rate": 6.0555139366976935e-05,
"loss_nats": 0.9893859028816223,
"step": 6340,
"unclipped_grad_norm": 0.7612236738204956
},
{
"bits_per_byte": 1.541960503261912,
"learning_rate": 6.020245801662274e-05,
"loss_nats": 1.0688055753707886,
"step": 6350,
"unclipped_grad_norm": 0.783340573310852
},
{
"bits_per_byte": 1.5465876964833758,
"learning_rate": 5.985156736711483e-05,
"loss_nats": 1.0720129013061523,
"step": 6360,
"unclipped_grad_norm": 0.8162001371383667
},
{
"bits_per_byte": 1.4662069610710688,
"learning_rate": 5.950247341420618e-05,
"loss_nats": 1.0162972211837769,
"step": 6370,
"unclipped_grad_norm": 0.7845266461372375
},
{
"bits_per_byte": 1.4477447954835971,
"learning_rate": 5.915518212294935e-05,
"loss_nats": 1.00350022315979,
"step": 6380,
"unclipped_grad_norm": 0.7937688231468201
},
{
"bits_per_byte": 1.5785561755910564,
"learning_rate": 5.880969942759425e-05,
"loss_nats": 1.0941717624664307,
"step": 6390,
"unclipped_grad_norm": 0.826732337474823
},
{
"bits_per_byte": 1.5478830698096744,
"learning_rate": 5.846603123148688e-05,
"loss_nats": 1.0729107856750488,
"step": 6400,
"unclipped_grad_norm": 0.8436270952224731
},
{
"bits_per_byte": 1.550731790438566,
"learning_rate": 5.812418340696858e-05,
"loss_nats": 1.074885368347168,
"step": 6410,
"unclipped_grad_norm": 0.8814867734909058
},
{
"bits_per_byte": 1.4786806907554975,
"learning_rate": 5.778416179527541e-05,
"loss_nats": 1.0249433517456055,
"step": 6420,
"unclipped_grad_norm": 0.8324126601219177
},
{
"bits_per_byte": 1.5252604719151805,
"learning_rate": 5.7445972206438635e-05,
"loss_nats": 1.057229995727539,
"step": 6430,
"unclipped_grad_norm": 0.7683992981910706
},
{
"bits_per_byte": 1.4432244034883193,
"learning_rate": 5.710962041918516e-05,
"loss_nats": 1.0003669261932373,
"step": 6440,
"unclipped_grad_norm": 0.8131992816925049
},
{
"bits_per_byte": 1.465075831176344,
"learning_rate": 5.677511218083908e-05,
"loss_nats": 1.0155131816864014,
"step": 6450,
"unclipped_grad_norm": 0.758244514465332
},
{
"bits_per_byte": 1.4922360193312023,
"learning_rate": 5.644245320722316e-05,
"loss_nats": 1.034339189529419,
"step": 6460,
"unclipped_grad_norm": 0.798247754573822
},
{
"bits_per_byte": 1.498638761440174,
"learning_rate": 5.61116491825615e-05,
"loss_nats": 1.038777232170105,
"step": 6470,
"unclipped_grad_norm": 0.8465205430984497
},
{
"bits_per_byte": 1.456987831071568,
"learning_rate": 5.5782705759382104e-05,
"loss_nats": 1.0099070072174072,
"step": 6480,
"unclipped_grad_norm": 0.7985926270484924
},
{
"bits_per_byte": 1.542977092711162,
"learning_rate": 5.5455628558420546e-05,
"loss_nats": 1.0695102214813232,
"step": 6490,
"unclipped_grad_norm": 0.8188008666038513
},
{
"bits_per_byte": 1.582791420351204,
"learning_rate": 5.513042316852368e-05,
"loss_nats": 1.0971074104309082,
"step": 6500,
"unclipped_grad_norm": 0.8359130620956421
},
{
"bits_per_byte": 1.5352007251697493,
"learning_rate": 5.480709514655443e-05,
"loss_nats": 1.0641200542449951,
"step": 6510,
"unclipped_grad_norm": 0.854469895362854
},
{
"bits_per_byte": 1.5259941499037615,
"learning_rate": 5.448565001729653e-05,
"loss_nats": 1.0577385425567627,
"step": 6520,
"unclipped_grad_norm": 0.8313364386558533
},
{
"bits_per_byte": 1.4871191915027322,
"learning_rate": 5.41660932733604e-05,
"loss_nats": 1.030792474746704,
"step": 6530,
"unclipped_grad_norm": 0.7717711329460144
},
{
"bits_per_byte": 1.4761642406080728,
"learning_rate": 5.384843037508902e-05,
"loss_nats": 1.0231990814208984,
"step": 6540,
"unclipped_grad_norm": 0.8019536733627319
},
{
"bits_per_byte": 1.5028273989019376,
"learning_rate": 5.353266675046498e-05,
"loss_nats": 1.0416805744171143,
"step": 6550,
"unclipped_grad_norm": 0.7671847343444824
},
{
"bits_per_byte": 1.5359399066031576,
"learning_rate": 5.321880779501729e-05,
"loss_nats": 1.0646324157714844,
"step": 6560,
"unclipped_grad_norm": 0.8164054751396179
},
{
"bits_per_byte": 1.504123116193538,
"learning_rate": 5.290685887172962e-05,
"loss_nats": 1.0425786972045898,
"step": 6570,
"unclipped_grad_norm": 0.7928623557090759
},
{
"bits_per_byte": 1.4852913598893622,
"learning_rate": 5.2596825310948295e-05,
"loss_nats": 1.0295255184173584,
"step": 6580,
"unclipped_grad_norm": 0.8092948794364929
},
{
"bits_per_byte": 1.5248497773449203,
"learning_rate": 5.228871241029154e-05,
"loss_nats": 1.0569453239440918,
"step": 6590,
"unclipped_grad_norm": 0.8197707533836365
},
{
"bits_per_byte": 1.426588779644336,
"learning_rate": 5.198252543455865e-05,
"loss_nats": 0.9888359904289246,
"step": 6600,
"unclipped_grad_norm": 0.7768723964691162
},
{
"bits_per_byte": 1.4662528804388493,
"learning_rate": 5.167826961564027e-05,
"loss_nats": 1.016329050064087,
"step": 6610,
"unclipped_grad_norm": 0.8190019726753235
},
{
"bits_per_byte": 1.5755777800434143,
"learning_rate": 5.137595015242884e-05,
"loss_nats": 1.0921072959899902,
"step": 6620,
"unclipped_grad_norm": 0.8630855083465576
},
{
"bits_per_byte": 1.5211641891369303,
"learning_rate": 5.1075572210729925e-05,
"loss_nats": 1.0543906688690186,
"step": 6630,
"unclipped_grad_norm": 0.8092710971832275
},
{
"bits_per_byte": 1.4689426890983437,
"learning_rate": 5.07771409231737e-05,
"loss_nats": 1.0181934833526611,
"step": 6640,
"unclipped_grad_norm": 0.7870210409164429
},
{
"bits_per_byte": 1.4737494322073081,
"learning_rate": 5.048066138912756e-05,
"loss_nats": 1.021525263786316,
"step": 6650,
"unclipped_grad_norm": 0.8340798020362854
},
{
"bits_per_byte": 1.5123179175245,
"learning_rate": 5.018613867460863e-05,
"loss_nats": 1.048258900642395,
"step": 6660,
"unclipped_grad_norm": 0.8463515043258667
},
{
"bits_per_byte": 1.4946623505695749,
"learning_rate": 4.989357781219755e-05,
"loss_nats": 1.0360209941864014,
"step": 6670,
"unclipped_grad_norm": 0.8076959252357483
},
{
"bits_per_byte": 1.4999131528830674,
"learning_rate": 4.960298380095222e-05,
"loss_nats": 1.0396605730056763,
"step": 6680,
"unclipped_grad_norm": 0.8417643904685974
},
{
"bits_per_byte": 1.4927383806543726,
"learning_rate": 4.931436160632256e-05,
"loss_nats": 1.0346873998641968,
"step": 6690,
"unclipped_grad_norm": 0.8966479301452637
},
{
"bits_per_byte": 1.507631906236441,
"learning_rate": 4.9027716160065485e-05,
"loss_nats": 1.0450108051300049,
"step": 6700,
"unclipped_grad_norm": 0.8065080642700195
},
{
"bits_per_byte": 1.526726795996437,
"learning_rate": 4.874305236016084e-05,
"loss_nats": 1.058246374130249,
"step": 6710,
"unclipped_grad_norm": 0.8376452326774597
},
{
"bits_per_byte": 1.5822010039107919,
"learning_rate": 4.846037507072752e-05,
"loss_nats": 1.0966981649398804,
"step": 6720,
"unclipped_grad_norm": 0.8624657392501831
},
{
"bits_per_byte": 1.447227471669802,
"learning_rate": 4.817968912194049e-05,
"loss_nats": 1.0031416416168213,
"step": 6730,
"unclipped_grad_norm": 0.8040223717689514
},
{
"bits_per_byte": 1.4848161718250283,
"learning_rate": 4.790099930994813e-05,
"loss_nats": 1.0291961431503296,
"step": 6740,
"unclipped_grad_norm": 0.8131929636001587
},
{
"bits_per_byte": 1.4599282184533695,
"learning_rate": 4.762431039679047e-05,
"loss_nats": 1.011945128440857,
"step": 6750,
"unclipped_grad_norm": 0.8242661356925964
},
{
"bits_per_byte": 1.4561461479782454,
"learning_rate": 4.734962711031755e-05,
"loss_nats": 1.0093235969543457,
"step": 6760,
"unclipped_grad_norm": 0.858147919178009
},
{
"bits_per_byte": 1.4679376224867013,
"learning_rate": 4.707695414410892e-05,
"loss_nats": 1.0174968242645264,
"step": 6770,
"unclipped_grad_norm": 0.8389740586280823
},
{
"bits_per_byte": 1.446337461451587,
"learning_rate": 4.680629615739322e-05,
"loss_nats": 1.002524733543396,
"step": 6780,
"unclipped_grad_norm": 0.806380033493042
},
{
"bits_per_byte": 1.4957231395600967,
"learning_rate": 4.653765777496863e-05,
"loss_nats": 1.0367562770843506,
"step": 6790,
"unclipped_grad_norm": 0.8443998694419861
},
{
"bits_per_byte": 1.483685729860907,
"learning_rate": 4.6271043587123986e-05,
"loss_nats": 1.0284125804901123,
"step": 6800,
"unclipped_grad_norm": 0.8614793419837952
},
{
"bits_per_byte": 1.4736859706091399,
"learning_rate": 4.600645814956006e-05,
"loss_nats": 1.0214812755584717,
"step": 6810,
"unclipped_grad_norm": 0.8073354959487915
},
{
"bits_per_byte": 1.4900140034820548,
"learning_rate": 4.574390598331204e-05,
"loss_nats": 1.0327990055084229,
"step": 6820,
"unclipped_grad_norm": 0.8066654801368713
},
{
"bits_per_byte": 1.4893948659389489,
"learning_rate": 4.548339157467199e-05,
"loss_nats": 1.03236985206604,
"step": 6830,
"unclipped_grad_norm": 0.8152909278869629
},
{
"bits_per_byte": 1.4517927511369553,
"learning_rate": 4.52249193751124e-05,
"loss_nats": 1.0063060522079468,
"step": 6840,
"unclipped_grad_norm": 0.8145732283592224
},
{
"bits_per_byte": 1.5401706798143826,
"learning_rate": 4.4968493801209985e-05,
"loss_nats": 1.0675649642944336,
"step": 6850,
"unclipped_grad_norm": 0.8944696187973022
},
{
"bits_per_byte": 1.5020013662298435,
"learning_rate": 4.471411923457034e-05,
"loss_nats": 1.0411080121994019,
"step": 6860,
"unclipped_grad_norm": 0.8296581506729126
},
{
"bits_per_byte": 1.4315734388056183,
"learning_rate": 4.446180002175291e-05,
"loss_nats": 0.9922910928726196,
"step": 6870,
"unclipped_grad_norm": 0.8298041224479675
},
{
"bits_per_byte": 1.4506086505857647,
"learning_rate": 4.421154047419693e-05,
"loss_nats": 1.0054852962493896,
"step": 6880,
"unclipped_grad_norm": 0.8620857000350952
},
{
"bits_per_byte": 1.4600797351687795,
"learning_rate": 4.396334486814753e-05,
"loss_nats": 1.0120501518249512,
"step": 6890,
"unclipped_grad_norm": 0.8483932614326477
},
{
"bits_per_byte": 1.5166142161257037,
"learning_rate": 4.371721744458285e-05,
"loss_nats": 1.051236867904663,
"step": 6900,
"unclipped_grad_norm": 0.8512884378433228
},
{
"bits_per_byte": 1.5060785589338481,
"learning_rate": 4.347316240914145e-05,
"loss_nats": 1.0439341068267822,
"step": 6910,
"unclipped_grad_norm": 0.8712192177772522
},
{
"bits_per_byte": 1.4495721111490145,
"learning_rate": 4.3231183932050546e-05,
"loss_nats": 1.004766821861267,
"step": 6920,
"unclipped_grad_norm": 0.8044716715812683
},
{
"bits_per_byte": 1.4331765750856358,
"learning_rate": 4.299128614805464e-05,
"loss_nats": 0.9934023022651672,
"step": 6930,
"unclipped_grad_norm": 0.8494104743003845
},
{
"bits_per_byte": 1.5111775005666286,
"learning_rate": 4.2753473156344965e-05,
"loss_nats": 1.0474684238433838,
"step": 6940,
"unclipped_grad_norm": 0.8407061696052551
},
{
"bits_per_byte": 1.4275953081085107,
"learning_rate": 4.251774902048942e-05,
"loss_nats": 0.9895336627960205,
"step": 6950,
"unclipped_grad_norm": 0.8335278630256653
},
{
"bits_per_byte": 1.4337008641967912,
"learning_rate": 4.228411776836306e-05,
"loss_nats": 0.9937657117843628,
"step": 6960,
"unclipped_grad_norm": 0.8460164666175842
},
{
"bits_per_byte": 1.404613954404317,
"learning_rate": 4.2052583392079427e-05,
"loss_nats": 0.9736042022705078,
"step": 6970,
"unclipped_grad_norm": 0.8476742506027222
},
{
"bits_per_byte": 1.5285965913766173,
"learning_rate": 4.182314984792214e-05,
"loss_nats": 1.0595424175262451,
"step": 6980,
"unclipped_grad_norm": 0.9341830611228943
},
{
"bits_per_byte": 1.4927865357966141,
"learning_rate": 4.159582105627748e-05,
"loss_nats": 1.034720778465271,
"step": 6990,
"unclipped_grad_norm": 0.8535965085029602
},
{
"bits_per_byte": 1.5015640143486995,
"learning_rate": 4.137060090156724e-05,
"loss_nats": 1.0408048629760742,
"step": 7000,
"unclipped_grad_norm": 0.8839671015739441
},
{
"bits_per_byte": 1.409006391307353,
"learning_rate": 4.114749323218249e-05,
"loss_nats": 0.9766488075256348,
"step": 7010,
"unclipped_grad_norm": 0.8374313712120056
},
{
"bits_per_byte": 1.4918236049344333,
"learning_rate": 4.0926501860417706e-05,
"loss_nats": 1.0340533256530762,
"step": 7020,
"unclipped_grad_norm": 0.8843756318092346
},
{
"bits_per_byte": 1.4765720114732686,
"learning_rate": 4.070763056240571e-05,
"loss_nats": 1.0234817266464233,
"step": 7030,
"unclipped_grad_norm": 0.8468295931816101
},
{
"bits_per_byte": 1.4469567709773439,
"learning_rate": 4.049088307805306e-05,
"loss_nats": 1.0029540061950684,
"step": 7040,
"unclipped_grad_norm": 0.8328400254249573
},
{
"bits_per_byte": 1.4604876780166263,
"learning_rate": 4.027626311097629e-05,
"loss_nats": 1.0123329162597656,
"step": 7050,
"unclipped_grad_norm": 0.8563967943191528
},
{
"bits_per_byte": 1.4275743262251053,
"learning_rate": 4.006377432843846e-05,
"loss_nats": 0.9895191192626953,
"step": 7060,
"unclipped_grad_norm": 0.830858588218689
},
{
"bits_per_byte": 1.4821155282585297,
"learning_rate": 3.985342036128656e-05,
"loss_nats": 1.0273241996765137,
"step": 7070,
"unclipped_grad_norm": 0.8345243334770203
},
{
"bits_per_byte": 1.482162651504866,
"learning_rate": 3.964520480388956e-05,
"loss_nats": 1.0273568630218506,
"step": 7080,
"unclipped_grad_norm": 0.9049980044364929
},
{
"bits_per_byte": 1.4400559671114737,
"learning_rate": 3.943913121407678e-05,
"loss_nats": 0.9981707334518433,
"step": 7090,
"unclipped_grad_norm": 0.86240553855896
},
{
"bits_per_byte": 1.4348600272636065,
"learning_rate": 3.923520311307735e-05,
"loss_nats": 0.9945691823959351,
"step": 7100,
"unclipped_grad_norm": 0.8350730538368225
},
{
"bits_per_byte": 1.4152483296377678,
"learning_rate": 3.9033423985459816e-05,
"loss_nats": 0.9809753894805908,
"step": 7110,
"unclipped_grad_norm": 0.8401304483413696
},
{
"bits_per_byte": 1.4797989219514076,
"learning_rate": 3.88337972790728e-05,
"loss_nats": 1.0257184505462646,
"step": 7120,
"unclipped_grad_norm": 0.8903806805610657
},
{
"bits_per_byte": 1.5240091261475028,
"learning_rate": 3.8636326404985845e-05,
"loss_nats": 1.0563626289367676,
"step": 7130,
"unclipped_grad_norm": 0.8470056056976318
},
{
"bits_per_byte": 1.4628170110399121,
"learning_rate": 3.844101473743143e-05,
"loss_nats": 1.0139474868774414,
"step": 7140,
"unclipped_grad_norm": 0.8429069519042969
},
{
"bits_per_byte": 1.5096671489267515,
"learning_rate": 3.824786561374701e-05,
"loss_nats": 1.0464215278625488,
"step": 7150,
"unclipped_grad_norm": 0.8920191526412964
},
{
"bits_per_byte": 1.5009496923198173,
"learning_rate": 3.805688233431824e-05,
"loss_nats": 1.0403790473937988,
"step": 7160,
"unclipped_grad_norm": 0.8647960424423218
},
{
"bits_per_byte": 1.5036095759980617,
"learning_rate": 3.786806816252236e-05,
"loss_nats": 1.0422227382659912,
"step": 7170,
"unclipped_grad_norm": 0.9017168283462524
},
{
"bits_per_byte": 1.4616292988530535,
"learning_rate": 3.768142632467269e-05,
"loss_nats": 1.0131242275238037,
"step": 7180,
"unclipped_grad_norm": 0.8551055788993835
},
{
"bits_per_byte": 1.5122007973392624,
"learning_rate": 3.7496960009963236e-05,
"loss_nats": 1.048177719116211,
"step": 7190,
"unclipped_grad_norm": 0.8699921369552612
},
{
"bits_per_byte": 1.441016404225217,
"learning_rate": 3.7314672370414325e-05,
"loss_nats": 0.9988364577293396,
"step": 7200,
"unclipped_grad_norm": 0.8101796507835388
},
{
"bits_per_byte": 1.4373777672809125,
"learning_rate": 3.713456652081882e-05,
"loss_nats": 0.9963143467903137,
"step": 7210,
"unclipped_grad_norm": 0.8248227834701538
},
{
"bits_per_byte": 1.4388484769197658,
"learning_rate": 3.695664553868873e-05,
"loss_nats": 0.9973337650299072,
"step": 7220,
"unclipped_grad_norm": 0.8497165441513062
},
{
"bits_per_byte": 1.4764554072359837,
"learning_rate": 3.678091246420277e-05,
"loss_nats": 1.023400902748108,
"step": 7230,
"unclipped_grad_norm": 0.869898796081543
},
{
"bits_per_byte": 1.5094511387172678,
"learning_rate": 3.660737030015427e-05,
"loss_nats": 1.046271800994873,
"step": 7240,
"unclipped_grad_norm": 0.8670166730880737
},
{
"bits_per_byte": 1.4896672864579155,
"learning_rate": 3.6436022011900056e-05,
"loss_nats": 1.0325586795806885,
"step": 7250,
"unclipped_grad_norm": 0.8726136684417725
},
{
"bits_per_byte": 1.4660583680607233,
"learning_rate": 3.626687052730954e-05,
"loss_nats": 1.016194224357605,
"step": 7260,
"unclipped_grad_norm": 0.8553854823112488
},
{
"bits_per_byte": 1.4854459722924878,
"learning_rate": 3.6099918736714955e-05,
"loss_nats": 1.0296326875686646,
"step": 7270,
"unclipped_grad_norm": 0.8842402100563049
},
{
"bits_per_byte": 1.4706281190767991,
"learning_rate": 3.5935169492861716e-05,
"loss_nats": 1.0193617343902588,
"step": 7280,
"unclipped_grad_norm": 0.8827332854270935
},
{
"bits_per_byte": 1.465586275684105,
"learning_rate": 3.5772625610859886e-05,
"loss_nats": 1.015866994857788,
"step": 7290,
"unclipped_grad_norm": 0.9209113121032715
},
{
"bits_per_byte": 1.4604543133823589,
"learning_rate": 3.5612289868135887e-05,
"loss_nats": 1.0123097896575928,
"step": 7300,
"unclipped_grad_norm": 0.8443734645843506
},
{
"bits_per_byte": 1.44415740936925,
"learning_rate": 3.545416500438522e-05,
"loss_nats": 1.0010136365890503,
"step": 7310,
"unclipped_grad_norm": 0.8828385472297668
},
{
"bits_per_byte": 1.4200490533539523,
"learning_rate": 3.52982537215255e-05,
"loss_nats": 0.9843029975891113,
"step": 7320,
"unclipped_grad_norm": 0.878987193107605
},
{
"bits_per_byte": 1.4725175204791778,
"learning_rate": 3.5144558683650396e-05,
"loss_nats": 1.0206713676452637,
"step": 7330,
"unclipped_grad_norm": 0.9326861500740051
},
{
"bits_per_byte": 1.4404713912046327,
"learning_rate": 3.499308251698404e-05,
"loss_nats": 0.9984586834907532,
"step": 7340,
"unclipped_grad_norm": 0.8777600526809692
},
{
"bits_per_byte": 1.448391966198794,
"learning_rate": 3.484382780983616e-05,
"loss_nats": 1.0039488077163696,
"step": 7350,
"unclipped_grad_norm": 0.8631419539451599
},
{
"bits_per_byte": 1.453881480431684,
"learning_rate": 3.469679711255795e-05,
"loss_nats": 1.007753849029541,
"step": 7360,
"unclipped_grad_norm": 0.855156421661377
},
{
"bits_per_byte": 1.4104983408035878,
"learning_rate": 3.4551992937498284e-05,
"loss_nats": 0.9776829481124878,
"step": 7370,
"unclipped_grad_norm": 0.8616177439689636
},
{
"bits_per_byte": 1.406926175153842,
"learning_rate": 3.440941775896104e-05,
"loss_nats": 0.9752069115638733,
"step": 7380,
"unclipped_grad_norm": 0.8499169945716858
},
{
"bits_per_byte": 1.4229885808224993,
"learning_rate": 3.4269074013162596e-05,
"loss_nats": 0.9863405227661133,
"step": 7390,
"unclipped_grad_norm": 0.8316128849983215
},
{
"bits_per_byte": 1.4424061960268393,
"learning_rate": 3.41309640981904e-05,
"loss_nats": 0.9997997879981995,
"step": 7400,
"unclipped_grad_norm": 0.8269481062889099
},
{
"bits_per_byte": 1.4779136481326494,
"learning_rate": 3.3995090373961795e-05,
"loss_nats": 1.024411678314209,
"step": 7410,
"unclipped_grad_norm": 0.882456362247467
},
{
"bits_per_byte": 1.464608382331299,
"learning_rate": 3.3861455162183876e-05,
"loss_nats": 1.0151891708374023,
"step": 7420,
"unclipped_grad_norm": 0.8758774995803833
},
{
"bits_per_byte": 1.3991048341492285,
"learning_rate": 3.3730060746313664e-05,
"loss_nats": 0.9697855710983276,
"step": 7430,
"unclipped_grad_norm": 0.8292478919029236
},
{
"bits_per_byte": 1.401245502204518,
"learning_rate": 3.3600909371519225e-05,
"loss_nats": 0.9712693691253662,
"step": 7440,
"unclipped_grad_norm": 0.875770092010498
},
{
"bits_per_byte": 1.5201309173705464,
"learning_rate": 3.3474003244641194e-05,
"loss_nats": 1.0536744594573975,
"step": 7450,
"unclipped_grad_norm": 0.9047821760177612
},
{
"bits_per_byte": 1.4276059710328641,
"learning_rate": 3.334934453415514e-05,
"loss_nats": 0.9895410537719727,
"step": 7460,
"unclipped_grad_norm": 0.8659260272979736
},
{
"bits_per_byte": 1.4078822267099882,
"learning_rate": 3.3226935370134486e-05,
"loss_nats": 0.9758695960044861,
"step": 7470,
"unclipped_grad_norm": 0.8766598105430603
},
{
"bits_per_byte": 1.486473568631393,
"learning_rate": 3.31067778442141e-05,
"loss_nats": 1.0303449630737305,
"step": 7480,
"unclipped_grad_norm": 0.8612073659896851
},
{
"bits_per_byte": 1.450941608997835,
"learning_rate": 3.298887400955457e-05,
"loss_nats": 1.00571608543396,
"step": 7490,
"unclipped_grad_norm": 0.888404905796051
},
{
"bits_per_byte": 1.4814055838757678,
"learning_rate": 3.287322588080714e-05,
"loss_nats": 1.026832103729248,
"step": 7500,
"unclipped_grad_norm": 0.9002490639686584
},
{
"bits_per_byte": 1.4606080658722302,
"learning_rate": 3.275983543407924e-05,
"loss_nats": 1.0124163627624512,
"step": 7510,
"unclipped_grad_norm": 0.8679240345954895
},
{
"bits_per_byte": 1.4798904167216664,
"learning_rate": 3.2648704606900735e-05,
"loss_nats": 1.0257818698883057,
"step": 7520,
"unclipped_grad_norm": 0.9162602424621582
},
{
"bits_per_byte": 1.4597773896685629,
"learning_rate": 3.253983529819086e-05,
"loss_nats": 1.011840581893921,
"step": 7530,
"unclipped_grad_norm": 0.8853410482406616
},
{
"bits_per_byte": 1.4621115382060728,
"learning_rate": 3.243322936822571e-05,
"loss_nats": 1.013458490371704,
"step": 7540,
"unclipped_grad_norm": 0.869069516658783
},
{
"bits_per_byte": 1.4571945542179052,
"learning_rate": 3.232888863860648e-05,
"loss_nats": 1.0100502967834473,
"step": 7550,
"unclipped_grad_norm": 0.8861801028251648
},
{
"bits_per_byte": 1.38000641874489,
"learning_rate": 3.222681489222838e-05,
"loss_nats": 0.9565475583076477,
"step": 7560,
"unclipped_grad_norm": 0.8688139915466309
},
{
"bits_per_byte": 1.4141796294453064,
"learning_rate": 3.212700987325009e-05,
"loss_nats": 0.9802346229553223,
"step": 7570,
"unclipped_grad_norm": 0.8542686700820923
},
{
"bits_per_byte": 1.4002238392670674,
"learning_rate": 3.202947528706403e-05,
"loss_nats": 0.9705612063407898,
"step": 7580,
"unclipped_grad_norm": 0.898608922958374
},
{
"bits_per_byte": 1.4730534184192665,
"learning_rate": 3.1934212800267116e-05,
"loss_nats": 1.021042823791504,
"step": 7590,
"unclipped_grad_norm": 0.8726481199264526
},
{
"bits_per_byte": 1.5103581752179183,
"learning_rate": 3.1841224040632484e-05,
"loss_nats": 1.0469005107879639,
"step": 7600,
"unclipped_grad_norm": 0.8932017087936401
},
{
"bits_per_byte": 1.4571630813927972,
"learning_rate": 3.175051059708143e-05,
"loss_nats": 1.0100284814834595,
"step": 7610,
"unclipped_grad_norm": 0.8839172124862671
},
{
"bits_per_byte": 1.4709910024701196,
"learning_rate": 3.1662074019656414e-05,
"loss_nats": 1.019613265991211,
"step": 7620,
"unclipped_grad_norm": 0.8883293271064758
},
{
"bits_per_byte": 1.436151788954237,
"learning_rate": 3.157591581949455e-05,
"loss_nats": 0.995464563369751,
"step": 7630,
"unclipped_grad_norm": 0.8680411577224731
},
{
"bits_per_byte": 1.4213451146108544,
"learning_rate": 3.149203746880175e-05,
"loss_nats": 0.985201358795166,
"step": 7640,
"unclipped_grad_norm": 0.9127815961837769
},
{
"bits_per_byte": 1.3999687889958379,
"learning_rate": 3.141044040082763e-05,
"loss_nats": 0.970384418964386,
"step": 7650,
"unclipped_grad_norm": 0.8844171166419983
},
{
"bits_per_byte": 1.4213168234647875,
"learning_rate": 3.133112600984091e-05,
"loss_nats": 0.9851817488670349,
"step": 7660,
"unclipped_grad_norm": 0.8884557485580444
},
{
"bits_per_byte": 1.467116749294133,
"learning_rate": 3.125409565110571e-05,
"loss_nats": 1.0169278383255005,
"step": 7670,
"unclipped_grad_norm": 0.9028065204620361
},
{
"bits_per_byte": 1.4497504571579594,
"learning_rate": 3.117935064085835e-05,
"loss_nats": 1.0048904418945312,
"step": 7680,
"unclipped_grad_norm": 0.8733177781105042
},
{
"bits_per_byte": 1.436977993608982,
"learning_rate": 3.110689225628479e-05,
"loss_nats": 0.9960372447967529,
"step": 7690,
"unclipped_grad_norm": 0.8574365973472595
},
{
"bits_per_byte": 1.4762694939904009,
"learning_rate": 3.1036721735498926e-05,
"loss_nats": 1.0232720375061035,
"step": 7700,
"unclipped_grad_norm": 0.8926270008087158
},
{
"bits_per_byte": 1.4168368473924622,
"learning_rate": 3.096884027752132e-05,
"loss_nats": 0.9820764660835266,
"step": 7710,
"unclipped_grad_norm": 0.8520281314849854
},
{
"bits_per_byte": 1.4226952644114528,
"learning_rate": 3.090324904225885e-05,
"loss_nats": 0.9861372113227844,
"step": 7720,
"unclipped_grad_norm": 0.8500117063522339
},
{
"bits_per_byte": 1.4242657239878067,
"learning_rate": 3.083994915048472e-05,
"loss_nats": 0.9872257709503174,
"step": 7730,
"unclipped_grad_norm": 0.8397486209869385
},
{
"bits_per_byte": 1.4715379072998633,
"learning_rate": 3.077894168381942e-05,
"loss_nats": 1.0199923515319824,
"step": 7740,
"unclipped_grad_norm": 0.9084712266921997
},
{
"bits_per_byte": 1.4017008262726773,
"learning_rate": 3.0720227684712254e-05,
"loss_nats": 0.9715849757194519,
"step": 7750,
"unclipped_grad_norm": 0.8779424428939819
},
{
"bits_per_byte": 1.4074879564828853,
"learning_rate": 3.0663808156423456e-05,
"loss_nats": 0.9755963087081909,
"step": 7760,
"unclipped_grad_norm": 0.8754757642745972
},
{
"bits_per_byte": 1.4701193943955468,
"learning_rate": 3.060968406300711e-05,
"loss_nats": 1.0190091133117676,
"step": 7770,
"unclipped_grad_norm": 0.9014414548873901
},
{
"bits_per_byte": 1.4337202982363386,
"learning_rate": 3.055785632929463e-05,
"loss_nats": 0.993779182434082,
"step": 7780,
"unclipped_grad_norm": 0.9682468175888062
},
{
"bits_per_byte": 1.461853220264477,
"learning_rate": 3.0508325840878984e-05,
"loss_nats": 1.0132794380187988,
"step": 7790,
"unclipped_grad_norm": 0.8698559403419495
},
{
"bits_per_byte": 1.446109240473892,
"learning_rate": 3.046109344409957e-05,
"loss_nats": 1.002366542816162,
"step": 7800,
"unclipped_grad_norm": 0.8869026303291321
},
{
"bits_per_byte": 1.4204974121247516,
"learning_rate": 3.0416159946027727e-05,
"loss_nats": 0.9846137762069702,
"step": 7810,
"unclipped_grad_norm": 0.893868088722229
},
{
"bits_per_byte": 1.471287500560207,
"learning_rate": 3.0373526114452946e-05,
"loss_nats": 1.0198187828063965,
"step": 7820,
"unclipped_grad_norm": 0.9250245690345764
},
{
"bits_per_byte": 1.457534563918661,
"learning_rate": 3.0333192677869803e-05,
"loss_nats": 1.0102859735488892,
"step": 7830,
"unclipped_grad_norm": 0.902686595916748
},
{
"bits_per_byte": 1.4368130622468045,
"learning_rate": 3.029516032546544e-05,
"loss_nats": 0.9959229230880737,
"step": 7840,
"unclipped_grad_norm": 0.880992591381073
},
{
"bits_per_byte": 1.4160461571551204,
"learning_rate": 3.02594297071078e-05,
"loss_nats": 0.9815284013748169,
"step": 7850,
"unclipped_grad_norm": 0.8318398594856262
},
{
"bits_per_byte": 1.463385069735712,
"learning_rate": 3.0226001433334603e-05,
"loss_nats": 1.0143412351608276,
"step": 7860,
"unclipped_grad_norm": 0.8814117908477783
},
{
"bits_per_byte": 1.4569142224969986,
"learning_rate": 3.019487607534278e-05,
"loss_nats": 1.0098559856414795,
"step": 7870,
"unclipped_grad_norm": 0.8954723477363586
},
{
"bits_per_byte": 1.4148711716844258,
"learning_rate": 3.016605416497886e-05,
"loss_nats": 0.980713963508606,
"step": 7880,
"unclipped_grad_norm": 0.9006734490394592
},
{
"bits_per_byte": 1.44079876018055,
"learning_rate": 3.0139536194729734e-05,
"loss_nats": 0.9986855983734131,
"step": 7890,
"unclipped_grad_norm": 0.8986613154411316
},
{
"bits_per_byte": 1.4040539788931077,
"learning_rate": 3.0115322617714362e-05,
"loss_nats": 0.9732160568237305,
"step": 7900,
"unclipped_grad_norm": 0.8486817479133606
},
{
"bits_per_byte": 1.4565468675547557,
"learning_rate": 3.0093413847675965e-05,
"loss_nats": 1.009601354598999,
"step": 7910,
"unclipped_grad_norm": 0.960801362991333
},
{
"bits_per_byte": 1.4869515084181408,
"learning_rate": 3.0073810258974985e-05,
"loss_nats": 1.030676245689392,
"step": 7920,
"unclipped_grad_norm": 0.9004883766174316
},
{
"bits_per_byte": 1.426570807457321,
"learning_rate": 3.0056512186582658e-05,
"loss_nats": 0.9888235330581665,
"step": 7930,
"unclipped_grad_norm": 0.9118951559066772
},
{
"bits_per_byte": 1.4575464307215704,
"learning_rate": 3.0041519926075316e-05,
"loss_nats": 1.0102941989898682,
"step": 7940,
"unclipped_grad_norm": 0.9135404825210571
},
{
"bits_per_byte": 1.4061704833859507,
"learning_rate": 3.0028833733629313e-05,
"loss_nats": 0.9746831059455872,
"step": 7950,
"unclipped_grad_norm": 0.8805200457572937
},
{
"bits_per_byte": 1.4792671515949398,
"learning_rate": 3.0018453826016686e-05,
"loss_nats": 1.0253498554229736,
"step": 7960,
"unclipped_grad_norm": 0.9637572765350342
},
{
"bits_per_byte": 1.4296003678435947,
"learning_rate": 3.001038038060143e-05,
"loss_nats": 0.9909234642982483,
"step": 7970,
"unclipped_grad_norm": 0.8976891040802002
},
{
"bits_per_byte": 1.4664819613297986,
"learning_rate": 3.0004613535336433e-05,
"loss_nats": 1.0164878368377686,
"step": 7980,
"unclipped_grad_norm": 0.9626050591468811
},
{
"bits_per_byte": 1.4209173077668331,
"learning_rate": 3.000115338876117e-05,
"loss_nats": 0.9849048256874084,
"step": 7990,
"unclipped_grad_norm": 0.9410001635551453
},
{
"bits_per_byte": 1.4482875727297202,
"learning_rate": 3e-05,
"loss_nats": 1.0038764476776123,
"step": 8000,
"unclipped_grad_norm": 0.9042860269546509
}
]
}
Reference in New Issue View Git Blame Copy Permalink