wuyang
9617 lines
272 KiB
JSON
9617 lines
272 KiB
JSON
{
|
|
"architecture": "block",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "a4c4ddb026a74a7ed70ffc58516bae9bdba5550dc232a1cb003063f2c7f9ca1d",
|
|
"depth": 16,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.0003044046170543879,
|
|
0.0002627764770295471,
|
|
0.0002327302354387939,
|
|
0.0002064981235889718,
|
|
0.0001888095575850457,
|
|
0.00017376731557305902,
|
|
0.00016044666699599475,
|
|
0.0001515327749075368,
|
|
0.000144502759212628,
|
|
0.00014044572890270501,
|
|
0.00013478762411978096,
|
|
0.00013080857752356678,
|
|
0.00012769536988344043,
|
|
0.00012482800229918212,
|
|
0.00012209055421408266,
|
|
0.0001202532512252219
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.0002516023632779252,
|
|
"first_to_last_ratio": 2.0336961080101905,
|
|
"imbalance_abs_log_ratio": 0.7098548803728145,
|
|
"last_quartile_mean": 0.00012371679440548178,
|
|
"mean": 0.00017039860222212155,
|
|
"normalized": [
|
|
1.7864267258341946,
|
|
1.5421281254819637,
|
|
1.3657989702017657,
|
|
1.2118533890306977,
|
|
1.1080463989893807,
|
|
1.0197696067162934,
|
|
0.941596145177564,
|
|
0.8892841427772256,
|
|
0.8480278437041564,
|
|
0.8242187850791655,
|
|
0.7910136724248463,
|
|
0.7676622684560079,
|
|
0.7493921206993488,
|
|
0.7325647081098923,
|
|
0.7164997401500549,
|
|
0.7057173571674424
|
|
],
|
|
"population_cv": 0.31342321744292045
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.005092422943562269,
|
|
0.007266214583069086,
|
|
0.00560376700013876,
|
|
0.007757550571113825,
|
|
0.005751485005021095,
|
|
0.008497623726725578,
|
|
0.006417509168386459,
|
|
0.009013078175485134,
|
|
0.006685016211122274,
|
|
0.010579346679151058,
|
|
0.007517023477703333,
|
|
0.010370824486017227,
|
|
0.008143824525177479,
|
|
0.011447666212916374,
|
|
0.009210918098688126,
|
|
0.013363312929868698
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.006429988774470985,
|
|
"first_to_last_ratio": 0.6099730781372781,
|
|
"imbalance_abs_log_ratio": 0.4943404569899061,
|
|
"last_quartile_mean": 0.01054143044166267,
|
|
"mean": 0.008294848987134174,
|
|
"normalized": [
|
|
0.6139259378273111,
|
|
0.8759911837261217,
|
|
0.675571913223562,
|
|
0.93522505149235,
|
|
0.6933803151741528,
|
|
1.0244458627162376,
|
|
0.7736740208701105,
|
|
1.0865873736176486,
|
|
0.8059237993954019,
|
|
1.275411607319227,
|
|
0.9062278878569946,
|
|
1.250272850307826,
|
|
0.9817929823447126,
|
|
1.3800933845417096,
|
|
1.1104383109294493,
|
|
1.6110375186571844
|
|
],
|
|
"population_cv": 0.2672227042610828
|
|
},
|
|
"bits_per_byte": 8.019773211317682,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0034366261679679155,
|
|
0.003824356012046337,
|
|
0.0035537455696612597,
|
|
0.003800130682066083,
|
|
0.004073208663612604,
|
|
0.0038295022677630186,
|
|
0.004064237233251333,
|
|
0.0037886372301727533,
|
|
0.004280691966414452,
|
|
0.0038349912501871586,
|
|
0.0050888159312307835,
|
|
0.0038198749534785748,
|
|
0.005177160259336233,
|
|
0.0037978272885084152,
|
|
0.005646859761327505,
|
|
0.003781275823712349,
|
|
0.0056319693103432655,
|
|
0.003643249161541462,
|
|
0.006857539992779493,
|
|
0.0037886397913098335,
|
|
0.006555364932864904,
|
|
0.003700864966958761,
|
|
0.0065165311098098755,
|
|
0.0036778782960027456,
|
|
0.0071639628149569035,
|
|
0.003695491934195161,
|
|
0.007599730510264635,
|
|
0.0037103460635989904,
|
|
0.00840203370898962,
|
|
0.0037034295964986086,
|
|
0.008103356696665287,
|
|
0.0036014309152960777
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.006980641236819168,
|
|
0.006790815718474037,
|
|
0.006855853249106197,
|
|
0.00601204909014858,
|
|
0.005899153963056259,
|
|
0.006300783956032697,
|
|
0.005704077288608638,
|
|
0.005731973180528048,
|
|
0.005353243010845193,
|
|
0.00629832195028187,
|
|
0.005348066742481173,
|
|
0.005021425359765903,
|
|
0.004977505888369926,
|
|
0.004996266682376707,
|
|
0.0052721592596541,
|
|
0.004713786845169383
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.006659839823636995,
|
|
"first_to_last_ratio": 1.3346560503958143,
|
|
"imbalance_abs_log_ratio": 0.2886736185043388,
|
|
"last_quartile_mean": 0.004989929668892529,
|
|
"mean": 0.005766007713857367,
|
|
"normalized": [
|
|
1.2106541619850226,
|
|
1.1777326801269727,
|
|
1.189012153526888,
|
|
1.0426710106023453,
|
|
1.0230915836062624,
|
|
1.092746362598529,
|
|
0.9892593925776594,
|
|
0.9940973833164453,
|
|
0.9284141257702132,
|
|
1.0923193763936871,
|
|
0.9275164043967956,
|
|
0.8708669167573224,
|
|
0.8632499530667561,
|
|
0.8665036417431853,
|
|
0.914351752770568,
|
|
0.8175131007613471
|
|
],
|
|
"population_cv": 0.12132893681311732
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028142472729086876,
|
|
0.014130397699773312,
|
|
0.014246499165892601,
|
|
0.01435180939733982,
|
|
0.01446386519819498,
|
|
0.009747256524860859,
|
|
0.009819178842008114,
|
|
0.00989843625575304,
|
|
0.009975779801607132,
|
|
0.007541276980191469,
|
|
0.007608718704432249,
|
|
0.007703919429332018,
|
|
0.0077462210319936275,
|
|
0.00631322804838419,
|
|
0.006355294957756996,
|
|
0.006443085614591837,
|
|
0.006486334837973118,
|
|
0.005488897208124399,
|
|
0.00551867438480258,
|
|
0.005632160697132349,
|
|
0.0056700571440160275,
|
|
0.004926848225295544,
|
|
0.004952999297529459,
|
|
0.005005215294659138,
|
|
0.005028663668781519,
|
|
0.004523467272520065,
|
|
0.004554289858788252,
|
|
0.004648423288017511,
|
|
0.004657648038119078,
|
|
0.004240449517965317,
|
|
0.0042638350278139114,
|
|
0.004358324687927961
|
|
],
|
|
"loss_nats": 5.558883190155029,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.0034366261679679155,
|
|
0.005092422943562269,
|
|
0.006235890556126833,
|
|
0.007266214583069086,
|
|
0.004073208663612604,
|
|
0.00560376700013876,
|
|
0.006768420804291964,
|
|
0.007757550571113825,
|
|
0.004280691966414452,
|
|
0.005751485005021095,
|
|
0.00762521754950285,
|
|
0.008497623726725578,
|
|
0.005177160259336233,
|
|
0.006417509168386459,
|
|
0.008189847692847252,
|
|
0.009013078175485134,
|
|
0.0056319693103432655,
|
|
0.006685016211122274,
|
|
0.009837556630373001,
|
|
0.010579346679151058,
|
|
0.006555364932864904,
|
|
0.007517023477703333,
|
|
0.009742370806634426,
|
|
0.010370824486017227,
|
|
0.0071639628149569035,
|
|
0.008143824525177479,
|
|
0.01089458167552948,
|
|
0.011447666212916374,
|
|
0.00840203370898962,
|
|
0.009210918098688126,
|
|
0.012791652232408524,
|
|
0.013363312929868698
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
7.495960016967729e-05,
|
|
6.749085150659084e-05,
|
|
5.70677402720321e-05,
|
|
5.0705322792055085e-05,
|
|
4.370361421024427e-05,
|
|
3.821287100436166e-05,
|
|
3.268615546403453e-05,
|
|
2.8849746740888804e-05,
|
|
2.5781664589885622e-05,
|
|
2.37614458455937e-05,
|
|
2.279552791151218e-05,
|
|
2.1788309823023155e-05,
|
|
2.0940320609952323e-05,
|
|
2.0449986550374888e-05,
|
|
1.9821660316665657e-05,
|
|
1.9541532310540788e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 6.255587868508883e-05,
|
|
"first_to_last_ratio": 3.098608919721194,
|
|
"imbalance_abs_log_ratio": 1.1309532752053548,
|
|
"last_quartile_mean": 2.0188374946883414e-05,
|
|
"mean": 3.5534771882339555e-05,
|
|
"normalized": [
|
|
2.1094718271409953,
|
|
1.8992904113768394,
|
|
1.605968949540217,
|
|
1.4269212972563121,
|
|
1.229883066506037,
|
|
1.0753655920710468,
|
|
0.9198358039911673,
|
|
0.811873700397296,
|
|
0.7255334204832441,
|
|
0.6686815360535046,
|
|
0.6414992049756577,
|
|
0.6131546276747519,
|
|
0.5892909817822544,
|
|
0.575492270446749,
|
|
0.5578102592665534,
|
|
0.549927051037374
|
|
],
|
|
"population_cv": 0.4936925451995582
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.009570118971168995,
|
|
0.01747945137321949,
|
|
0.011283415369689465,
|
|
0.020727796480059624,
|
|
0.012305145151913166,
|
|
0.023130038753151894,
|
|
0.015205477364361286,
|
|
0.028237100690603256,
|
|
0.017268717288970947,
|
|
0.03324708342552185,
|
|
0.021605173125863075,
|
|
0.04313390702009201,
|
|
0.02657386288046837,
|
|
0.05039682611823082,
|
|
0.03066091425716877,
|
|
0.0630461722612381
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.014765195548534393,
|
|
"first_to_last_ratio": 0.34603674681838265,
|
|
"imbalance_abs_log_ratio": 1.0612103048863237,
|
|
"last_quartile_mean": 0.042669443879276514,
|
|
"mean": 0.02649195003323257,
|
|
"normalized": [
|
|
0.3612463015808142,
|
|
0.6598023683153774,
|
|
0.42591864153205383,
|
|
0.7824186763925585,
|
|
0.46448619812724606,
|
|
0.8730968737347248,
|
|
0.5739659536307038,
|
|
1.0658747527147492,
|
|
0.6518477223197376,
|
|
1.2549881523940432,
|
|
0.8155372895827101,
|
|
1.628189203361138,
|
|
1.0030919901001265,
|
|
1.9023449030747457,
|
|
1.1573672084805566,
|
|
2.3798237646587146
|
|
],
|
|
"population_cv": 0.544699846184418
|
|
},
|
|
"bits_per_byte": 5.090461856197109,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.004518123809248209,
|
|
0.008520865812897682,
|
|
0.004510615952312946,
|
|
0.009082728065550327,
|
|
0.005324283614754677,
|
|
0.0099419467151165,
|
|
0.004656728357076645,
|
|
0.010662632063031197,
|
|
0.00614966033026576,
|
|
0.010214399546384811,
|
|
0.0060766469687223434,
|
|
0.011413711123168468,
|
|
0.0087440712377429,
|
|
0.011133860796689987,
|
|
0.009014265611767769,
|
|
0.011885596439242363,
|
|
0.010757530108094215,
|
|
0.011522865854203701,
|
|
0.011306028813123703,
|
|
0.012577465735375881,
|
|
0.014357471838593483,
|
|
0.01213979721069336,
|
|
0.01672077551484108,
|
|
0.013535024598240852,
|
|
0.01905498467385769,
|
|
0.012188552878797054,
|
|
0.018409499898552895,
|
|
0.013168000616133213,
|
|
0.021850280463695526,
|
|
0.012649565935134888,
|
|
0.02609730139374733,
|
|
0.013150178827345371
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00025280350140969286,
|
|
0.00022135664689195577,
|
|
0.00022214031151724322,
|
|
0.0001810293778015987,
|
|
0.00018338803861517842,
|
|
0.00016050318916432303,
|
|
0.0001626860438404638,
|
|
0.00015310192441761306,
|
|
0.00014075054270255738,
|
|
0.00013076915091262195,
|
|
0.00013639133469956174,
|
|
0.0001377615870880476,
|
|
0.00013681686670328103,
|
|
0.0001370125878547912,
|
|
0.0001387461931789955,
|
|
0.0001464231652858264
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00021933245940512265,
|
|
"first_to_last_ratio": 1.5694663694832551,
|
|
"imbalance_abs_log_ratio": 0.45073566953494,
|
|
"last_quartile_mean": 0.00013974970325572353,
|
|
"mean": 0.0001651050288802345,
|
|
"normalized": [
|
|
1.531167785283354,
|
|
1.3407020270262369,
|
|
1.3454484882976008,
|
|
1.0964498115494445,
|
|
1.1107356320939579,
|
|
0.9721278040582907,
|
|
0.9853488106560013,
|
|
0.9273001885888748,
|
|
0.8524909486836794,
|
|
0.7920361469273026,
|
|
0.8260883125401264,
|
|
0.8343875896595401,
|
|
0.8286656538034743,
|
|
0.8298510880257848,
|
|
0.8403511032946221,
|
|
0.8868486095117084
|
|
],
|
|
"population_cv": 0.21861370307121916
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6929093599319458,
|
|
"mean_weights": [
|
|
0.4913179874420166,
|
|
0.5086820125579834
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6919252872467041,
|
|
"mean_weights": [
|
|
0.5235024690628052,
|
|
0.4764975309371948
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6930201053619385,
|
|
"mean_weights": [
|
|
0.4991857707500458,
|
|
0.5008142590522766
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6926943063735962,
|
|
"mean_weights": [
|
|
0.5129889249801636,
|
|
0.48701101541519165
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.097741961479187,
|
|
"mean_weights": [
|
|
0.3480756878852844,
|
|
0.33060726523399353,
|
|
0.32131701707839966
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0982271432876587,
|
|
"mean_weights": [
|
|
0.3435906767845154,
|
|
0.327891081571579,
|
|
0.328518271446228
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.097235918045044,
|
|
"mean_weights": [
|
|
0.35132721066474915,
|
|
0.33742234110832214,
|
|
0.3112504482269287
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.098421573638916,
|
|
"mean_weights": [
|
|
0.340750515460968,
|
|
0.3319299519062042,
|
|
0.32731953263282776
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3854012489318848,
|
|
"mean_weights": [
|
|
0.26307183504104614,
|
|
0.2547699213027954,
|
|
0.2451326996088028,
|
|
0.23702551424503326
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862006664276123,
|
|
"mean_weights": [
|
|
0.25451934337615967,
|
|
0.25035756826400757,
|
|
0.24778643250465393,
|
|
0.24733665585517883
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3854570388793945,
|
|
"mean_weights": [
|
|
0.26253488659858704,
|
|
0.2546469569206238,
|
|
0.2452046275138855,
|
|
0.2376135289669037
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3861933946609497,
|
|
"mean_weights": [
|
|
0.2505362629890442,
|
|
0.254558801651001,
|
|
0.2493266463279724,
|
|
0.24557828903198242
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6085797548294067,
|
|
"mean_weights": [
|
|
0.2100008726119995,
|
|
0.20838744938373566,
|
|
0.19692982733249664,
|
|
0.19323000311851501,
|
|
0.19145184755325317
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6093010902404785,
|
|
"mean_weights": [
|
|
0.19938474893569946,
|
|
0.20374906063079834,
|
|
0.20229801535606384,
|
|
0.19861868023872375,
|
|
0.1959494650363922
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6085379123687744,
|
|
"mean_weights": [
|
|
0.20589761435985565,
|
|
0.21123698353767395,
|
|
0.20005744695663452,
|
|
0.1941205859184265,
|
|
0.18868738412857056
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6093740463256836,
|
|
"mean_weights": [
|
|
0.19956088066101074,
|
|
0.2027459442615509,
|
|
0.20118315517902374,
|
|
0.1998024433851242,
|
|
0.1967075765132904
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.789860725402832,
|
|
"mean_weights": [
|
|
0.17310243844985962,
|
|
0.1810513734817505,
|
|
0.17217648029327393,
|
|
0.1649792194366455,
|
|
0.1559385061264038,
|
|
0.15275199711322784
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791734218597412,
|
|
"mean_weights": [
|
|
0.16535353660583496,
|
|
0.16847901046276093,
|
|
0.1662793755531311,
|
|
0.16693073511123657,
|
|
0.16613470017910004,
|
|
0.1668226420879364
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7890782356262207,
|
|
"mean_weights": [
|
|
0.17188119888305664,
|
|
0.18350960314273834,
|
|
0.17409591376781464,
|
|
0.16590778529644012,
|
|
0.15644267201423645,
|
|
0.148162841796875
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791684627532959,
|
|
"mean_weights": [
|
|
0.16539111733436584,
|
|
0.16517111659049988,
|
|
0.16458021104335785,
|
|
0.16693061590194702,
|
|
0.16811248660087585,
|
|
0.16981446743011475
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9417986869812012,
|
|
"mean_weights": [
|
|
0.14831508696079254,
|
|
0.16164575517177582,
|
|
0.15476199984550476,
|
|
0.14618372917175293,
|
|
0.13434207439422607,
|
|
0.1278611123561859,
|
|
0.12689024209976196
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9456918239593506,
|
|
"mean_weights": [
|
|
0.14120136201381683,
|
|
0.1386352777481079,
|
|
0.14049293100833893,
|
|
0.1421712040901184,
|
|
0.14506074786186218,
|
|
0.1458473801612854,
|
|
0.14659109711647034
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9406665563583374,
|
|
"mean_weights": [
|
|
0.14782890677452087,
|
|
0.1637636125087738,
|
|
0.15569329261779785,
|
|
0.14784488081932068,
|
|
0.13445764780044556,
|
|
0.12887068092823029,
|
|
0.12154098600149155
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9455351829528809,
|
|
"mean_weights": [
|
|
0.14214086532592773,
|
|
0.1382882297039032,
|
|
0.1401882916688919,
|
|
0.1396511197090149,
|
|
0.14402973651885986,
|
|
0.14592906832695007,
|
|
0.14977267384529114
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.0713937282562256,
|
|
"mean_weights": [
|
|
0.1321096122264862,
|
|
0.1488187313079834,
|
|
0.13977529108524323,
|
|
0.13292929530143738,
|
|
0.12093928456306458,
|
|
0.11470195651054382,
|
|
0.10486428439617157,
|
|
0.10586154460906982
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0790562629699707,
|
|
"mean_weights": [
|
|
0.12411445379257202,
|
|
0.1212267130613327,
|
|
0.12224400043487549,
|
|
0.12172792106866837,
|
|
0.12399555742740631,
|
|
0.12670958042144775,
|
|
0.1301731914281845,
|
|
0.12980858981609344
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.068070888519287,
|
|
"mean_weights": [
|
|
0.13267414271831512,
|
|
0.15216735005378723,
|
|
0.1437777280807495,
|
|
0.13539820909500122,
|
|
0.1199566125869751,
|
|
0.11211173236370087,
|
|
0.10534092038869858,
|
|
0.09857328981161118
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079178810119629,
|
|
"mean_weights": [
|
|
0.12415283918380737,
|
|
0.12161671370267868,
|
|
0.12301617860794067,
|
|
0.12307952344417572,
|
|
0.12411122024059296,
|
|
0.12585631012916565,
|
|
0.12885767221450806,
|
|
0.1293095499277115
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1886847019195557,
|
|
"mean_weights": [
|
|
0.11892519891262054,
|
|
0.13333868980407715,
|
|
0.12725916504859924,
|
|
0.12129361927509308,
|
|
0.10894724726676941,
|
|
0.10413059592247009,
|
|
0.09661625325679779,
|
|
0.09465864300727844,
|
|
0.09483058750629425
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1969594955444336,
|
|
"mean_weights": [
|
|
0.1101110577583313,
|
|
0.10734203457832336,
|
|
0.10833269357681274,
|
|
0.10946638882160187,
|
|
0.11091698706150055,
|
|
0.11324633657932281,
|
|
0.11330728977918625,
|
|
0.11495929211378098,
|
|
0.11231791973114014
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.186551570892334,
|
|
"mean_weights": [
|
|
0.11992302536964417,
|
|
0.13721907138824463,
|
|
0.12899263203144073,
|
|
0.12160105258226395,
|
|
0.10788539052009583,
|
|
0.10353681445121765,
|
|
0.09616796672344208,
|
|
0.09295656532049179,
|
|
0.09171747416257858
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02856776863336563,
|
|
0.014094225130975246,
|
|
0.01573058031499386,
|
|
0.015321774408221245,
|
|
0.017058780416846275,
|
|
0.011773223988711834,
|
|
0.013085381127893925,
|
|
0.013459180481731892,
|
|
0.015175317414104939,
|
|
0.011872655712068081,
|
|
0.013221408240497112,
|
|
0.013565259985625744,
|
|
0.015276176854968071,
|
|
0.012822598218917847,
|
|
0.014198821038007736,
|
|
0.014604844152927399,
|
|
0.0162027720361948,
|
|
0.014113145880401134,
|
|
0.015384308062493801,
|
|
0.015927067026495934,
|
|
0.017611440271139145,
|
|
0.01576142944395542,
|
|
0.017285119742155075,
|
|
0.01772329770028591,
|
|
0.019923174753785133,
|
|
0.017852960154414177,
|
|
0.019824320450425148,
|
|
0.01966804265975952,
|
|
0.022481828927993774,
|
|
0.020090021193027496,
|
|
0.02255004271864891,
|
|
0.022322168573737144
|
|
],
|
|
"loss_nats": 3.5284392833709717,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.193685531616211,
|
|
"mean_weights": [
|
|
0.10035791993141174,
|
|
0.09579961001873016,
|
|
0.10334210842847824,
|
|
0.108812615275383,
|
|
0.11718772351741791,
|
|
0.11711248755455017,
|
|
0.12047697603702545,
|
|
0.11920724809169769,
|
|
0.11770330369472504
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 100,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.004518123809248209,
|
|
0.009570118971168995,
|
|
0.010795549489557743,
|
|
0.01747945137321949,
|
|
0.005324283614754677,
|
|
0.011283415369689465,
|
|
0.012478979304432869,
|
|
0.020727796480059624,
|
|
0.00614966033026576,
|
|
0.012305145151913166,
|
|
0.014650273136794567,
|
|
0.023130038753151894,
|
|
0.0087440712377429,
|
|
0.015205477364361286,
|
|
0.019528985023498535,
|
|
0.028237100690603256,
|
|
0.010757530108094215,
|
|
0.017268717288970947,
|
|
0.02412688545882702,
|
|
0.03324708342552185,
|
|
0.014357471838593483,
|
|
0.021605173125863075,
|
|
0.03339937701821327,
|
|
0.04313390702009201,
|
|
0.01905498467385769,
|
|
0.02657386288046837,
|
|
0.04070562124252319,
|
|
0.05039682611823082,
|
|
0.021850280463695526,
|
|
0.03066091425716877,
|
|
0.052838243544101715,
|
|
0.0630461722612381
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.00015504666953347623,
|
|
0.00015570323739666492,
|
|
0.00013642148405779153,
|
|
0.00012308824807405472,
|
|
0.00010802906763274223,
|
|
9.803942521102726e-05,
|
|
8.508376777172089e-05,
|
|
7.592589827254415e-05,
|
|
7.549385918537155e-05,
|
|
6.5492422436364e-05,
|
|
6.254414620343596e-05,
|
|
5.1708771934499964e-05,
|
|
4.589578747982159e-05,
|
|
4.085309410584159e-05,
|
|
3.8330261304508895e-05,
|
|
3.7662506656488404e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.00014256490976549685,
|
|
"first_to_last_ratio": 3.504079260905398,
|
|
"imbalance_abs_log_ratio": 1.2539277929397175,
|
|
"last_quartile_mean": 4.068541238666512e-05,
|
|
"mean": 8.470741545352212e-05,
|
|
"normalized": [
|
|
1.8303789426623265,
|
|
1.8381299507608906,
|
|
1.610502260367562,
|
|
1.4530988510869118,
|
|
1.2753200774023898,
|
|
1.1573889332607517,
|
|
1.0044429677872213,
|
|
0.8963311873705286,
|
|
0.8912308182368528,
|
|
0.7731604380291695,
|
|
0.7383550291149319,
|
|
0.6104397313700582,
|
|
0.5418154624845569,
|
|
0.4822847431610892,
|
|
0.45250183941145294,
|
|
0.44461876749330576
|
|
],
|
|
"population_cv": 0.46442228143345987
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.02604864165186882,
|
|
0.04358876869082451,
|
|
0.022780675441026688,
|
|
0.03670342266559601,
|
|
0.027025330811738968,
|
|
0.05019363388419151,
|
|
0.040244098752737045,
|
|
0.04879867285490036,
|
|
0.029798155650496483,
|
|
0.04518449306488037,
|
|
0.02838377095758915,
|
|
0.051337067037820816,
|
|
0.03289651870727539,
|
|
0.0653979703783989,
|
|
0.04346369579434395,
|
|
0.08895192295312881
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.032280377112329006,
|
|
"first_to_last_ratio": 0.5596699237065876,
|
|
"imbalance_abs_log_ratio": 0.5804080909828722,
|
|
"last_quartile_mean": 0.05767752695828676,
|
|
"mean": 0.04254980245605111,
|
|
"normalized": [
|
|
0.6121918351741814,
|
|
1.0244176511946566,
|
|
0.5353885124274412,
|
|
0.8625991320055194,
|
|
0.6351458585419503,
|
|
1.1796443458470947,
|
|
0.9458116472880084,
|
|
1.1468601506506073,
|
|
0.7003124322674456,
|
|
1.0619201607704427,
|
|
0.6670717446198772,
|
|
1.2065171651700595,
|
|
0.7731297634402319,
|
|
1.5369747120670472,
|
|
1.0214782040230805,
|
|
2.0905366845123567
|
|
],
|
|
"population_cv": 0.38343105399742256
|
|
},
|
|
"bits_per_byte": 3.1146838872489564,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.01352347619831562,
|
|
0.020455393940210342,
|
|
0.016984812915325165,
|
|
0.0142062958329916,
|
|
0.01867547444999218,
|
|
0.011138904839754105,
|
|
0.019149528816342354,
|
|
0.0099713783711195,
|
|
0.023481043055653572,
|
|
0.011000796221196651,
|
|
0.02560650371015072,
|
|
0.010696547105908394,
|
|
0.0353584885597229,
|
|
0.012510747648775578,
|
|
0.023770872503519058,
|
|
0.012255984358489513,
|
|
0.023732667788863182,
|
|
0.014518127776682377,
|
|
0.021041536703705788,
|
|
0.015058746561408043,
|
|
0.021396929398179054,
|
|
0.018767673522233963,
|
|
0.022012433037161827,
|
|
0.023688357323408127,
|
|
0.020202528685331345,
|
|
0.02677428349852562,
|
|
0.015811076387763023,
|
|
0.03653915971517563,
|
|
0.014067054726183414,
|
|
0.0391332171857357,
|
|
0.010850721038877964,
|
|
0.043844256550073624
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0006567095640112879,
|
|
0.00045868421249221203,
|
|
0.00039439577454117915,
|
|
0.0003702348543320087,
|
|
0.0003691077163774564,
|
|
0.00041151853265381964,
|
|
0.0004306774133910072,
|
|
0.00035317417015185023,
|
|
0.00038561746855580054,
|
|
0.0003833308661148803,
|
|
0.00032325470855348767,
|
|
0.0002967607909467712,
|
|
0.00021968458301266372,
|
|
0.00017220390788738176,
|
|
0.00016255203232502606,
|
|
0.0001597780110258732
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00047000610134417196,
|
|
"first_to_last_ratio": 2.6322817390176136,
|
|
"imbalance_abs_log_ratio": 0.967851051434004,
|
|
"last_quartile_mean": 0.0001785546335627362,
|
|
"mean": 0.00034673028789829414,
|
|
"normalized": [
|
|
1.8940069181493584,
|
|
1.3228847565424027,
|
|
1.137471366957324,
|
|
1.0677891930819992,
|
|
1.0645384301867686,
|
|
1.1868548754371575,
|
|
1.242110736854166,
|
|
1.0185847111673332,
|
|
1.1121539767789572,
|
|
1.105559218487778,
|
|
0.9322944081778846,
|
|
0.8558836689623711,
|
|
0.6335892498583898,
|
|
0.49665089522809175,
|
|
0.46881405518489694,
|
|
0.46081353894511995
|
|
],
|
|
"population_cv": 0.3565995806724142
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6422993540763855,
|
|
"mean_weights": [
|
|
0.370745986700058,
|
|
0.6292540431022644
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6814000606536865,
|
|
"mean_weights": [
|
|
0.4577118158340454,
|
|
0.5422881841659546
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6871446371078491,
|
|
"mean_weights": [
|
|
0.49052417278289795,
|
|
0.5094758868217468
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6793373227119446,
|
|
"mean_weights": [
|
|
0.4676857590675354,
|
|
0.5323142409324646
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0915337800979614,
|
|
"mean_weights": [
|
|
0.29751554131507874,
|
|
0.351987361907959,
|
|
0.3504970669746399
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0892891883850098,
|
|
"mean_weights": [
|
|
0.3054355978965759,
|
|
0.34222692251205444,
|
|
0.35233747959136963
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.093106746673584,
|
|
"mean_weights": [
|
|
0.30644628405570984,
|
|
0.34674984216690063,
|
|
0.3468038737773895
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0885257720947266,
|
|
"mean_weights": [
|
|
0.3052448034286499,
|
|
0.3325393795967102,
|
|
0.3622158169746399
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.381313443183899,
|
|
"mean_weights": [
|
|
0.2415536344051361,
|
|
0.24482353031635284,
|
|
0.25347256660461426,
|
|
0.2601502537727356
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3802430629730225,
|
|
"mean_weights": [
|
|
0.24572397768497467,
|
|
0.2491092085838318,
|
|
0.25083041191101074,
|
|
0.254336416721344
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3813848495483398,
|
|
"mean_weights": [
|
|
0.2464662790298462,
|
|
0.23372246325016022,
|
|
0.2595708668231964,
|
|
0.260240375995636
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3772881031036377,
|
|
"mean_weights": [
|
|
0.24203142523765564,
|
|
0.2443695217370987,
|
|
0.2359469085931778,
|
|
0.27765214443206787
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6055738925933838,
|
|
"mean_weights": [
|
|
0.20364060997962952,
|
|
0.18423235416412354,
|
|
0.18416231870651245,
|
|
0.20781230926513672,
|
|
0.22015240788459778
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.602186679840088,
|
|
"mean_weights": [
|
|
0.19954648613929749,
|
|
0.1888481080532074,
|
|
0.18637022376060486,
|
|
0.21049518883228302,
|
|
0.21474000811576843
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6062731742858887,
|
|
"mean_weights": [
|
|
0.19444647431373596,
|
|
0.1968603879213333,
|
|
0.18490161001682281,
|
|
0.20200669765472412,
|
|
0.2217848002910614
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6028468608856201,
|
|
"mean_weights": [
|
|
0.20196445286273956,
|
|
0.19006873667240143,
|
|
0.1860634982585907,
|
|
0.1993759572505951,
|
|
0.22252735495567322
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7881723642349243,
|
|
"mean_weights": [
|
|
0.15337401628494263,
|
|
0.17425060272216797,
|
|
0.1601826399564743,
|
|
0.1535373479127884,
|
|
0.17563557624816895,
|
|
0.18301980197429657
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7856398820877075,
|
|
"mean_weights": [
|
|
0.1621149629354477,
|
|
0.15582670271396637,
|
|
0.15492847561836243,
|
|
0.1618887484073639,
|
|
0.1784553974866867,
|
|
0.1867857277393341
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7870616912841797,
|
|
"mean_weights": [
|
|
0.1564463973045349,
|
|
0.18363508582115173,
|
|
0.15811879932880402,
|
|
0.15924012660980225,
|
|
0.15980733931064606,
|
|
0.18275225162506104
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7877882719039917,
|
|
"mean_weights": [
|
|
0.17034202814102173,
|
|
0.15881556272506714,
|
|
0.1534808874130249,
|
|
0.16709929704666138,
|
|
0.16602207720279694,
|
|
0.1842401623725891
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9344044923782349,
|
|
"mean_weights": [
|
|
0.12371677905321121,
|
|
0.1564711332321167,
|
|
0.13837161660194397,
|
|
0.1230623871088028,
|
|
0.12980599701404572,
|
|
0.152274027466774,
|
|
0.17629806697368622
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9417431354522705,
|
|
"mean_weights": [
|
|
0.14328667521476746,
|
|
0.1450044810771942,
|
|
0.14678549766540527,
|
|
0.13414078950881958,
|
|
0.1328907310962677,
|
|
0.1347782164812088,
|
|
0.16311360895633698
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9288355112075806,
|
|
"mean_weights": [
|
|
0.12207633256912231,
|
|
0.16755375266075134,
|
|
0.13929706811904907,
|
|
0.12109649926424026,
|
|
0.12036697566509247,
|
|
0.15116539597511292,
|
|
0.1784439980983734
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.938415288925171,
|
|
"mean_weights": [
|
|
0.1377446949481964,
|
|
0.1613207459449768,
|
|
0.156096950173378,
|
|
0.13170479238033295,
|
|
0.12767548859119415,
|
|
0.13129547238349915,
|
|
0.15416184067726135
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.0616660118103027,
|
|
"mean_weights": [
|
|
0.10379506647586823,
|
|
0.1403447985649109,
|
|
0.12122530490159988,
|
|
0.10852085053920746,
|
|
0.10464349389076233,
|
|
0.1186104342341423,
|
|
0.13291975855827332,
|
|
0.1699402928352356
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0691847801208496,
|
|
"mean_weights": [
|
|
0.11521603912115097,
|
|
0.14423911273479462,
|
|
0.13052678108215332,
|
|
0.12005230039358139,
|
|
0.10937757790088654,
|
|
0.12122473865747452,
|
|
0.12272199243307114,
|
|
0.1366414725780487
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.054232597351074,
|
|
"mean_weights": [
|
|
0.10057584941387177,
|
|
0.16718831658363342,
|
|
0.1352493166923523,
|
|
0.11277885735034943,
|
|
0.09778623282909393,
|
|
0.11311028897762299,
|
|
0.13487182557582855,
|
|
0.1384393274784088
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.060153007507324,
|
|
"mean_weights": [
|
|
0.11611463129520416,
|
|
0.15179042518138885,
|
|
0.14188171923160553,
|
|
0.1159345731139183,
|
|
0.10906632244586945,
|
|
0.12251802533864975,
|
|
0.12206383049488068,
|
|
0.12063045799732208
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1772594451904297,
|
|
"mean_weights": [
|
|
0.09167024493217468,
|
|
0.14425000548362732,
|
|
0.1316833794116974,
|
|
0.09630900621414185,
|
|
0.09528810530900955,
|
|
0.1022176742553711,
|
|
0.12638677656650543,
|
|
0.10320886224508286,
|
|
0.10898594558238983
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.191457986831665,
|
|
"mean_weights": [
|
|
0.10587693750858307,
|
|
0.12391996383666992,
|
|
0.11278913170099258,
|
|
0.11095453053712845,
|
|
0.10569566488265991,
|
|
0.11266011744737625,
|
|
0.11442379653453827,
|
|
0.11124123632907867,
|
|
0.10243862122297287
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1550254821777344,
|
|
"mean_weights": [
|
|
0.08980270475149155,
|
|
0.17290523648262024,
|
|
0.13377593457698822,
|
|
0.10608047246932983,
|
|
0.09299036115407944,
|
|
0.1071130782365799,
|
|
0.12428411841392517,
|
|
0.09728670865297318,
|
|
0.07576138526201248
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.04350041598081589,
|
|
0.017946822568774223,
|
|
0.025110680609941483,
|
|
0.028106264770030975,
|
|
0.031104078516364098,
|
|
0.02212802693247795,
|
|
0.02379882149398327,
|
|
0.02537565305829048,
|
|
0.026546044275164604,
|
|
0.02055767923593521,
|
|
0.022140897810459137,
|
|
0.02340756356716156,
|
|
0.02546003647148609,
|
|
0.020307816565036774,
|
|
0.02165406011044979,
|
|
0.021843569353222847,
|
|
0.023044398054480553,
|
|
0.018649185076355934,
|
|
0.019831404089927673,
|
|
0.019260063767433167,
|
|
0.02048473060131073,
|
|
0.01540753897279501,
|
|
0.01695212349295616,
|
|
0.01553493645042181,
|
|
0.017758244648575783,
|
|
0.013987960293889046,
|
|
0.016796667128801346,
|
|
0.01665738970041275,
|
|
0.01972099393606186,
|
|
0.01747863180935383,
|
|
0.020752927288413048,
|
|
0.019990377128124237
|
|
],
|
|
"loss_nats": 2.1589343547821045,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1796395778656006,
|
|
"mean_weights": [
|
|
0.12411575019359589,
|
|
0.09159804880619049,
|
|
0.10466736555099487,
|
|
0.10315616428852081,
|
|
0.09333926439285278,
|
|
0.09817550331354141,
|
|
0.11351706087589264,
|
|
0.1361408531665802,
|
|
0.1352899968624115
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 500,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.01352347619831562,
|
|
0.02604864165186882,
|
|
0.03525395691394806,
|
|
0.04358876869082451,
|
|
0.01867547444999218,
|
|
0.022780675441026688,
|
|
0.03279132395982742,
|
|
0.03670342266559601,
|
|
0.023481043055653572,
|
|
0.027025330811738968,
|
|
0.04530210420489311,
|
|
0.05019363388419151,
|
|
0.0353584885597229,
|
|
0.040244098752737045,
|
|
0.04290904477238655,
|
|
0.04879867285490036,
|
|
0.023732667788863182,
|
|
0.029798155650496483,
|
|
0.03790274262428284,
|
|
0.04518449306488037,
|
|
0.021396929398179054,
|
|
0.02838377095758915,
|
|
0.03953101858496666,
|
|
0.051337067037820816,
|
|
0.020202528685331345,
|
|
0.03289651870727539,
|
|
0.039050742983818054,
|
|
0.0653979703783989,
|
|
0.014067054726183414,
|
|
0.04346369579434395,
|
|
0.048713527619838715,
|
|
0.08895192295312881
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
7.91841812315397e-05,
|
|
8.04664014140144e-05,
|
|
6.887180643389001e-05,
|
|
6.264931289479136e-05,
|
|
6.203571683727205e-05,
|
|
6.256609776755795e-05,
|
|
7.726731564616784e-05,
|
|
7.684857700951397e-05,
|
|
0.0001297412090934813,
|
|
0.00010362039029132575,
|
|
0.0001698000414762646,
|
|
0.00012187293032184243,
|
|
0.00013834609126206487,
|
|
8.384700049646199e-05,
|
|
4.917434125673026e-05,
|
|
4.311763041187078e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 7.279292549355887e-05,
|
|
"first_to_last_ratio": 0.9258681439467011,
|
|
"imbalance_abs_log_ratio": 0.07702344762093526,
|
|
"last_quartile_mean": 7.862126585678197e-05,
|
|
"mean": 8.808806524029933e-05,
|
|
"normalized": [
|
|
0.8989206541831708,
|
|
0.91347677116652,
|
|
0.7818517326497246,
|
|
0.7112122706281212,
|
|
0.7042465590320558,
|
|
0.7102675895637067,
|
|
0.8771598676323168,
|
|
0.8724062311946045,
|
|
1.4728579716168646,
|
|
1.1763272358026606,
|
|
1.9276168799151117,
|
|
1.3835351019389504,
|
|
1.570542966117651,
|
|
0.9518542638861694,
|
|
0.5582406779236824,
|
|
0.4894832267486894
|
|
],
|
|
"population_cv": 0.3870844002085187
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.10205306112766266,
|
|
0.1637924462556839,
|
|
0.1347782015800476,
|
|
0.18643726408481598,
|
|
0.08823998272418976,
|
|
0.14544272422790527,
|
|
0.051831819117069244,
|
|
0.08038940280675888,
|
|
0.045988935977220535,
|
|
0.08743251860141754,
|
|
0.060310039669275284,
|
|
0.09679152816534042,
|
|
0.05415637046098709,
|
|
0.0906868502497673,
|
|
0.09274918586015701,
|
|
0.19966518878936768
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.14676524326205254,
|
|
"first_to_last_ratio": 1.3425975426784762,
|
|
"imbalance_abs_log_ratio": 0.2946062022500781,
|
|
"last_quartile_mean": 0.10931439884006977,
|
|
"mean": 0.10504659498110414,
|
|
"normalized": [
|
|
0.9715027997434856,
|
|
1.5592361302634037,
|
|
1.2830325590685887,
|
|
1.7748054005782146,
|
|
0.840008024439654,
|
|
1.3845543899263715,
|
|
0.49341741278137374,
|
|
0.7652737608603057,
|
|
0.4377955895238019,
|
|
0.8323213010107083,
|
|
0.5741265547933648,
|
|
0.9214151889716308,
|
|
0.5155461771105363,
|
|
0.8633011880687815,
|
|
0.8829337674090323,
|
|
1.9007297554507465
|
|
],
|
|
"population_cv": 0.43941183630186603
|
|
},
|
|
"bits_per_byte": 2.0898630043632984,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.04098131135106087,
|
|
0.09075035154819489,
|
|
0.05343177169561386,
|
|
0.04652838781476021,
|
|
0.13196392357349396,
|
|
0.018610643222928047,
|
|
0.06760989129543304,
|
|
0.01781350187957287,
|
|
0.0839577317237854,
|
|
0.0164200272411108,
|
|
0.07349737733602524,
|
|
0.016965076327323914,
|
|
0.04464862495660782,
|
|
0.016964683309197426,
|
|
0.0346895232796669,
|
|
0.021388454362750053,
|
|
0.03376077860593796,
|
|
0.026654686778783798,
|
|
0.04212235286831856,
|
|
0.042033009231090546,
|
|
0.0419253334403038,
|
|
0.040410157293081284,
|
|
0.028147248551249504,
|
|
0.05647630989551544,
|
|
0.025301706045866013,
|
|
0.04489820450544357,
|
|
0.015376480296254158,
|
|
0.058749374002218246,
|
|
0.025178087875247,
|
|
0.0875716358423233,
|
|
0.01239082682877779,
|
|
0.1223064437508583
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00040335839120789034,
|
|
0.00023897821552144955,
|
|
0.00022638555923562337,
|
|
0.00017534070905568056,
|
|
0.00017358160175308344,
|
|
0.00015895432307312814,
|
|
0.00019058997613466408,
|
|
0.00021161798453125184,
|
|
0.00037807554491806804,
|
|
0.0004740713531015684,
|
|
0.0009894587672613622,
|
|
0.0007545475239678793,
|
|
0.0006624146206615605,
|
|
0.0004575991184432586,
|
|
0.0002840295434450018,
|
|
0.00025831777406547436
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00026101571875516093,
|
|
"first_to_last_ratio": 0.6280602344874717,
|
|
"imbalance_abs_log_ratio": 0.46511920232449994,
|
|
"last_quartile_mean": 0.00041559026415382384,
|
|
"mean": 0.00037733256289855906,
|
|
"normalized": [
|
|
1.0689731840512475,
|
|
0.633335786568984,
|
|
0.5999629544203535,
|
|
0.464684806709403,
|
|
0.460022852042453,
|
|
0.4212579000658921,
|
|
0.5050981412009801,
|
|
0.5608261924326488,
|
|
1.001969037640965,
|
|
1.2563754091613246,
|
|
2.6222459033501577,
|
|
1.9996883337384523,
|
|
1.7555193635372572,
|
|
1.2127209878949095,
|
|
0.7527300089426935,
|
|
0.6845891382422771
|
|
],
|
|
"population_cv": 0.6186471183066214
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.4914427399635315,
|
|
"mean_weights": [
|
|
0.7962706685066223,
|
|
0.2037293016910553
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6456852555274963,
|
|
"mean_weights": [
|
|
0.6051734089851379,
|
|
0.39482662081718445
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5276303291320801,
|
|
"mean_weights": [
|
|
0.7698293924331665,
|
|
0.2301705777645111
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.667390763759613,
|
|
"mean_weights": [
|
|
0.5093090534210205,
|
|
0.4906909465789795
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9880077838897705,
|
|
"mean_weights": [
|
|
0.5180364847183228,
|
|
0.2465059459209442,
|
|
0.23545756936073303
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.08414626121521,
|
|
"mean_weights": [
|
|
0.2851155400276184,
|
|
0.3203214704990387,
|
|
0.3945629596710205
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.995106041431427,
|
|
"mean_weights": [
|
|
0.5032885074615479,
|
|
0.23357318341732025,
|
|
0.2631382942199707
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0889081954956055,
|
|
"mean_weights": [
|
|
0.2929039001464844,
|
|
0.3457888960838318,
|
|
0.36130720376968384
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3091163635253906,
|
|
"mean_weights": [
|
|
0.3480933904647827,
|
|
0.15371239185333252,
|
|
0.20259280502796173,
|
|
0.29560136795043945
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3786191940307617,
|
|
"mean_weights": [
|
|
0.22527548670768738,
|
|
0.26118987798690796,
|
|
0.22951865196228027,
|
|
0.284015953540802
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.308091163635254,
|
|
"mean_weights": [
|
|
0.28280675411224365,
|
|
0.12650004029273987,
|
|
0.24999231100082397,
|
|
0.3407008647918701
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3780848979949951,
|
|
"mean_weights": [
|
|
0.23603014647960663,
|
|
0.2631712555885315,
|
|
0.21569286286830902,
|
|
0.28510570526123047
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5637545585632324,
|
|
"mean_weights": [
|
|
0.21651385724544525,
|
|
0.1117275133728981,
|
|
0.18637539446353912,
|
|
0.22181779146194458,
|
|
0.2635654807090759
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.585724115371704,
|
|
"mean_weights": [
|
|
0.18968582153320312,
|
|
0.1962759643793106,
|
|
0.1558789312839508,
|
|
0.1798120141029358,
|
|
0.2783472537994385
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5276048183441162,
|
|
"mean_weights": [
|
|
0.2346682846546173,
|
|
0.15222865343093872,
|
|
0.12689977884292603,
|
|
0.14891111850738525,
|
|
0.3372921645641327
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.59938383102417,
|
|
"mean_weights": [
|
|
0.20912373065948486,
|
|
0.21754522621631622,
|
|
0.17016828060150146,
|
|
0.17554700374603271,
|
|
0.22761575877666473
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.762988805770874,
|
|
"mean_weights": [
|
|
0.15954776108264923,
|
|
0.12860070168972015,
|
|
0.13993129134178162,
|
|
0.14435188472270966,
|
|
0.1894117146730423,
|
|
0.23815666139125824
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.730720043182373,
|
|
"mean_weights": [
|
|
0.17182672023773193,
|
|
0.16310763359069824,
|
|
0.12390270084142685,
|
|
0.11688461899757385,
|
|
0.1326894462108612,
|
|
0.2915888726711273
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.4424301385879517,
|
|
"mean_weights": [
|
|
0.1671508103609085,
|
|
0.1365666687488556,
|
|
0.058271970599889755,
|
|
0.05150005966424942,
|
|
0.10222888737916946,
|
|
0.48428159952163696
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7727878093719482,
|
|
"mean_weights": [
|
|
0.1808200478553772,
|
|
0.17958280444145203,
|
|
0.1469326913356781,
|
|
0.13797307014465332,
|
|
0.14092227816581726,
|
|
0.2137690931558609
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.5771737098693848,
|
|
"mean_weights": [
|
|
0.09523316472768784,
|
|
0.08390435576438904,
|
|
0.04673727601766586,
|
|
0.04574791342020035,
|
|
0.07810044288635254,
|
|
0.19550111889839172,
|
|
0.45477572083473206
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.866133689880371,
|
|
"mean_weights": [
|
|
0.13646650314331055,
|
|
0.12685352563858032,
|
|
0.11049237847328186,
|
|
0.10180146992206573,
|
|
0.1026906818151474,
|
|
0.1334708034992218,
|
|
0.28822463750839233
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.5755802392959595,
|
|
"mean_weights": [
|
|
0.09840903431177139,
|
|
0.08739377558231354,
|
|
0.04908968508243561,
|
|
0.048939045518636703,
|
|
0.07940414547920227,
|
|
0.16869762539863586,
|
|
0.46806666254997253
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8938008546829224,
|
|
"mean_weights": [
|
|
0.14332836866378784,
|
|
0.14485859870910645,
|
|
0.1094474121928215,
|
|
0.10118386149406433,
|
|
0.10821370780467987,
|
|
0.14364898204803467,
|
|
0.24931909143924713
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.7736961841583252,
|
|
"mean_weights": [
|
|
0.08116129040718079,
|
|
0.07111318409442902,
|
|
0.04280973970890045,
|
|
0.041376810520887375,
|
|
0.06107718497514725,
|
|
0.1160595640540123,
|
|
0.31332647800445557,
|
|
0.2730756998062134
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9035927057266235,
|
|
"mean_weights": [
|
|
0.10087604820728302,
|
|
0.09542917460203171,
|
|
0.06220542639493942,
|
|
0.057604290544986725,
|
|
0.0788167417049408,
|
|
0.11659102141857147,
|
|
0.18421337008476257,
|
|
0.3042638897895813
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.7131891250610352,
|
|
"mean_weights": [
|
|
0.0740746557712555,
|
|
0.07079474627971649,
|
|
0.04342404007911682,
|
|
0.04123558849096298,
|
|
0.055328819900751114,
|
|
0.09829520434141159,
|
|
0.21916092932224274,
|
|
0.39768603444099426
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9703106880187988,
|
|
"mean_weights": [
|
|
0.11310406029224396,
|
|
0.11765976250171661,
|
|
0.07340675592422485,
|
|
0.0660770833492279,
|
|
0.08407266438007355,
|
|
0.12691649794578552,
|
|
0.20392963290214539,
|
|
0.2148335576057434
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9610494375228882,
|
|
"mean_weights": [
|
|
0.07792562991380692,
|
|
0.0730937123298645,
|
|
0.040804408490657806,
|
|
0.03727761656045914,
|
|
0.05315954610705376,
|
|
0.0937754213809967,
|
|
0.19250980019569397,
|
|
0.26475197076797485,
|
|
0.16670189797878265
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.9782097339630127,
|
|
"mean_weights": [
|
|
0.08397573232650757,
|
|
0.08872128278017044,
|
|
0.0407399907708168,
|
|
0.036753468215465546,
|
|
0.058452628552913666,
|
|
0.11618129909038544,
|
|
0.22768989205360413,
|
|
0.21488600969314575,
|
|
0.13259968161582947
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.0315680503845215,
|
|
"mean_weights": [
|
|
0.08375388383865356,
|
|
0.10381080955266953,
|
|
0.0493716336786747,
|
|
0.043391548097133636,
|
|
0.06576120853424072,
|
|
0.10571172833442688,
|
|
0.18509769439697266,
|
|
0.2181759774684906,
|
|
0.14492549002170563
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.05570797994732857,
|
|
0.04578034579753876,
|
|
0.058472853153944016,
|
|
0.058589257299900055,
|
|
0.09159071743488312,
|
|
0.0729803666472435,
|
|
0.0950615182518959,
|
|
0.08442825824022293,
|
|
0.11022263020277023,
|
|
0.07856440544128418,
|
|
0.0938955619931221,
|
|
0.10307082533836365,
|
|
0.10586344450712204,
|
|
0.08232943713665009,
|
|
0.08149705082178116,
|
|
0.07412237673997879,
|
|
0.0869104191660881,
|
|
0.0654435008764267,
|
|
0.0601484477519989,
|
|
0.04590349644422531,
|
|
0.06966738402843475,
|
|
0.028740333393216133,
|
|
0.04708963632583618,
|
|
0.03821079805493355,
|
|
0.0513334795832634,
|
|
0.034749191254377365,
|
|
0.034582361578941345,
|
|
0.034875962883234024,
|
|
0.041408468037843704,
|
|
0.03493491932749748,
|
|
0.0412786528468132,
|
|
0.04175883159041405
|
|
],
|
|
"loss_nats": 1.448582649230957,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.893564224243164,
|
|
"mean_weights": [
|
|
0.09584413468837738,
|
|
0.060246460139751434,
|
|
0.05342919006943703,
|
|
0.050279051065444946,
|
|
0.05668472498655319,
|
|
0.06780324131250381,
|
|
0.0837026908993721,
|
|
0.2589869201183319,
|
|
0.2730236053466797
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 2000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.04098131135106087,
|
|
0.10205306112766266,
|
|
0.13058459758758545,
|
|
0.1637924462556839,
|
|
0.13196392357349396,
|
|
0.1347782015800476,
|
|
0.18347561359405518,
|
|
0.18643726408481598,
|
|
0.0839577317237854,
|
|
0.08823998272418976,
|
|
0.14121752977371216,
|
|
0.14544272422790527,
|
|
0.04464862495660782,
|
|
0.051831819117069244,
|
|
0.07186394184827805,
|
|
0.08038940280675888,
|
|
0.03376077860593796,
|
|
0.045988935977220535,
|
|
0.06799308955669403,
|
|
0.08743251860141754,
|
|
0.0419253334403038,
|
|
0.060310039669275284,
|
|
0.07043248414993286,
|
|
0.09679152816534042,
|
|
0.025301706045866013,
|
|
0.05415637046098709,
|
|
0.05762659013271332,
|
|
0.0906868502497673,
|
|
0.025178087875247,
|
|
0.09274918586015701,
|
|
0.0948956087231636,
|
|
0.19966518878936768
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
5.6704131566220894e-05,
|
|
5.7463053963147104e-05,
|
|
5.5973152484511957e-05,
|
|
4.9604332161834463e-05,
|
|
4.9622754886513576e-05,
|
|
5.175438491278328e-05,
|
|
6.153700815048069e-05,
|
|
5.993711238261312e-05,
|
|
0.00010211551852989942,
|
|
8.014418563107029e-05,
|
|
0.00014815112808719277,
|
|
0.00010670819756342098,
|
|
0.00014386174734681845,
|
|
8.734197035664693e-05,
|
|
5.547845285036601e-05,
|
|
4.4348533265292645e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 5.4936167543928605e-05,
|
|
"first_to_last_ratio": 0.6638196023526066,
|
|
"imbalance_abs_log_ratio": 0.4097448496221764,
|
|
"last_quartile_mean": 8.275767595478101e-05,
|
|
"mean": 7.567160400867579e-05,
|
|
"normalized": [
|
|
0.7493449135784109,
|
|
0.7593740705768433,
|
|
0.7396850273993744,
|
|
0.6555210876215509,
|
|
0.6557645438680578,
|
|
0.6839340277080636,
|
|
0.8132113618660104,
|
|
0.792068744515331,
|
|
1.3494562440911366,
|
|
1.0591051515424692,
|
|
1.9578166741411636,
|
|
1.4101484825296797,
|
|
1.9011325216566657,
|
|
1.1542238531990563,
|
|
0.7331475733487212,
|
|
0.5860657223574653
|
|
],
|
|
"population_cv": 0.42524285954419916
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.16565655171871185,
|
|
0.24343863129615784,
|
|
0.21182885766029358,
|
|
0.28418517112731934,
|
|
0.12311498820781708,
|
|
0.18273214995861053,
|
|
0.06521740555763245,
|
|
0.10707039386034012,
|
|
0.06429450213909149,
|
|
0.1264011114835739,
|
|
0.0799982100725174,
|
|
0.12911832332611084,
|
|
0.06644809991121292,
|
|
0.1084718257188797,
|
|
0.10412788391113281,
|
|
0.23786598443984985
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.22627730295062065,
|
|
"first_to_last_ratio": 1.750986764798193,
|
|
"imbalance_abs_log_ratio": 0.5601794946219173,
|
|
"last_quartile_mean": 0.12922844849526882,
|
|
"mean": 0.14374813064932823,
|
|
"normalized": [
|
|
1.152408389385104,
|
|
1.6935081534383452,
|
|
1.4736112163924233,
|
|
1.976966029705009,
|
|
0.8564632294812553,
|
|
1.2711967044940802,
|
|
0.4536921994257406,
|
|
0.7448472086328344,
|
|
0.4472719182410596,
|
|
0.8793235147657527,
|
|
0.5565165245012613,
|
|
0.8982261038308273,
|
|
0.4622536627854468,
|
|
0.7545964266032466,
|
|
0.7243773080093228,
|
|
1.654741410308291
|
|
],
|
|
"population_cv": 0.4691623243633533
|
|
},
|
|
"bits_per_byte": 1.9440733112268958,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.06033274158835411,
|
|
0.15077131986618042,
|
|
0.0633837878704071,
|
|
0.06536085903644562,
|
|
0.20718541741371155,
|
|
0.025066526606678963,
|
|
0.08910981565713882,
|
|
0.0229792483150959,
|
|
0.11816111207008362,
|
|
0.01873566210269928,
|
|
0.06798963993787766,
|
|
0.025694282725453377,
|
|
0.05406087636947632,
|
|
0.025319905951619148,
|
|
0.05011621490120888,
|
|
0.03549748659133911,
|
|
0.04759044572710991,
|
|
0.038602542132139206,
|
|
0.05848398804664612,
|
|
0.0661846399307251,
|
|
0.052182719111442566,
|
|
0.057186078280210495,
|
|
0.03716399520635605,
|
|
0.08089981228113174,
|
|
0.03087303601205349,
|
|
0.055707309395074844,
|
|
0.02278325706720352,
|
|
0.07142244279384613,
|
|
0.0293489471077919,
|
|
0.09790552407503128,
|
|
0.01774938404560089,
|
|
0.15747468173503876
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00039487297360913916,
|
|
0.00021664788561845028,
|
|
0.0001999236891396226,
|
|
0.000155700993556381,
|
|
0.00014909408964546676,
|
|
0.0001300025003667294,
|
|
0.0001780845905877945,
|
|
0.00021758503566474802,
|
|
0.0003627569164977402,
|
|
0.0004938025952783663,
|
|
0.0010348495424230683,
|
|
0.0008427402552292494,
|
|
0.0008322795274123101,
|
|
0.0005665088497243569,
|
|
0.00036612876328907985,
|
|
0.0003194282437589591
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00024178638548089825,
|
|
"first_to_last_ratio": 0.464004453994026,
|
|
"imbalance_abs_log_ratio": 0.7678611276768971,
|
|
"last_quartile_mean": 0.0005210863460461766,
|
|
"mean": 0.00040377540323759136,
|
|
"normalized": [
|
|
0.9779520259107666,
|
|
0.5365554312652605,
|
|
0.49513587884892185,
|
|
0.3856128736617538,
|
|
0.36925005448570164,
|
|
0.3219673593892314,
|
|
0.44104863535485134,
|
|
0.5388763998997622,
|
|
0.8984126164918598,
|
|
1.2229635369537375,
|
|
2.5629335866556917,
|
|
2.087151046031797,
|
|
2.061243752687373,
|
|
1.4030296179061932,
|
|
0.9067634143965939,
|
|
0.7911037700605048
|
|
],
|
|
"population_cv": 0.6734220009206653
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.2750788629055023,
|
|
"mean_weights": [
|
|
0.9180313944816589,
|
|
0.08196862041950226
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5785284042358398,
|
|
"mean_weights": [
|
|
0.7040213346481323,
|
|
0.2959786057472229
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.4888499975204468,
|
|
"mean_weights": [
|
|
0.7967724800109863,
|
|
0.20322750508785248
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.651995062828064,
|
|
"mean_weights": [
|
|
0.5629274845123291,
|
|
0.4370725154876709
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9925327301025391,
|
|
"mean_weights": [
|
|
0.4965394139289856,
|
|
0.22532734274864197,
|
|
0.27813321352005005
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0594394207000732,
|
|
"mean_weights": [
|
|
0.2700359523296356,
|
|
0.27400293946266174,
|
|
0.455961138010025
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.9841134548187256,
|
|
"mean_weights": [
|
|
0.49647465348243713,
|
|
0.21238623559474945,
|
|
0.2911390960216522
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0744149684906006,
|
|
"mean_weights": [
|
|
0.26788532733917236,
|
|
0.3177185654640198,
|
|
0.41439610719680786
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.307446002960205,
|
|
"mean_weights": [
|
|
0.33086591958999634,
|
|
0.13986191153526306,
|
|
0.23523268103599548,
|
|
0.2940394878387451
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.376603603363037,
|
|
"mean_weights": [
|
|
0.21549805998802185,
|
|
0.245200514793396,
|
|
0.24268285930156708,
|
|
0.29661858081817627
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.270529866218567,
|
|
"mean_weights": [
|
|
0.2375539094209671,
|
|
0.10673755407333374,
|
|
0.29245367646217346,
|
|
0.3632548451423645
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3762142658233643,
|
|
"mean_weights": [
|
|
0.22679615020751953,
|
|
0.24722415208816528,
|
|
0.2286011129617691,
|
|
0.2973785698413849
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5536365509033203,
|
|
"mean_weights": [
|
|
0.20543108880519867,
|
|
0.10043682903051376,
|
|
0.22657056152820587,
|
|
0.26188796758651733,
|
|
0.20567354559898376
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5861575603485107,
|
|
"mean_weights": [
|
|
0.1991366446018219,
|
|
0.19079142808914185,
|
|
0.16120794415473938,
|
|
0.1751076579093933,
|
|
0.27375632524490356
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.4829623699188232,
|
|
"mean_weights": [
|
|
0.2499978393316269,
|
|
0.13764306902885437,
|
|
0.10969669371843338,
|
|
0.13450437784194946,
|
|
0.3681580424308777
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5994092226028442,
|
|
"mean_weights": [
|
|
0.2161637395620346,
|
|
0.20309701561927795,
|
|
0.17752361297607422,
|
|
0.17640036344528198,
|
|
0.22681528329849243
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.757448434829712,
|
|
"mean_weights": [
|
|
0.1632193624973297,
|
|
0.11841119825839996,
|
|
0.1406504064798355,
|
|
0.1471654176712036,
|
|
0.19204986095428467,
|
|
0.23850372433662415
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.719214677810669,
|
|
"mean_weights": [
|
|
0.17631781101226807,
|
|
0.15250492095947266,
|
|
0.11967972666025162,
|
|
0.11400137841701508,
|
|
0.13607722520828247,
|
|
0.3014189600944519
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.3921294212341309,
|
|
"mean_weights": [
|
|
0.18369951844215393,
|
|
0.1281489133834839,
|
|
0.04485193267464638,
|
|
0.042511094361543655,
|
|
0.10842494666576385,
|
|
0.4923636019229889
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.769790768623352,
|
|
"mean_weights": [
|
|
0.1860106885433197,
|
|
0.17010365426540375,
|
|
0.1509144902229309,
|
|
0.1396978795528412,
|
|
0.13775232434272766,
|
|
0.215520977973938
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.5072013139724731,
|
|
"mean_weights": [
|
|
0.09493429213762283,
|
|
0.07214362919330597,
|
|
0.03675283491611481,
|
|
0.03880048543214798,
|
|
0.07758605480194092,
|
|
0.19188526272773743,
|
|
0.48789745569229126
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8300648927688599,
|
|
"mean_weights": [
|
|
0.13595148921012878,
|
|
0.11550325155258179,
|
|
0.1085444986820221,
|
|
0.09914489090442657,
|
|
0.09707978367805481,
|
|
0.12102536857128143,
|
|
0.3227507174015045
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.5086650848388672,
|
|
"mean_weights": [
|
|
0.10266698896884918,
|
|
0.07744058966636658,
|
|
0.03928206115961075,
|
|
0.0407833606004715,
|
|
0.07880876958370209,
|
|
0.16221223771572113,
|
|
0.4988059997558594
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8771061897277832,
|
|
"mean_weights": [
|
|
0.1470566689968109,
|
|
0.13075211644172668,
|
|
0.10456536710262299,
|
|
0.09934695065021515,
|
|
0.1088101863861084,
|
|
0.1398048996925354,
|
|
0.26966381072998047
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.6780993938446045,
|
|
"mean_weights": [
|
|
0.08037982881069183,
|
|
0.05826176330447197,
|
|
0.03183364123106003,
|
|
0.03276165574789047,
|
|
0.053941305726766586,
|
|
0.10113498568534851,
|
|
0.30644336342811584,
|
|
0.33524346351623535
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8294007778167725,
|
|
"mean_weights": [
|
|
0.09819990396499634,
|
|
0.08131108433008194,
|
|
0.05535554885864258,
|
|
0.052511729300022125,
|
|
0.07309524714946747,
|
|
0.1031290739774704,
|
|
0.1768367737531662,
|
|
0.35956063866615295
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.620237112045288,
|
|
"mean_weights": [
|
|
0.07192602753639221,
|
|
0.0598294772207737,
|
|
0.03551005944609642,
|
|
0.03596649691462517,
|
|
0.052555397152900696,
|
|
0.08522239327430725,
|
|
0.2067522257566452,
|
|
0.45223796367645264
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.922456979751587,
|
|
"mean_weights": [
|
|
0.11432191729545593,
|
|
0.10134892165660858,
|
|
0.06334333121776581,
|
|
0.05966266989707947,
|
|
0.07981826364994049,
|
|
0.11505833268165588,
|
|
0.20316767692565918,
|
|
0.26327887177467346
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8568212985992432,
|
|
"mean_weights": [
|
|
0.07325166463851929,
|
|
0.05971681326627731,
|
|
0.03186998516321182,
|
|
0.03226577863097191,
|
|
0.04611574858427048,
|
|
0.07693300396203995,
|
|
0.17066776752471924,
|
|
0.3028550446033478,
|
|
0.20632418990135193
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.9784047603607178,
|
|
"mean_weights": [
|
|
0.08589711785316467,
|
|
0.07969395816326141,
|
|
0.03702075034379959,
|
|
0.037476278841495514,
|
|
0.0731067955493927,
|
|
0.12099474668502808,
|
|
0.22441011667251587,
|
|
0.20659996569156647,
|
|
0.1348002701997757
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.9687256813049316,
|
|
"mean_weights": [
|
|
0.07957327365875244,
|
|
0.0893080085515976,
|
|
0.03895219415426254,
|
|
0.03689703717827797,
|
|
0.0598301999270916,
|
|
0.09126800298690796,
|
|
0.16929352283477783,
|
|
0.23949456214904785,
|
|
0.1953832060098648
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.05895887315273285,
|
|
0.05461842566728592,
|
|
0.07232892513275146,
|
|
0.07006148993968964,
|
|
0.12035266309976578,
|
|
0.10343272238969803,
|
|
0.14036840200424194,
|
|
0.12118023633956909,
|
|
0.16676878929138184,
|
|
0.11889537423849106,
|
|
0.13824602961540222,
|
|
0.15437477827072144,
|
|
0.1504182070493698,
|
|
0.12633125483989716,
|
|
0.11159109324216843,
|
|
0.09292281419038773,
|
|
0.11971895396709442,
|
|
0.08979442715644836,
|
|
0.08137664943933487,
|
|
0.06381704658269882,
|
|
0.09776479750871658,
|
|
0.03986503183841705,
|
|
0.06484787911176682,
|
|
0.05485111102461815,
|
|
0.07206991314888,
|
|
0.046776797622442245,
|
|
0.04656966030597687,
|
|
0.04702001437544823,
|
|
0.055061157792806625,
|
|
0.04538688063621521,
|
|
0.05328623950481415,
|
|
0.053778134286403656
|
|
],
|
|
"loss_nats": 1.3475289344787598,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.7764334678649902,
|
|
"mean_weights": [
|
|
0.08629432320594788,
|
|
0.05034743621945381,
|
|
0.046479545533657074,
|
|
0.04277106374502182,
|
|
0.04524604231119156,
|
|
0.05579439178109169,
|
|
0.07300648093223572,
|
|
0.2586928606033325,
|
|
0.3413679003715515
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 4000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.06033274158835411,
|
|
0.16565655171871185,
|
|
0.19572967290878296,
|
|
0.24343863129615784,
|
|
0.20718541741371155,
|
|
0.21182885766029358,
|
|
0.280988872051239,
|
|
0.28418517112731934,
|
|
0.11816111207008362,
|
|
0.12311498820781708,
|
|
0.17678114771842957,
|
|
0.18273214995861053,
|
|
0.05406087636947632,
|
|
0.06521740555763245,
|
|
0.0933222770690918,
|
|
0.10707039386034012,
|
|
0.04759044572710991,
|
|
0.06429450213909149,
|
|
0.09436589479446411,
|
|
0.1264011114835739,
|
|
0.052182719111442566,
|
|
0.0799982100725174,
|
|
0.0912039652466774,
|
|
0.12911832332611084,
|
|
0.03087303601205349,
|
|
0.06644809991121292,
|
|
0.07178688794374466,
|
|
0.1084718257188797,
|
|
0.0293489471077919,
|
|
0.10412788391113281,
|
|
0.10697942227125168,
|
|
0.23786598443984985
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
7.570489106001332e-05,
|
|
7.69118414609693e-05,
|
|
7.822692714398727e-05,
|
|
6.853767990833148e-05,
|
|
6.88084473949857e-05,
|
|
7.10485182935372e-05,
|
|
8.91982635948807e-05,
|
|
8.665756467962638e-05,
|
|
0.000171030915225856,
|
|
0.00012665522808674723,
|
|
0.00023480529489461333,
|
|
0.00016227969899773598,
|
|
0.00023323859204538167,
|
|
0.000136202055728063,
|
|
9.528713417239487e-05,
|
|
6.303648115135729e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 7.484533489332534e-05,
|
|
"first_to_last_ratio": 0.5672633797074002,
|
|
"imbalance_abs_log_ratio": 0.5669315686327029,
|
|
"last_quartile_mean": 0.0001319410657742992,
|
|
"mean": 0.00011485184586490504,
|
|
"normalized": [
|
|
0.6591525847051819,
|
|
0.6696613439842941,
|
|
0.6811116230208613,
|
|
0.5967486146365397,
|
|
0.5991061517280438,
|
|
0.6186101560536372,
|
|
0.7766376145125305,
|
|
0.7545160813658816,
|
|
1.4891438090340474,
|
|
1.1027705051926284,
|
|
2.0444189914962623,
|
|
1.4129481139434028,
|
|
2.0307779147035174,
|
|
1.1858934847966764,
|
|
0.8296526142424977,
|
|
0.5488503965839976
|
|
],
|
|
"population_cv": 0.4839100882418841
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.16869333386421204,
|
|
0.24958865344524384,
|
|
0.23942448198795319,
|
|
0.31786683201789856,
|
|
0.1331564038991928,
|
|
0.19950589537620544,
|
|
0.06165311112999916,
|
|
0.10097606480121613,
|
|
0.05752932280302048,
|
|
0.11994150280952454,
|
|
0.07362622767686844,
|
|
0.12106145173311234,
|
|
0.06137929856777191,
|
|
0.09901073575019836,
|
|
0.0852273628115654,
|
|
0.19495683908462524
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.2438933253288269,
|
|
"first_to_last_ratio": 2.2143221757549307,
|
|
"imbalance_abs_log_ratio": 0.7949463412042855,
|
|
"last_quartile_mean": 0.11014355905354023,
|
|
"mean": 0.142724844859913,
|
|
"normalized": [
|
|
1.1819479224502754,
|
|
1.7487400577679353,
|
|
1.6775249062134392,
|
|
2.2271303383085868,
|
|
0.9329588273848762,
|
|
1.3978357837559683,
|
|
0.43197182095740094,
|
|
0.7074876480007805,
|
|
0.4030785450107621,
|
|
0.8403687734062649,
|
|
0.5158613256797294,
|
|
0.8482156827841455,
|
|
0.430053356358641,
|
|
0.6937175923882012,
|
|
0.5971445468742151,
|
|
1.3659628726587785
|
|
],
|
|
"population_cv": 0.529368044298578
|
|
},
|
|
"bits_per_byte": 1.883020158101202,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.06029115617275238,
|
|
0.1536618322134018,
|
|
0.06518179178237915,
|
|
0.06745198369026184,
|
|
0.23389145731925964,
|
|
0.026561493054032326,
|
|
0.09204838424921036,
|
|
0.022710271179676056,
|
|
0.12803514301776886,
|
|
0.017566412687301636,
|
|
0.07296492159366608,
|
|
0.024595333263278008,
|
|
0.05156426504254341,
|
|
0.023977436125278473,
|
|
0.0450289212167263,
|
|
0.03369171544909477,
|
|
0.04221063107252121,
|
|
0.034665971994400024,
|
|
0.05162743479013443,
|
|
0.07154636830091476,
|
|
0.04606551304459572,
|
|
0.05490076169371605,
|
|
0.03287900984287262,
|
|
0.08022980391979218,
|
|
0.027681373059749603,
|
|
0.05210135132074356,
|
|
0.019983114674687386,
|
|
0.06804364919662476,
|
|
0.02240605279803276,
|
|
0.08063697069883347,
|
|
0.016830744221806526,
|
|
0.13339926302433014
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0005343193187437567,
|
|
0.0003255019836390766,
|
|
0.00031277964721318437,
|
|
0.00022314651478922115,
|
|
0.0002396348782003974,
|
|
0.00021519463019459167,
|
|
0.00025456643186586784,
|
|
0.0003008996254324453,
|
|
0.00059437697364117,
|
|
0.0008147654749400623,
|
|
0.001560733102975404,
|
|
0.001320101058102649,
|
|
0.0015032932063515777,
|
|
0.0009169853654825589,
|
|
0.0006174439085696804,
|
|
0.00045539974927406244
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.0003489368660963097,
|
|
"first_to_last_ratio": 0.3995701760811125,
|
|
"imbalance_abs_log_ratio": 0.9173658694246768,
|
|
"last_quartile_mean": 0.0008732805574194699,
|
|
"mean": 0.0006368213668384816,
|
|
"normalized": [
|
|
0.8390411292202721,
|
|
0.5111354621391565,
|
|
0.4911575890834005,
|
|
0.3504067646113047,
|
|
0.3762984263390404,
|
|
0.33791992762889184,
|
|
0.39974543117117817,
|
|
0.4725024019314401,
|
|
0.9333496088423855,
|
|
1.2794254674352237,
|
|
2.450817739869046,
|
|
2.072953463631928,
|
|
2.3606199236290095,
|
|
1.4399412664731395,
|
|
0.9695715953046595,
|
|
0.7151138026899253
|
|
],
|
|
"population_cv": 0.7016607196207993
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.2716493308544159,
|
|
"mean_weights": [
|
|
0.9192699193954468,
|
|
0.0807301327586174
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5622239112854004,
|
|
"mean_weights": [
|
|
0.72159743309021,
|
|
0.27840256690979004
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.48637402057647705,
|
|
"mean_weights": [
|
|
0.7995253801345825,
|
|
0.20047461986541748
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6475558280944824,
|
|
"mean_weights": [
|
|
0.5858595371246338,
|
|
0.41414040327072144
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9933564066886902,
|
|
"mean_weights": [
|
|
0.4998217821121216,
|
|
0.2229493409395218,
|
|
0.2772288918495178
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0489425659179688,
|
|
"mean_weights": [
|
|
0.2563193440437317,
|
|
0.269240140914917,
|
|
0.4744405150413513
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.9815441966056824,
|
|
"mean_weights": [
|
|
0.5070900917053223,
|
|
0.19802016019821167,
|
|
0.29488980770111084
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.071545958518982,
|
|
"mean_weights": [
|
|
0.2598530650138855,
|
|
0.3191677927970886,
|
|
0.4209791421890259
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3049530982971191,
|
|
"mean_weights": [
|
|
0.3487145006656647,
|
|
0.13604819774627686,
|
|
0.23624856770038605,
|
|
0.27898871898651123
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3723504543304443,
|
|
"mean_weights": [
|
|
0.2135777771472931,
|
|
0.23673927783966064,
|
|
0.23828449845314026,
|
|
0.311398446559906
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.265662670135498,
|
|
"mean_weights": [
|
|
0.24546673893928528,
|
|
0.10275381803512573,
|
|
0.2848597466945648,
|
|
0.36691969633102417
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3751296997070312,
|
|
"mean_weights": [
|
|
0.23200613260269165,
|
|
0.24026888608932495,
|
|
0.22661332786083221,
|
|
0.30111163854599
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5548532009124756,
|
|
"mean_weights": [
|
|
0.2040066421031952,
|
|
0.09975787997245789,
|
|
0.22992464900016785,
|
|
0.259371280670166,
|
|
0.20693954825401306
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5716381072998047,
|
|
"mean_weights": [
|
|
0.1984836459159851,
|
|
0.17718908190727234,
|
|
0.15249167382717133,
|
|
0.17045220732688904,
|
|
0.301383376121521
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.4117064476013184,
|
|
"mean_weights": [
|
|
0.25685301423072815,
|
|
0.12795110046863556,
|
|
0.08630626648664474,
|
|
0.10788753628730774,
|
|
0.4210020899772644
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5975096225738525,
|
|
"mean_weights": [
|
|
0.2208021879196167,
|
|
0.19316574931144714,
|
|
0.17618897557258606,
|
|
0.17661380767822266,
|
|
0.23322927951812744
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7283259630203247,
|
|
"mean_weights": [
|
|
0.1650083363056183,
|
|
0.10661210119724274,
|
|
0.12329171597957611,
|
|
0.13269363343715668,
|
|
0.19600069522857666,
|
|
0.27639350295066833
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7028545141220093,
|
|
"mean_weights": [
|
|
0.17663681507110596,
|
|
0.13835734128952026,
|
|
0.11904168128967285,
|
|
0.11434897780418396,
|
|
0.12915222346782684,
|
|
0.3224629759788513
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.27894127368927,
|
|
"mean_weights": [
|
|
0.17268770933151245,
|
|
0.10967432707548141,
|
|
0.03322022780776024,
|
|
0.031795307993888855,
|
|
0.09717383980751038,
|
|
0.555448591709137
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7646939754486084,
|
|
"mean_weights": [
|
|
0.1884503960609436,
|
|
0.1533900499343872,
|
|
0.15274858474731445,
|
|
0.14330559968948364,
|
|
0.1338358074426651,
|
|
0.2282695174217224
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.4344873428344727,
|
|
"mean_weights": [
|
|
0.09210026264190674,
|
|
0.06219030171632767,
|
|
0.029036182910203934,
|
|
0.0315997451543808,
|
|
0.07091090828180313,
|
|
0.1949847936630249,
|
|
0.5191777944564819
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.802895426750183,
|
|
"mean_weights": [
|
|
0.13523127138614655,
|
|
0.10325458645820618,
|
|
0.10260490328073502,
|
|
0.09636874496936798,
|
|
0.09404134750366211,
|
|
0.12264730036258698,
|
|
0.3458518385887146
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.4361302852630615,
|
|
"mean_weights": [
|
|
0.10137400776147842,
|
|
0.06668317317962646,
|
|
0.03099493309855461,
|
|
0.033589236438274384,
|
|
0.07286111265420914,
|
|
0.16290563344955444,
|
|
0.5315918922424316
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8628063201904297,
|
|
"mean_weights": [
|
|
0.14754432439804077,
|
|
0.11970964819192886,
|
|
0.10172373056411743,
|
|
0.09891469776630402,
|
|
0.10695713013410568,
|
|
0.14022418856620789,
|
|
0.28492626547813416
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.6038438081741333,
|
|
"mean_weights": [
|
|
0.07592754065990448,
|
|
0.048247914761304855,
|
|
0.024802275002002716,
|
|
0.026826059445738792,
|
|
0.049207836389541626,
|
|
0.09689367562532425,
|
|
0.29891306161880493,
|
|
0.3791816532611847
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.7748796939849854,
|
|
"mean_weights": [
|
|
0.09652430564165115,
|
|
0.07308399677276611,
|
|
0.04972345381975174,
|
|
0.050231896340847015,
|
|
0.0702696442604065,
|
|
0.09669448435306549,
|
|
0.16264159977436066,
|
|
0.40083062648773193
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.526896357536316,
|
|
"mean_weights": [
|
|
0.06852421164512634,
|
|
0.05069870874285698,
|
|
0.02793668583035469,
|
|
0.029434790834784508,
|
|
0.046449482440948486,
|
|
0.08027796447277069,
|
|
0.19853916764259338,
|
|
0.49813902378082275
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9032639265060425,
|
|
"mean_weights": [
|
|
0.11537295579910278,
|
|
0.09104456007480621,
|
|
0.05989263206720352,
|
|
0.057612620294094086,
|
|
0.07744243741035461,
|
|
0.11209231615066528,
|
|
0.20047283172607422,
|
|
0.2860696315765381
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.7657151222229004,
|
|
"mean_weights": [
|
|
0.06711487472057343,
|
|
0.04881071671843529,
|
|
0.02479729615151882,
|
|
0.025201791897416115,
|
|
0.040198031812906265,
|
|
0.06934815645217896,
|
|
0.15364184975624084,
|
|
0.3216337561607361,
|
|
0.24925348162651062
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.9778118133544922,
|
|
"mean_weights": [
|
|
0.08567827194929123,
|
|
0.08225203305482864,
|
|
0.030744101852178574,
|
|
0.03173041343688965,
|
|
0.07107861340045929,
|
|
0.128017395734787,
|
|
0.2095082402229309,
|
|
0.20655101537704468,
|
|
0.15443992614746094
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.88002610206604,
|
|
"mean_weights": [
|
|
0.07342863082885742,
|
|
0.07307932525873184,
|
|
0.030178282409906387,
|
|
0.029849819839000702,
|
|
0.050495631992816925,
|
|
0.08296258747577667,
|
|
0.14574018120765686,
|
|
0.2550249397754669,
|
|
0.25924059748649597
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.061662707477808,
|
|
0.0571652427315712,
|
|
0.07283993810415268,
|
|
0.07206818461418152,
|
|
0.1187756210565567,
|
|
0.10985641181468964,
|
|
0.15539979934692383,
|
|
0.12964090704917908,
|
|
0.18306857347488403,
|
|
0.129257932305336,
|
|
0.14960098266601562,
|
|
0.16845102608203888,
|
|
0.16322582960128784,
|
|
0.13869591057300568,
|
|
0.11515536159276962,
|
|
0.08842873573303223,
|
|
0.1271047443151474,
|
|
0.08747127652168274,
|
|
0.08377104997634888,
|
|
0.05908897519111633,
|
|
0.10309968143701553,
|
|
0.03653678670525551,
|
|
0.06588983535766602,
|
|
0.052161477506160736,
|
|
0.07435949891805649,
|
|
0.04259056970477104,
|
|
0.04427788779139519,
|
|
0.043995074927806854,
|
|
0.053472407162189484,
|
|
0.04156629368662834,
|
|
0.04980214312672615,
|
|
0.0496271476149559
|
|
],
|
|
"loss_nats": 1.3052101135253906,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.609398603439331,
|
|
"mean_weights": [
|
|
0.0809592753648758,
|
|
0.04013277217745781,
|
|
0.03442784771323204,
|
|
0.03263507038354874,
|
|
0.03443540260195732,
|
|
0.047240644693374634,
|
|
0.05792279541492462,
|
|
0.2393622249364853,
|
|
0.43288397789001465
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 8000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.06029115617275238,
|
|
0.16869333386421204,
|
|
0.20056062936782837,
|
|
0.24958865344524384,
|
|
0.23389145731925964,
|
|
0.23942448198795319,
|
|
0.31467917561531067,
|
|
0.31786683201789856,
|
|
0.12803514301776886,
|
|
0.1331564038991928,
|
|
0.19432871043682098,
|
|
0.19950589537620544,
|
|
0.05156426504254341,
|
|
0.06165311112999916,
|
|
0.08753261715173721,
|
|
0.10097606480121613,
|
|
0.04221063107252121,
|
|
0.05752932280302048,
|
|
0.08379971981048584,
|
|
0.11994150280952454,
|
|
0.04606551304459572,
|
|
0.07362622767686844,
|
|
0.08261217921972275,
|
|
0.12106145173311234,
|
|
0.027681373059749603,
|
|
0.06137929856777191,
|
|
0.06537999957799911,
|
|
0.09901073575019836,
|
|
0.02240605279803276,
|
|
0.0852273628115654,
|
|
0.08763451874256134,
|
|
0.19495683908462524
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 8.015965601418904,
|
|
"cross_entropy_nats": 5.55624395608902,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 5.1077262486126465,
|
|
"cross_entropy_nats": 3.540406048297882,
|
|
"step": 100
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1449978072640112,
|
|
"cross_entropy_nats": 2.1799463629722595,
|
|
"step": 500
|
|
},
|
|
{
|
|
"bits_per_byte": 1.978178932745523,
|
|
"cross_entropy_nats": 1.3711691498756409,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8301036761704788,
|
|
"cross_entropy_nats": 1.2685312032699585,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"bits_per_byte": 1.72661337398776,
|
|
"cross_entropy_nats": 1.1967971920967102,
|
|
"step": 8000
|
|
}
|
|
],
|
|
"gradient_gate": null,
|
|
"hashes": {
|
|
"final_mixer_parameters": "ad1807a65c19f598a4bde3ce03f75bb64bba14b79befbf1740b5aaddab905279",
|
|
"final_model_state": "6df5cb12d848445089dff93c7b22def426897ab70173e1fcca78c3db41e914dc",
|
|
"final_optimizer_state": "a8e9a566e68720356f0b3f878d687f8c4cac616f59f729e02056853409f8d145",
|
|
"final_public_parameters": "16b3a7903fb4172275d1673bd0f390d3b3c03a9d0d151ac66d732eb978cff06f",
|
|
"initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f",
|
|
"initial_public_parameter_elements": 9541824,
|
|
"initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
|
|
"initial_public_parameter_tensors": 115,
|
|
"initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
|
|
"1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
|
|
"7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 16,
|
|
"parameters": {
|
|
"core": 9541824,
|
|
"embedding": 98304,
|
|
"mixer": 12672,
|
|
"total": 9554496
|
|
},
|
|
"sublayers": 32,
|
|
"sublayers_per_attnres_group": 4,
|
|
"transformer_blocks_per_attnres_group": 2,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"run_kind": "formal",
|
|
"schema_version": 1,
|
|
"seed": 2026073003,
|
|
"steps": 8000,
|
|
"target_bytes_seen": 65536000,
|
|
"timing": {
|
|
"mean_ms": 54.54310071476275,
|
|
"measured_steps": 7980,
|
|
"median_ms": 54.49852900346741,
|
|
"p95_ms": 54.8562506082817,
|
|
"peak_allocated_bytes": 6536921088,
|
|
"peak_reserved_bytes": 6889144320,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 8.030921126746609,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.566610336303711,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 17.857467651367188
|
|
},
|
|
{
|
|
"bits_per_byte": 7.460191884344154,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.171010971069336,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 12.879206657409668
|
|
},
|
|
{
|
|
"bits_per_byte": 6.7765918040416455,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.6971755027771,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 4.5564117431640625
|
|
},
|
|
{
|
|
"bits_per_byte": 6.553465640395858,
|
|
"learning_rate": 2.2499999999999998e-05,
|
|
"loss_nats": 4.542516231536865,
|
|
"step": 30,
|
|
"unclipped_grad_norm": 3.0218148231506348
|
|
},
|
|
{
|
|
"bits_per_byte": 6.42131485940348,
|
|
"learning_rate": 2.9999999999999997e-05,
|
|
"loss_nats": 4.450916290283203,
|
|
"step": 40,
|
|
"unclipped_grad_norm": 2.865441083908081
|
|
},
|
|
{
|
|
"bits_per_byte": 6.256338093806218,
|
|
"learning_rate": 3.75e-05,
|
|
"loss_nats": 4.3365631103515625,
|
|
"step": 50,
|
|
"unclipped_grad_norm": 2.8344333171844482
|
|
},
|
|
{
|
|
"bits_per_byte": 5.946641367160921,
|
|
"learning_rate": 4.4999999999999996e-05,
|
|
"loss_nats": 4.1218976974487305,
|
|
"step": 60,
|
|
"unclipped_grad_norm": 2.6930737495422363
|
|
},
|
|
{
|
|
"bits_per_byte": 5.708490111205155,
|
|
"learning_rate": 5.2499999999999995e-05,
|
|
"loss_nats": 3.9568238258361816,
|
|
"step": 70,
|
|
"unclipped_grad_norm": 2.6865241527557373
|
|
},
|
|
{
|
|
"bits_per_byte": 5.509431199721109,
|
|
"learning_rate": 5.9999999999999995e-05,
|
|
"loss_nats": 3.8188467025756836,
|
|
"step": 80,
|
|
"unclipped_grad_norm": 2.448140859603882
|
|
},
|
|
{
|
|
"bits_per_byte": 5.311447867735559,
|
|
"learning_rate": 6.749999999999999e-05,
|
|
"loss_nats": 3.681615114212036,
|
|
"step": 90,
|
|
"unclipped_grad_norm": 2.2759759426116943
|
|
},
|
|
{
|
|
"bits_per_byte": 5.0864742664542035,
|
|
"learning_rate": 7.5e-05,
|
|
"loss_nats": 3.5256752967834473,
|
|
"step": 100,
|
|
"unclipped_grad_norm": 2.1487066745758057
|
|
},
|
|
{
|
|
"bits_per_byte": 4.9338616775927715,
|
|
"learning_rate": 8.249999999999999e-05,
|
|
"loss_nats": 3.4198923110961914,
|
|
"step": 110,
|
|
"unclipped_grad_norm": 1.9959912300109863
|
|
},
|
|
{
|
|
"bits_per_byte": 4.731707078393195,
|
|
"learning_rate": 8.999999999999999e-05,
|
|
"loss_nats": 3.2797694206237793,
|
|
"step": 120,
|
|
"unclipped_grad_norm": 1.7048262357711792
|
|
},
|
|
{
|
|
"bits_per_byte": 4.582112785054405,
|
|
"learning_rate": 9.75e-05,
|
|
"loss_nats": 3.1760785579681396,
|
|
"step": 130,
|
|
"unclipped_grad_norm": 1.5641754865646362
|
|
},
|
|
{
|
|
"bits_per_byte": 4.436114304979855,
|
|
"learning_rate": 0.00010499999999999999,
|
|
"loss_nats": 3.0748801231384277,
|
|
"step": 140,
|
|
"unclipped_grad_norm": 1.380750060081482
|
|
},
|
|
{
|
|
"bits_per_byte": 4.229324085408923,
|
|
"learning_rate": 0.0001125,
|
|
"loss_nats": 2.931544065475464,
|
|
"step": 150,
|
|
"unclipped_grad_norm": 1.1272099018096924
|
|
},
|
|
{
|
|
"bits_per_byte": 4.108468093029308,
|
|
"learning_rate": 0.00011999999999999999,
|
|
"loss_nats": 2.8477730751037598,
|
|
"step": 160,
|
|
"unclipped_grad_norm": 0.8926234245300293
|
|
},
|
|
{
|
|
"bits_per_byte": 4.025982461953091,
|
|
"learning_rate": 0.00012749999999999998,
|
|
"loss_nats": 2.7905983924865723,
|
|
"step": 170,
|
|
"unclipped_grad_norm": 0.915916919708252
|
|
},
|
|
{
|
|
"bits_per_byte": 3.9299425341970746,
|
|
"learning_rate": 0.00013499999999999997,
|
|
"loss_nats": 2.7240285873413086,
|
|
"step": 180,
|
|
"unclipped_grad_norm": 0.7168256640434265
|
|
},
|
|
{
|
|
"bits_per_byte": 3.8518424607178594,
|
|
"learning_rate": 0.0001425,
|
|
"loss_nats": 2.669893741607666,
|
|
"step": 190,
|
|
"unclipped_grad_norm": 0.8937639594078064
|
|
},
|
|
{
|
|
"bits_per_byte": 3.7890044716415114,
|
|
"learning_rate": 0.00015,
|
|
"loss_nats": 2.626337766647339,
|
|
"step": 200,
|
|
"unclipped_grad_norm": 0.7755346894264221
|
|
},
|
|
{
|
|
"bits_per_byte": 3.7186869570791403,
|
|
"learning_rate": 0.0001575,
|
|
"loss_nats": 2.5775973796844482,
|
|
"step": 210,
|
|
"unclipped_grad_norm": 0.731957197189331
|
|
},
|
|
{
|
|
"bits_per_byte": 3.706538446587493,
|
|
"learning_rate": 0.00016499999999999997,
|
|
"loss_nats": 2.56917667388916,
|
|
"step": 220,
|
|
"unclipped_grad_norm": 0.8748186230659485
|
|
},
|
|
{
|
|
"bits_per_byte": 3.655573795761412,
|
|
"learning_rate": 0.0001725,
|
|
"loss_nats": 2.53385066986084,
|
|
"step": 230,
|
|
"unclipped_grad_norm": 0.8331225514411926
|
|
},
|
|
{
|
|
"bits_per_byte": 3.652975825837478,
|
|
"learning_rate": 0.00017999999999999998,
|
|
"loss_nats": 2.5320498943328857,
|
|
"step": 240,
|
|
"unclipped_grad_norm": 1.103366494178772
|
|
},
|
|
{
|
|
"bits_per_byte": 3.571490789823981,
|
|
"learning_rate": 0.0001875,
|
|
"loss_nats": 2.4755687713623047,
|
|
"step": 250,
|
|
"unclipped_grad_norm": 0.6647337675094604
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5855342051628347,
|
|
"learning_rate": 0.000195,
|
|
"loss_nats": 2.4853029251098633,
|
|
"step": 260,
|
|
"unclipped_grad_norm": 0.8029841780662537
|
|
},
|
|
{
|
|
"bits_per_byte": 3.576924065730039,
|
|
"learning_rate": 0.00020249999999999996,
|
|
"loss_nats": 2.479334831237793,
|
|
"step": 270,
|
|
"unclipped_grad_norm": 0.7433680891990662
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5153632198189984,
|
|
"learning_rate": 0.00020999999999999998,
|
|
"loss_nats": 2.43666410446167,
|
|
"step": 280,
|
|
"unclipped_grad_norm": 0.824142336845398
|
|
},
|
|
{
|
|
"bits_per_byte": 3.511655617831698,
|
|
"learning_rate": 0.00021749999999999997,
|
|
"loss_nats": 2.434094190597534,
|
|
"step": 290,
|
|
"unclipped_grad_norm": 0.8560227751731873
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4998641433232422,
|
|
"learning_rate": 0.000225,
|
|
"loss_nats": 2.4259209632873535,
|
|
"step": 300,
|
|
"unclipped_grad_norm": 0.8836220502853394
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4156380478202935,
|
|
"learning_rate": 0.00023249999999999996,
|
|
"loss_nats": 2.367539882659912,
|
|
"step": 310,
|
|
"unclipped_grad_norm": 0.9033657908439636
|
|
},
|
|
{
|
|
"bits_per_byte": 3.481727196928552,
|
|
"learning_rate": 0.00023999999999999998,
|
|
"loss_nats": 2.4133493900299072,
|
|
"step": 320,
|
|
"unclipped_grad_norm": 1.119826316833496
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4809216301919106,
|
|
"learning_rate": 0.0002475,
|
|
"loss_nats": 2.4127910137176514,
|
|
"step": 330,
|
|
"unclipped_grad_norm": 0.9842678308486938
|
|
},
|
|
{
|
|
"bits_per_byte": 3.422977579428515,
|
|
"learning_rate": 0.00025499999999999996,
|
|
"loss_nats": 2.3726272583007812,
|
|
"step": 340,
|
|
"unclipped_grad_norm": 1.075486421585083
|
|
},
|
|
{
|
|
"bits_per_byte": 3.385370477129646,
|
|
"learning_rate": 0.0002625,
|
|
"loss_nats": 2.346560001373291,
|
|
"step": 350,
|
|
"unclipped_grad_norm": 0.7245537042617798
|
|
},
|
|
{
|
|
"bits_per_byte": 3.385430671057448,
|
|
"learning_rate": 0.00026999999999999995,
|
|
"loss_nats": 2.346601724624634,
|
|
"step": 360,
|
|
"unclipped_grad_norm": 1.202561616897583
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3953991294667585,
|
|
"learning_rate": 0.00027749999999999997,
|
|
"loss_nats": 2.353511333465576,
|
|
"step": 370,
|
|
"unclipped_grad_norm": 0.7641725540161133
|
|
},
|
|
{
|
|
"bits_per_byte": 3.295281837024086,
|
|
"learning_rate": 0.000285,
|
|
"loss_nats": 2.2841153144836426,
|
|
"step": 380,
|
|
"unclipped_grad_norm": 1.061362862586975
|
|
},
|
|
{
|
|
"bits_per_byte": 3.380484450018634,
|
|
"learning_rate": 0.0002925,
|
|
"loss_nats": 2.3431732654571533,
|
|
"step": 390,
|
|
"unclipped_grad_norm": 1.0704491138458252
|
|
},
|
|
{
|
|
"bits_per_byte": 3.354825670405808,
|
|
"learning_rate": 0.0003,
|
|
"loss_nats": 2.325387954711914,
|
|
"step": 400,
|
|
"unclipped_grad_norm": 1.066178560256958
|
|
},
|
|
{
|
|
"bits_per_byte": 3.38879568360423,
|
|
"learning_rate": 0.0002999988466112388,
|
|
"loss_nats": 2.3489341735839844,
|
|
"step": 410,
|
|
"unclipped_grad_norm": 0.9359472990036011
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3128674070401103,
|
|
"learning_rate": 0.00029999538646466355,
|
|
"loss_nats": 2.296304702758789,
|
|
"step": 420,
|
|
"unclipped_grad_norm": 0.9575046896934509
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3331142365955855,
|
|
"learning_rate": 0.0002999896196193985,
|
|
"loss_nats": 2.3103387355804443,
|
|
"step": 430,
|
|
"unclipped_grad_norm": 0.9350728988647461
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2769842588334903,
|
|
"learning_rate": 0.00029998154617398326,
|
|
"loss_nats": 2.271432399749756,
|
|
"step": 440,
|
|
"unclipped_grad_norm": 1.028943419456482
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3396626480098375,
|
|
"learning_rate": 0.0002999711662663707,
|
|
"loss_nats": 2.31487774848938,
|
|
"step": 450,
|
|
"unclipped_grad_norm": 1.1271060705184937
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2591895579140173,
|
|
"learning_rate": 0.0002999584800739247,
|
|
"loss_nats": 2.2590980529785156,
|
|
"step": 460,
|
|
"unclipped_grad_norm": 1.1319539546966553
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2327510089621794,
|
|
"learning_rate": 0.0002999434878134173,
|
|
"loss_nats": 2.240772247314453,
|
|
"step": 470,
|
|
"unclipped_grad_norm": 1.1909105777740479
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2186394884935843,
|
|
"learning_rate": 0.000299926189741025,
|
|
"loss_nats": 2.2309908866882324,
|
|
"step": 480,
|
|
"unclipped_grad_norm": 1.3586266040802002
|
|
},
|
|
{
|
|
"bits_per_byte": 3.169725902761693,
|
|
"learning_rate": 0.000299906586152324,
|
|
"loss_nats": 2.1970865726470947,
|
|
"step": 490,
|
|
"unclipped_grad_norm": 1.5543664693832397
|
|
},
|
|
{
|
|
"bits_per_byte": 3.101864644418944,
|
|
"learning_rate": 0.0002998846773822856,
|
|
"loss_nats": 2.1500487327575684,
|
|
"step": 500,
|
|
"unclipped_grad_norm": 1.9330527782440186
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1523112835006266,
|
|
"learning_rate": 0.0002998604638052702,
|
|
"loss_nats": 2.1850156784057617,
|
|
"step": 510,
|
|
"unclipped_grad_norm": 1.367343544960022
|
|
},
|
|
{
|
|
"bits_per_byte": 3.137257298105303,
|
|
"learning_rate": 0.0002998339458350211,
|
|
"loss_nats": 2.1745810508728027,
|
|
"step": 520,
|
|
"unclipped_grad_norm": 1.4051828384399414
|
|
},
|
|
{
|
|
"bits_per_byte": 2.96818390576449,
|
|
"learning_rate": 0.0002998051239246572,
|
|
"loss_nats": 2.0573883056640625,
|
|
"step": 530,
|
|
"unclipped_grad_norm": 0.88100665807724
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9700385666713944,
|
|
"learning_rate": 0.00029977399856666537,
|
|
"loss_nats": 2.058673858642578,
|
|
"step": 540,
|
|
"unclipped_grad_norm": 1.1694613695144653
|
|
},
|
|
{
|
|
"bits_per_byte": 2.981698990399893,
|
|
"learning_rate": 0.0002997405702928922,
|
|
"loss_nats": 2.066756248474121,
|
|
"step": 550,
|
|
"unclipped_grad_norm": 2.2637624740600586
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9860140351100406,
|
|
"learning_rate": 0.0002997048396745345,
|
|
"loss_nats": 2.06974720954895,
|
|
"step": 560,
|
|
"unclipped_grad_norm": 2.0154106616973877
|
|
},
|
|
{
|
|
"bits_per_byte": 2.908624249978889,
|
|
"learning_rate": 0.0002996668073221302,
|
|
"loss_nats": 2.0161046981811523,
|
|
"step": 570,
|
|
"unclipped_grad_norm": 1.2818057537078857
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9258029090429716,
|
|
"learning_rate": 0.000299626473885547,
|
|
"loss_nats": 2.0280120372772217,
|
|
"step": 580,
|
|
"unclipped_grad_norm": 1.5401909351348877
|
|
},
|
|
{
|
|
"bits_per_byte": 2.865440266260492,
|
|
"learning_rate": 0.00029958384005397226,
|
|
"loss_nats": 1.986171841621399,
|
|
"step": 590,
|
|
"unclipped_grad_norm": 1.2538588047027588
|
|
},
|
|
{
|
|
"bits_per_byte": 2.8203701488100825,
|
|
"learning_rate": 0.0002995389065559004,
|
|
"loss_nats": 1.954931616783142,
|
|
"step": 600,
|
|
"unclipped_grad_norm": 1.5663292407989502
|
|
},
|
|
{
|
|
"bits_per_byte": 2.8417604910111445,
|
|
"learning_rate": 0.000299491674159121,
|
|
"loss_nats": 1.9697582721710205,
|
|
"step": 610,
|
|
"unclipped_grad_norm": 1.3150807619094849
|
|
},
|
|
{
|
|
"bits_per_byte": 2.806068931477586,
|
|
"learning_rate": 0.00029944214367070535,
|
|
"loss_nats": 1.9450187683105469,
|
|
"step": 620,
|
|
"unclipped_grad_norm": 1.0626381635665894
|
|
},
|
|
{
|
|
"bits_per_byte": 2.766462358879786,
|
|
"learning_rate": 0.00029939031593699284,
|
|
"loss_nats": 1.9175655841827393,
|
|
"step": 630,
|
|
"unclipped_grad_norm": 2.55611515045166
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7411267346679313,
|
|
"learning_rate": 0.00029933619184357654,
|
|
"loss_nats": 1.900004267692566,
|
|
"step": 640,
|
|
"unclipped_grad_norm": 0.9799621105194092
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7289859633955733,
|
|
"learning_rate": 0.00029927977231528774,
|
|
"loss_nats": 1.8915889263153076,
|
|
"step": 650,
|
|
"unclipped_grad_norm": 1.4919346570968628
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7446909031244124,
|
|
"learning_rate": 0.00029922105831618057,
|
|
"loss_nats": 1.9024747610092163,
|
|
"step": 660,
|
|
"unclipped_grad_norm": 1.184576153755188
|
|
},
|
|
{
|
|
"bits_per_byte": 2.706914225931793,
|
|
"learning_rate": 0.0002991600508495153,
|
|
"loss_nats": 1.876289963722229,
|
|
"step": 670,
|
|
"unclipped_grad_norm": 1.2165472507476807
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7108362902847194,
|
|
"learning_rate": 0.00029909675095774114,
|
|
"loss_nats": 1.8790085315704346,
|
|
"step": 680,
|
|
"unclipped_grad_norm": 2.3380022048950195
|
|
},
|
|
{
|
|
"bits_per_byte": 2.6699751042488753,
|
|
"learning_rate": 0.0002990311597224787,
|
|
"loss_nats": 1.850685715675354,
|
|
"step": 690,
|
|
"unclipped_grad_norm": 1.790702223777771
|
|
},
|
|
{
|
|
"bits_per_byte": 2.6284332108809165,
|
|
"learning_rate": 0.00029896327826450103,
|
|
"loss_nats": 1.8218910694122314,
|
|
"step": 700,
|
|
"unclipped_grad_norm": 1.6820380687713623
|
|
},
|
|
{
|
|
"bits_per_byte": 2.6224609413470548,
|
|
"learning_rate": 0.0002988931077437152,
|
|
"loss_nats": 1.817751407623291,
|
|
"step": 710,
|
|
"unclipped_grad_norm": 1.3093626499176025
|
|
},
|
|
{
|
|
"bits_per_byte": 2.6378994799497075,
|
|
"learning_rate": 0.00029882064935914164,
|
|
"loss_nats": 1.8284525871276855,
|
|
"step": 720,
|
|
"unclipped_grad_norm": 1.689491868019104
|
|
},
|
|
{
|
|
"bits_per_byte": 2.604935393276182,
|
|
"learning_rate": 0.0002987459043488943,
|
|
"loss_nats": 1.8056036233901978,
|
|
"step": 730,
|
|
"unclipped_grad_norm": 1.4546315670013428
|
|
},
|
|
{
|
|
"bits_per_byte": 2.542083817570416,
|
|
"learning_rate": 0.0002986688739901591,
|
|
"loss_nats": 1.762038230895996,
|
|
"step": 740,
|
|
"unclipped_grad_norm": 1.6848912239074707
|
|
},
|
|
{
|
|
"bits_per_byte": 2.5572903515770546,
|
|
"learning_rate": 0.0002985895595991723,
|
|
"loss_nats": 1.7725785970687866,
|
|
"step": 750,
|
|
"unclipped_grad_norm": 1.477650761604309
|
|
},
|
|
{
|
|
"bits_per_byte": 2.448532478772384,
|
|
"learning_rate": 0.00029850796253119824,
|
|
"loss_nats": 1.6971933841705322,
|
|
"step": 760,
|
|
"unclipped_grad_norm": 1.4033355712890625
|
|
},
|
|
{
|
|
"bits_per_byte": 2.5017822630804796,
|
|
"learning_rate": 0.00029842408418050544,
|
|
"loss_nats": 1.7341033220291138,
|
|
"step": 770,
|
|
"unclipped_grad_norm": 1.3305320739746094
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4567315796696176,
|
|
"learning_rate": 0.0002983379259803436,
|
|
"loss_nats": 1.7028765678405762,
|
|
"step": 780,
|
|
"unclipped_grad_norm": 1.7414066791534424
|
|
},
|
|
{
|
|
"bits_per_byte": 2.463544844366199,
|
|
"learning_rate": 0.0002982494894029186,
|
|
"loss_nats": 1.70759916305542,
|
|
"step": 790,
|
|
"unclipped_grad_norm": 1.3770233392715454
|
|
},
|
|
{
|
|
"bits_per_byte": 2.390281954925151,
|
|
"learning_rate": 0.0002981587759593675,
|
|
"loss_nats": 1.6568171977996826,
|
|
"step": 800,
|
|
"unclipped_grad_norm": 1.4013909101486206
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4192710065892844,
|
|
"learning_rate": 0.00029806578719973283,
|
|
"loss_nats": 1.6769108772277832,
|
|
"step": 810,
|
|
"unclipped_grad_norm": 1.360757827758789
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3909581907083437,
|
|
"learning_rate": 0.00029797052471293596,
|
|
"loss_nats": 1.6572859287261963,
|
|
"step": 820,
|
|
"unclipped_grad_norm": 1.4178366661071777
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4097108349931227,
|
|
"learning_rate": 0.0002978729901267499,
|
|
"loss_nats": 1.6702842712402344,
|
|
"step": 830,
|
|
"unclipped_grad_norm": 1.1894793510437012
|
|
},
|
|
{
|
|
"bits_per_byte": 2.395687713607071,
|
|
"learning_rate": 0.0002977731851077716,
|
|
"loss_nats": 1.6605641841888428,
|
|
"step": 840,
|
|
"unclipped_grad_norm": 1.7937202453613281
|
|
},
|
|
{
|
|
"bits_per_byte": 2.326945216161301,
|
|
"learning_rate": 0.0002976711113613935,
|
|
"loss_nats": 1.6129155158996582,
|
|
"step": 850,
|
|
"unclipped_grad_norm": 1.4669246673583984
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3940583499727963,
|
|
"learning_rate": 0.0002975667706317743,
|
|
"loss_nats": 1.6594347953796387,
|
|
"step": 860,
|
|
"unclipped_grad_norm": 1.3730710744857788
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3742401011832714,
|
|
"learning_rate": 0.00029746016470180915,
|
|
"loss_nats": 1.645697832107544,
|
|
"step": 870,
|
|
"unclipped_grad_norm": 1.3468742370605469
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3406023584663167,
|
|
"learning_rate": 0.00029735129539309926,
|
|
"loss_nats": 1.6223819255828857,
|
|
"step": 880,
|
|
"unclipped_grad_norm": 1.3746341466903687
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2996515694693733,
|
|
"learning_rate": 0.0002972401645659207,
|
|
"loss_nats": 1.5939970016479492,
|
|
"step": 890,
|
|
"unclipped_grad_norm": 1.5287024974822998
|
|
},
|
|
{
|
|
"bits_per_byte": 2.338895602639154,
|
|
"learning_rate": 0.00029712677411919283,
|
|
"loss_nats": 1.6211988925933838,
|
|
"step": 900,
|
|
"unclipped_grad_norm": 1.2028048038482666
|
|
},
|
|
{
|
|
"bits_per_byte": 2.322601450348459,
|
|
"learning_rate": 0.0002970111259904454,
|
|
"loss_nats": 1.6099046468734741,
|
|
"step": 910,
|
|
"unclipped_grad_norm": 1.5355594158172607
|
|
},
|
|
{
|
|
"bits_per_byte": 2.307329906899794,
|
|
"learning_rate": 0.0002968932221557859,
|
|
"loss_nats": 1.5993192195892334,
|
|
"step": 920,
|
|
"unclipped_grad_norm": 1.5230286121368408
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2540851100885737,
|
|
"learning_rate": 0.0002967730646298655,
|
|
"loss_nats": 1.5624127388000488,
|
|
"step": 930,
|
|
"unclipped_grad_norm": 1.3632712364196777
|
|
},
|
|
{
|
|
"bits_per_byte": 2.306159220995371,
|
|
"learning_rate": 0.0002966506554658449,
|
|
"loss_nats": 1.5985077619552612,
|
|
"step": 940,
|
|
"unclipped_grad_norm": 1.4536393880844116
|
|
},
|
|
{
|
|
"bits_per_byte": 2.330239371855927,
|
|
"learning_rate": 0.00029652599675535876,
|
|
"loss_nats": 1.6151988506317139,
|
|
"step": 950,
|
|
"unclipped_grad_norm": 1.4820353984832764
|
|
},
|
|
{
|
|
"bits_per_byte": 2.256347369878023,
|
|
"learning_rate": 0.0002963990906284808,
|
|
"loss_nats": 1.5639808177947998,
|
|
"step": 960,
|
|
"unclipped_grad_norm": 1.5091183185577393
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2135099312011146,
|
|
"learning_rate": 0.0002962699392536863,
|
|
"loss_nats": 1.5342881679534912,
|
|
"step": 970,
|
|
"unclipped_grad_norm": 1.3972843885421753
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3381242604500345,
|
|
"learning_rate": 0.0002961385448378161,
|
|
"loss_nats": 1.6206642389297485,
|
|
"step": 980,
|
|
"unclipped_grad_norm": 1.3591992855072021
|
|
},
|
|
{
|
|
"bits_per_byte": 2.202303885636196,
|
|
"learning_rate": 0.00029600490962603817,
|
|
"loss_nats": 1.5265207290649414,
|
|
"step": 990,
|
|
"unclipped_grad_norm": 1.2794140577316284
|
|
},
|
|
{
|
|
"bits_per_byte": 2.194666824041982,
|
|
"learning_rate": 0.00029586903590180956,
|
|
"loss_nats": 1.5212271213531494,
|
|
"step": 1000,
|
|
"unclipped_grad_norm": 1.2117714881896973
|
|
},
|
|
{
|
|
"bits_per_byte": 2.205168084703665,
|
|
"learning_rate": 0.00029573092598683735,
|
|
"loss_nats": 1.5285060405731201,
|
|
"step": 1010,
|
|
"unclipped_grad_norm": 1.4173126220703125
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1627664500640433,
|
|
"learning_rate": 0.00029559058224103895,
|
|
"loss_nats": 1.4991154670715332,
|
|
"step": 1020,
|
|
"unclipped_grad_norm": 1.5233873128890991
|
|
},
|
|
{
|
|
"bits_per_byte": 2.171931921476475,
|
|
"learning_rate": 0.0002954480070625017,
|
|
"loss_nats": 1.505468487739563,
|
|
"step": 1030,
|
|
"unclipped_grad_norm": 1.5871013402938843
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2533218510840447,
|
|
"learning_rate": 0.00029530320288744205,
|
|
"loss_nats": 1.5618836879730225,
|
|
"step": 1040,
|
|
"unclipped_grad_norm": 1.408625602722168
|
|
},
|
|
{
|
|
"bits_per_byte": 2.203781560572409,
|
|
"learning_rate": 0.0002951561721901638,
|
|
"loss_nats": 1.5275449752807617,
|
|
"step": 1050,
|
|
"unclipped_grad_norm": 1.9470032453536987
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2074819392883733,
|
|
"learning_rate": 0.00029500691748301594,
|
|
"loss_nats": 1.5301098823547363,
|
|
"step": 1060,
|
|
"unclipped_grad_norm": 1.358320713043213
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2549842353872847,
|
|
"learning_rate": 0.00029485544131634963,
|
|
"loss_nats": 1.5630359649658203,
|
|
"step": 1070,
|
|
"unclipped_grad_norm": 1.3524690866470337
|
|
},
|
|
{
|
|
"bits_per_byte": 2.232926944466177,
|
|
"learning_rate": 0.0002947017462784745,
|
|
"loss_nats": 1.547747015953064,
|
|
"step": 1080,
|
|
"unclipped_grad_norm": 1.3045321702957153
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1562297334700493,
|
|
"learning_rate": 0.00029454583499561476,
|
|
"loss_nats": 1.494584560394287,
|
|
"step": 1090,
|
|
"unclipped_grad_norm": 1.3637882471084595
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1766043461351154,
|
|
"learning_rate": 0.00029438771013186413,
|
|
"loss_nats": 1.5087071657180786,
|
|
"step": 1100,
|
|
"unclipped_grad_norm": 1.3327077627182007
|
|
},
|
|
{
|
|
"bits_per_byte": 2.113153238908442,
|
|
"learning_rate": 0.0002942273743891401,
|
|
"loss_nats": 1.464726209640503,
|
|
"step": 1110,
|
|
"unclipped_grad_norm": 1.3556562662124634
|
|
},
|
|
{
|
|
"bits_per_byte": 2.158234707213808,
|
|
"learning_rate": 0.00029406483050713824,
|
|
"loss_nats": 1.4959743022918701,
|
|
"step": 1120,
|
|
"unclipped_grad_norm": 1.5276546478271484
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1778080527085044,
|
|
"learning_rate": 0.00029390008126328504,
|
|
"loss_nats": 1.5095415115356445,
|
|
"step": 1130,
|
|
"unclipped_grad_norm": 1.3574931621551514
|
|
},
|
|
{
|
|
"bits_per_byte": 2.120840347454057,
|
|
"learning_rate": 0.00029373312947269045,
|
|
"loss_nats": 1.4700545072555542,
|
|
"step": 1140,
|
|
"unclipped_grad_norm": 1.49063241481781
|
|
},
|
|
{
|
|
"bits_per_byte": 2.172341240185528,
|
|
"learning_rate": 0.00029356397798809995,
|
|
"loss_nats": 1.5057522058486938,
|
|
"step": 1150,
|
|
"unclipped_grad_norm": 1.270938515663147
|
|
},
|
|
{
|
|
"bits_per_byte": 2.081790654679751,
|
|
"learning_rate": 0.0002933926296998457,
|
|
"loss_nats": 1.442987322807312,
|
|
"step": 1160,
|
|
"unclipped_grad_norm": 1.387803077697754
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1352082940550883,
|
|
"learning_rate": 0.0002932190875357972,
|
|
"loss_nats": 1.4800136089324951,
|
|
"step": 1170,
|
|
"unclipped_grad_norm": 1.2486525774002075
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1410305947173978,
|
|
"learning_rate": 0.00029304335446131126,
|
|
"loss_nats": 1.4840493202209473,
|
|
"step": 1180,
|
|
"unclipped_grad_norm": 1.3077996969223022
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1241448221077346,
|
|
"learning_rate": 0.00029286543347918116,
|
|
"loss_nats": 1.472344994544983,
|
|
"step": 1190,
|
|
"unclipped_grad_norm": 1.846550464630127
|
|
},
|
|
{
|
|
"bits_per_byte": 2.128028362346868,
|
|
"learning_rate": 0.0002926853276295856,
|
|
"loss_nats": 1.475036859512329,
|
|
"step": 1200,
|
|
"unclipped_grad_norm": 1.1341090202331543
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1484036629425374,
|
|
"learning_rate": 0.0002925030399900368,
|
|
"loss_nats": 1.4891599416732788,
|
|
"step": 1210,
|
|
"unclipped_grad_norm": 1.2377034425735474
|
|
},
|
|
{
|
|
"bits_per_byte": 2.096246140450072,
|
|
"learning_rate": 0.00029231857367532726,
|
|
"loss_nats": 1.4530071020126343,
|
|
"step": 1220,
|
|
"unclipped_grad_norm": 1.182817816734314
|
|
},
|
|
{
|
|
"bits_per_byte": 2.11190705262029,
|
|
"learning_rate": 0.0002921319318374776,
|
|
"loss_nats": 1.463862419128418,
|
|
"step": 1230,
|
|
"unclipped_grad_norm": 1.3176664113998413
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0884407078406633,
|
|
"learning_rate": 0.00029194311766568174,
|
|
"loss_nats": 1.447596788406372,
|
|
"step": 1240,
|
|
"unclipped_grad_norm": 1.3174140453338623
|
|
},
|
|
{
|
|
"bits_per_byte": 2.01052448381522,
|
|
"learning_rate": 0.000291752134386253,
|
|
"loss_nats": 1.3935893774032593,
|
|
"step": 1250,
|
|
"unclipped_grad_norm": 1.3046211004257202
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0807407005962335,
|
|
"learning_rate": 0.00029155898526256856,
|
|
"loss_nats": 1.4422595500946045,
|
|
"step": 1260,
|
|
"unclipped_grad_norm": 1.3089001178741455
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0962698740558907,
|
|
"learning_rate": 0.0002913636735950141,
|
|
"loss_nats": 1.4530235528945923,
|
|
"step": 1270,
|
|
"unclipped_grad_norm": 1.318946361541748
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1189218809836823,
|
|
"learning_rate": 0.0002911662027209272,
|
|
"loss_nats": 1.4687247276306152,
|
|
"step": 1280,
|
|
"unclipped_grad_norm": 1.1660429239273071
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0943455601753866,
|
|
"learning_rate": 0.00029096657601454013,
|
|
"loss_nats": 1.4516897201538086,
|
|
"step": 1290,
|
|
"unclipped_grad_norm": 1.5281683206558228
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0472271293532014,
|
|
"learning_rate": 0.00029076479688692266,
|
|
"loss_nats": 1.419029712677002,
|
|
"step": 1300,
|
|
"unclipped_grad_norm": 1.416712999343872
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0644360573638356,
|
|
"learning_rate": 0.00029056086878592323,
|
|
"loss_nats": 1.4309580326080322,
|
|
"step": 1310,
|
|
"unclipped_grad_norm": 1.2950114011764526
|
|
},
|
|
{
|
|
"bits_per_byte": 2.044286569988749,
|
|
"learning_rate": 0.00029035479519611045,
|
|
"loss_nats": 1.4169914722442627,
|
|
"step": 1320,
|
|
"unclipped_grad_norm": 1.1484317779541016
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0443660259734475,
|
|
"learning_rate": 0.0002901465796387134,
|
|
"loss_nats": 1.4170465469360352,
|
|
"step": 1330,
|
|
"unclipped_grad_norm": 1.206559181213379
|
|
},
|
|
{
|
|
"bits_per_byte": 2.040463051694767,
|
|
"learning_rate": 0.00028993622567156154,
|
|
"loss_nats": 1.4143412113189697,
|
|
"step": 1340,
|
|
"unclipped_grad_norm": 1.1644346714019775
|
|
},
|
|
{
|
|
"bits_per_byte": 2.073596541279392,
|
|
"learning_rate": 0.0002897237368890237,
|
|
"loss_nats": 1.437307596206665,
|
|
"step": 1350,
|
|
"unclipped_grad_norm": 1.295727014541626
|
|
},
|
|
{
|
|
"bits_per_byte": 2.028423750186418,
|
|
"learning_rate": 0.0002895091169219469,
|
|
"loss_nats": 1.4059962034225464,
|
|
"step": 1360,
|
|
"unclipped_grad_norm": 1.2224998474121094
|
|
},
|
|
{
|
|
"bits_per_byte": 2.049744783379182,
|
|
"learning_rate": 0.00028929236943759425,
|
|
"loss_nats": 1.4207748174667358,
|
|
"step": 1370,
|
|
"unclipped_grad_norm": 1.2044527530670166
|
|
},
|
|
{
|
|
"bits_per_byte": 2.01407248590252,
|
|
"learning_rate": 0.00028907349813958225,
|
|
"loss_nats": 1.396048665046692,
|
|
"step": 1380,
|
|
"unclipped_grad_norm": 1.241443157196045
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0615071928196422,
|
|
"learning_rate": 0.0002888525067678175,
|
|
"loss_nats": 1.4289278984069824,
|
|
"step": 1390,
|
|
"unclipped_grad_norm": 1.1473987102508545
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1662895146669676,
|
|
"learning_rate": 0.00028862939909843273,
|
|
"loss_nats": 1.501557469367981,
|
|
"step": 1400,
|
|
"unclipped_grad_norm": 1.1963618993759155
|
|
},
|
|
{
|
|
"bits_per_byte": 2.064517577140345,
|
|
"learning_rate": 0.0002884041789437225,
|
|
"loss_nats": 1.4310145378112793,
|
|
"step": 1410,
|
|
"unclipped_grad_norm": 1.5827157497406006
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0563607839752236,
|
|
"learning_rate": 0.0002881768501520778,
|
|
"loss_nats": 1.4253606796264648,
|
|
"step": 1420,
|
|
"unclipped_grad_norm": 1.2223409414291382
|
|
},
|
|
{
|
|
"bits_per_byte": 1.996439276692207,
|
|
"learning_rate": 0.0002879474166079205,
|
|
"loss_nats": 1.3838262557983398,
|
|
"step": 1430,
|
|
"unclipped_grad_norm": 1.1598049402236938
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0215600945731342,
|
|
"learning_rate": 0.00028771588223163694,
|
|
"loss_nats": 1.4012386798858643,
|
|
"step": 1440,
|
|
"unclipped_grad_norm": 1.1466177701950073
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0019069491284376,
|
|
"learning_rate": 0.00028748225097951054,
|
|
"loss_nats": 1.3876161575317383,
|
|
"step": 1450,
|
|
"unclipped_grad_norm": 1.1313462257385254
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1132414660083345,
|
|
"learning_rate": 0.000287246526843655,
|
|
"loss_nats": 1.4647873640060425,
|
|
"step": 1460,
|
|
"unclipped_grad_norm": 1.1495121717453003
|
|
},
|
|
{
|
|
"bits_per_byte": 2.102186421210873,
|
|
"learning_rate": 0.00028700871385194533,
|
|
"loss_nats": 1.4571245908737183,
|
|
"step": 1470,
|
|
"unclipped_grad_norm": 1.337440013885498
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0400563127254765,
|
|
"learning_rate": 0.00028676881606794945,
|
|
"loss_nats": 1.4140592813491821,
|
|
"step": 1480,
|
|
"unclipped_grad_norm": 1.06326162815094
|
|
},
|
|
{
|
|
"bits_per_byte": 2.057447370363375,
|
|
"learning_rate": 0.00028652683759085854,
|
|
"loss_nats": 1.4261138439178467,
|
|
"step": 1490,
|
|
"unclipped_grad_norm": 1.1691441535949707
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9426222936015662,
|
|
"learning_rate": 0.0002862827825554171,
|
|
"loss_nats": 1.3465231657028198,
|
|
"step": 1500,
|
|
"unclipped_grad_norm": 1.0966376066207886
|
|
},
|
|
{
|
|
"bits_per_byte": 1.953619208263036,
|
|
"learning_rate": 0.00028603665513185245,
|
|
"loss_nats": 1.3541456460952759,
|
|
"step": 1510,
|
|
"unclipped_grad_norm": 1.2253862619400024
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9856435817322469,
|
|
"learning_rate": 0.000285788459525803,
|
|
"loss_nats": 1.3763432502746582,
|
|
"step": 1520,
|
|
"unclipped_grad_norm": 1.1588174104690552
|
|
},
|
|
{
|
|
"bits_per_byte": 2.043245215037775,
|
|
"learning_rate": 0.00028553819997824707,
|
|
"loss_nats": 1.4162696599960327,
|
|
"step": 1530,
|
|
"unclipped_grad_norm": 1.1396243572235107
|
|
},
|
|
{
|
|
"bits_per_byte": 2.030725050037613,
|
|
"learning_rate": 0.00028528588076542964,
|
|
"loss_nats": 1.4075913429260254,
|
|
"step": 1540,
|
|
"unclipped_grad_norm": 1.1333942413330078
|
|
},
|
|
{
|
|
"bits_per_byte": 2.081739919797746,
|
|
"learning_rate": 0.00028503150619878996,
|
|
"loss_nats": 1.4429521560668945,
|
|
"step": 1550,
|
|
"unclipped_grad_norm": 1.1568241119384766
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9638389333252566,
|
|
"learning_rate": 0.0002847750806248876,
|
|
"loss_nats": 1.361229419708252,
|
|
"step": 1560,
|
|
"unclipped_grad_norm": 1.0923500061035156
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9637097743544585,
|
|
"learning_rate": 0.000284516608425328,
|
|
"loss_nats": 1.3611398935317993,
|
|
"step": 1570,
|
|
"unclipped_grad_norm": 1.181723952293396
|
|
},
|
|
{
|
|
"bits_per_byte": 1.984943956308537,
|
|
"learning_rate": 0.000284256094016688,
|
|
"loss_nats": 1.3758583068847656,
|
|
"step": 1580,
|
|
"unclipped_grad_norm": 1.15809965133667
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9967120411764756,
|
|
"learning_rate": 0.0002839935418504399,
|
|
"loss_nats": 1.3840153217315674,
|
|
"step": 1590,
|
|
"unclipped_grad_norm": 1.2961064577102661
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9713934432470563,
|
|
"learning_rate": 0.000283728956412876,
|
|
"loss_nats": 1.3664658069610596,
|
|
"step": 1600,
|
|
"unclipped_grad_norm": 1.1767901182174683
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9459826626167747,
|
|
"learning_rate": 0.00028346234222503136,
|
|
"loss_nats": 1.3488523960113525,
|
|
"step": 1610,
|
|
"unclipped_grad_norm": 1.1538722515106201
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9305324291938633,
|
|
"learning_rate": 0.00028319370384260675,
|
|
"loss_nats": 1.3381431102752686,
|
|
"step": 1620,
|
|
"unclipped_grad_norm": 1.0682140588760376
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9835519287324535,
|
|
"learning_rate": 0.00028292304585589105,
|
|
"loss_nats": 1.3748934268951416,
|
|
"step": 1630,
|
|
"unclipped_grad_norm": 1.1328599452972412
|
|
},
|
|
{
|
|
"bits_per_byte": 1.952673819631243,
|
|
"learning_rate": 0.0002826503728896824,
|
|
"loss_nats": 1.3534903526306152,
|
|
"step": 1640,
|
|
"unclipped_grad_norm": 1.1611915826797485
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9528083100642177,
|
|
"learning_rate": 0.0002823756896032095,
|
|
"loss_nats": 1.353583574295044,
|
|
"step": 1650,
|
|
"unclipped_grad_norm": 1.2886978387832642
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9660846827802232,
|
|
"learning_rate": 0.0002820990006900518,
|
|
"loss_nats": 1.362786054611206,
|
|
"step": 1660,
|
|
"unclipped_grad_norm": 1.1841517686843872
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9119278619714772,
|
|
"learning_rate": 0.00028182031087805947,
|
|
"loss_nats": 1.3252474069595337,
|
|
"step": 1670,
|
|
"unclipped_grad_norm": 1.1872814893722534
|
|
},
|
|
{
|
|
"bits_per_byte": 1.93973676868539,
|
|
"learning_rate": 0.0002815396249292725,
|
|
"loss_nats": 1.3445230722427368,
|
|
"step": 1680,
|
|
"unclipped_grad_norm": 1.1895164251327515
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9710771671521197,
|
|
"learning_rate": 0.0002812569476398391,
|
|
"loss_nats": 1.3662465810775757,
|
|
"step": 1690,
|
|
"unclipped_grad_norm": 1.0209845304489136
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9189868898761413,
|
|
"learning_rate": 0.00028097228383993446,
|
|
"loss_nats": 1.3301403522491455,
|
|
"step": 1700,
|
|
"unclipped_grad_norm": 1.2754039764404297
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9587719804655366,
|
|
"learning_rate": 0.0002806856383936774,
|
|
"loss_nats": 1.3577172756195068,
|
|
"step": 1710,
|
|
"unclipped_grad_norm": 0.9888415336608887
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8892168649944399,
|
|
"learning_rate": 0.00028039701619904774,
|
|
"loss_nats": 1.3095053434371948,
|
|
"step": 1720,
|
|
"unclipped_grad_norm": 1.1411211490631104
|
|
},
|
|
{
|
|
"bits_per_byte": 1.970308060737461,
|
|
"learning_rate": 0.0002801064221878024,
|
|
"loss_nats": 1.3657134771347046,
|
|
"step": 1730,
|
|
"unclipped_grad_norm": 1.1278765201568604
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9812284431192968,
|
|
"learning_rate": 0.00027981386132539134,
|
|
"loss_nats": 1.3732829093933105,
|
|
"step": 1740,
|
|
"unclipped_grad_norm": 1.0859284400939941
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9480999410315467,
|
|
"learning_rate": 0.0002795193386108724,
|
|
"loss_nats": 1.3503199815750122,
|
|
"step": 1750,
|
|
"unclipped_grad_norm": 1.063671588897705
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9687923776354073,
|
|
"learning_rate": 0.00027922285907682625,
|
|
"loss_nats": 1.3646628856658936,
|
|
"step": 1760,
|
|
"unclipped_grad_norm": 1.02903151512146
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9746323925107556,
|
|
"learning_rate": 0.00027892442778927005,
|
|
"loss_nats": 1.3687108755111694,
|
|
"step": 1770,
|
|
"unclipped_grad_norm": 1.1287299394607544
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0187137817013547,
|
|
"learning_rate": 0.00027862404984757116,
|
|
"loss_nats": 1.3992657661437988,
|
|
"step": 1780,
|
|
"unclipped_grad_norm": 1.1014562845230103
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8992608237874788,
|
|
"learning_rate": 0.0002783217303843597,
|
|
"loss_nats": 1.31646728515625,
|
|
"step": 1790,
|
|
"unclipped_grad_norm": 1.2110038995742798
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0498698147663594,
|
|
"learning_rate": 0.00027801747456544134,
|
|
"loss_nats": 1.4208614826202393,
|
|
"step": 1800,
|
|
"unclipped_grad_norm": 1.5255318880081177
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8942502812372417,
|
|
"learning_rate": 0.00027771128758970843,
|
|
"loss_nats": 1.3129942417144775,
|
|
"step": 1810,
|
|
"unclipped_grad_norm": 1.0947346687316895
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9627021280030534,
|
|
"learning_rate": 0.00027740317468905165,
|
|
"loss_nats": 1.3604414463043213,
|
|
"step": 1820,
|
|
"unclipped_grad_norm": 1.1853052377700806
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9780426364947141,
|
|
"learning_rate": 0.0002770931411282704,
|
|
"loss_nats": 1.3710746765136719,
|
|
"step": 1830,
|
|
"unclipped_grad_norm": 1.0857312679290771
|
|
},
|
|
{
|
|
"bits_per_byte": 1.975353859731125,
|
|
"learning_rate": 0.0002767811922049827,
|
|
"loss_nats": 1.369210958480835,
|
|
"step": 1840,
|
|
"unclipped_grad_norm": 1.1014374494552612
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8969483450639775,
|
|
"learning_rate": 0.00027646733324953503,
|
|
"loss_nats": 1.3148643970489502,
|
|
"step": 1850,
|
|
"unclipped_grad_norm": 1.0865846872329712
|
|
},
|
|
{
|
|
"bits_per_byte": 1.943441103002324,
|
|
"learning_rate": 0.00027615156962491097,
|
|
"loss_nats": 1.347090721130371,
|
|
"step": 1860,
|
|
"unclipped_grad_norm": 0.9979931116104126
|
|
},
|
|
{
|
|
"bits_per_byte": 2.018713093770751,
|
|
"learning_rate": 0.0002758339067266396,
|
|
"loss_nats": 1.3992652893066406,
|
|
"step": 1870,
|
|
"unclipped_grad_norm": 1.1247007846832275
|
|
},
|
|
{
|
|
"bits_per_byte": 1.895716605318498,
|
|
"learning_rate": 0.0002755143499827035,
|
|
"loss_nats": 1.3140106201171875,
|
|
"step": 1880,
|
|
"unclipped_grad_norm": 1.2656805515289307
|
|
},
|
|
{
|
|
"bits_per_byte": 1.939341724536358,
|
|
"learning_rate": 0.0002751929048534456,
|
|
"loss_nats": 1.3442492485046387,
|
|
"step": 1890,
|
|
"unclipped_grad_norm": 1.1298930644989014
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9376002282137212,
|
|
"learning_rate": 0.0002748695768314763,
|
|
"loss_nats": 1.3430421352386475,
|
|
"step": 1900,
|
|
"unclipped_grad_norm": 1.1912769079208374
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9616757355427041,
|
|
"learning_rate": 0.00027454437144157947,
|
|
"loss_nats": 1.3597300052642822,
|
|
"step": 1910,
|
|
"unclipped_grad_norm": 1.0634745359420776
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8880833273426032,
|
|
"learning_rate": 0.00027421729424061787,
|
|
"loss_nats": 1.3087196350097656,
|
|
"step": 1920,
|
|
"unclipped_grad_norm": 1.0458215475082397
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9484389188363973,
|
|
"learning_rate": 0.0002738883508174385,
|
|
"loss_nats": 1.3505549430847168,
|
|
"step": 1930,
|
|
"unclipped_grad_norm": 1.0484954118728638
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9728711181832694,
|
|
"learning_rate": 0.0002735575467927768,
|
|
"loss_nats": 1.3674900531768799,
|
|
"step": 1940,
|
|
"unclipped_grad_norm": 1.0497392416000366
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9061248233660644,
|
|
"learning_rate": 0.0002732248878191609,
|
|
"loss_nats": 1.3212250471115112,
|
|
"step": 1950,
|
|
"unclipped_grad_norm": 1.2195848226547241
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8858399856447488,
|
|
"learning_rate": 0.0002728903795808148,
|
|
"loss_nats": 1.3071646690368652,
|
|
"step": 1960,
|
|
"unclipped_grad_norm": 1.0152307748794556
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8880905506139394,
|
|
"learning_rate": 0.0002725540277935614,
|
|
"loss_nats": 1.3087246417999268,
|
|
"step": 1970,
|
|
"unclipped_grad_norm": 1.1332975625991821
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9225495104887642,
|
|
"learning_rate": 0.00027221583820472456,
|
|
"loss_nats": 1.33260977268219,
|
|
"step": 1980,
|
|
"unclipped_grad_norm": 1.1374293565750122
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8947043154355196,
|
|
"learning_rate": 0.0002718758165930314,
|
|
"loss_nats": 1.3133089542388916,
|
|
"step": 1990,
|
|
"unclipped_grad_norm": 1.0456688404083252
|
|
},
|
|
{
|
|
"bits_per_byte": 1.977381019236845,
|
|
"learning_rate": 0.0002715339687685131,
|
|
"loss_nats": 1.37061607837677,
|
|
"step": 2000,
|
|
"unclipped_grad_norm": 1.1185020208358765
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9636655748131868,
|
|
"learning_rate": 0.00027119030057240576,
|
|
"loss_nats": 1.3611092567443848,
|
|
"step": 2010,
|
|
"unclipped_grad_norm": 1.1275893449783325
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8762688071589317,
|
|
"learning_rate": 0.0002708448178770506,
|
|
"loss_nats": 1.3005304336547852,
|
|
"step": 2020,
|
|
"unclipped_grad_norm": 1.0362930297851562
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8977705941177525,
|
|
"learning_rate": 0.00027049752658579376,
|
|
"loss_nats": 1.3154343366622925,
|
|
"step": 2030,
|
|
"unclipped_grad_norm": 1.0457504987716675
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8942691993288368,
|
|
"learning_rate": 0.00027014843263288514,
|
|
"loss_nats": 1.3130073547363281,
|
|
"step": 2040,
|
|
"unclipped_grad_norm": 0.9790334105491638
|
|
},
|
|
{
|
|
"bits_per_byte": 1.854481013051726,
|
|
"learning_rate": 0.00026979754198337724,
|
|
"loss_nats": 1.2854282855987549,
|
|
"step": 2050,
|
|
"unclipped_grad_norm": 1.067064881324768
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8548413167052835,
|
|
"learning_rate": 0.00026944486063302305,
|
|
"loss_nats": 1.2856780290603638,
|
|
"step": 2060,
|
|
"unclipped_grad_norm": 1.0005096197128296
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9705749778116002,
|
|
"learning_rate": 0.0002690903946081738,
|
|
"loss_nats": 1.3658984899520874,
|
|
"step": 2070,
|
|
"unclipped_grad_norm": 1.1014602184295654
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9435265783798028,
|
|
"learning_rate": 0.00026873414996567596,
|
|
"loss_nats": 1.3471499681472778,
|
|
"step": 2080,
|
|
"unclipped_grad_norm": 1.1029977798461914
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8586646630166146,
|
|
"learning_rate": 0.00026837613279276736,
|
|
"loss_nats": 1.2883281707763672,
|
|
"step": 2090,
|
|
"unclipped_grad_norm": 1.0793615579605103
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8191048696998495,
|
|
"learning_rate": 0.0002680163492069739,
|
|
"loss_nats": 1.2609074115753174,
|
|
"step": 2100,
|
|
"unclipped_grad_norm": 1.0433158874511719
|
|
},
|
|
{
|
|
"bits_per_byte": 1.851254274556238,
|
|
"learning_rate": 0.00026765480535600414,
|
|
"loss_nats": 1.2831916809082031,
|
|
"step": 2110,
|
|
"unclipped_grad_norm": 1.1101045608520508
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8581610978148881,
|
|
"learning_rate": 0.0002672915074176452,
|
|
"loss_nats": 1.2879791259765625,
|
|
"step": 2120,
|
|
"unclipped_grad_norm": 1.1086524724960327
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9049420986760808,
|
|
"learning_rate": 0.00026692646159965624,
|
|
"loss_nats": 1.3204052448272705,
|
|
"step": 2130,
|
|
"unclipped_grad_norm": 1.0729707479476929
|
|
},
|
|
{
|
|
"bits_per_byte": 1.941282376768694,
|
|
"learning_rate": 0.0002665596741396633,
|
|
"loss_nats": 1.3455944061279297,
|
|
"step": 2140,
|
|
"unclipped_grad_norm": 1.0528757572174072
|
|
},
|
|
{
|
|
"bits_per_byte": 1.843757550805128,
|
|
"learning_rate": 0.000266191151305052,
|
|
"loss_nats": 1.2779953479766846,
|
|
"step": 2150,
|
|
"unclipped_grad_norm": 1.1246051788330078
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9154739722496175,
|
|
"learning_rate": 0.00026582089939286075,
|
|
"loss_nats": 1.3277053833007812,
|
|
"step": 2160,
|
|
"unclipped_grad_norm": 1.0607515573501587
|
|
},
|
|
{
|
|
"bits_per_byte": 1.850249895875199,
|
|
"learning_rate": 0.0002654489247296732,
|
|
"loss_nats": 1.2824954986572266,
|
|
"step": 2170,
|
|
"unclipped_grad_norm": 1.0301073789596558
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9307078514977434,
|
|
"learning_rate": 0.00026507523367151007,
|
|
"loss_nats": 1.3382647037506104,
|
|
"step": 2180,
|
|
"unclipped_grad_norm": 1.0744229555130005
|
|
},
|
|
{
|
|
"bits_per_byte": 1.895962540509232,
|
|
"learning_rate": 0.00026469983260372037,
|
|
"loss_nats": 1.3141810894012451,
|
|
"step": 2190,
|
|
"unclipped_grad_norm": 1.1302496194839478
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8514286649642127,
|
|
"learning_rate": 0.0002643227279408727,
|
|
"loss_nats": 1.2833125591278076,
|
|
"step": 2200,
|
|
"unclipped_grad_norm": 1.043583869934082
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8687825743495246,
|
|
"learning_rate": 0.0002639439261266453,
|
|
"loss_nats": 1.2953413724899292,
|
|
"step": 2210,
|
|
"unclipped_grad_norm": 1.1252996921539307
|
|
},
|
|
{
|
|
"bits_per_byte": 1.862911258631719,
|
|
"learning_rate": 0.000263563433633716,
|
|
"loss_nats": 1.291271686553955,
|
|
"step": 2220,
|
|
"unclipped_grad_norm": 1.0556633472442627
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8208848901362797,
|
|
"learning_rate": 0.00026318125696365156,
|
|
"loss_nats": 1.262141227722168,
|
|
"step": 2230,
|
|
"unclipped_grad_norm": 1.0693137645721436
|
|
},
|
|
{
|
|
"bits_per_byte": 1.90406292336487,
|
|
"learning_rate": 0.000262797402646797,
|
|
"loss_nats": 1.319795846939087,
|
|
"step": 2240,
|
|
"unclipped_grad_norm": 1.0752713680267334
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8964874315596651,
|
|
"learning_rate": 0.00026241187724216334,
|
|
"loss_nats": 1.314544916152954,
|
|
"step": 2250,
|
|
"unclipped_grad_norm": 1.0330802202224731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.935825367256817,
|
|
"learning_rate": 0.0002620246873373162,
|
|
"loss_nats": 1.3418118953704834,
|
|
"step": 2260,
|
|
"unclipped_grad_norm": 1.078423023223877
|
|
},
|
|
{
|
|
"bits_per_byte": 1.832309869615747,
|
|
"learning_rate": 0.00026163583954826264,
|
|
"loss_nats": 1.270060420036316,
|
|
"step": 2270,
|
|
"unclipped_grad_norm": 1.056177020072937
|
|
},
|
|
{
|
|
"bits_per_byte": 1.887026837918353,
|
|
"learning_rate": 0.0002612453405193386,
|
|
"loss_nats": 1.3079873323440552,
|
|
"step": 2280,
|
|
"unclipped_grad_norm": 1.0121575593948364
|
|
},
|
|
{
|
|
"bits_per_byte": 1.834445722156812,
|
|
"learning_rate": 0.000260853196923095,
|
|
"loss_nats": 1.271540880203247,
|
|
"step": 2290,
|
|
"unclipped_grad_norm": 1.0369646549224854
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8213642057842343,
|
|
"learning_rate": 0.0002604594154601839,
|
|
"loss_nats": 1.262473464012146,
|
|
"step": 2300,
|
|
"unclipped_grad_norm": 0.9950895309448242
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8373876573824712,
|
|
"learning_rate": 0.0002600640028592441,
|
|
"loss_nats": 1.2735800743103027,
|
|
"step": 2310,
|
|
"unclipped_grad_norm": 1.0178841352462769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8979250345382275,
|
|
"learning_rate": 0.0002596669658767859,
|
|
"loss_nats": 1.315541386604309,
|
|
"step": 2320,
|
|
"unclipped_grad_norm": 0.9894719123840332
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8294843666447218,
|
|
"learning_rate": 0.000259268311297076,
|
|
"loss_nats": 1.2681019306182861,
|
|
"step": 2330,
|
|
"unclipped_grad_norm": 0.9816614389419556
|
|
},
|
|
{
|
|
"bits_per_byte": 1.848164090285535,
|
|
"learning_rate": 0.00025886804593202114,
|
|
"loss_nats": 1.2810497283935547,
|
|
"step": 2340,
|
|
"unclipped_grad_norm": 1.0238531827926636
|
|
},
|
|
{
|
|
"bits_per_byte": 1.905264050198496,
|
|
"learning_rate": 0.00025846617662105193,
|
|
"loss_nats": 1.3206284046173096,
|
|
"step": 2350,
|
|
"unclipped_grad_norm": 1.0559433698654175
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8307938425483912,
|
|
"learning_rate": 0.00025806271023100637,
|
|
"loss_nats": 1.2690095901489258,
|
|
"step": 2360,
|
|
"unclipped_grad_norm": 1.0712460279464722
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7703326537069675,
|
|
"learning_rate": 0.00025765765365601165,
|
|
"loss_nats": 1.2271010875701904,
|
|
"step": 2370,
|
|
"unclipped_grad_norm": 1.1017504930496216
|
|
},
|
|
{
|
|
"bits_per_byte": 1.886526368404342,
|
|
"learning_rate": 0.00025725101381736707,
|
|
"loss_nats": 1.3076404333114624,
|
|
"step": 2380,
|
|
"unclipped_grad_norm": 1.1038753986358643
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8630619154338748,
|
|
"learning_rate": 0.00025684279766342567,
|
|
"loss_nats": 1.2913761138916016,
|
|
"step": 2390,
|
|
"unclipped_grad_norm": 1.0329313278198242
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9135943738583379,
|
|
"learning_rate": 0.00025643301216947504,
|
|
"loss_nats": 1.3264025449752808,
|
|
"step": 2400,
|
|
"unclipped_grad_norm": 1.1860445737838745
|
|
},
|
|
{
|
|
"bits_per_byte": 1.835037686441082,
|
|
"learning_rate": 0.00025602166433761866,
|
|
"loss_nats": 1.2719511985778809,
|
|
"step": 2410,
|
|
"unclipped_grad_norm": 1.153725266456604
|
|
},
|
|
{
|
|
"bits_per_byte": 1.871177432762789,
|
|
"learning_rate": 0.0002556087611966561,
|
|
"loss_nats": 1.2970013618469238,
|
|
"step": 2420,
|
|
"unclipped_grad_norm": 1.0672882795333862
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8586395535495885,
|
|
"learning_rate": 0.00025519430980196277,
|
|
"loss_nats": 1.2883107662200928,
|
|
"step": 2430,
|
|
"unclipped_grad_norm": 1.0162642002105713
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8218378460047104,
|
|
"learning_rate": 0.0002547783172353695,
|
|
"loss_nats": 1.2628017663955688,
|
|
"step": 2440,
|
|
"unclipped_grad_norm": 1.065905213356018
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9384147380482075,
|
|
"learning_rate": 0.0002543607906050415,
|
|
"loss_nats": 1.34360671043396,
|
|
"step": 2450,
|
|
"unclipped_grad_norm": 1.218255639076233
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8158471743272062,
|
|
"learning_rate": 0.0002539417370453568,
|
|
"loss_nats": 1.2586493492126465,
|
|
"step": 2460,
|
|
"unclipped_grad_norm": 1.0468804836273193
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8206396428761493,
|
|
"learning_rate": 0.00025352116371678464,
|
|
"loss_nats": 1.2619712352752686,
|
|
"step": 2470,
|
|
"unclipped_grad_norm": 1.1381754875183105
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8851444878046595,
|
|
"learning_rate": 0.0002530990778057627,
|
|
"loss_nats": 1.3066825866699219,
|
|
"step": 2480,
|
|
"unclipped_grad_norm": 1.035730242729187
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8941151028736636,
|
|
"learning_rate": 0.00025267548652457475,
|
|
"loss_nats": 1.3129005432128906,
|
|
"step": 2490,
|
|
"unclipped_grad_norm": 1.0220366716384888
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8318142156159603,
|
|
"learning_rate": 0.00025225039711122696,
|
|
"loss_nats": 1.2697168588638306,
|
|
"step": 2500,
|
|
"unclipped_grad_norm": 0.982593834400177
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8129355080480989,
|
|
"learning_rate": 0.00025182381682932464,
|
|
"loss_nats": 1.2566311359405518,
|
|
"step": 2510,
|
|
"unclipped_grad_norm": 1.091998815536499
|
|
},
|
|
{
|
|
"bits_per_byte": 1.841321932503609,
|
|
"learning_rate": 0.0002513957529679477,
|
|
"loss_nats": 1.2763071060180664,
|
|
"step": 2520,
|
|
"unclipped_grad_norm": 1.000153660774231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.848013777448681,
|
|
"learning_rate": 0.00025096621284152656,
|
|
"loss_nats": 1.2809455394744873,
|
|
"step": 2530,
|
|
"unclipped_grad_norm": 1.0528815984725952
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8343394368785788,
|
|
"learning_rate": 0.0002505352037897168,
|
|
"loss_nats": 1.2714672088623047,
|
|
"step": 2540,
|
|
"unclipped_grad_norm": 0.9793087840080261
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7887870800751502,
|
|
"learning_rate": 0.0002501027331772739,
|
|
"loss_nats": 1.2398927211761475,
|
|
"step": 2550,
|
|
"unclipped_grad_norm": 1.0650514364242554
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7584782335482958,
|
|
"learning_rate": 0.00024966880839392746,
|
|
"loss_nats": 1.2188842296600342,
|
|
"step": 2560,
|
|
"unclipped_grad_norm": 0.9875280857086182
|
|
},
|
|
{
|
|
"bits_per_byte": 1.81654955147333,
|
|
"learning_rate": 0.0002492334368542548,
|
|
"loss_nats": 1.2591361999511719,
|
|
"step": 2570,
|
|
"unclipped_grad_norm": 1.0060385465621948
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7945154782100887,
|
|
"learning_rate": 0.00024879662599755416,
|
|
"loss_nats": 1.2438633441925049,
|
|
"step": 2580,
|
|
"unclipped_grad_norm": 1.0824097394943237
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7956424805211926,
|
|
"learning_rate": 0.00024835838328771803,
|
|
"loss_nats": 1.2446445226669312,
|
|
"step": 2590,
|
|
"unclipped_grad_norm": 0.9988609552383423
|
|
},
|
|
{
|
|
"bits_per_byte": 1.815006351147138,
|
|
"learning_rate": 0.0002479187162131051,
|
|
"loss_nats": 1.2580665349960327,
|
|
"step": 2600,
|
|
"unclipped_grad_norm": 0.9940101504325867
|
|
},
|
|
{
|
|
"bits_per_byte": 1.840550934279791,
|
|
"learning_rate": 0.00024747763228641264,
|
|
"loss_nats": 1.2757726907730103,
|
|
"step": 2610,
|
|
"unclipped_grad_norm": 1.0804052352905273
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7607050649116673,
|
|
"learning_rate": 0.00024703513904454795,
|
|
"loss_nats": 1.2204277515411377,
|
|
"step": 2620,
|
|
"unclipped_grad_norm": 1.0194178819656372
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7479795526263755,
|
|
"learning_rate": 0.00024659124404849956,
|
|
"loss_nats": 1.2116070985794067,
|
|
"step": 2630,
|
|
"unclipped_grad_norm": 0.9564489126205444
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7532458333784458,
|
|
"learning_rate": 0.00024614595488320833,
|
|
"loss_nats": 1.2152574062347412,
|
|
"step": 2640,
|
|
"unclipped_grad_norm": 0.9848498106002808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.813315245741204,
|
|
"learning_rate": 0.00024569927915743745,
|
|
"loss_nats": 1.2568943500518799,
|
|
"step": 2650,
|
|
"unclipped_grad_norm": 0.9513642191886902
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7231679595517,
|
|
"learning_rate": 0.0002452512245036427,
|
|
"loss_nats": 1.1944090127944946,
|
|
"step": 2660,
|
|
"unclipped_grad_norm": 1.0000529289245605
|
|
},
|
|
{
|
|
"bits_per_byte": 1.802969457395901,
|
|
"learning_rate": 0.00024480179857784177,
|
|
"loss_nats": 1.249723196029663,
|
|
"step": 2670,
|
|
"unclipped_grad_norm": 0.9922699928283691
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9163316497294702,
|
|
"learning_rate": 0.00024435100905948387,
|
|
"loss_nats": 1.328299880027771,
|
|
"step": 2680,
|
|
"unclipped_grad_norm": 1.192367672920227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8243539521868333,
|
|
"learning_rate": 0.00024389886365131793,
|
|
"loss_nats": 1.2645457983016968,
|
|
"step": 2690,
|
|
"unclipped_grad_norm": 1.0074772834777832
|
|
},
|
|
{
|
|
"bits_per_byte": 1.751833683832211,
|
|
"learning_rate": 0.00024344537007926163,
|
|
"loss_nats": 1.2142785787582397,
|
|
"step": 2700,
|
|
"unclipped_grad_norm": 0.9746342301368713
|
|
},
|
|
{
|
|
"bits_per_byte": 1.826312318632208,
|
|
"learning_rate": 0.00024299053609226878,
|
|
"loss_nats": 1.2659032344818115,
|
|
"step": 2710,
|
|
"unclipped_grad_norm": 1.0221014022827148
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8721807795479226,
|
|
"learning_rate": 0.00024253436946219733,
|
|
"loss_nats": 1.297696828842163,
|
|
"step": 2720,
|
|
"unclipped_grad_norm": 1.0561023950576782
|
|
},
|
|
{
|
|
"bits_per_byte": 1.742347808741222,
|
|
"learning_rate": 0.0002420768779836764,
|
|
"loss_nats": 1.2077034711837769,
|
|
"step": 2730,
|
|
"unclipped_grad_norm": 0.946500301361084
|
|
},
|
|
{
|
|
"bits_per_byte": 1.826287381147833,
|
|
"learning_rate": 0.00024161806947397305,
|
|
"loss_nats": 1.2658859491348267,
|
|
"step": 2740,
|
|
"unclipped_grad_norm": 1.0504826307296753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7872662374935704,
|
|
"learning_rate": 0.00024115795177285886,
|
|
"loss_nats": 1.23883855342865,
|
|
"step": 2750,
|
|
"unclipped_grad_norm": 1.0072195529937744
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9127833036768689,
|
|
"learning_rate": 0.0002406965327424758,
|
|
"loss_nats": 1.3258403539657593,
|
|
"step": 2760,
|
|
"unclipped_grad_norm": 1.1406866312026978
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8309943743192973,
|
|
"learning_rate": 0.00024023382026720197,
|
|
"loss_nats": 1.269148588180542,
|
|
"step": 2770,
|
|
"unclipped_grad_norm": 1.0432147979736328
|
|
},
|
|
{
|
|
"bits_per_byte": 1.835324725485372,
|
|
"learning_rate": 0.00023976982225351698,
|
|
"loss_nats": 1.2721501588821411,
|
|
"step": 2780,
|
|
"unclipped_grad_norm": 1.0946613550186157
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8014186898330709,
|
|
"learning_rate": 0.0002393045466298666,
|
|
"loss_nats": 1.2486482858657837,
|
|
"step": 2790,
|
|
"unclipped_grad_norm": 1.0307797193527222
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8122052697125355,
|
|
"learning_rate": 0.00023883800134652763,
|
|
"loss_nats": 1.2561249732971191,
|
|
"step": 2800,
|
|
"unclipped_grad_norm": 1.0519623756408691
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7642310532196568,
|
|
"learning_rate": 0.0002383701943754715,
|
|
"loss_nats": 1.2228717803955078,
|
|
"step": 2810,
|
|
"unclipped_grad_norm": 1.0939170122146606
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8221561858914574,
|
|
"learning_rate": 0.00023790113371022884,
|
|
"loss_nats": 1.2630224227905273,
|
|
"step": 2820,
|
|
"unclipped_grad_norm": 1.006143569946289
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7646739085456284,
|
|
"learning_rate": 0.0002374308273657521,
|
|
"loss_nats": 1.223178744316101,
|
|
"step": 2830,
|
|
"unclipped_grad_norm": 0.9335127472877502
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8147093371090979,
|
|
"learning_rate": 0.00023695928337827911,
|
|
"loss_nats": 1.2578606605529785,
|
|
"step": 2840,
|
|
"unclipped_grad_norm": 0.9985573887825012
|
|
},
|
|
{
|
|
"bits_per_byte": 1.758499215431701,
|
|
"learning_rate": 0.00023648650980519564,
|
|
"loss_nats": 1.2188987731933594,
|
|
"step": 2850,
|
|
"unclipped_grad_norm": 0.9980565905570984
|
|
},
|
|
{
|
|
"bits_per_byte": 1.783597331602714,
|
|
"learning_rate": 0.00023601251472489757,
|
|
"loss_nats": 1.236295461654663,
|
|
"step": 2860,
|
|
"unclipped_grad_norm": 1.0390338897705078
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7758223399225082,
|
|
"learning_rate": 0.00023553730623665305,
|
|
"loss_nats": 1.2309062480926514,
|
|
"step": 2870,
|
|
"unclipped_grad_norm": 1.0219858884811401
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7873183482367818,
|
|
"learning_rate": 0.00023506089246046403,
|
|
"loss_nats": 1.2388746738433838,
|
|
"step": 2880,
|
|
"unclipped_grad_norm": 1.002677083015442
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7586128959639213,
|
|
"learning_rate": 0.00023458328153692736,
|
|
"loss_nats": 1.2189775705337524,
|
|
"step": 2890,
|
|
"unclipped_grad_norm": 1.0140609741210938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7806794739481757,
|
|
"learning_rate": 0.0002341044816270961,
|
|
"loss_nats": 1.2342729568481445,
|
|
"step": 2900,
|
|
"unclipped_grad_norm": 0.9781115651130676
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7540770255300655,
|
|
"learning_rate": 0.00023362450091233953,
|
|
"loss_nats": 1.2158335447311401,
|
|
"step": 2910,
|
|
"unclipped_grad_norm": 0.9601871967315674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.798447689539416,
|
|
"learning_rate": 0.00023314334759420388,
|
|
"loss_nats": 1.246588945388794,
|
|
"step": 2920,
|
|
"unclipped_grad_norm": 1.0120164155960083
|
|
},
|
|
{
|
|
"bits_per_byte": 1.749404256906123,
|
|
"learning_rate": 0.0002326610298942717,
|
|
"loss_nats": 1.2125946283340454,
|
|
"step": 2930,
|
|
"unclipped_grad_norm": 0.9387297630310059
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7688945347804526,
|
|
"learning_rate": 0.00023217755605402175,
|
|
"loss_nats": 1.2261042594909668,
|
|
"step": 2940,
|
|
"unclipped_grad_norm": 1.073227047920227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8185380148826058,
|
|
"learning_rate": 0.00023169293433468802,
|
|
"loss_nats": 1.260514497756958,
|
|
"step": 2950,
|
|
"unclipped_grad_norm": 1.033491849899292
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8193900169349801,
|
|
"learning_rate": 0.00023120717301711857,
|
|
"loss_nats": 1.2611050605773926,
|
|
"step": 2960,
|
|
"unclipped_grad_norm": 1.0577077865600586
|
|
},
|
|
{
|
|
"bits_per_byte": 1.799499191466791,
|
|
"learning_rate": 0.00023072028040163408,
|
|
"loss_nats": 1.2473177909851074,
|
|
"step": 2970,
|
|
"unclipped_grad_norm": 1.0431829690933228
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8318936716006589,
|
|
"learning_rate": 0.00023023226480788593,
|
|
"loss_nats": 1.269771933555603,
|
|
"step": 2980,
|
|
"unclipped_grad_norm": 1.3554320335388184
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7557352822496848,
|
|
"learning_rate": 0.00022974313457471416,
|
|
"loss_nats": 1.2169829607009888,
|
|
"step": 2990,
|
|
"unclipped_grad_norm": 1.0419063568115234
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8299745171996809,
|
|
"learning_rate": 0.0002292528980600049,
|
|
"loss_nats": 1.2684416770935059,
|
|
"step": 3000,
|
|
"unclipped_grad_norm": 1.0811892747879028
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7247794369902845,
|
|
"learning_rate": 0.00022876156364054758,
|
|
"loss_nats": 1.1955260038375854,
|
|
"step": 3010,
|
|
"unclipped_grad_norm": 1.0030642747879028
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7923558920632043,
|
|
"learning_rate": 0.0002282691397118918,
|
|
"loss_nats": 1.2423664331436157,
|
|
"step": 3020,
|
|
"unclipped_grad_norm": 1.002471685409546
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7403096423458468,
|
|
"learning_rate": 0.00022777563468820376,
|
|
"loss_nats": 1.2062907218933105,
|
|
"step": 3030,
|
|
"unclipped_grad_norm": 1.0230085849761963
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7392983843587735,
|
|
"learning_rate": 0.00022728105700212282,
|
|
"loss_nats": 1.205589771270752,
|
|
"step": 3040,
|
|
"unclipped_grad_norm": 1.0195822715759277
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8747281865725025,
|
|
"learning_rate": 0.00022678541510461698,
|
|
"loss_nats": 1.2994625568389893,
|
|
"step": 3050,
|
|
"unclipped_grad_norm": 1.0621304512023926
|
|
},
|
|
{
|
|
"bits_per_byte": 1.80563828417199,
|
|
"learning_rate": 0.00022628871746483876,
|
|
"loss_nats": 1.251573085784912,
|
|
"step": 3060,
|
|
"unclipped_grad_norm": 1.0812959671020508
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7309672007856773,
|
|
"learning_rate": 0.0002257909725699804,
|
|
"loss_nats": 1.199815034866333,
|
|
"step": 3070,
|
|
"unclipped_grad_norm": 1.048651099205017
|
|
},
|
|
{
|
|
"bits_per_byte": 1.80954315025983,
|
|
"learning_rate": 0.00022529218892512886,
|
|
"loss_nats": 1.2542797327041626,
|
|
"step": 3080,
|
|
"unclipped_grad_norm": 1.0254682302474976
|
|
},
|
|
{
|
|
"bits_per_byte": 1.787755528135275,
|
|
"learning_rate": 0.00022479237505312034,
|
|
"loss_nats": 1.2391777038574219,
|
|
"step": 3090,
|
|
"unclipped_grad_norm": 1.0376405715942383
|
|
},
|
|
{
|
|
"bits_per_byte": 1.748834134418513,
|
|
"learning_rate": 0.00022429153949439496,
|
|
"loss_nats": 1.2121994495391846,
|
|
"step": 3100,
|
|
"unclipped_grad_norm": 1.0169665813446045
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8895020122295705,
|
|
"learning_rate": 0.00022378969080685053,
|
|
"loss_nats": 1.30970299243927,
|
|
"step": 3110,
|
|
"unclipped_grad_norm": 1.009265661239624
|
|
},
|
|
{
|
|
"bits_per_byte": 1.772648744066137,
|
|
"learning_rate": 0.0002232868375656965,
|
|
"loss_nats": 1.2287064790725708,
|
|
"step": 3120,
|
|
"unclipped_grad_norm": 1.0459308624267578
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7184131552032957,
|
|
"learning_rate": 0.00022278298836330733,
|
|
"loss_nats": 1.1911132335662842,
|
|
"step": 3130,
|
|
"unclipped_grad_norm": 0.9758033752441406
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7503415623533254,
|
|
"learning_rate": 0.00022227815180907572,
|
|
"loss_nats": 1.2132443189620972,
|
|
"step": 3140,
|
|
"unclipped_grad_norm": 1.0032713413238525
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7368366246943234,
|
|
"learning_rate": 0.00022177233652926546,
|
|
"loss_nats": 1.203883409500122,
|
|
"step": 3150,
|
|
"unclipped_grad_norm": 0.9544663429260254
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7448823170669872,
|
|
"learning_rate": 0.00022126555116686416,
|
|
"loss_nats": 1.2094602584838867,
|
|
"step": 3160,
|
|
"unclipped_grad_norm": 0.9832491278648376
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8183364512157945,
|
|
"learning_rate": 0.00022075780438143532,
|
|
"loss_nats": 1.2603747844696045,
|
|
"step": 3170,
|
|
"unclipped_grad_norm": 1.080575942993164
|
|
},
|
|
{
|
|
"bits_per_byte": 1.796097030667423,
|
|
"learning_rate": 0.00022024910484897065,
|
|
"loss_nats": 1.2449595928192139,
|
|
"step": 3180,
|
|
"unclipped_grad_norm": 1.0869723558425903
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7798367589589479,
|
|
"learning_rate": 0.00021973946126174158,
|
|
"loss_nats": 1.2336888313293457,
|
|
"step": 3190,
|
|
"unclipped_grad_norm": 1.0730317831039429
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7660776309419706,
|
|
"learning_rate": 0.00021922888232815086,
|
|
"loss_nats": 1.2241517305374146,
|
|
"step": 3200,
|
|
"unclipped_grad_norm": 1.0053741931915283
|
|
},
|
|
{
|
|
"bits_per_byte": 1.736418362887425,
|
|
"learning_rate": 0.0002187173767725837,
|
|
"loss_nats": 1.2035934925079346,
|
|
"step": 3210,
|
|
"unclipped_grad_norm": 0.9884828925132751
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7094793444215763,
|
|
"learning_rate": 0.0002182049533352588,
|
|
"loss_nats": 1.1849207878112793,
|
|
"step": 3220,
|
|
"unclipped_grad_norm": 0.9791789054870605
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7767474346014993,
|
|
"learning_rate": 0.00021769162077207884,
|
|
"loss_nats": 1.231547474861145,
|
|
"step": 3230,
|
|
"unclipped_grad_norm": 1.0538209676742554
|
|
},
|
|
{
|
|
"bits_per_byte": 1.783878351254224,
|
|
"learning_rate": 0.00021717738785448102,
|
|
"loss_nats": 1.236490249633789,
|
|
"step": 3240,
|
|
"unclipped_grad_norm": 1.0962731838226318
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7619694813608107,
|
|
"learning_rate": 0.00021666226336928708,
|
|
"loss_nats": 1.221304178237915,
|
|
"step": 3250,
|
|
"unclipped_grad_norm": 0.9704980850219727
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7647327266122235,
|
|
"learning_rate": 0.0002161462561185532,
|
|
"loss_nats": 1.2232195138931274,
|
|
"step": 3260,
|
|
"unclipped_grad_norm": 0.9566147923469543
|
|
},
|
|
{
|
|
"bits_per_byte": 1.793783004100064,
|
|
"learning_rate": 0.00021562937491941957,
|
|
"loss_nats": 1.243355631828308,
|
|
"step": 3270,
|
|
"unclipped_grad_norm": 1.0367501974105835
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7882781834112471,
|
|
"learning_rate": 0.00021511162860395975,
|
|
"loss_nats": 1.2395399808883667,
|
|
"step": 3280,
|
|
"unclipped_grad_norm": 1.1419988870620728
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7570841421804018,
|
|
"learning_rate": 0.0002145930260190298,
|
|
"loss_nats": 1.2179179191589355,
|
|
"step": 3290,
|
|
"unclipped_grad_norm": 1.0371057987213135
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7704131415875712,
|
|
"learning_rate": 0.00021407357602611698,
|
|
"loss_nats": 1.2271568775177002,
|
|
"step": 3300,
|
|
"unclipped_grad_norm": 1.0890001058578491
|
|
},
|
|
{
|
|
"bits_per_byte": 1.73421681297373,
|
|
"learning_rate": 0.0002135532875011885,
|
|
"loss_nats": 1.202067494392395,
|
|
"step": 3310,
|
|
"unclipped_grad_norm": 1.0024704933166504
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7556792159055035,
|
|
"learning_rate": 0.0002130321693345397,
|
|
"loss_nats": 1.2169440984725952,
|
|
"step": 3320,
|
|
"unclipped_grad_norm": 1.0351438522338867
|
|
},
|
|
{
|
|
"bits_per_byte": 1.751791376100099,
|
|
"learning_rate": 0.00021251023043064224,
|
|
"loss_nats": 1.2142492532730103,
|
|
"step": 3330,
|
|
"unclipped_grad_norm": 0.967126727104187
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7023927992927743,
|
|
"learning_rate": 0.00021198747970799188,
|
|
"loss_nats": 1.1800087690353394,
|
|
"step": 3340,
|
|
"unclipped_grad_norm": 0.9899824857711792
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7427576433982281,
|
|
"learning_rate": 0.00021146392609895618,
|
|
"loss_nats": 1.2079875469207764,
|
|
"step": 3350,
|
|
"unclipped_grad_norm": 1.044270396232605
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7489617455454531,
|
|
"learning_rate": 0.00021093957854962178,
|
|
"loss_nats": 1.2122879028320312,
|
|
"step": 3360,
|
|
"unclipped_grad_norm": 1.0146241188049316
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6929069242017853,
|
|
"learning_rate": 0.00021041444601964157,
|
|
"loss_nats": 1.1734336614608765,
|
|
"step": 3370,
|
|
"unclipped_grad_norm": 0.9934349656105042
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7374774320514381,
|
|
"learning_rate": 0.00020988853748208164,
|
|
"loss_nats": 1.2043275833129883,
|
|
"step": 3380,
|
|
"unclipped_grad_norm": 1.0039739608764648
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7582206035373031,
|
|
"learning_rate": 0.00020936186192326782,
|
|
"loss_nats": 1.218705654144287,
|
|
"step": 3390,
|
|
"unclipped_grad_norm": 1.007129430770874
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7465608677394082,
|
|
"learning_rate": 0.00020883442834263225,
|
|
"loss_nats": 1.2106237411499023,
|
|
"step": 3400,
|
|
"unclipped_grad_norm": 0.9504732489585876
|
|
},
|
|
{
|
|
"bits_per_byte": 1.744059208099958,
|
|
"learning_rate": 0.0002083062457525596,
|
|
"loss_nats": 1.2088897228240967,
|
|
"step": 3410,
|
|
"unclipped_grad_norm": 1.002007007598877
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7186439559207538,
|
|
"learning_rate": 0.00020777732317823306,
|
|
"loss_nats": 1.1912732124328613,
|
|
"step": 3420,
|
|
"unclipped_grad_norm": 0.9931132197380066
|
|
},
|
|
{
|
|
"bits_per_byte": 1.762099672227514,
|
|
"learning_rate": 0.00020724766965748005,
|
|
"loss_nats": 1.221394419670105,
|
|
"step": 3430,
|
|
"unclipped_grad_norm": 0.943431556224823
|
|
},
|
|
{
|
|
"bits_per_byte": 1.763225298677411,
|
|
"learning_rate": 0.00020671729424061788,
|
|
"loss_nats": 1.2221746444702148,
|
|
"step": 3440,
|
|
"unclipped_grad_norm": 0.9422610402107239
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8310394337738234,
|
|
"learning_rate": 0.00020618620599029906,
|
|
"loss_nats": 1.2691798210144043,
|
|
"step": 3450,
|
|
"unclipped_grad_norm": 1.1188790798187256
|
|
},
|
|
{
|
|
"bits_per_byte": 1.822499291279929,
|
|
"learning_rate": 0.00020565441398135649,
|
|
"loss_nats": 1.2632602453231812,
|
|
"step": 3460,
|
|
"unclipped_grad_norm": 1.14117431640625
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7955301758501792,
|
|
"learning_rate": 0.00020512192730064827,
|
|
"loss_nats": 1.2445666790008545,
|
|
"step": 3470,
|
|
"unclipped_grad_norm": 1.0871022939682007
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7607453088519693,
|
|
"learning_rate": 0.00020458875504690273,
|
|
"loss_nats": 1.2204556465148926,
|
|
"step": 3480,
|
|
"unclipped_grad_norm": 1.0999677181243896
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6814413568167144,
|
|
"learning_rate": 0.00020405490633056245,
|
|
"loss_nats": 1.1654863357543945,
|
|
"step": 3490,
|
|
"unclipped_grad_norm": 1.0303022861480713
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6926489502254911,
|
|
"learning_rate": 0.00020352039027362917,
|
|
"loss_nats": 1.1732548475265503,
|
|
"step": 3500,
|
|
"unclipped_grad_norm": 1.0132246017456055
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7370531508517597,
|
|
"learning_rate": 0.0002029852160095075,
|
|
"loss_nats": 1.2040334939956665,
|
|
"step": 3510,
|
|
"unclipped_grad_norm": 1.0184191465377808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7040419409318979,
|
|
"learning_rate": 0.000202449392682849,
|
|
"loss_nats": 1.1811518669128418,
|
|
"step": 3520,
|
|
"unclipped_grad_norm": 0.9375280141830444
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7216300906876847,
|
|
"learning_rate": 0.0002019129294493959,
|
|
"loss_nats": 1.1933430433273315,
|
|
"step": 3530,
|
|
"unclipped_grad_norm": 1.0306724309921265
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7320362449434406,
|
|
"learning_rate": 0.00020137583547582478,
|
|
"loss_nats": 1.2005560398101807,
|
|
"step": 3540,
|
|
"unclipped_grad_norm": 0.9943498373031616
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7834315403272825,
|
|
"learning_rate": 0.00020083811993958973,
|
|
"loss_nats": 1.2361805438995361,
|
|
"step": 3550,
|
|
"unclipped_grad_norm": 1.066209316253662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7059953198803977,
|
|
"learning_rate": 0.00020029979202876563,
|
|
"loss_nats": 1.1825058460235596,
|
|
"step": 3560,
|
|
"unclipped_grad_norm": 1.0326995849609375
|
|
},
|
|
{
|
|
"bits_per_byte": 1.768657198722262,
|
|
"learning_rate": 0.00019976086094189118,
|
|
"loss_nats": 1.2259397506713867,
|
|
"step": 3570,
|
|
"unclipped_grad_norm": 1.0416048765182495
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8330426876910733,
|
|
"learning_rate": 0.00019922133588781175,
|
|
"loss_nats": 1.2705683708190918,
|
|
"step": 3580,
|
|
"unclipped_grad_norm": 1.1256591081619263
|
|
},
|
|
{
|
|
"bits_per_byte": 1.759539710469421,
|
|
"learning_rate": 0.0001986812260855219,
|
|
"loss_nats": 1.2196199893951416,
|
|
"step": 3590,
|
|
"unclipped_grad_norm": 1.0512219667434692
|
|
},
|
|
{
|
|
"bits_per_byte": 1.727328735824024,
|
|
"learning_rate": 0.00019814054076400787,
|
|
"loss_nats": 1.1972930431365967,
|
|
"step": 3600,
|
|
"unclipped_grad_norm": 0.9710429906845093
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7248793589104356,
|
|
"learning_rate": 0.00019759928916209004,
|
|
"loss_nats": 1.1955952644348145,
|
|
"step": 3610,
|
|
"unclipped_grad_norm": 1.0369946956634521
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7248757472747676,
|
|
"learning_rate": 0.00019705748052826493,
|
|
"loss_nats": 1.1955927610397339,
|
|
"step": 3620,
|
|
"unclipped_grad_norm": 1.0838762521743774
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7412607064051182,
|
|
"learning_rate": 0.00019651512412054723,
|
|
"loss_nats": 1.2069499492645264,
|
|
"step": 3630,
|
|
"unclipped_grad_norm": 0.9884288311004639
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6889442719432552,
|
|
"learning_rate": 0.00019597222920631144,
|
|
"loss_nats": 1.170686960220337,
|
|
"step": 3640,
|
|
"unclipped_grad_norm": 1.0365123748779297
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7701531038194667,
|
|
"learning_rate": 0.0001954288050621338,
|
|
"loss_nats": 1.2269766330718994,
|
|
"step": 3650,
|
|
"unclipped_grad_norm": 1.1143370866775513
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6641216719789245,
|
|
"learning_rate": 0.00019488486097363363,
|
|
"loss_nats": 1.1534812450408936,
|
|
"step": 3660,
|
|
"unclipped_grad_norm": 1.0202206373214722
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6856676584790171,
|
|
"learning_rate": 0.00019434040623531452,
|
|
"loss_nats": 1.1684157848358154,
|
|
"step": 3670,
|
|
"unclipped_grad_norm": 1.0151705741882324
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7531821997976265,
|
|
"learning_rate": 0.0001937954501504058,
|
|
"loss_nats": 1.2152132987976074,
|
|
"step": 3680,
|
|
"unclipped_grad_norm": 1.013437271118164
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7126154480600597,
|
|
"learning_rate": 0.00019325000203070333,
|
|
"loss_nats": 1.1870945692062378,
|
|
"step": 3690,
|
|
"unclipped_grad_norm": 1.0570411682128906
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6629353356532728,
|
|
"learning_rate": 0.0001927040711964106,
|
|
"loss_nats": 1.1526589393615723,
|
|
"step": 3700,
|
|
"unclipped_grad_norm": 1.0999462604522705
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7333302424085324,
|
|
"learning_rate": 0.00019215766697597923,
|
|
"loss_nats": 1.2014529705047607,
|
|
"step": 3710,
|
|
"unclipped_grad_norm": 1.0060557126998901
|
|
},
|
|
{
|
|
"bits_per_byte": 1.714764887230543,
|
|
"learning_rate": 0.00019161079870594978,
|
|
"loss_nats": 1.1885844469070435,
|
|
"step": 3720,
|
|
"unclipped_grad_norm": 1.061936378479004
|
|
},
|
|
{
|
|
"bits_per_byte": 1.761044214699169,
|
|
"learning_rate": 0.0001910634757307922,
|
|
"loss_nats": 1.2206628322601318,
|
|
"step": 3730,
|
|
"unclipped_grad_norm": 1.0892959833145142
|
|
},
|
|
{
|
|
"bits_per_byte": 1.704870553343755,
|
|
"learning_rate": 0.0001905157074027459,
|
|
"loss_nats": 1.1817262172698975,
|
|
"step": 3740,
|
|
"unclipped_grad_norm": 1.0097936391830444
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6862241943372092,
|
|
"learning_rate": 0.00018996750308166037,
|
|
"loss_nats": 1.1688015460968018,
|
|
"step": 3750,
|
|
"unclipped_grad_norm": 1.0313822031021118
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6824576023006628,
|
|
"learning_rate": 0.00018941887213483482,
|
|
"loss_nats": 1.16619074344635,
|
|
"step": 3760,
|
|
"unclipped_grad_norm": 1.0350043773651123
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7407710717981117,
|
|
"learning_rate": 0.00018886982393685845,
|
|
"loss_nats": 1.2066105604171753,
|
|
"step": 3770,
|
|
"unclipped_grad_norm": 1.0363857746124268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6889002443846344,
|
|
"learning_rate": 0.00018832036786945008,
|
|
"loss_nats": 1.170656442642212,
|
|
"step": 3780,
|
|
"unclipped_grad_norm": 0.9539183974266052
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7337478162848272,
|
|
"learning_rate": 0.0001877705133212979,
|
|
"loss_nats": 1.20174241065979,
|
|
"step": 3790,
|
|
"unclipped_grad_norm": 0.9895081520080566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6773657119565675,
|
|
"learning_rate": 0.00018722026968789906,
|
|
"loss_nats": 1.1626613140106201,
|
|
"step": 3800,
|
|
"unclipped_grad_norm": 0.9700848460197449
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6665356204664348,
|
|
"learning_rate": 0.00018666964637139907,
|
|
"loss_nats": 1.1551544666290283,
|
|
"step": 3810,
|
|
"unclipped_grad_norm": 1.059192180633545
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7701960994821824,
|
|
"learning_rate": 0.00018611865278043115,
|
|
"loss_nats": 1.227006435394287,
|
|
"step": 3820,
|
|
"unclipped_grad_norm": 0.9839220643043518
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6973205149708774,
|
|
"learning_rate": 0.00018556729832995555,
|
|
"loss_nats": 1.1764929294586182,
|
|
"step": 3830,
|
|
"unclipped_grad_norm": 1.0503867864608765
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6894750104038179,
|
|
"learning_rate": 0.00018501559244109864,
|
|
"loss_nats": 1.1710548400878906,
|
|
"step": 3840,
|
|
"unclipped_grad_norm": 1.0203979015350342
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7223587811793903,
|
|
"learning_rate": 0.00018446354454099178,
|
|
"loss_nats": 1.1938481330871582,
|
|
"step": 3850,
|
|
"unclipped_grad_norm": 1.0148154497146606
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6589467140144625,
|
|
"learning_rate": 0.00018391116406261048,
|
|
"loss_nats": 1.1498942375183105,
|
|
"step": 3860,
|
|
"unclipped_grad_norm": 0.9799250364303589
|
|
},
|
|
{
|
|
"bits_per_byte": 1.719725382829379,
|
|
"learning_rate": 0.00018335846044461305,
|
|
"loss_nats": 1.1920228004455566,
|
|
"step": 3870,
|
|
"unclipped_grad_norm": 0.9761002659797668
|
|
},
|
|
{
|
|
"bits_per_byte": 1.70201718918329,
|
|
"learning_rate": 0.0001828054431311793,
|
|
"loss_nats": 1.1797484159469604,
|
|
"step": 3880,
|
|
"unclipped_grad_norm": 1.047680139541626
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7301960305792086,
|
|
"learning_rate": 0.00018225212157184934,
|
|
"loss_nats": 1.1992805004119873,
|
|
"step": 3890,
|
|
"unclipped_grad_norm": 1.0218281745910645
|
|
},
|
|
{
|
|
"bits_per_byte": 1.728929550338255,
|
|
"learning_rate": 0.00018169850522136192,
|
|
"loss_nats": 1.1984026432037354,
|
|
"step": 3900,
|
|
"unclipped_grad_norm": 1.1127806901931763
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7357486624449652,
|
|
"learning_rate": 0.000181144603539493,
|
|
"loss_nats": 1.2031292915344238,
|
|
"step": 3910,
|
|
"unclipped_grad_norm": 0.9941173791885376
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7061793413168207,
|
|
"learning_rate": 0.00018059042599089412,
|
|
"loss_nats": 1.182633399963379,
|
|
"step": 3920,
|
|
"unclipped_grad_norm": 0.9868062734603882
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6839027725158628,
|
|
"learning_rate": 0.00018003598204493048,
|
|
"loss_nats": 1.1671924591064453,
|
|
"step": 3930,
|
|
"unclipped_grad_norm": 1.0294983386993408
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6929858642385314,
|
|
"learning_rate": 0.00017948128117551936,
|
|
"loss_nats": 1.1734883785247803,
|
|
"step": 3940,
|
|
"unclipped_grad_norm": 1.0079408884048462
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7648030675164263,
|
|
"learning_rate": 0.00017892633286096816,
|
|
"loss_nats": 1.2232682704925537,
|
|
"step": 3950,
|
|
"unclipped_grad_norm": 1.091759204864502
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7722222270919972,
|
|
"learning_rate": 0.00017837114658381247,
|
|
"loss_nats": 1.2284108400344849,
|
|
"step": 3960,
|
|
"unclipped_grad_norm": 1.075370192527771
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7441799399208635,
|
|
"learning_rate": 0.00017781573183065386,
|
|
"loss_nats": 1.2089734077453613,
|
|
"step": 3970,
|
|
"unclipped_grad_norm": 1.0237821340560913
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7199797451700052,
|
|
"learning_rate": 0.0001772600980919981,
|
|
"loss_nats": 1.1921991109848022,
|
|
"step": 3980,
|
|
"unclipped_grad_norm": 1.0290383100509644
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7549034021674614,
|
|
"learning_rate": 0.0001767042548620927,
|
|
"loss_nats": 1.2164063453674316,
|
|
"step": 3990,
|
|
"unclipped_grad_norm": 1.0765279531478882
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7501554771250918,
|
|
"learning_rate": 0.00017614821163876486,
|
|
"loss_nats": 1.2131153345108032,
|
|
"step": 4000,
|
|
"unclipped_grad_norm": 1.037840485572815
|
|
},
|
|
{
|
|
"bits_per_byte": 1.783230320625773,
|
|
"learning_rate": 0.00017559197792325906,
|
|
"loss_nats": 1.2360410690307617,
|
|
"step": 4010,
|
|
"unclipped_grad_norm": 1.1067007780075073
|
|
},
|
|
{
|
|
"bits_per_byte": 1.634854696350997,
|
|
"learning_rate": 0.0001750355632200748,
|
|
"loss_nats": 1.133194923400879,
|
|
"step": 4020,
|
|
"unclipped_grad_norm": 0.9555695652961731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6858748975733067,
|
|
"learning_rate": 0.00017447897703680418,
|
|
"loss_nats": 1.1685594320297241,
|
|
"step": 4030,
|
|
"unclipped_grad_norm": 0.9827634692192078
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6858767893824662,
|
|
"learning_rate": 0.0001739222288839694,
|
|
"loss_nats": 1.1685607433319092,
|
|
"step": 4040,
|
|
"unclipped_grad_norm": 1.0647450685501099
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6753074235910415,
|
|
"learning_rate": 0.00017336532827486026,
|
|
"loss_nats": 1.1612346172332764,
|
|
"step": 4050,
|
|
"unclipped_grad_norm": 1.0498672723770142
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7132395731000407,
|
|
"learning_rate": 0.00017280828472537163,
|
|
"loss_nats": 1.1875271797180176,
|
|
"step": 4060,
|
|
"unclipped_grad_norm": 1.0244723558425903
|
|
},
|
|
{
|
|
"bits_per_byte": 1.659084988065756,
|
|
"learning_rate": 0.0001722511077538408,
|
|
"loss_nats": 1.1499900817871094,
|
|
"step": 4070,
|
|
"unclipped_grad_norm": 0.954911470413208
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7001991605810192,
|
|
"learning_rate": 0.00017169380688088497,
|
|
"loss_nats": 1.1784882545471191,
|
|
"step": 4080,
|
|
"unclipped_grad_norm": 1.0178123712539673
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6512532421107655,
|
|
"learning_rate": 0.0001711363916292384,
|
|
"loss_nats": 1.144561529159546,
|
|
"step": 4090,
|
|
"unclipped_grad_norm": 0.988084614276886
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6312373852553994,
|
|
"learning_rate": 0.00017057887152358974,
|
|
"loss_nats": 1.1306875944137573,
|
|
"step": 4100,
|
|
"unclipped_grad_norm": 0.9640962481498718
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7389465078551083,
|
|
"learning_rate": 0.0001700212560904194,
|
|
"loss_nats": 1.205345869064331,
|
|
"step": 4110,
|
|
"unclipped_grad_norm": 1.0591799020767212
|
|
},
|
|
{
|
|
"bits_per_byte": 1.670625711869254,
|
|
"learning_rate": 0.00016946355485783656,
|
|
"loss_nats": 1.157989501953125,
|
|
"step": 4120,
|
|
"unclipped_grad_norm": 1.0143104791641235
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6531302607622822,
|
|
"learning_rate": 0.0001689057773554166,
|
|
"loss_nats": 1.1458625793457031,
|
|
"step": 4130,
|
|
"unclipped_grad_norm": 1.0070092678070068
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6853878427060633,
|
|
"learning_rate": 0.00016834793311403806,
|
|
"loss_nats": 1.1682218313217163,
|
|
"step": 4140,
|
|
"unclipped_grad_norm": 1.0397000312805176
|
|
},
|
|
{
|
|
"bits_per_byte": 1.716280570332597,
|
|
"learning_rate": 0.00016779003166571981,
|
|
"loss_nats": 1.1896350383758545,
|
|
"step": 4150,
|
|
"unclipped_grad_norm": 1.0394762754440308
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7032425655706875,
|
|
"learning_rate": 0.00016723208254345838,
|
|
"loss_nats": 1.1805977821350098,
|
|
"step": 4160,
|
|
"unclipped_grad_norm": 1.0539857149124146
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6362312454885026,
|
|
"learning_rate": 0.00016667409528106481,
|
|
"loss_nats": 1.1341490745544434,
|
|
"step": 4170,
|
|
"unclipped_grad_norm": 1.0312221050262451
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6176134356020002,
|
|
"learning_rate": 0.00016611607941300184,
|
|
"loss_nats": 1.121244192123413,
|
|
"step": 4180,
|
|
"unclipped_grad_norm": 0.9745424389839172
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7779335989445002,
|
|
"learning_rate": 0.0001655580444742211,
|
|
"loss_nats": 1.2323696613311768,
|
|
"step": 4190,
|
|
"unclipped_grad_norm": 1.067311406135559
|
|
},
|
|
{
|
|
"bits_per_byte": 1.675829562919061,
|
|
"learning_rate": 0.00016499999999999997,
|
|
"loss_nats": 1.1615965366363525,
|
|
"step": 4200,
|
|
"unclipped_grad_norm": 1.045929193496704
|
|
},
|
|
{
|
|
"bits_per_byte": 1.681460618873611,
|
|
"learning_rate": 0.0001644419555257789,
|
|
"loss_nats": 1.1654996871948242,
|
|
"step": 4210,
|
|
"unclipped_grad_norm": 1.0254682302474976
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7232076875440492,
|
|
"learning_rate": 0.0001638839205869981,
|
|
"loss_nats": 1.1944365501403809,
|
|
"step": 4220,
|
|
"unclipped_grad_norm": 1.0498327016830444
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6892706950145928,
|
|
"learning_rate": 0.0001633259047189352,
|
|
"loss_nats": 1.1709132194519043,
|
|
"step": 4230,
|
|
"unclipped_grad_norm": 1.0495660305023193
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7317139494557237,
|
|
"learning_rate": 0.0001627679174565416,
|
|
"loss_nats": 1.2003326416015625,
|
|
"step": 4240,
|
|
"unclipped_grad_norm": 1.1651132106781006
|
|
},
|
|
{
|
|
"bits_per_byte": 1.744415040204593,
|
|
"learning_rate": 0.00016220996833428016,
|
|
"loss_nats": 1.2091363668441772,
|
|
"step": 4250,
|
|
"unclipped_grad_norm": 1.0255383253097534
|
|
},
|
|
{
|
|
"bits_per_byte": 1.73589312787169,
|
|
"learning_rate": 0.00016165206688596191,
|
|
"loss_nats": 1.2032294273376465,
|
|
"step": 4260,
|
|
"unclipped_grad_norm": 1.1120972633361816
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7234627378152787,
|
|
"learning_rate": 0.00016109422264458335,
|
|
"loss_nats": 1.1946133375167847,
|
|
"step": 4270,
|
|
"unclipped_grad_norm": 1.016249656677246
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7430716837187037,
|
|
"learning_rate": 0.00016053644514216336,
|
|
"loss_nats": 1.208205223083496,
|
|
"step": 4280,
|
|
"unclipped_grad_norm": 1.0722230672836304
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7030208799337254,
|
|
"learning_rate": 0.00015997874390958058,
|
|
"loss_nats": 1.1804441213607788,
|
|
"step": 4290,
|
|
"unclipped_grad_norm": 1.03997004032135
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6244537015747667,
|
|
"learning_rate": 0.00015942112847641018,
|
|
"loss_nats": 1.1259855031967163,
|
|
"step": 4300,
|
|
"unclipped_grad_norm": 1.037449836730957
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7698973656176336,
|
|
"learning_rate": 0.00015886360837076156,
|
|
"loss_nats": 1.2267993688583374,
|
|
"step": 4310,
|
|
"unclipped_grad_norm": 1.235188603401184
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6511122163370582,
|
|
"learning_rate": 0.00015830619311911495,
|
|
"loss_nats": 1.1444637775421143,
|
|
"step": 4320,
|
|
"unclipped_grad_norm": 1.1049147844314575
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6449793150072904,
|
|
"learning_rate": 0.00015774889224615915,
|
|
"loss_nats": 1.1402127742767334,
|
|
"step": 4330,
|
|
"unclipped_grad_norm": 1.0132696628570557
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6583323919855806,
|
|
"learning_rate": 0.00015719171527462834,
|
|
"loss_nats": 1.1494684219360352,
|
|
"step": 4340,
|
|
"unclipped_grad_norm": 1.0222140550613403
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5761074866080718,
|
|
"learning_rate": 0.0001566346717251397,
|
|
"loss_nats": 1.0924744606018066,
|
|
"step": 4350,
|
|
"unclipped_grad_norm": 1.0156723260879517
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7219245249859618,
|
|
"learning_rate": 0.00015607777111603056,
|
|
"loss_nats": 1.1935471296310425,
|
|
"step": 4360,
|
|
"unclipped_grad_norm": 1.0623725652694702
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6842240361076746,
|
|
"learning_rate": 0.00015552102296319576,
|
|
"loss_nats": 1.1674151420593262,
|
|
"step": 4370,
|
|
"unclipped_grad_norm": 1.070603847503662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6398215553079147,
|
|
"learning_rate": 0.00015496443677992516,
|
|
"loss_nats": 1.1366376876831055,
|
|
"step": 4380,
|
|
"unclipped_grad_norm": 1.0265616178512573
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6765849107216504,
|
|
"learning_rate": 0.0001544080220767409,
|
|
"loss_nats": 1.1621201038360596,
|
|
"step": 4390,
|
|
"unclipped_grad_norm": 1.020019769668579
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6365060737645816,
|
|
"learning_rate": 0.00015385178836123511,
|
|
"loss_nats": 1.1343395709991455,
|
|
"step": 4400,
|
|
"unclipped_grad_norm": 1.0334678888320923
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6461109608499676,
|
|
"learning_rate": 0.00015329574513790726,
|
|
"loss_nats": 1.1409971714019775,
|
|
"step": 4410,
|
|
"unclipped_grad_norm": 1.064432144165039
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6319834459948421,
|
|
"learning_rate": 0.00015273990190800186,
|
|
"loss_nats": 1.1312047243118286,
|
|
"step": 4420,
|
|
"unclipped_grad_norm": 1.0356578826904297
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6589714795161867,
|
|
"learning_rate": 0.0001521842681693461,
|
|
"loss_nats": 1.1499114036560059,
|
|
"step": 4430,
|
|
"unclipped_grad_norm": 1.0542991161346436
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6678407968038327,
|
|
"learning_rate": 0.00015162885341618752,
|
|
"loss_nats": 1.1560591459274292,
|
|
"step": 4440,
|
|
"unclipped_grad_norm": 1.1220062971115112
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6463864770566499,
|
|
"learning_rate": 0.00015107366713903178,
|
|
"loss_nats": 1.141188144683838,
|
|
"step": 4450,
|
|
"unclipped_grad_norm": 1.069725751876831
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6193611233000682,
|
|
"learning_rate": 0.0001505187188244806,
|
|
"loss_nats": 1.1224555969238281,
|
|
"step": 4460,
|
|
"unclipped_grad_norm": 1.0096163749694824
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7261988098078551,
|
|
"learning_rate": 0.00014996401795506952,
|
|
"loss_nats": 1.196509838104248,
|
|
"step": 4470,
|
|
"unclipped_grad_norm": 1.0757471323013306
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6253762165139949,
|
|
"learning_rate": 0.00014940957400910585,
|
|
"loss_nats": 1.1266249418258667,
|
|
"step": 4480,
|
|
"unclipped_grad_norm": 1.0172395706176758
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5785597872267245,
|
|
"learning_rate": 0.00014885539646050699,
|
|
"loss_nats": 1.0941742658615112,
|
|
"step": 4490,
|
|
"unclipped_grad_norm": 1.081101894378662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6683827141367014,
|
|
"learning_rate": 0.00014830149477863805,
|
|
"loss_nats": 1.1564347743988037,
|
|
"step": 4500,
|
|
"unclipped_grad_norm": 1.027239441871643
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6472935135573004,
|
|
"learning_rate": 0.00014774787842815066,
|
|
"loss_nats": 1.1418168544769287,
|
|
"step": 4510,
|
|
"unclipped_grad_norm": 1.0604325532913208
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6464067710094517,
|
|
"learning_rate": 0.00014719455686882066,
|
|
"loss_nats": 1.1412022113800049,
|
|
"step": 4520,
|
|
"unclipped_grad_norm": 1.0223703384399414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6663331968863693,
|
|
"learning_rate": 0.00014664153955538695,
|
|
"loss_nats": 1.155014157295227,
|
|
"step": 4530,
|
|
"unclipped_grad_norm": 1.096794843673706
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5983706407622622,
|
|
"learning_rate": 0.00014608883593738946,
|
|
"loss_nats": 1.1079061031341553,
|
|
"step": 4540,
|
|
"unclipped_grad_norm": 0.991118311882019
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6821887934173643,
|
|
"learning_rate": 0.00014553645545900817,
|
|
"loss_nats": 1.1660044193267822,
|
|
"step": 4550,
|
|
"unclipped_grad_norm": 1.0131337642669678
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6779839895864195,
|
|
"learning_rate": 0.0001449844075589013,
|
|
"loss_nats": 1.1630898714065552,
|
|
"step": 4560,
|
|
"unclipped_grad_norm": 1.0416655540466309
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6489940780090317,
|
|
"learning_rate": 0.0001444327016700444,
|
|
"loss_nats": 1.1429955959320068,
|
|
"step": 4570,
|
|
"unclipped_grad_norm": 1.0762525796890259
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5938887728807776,
|
|
"learning_rate": 0.0001438813472195688,
|
|
"loss_nats": 1.104799509048462,
|
|
"step": 4580,
|
|
"unclipped_grad_norm": 1.039039969444275
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7129355077733153,
|
|
"learning_rate": 0.00014333035362860093,
|
|
"loss_nats": 1.1873164176940918,
|
|
"step": 4590,
|
|
"unclipped_grad_norm": 1.1662163734436035
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6753789683738005,
|
|
"learning_rate": 0.00014277973031210088,
|
|
"loss_nats": 1.1612842082977295,
|
|
"step": 4600,
|
|
"unclipped_grad_norm": 1.0834790468215942
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6489213293477167,
|
|
"learning_rate": 0.00014222948667870207,
|
|
"loss_nats": 1.1429451704025269,
|
|
"step": 4610,
|
|
"unclipped_grad_norm": 1.0408029556274414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5772676815707922,
|
|
"learning_rate": 0.00014167963213054991,
|
|
"loss_nats": 1.0932786464691162,
|
|
"step": 4620,
|
|
"unclipped_grad_norm": 0.9947658777236938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6629637127906651,
|
|
"learning_rate": 0.00014113017606314152,
|
|
"loss_nats": 1.1526786088943481,
|
|
"step": 4630,
|
|
"unclipped_grad_norm": 1.0645713806152344
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6351708004632826,
|
|
"learning_rate": 0.00014058112786516518,
|
|
"loss_nats": 1.1334140300750732,
|
|
"step": 4640,
|
|
"unclipped_grad_norm": 1.0213215351104736
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6143973600308656,
|
|
"learning_rate": 0.0001400324969183396,
|
|
"loss_nats": 1.1190149784088135,
|
|
"step": 4650,
|
|
"unclipped_grad_norm": 1.0158205032348633
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6442494206370286,
|
|
"learning_rate": 0.00013948429259725406,
|
|
"loss_nats": 1.1397068500518799,
|
|
"step": 4660,
|
|
"unclipped_grad_norm": 1.032319188117981
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6788175894951514,
|
|
"learning_rate": 0.00013893652426920776,
|
|
"loss_nats": 1.1636676788330078,
|
|
"step": 4670,
|
|
"unclipped_grad_norm": 1.1467437744140625
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6228953667752988,
|
|
"learning_rate": 0.00013838920129405016,
|
|
"loss_nats": 1.1249053478240967,
|
|
"step": 4680,
|
|
"unclipped_grad_norm": 1.0684930086135864
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6116814100084402,
|
|
"learning_rate": 0.0001378423330240207,
|
|
"loss_nats": 1.1171324253082275,
|
|
"step": 4690,
|
|
"unclipped_grad_norm": 1.0818438529968262
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7118176205427071,
|
|
"learning_rate": 0.00013729592880358936,
|
|
"loss_nats": 1.1865415573120117,
|
|
"step": 4700,
|
|
"unclipped_grad_norm": 1.0913734436035156
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6314229545356804,
|
|
"learning_rate": 0.00013674999796929659,
|
|
"loss_nats": 1.1308162212371826,
|
|
"step": 4710,
|
|
"unclipped_grad_norm": 1.0516728162765503
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6067035441618671,
|
|
"learning_rate": 0.00013620454984959417,
|
|
"loss_nats": 1.1136820316314697,
|
|
"step": 4720,
|
|
"unclipped_grad_norm": 1.027355670928955
|
|
},
|
|
{
|
|
"bits_per_byte": 1.723685455348146,
|
|
"learning_rate": 0.00013565959376468542,
|
|
"loss_nats": 1.194767713546753,
|
|
"step": 4730,
|
|
"unclipped_grad_norm": 1.0737906694412231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6626813172779484,
|
|
"learning_rate": 0.00013511513902636634,
|
|
"loss_nats": 1.1524828672409058,
|
|
"step": 4740,
|
|
"unclipped_grad_norm": 1.093795657157898
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6942468410346574,
|
|
"learning_rate": 0.0001345711949378662,
|
|
"loss_nats": 1.1743624210357666,
|
|
"step": 4750,
|
|
"unclipped_grad_norm": 1.098819375038147
|
|
},
|
|
{
|
|
"bits_per_byte": 1.567997816689287,
|
|
"learning_rate": 0.00013402777079368853,
|
|
"loss_nats": 1.086853265762329,
|
|
"step": 4760,
|
|
"unclipped_grad_norm": 0.9886804819107056
|
|
},
|
|
{
|
|
"bits_per_byte": 1.674730077832095,
|
|
"learning_rate": 0.0001334848758794528,
|
|
"loss_nats": 1.1608344316482544,
|
|
"step": 4770,
|
|
"unclipped_grad_norm": 1.0352309942245483
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6338455021557339,
|
|
"learning_rate": 0.00013294251947173504,
|
|
"loss_nats": 1.132495403289795,
|
|
"step": 4780,
|
|
"unclipped_grad_norm": 1.037882685661316
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6704422063807836,
|
|
"learning_rate": 0.00013240071083790996,
|
|
"loss_nats": 1.1578623056411743,
|
|
"step": 4790,
|
|
"unclipped_grad_norm": 1.117195725440979
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6306944360266256,
|
|
"learning_rate": 0.0001318594592359921,
|
|
"loss_nats": 1.1303112506866455,
|
|
"step": 4800,
|
|
"unclipped_grad_norm": 1.0563780069351196
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6131430905581232,
|
|
"learning_rate": 0.0001313187739144781,
|
|
"loss_nats": 1.1181455850601196,
|
|
"step": 4810,
|
|
"unclipped_grad_norm": 1.0533764362335205
|
|
},
|
|
{
|
|
"bits_per_byte": 1.675093477173368,
|
|
"learning_rate": 0.0001307786641121882,
|
|
"loss_nats": 1.1610863208770752,
|
|
"step": 4820,
|
|
"unclipped_grad_norm": 1.0993541479110718
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6710443176414542,
|
|
"learning_rate": 0.00013023913905810876,
|
|
"loss_nats": 1.1582796573638916,
|
|
"step": 4830,
|
|
"unclipped_grad_norm": 1.2536643743515015
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6771435103716528,
|
|
"learning_rate": 0.00012970020797123432,
|
|
"loss_nats": 1.1625072956085205,
|
|
"step": 4840,
|
|
"unclipped_grad_norm": 1.074759840965271
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6143529885069432,
|
|
"learning_rate": 0.00012916188006041027,
|
|
"loss_nats": 1.1189842224121094,
|
|
"step": 4850,
|
|
"unclipped_grad_norm": 1.0702130794525146
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6332604171774985,
|
|
"learning_rate": 0.00012862416452417516,
|
|
"loss_nats": 1.1320898532867432,
|
|
"step": 4860,
|
|
"unclipped_grad_norm": 1.0851078033447266
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6331224870915066,
|
|
"learning_rate": 0.00012808707055060405,
|
|
"loss_nats": 1.1319942474365234,
|
|
"step": 4870,
|
|
"unclipped_grad_norm": 1.08431875705719
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7854224114936703,
|
|
"learning_rate": 0.000127550607317151,
|
|
"loss_nats": 1.237560510635376,
|
|
"step": 4880,
|
|
"unclipped_grad_norm": 1.1104563474655151
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5755340964500952,
|
|
"learning_rate": 0.00012701478399049248,
|
|
"loss_nats": 1.0920770168304443,
|
|
"step": 4890,
|
|
"unclipped_grad_norm": 1.042919635772705
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6730817961062259,
|
|
"learning_rate": 0.0001264796097263708,
|
|
"loss_nats": 1.1596919298171997,
|
|
"step": 4900,
|
|
"unclipped_grad_norm": 1.0552027225494385
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6497614645971816,
|
|
"learning_rate": 0.0001259450936694375,
|
|
"loss_nats": 1.1435275077819824,
|
|
"step": 4910,
|
|
"unclipped_grad_norm": 1.0571706295013428
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5965801293841295,
|
|
"learning_rate": 0.00012541124495309727,
|
|
"loss_nats": 1.106665015220642,
|
|
"step": 4920,
|
|
"unclipped_grad_norm": 1.0640807151794434
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6007018655947076,
|
|
"learning_rate": 0.00012487807269935167,
|
|
"loss_nats": 1.1095219850540161,
|
|
"step": 4930,
|
|
"unclipped_grad_norm": 1.0152149200439453
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6331477685411835,
|
|
"learning_rate": 0.00012434558601864349,
|
|
"loss_nats": 1.1320117712020874,
|
|
"step": 4940,
|
|
"unclipped_grad_norm": 1.100314736366272
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6360496318091915,
|
|
"learning_rate": 0.0001238137940097009,
|
|
"loss_nats": 1.1340231895446777,
|
|
"step": 4950,
|
|
"unclipped_grad_norm": 1.079222559928894
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6569551549174713,
|
|
"learning_rate": 0.0001232827057593821,
|
|
"loss_nats": 1.1485137939453125,
|
|
"step": 4960,
|
|
"unclipped_grad_norm": 1.0604397058486938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.638398398872025,
|
|
"learning_rate": 0.00012275233034251992,
|
|
"loss_nats": 1.1356512308120728,
|
|
"step": 4970,
|
|
"unclipped_grad_norm": 1.0775542259216309
|
|
},
|
|
{
|
|
"bits_per_byte": 1.680847672705936,
|
|
"learning_rate": 0.0001222226768217669,
|
|
"loss_nats": 1.1650748252868652,
|
|
"step": 4980,
|
|
"unclipped_grad_norm": 1.1589142084121704
|
|
},
|
|
{
|
|
"bits_per_byte": 1.633999426628256,
|
|
"learning_rate": 0.00012169375424744034,
|
|
"loss_nats": 1.1326020956039429,
|
|
"step": 4990,
|
|
"unclipped_grad_norm": 1.075969934463501
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6444650868812105,
|
|
"learning_rate": 0.00012116557165736772,
|
|
"loss_nats": 1.1398563385009766,
|
|
"step": 5000,
|
|
"unclipped_grad_norm": 1.0691334009170532
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6738108305632333,
|
|
"learning_rate": 0.00012063813807673214,
|
|
"loss_nats": 1.1601972579956055,
|
|
"step": 5010,
|
|
"unclipped_grad_norm": 1.0487086772918701
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6477535671483583,
|
|
"learning_rate": 0.00012011146251791832,
|
|
"loss_nats": 1.142135739326477,
|
|
"step": 5020,
|
|
"unclipped_grad_norm": 1.1224150657653809
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5805776596692978,
|
|
"learning_rate": 0.0001195855539803584,
|
|
"loss_nats": 1.0955729484558105,
|
|
"step": 5030,
|
|
"unclipped_grad_norm": 0.9987351894378662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6501410303076358,
|
|
"learning_rate": 0.00011906042145037817,
|
|
"loss_nats": 1.143790602684021,
|
|
"step": 5040,
|
|
"unclipped_grad_norm": 1.069442868232727
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6202205206064295,
|
|
"learning_rate": 0.0001185360739010438,
|
|
"loss_nats": 1.1230512857437134,
|
|
"step": 5050,
|
|
"unclipped_grad_norm": 1.1084743738174438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6928149134835737,
|
|
"learning_rate": 0.00011801252029200808,
|
|
"loss_nats": 1.1733698844909668,
|
|
"step": 5060,
|
|
"unclipped_grad_norm": 1.0974880456924438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6259845191500966,
|
|
"learning_rate": 0.00011748976956935777,
|
|
"loss_nats": 1.1270465850830078,
|
|
"step": 5070,
|
|
"unclipped_grad_norm": 1.0709881782531738
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6998562271751987,
|
|
"learning_rate": 0.00011696783066546024,
|
|
"loss_nats": 1.1782505512237549,
|
|
"step": 5080,
|
|
"unclipped_grad_norm": 1.1272931098937988
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6260173678364114,
|
|
"learning_rate": 0.00011644671249881147,
|
|
"loss_nats": 1.127069354057312,
|
|
"step": 5090,
|
|
"unclipped_grad_norm": 1.0618640184402466
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6234274810970684,
|
|
"learning_rate": 0.00011592642397388293,
|
|
"loss_nats": 1.1252741813659668,
|
|
"step": 5100,
|
|
"unclipped_grad_norm": 1.1292778253555298
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6568031222541084,
|
|
"learning_rate": 0.00011540697398097015,
|
|
"loss_nats": 1.1484084129333496,
|
|
"step": 5110,
|
|
"unclipped_grad_norm": 1.1496297121047974
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5598939941806313,
|
|
"learning_rate": 0.00011488837139604016,
|
|
"loss_nats": 1.0812361240386963,
|
|
"step": 5120,
|
|
"unclipped_grad_norm": 1.035429835319519
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6914442117561972,
|
|
"learning_rate": 0.00011437062508058039,
|
|
"loss_nats": 1.172419786453247,
|
|
"step": 5130,
|
|
"unclipped_grad_norm": 1.0999172925949097
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6623124144918477,
|
|
"learning_rate": 0.00011385374388144673,
|
|
"loss_nats": 1.1522271633148193,
|
|
"step": 5140,
|
|
"unclipped_grad_norm": 1.0919309854507446
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6100444791375281,
|
|
"learning_rate": 0.00011333773663071286,
|
|
"loss_nats": 1.1159977912902832,
|
|
"step": 5150,
|
|
"unclipped_grad_norm": 1.0898085832595825
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6321538808018472,
|
|
"learning_rate": 0.00011282261214551897,
|
|
"loss_nats": 1.1313228607177734,
|
|
"step": 5160,
|
|
"unclipped_grad_norm": 1.0807921886444092
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6844840738757791,
|
|
"learning_rate": 0.00011230837922792112,
|
|
"loss_nats": 1.167595386505127,
|
|
"step": 5170,
|
|
"unclipped_grad_norm": 1.176076889038086
|
|
},
|
|
{
|
|
"bits_per_byte": 1.65267932225172,
|
|
"learning_rate": 0.0001117950466647412,
|
|
"loss_nats": 1.145550012588501,
|
|
"step": 5180,
|
|
"unclipped_grad_norm": 1.110230803489685
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5426296877564192,
|
|
"learning_rate": 0.00011128262322741627,
|
|
"loss_nats": 1.0692694187164307,
|
|
"step": 5190,
|
|
"unclipped_grad_norm": 1.0403128862380981
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6380544335702993,
|
|
"learning_rate": 0.00011077111767184913,
|
|
"loss_nats": 1.1354128122329712,
|
|
"step": 5200,
|
|
"unclipped_grad_norm": 1.1156731843948364
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6025732088187459,
|
|
"learning_rate": 0.00011026053873825837,
|
|
"loss_nats": 1.1108191013336182,
|
|
"step": 5210,
|
|
"unclipped_grad_norm": 1.112536907196045
|
|
},
|
|
{
|
|
"bits_per_byte": 1.595609287320009,
|
|
"learning_rate": 0.0001097508951510293,
|
|
"loss_nats": 1.105992078781128,
|
|
"step": 5220,
|
|
"unclipped_grad_norm": 1.0583246946334839
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6907153492818405,
|
|
"learning_rate": 0.0001092421956185646,
|
|
"loss_nats": 1.1719145774841309,
|
|
"step": 5230,
|
|
"unclipped_grad_norm": 1.1788045167922974
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6472453584150588,
|
|
"learning_rate": 0.00010873444883313581,
|
|
"loss_nats": 1.1417834758758545,
|
|
"step": 5240,
|
|
"unclipped_grad_norm": 1.1392273902893066
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6431225903085755,
|
|
"learning_rate": 0.00010822766347073447,
|
|
"loss_nats": 1.1389257907867432,
|
|
"step": 5250,
|
|
"unclipped_grad_norm": 1.087467074394226
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5745493237912547,
|
|
"learning_rate": 0.00010772184819092424,
|
|
"loss_nats": 1.0913944244384766,
|
|
"step": 5260,
|
|
"unclipped_grad_norm": 1.0672780275344849
|
|
},
|
|
{
|
|
"bits_per_byte": 1.625834722261195,
|
|
"learning_rate": 0.0001072170116366926,
|
|
"loss_nats": 1.126942753791809,
|
|
"step": 5270,
|
|
"unclipped_grad_norm": 1.0955538749694824
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5722745092682924,
|
|
"learning_rate": 0.00010671316243430345,
|
|
"loss_nats": 1.0898176431655884,
|
|
"step": 5280,
|
|
"unclipped_grad_norm": 1.1191643476486206
|
|
},
|
|
{
|
|
"bits_per_byte": 1.581137635180507,
|
|
"learning_rate": 0.00010621030919314944,
|
|
"loss_nats": 1.095961093902588,
|
|
"step": 5290,
|
|
"unclipped_grad_norm": 1.0905182361602783
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5625956696430352,
|
|
"learning_rate": 0.000105708460505605,
|
|
"loss_nats": 1.0831087827682495,
|
|
"step": 5300,
|
|
"unclipped_grad_norm": 1.0970019102096558
|
|
},
|
|
{
|
|
"bits_per_byte": 1.575699715742876,
|
|
"learning_rate": 0.00010520762494687964,
|
|
"loss_nats": 1.0921918153762817,
|
|
"step": 5310,
|
|
"unclipped_grad_norm": 1.085243582725525
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5934223559316376,
|
|
"learning_rate": 0.00010470781107487111,
|
|
"loss_nats": 1.1044762134552002,
|
|
"step": 5320,
|
|
"unclipped_grad_norm": 1.1693148612976074
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6214211314921028,
|
|
"learning_rate": 0.00010420902743001955,
|
|
"loss_nats": 1.1238834857940674,
|
|
"step": 5330,
|
|
"unclipped_grad_norm": 1.1232852935791016
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6839337293930183,
|
|
"learning_rate": 0.00010371128253516117,
|
|
"loss_nats": 1.1672139167785645,
|
|
"step": 5340,
|
|
"unclipped_grad_norm": 1.1369119882583618
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5416975417887426,
|
|
"learning_rate": 0.000103214584895383,
|
|
"loss_nats": 1.0686233043670654,
|
|
"step": 5350,
|
|
"unclipped_grad_norm": 1.0887452363967896
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5866414239734188,
|
|
"learning_rate": 0.00010271894299787715,
|
|
"loss_nats": 1.099776029586792,
|
|
"step": 5360,
|
|
"unclipped_grad_norm": 1.0925503969192505
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6646420914804352,
|
|
"learning_rate": 0.00010222436531179622,
|
|
"loss_nats": 1.1538419723510742,
|
|
"step": 5370,
|
|
"unclipped_grad_norm": 1.1924686431884766
|
|
},
|
|
{
|
|
"bits_per_byte": 1.586491799067168,
|
|
"learning_rate": 0.0001017308602881082,
|
|
"loss_nats": 1.0996723175048828,
|
|
"step": 5380,
|
|
"unclipped_grad_norm": 1.0599968433380127
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5783984675002152,
|
|
"learning_rate": 0.00010123843635945242,
|
|
"loss_nats": 1.0940624475479126,
|
|
"step": 5390,
|
|
"unclipped_grad_norm": 1.0973831415176392
|
|
},
|
|
{
|
|
"bits_per_byte": 1.642996527025493,
|
|
"learning_rate": 0.00010074710193999505,
|
|
"loss_nats": 1.1388384103775024,
|
|
"step": 5400,
|
|
"unclipped_grad_norm": 1.1477714776992798
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5308346016122951,
|
|
"learning_rate": 0.0001002568654252858,
|
|
"loss_nats": 1.0610936880111694,
|
|
"step": 5410,
|
|
"unclipped_grad_norm": 1.0651572942733765
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6877225071915258,
|
|
"learning_rate": 9.976773519211404e-05,
|
|
"loss_nats": 1.1698400974273682,
|
|
"step": 5420,
|
|
"unclipped_grad_norm": 1.230097770690918
|
|
},
|
|
{
|
|
"bits_per_byte": 1.606558218821888,
|
|
"learning_rate": 9.92797195983659e-05,
|
|
"loss_nats": 1.1135812997817993,
|
|
"step": 5430,
|
|
"unclipped_grad_norm": 1.1141976118087769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6079992614534675,
|
|
"learning_rate": 9.879282698288142e-05,
|
|
"loss_nats": 1.1145801544189453,
|
|
"step": 5440,
|
|
"unclipped_grad_norm": 1.0613634586334229
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6121270170568258,
|
|
"learning_rate": 9.830706566531197e-05,
|
|
"loss_nats": 1.1174412965774536,
|
|
"step": 5450,
|
|
"unclipped_grad_norm": 1.1033883094787598
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6549515570349194,
|
|
"learning_rate": 9.782244394597826e-05,
|
|
"loss_nats": 1.147125005722046,
|
|
"step": 5460,
|
|
"unclipped_grad_norm": 1.0616251230239868
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6138821000088808,
|
|
"learning_rate": 9.733897010572829e-05,
|
|
"loss_nats": 1.1186578273773193,
|
|
"step": 5470,
|
|
"unclipped_grad_norm": 1.1187942028045654
|
|
},
|
|
{
|
|
"bits_per_byte": 1.566549894751673,
|
|
"learning_rate": 9.685665240579612e-05,
|
|
"loss_nats": 1.085849642753601,
|
|
"step": 5480,
|
|
"unclipped_grad_norm": 1.095421314239502
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6100902265226575,
|
|
"learning_rate": 9.637549908766041e-05,
|
|
"loss_nats": 1.1160295009613037,
|
|
"step": 5490,
|
|
"unclipped_grad_norm": 1.1169681549072266
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6345392801693144,
|
|
"learning_rate": 9.589551837290389e-05,
|
|
"loss_nats": 1.1329762935638428,
|
|
"step": 5500,
|
|
"unclipped_grad_norm": 1.0606133937835693
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6112232482265416,
|
|
"learning_rate": 9.541671846307257e-05,
|
|
"loss_nats": 1.1168148517608643,
|
|
"step": 5510,
|
|
"unclipped_grad_norm": 1.082831859588623
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6200284159854157,
|
|
"learning_rate": 9.493910753953597e-05,
|
|
"loss_nats": 1.1229181289672852,
|
|
"step": 5520,
|
|
"unclipped_grad_norm": 1.1288034915924072
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6500037881522474,
|
|
"learning_rate": 9.446269376334689e-05,
|
|
"loss_nats": 1.1436954736709595,
|
|
"step": 5530,
|
|
"unclipped_grad_norm": 1.1526131629943848
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6647621353707376,
|
|
"learning_rate": 9.39874852751024e-05,
|
|
"loss_nats": 1.1539251804351807,
|
|
"step": 5540,
|
|
"unclipped_grad_norm": 1.1722888946533203
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5974680758105342,
|
|
"learning_rate": 9.351349019480433e-05,
|
|
"loss_nats": 1.1072804927825928,
|
|
"step": 5550,
|
|
"unclipped_grad_norm": 1.073315143585205
|
|
},
|
|
{
|
|
"bits_per_byte": 1.594942510582614,
|
|
"learning_rate": 9.304071662172086e-05,
|
|
"loss_nats": 1.1055299043655396,
|
|
"step": 5560,
|
|
"unclipped_grad_norm": 1.0879392623901367
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6466642290377933,
|
|
"learning_rate": 9.256917263424787e-05,
|
|
"loss_nats": 1.1413806676864624,
|
|
"step": 5570,
|
|
"unclipped_grad_norm": 1.21271812915802
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5387917229197647,
|
|
"learning_rate": 9.209886628977114e-05,
|
|
"loss_nats": 1.0666091442108154,
|
|
"step": 5580,
|
|
"unclipped_grad_norm": 1.0752229690551758
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5950678859350929,
|
|
"learning_rate": 9.162980562452843e-05,
|
|
"loss_nats": 1.105616807937622,
|
|
"step": 5590,
|
|
"unclipped_grad_norm": 1.115165114402771
|
|
},
|
|
{
|
|
"bits_per_byte": 1.60302053569364,
|
|
"learning_rate": 9.116199865347238e-05,
|
|
"loss_nats": 1.1111291646957397,
|
|
"step": 5600,
|
|
"unclipped_grad_norm": 1.0947556495666504
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6150159816260192,
|
|
"learning_rate": 9.069545337013333e-05,
|
|
"loss_nats": 1.1194437742233276,
|
|
"step": 5610,
|
|
"unclipped_grad_norm": 1.0436956882476807
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5445598490470525,
|
|
"learning_rate": 9.0230177746483e-05,
|
|
"loss_nats": 1.070607304573059,
|
|
"step": 5620,
|
|
"unclipped_grad_norm": 1.1328603029251099
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5627000631121089,
|
|
"learning_rate": 8.976617973279797e-05,
|
|
"loss_nats": 1.0831811428070068,
|
|
"step": 5630,
|
|
"unclipped_grad_norm": 1.0480417013168335
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5462631652211978,
|
|
"learning_rate": 8.930346725752419e-05,
|
|
"loss_nats": 1.07178795337677,
|
|
"step": 5640,
|
|
"unclipped_grad_norm": 1.1326069831848145
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6566736193180087,
|
|
"learning_rate": 8.88420482271411e-05,
|
|
"loss_nats": 1.1483186483383179,
|
|
"step": 5650,
|
|
"unclipped_grad_norm": 1.174943447113037
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6152889180929386,
|
|
"learning_rate": 8.838193052602691e-05,
|
|
"loss_nats": 1.1196329593658447,
|
|
"step": 5660,
|
|
"unclipped_grad_norm": 1.1866281032562256
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6043573568387968,
|
|
"learning_rate": 8.792312201632352e-05,
|
|
"loss_nats": 1.112055778503418,
|
|
"step": 5670,
|
|
"unclipped_grad_norm": 1.1592957973480225
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6221047625292826,
|
|
"learning_rate": 8.746563053780261e-05,
|
|
"loss_nats": 1.1243573427200317,
|
|
"step": 5680,
|
|
"unclipped_grad_norm": 1.1609989404678345
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6421648189259206,
|
|
"learning_rate": 8.700946390773114e-05,
|
|
"loss_nats": 1.1382619142532349,
|
|
"step": 5690,
|
|
"unclipped_grad_norm": 1.1130645275115967
|
|
},
|
|
{
|
|
"bits_per_byte": 1.598431178655366,
|
|
"learning_rate": 8.655462992073833e-05,
|
|
"loss_nats": 1.1079480648040771,
|
|
"step": 5700,
|
|
"unclipped_grad_norm": 1.1522045135498047
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6208697551134366,
|
|
"learning_rate": 8.610113634868199e-05,
|
|
"loss_nats": 1.1235013008117676,
|
|
"step": 5710,
|
|
"unclipped_grad_norm": 1.1070868968963623
|
|
},
|
|
{
|
|
"bits_per_byte": 1.648464543427025,
|
|
"learning_rate": 8.564899094051613e-05,
|
|
"loss_nats": 1.14262855052948,
|
|
"step": 5720,
|
|
"unclipped_grad_norm": 1.2136235237121582
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5559942875723172,
|
|
"learning_rate": 8.519820142215816e-05,
|
|
"loss_nats": 1.0785330533981323,
|
|
"step": 5730,
|
|
"unclipped_grad_norm": 1.1237554550170898
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6165538504900345,
|
|
"learning_rate": 8.474877549635728e-05,
|
|
"loss_nats": 1.1205097436904907,
|
|
"step": 5740,
|
|
"unclipped_grad_norm": 1.1293457746505737
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5847578699811693,
|
|
"learning_rate": 8.430072084256247e-05,
|
|
"loss_nats": 1.0984704494476318,
|
|
"step": 5750,
|
|
"unclipped_grad_norm": 1.1838749647140503
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5751051717188433,
|
|
"learning_rate": 8.385404511679161e-05,
|
|
"loss_nats": 1.0917797088623047,
|
|
"step": 5760,
|
|
"unclipped_grad_norm": 1.1379411220550537
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5366526027083331,
|
|
"learning_rate": 8.340875595150034e-05,
|
|
"loss_nats": 1.0651264190673828,
|
|
"step": 5770,
|
|
"unclipped_grad_norm": 1.1725109815597534
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6123841311198657,
|
|
"learning_rate": 8.296486095545203e-05,
|
|
"loss_nats": 1.117619514465332,
|
|
"step": 5780,
|
|
"unclipped_grad_norm": 1.1070845127105713
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5625800192218067,
|
|
"learning_rate": 8.252236771358729e-05,
|
|
"loss_nats": 1.0830979347229004,
|
|
"step": 5790,
|
|
"unclipped_grad_norm": 1.0731046199798584
|
|
},
|
|
{
|
|
"bits_per_byte": 1.581076581339451,
|
|
"learning_rate": 8.208128378689484e-05,
|
|
"loss_nats": 1.0959187746047974,
|
|
"step": 5800,
|
|
"unclipped_grad_norm": 1.1906232833862305
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6024834338749956,
|
|
"learning_rate": 8.16416167122819e-05,
|
|
"loss_nats": 1.1107568740844727,
|
|
"step": 5810,
|
|
"unclipped_grad_norm": 1.198943853378296
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6068543729466738,
|
|
"learning_rate": 8.120337400244578e-05,
|
|
"loss_nats": 1.1137865781784058,
|
|
"step": 5820,
|
|
"unclipped_grad_norm": 1.1112759113311768
|
|
},
|
|
{
|
|
"bits_per_byte": 1.567694095327863,
|
|
"learning_rate": 8.076656314574521e-05,
|
|
"loss_nats": 1.0866427421569824,
|
|
"step": 5830,
|
|
"unclipped_grad_norm": 1.2037192583084106
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5486041930647423,
|
|
"learning_rate": 8.03311916060725e-05,
|
|
"loss_nats": 1.0734106302261353,
|
|
"step": 5840,
|
|
"unclipped_grad_norm": 1.1501859426498413
|
|
},
|
|
{
|
|
"bits_per_byte": 1.540087784176667,
|
|
"learning_rate": 7.989726682272606e-05,
|
|
"loss_nats": 1.0675075054168701,
|
|
"step": 5850,
|
|
"unclipped_grad_norm": 1.1106252670288086
|
|
},
|
|
{
|
|
"bits_per_byte": 1.591153904766757,
|
|
"learning_rate": 7.946479621028318e-05,
|
|
"loss_nats": 1.1029038429260254,
|
|
"step": 5860,
|
|
"unclipped_grad_norm": 1.267377257347107
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6187461133405827,
|
|
"learning_rate": 7.903378715847343e-05,
|
|
"loss_nats": 1.1220293045043945,
|
|
"step": 5870,
|
|
"unclipped_grad_norm": 1.1557857990264893
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6194158137830426,
|
|
"learning_rate": 7.860424703205224e-05,
|
|
"loss_nats": 1.1224935054779053,
|
|
"step": 5880,
|
|
"unclipped_grad_norm": 1.1605706214904785
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6297550667876128,
|
|
"learning_rate": 7.817618317067534e-05,
|
|
"loss_nats": 1.1296601295471191,
|
|
"step": 5890,
|
|
"unclipped_grad_norm": 1.1718353033065796
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5804839291245776,
|
|
"learning_rate": 7.774960288877295e-05,
|
|
"loss_nats": 1.0955079793930054,
|
|
"step": 5900,
|
|
"unclipped_grad_norm": 1.1900224685668945
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5770434161940672,
|
|
"learning_rate": 7.732451347542522e-05,
|
|
"loss_nats": 1.093123197555542,
|
|
"step": 5910,
|
|
"unclipped_grad_norm": 1.1988046169281006
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5761890063845807,
|
|
"learning_rate": 7.690092219423722e-05,
|
|
"loss_nats": 1.0925309658050537,
|
|
"step": 5920,
|
|
"unclipped_grad_norm": 1.1799134016036987
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6240858306845711,
|
|
"learning_rate": 7.647883628321536e-05,
|
|
"loss_nats": 1.1257305145263672,
|
|
"step": 5930,
|
|
"unclipped_grad_norm": 1.1704758405685425
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6230097352381225,
|
|
"learning_rate": 7.605826295464316e-05,
|
|
"loss_nats": 1.124984622001648,
|
|
"step": 5940,
|
|
"unclipped_grad_norm": 1.1741182804107666
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5942451209333652,
|
|
"learning_rate": 7.563920939495851e-05,
|
|
"loss_nats": 1.1050465106964111,
|
|
"step": 5950,
|
|
"unclipped_grad_norm": 1.223347544670105
|
|
},
|
|
{
|
|
"bits_per_byte": 1.587844614598855,
|
|
"learning_rate": 7.522168276463047e-05,
|
|
"loss_nats": 1.1006100177764893,
|
|
"step": 5960,
|
|
"unclipped_grad_norm": 1.1676011085510254
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6162547726601841,
|
|
"learning_rate": 7.480569019803719e-05,
|
|
"loss_nats": 1.120302438735962,
|
|
"step": 5970,
|
|
"unclipped_grad_norm": 1.2063930034637451
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5550433954956966,
|
|
"learning_rate": 7.439123880334388e-05,
|
|
"loss_nats": 1.077873945236206,
|
|
"step": 5980,
|
|
"unclipped_grad_norm": 1.1823816299438477
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5403349232459567,
|
|
"learning_rate": 7.39783356623813e-05,
|
|
"loss_nats": 1.0676788091659546,
|
|
"step": 5990,
|
|
"unclipped_grad_norm": 1.1445503234863281
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6087774829486217,
|
|
"learning_rate": 7.356698783052497e-05,
|
|
"loss_nats": 1.1151195764541626,
|
|
"step": 6000,
|
|
"unclipped_grad_norm": 1.182093858718872
|
|
},
|
|
{
|
|
"bits_per_byte": 1.585380103211991,
|
|
"learning_rate": 7.315720233657431e-05,
|
|
"loss_nats": 1.0989017486572266,
|
|
"step": 6010,
|
|
"unclipped_grad_norm": 1.1548501253128052
|
|
},
|
|
{
|
|
"bits_per_byte": 1.57159036228316,
|
|
"learning_rate": 7.274898618263286e-05,
|
|
"loss_nats": 1.0893434286117554,
|
|
"step": 6020,
|
|
"unclipped_grad_norm": 1.132990837097168
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6225496816470646,
|
|
"learning_rate": 7.234234634398831e-05,
|
|
"loss_nats": 1.1246657371520996,
|
|
"step": 6030,
|
|
"unclipped_grad_norm": 1.1933883428573608
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5568839538252304,
|
|
"learning_rate": 7.193728976899362e-05,
|
|
"loss_nats": 1.0791497230529785,
|
|
"step": 6040,
|
|
"unclipped_grad_norm": 1.1697132587432861
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5810753774608948,
|
|
"learning_rate": 7.153382337894797e-05,
|
|
"loss_nats": 1.0959179401397705,
|
|
"step": 6050,
|
|
"unclipped_grad_norm": 1.1444475650787354
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6146938581209531,
|
|
"learning_rate": 7.113195406797887e-05,
|
|
"loss_nats": 1.119220495223999,
|
|
"step": 6060,
|
|
"unclipped_grad_norm": 1.1966716051101685
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5827436091742642,
|
|
"learning_rate": 7.073168870292397e-05,
|
|
"loss_nats": 1.097074270248413,
|
|
"step": 6070,
|
|
"unclipped_grad_norm": 1.1267356872558594
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5673551175230125,
|
|
"learning_rate": 7.033303412321404e-05,
|
|
"loss_nats": 1.0864077806472778,
|
|
"step": 6080,
|
|
"unclipped_grad_norm": 1.097261905670166
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5774123189801679,
|
|
"learning_rate": 6.993599714075586e-05,
|
|
"loss_nats": 1.0933789014816284,
|
|
"step": 6090,
|
|
"unclipped_grad_norm": 1.1282033920288086
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5779040173789847,
|
|
"learning_rate": 6.954058453981609e-05,
|
|
"loss_nats": 1.093719720840454,
|
|
"step": 6100,
|
|
"unclipped_grad_norm": 1.1281672716140747
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5504769121499873,
|
|
"learning_rate": 6.914680307690504e-05,
|
|
"loss_nats": 1.0747087001800537,
|
|
"step": 6110,
|
|
"unclipped_grad_norm": 1.1312857866287231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5543487575688617,
|
|
"learning_rate": 6.875465948066139e-05,
|
|
"loss_nats": 1.0773924589157104,
|
|
"step": 6120,
|
|
"unclipped_grad_norm": 1.157880187034607
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5931889754744168,
|
|
"learning_rate": 6.836416045173733e-05,
|
|
"loss_nats": 1.1043144464492798,
|
|
"step": 6130,
|
|
"unclipped_grad_norm": 1.1723217964172363
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6084328297162926,
|
|
"learning_rate": 6.797531266268377e-05,
|
|
"loss_nats": 1.1148806810379028,
|
|
"step": 6140,
|
|
"unclipped_grad_norm": 1.1961045265197754
|
|
},
|
|
{
|
|
"bits_per_byte": 1.542341444833573,
|
|
"learning_rate": 6.758812275783663e-05,
|
|
"loss_nats": 1.0690696239471436,
|
|
"step": 6150,
|
|
"unclipped_grad_norm": 1.1969319581985474
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6357159854665178,
|
|
"learning_rate": 6.720259735320298e-05,
|
|
"loss_nats": 1.1337919235229492,
|
|
"step": 6160,
|
|
"unclipped_grad_norm": 1.2412800788879395
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4931134748159045,
|
|
"learning_rate": 6.681874303634842e-05,
|
|
"loss_nats": 1.034947395324707,
|
|
"step": 6170,
|
|
"unclipped_grad_norm": 1.0997314453125
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5659319610871227,
|
|
"learning_rate": 6.643656636628402e-05,
|
|
"loss_nats": 1.0854213237762451,
|
|
"step": 6180,
|
|
"unclipped_grad_norm": 1.128227710723877
|
|
},
|
|
{
|
|
"bits_per_byte": 1.581150533879322,
|
|
"learning_rate": 6.605607387335466e-05,
|
|
"loss_nats": 1.0959700345993042,
|
|
"step": 6190,
|
|
"unclipped_grad_norm": 1.1780462265014648
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5351257407339731,
|
|
"learning_rate": 6.567727205912724e-05,
|
|
"loss_nats": 1.064068078994751,
|
|
"step": 6200,
|
|
"unclipped_grad_norm": 1.13433837890625
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5793001725386888,
|
|
"learning_rate": 6.530016739627959e-05,
|
|
"loss_nats": 1.0946874618530273,
|
|
"step": 6210,
|
|
"unclipped_grad_norm": 1.1643598079681396
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5609047362197521,
|
|
"learning_rate": 6.492476632848993e-05,
|
|
"loss_nats": 1.0819367170333862,
|
|
"step": 6220,
|
|
"unclipped_grad_norm": 1.131072998046875
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5253499028936293,
|
|
"learning_rate": 6.455107527032678e-05,
|
|
"loss_nats": 1.0572919845581055,
|
|
"step": 6230,
|
|
"unclipped_grad_norm": 1.1126173734664917
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5400140036194467,
|
|
"learning_rate": 6.417910060713926e-05,
|
|
"loss_nats": 1.0674563646316528,
|
|
"step": 6240,
|
|
"unclipped_grad_norm": 1.1826318502426147
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5344780540708238,
|
|
"learning_rate": 6.380884869494799e-05,
|
|
"loss_nats": 1.0636191368103027,
|
|
"step": 6250,
|
|
"unclipped_grad_norm": 1.1302508115768433
|
|
},
|
|
{
|
|
"bits_per_byte": 1.600364263651064,
|
|
"learning_rate": 6.34403258603367e-05,
|
|
"loss_nats": 1.109287977218628,
|
|
"step": 6260,
|
|
"unclipped_grad_norm": 1.1724376678466797
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5544997583363194,
|
|
"learning_rate": 6.30735384003437e-05,
|
|
"loss_nats": 1.077497124671936,
|
|
"step": 6270,
|
|
"unclipped_grad_norm": 1.081700086593628
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6754886933050508,
|
|
"learning_rate": 6.270849258235484e-05,
|
|
"loss_nats": 1.161360263824463,
|
|
"step": 6280,
|
|
"unclipped_grad_norm": 1.2664587497711182
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5413214157313058,
|
|
"learning_rate": 6.234519464399581e-05,
|
|
"loss_nats": 1.0683625936508179,
|
|
"step": 6290,
|
|
"unclipped_grad_norm": 1.2083820104599
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5888871734283854,
|
|
"learning_rate": 6.198365079302614e-05,
|
|
"loss_nats": 1.101332664489746,
|
|
"step": 6300,
|
|
"unclipped_grad_norm": 1.2475577592849731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6228484155116132,
|
|
"learning_rate": 6.16238672072326e-05,
|
|
"loss_nats": 1.1248728036880493,
|
|
"step": 6310,
|
|
"unclipped_grad_norm": 1.2506906986236572
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5728649257087044,
|
|
"learning_rate": 6.126585003432406e-05,
|
|
"loss_nats": 1.0902268886566162,
|
|
"step": 6320,
|
|
"unclipped_grad_norm": 1.1333374977111816
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5253509347895344,
|
|
"learning_rate": 6.090960539182614e-05,
|
|
"loss_nats": 1.0572926998138428,
|
|
"step": 6330,
|
|
"unclipped_grad_norm": 1.180550217628479
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5807446548232857,
|
|
"learning_rate": 6.0555139366976935e-05,
|
|
"loss_nats": 1.0956887006759644,
|
|
"step": 6340,
|
|
"unclipped_grad_norm": 1.1644010543823242
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5717518539923203,
|
|
"learning_rate": 6.020245801662274e-05,
|
|
"loss_nats": 1.0894553661346436,
|
|
"step": 6350,
|
|
"unclipped_grad_norm": 1.142453908920288
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5231024336121541,
|
|
"learning_rate": 5.985156736711483e-05,
|
|
"loss_nats": 1.0557341575622559,
|
|
"step": 6360,
|
|
"unclipped_grad_norm": 1.1504549980163574
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6077399116159663,
|
|
"learning_rate": 5.950247341420618e-05,
|
|
"loss_nats": 1.1144003868103027,
|
|
"step": 6370,
|
|
"unclipped_grad_norm": 1.2130403518676758
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5316929670227515,
|
|
"learning_rate": 5.915518212294935e-05,
|
|
"loss_nats": 1.0616886615753174,
|
|
"step": 6380,
|
|
"unclipped_grad_norm": 1.1643673181533813
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5801346323606753,
|
|
"learning_rate": 5.880969942759425e-05,
|
|
"loss_nats": 1.0952658653259277,
|
|
"step": 6390,
|
|
"unclipped_grad_norm": 1.1411337852478027
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5548263533903077,
|
|
"learning_rate": 5.846603123148688e-05,
|
|
"loss_nats": 1.077723503112793,
|
|
"step": 6400,
|
|
"unclipped_grad_norm": 1.1154334545135498
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5450890396637573,
|
|
"learning_rate": 5.812418340696858e-05,
|
|
"loss_nats": 1.0709741115570068,
|
|
"step": 6410,
|
|
"unclipped_grad_norm": 1.137816071510315
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5323659351355776,
|
|
"learning_rate": 5.778416179527541e-05,
|
|
"loss_nats": 1.0621551275253296,
|
|
"step": 6420,
|
|
"unclipped_grad_norm": 1.1425901651382446
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5643414655319436,
|
|
"learning_rate": 5.7445972206438635e-05,
|
|
"loss_nats": 1.0843188762664795,
|
|
"step": 6430,
|
|
"unclipped_grad_norm": 1.2278902530670166
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5636070996127593,
|
|
"learning_rate": 5.710962041918516e-05,
|
|
"loss_nats": 1.0838098526000977,
|
|
"step": 6440,
|
|
"unclipped_grad_norm": 1.204923152923584
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5427707135301267,
|
|
"learning_rate": 5.677511218083908e-05,
|
|
"loss_nats": 1.0693671703338623,
|
|
"step": 6450,
|
|
"unclipped_grad_norm": 1.1532135009765625
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5522318231193915,
|
|
"learning_rate": 5.644245320722316e-05,
|
|
"loss_nats": 1.0759251117706299,
|
|
"step": 6460,
|
|
"unclipped_grad_norm": 1.5724140405654907
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5677630603708592,
|
|
"learning_rate": 5.61116491825615e-05,
|
|
"loss_nats": 1.0866905450820923,
|
|
"step": 6470,
|
|
"unclipped_grad_norm": 1.182875394821167
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5839062118940967,
|
|
"learning_rate": 5.5782705759382104e-05,
|
|
"loss_nats": 1.0978801250457764,
|
|
"step": 6480,
|
|
"unclipped_grad_norm": 1.211335301399231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.561875234318571,
|
|
"learning_rate": 5.5455628558420546e-05,
|
|
"loss_nats": 1.0826094150543213,
|
|
"step": 6490,
|
|
"unclipped_grad_norm": 1.2880244255065918
|
|
},
|
|
{
|
|
"bits_per_byte": 1.577221590220361,
|
|
"learning_rate": 5.513042316852368e-05,
|
|
"loss_nats": 1.0932466983795166,
|
|
"step": 6500,
|
|
"unclipped_grad_norm": 1.2181490659713745
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5067461955844976,
|
|
"learning_rate": 5.480709514655443e-05,
|
|
"loss_nats": 1.0443968772888184,
|
|
"step": 6510,
|
|
"unclipped_grad_norm": 1.1337521076202393
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4936307986296997,
|
|
"learning_rate": 5.448565001729653e-05,
|
|
"loss_nats": 1.0353059768676758,
|
|
"step": 6520,
|
|
"unclipped_grad_norm": 1.2117457389831543
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5784743118492457,
|
|
"learning_rate": 5.41660932733604e-05,
|
|
"loss_nats": 1.0941150188446045,
|
|
"step": 6530,
|
|
"unclipped_grad_norm": 1.2337595224380493
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5448394928373554,
|
|
"learning_rate": 5.384843037508902e-05,
|
|
"loss_nats": 1.0708011388778687,
|
|
"step": 6540,
|
|
"unclipped_grad_norm": 1.1313438415527344
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5964084906985685,
|
|
"learning_rate": 5.353266675046498e-05,
|
|
"loss_nats": 1.1065460443496704,
|
|
"step": 6550,
|
|
"unclipped_grad_norm": 1.2166260480880737
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5631484218829081,
|
|
"learning_rate": 5.321880779501729e-05,
|
|
"loss_nats": 1.0834919214248657,
|
|
"step": 6560,
|
|
"unclipped_grad_norm": 1.1645575761795044
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4810062401604644,
|
|
"learning_rate": 5.290685887172962e-05,
|
|
"loss_nats": 1.0265552997589111,
|
|
"step": 6570,
|
|
"unclipped_grad_norm": 1.2005696296691895
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5898447728283895,
|
|
"learning_rate": 5.2596825310948295e-05,
|
|
"loss_nats": 1.1019964218139648,
|
|
"step": 6580,
|
|
"unclipped_grad_norm": 1.1795482635498047
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5634753609021985,
|
|
"learning_rate": 5.228871241029154e-05,
|
|
"loss_nats": 1.0837185382843018,
|
|
"step": 6590,
|
|
"unclipped_grad_norm": 1.1887481212615967
|
|
},
|
|
{
|
|
"bits_per_byte": 1.47844799822888,
|
|
"learning_rate": 5.198252543455865e-05,
|
|
"loss_nats": 1.0247820615768433,
|
|
"step": 6600,
|
|
"unclipped_grad_norm": 1.1429756879806519
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6001293353499855,
|
|
"learning_rate": 5.167826961564027e-05,
|
|
"loss_nats": 1.1091251373291016,
|
|
"step": 6610,
|
|
"unclipped_grad_norm": 1.1761175394058228
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6007649832275743,
|
|
"learning_rate": 5.137595015242884e-05,
|
|
"loss_nats": 1.1095657348632812,
|
|
"step": 6620,
|
|
"unclipped_grad_norm": 1.2725090980529785
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5866947385951862,
|
|
"learning_rate": 5.1075572210729925e-05,
|
|
"loss_nats": 1.0998129844665527,
|
|
"step": 6630,
|
|
"unclipped_grad_norm": 1.242309808731079
|
|
},
|
|
{
|
|
"bits_per_byte": 1.557243053600232,
|
|
"learning_rate": 5.07771409231737e-05,
|
|
"loss_nats": 1.0793986320495605,
|
|
"step": 6640,
|
|
"unclipped_grad_norm": 1.2104532718658447
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5462134622350983,
|
|
"learning_rate": 5.048066138912756e-05,
|
|
"loss_nats": 1.0717535018920898,
|
|
"step": 6650,
|
|
"unclipped_grad_norm": 1.1630686521530151
|
|
},
|
|
{
|
|
"bits_per_byte": 1.588834890702523,
|
|
"learning_rate": 5.018613867460863e-05,
|
|
"loss_nats": 1.1012964248657227,
|
|
"step": 6660,
|
|
"unclipped_grad_norm": 1.204805612564087
|
|
},
|
|
{
|
|
"bits_per_byte": 1.587033544417386,
|
|
"learning_rate": 4.989357781219755e-05,
|
|
"loss_nats": 1.1000478267669678,
|
|
"step": 6670,
|
|
"unclipped_grad_norm": 1.2499597072601318
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5912992301067361,
|
|
"learning_rate": 4.960298380095222e-05,
|
|
"loss_nats": 1.1030045747756958,
|
|
"step": 6680,
|
|
"unclipped_grad_norm": 1.2160732746124268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.56477726956923,
|
|
"learning_rate": 4.931436160632256e-05,
|
|
"loss_nats": 1.0846209526062012,
|
|
"step": 6690,
|
|
"unclipped_grad_norm": 1.2467725276947021
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6040180350686444,
|
|
"learning_rate": 4.9027716160065485e-05,
|
|
"loss_nats": 1.1118205785751343,
|
|
"step": 6700,
|
|
"unclipped_grad_norm": 1.267776370048523
|
|
},
|
|
{
|
|
"bits_per_byte": 1.537844786444114,
|
|
"learning_rate": 4.874305236016084e-05,
|
|
"loss_nats": 1.0659527778625488,
|
|
"step": 6710,
|
|
"unclipped_grad_norm": 1.2301596403121948
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5327245189626266,
|
|
"learning_rate": 4.846037507072752e-05,
|
|
"loss_nats": 1.062403678894043,
|
|
"step": 6720,
|
|
"unclipped_grad_norm": 1.180388331413269
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5337890915714674,
|
|
"learning_rate": 4.817968912194049e-05,
|
|
"loss_nats": 1.0631415843963623,
|
|
"step": 6730,
|
|
"unclipped_grad_norm": 1.2269387245178223
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5501341507268178,
|
|
"learning_rate": 4.790099930994813e-05,
|
|
"loss_nats": 1.074471116065979,
|
|
"step": 6740,
|
|
"unclipped_grad_norm": 1.1682806015014648
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5184903748639662,
|
|
"learning_rate": 4.762431039679047e-05,
|
|
"loss_nats": 1.0525373220443726,
|
|
"step": 6750,
|
|
"unclipped_grad_norm": 1.23702871799469
|
|
},
|
|
{
|
|
"bits_per_byte": 1.500455070215936,
|
|
"learning_rate": 4.734962711031755e-05,
|
|
"loss_nats": 1.0400362014770508,
|
|
"step": 6760,
|
|
"unclipped_grad_norm": 1.1704752445220947
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5177950490065277,
|
|
"learning_rate": 4.707695414410892e-05,
|
|
"loss_nats": 1.0520553588867188,
|
|
"step": 6770,
|
|
"unclipped_grad_norm": 1.209053874015808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5265771710901863,
|
|
"learning_rate": 4.680629615739322e-05,
|
|
"loss_nats": 1.0581426620483398,
|
|
"step": 6780,
|
|
"unclipped_grad_norm": 1.2882649898529053
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6213150181965206,
|
|
"learning_rate": 4.653765777496863e-05,
|
|
"loss_nats": 1.1238099336624146,
|
|
"step": 6790,
|
|
"unclipped_grad_norm": 1.1679272651672363
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5863846538756805,
|
|
"learning_rate": 4.6271043587123986e-05,
|
|
"loss_nats": 1.0995980501174927,
|
|
"step": 6800,
|
|
"unclipped_grad_norm": 1.280809998512268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.578056394007649,
|
|
"learning_rate": 4.600645814956006e-05,
|
|
"loss_nats": 1.093825340270996,
|
|
"step": 6810,
|
|
"unclipped_grad_norm": 1.24360191822052
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5169633409069554,
|
|
"learning_rate": 4.574390598331204e-05,
|
|
"loss_nats": 1.0514788627624512,
|
|
"step": 6820,
|
|
"unclipped_grad_norm": 1.2292150259017944
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5347601056182387,
|
|
"learning_rate": 4.548339157467199e-05,
|
|
"loss_nats": 1.063814640045166,
|
|
"step": 6830,
|
|
"unclipped_grad_norm": 1.242634892463684
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5555476486280264,
|
|
"learning_rate": 4.52249193751124e-05,
|
|
"loss_nats": 1.078223466873169,
|
|
"step": 6840,
|
|
"unclipped_grad_norm": 1.2094796895980835
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5217322478327302,
|
|
"learning_rate": 4.4968493801209985e-05,
|
|
"loss_nats": 1.0547844171524048,
|
|
"step": 6850,
|
|
"unclipped_grad_norm": 1.1779916286468506
|
|
},
|
|
{
|
|
"bits_per_byte": 1.549958384457636,
|
|
"learning_rate": 4.471411923457034e-05,
|
|
"loss_nats": 1.074349284172058,
|
|
"step": 6860,
|
|
"unclipped_grad_norm": 1.2639429569244385
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5571720247654257,
|
|
"learning_rate": 4.446180002175291e-05,
|
|
"loss_nats": 1.079349398612976,
|
|
"step": 6870,
|
|
"unclipped_grad_norm": 1.2982949018478394
|
|
},
|
|
{
|
|
"bits_per_byte": 1.597770421310751,
|
|
"learning_rate": 4.421154047419693e-05,
|
|
"loss_nats": 1.107490062713623,
|
|
"step": 6880,
|
|
"unclipped_grad_norm": 1.2857638597488403
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4860104193526196,
|
|
"learning_rate": 4.396334486814753e-05,
|
|
"loss_nats": 1.0300239324569702,
|
|
"step": 6890,
|
|
"unclipped_grad_norm": 1.2555214166641235
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5064882216082034,
|
|
"learning_rate": 4.371721744458285e-05,
|
|
"loss_nats": 1.0442180633544922,
|
|
"step": 6900,
|
|
"unclipped_grad_norm": 1.202001690864563
|
|
},
|
|
{
|
|
"bits_per_byte": 1.685866814388716,
|
|
"learning_rate": 4.347316240914145e-05,
|
|
"loss_nats": 1.1685538291931152,
|
|
"step": 6910,
|
|
"unclipped_grad_norm": 1.3379162549972534
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5053493524941899,
|
|
"learning_rate": 4.3231183932050546e-05,
|
|
"loss_nats": 1.043428659439087,
|
|
"step": 6920,
|
|
"unclipped_grad_norm": 1.1770343780517578
|
|
},
|
|
{
|
|
"bits_per_byte": 1.516240325842728,
|
|
"learning_rate": 4.299128614805464e-05,
|
|
"loss_nats": 1.0509777069091797,
|
|
"step": 6930,
|
|
"unclipped_grad_norm": 1.2326538562774658
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5463919802266939,
|
|
"learning_rate": 4.2753473156344965e-05,
|
|
"loss_nats": 1.0718772411346436,
|
|
"step": 6940,
|
|
"unclipped_grad_norm": 1.2218632698059082
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5410003241221448,
|
|
"learning_rate": 4.251774902048942e-05,
|
|
"loss_nats": 1.0681400299072266,
|
|
"step": 6950,
|
|
"unclipped_grad_norm": 1.2248505353927612
|
|
},
|
|
{
|
|
"bits_per_byte": 1.574234251574874,
|
|
"learning_rate": 4.228411776836306e-05,
|
|
"loss_nats": 1.0911760330200195,
|
|
"step": 6960,
|
|
"unclipped_grad_norm": 1.2728900909423828
|
|
},
|
|
{
|
|
"bits_per_byte": 1.556295085228676,
|
|
"learning_rate": 4.2052583392079427e-05,
|
|
"loss_nats": 1.0787415504455566,
|
|
"step": 6970,
|
|
"unclipped_grad_norm": 1.2423036098480225
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6109532354646872,
|
|
"learning_rate": 4.182314984792214e-05,
|
|
"loss_nats": 1.1166276931762695,
|
|
"step": 6980,
|
|
"unclipped_grad_norm": 1.2731002569198608
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5507145921734797,
|
|
"learning_rate": 4.159582105627748e-05,
|
|
"loss_nats": 1.074873447418213,
|
|
"step": 6990,
|
|
"unclipped_grad_norm": 1.2125478982925415
|
|
},
|
|
{
|
|
"bits_per_byte": 1.506560110356264,
|
|
"learning_rate": 4.137060090156724e-05,
|
|
"loss_nats": 1.0442678928375244,
|
|
"step": 7000,
|
|
"unclipped_grad_norm": 1.2366911172866821
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5357397187975532,
|
|
"learning_rate": 4.114749323218249e-05,
|
|
"loss_nats": 1.0644936561584473,
|
|
"step": 7010,
|
|
"unclipped_grad_norm": 1.2639423608779907
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5037485379799589,
|
|
"learning_rate": 4.0926501860417706e-05,
|
|
"loss_nats": 1.0423190593719482,
|
|
"step": 7020,
|
|
"unclipped_grad_norm": 1.2011712789535522
|
|
},
|
|
{
|
|
"bits_per_byte": 1.528263632964547,
|
|
"learning_rate": 4.070763056240571e-05,
|
|
"loss_nats": 1.0593116283416748,
|
|
"step": 7030,
|
|
"unclipped_grad_norm": 1.1849545240402222
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5983039114937276,
|
|
"learning_rate": 4.049088307805306e-05,
|
|
"loss_nats": 1.1078598499298096,
|
|
"step": 7040,
|
|
"unclipped_grad_norm": 1.3057087659835815
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5441888824691414,
|
|
"learning_rate": 4.027626311097629e-05,
|
|
"loss_nats": 1.070350170135498,
|
|
"step": 7050,
|
|
"unclipped_grad_norm": 1.2155181169509888
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5741694141154987,
|
|
"learning_rate": 4.006377432843846e-05,
|
|
"loss_nats": 1.0911310911178589,
|
|
"step": 7060,
|
|
"unclipped_grad_norm": 1.3050142526626587
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5094996378248111,
|
|
"learning_rate": 3.985342036128656e-05,
|
|
"loss_nats": 1.0463054180145264,
|
|
"step": 7070,
|
|
"unclipped_grad_norm": 1.1835323572158813
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4442604269771167,
|
|
"learning_rate": 3.964520480388956e-05,
|
|
"loss_nats": 1.0010850429534912,
|
|
"step": 7080,
|
|
"unclipped_grad_norm": 1.167786717414856
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5378045425038123,
|
|
"learning_rate": 3.943913121407678e-05,
|
|
"loss_nats": 1.065924882888794,
|
|
"step": 7090,
|
|
"unclipped_grad_norm": 1.240479588508606
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5019334330827527,
|
|
"learning_rate": 3.923520311307735e-05,
|
|
"loss_nats": 1.0410609245300293,
|
|
"step": 7100,
|
|
"unclipped_grad_norm": 1.1977860927581787
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5780336922977352,
|
|
"learning_rate": 3.9033423985459816e-05,
|
|
"loss_nats": 1.0938096046447754,
|
|
"step": 7110,
|
|
"unclipped_grad_norm": 1.2797318696975708
|
|
},
|
|
{
|
|
"bits_per_byte": 1.56051364767169,
|
|
"learning_rate": 3.88337972790728e-05,
|
|
"loss_nats": 1.0816656351089478,
|
|
"step": 7120,
|
|
"unclipped_grad_norm": 1.297847032546997
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4993894657111901,
|
|
"learning_rate": 3.8636326404985845e-05,
|
|
"loss_nats": 1.0392975807189941,
|
|
"step": 7130,
|
|
"unclipped_grad_norm": 1.224601149559021
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6077283887783587,
|
|
"learning_rate": 3.844101473743143e-05,
|
|
"loss_nats": 1.1143923997879028,
|
|
"step": 7140,
|
|
"unclipped_grad_norm": 1.3082928657531738
|
|
},
|
|
{
|
|
"bits_per_byte": 1.624090990164097,
|
|
"learning_rate": 3.824786561374701e-05,
|
|
"loss_nats": 1.1257340908050537,
|
|
"step": 7150,
|
|
"unclipped_grad_norm": 1.2189027070999146
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5477993142587043,
|
|
"learning_rate": 3.805688233431824e-05,
|
|
"loss_nats": 1.0728527307510376,
|
|
"step": 7160,
|
|
"unclipped_grad_norm": 1.2660733461380005
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5516317756505311,
|
|
"learning_rate": 3.786806816252236e-05,
|
|
"loss_nats": 1.0755091905593872,
|
|
"step": 7170,
|
|
"unclipped_grad_norm": 1.2526509761810303
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4692106380683878,
|
|
"learning_rate": 3.768142632467269e-05,
|
|
"loss_nats": 1.0183792114257812,
|
|
"step": 7180,
|
|
"unclipped_grad_norm": 1.1470739841461182
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4673153892558797,
|
|
"learning_rate": 3.7496960009963236e-05,
|
|
"loss_nats": 1.0170655250549316,
|
|
"step": 7190,
|
|
"unclipped_grad_norm": 1.2286535501480103
|
|
},
|
|
{
|
|
"bits_per_byte": 1.553365876719181,
|
|
"learning_rate": 3.7314672370414325e-05,
|
|
"loss_nats": 1.0767111778259277,
|
|
"step": 7200,
|
|
"unclipped_grad_norm": 1.2306180000305176
|
|
},
|
|
{
|
|
"bits_per_byte": 1.561453876823957,
|
|
"learning_rate": 3.713456652081882e-05,
|
|
"loss_nats": 1.0823173522949219,
|
|
"step": 7210,
|
|
"unclipped_grad_norm": 1.2458006143569946
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5086355969868763,
|
|
"learning_rate": 3.695664553868873e-05,
|
|
"loss_nats": 1.0457065105438232,
|
|
"step": 7220,
|
|
"unclipped_grad_norm": 1.1648746728897095
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5304512522835219,
|
|
"learning_rate": 3.678091246420277e-05,
|
|
"loss_nats": 1.0608279705047607,
|
|
"step": 7230,
|
|
"unclipped_grad_norm": 1.2855236530303955
|
|
},
|
|
{
|
|
"bits_per_byte": 1.493701999447157,
|
|
"learning_rate": 3.660737030015427e-05,
|
|
"loss_nats": 1.0353553295135498,
|
|
"step": 7240,
|
|
"unclipped_grad_norm": 1.2490638494491577
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5430964486708607,
|
|
"learning_rate": 3.6436022011900056e-05,
|
|
"loss_nats": 1.0695929527282715,
|
|
"step": 7250,
|
|
"unclipped_grad_norm": 1.286012887954712
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5196966611771179,
|
|
"learning_rate": 3.626687052730954e-05,
|
|
"loss_nats": 1.0533734560012817,
|
|
"step": 7260,
|
|
"unclipped_grad_norm": 1.2369425296783447
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5993731276341416,
|
|
"learning_rate": 3.6099918736714955e-05,
|
|
"loss_nats": 1.1086009740829468,
|
|
"step": 7270,
|
|
"unclipped_grad_norm": 1.2697807550430298
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5305549578219921,
|
|
"learning_rate": 3.5935169492861716e-05,
|
|
"loss_nats": 1.0608998537063599,
|
|
"step": 7280,
|
|
"unclipped_grad_norm": 1.2743209600448608
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5476247518680786,
|
|
"learning_rate": 3.5772625610859886e-05,
|
|
"loss_nats": 1.0727317333221436,
|
|
"step": 7290,
|
|
"unclipped_grad_norm": 1.2424356937408447
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5365045256459402,
|
|
"learning_rate": 3.5612289868135887e-05,
|
|
"loss_nats": 1.0650237798690796,
|
|
"step": 7300,
|
|
"unclipped_grad_norm": 1.298574686050415
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5764911799021466,
|
|
"learning_rate": 3.545416500438522e-05,
|
|
"loss_nats": 1.0927404165267944,
|
|
"step": 7310,
|
|
"unclipped_grad_norm": 1.3189024925231934
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5490052566065542,
|
|
"learning_rate": 3.52982537215255e-05,
|
|
"loss_nats": 1.0736886262893677,
|
|
"step": 7320,
|
|
"unclipped_grad_norm": 1.2497724294662476
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5554974296939745,
|
|
"learning_rate": 3.5144558683650396e-05,
|
|
"loss_nats": 1.0781886577606201,
|
|
"step": 7330,
|
|
"unclipped_grad_norm": 1.279703140258789
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5241250424541843,
|
|
"learning_rate": 3.499308251698404e-05,
|
|
"loss_nats": 1.0564429759979248,
|
|
"step": 7340,
|
|
"unclipped_grad_norm": 1.2488007545471191
|
|
},
|
|
{
|
|
"bits_per_byte": 1.57090466745417,
|
|
"learning_rate": 3.484382780983616e-05,
|
|
"loss_nats": 1.0888681411743164,
|
|
"step": 7350,
|
|
"unclipped_grad_norm": 1.238014578819275
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5500565865512788,
|
|
"learning_rate": 3.469679711255795e-05,
|
|
"loss_nats": 1.0744173526763916,
|
|
"step": 7360,
|
|
"unclipped_grad_norm": 1.2180218696594238
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5597980278614498,
|
|
"learning_rate": 3.4551992937498284e-05,
|
|
"loss_nats": 1.081169605255127,
|
|
"step": 7370,
|
|
"unclipped_grad_norm": 1.2368392944335938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6215841710451209,
|
|
"learning_rate": 3.440941775896104e-05,
|
|
"loss_nats": 1.1239964962005615,
|
|
"step": 7380,
|
|
"unclipped_grad_norm": 1.3172329664230347
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5928735592927343,
|
|
"learning_rate": 3.4269074013162596e-05,
|
|
"loss_nats": 1.1040958166122437,
|
|
"step": 7390,
|
|
"unclipped_grad_norm": 1.3043875694274902
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5057738056765193,
|
|
"learning_rate": 3.41309640981904e-05,
|
|
"loss_nats": 1.0437228679656982,
|
|
"step": 7400,
|
|
"unclipped_grad_norm": 1.2024799585342407
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5152624324899222,
|
|
"learning_rate": 3.3995090373961795e-05,
|
|
"loss_nats": 1.050299882888794,
|
|
"step": 7410,
|
|
"unclipped_grad_norm": 1.2088910341262817
|
|
},
|
|
{
|
|
"bits_per_byte": 1.597047062281222,
|
|
"learning_rate": 3.3861455162183876e-05,
|
|
"loss_nats": 1.1069886684417725,
|
|
"step": 7420,
|
|
"unclipped_grad_norm": 1.3177212476730347
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5299019396966662,
|
|
"learning_rate": 3.3730060746313664e-05,
|
|
"loss_nats": 1.0604472160339355,
|
|
"step": 7430,
|
|
"unclipped_grad_norm": 1.290786862373352
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5250764504787575,
|
|
"learning_rate": 3.3600909371519225e-05,
|
|
"loss_nats": 1.0571024417877197,
|
|
"step": 7440,
|
|
"unclipped_grad_norm": 1.2367006540298462
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4892672548120085,
|
|
"learning_rate": 3.3474003244641194e-05,
|
|
"loss_nats": 1.0322813987731934,
|
|
"step": 7450,
|
|
"unclipped_grad_norm": 1.3495734930038452
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5046927227331957,
|
|
"learning_rate": 3.334934453415514e-05,
|
|
"loss_nats": 1.042973518371582,
|
|
"step": 7460,
|
|
"unclipped_grad_norm": 1.2188435792922974
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5481360562890936,
|
|
"learning_rate": 3.3226935370134486e-05,
|
|
"loss_nats": 1.073086142539978,
|
|
"step": 7470,
|
|
"unclipped_grad_norm": 1.1978274583816528
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4779339420854511,
|
|
"learning_rate": 3.31067778442141e-05,
|
|
"loss_nats": 1.024425745010376,
|
|
"step": 7480,
|
|
"unclipped_grad_norm": 1.25648832321167
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5058897219832008,
|
|
"learning_rate": 3.298887400955457e-05,
|
|
"loss_nats": 1.0438032150268555,
|
|
"step": 7490,
|
|
"unclipped_grad_norm": 1.2401517629623413
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5728836718176484,
|
|
"learning_rate": 3.287322588080714e-05,
|
|
"loss_nats": 1.0902398824691772,
|
|
"step": 7500,
|
|
"unclipped_grad_norm": 1.2475262880325317
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5378353273983165,
|
|
"learning_rate": 3.275983543407924e-05,
|
|
"loss_nats": 1.0659462213516235,
|
|
"step": 7510,
|
|
"unclipped_grad_norm": 1.2484221458435059
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5312337733449477,
|
|
"learning_rate": 3.2648704606900735e-05,
|
|
"loss_nats": 1.0613703727722168,
|
|
"step": 7520,
|
|
"unclipped_grad_norm": 1.1939175128936768
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5599504044901142,
|
|
"learning_rate": 3.253983529819086e-05,
|
|
"loss_nats": 1.081275224685669,
|
|
"step": 7530,
|
|
"unclipped_grad_norm": 1.2631168365478516
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5976060058965262,
|
|
"learning_rate": 3.243322936822571e-05,
|
|
"loss_nats": 1.1073760986328125,
|
|
"step": 7540,
|
|
"unclipped_grad_norm": 1.3131186962127686
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5182279293387495,
|
|
"learning_rate": 3.232888863860648e-05,
|
|
"loss_nats": 1.052355408668518,
|
|
"step": 7550,
|
|
"unclipped_grad_norm": 1.2190253734588623
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5653272700866891,
|
|
"learning_rate": 3.222681489222838e-05,
|
|
"loss_nats": 1.0850021839141846,
|
|
"step": 7560,
|
|
"unclipped_grad_norm": 1.2437095642089844
|
|
},
|
|
{
|
|
"bits_per_byte": 1.566524613301996,
|
|
"learning_rate": 3.212700987325009e-05,
|
|
"loss_nats": 1.085832118988037,
|
|
"step": 7570,
|
|
"unclipped_grad_norm": 1.2195208072662354
|
|
},
|
|
{
|
|
"bits_per_byte": 1.514368810636039,
|
|
"learning_rate": 3.202947528706403e-05,
|
|
"loss_nats": 1.049680471420288,
|
|
"step": 7580,
|
|
"unclipped_grad_norm": 1.2027956247329712
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5218865162705542,
|
|
"learning_rate": 3.1934212800267116e-05,
|
|
"loss_nats": 1.0548913478851318,
|
|
"step": 7590,
|
|
"unclipped_grad_norm": 1.22147798538208
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5124059726417418,
|
|
"learning_rate": 3.1841224040632484e-05,
|
|
"loss_nats": 1.048319935798645,
|
|
"step": 7600,
|
|
"unclipped_grad_norm": 1.2925957441329956
|
|
},
|
|
{
|
|
"bits_per_byte": 1.451400458710337,
|
|
"learning_rate": 3.175051059708143e-05,
|
|
"loss_nats": 1.0060341358184814,
|
|
"step": 7610,
|
|
"unclipped_grad_norm": 1.22130286693573
|
|
},
|
|
{
|
|
"bits_per_byte": 1.554240580481469,
|
|
"learning_rate": 3.1662074019656414e-05,
|
|
"loss_nats": 1.077317476272583,
|
|
"step": 7620,
|
|
"unclipped_grad_norm": 1.2743861675262451
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5089792183233002,
|
|
"learning_rate": 3.157591581949455e-05,
|
|
"loss_nats": 1.0459446907043457,
|
|
"step": 7630,
|
|
"unclipped_grad_norm": 1.1950352191925049
|
|
},
|
|
{
|
|
"bits_per_byte": 1.523324463214418,
|
|
"learning_rate": 3.149203746880175e-05,
|
|
"loss_nats": 1.055888056755066,
|
|
"step": 7640,
|
|
"unclipped_grad_norm": 1.2816646099090576
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6003928127711071,
|
|
"learning_rate": 3.141044040082763e-05,
|
|
"loss_nats": 1.1093077659606934,
|
|
"step": 7650,
|
|
"unclipped_grad_norm": 1.3542457818984985
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5396506042781735,
|
|
"learning_rate": 3.133112600984091e-05,
|
|
"loss_nats": 1.067204475402832,
|
|
"step": 7660,
|
|
"unclipped_grad_norm": 1.277113437652588
|
|
},
|
|
{
|
|
"bits_per_byte": 1.515347907867401,
|
|
"learning_rate": 3.125409565110571e-05,
|
|
"loss_nats": 1.0503591299057007,
|
|
"step": 7670,
|
|
"unclipped_grad_norm": 1.3265182971954346
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4941618810555641,
|
|
"learning_rate": 3.117935064085835e-05,
|
|
"loss_nats": 1.0356740951538086,
|
|
"step": 7680,
|
|
"unclipped_grad_norm": 1.2858244180679321
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5394590156051124,
|
|
"learning_rate": 3.110689225628479e-05,
|
|
"loss_nats": 1.0670716762542725,
|
|
"step": 7690,
|
|
"unclipped_grad_norm": 1.2726463079452515
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5472200766905984,
|
|
"learning_rate": 3.1036721735498926e-05,
|
|
"loss_nats": 1.0724512338638306,
|
|
"step": 7700,
|
|
"unclipped_grad_norm": 1.3285934925079346
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4615679010466955,
|
|
"learning_rate": 3.096884027752132e-05,
|
|
"loss_nats": 1.013081669807434,
|
|
"step": 7710,
|
|
"unclipped_grad_norm": 1.2261079549789429
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4667946257890672,
|
|
"learning_rate": 3.090324904225885e-05,
|
|
"loss_nats": 1.0167045593261719,
|
|
"step": 7720,
|
|
"unclipped_grad_norm": 1.24894380569458
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5217016349208765,
|
|
"learning_rate": 3.083994915048472e-05,
|
|
"loss_nats": 1.0547631978988647,
|
|
"step": 7730,
|
|
"unclipped_grad_norm": 1.2570104598999023
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5186754281962946,
|
|
"learning_rate": 3.077894168381942e-05,
|
|
"loss_nats": 1.0526655912399292,
|
|
"step": 7740,
|
|
"unclipped_grad_norm": 1.2608413696289062
|
|
},
|
|
{
|
|
"bits_per_byte": 1.530832365837834,
|
|
"learning_rate": 3.0720227684712254e-05,
|
|
"loss_nats": 1.0610921382904053,
|
|
"step": 7750,
|
|
"unclipped_grad_norm": 1.268287181854248
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5045843736631521,
|
|
"learning_rate": 3.0663808156423456e-05,
|
|
"loss_nats": 1.042898416519165,
|
|
"step": 7760,
|
|
"unclipped_grad_norm": 1.2538435459136963
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5028014295216574,
|
|
"learning_rate": 3.060968406300711e-05,
|
|
"loss_nats": 1.041662573814392,
|
|
"step": 7770,
|
|
"unclipped_grad_norm": 1.3365142345428467
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6460007199707645,
|
|
"learning_rate": 3.055785632929463e-05,
|
|
"loss_nats": 1.1409207582473755,
|
|
"step": 7780,
|
|
"unclipped_grad_norm": 1.3043659925460815
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5196737874845532,
|
|
"learning_rate": 3.0508325840878984e-05,
|
|
"loss_nats": 1.0533576011657715,
|
|
"step": 7790,
|
|
"unclipped_grad_norm": 1.2850605249404907
|
|
},
|
|
{
|
|
"bits_per_byte": 1.559599903847656,
|
|
"learning_rate": 3.046109344409957e-05,
|
|
"loss_nats": 1.0810322761535645,
|
|
"step": 7800,
|
|
"unclipped_grad_norm": 1.3824384212493896
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4742115495901766,
|
|
"learning_rate": 3.0416159946027727e-05,
|
|
"loss_nats": 1.0218455791473389,
|
|
"step": 7810,
|
|
"unclipped_grad_norm": 1.2828388214111328
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5144011433744011,
|
|
"learning_rate": 3.0373526114452946e-05,
|
|
"loss_nats": 1.0497028827667236,
|
|
"step": 7820,
|
|
"unclipped_grad_norm": 1.3298958539962769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.549842984098907,
|
|
"learning_rate": 3.0333192677869803e-05,
|
|
"loss_nats": 1.0742692947387695,
|
|
"step": 7830,
|
|
"unclipped_grad_norm": 1.2448348999023438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5177059619933808,
|
|
"learning_rate": 3.029516032546544e-05,
|
|
"loss_nats": 1.0519936084747314,
|
|
"step": 7840,
|
|
"unclipped_grad_norm": 1.234107255935669
|
|
},
|
|
{
|
|
"bits_per_byte": 1.486972662284197,
|
|
"learning_rate": 3.02594297071078e-05,
|
|
"loss_nats": 1.0306909084320068,
|
|
"step": 7850,
|
|
"unclipped_grad_norm": 1.2269891500473022
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4944172752920954,
|
|
"learning_rate": 3.0226001433334603e-05,
|
|
"loss_nats": 1.0358511209487915,
|
|
"step": 7860,
|
|
"unclipped_grad_norm": 1.266727328300476
|
|
},
|
|
{
|
|
"bits_per_byte": 1.549148518154723,
|
|
"learning_rate": 3.019487607534278e-05,
|
|
"loss_nats": 1.0737879276275635,
|
|
"step": 7870,
|
|
"unclipped_grad_norm": 1.3428150415420532
|
|
},
|
|
{
|
|
"bits_per_byte": 1.522268833703422,
|
|
"learning_rate": 3.016605416497886e-05,
|
|
"loss_nats": 1.0551563501358032,
|
|
"step": 7880,
|
|
"unclipped_grad_norm": 1.305385947227478
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5751703531435204,
|
|
"learning_rate": 3.0139536194729734e-05,
|
|
"loss_nats": 1.0918248891830444,
|
|
"step": 7890,
|
|
"unclipped_grad_norm": 1.2760140895843506
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5609104116472305,
|
|
"learning_rate": 3.0115322617714362e-05,
|
|
"loss_nats": 1.0819406509399414,
|
|
"step": 7900,
|
|
"unclipped_grad_norm": 1.2964600324630737
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5527064952357728,
|
|
"learning_rate": 3.0093413847675965e-05,
|
|
"loss_nats": 1.07625412940979,
|
|
"step": 7910,
|
|
"unclipped_grad_norm": 1.3213040828704834
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4860267577044515,
|
|
"learning_rate": 3.0073810258974985e-05,
|
|
"loss_nats": 1.0300352573394775,
|
|
"step": 7920,
|
|
"unclipped_grad_norm": 1.2681679725646973
|
|
},
|
|
{
|
|
"bits_per_byte": 1.482107273091288,
|
|
"learning_rate": 3.0056512186582658e-05,
|
|
"loss_nats": 1.0273184776306152,
|
|
"step": 7930,
|
|
"unclipped_grad_norm": 1.2925180196762085
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5584990428994832,
|
|
"learning_rate": 3.0041519926075316e-05,
|
|
"loss_nats": 1.08026921749115,
|
|
"step": 7940,
|
|
"unclipped_grad_norm": 1.4040197134017944
|
|
},
|
|
{
|
|
"bits_per_byte": 1.488459280318255,
|
|
"learning_rate": 3.0028833733629313e-05,
|
|
"loss_nats": 1.0317213535308838,
|
|
"step": 7950,
|
|
"unclipped_grad_norm": 1.3141639232635498
|
|
},
|
|
{
|
|
"bits_per_byte": 1.530331896323823,
|
|
"learning_rate": 3.0018453826016686e-05,
|
|
"loss_nats": 1.0607452392578125,
|
|
"step": 7960,
|
|
"unclipped_grad_norm": 1.263742446899414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5034190192209056,
|
|
"learning_rate": 3.001038038060143e-05,
|
|
"loss_nats": 1.042090654373169,
|
|
"step": 7970,
|
|
"unclipped_grad_norm": 1.2354859113693237
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5362255697862408,
|
|
"learning_rate": 3.0004613535336433e-05,
|
|
"loss_nats": 1.0648304224014282,
|
|
"step": 7980,
|
|
"unclipped_grad_norm": 1.2929013967514038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5535658925421343,
|
|
"learning_rate": 3.000115338876117e-05,
|
|
"loss_nats": 1.0768498182296753,
|
|
"step": 7990,
|
|
"unclipped_grad_norm": 1.2544541358947754
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4716885641020192,
|
|
"learning_rate": 3e-05,
|
|
"loss_nats": 1.020096778869629,
|
|
"step": 8000,
|
|
"unclipped_grad_norm": 1.2711973190307617
|
|
}
|
|
]
|
|
}
|