wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: publish AttnRes forward training study

Browse Source
This commit is contained in:
wuyang
2026-07-30 18:41:18 +08:00
parent 7ea91caabb
commit f177fa676d
47 changed files with 192164 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/k3/attnres_forward/results/gates/step-zero.json
+21 -11
View File
@@ -1,5 +1,5 @@
{
"canonical_sha256_without_self": "00db0569d2fd00599383ecdc50578c52df784a937a96dd69576f1366e88b0dcb",
"canonical_sha256_without_self": "9a0716bc496d39b9447b79aba0aec42ca4c953cbabe30e3aa349c1bba0701c5f",
"environment": {
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
@@ -21,7 +21,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -1254,7 +1255,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -1267,7 +1269,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -2500,7 +2503,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -2562,7 +2566,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -3795,7 +3800,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -3857,7 +3863,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -5090,7 +5097,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -5128,7 +5136,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -6361,7 +6370,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
experiments/k3/attnres_forward/results/gates/wrapper-smoke.json
+259 -131
View File
@@ -1,7 +1,7 @@
{
"architecture": "block",
"batch_size": 32,
"canonical_sha256_without_self": "a467cc1c05e1b860abbf44086273071bcd2a891db4354cc6e79b319e44d1cce5",
"canonical_sha256_without_self": "81b011e2ab02c49abb6b2686b3e0701d71ab6e7f46cbb7c426eec91702836b7f",
"depth": 32,
"diagnostics": [
{
@@ -2479,74 +2479,74 @@
],
"forward_intervention": {
"depth_visit_counts": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39
],
"forward_calls": 0,
"forward_calls": 39,
"output_mixer_selected": false,
"output_visit_count": 0,
"output_visit_count": 39,
"passed": true,
"selected_depth_indices": [],
"selected_parameter_reachability_gate": true,
@@ -2555,70 +2555,198 @@
"selector_gate": true,
"semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel",
"source_counts_by_depth_index": {
"0": [],
"1": [],
"10": [],
"11": [],
"12": [],
"13": [],
"14": [],
"15": [],
"16": [],
"17": [],
"18": [],
"19": [],
"2": [],
"20": [],
"21": [],
"22": [],
"23": [],
"24": [],
"25": [],
"26": [],
"27": [],
"28": [],
"29": [],
"3": [],
"30": [],
"31": [],
"32": [],
"33": [],
"34": [],
"35": [],
"36": [],
"37": [],
"38": [],
"39": [],
"4": [],
"40": [],
"41": [],
"42": [],
"43": [],
"44": [],
"45": [],
"46": [],
"47": [],
"48": [],
"49": [],
"5": [],
"50": [],
"51": [],
"52": [],
"53": [],
"54": [],
"55": [],
"56": [],
"57": [],
"58": [],
"59": [],
"6": [],
"60": [],
"61": [],
"62": [],
"63": [],
"7": [],
"8": [],
"9": []
"0": [
1
],
"1": [
2
],
"10": [
3
],
"11": [
3
],
"12": [
3
],
"13": [
3
],
"14": [
3
],
"15": [
3
],
"16": [
3
],
"17": [
4
],
"18": [
4
],
"19": [
4
],
"2": [
2
],
"20": [
4
],
"21": [
4
],
"22": [
4
],
"23": [
4
],
"24": [
4
],
"25": [
5
],
"26": [
5
],
"27": [
5
],
"28": [
5
],
"29": [
5
],
"3": [
2
],
"30": [
5
],
"31": [
5
],
"32": [
5
],
"33": [
6
],
"34": [
6
],
"35": [
6
],
"36": [
6
],
"37": [
6
],
"38": [
6
],
"39": [
6
],
"4": [
2
],
"40": [
6
],
"41": [
7
],
"42": [
7
],
"43": [
7
],
"44": [
7
],
"45": [
7
],
"46": [
7
],
"47": [
7
],
"48": [
7
],
"49": [
8
],
"5": [
2
],
"50": [
8
],
"51": [
8
],
"52": [
8
],
"53": [
8
],
"54": [
8
],
"55": [
8
],
"56": [
8
],
"57": [
9
],
"58": [
9
],
"59": [
9
],
"6": [
2
],
"60": [
9
],
"61": [
9
],
"62": [
9
],
"63": [
9
],
"7": [
2
],
"8": [
2
],
"9": [
3
]
},
"uniform_weight_gate": true,
"uniform_weight_max_abs_error": 0.0,