wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: preregister AttnRes local path study

Browse Source
This commit is contained in:
wuyang
2026-07-30 12:46:59 +08:00
parent 5e64990a7d
commit 6911efc6e7
5 changed files with 1362 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/k3/attnres_local_path/README.md
+22
View File
@@ -0,0 +1,22 @@
# Attention Residuals local mixer-path diagnostics
This directory implements preregistered protocol
`llm-atlas-k3-attnres-local-path-v1`.
It is a targeted follow-up to Round 06. It exact-replays the same depth-32
Block training and keeps the learned forward unchanged while switching source
value-gradient coefficients only at frozen mixer scopes. It is not a Kimi K3
checkpoint run, a trainable variant, an additive attribution, or a reproduction
of unpublished Figure 5 telemetry.
## Frozen environment
```text
Python /home/wuyang/.pyenv/versions/3.10.14/envs/navi-router-cu128/bin/python
PyTorch 2.11.0+cu128
GPU NVIDIA GeForce RTX 5090
CUBLAS_WORKSPACE_CONFIG=:4096:8
```
The runner and exact commands will be added after this preregistration
manifest is committed. No result file may precede that commit.
experiments/k3/attnres_local_path/manifest.json
+540
View File
@@ -0,0 +1,540 @@
{
"schema_version": 1,
"protocol_id": "llm-atlas-k3-attnres-local-path-v1",
"parent_protocol_id": "llm-atlas-k3-attnres-spike-path-v1",
"study_identity": "targeted local-path follow-up informed by Round 05 and Round 06; not blind discovery",
"architecture": "block",
"depth": 32,
"aggregation_groups": 8,
"blocks_per_group": 4,
"depth_mixers": 64,
"output_mixers": 1,
"formal_seeds": [
2026073001,
2026073002,
2026073003
],
"replay": {
"architecture": "block",
"depth": 32,
"seed": 2026073001,
"environment_scope": "same host, GPU, Python, PyTorch, CUDA and CUBLAS_WORKSPACE_CONFIG"
},
"training": {
"steps": 8000,
"batch_size": 32,
"context": 256,
"target_bytes_per_cell": 65536000,
"parent_diagnostic_steps": [
0,
100,
500,
2000,
4000,
8000
],
"local_matrix_steps": [
0,
8000
],
"matrix_used_during_training": false
},
"primary_object": {
"position": "post_mlp_state",
"reduction": "element_rms",
"spike_layers_one_based": [
21,
22,
23,
24,
25
],
"metrics": [
"spike_contrast",
"peak_normalized"
]
},
"matrix_modes": [
"detached_learned",
"uniform_group_6_only",
"uniform_group_7_only",
"uniform_groups_6_7_only",
"uniform_group_6_attention_only",
"uniform_group_6_mlp_only",
"uniform_group_7_attention_only",
"uniform_group_7_mlp_only",
"uniform_output_only",
"uniform_depth_all",
"uniform_all",
"uniform_except_group_6",
"uniform_except_group_7",
"uniform_except_groups_6_7"
],
"selector": {
"coefficient_choices": [
"detached_learned",
"uniform"
],
"full_autograd_learned_in_matrix": false,
"depth_identity": "kind=depth,index=0..63; layer=floor(index/2)+1; attention iff index even",
"output_identity": "kind=output,index=64; parallel-schema alias only; parent mixer_index remains null",
"uniform_depth_indices": {
"detached_learned": [],
"uniform_group_6_only": [
40,
41,
42,
43,
44,
45,
46,
47
],
"uniform_group_7_only": [
48,
49,
50,
51,
52,
53,
54,
55
],
"uniform_groups_6_7_only": [
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55
],
"uniform_group_6_attention_only": [
40,
42,
44,
46
],
"uniform_group_6_mlp_only": [
41,
43,
45,
47
],
"uniform_group_7_attention_only": [
48,
50,
52,
54
],
"uniform_group_7_mlp_only": [
49,
51,
53,
55
],
"uniform_output_only": [],
"uniform_depth_all": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63
],
"uniform_all": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63
],
"uniform_except_group_6": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63
],
"uniform_except_group_7": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
56,
57,
58,
59,
60,
61,
62,
63
],
"uniform_except_groups_6_7": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
56,
57,
58,
59,
60,
61,
62,
63
]
},
"uniform_output": {
"detached_learned": false,
"uniform_group_6_only": false,
"uniform_group_7_only": false,
"uniform_groups_6_7_only": false,
"uniform_group_6_attention_only": false,
"uniform_group_6_mlp_only": false,
"uniform_group_7_attention_only": false,
"uniform_group_7_mlp_only": false,
"uniform_output_only": true,
"uniform_depth_all": false,
"uniform_all": true,
"uniform_except_group_6": true,
"uniform_except_group_7": true,
"uniform_except_groups_6_7": true
},
"expected_uniform_counts": {
"detached_learned": 0,
"uniform_group_6_only": 8,
"uniform_group_7_only": 8,
"uniform_groups_6_7_only": 16,
"uniform_group_6_attention_only": 4,
"uniform_group_6_mlp_only": 4,
"uniform_group_7_attention_only": 4,
"uniform_group_7_mlp_only": 4,
"uniform_output_only": 1,
"uniform_depth_all": 64,
"uniform_all": 65,
"uniform_except_group_6": 57,
"uniform_except_group_7": 57,
"uniform_except_groups_6_7": 49
}
},
"thresholds": {
"positive_denominator_epsilon": 1e-30,
"step0_spectrum_tolerance": 1e-6,
"loss_scale_tolerance": 1e-5,
"global_relative_drop_minimum": 0.2,
"groups_6_7_sufficiency_minimum": 0.5,
"groups_6_7_restoration_minimum": 0.5,
"single_group_material_minimum": 0.2,
"branch_material_minimum": 0.2,
"branch_dominance_margin": 0.15,
"output_half_gap_minimum": 0.5,
"formal_seed_gate": "3/3 independently for both metrics; means are display-only"
},
"formulas": {
"global_log_gap": "G_X = ln(X_ref / X_uniform_all)",
"global_relative_drop": "(X_ref - X_uniform_all) / X_ref",
"sufficiency": "S_X(m) = ln(X_ref / X_m) / G_X",
"restoration": "R_X(r) = ln(X_r / X_uniform_all) / G_X",
"score_clipping": false
},
"parent_artifacts": {
"manifest_path": "experiments/k3/attnres_spike/manifest.json",
"manifest_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef",
"runner_path": "experiments/k3/attnres_spike/train.py",
"runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084",
"protocol_path": "research/K3_ATTNRES_SPIKE_PROTOCOL.md",
"protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16",
"scoping_path": "research/K3_ATTNRES_SPIKE_SCOPING.md",
"scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338",
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716"
},
"current_artifacts": {
"protocol_path": "research/K3_ATTNRES_LOCAL_PATH_PROTOCOL.md",
"protocol_sha256": "5ecc7ca92314ddb50aecf0cb50e115814c8983aa8bffb30e3634f7b3ce6dca1d",
"scoping_path": "research/K3_ATTNRES_LOCAL_PATH_SCOPING.md",
"scoping_sha256": "670ca4edf31a4be1f54937d9c7a760dba7a96e1e820c38c6b10405e22b078fc8",
"grok_review_path": "research/K3_ATTNRES_LOCAL_PATH_GROK_REVIEW.md",
"grok_review_sha256": "2da1b6bf1f455c4121a7a2c5cfe40e102327dabafc7e24dccc23ed0d00ac6d71",
"grok_session": "019fb151-9627-76c1-b7d7-53012874f85c"
},
"round06_expected": {
"2026073001": {
"raw_file_sha256": "e39e93b7a7fce3c56f5f14f95cfdc04afdce53628affee1202fe62bd1bdb7f71",
"canonical_sha256": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515",
"final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59",
"final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637"
},
"2026073002": {
"raw_file_sha256": "1c6f6c731030ec0adb2a8e7a4d586e0c4005cc3319568a7ac83c08c2a4b8eaf8",
"canonical_sha256": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78",
"final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1",
"final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77"
},
"2026073003": {
"raw_file_sha256": "115f8245577ece6dfaaa8ada68445c186e6523a7f3b26efcc3eb4c0c4ce82406",
"canonical_sha256": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766",
"final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76",
"final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e"
}
}
}