research: audit AttnRes spike path study
This commit is contained in:
wuyang
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"artifacts": {
|
||||
"analyzer": "b9dd7f723e8c5765ecffd9ae3050ff7a95643f95853adaeeaa2c2c323d0ca057",
|
||||
"manifest": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef",
|
||||
"protocol": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16",
|
||||
"runner": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084",
|
||||
"scoping": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27"
|
||||
},
|
||||
"canonical_sha256_without_self": "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515",
|
||||
"protocol_id": "llm-atlas-k3-attnres-spike-path-v1",
|
||||
"raw_files": {
|
||||
"formal-seed-2026073001.json": {
|
||||
"canonical_sha256": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515",
|
||||
"file_sha256": "e39e93b7a7fce3c56f5f14f95cfdc04afdce53628affee1202fe62bd1bdb7f71"
|
||||
},
|
||||
"formal-seed-2026073002.json": {
|
||||
"canonical_sha256": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78",
|
||||
"file_sha256": "1c6f6c731030ec0adb2a8e7a4d586e0c4005cc3319568a7ac83c08c2a4b8eaf8"
|
||||
},
|
||||
"formal-seed-2026073003.json": {
|
||||
"canonical_sha256": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766",
|
||||
"file_sha256": "115f8245577ece6dfaaa8ada68445c186e6523a7f3b26efcc3eb4c0c4ce82406"
|
||||
},
|
||||
"replay-seed-2026073001.json": {
|
||||
"canonical_sha256": "c9ebccba0acf10b05fea65beade50e588fd9a83b1807f7ebba8ec1ab836b26fb",
|
||||
"file_sha256": "4f91c7a2698662ea69af9aa3425e144107834fd9b6b717fb82b84f153b867dc5"
|
||||
}
|
||||
},
|
||||
"replay_gate": {
|
||||
"checks": {
|
||||
"architecture": true,
|
||||
"artifacts": true,
|
||||
"batch_size": true,
|
||||
"depth": true,
|
||||
"diagnostics": true,
|
||||
"environment": true,
|
||||
"evaluations": true,
|
||||
"hashes": true,
|
||||
"manifest": true,
|
||||
"model": true,
|
||||
"optimizer": true,
|
||||
"round05_equivalence": true,
|
||||
"seed": true,
|
||||
"steps": true,
|
||||
"target_bytes_seen": true,
|
||||
"training_history": true
|
||||
},
|
||||
"frozen_compare_sha256": "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87",
|
||||
"passed": true
|
||||
},
|
||||
"schema_version": 1
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,7 @@
|
||||
"check:data:deepseek-chat-task-bootstrap": "node scripts/check-deepseek-chat-task-bootstrap-crn-data.mjs",
|
||||
"check:data:k3-attnres": "node scripts/check-k3-attnres-data.mjs",
|
||||
"check:data:k3-attnres-gradient": "node scripts/check-k3-attnres-gradient-data.mjs",
|
||||
"check:data:k3-attnres-spike": "node scripts/check-k3-attnres-spike-data.mjs",
|
||||
"check:site": "node scripts/check-site.mjs",
|
||||
"check:moe-browser": "node scripts/check-moe-browser.mjs",
|
||||
"check:reasoning-browser": "node scripts/check-reasoning-browser.mjs",
|
||||
|
||||
@@ -0,0 +1,503 @@
|
||||
# K3 Attention Residuals 局部梯度尖峰与反向路由审计
|
||||
|
||||
研究日期：2026-07-30
|
||||
协议 ID：`llm-atlas-k3-attnres-spike-path-v1`
|
||||
父协议：`llm-atlas-k3-attnres-gradient-scale-v1`
|
||||
研究身份：**Round 05 已知异常驱动的定向机制追踪，不是盲发现**
|
||||
|
||||
## 0. 一句话结果
|
||||
|
||||
在与 Round 05 **model / optimizer / BPC / post-MLP gradient 全部 exact** 的
|
||||
depth-32 Block AttnRes 三 seed 重放中：
|
||||
|
||||
1. layer 21–25 的局部峰在六个捕获位置最终都 3 / 3 可见；
|
||||
2. 四种预注册 Token-gradient reductions 的 12 / 12 单元全部过闸；
|
||||
3. 全局切断 64 个 depth mixers 与 output mixer 的 softmax / query / key
|
||||
source-gradient path，峰没有下降，反而在 3 / 3 seed 略增；
|
||||
4. 保持 learned forward 完全不变、只把所有 mixer 的 value backward coefficients
|
||||
从 learned `w` 改为 `1/N` 后，spike contrast 平均下降 **70.2%**、peak normalized
|
||||
平均下降 **37.0%**，两项都在 3 / 3 seed 超过预注册 20% 阈值；
|
||||
5. seed-1 从初始化完整重训 8,000 steps 的 replay，连六位置、七 reductions、三模式和
|
||||
65 个 mixer quantile summaries 都 exact。
|
||||
|
||||
最窄、最准确的结论是：
|
||||
|
||||
> 在这个已训练的缩小 Block AttnRes 图上，最终 layer 21–25 峰对 learned mixer 的
|
||||
> **value-backward routing coefficients** 有预注册阈值下的强全局敏感性；它不是由
|
||||
> softmax/key source-gradient derivative path 单独解释的。
|
||||
|
||||
这不等于：
|
||||
|
||||
- uniform backward 是更好的训练方法；
|
||||
- learned value weights 让模型训练变差；
|
||||
- 某一个 layer 或同层 MLP mixer 是唯一原因；
|
||||
- Kimi K3 真实 checkpoint 存在同样尖峰。
|
||||
|
||||
## 1. 研究问题从哪里来
|
||||
|
||||
Round 05 已知：
|
||||
|
||||
- Block 的首/末四分位失衡在 6 / 6 配对中改善；
|
||||
- 全层 CV 却在 6 / 6 配对中恶化；
|
||||
- depth-32 三 seed 平均的相对高值集中在 layer 21–25；
|
||||
- post-MLP gradient 与 MLP latest-source mean weight 在 96 个
|
||||
`seed × layer` 点上 Pearson `r≈0.651`，在 layer 19–28 的 30 点上 `r≈0.690`。
|
||||
|
||||
这些 observational correlations 不能回答：
|
||||
|
||||
- 是 softmax / key 的导数路径重要，还是 value coefficients 重要？
|
||||
- 峰是否只是 element RMS 的 reduction 产物？
|
||||
- 峰在一个 block 内到哪个位置才出现？
|
||||
|
||||
因此 Round 06 在看见新结果前固定：
|
||||
|
||||
- 六个 activation 位置；
|
||||
- 四个主 reductions、两个 cancellation-sensitive reductions 和一个代数控制；
|
||||
- 三种 same-forward backward rules；
|
||||
- fixed spike set `S={21,22,23,24,25}`；
|
||||
- `contrast≥1.5`、top-5 overlap、Spearman、20% material drop；
|
||||
- 三 formal seeds 和一个完整 replay。
|
||||
|
||||
前置 scoping、协议与 runner 分别在提交 `204f599`、`54c80fb` 前后进入公开仓库，
|
||||
formal 训练随后才开始。
|
||||
|
||||
## 2. 对抗审查如何改变协议
|
||||
|
||||
本机 Grok Headless 只用于一次只读、对抗性方法审查，不提供事实来源，也没有修改文件。
|
||||
主代理采纳并在结果前修正的关键问题包括：
|
||||
|
||||
- 把“最早生成位置”降级为“最早可见 tensor”；
|
||||
- 主判定时点固定为 step 8,000；
|
||||
- intervention 明确作用于全部 64 个 depth mixers 加 output mixer；
|
||||
- 结论从局部因果分解降级为全局 backward-rule sensitivity；
|
||||
- material 判定固定只用 `element_rms`；
|
||||
- median / p95 固定 Hyndman–Fan Type 7；
|
||||
- top-5 ties、Spearman ties、正分母与 invalid 条件全部写死；
|
||||
- 增加六位置 loss×2、optimizer isolation 和训练不得进入 custom autograd 的闸门。
|
||||
|
||||
这次审查的价值是发现协议漏洞，不是为结果背书。
|
||||
|
||||
## 3. 第一次 smoke 为什么被拦住
|
||||
|
||||
最初 intervention custom backward 直接用 elementwise multiplication 分配
|
||||
`weights × grad_output`。forward identity 已经 exact，但 step-0 uniform negative
|
||||
control 失败：
|
||||
|
||||
```text
|
||||
六位置 max raw relative difference：约 0.0037–0.0046
|
||||
六位置 max normalized difference：约 0.0011–0.0020
|
||||
```
|
||||
|
||||
这说明虽然代数公式一致，手写 backward 没有复用父 runner autocast einsum 的实际数值
|
||||
路径，不能满足预注册的“query=0 时三模式等价”。
|
||||
|
||||
正式训练尚未开始。实现随后改为：
|
||||
|
||||
1. forward 直接返回父 `DepthMixer.forward` 已算出的 `y_parent`；
|
||||
2. backward 在 `torch.enable_grad()` 下，用冻结的 learned / uniform weights
|
||||
重新走同一个 BF16-autocast einsum surrogate；
|
||||
3. 只把 surrogate 对 source values 的梯度返回；
|
||||
4. 不向 `y_parent`、query、key path 回传。
|
||||
|
||||
修正后：
|
||||
|
||||
```text
|
||||
三模式 logits / loss / 六位置 activations：byte-exact
|
||||
65 个 mixer summaries：exact
|
||||
step-0 六位置 raw gradient spectra：max difference 0
|
||||
step-0 normalized spectra：max difference 0
|
||||
六位置 × 七 reductions loss×2：scale / shape error 0
|
||||
两个全新进程非计时字段：exact
|
||||
```
|
||||
|
||||
这个失败不是需要隐藏的“调试噪声”；它正是负控制在正式结果前发挥作用的证据。
|
||||
|
||||
## 4. 训练合同与工作量
|
||||
|
||||
Round 06 没有创建新的训练分布：
|
||||
|
||||
| 字段 | 值 |
|
||||
|---|---:|
|
||||
| architecture | Block AttnRes |
|
||||
| depth | 32 Transformer blocks |
|
||||
| groups | 8，每组 4 blocks |
|
||||
| width / heads / FFN | 192 / 6 / 768 |
|
||||
| context / batch | 256 / 32 |
|
||||
| steps | 8,000 |
|
||||
| formal seeds | 2026073001 / 2 / 3 |
|
||||
| replay | seed-1，从初始化完整 8,000 steps |
|
||||
| formal target bytes | 196,608,000 |
|
||||
| 含 replay target bytes | 262,144,000 |
|
||||
| diagnostic tensor | 固定 16 × 256 |
|
||||
| diagnostic steps | 0 / 100 / 500 / 2,000 / 4,000 / 8,000 |
|
||||
| intervention steps | 0 / 8,000 |
|
||||
|
||||
数据与父协议 exact，完整值为：
|
||||
|
||||
```text
|
||||
formal schedule:
|
||||
5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e
|
||||
|
||||
validation:
|
||||
f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338
|
||||
|
||||
diagnostic:
|
||||
21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716
|
||||
```
|
||||
|
||||
## 5. 父协议训练等价闸门
|
||||
|
||||
三个 formal seed 全部满足：
|
||||
|
||||
| 字段 | seed-1 | seed-2 | seed-3 |
|
||||
|---|---:|---:|---:|
|
||||
| final model-state hash | exact | exact | exact |
|
||||
| final optimizer-state hash | exact | exact | exact |
|
||||
| 六个 validation BPC | exact | exact | exact |
|
||||
| 六条 post-MLP element-RMS arrays | exact | exact | exact |
|
||||
| training history | exact | exact | exact |
|
||||
|
||||
最终 BPC：
|
||||
|
||||
| seed | Round 06 | Round 05 |
|
||||
|---:|---:|---:|
|
||||
| 2026073001 | 1.7123525940760473 | exact |
|
||||
| 2026073002 | 1.7093240655856785 | exact |
|
||||
| 2026073003 | 1.7030966812870931 | exact |
|
||||
|
||||
因此新增 capture / diagnostics 没有改变 optimizer trajectory。Round 06 不是一组“看起来
|
||||
相近”的新模型，而是同一冻结训练的独立数值重放。
|
||||
|
||||
## 6. 峰在训练中形成，不是初始化常数
|
||||
|
||||
最终主指标是 post-MLP element-RMS：
|
||||
|
||||
```text
|
||||
spike_contrast =
|
||||
mean(layers 21–25) / mean(other 27 layers)
|
||||
```
|
||||
|
||||
时间轨迹：
|
||||
|
||||
| step | seed-1 | seed-2 | seed-3 |
|
||||
|---:|---:|---:|---:|
|
||||
| 0 | 0.782 | 0.691 | 0.797 |
|
||||
| 100 | 0.560 | 0.556 | 0.575 |
|
||||
| 500 | 0.522 | 0.573 | 0.478 |
|
||||
| 2,000 | 2.497 | 2.740 | **1.461** |
|
||||
| 4,000 | 2.851 | 3.157 | 1.773 |
|
||||
| 8,000 | **3.046** | **3.333** | **1.881** |
|
||||
|
||||
step 500 时三个 seed 的目标层都低于其他层均值；到 step 2,000，seed-1/2 已形成明显峰，
|
||||
seed-3 仍略低于预注册 1.5 阈值；seed-3 到 step 4,000 才跨过。
|
||||
|
||||
所以允许说：
|
||||
|
||||
> 尖峰在这个训练过程中发展出来，且 onset 有 seed 差异。
|
||||
|
||||
不允许说：
|
||||
|
||||
> aggregation boundary 本身在初始化就必然产生 layer 21 峰。
|
||||
|
||||
## 7. 六个位置：处处可见，不等于找到 origin
|
||||
|
||||
最终 element-RMS spike contrast：
|
||||
|
||||
| 位置 | seed-1 | seed-2 | seed-3 | 三 seed mean | 3/3 ≥1.5 |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| pre-attention input | 3.724 | 3.189 | 2.040 | 2.984 | 是 |
|
||||
| attention branch output | 3.200 | 3.662 | 2.005 | 2.955 | 是 |
|
||||
| post-attention state | 3.200 | 3.662 | 2.005 | 2.955 | 是 |
|
||||
| pre-MLP input | 3.829 | 4.527 | 2.978 | **3.778** | 是 |
|
||||
| MLP branch output | 3.046 | 3.333 | 1.881 | 2.754 | 是 |
|
||||
| post-MLP state | 3.046 | 3.333 | 1.881 | 2.754 | 是 |
|
||||
|
||||
协议顺序中的第一个可见 tensor 是 `pre_attention_input`。但 activation gradient
|
||||
本来就汇总全部下游路径，所以这只说明：
|
||||
|
||||
> 在进入该层 attention 之前，固定深度模式已经能在损失对该 tensor 的敏感性中看到。
|
||||
|
||||
它不说明峰在 pre-attention mixer “生成”。六个位置全部可见，反而证明单靠位置 trace
|
||||
无法定位唯一 origin。
|
||||
|
||||
值得注意的是，`pre_mlp_input` 的平均 contrast 最高，但这仍是读数，不是“MLP 是唯一
|
||||
原因”的消融证据。
|
||||
|
||||
## 8. reduction robustness：12 / 12 过闸
|
||||
|
||||
主 family：
|
||||
|
||||
1. element RMS；
|
||||
2. mean token RMS；
|
||||
3. median token RMS；
|
||||
4. p95 token RMS。
|
||||
|
||||
每个 seed / reduction 必须同时满足：
|
||||
|
||||
- contrast ≥1.5；
|
||||
- top-5 与固定集合至少重合 3；
|
||||
- 与 element-RMS 32-layer Spearman ≥0.8。
|
||||
|
||||
结果：
|
||||
|
||||
| reduction | seed contrasts | 最低 top-5 overlap | 最低 Spearman | verdict |
|
||||
|---|---|---:|---:|---:|
|
||||
| element RMS | 3.046 / 3.333 / 1.881 | 3 | 1.000 | pass |
|
||||
| token RMS mean | 2.869 / 3.182 / 1.808 | 3 | 0.958 | pass |
|
||||
| token RMS median | 2.646 / 2.905 / 1.666 | 3 | **0.895** | pass |
|
||||
| token RMS p95 | 3.143 / 3.532 / 2.012 | 3 | 0.993 | pass |
|
||||
|
||||
总判定：
|
||||
|
||||
```text
|
||||
12 / 12 cells pass
|
||||
robust within the preregistered reduction family
|
||||
```
|
||||
|
||||
`global_l2` 与 element RMS 的固定 shape 代数控制也通过。batch-mean / token-mean
|
||||
reduction 同样保存，但它们允许正负梯度先抵消，按协议只作探索性读数，不进入 robustness
|
||||
判定。
|
||||
|
||||
这个结果支持：
|
||||
|
||||
> layer 21–25 峰不是只由“先平方再平均”的单一 reduction 选择制造的。
|
||||
|
||||
它不支持：
|
||||
|
||||
> 论文所有可能的 gradient magnitude 定义都会给出同一曲线。
|
||||
|
||||
## 9. same-forward interventions 到底改了什么
|
||||
|
||||
三模式：
|
||||
|
||||
### learned
|
||||
|
||||
原计算图：
|
||||
|
||||
```text
|
||||
w = softmax(qᵀ RMSNorm(sources))
|
||||
y = Σ w_i source_i
|
||||
```
|
||||
|
||||
### detached learned
|
||||
|
||||
forward 的 `w`、`y`、logits、loss 完全相同；backward 对 sources 保留 learned
|
||||
value coefficients，但切断 softmax / query / key 的 source-gradient path。
|
||||
|
||||
### uniform value backward
|
||||
|
||||
forward 仍逐字节返回 learned `y`；backward 对每个 source 返回 `grad_y/N`，不向
|
||||
softmax / query / key 回传。
|
||||
|
||||
作用域是全部 64 个 depth mixers 加 output mixer。它不是只改 layer 21 附近，也不是一次
|
||||
训练消融。
|
||||
|
||||
三个 seed 的最终 forward identity 均满足：
|
||||
|
||||
```text
|
||||
logits hash exact
|
||||
loss exact
|
||||
六位置 activation hashes exact
|
||||
65 mixer summary hash exact
|
||||
```
|
||||
|
||||
所以三模式的梯度差不能归因于不同 forward activation 或不同预测。
|
||||
|
||||
## 10. softmax / key path：没有降低峰
|
||||
|
||||
learned → detached learned：
|
||||
|
||||
| seed | contrast | detached | relative drop | peak relative drop |
|
||||
|---:|---:|---:|---:|---:|
|
||||
| 2026073001 | 3.046 | 3.093 | −1.54% | −0.67% |
|
||||
| 2026073002 | 3.333 | 3.397 | −1.93% | −0.38% |
|
||||
| 2026073003 | 1.881 | 1.929 | −2.53% | −1.47% |
|
||||
| mean | — | — | **−2.00%** | **−0.84%** |
|
||||
|
||||
“relative drop”为负表示 detaching 后峰略高。预注册规则要求 contrast 与 peak 都下降至少
|
||||
20%、3 / 3 seed 同向；实际是两项在 3 / 3 seed 都朝相反方向。
|
||||
|
||||
结论：
|
||||
|
||||
> 全局移除 softmax / query / key 的 source-gradient derivative path，没有在本阈值下
|
||||
> 降低固定峰；它在三个 seed 中都使峰略增。
|
||||
|
||||
这不证明该路径处处为零，也不证明它对训练没有作用。它只否定本轮预注册的
|
||||
“material peak reduction”方向。
|
||||
|
||||
## 11. learned value coefficients：3 / 3 过 material 闸门
|
||||
|
||||
detached learned → uniform value backward：
|
||||
|
||||
| seed | detached contrast | uniform contrast | contrast drop | peak drop |
|
||||
|---:|---:|---:|---:|---:|
|
||||
| 2026073001 | 3.093 | 0.780 | **74.8%** | **39.3%** |
|
||||
| 2026073002 | 3.397 | 0.786 | **76.9%** | **43.2%** |
|
||||
| 2026073003 | 1.929 | 0.792 | **58.9%** | **28.6%** |
|
||||
| mean | — | — | **70.2%** | **37.0%** |
|
||||
|
||||
六个相对下降全部超过预注册 20%。uniform value backward 下：
|
||||
|
||||
- 固定 target-layer contrast 都低于 0.8；
|
||||
- peak layer 从 learned / detached 的 layer 21 移到 layer 2；
|
||||
- forward logits、loss 与 activation 完全不变。
|
||||
|
||||
预注册判定：
|
||||
|
||||
```text
|
||||
material sensitivity at the preregistered threshold
|
||||
3 / 3 seeds
|
||||
```
|
||||
|
||||
允许说：
|
||||
|
||||
> 在最终已训练计算图上，learned source value-backward coefficients 是该局部梯度形状的
|
||||
> material carrier / amplifier。
|
||||
|
||||
禁止说：
|
||||
|
||||
- “把训练改成 uniform backward 一定更好”；
|
||||
- “learned mixer 导致 BPC 下降或上升”；
|
||||
- “峰完全由同层 MLP latest weight造成”；
|
||||
- “70.2% 是因果贡献率”。
|
||||
|
||||
相对下降是对一个**全局反向规则干预**的指标变化，不是可加的路径归因百分比。
|
||||
|
||||
## 12. mixer association 与 intervention 如何一起读
|
||||
|
||||
最终 observational association：
|
||||
|
||||
| scope | metric | Pearson | Spearman |
|
||||
|---|---|---:|---:|
|
||||
| 96 seed×layer | MLP latest mass | 0.651 | 0.692 |
|
||||
| 96 seed×layer | MLP normalized entropy | −0.274 | −0.417 |
|
||||
| layer 19–28 | MLP latest mass | 0.690 | 0.693 |
|
||||
| layer 19–28 | MLP max mass | 0.702 | 0.705 |
|
||||
| layer 19–28 | MLP normalized entropy | −0.636 | −0.640 |
|
||||
| layer 19–28 | attention latest mass | 0.158 | 0.339 |
|
||||
|
||||
Round 06 的 intervention 让“value coefficients 值得关注”从单纯相关线索升级为
|
||||
same-forward backward sensitivity。但它仍然是**全部 mixers 同时改写**，所以不能把
|
||||
全局结果定向归因给表中同层 MLP latest mass。
|
||||
|
||||
更窄的下一步需要：
|
||||
|
||||
- 逐 group / 逐 mixer intervention；
|
||||
- output mixer 与 64 个 depth mixers 分开；
|
||||
- current partial、completed group、embedding source 分开；
|
||||
- 保持 forward exact 的局部 backward path ablation。
|
||||
|
||||
这些没有在本轮结果后追加，以免事后挑选 layer。
|
||||
|
||||
## 13. replay 与完整复现
|
||||
|
||||
seed-1 replay 从初始化完整训练 8,000 steps，不加载 formal checkpoint。
|
||||
|
||||
exact 字段：
|
||||
|
||||
```text
|
||||
architecture / depth / seed / steps / batch / target bytes
|
||||
manifest / model / optimizer contracts
|
||||
model and optimizer state hashes
|
||||
six validation evaluations
|
||||
all training-history checkpoints
|
||||
six diagnostic steps
|
||||
six positions
|
||||
seven reductions
|
||||
three intervention modes at step 0 / 8,000
|
||||
65 mixer summaries with quantiles
|
||||
forward identity / negative control / loss-scale gates
|
||||
environment / artifact identities
|
||||
Round 05 equivalence record
|
||||
```
|
||||
|
||||
冻结 compare SHA-256：
|
||||
|
||||
```text
|
||||
05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87
|
||||
```
|
||||
|
||||
排除计时、run kind 和输出路径。replay verdict：`exact`。
|
||||
|
||||
## 14. 成本数字的正确解释
|
||||
|
||||
formal 三个进程的 mean step time 都约 224 ms；peak allocated 约 12.84 GB，
|
||||
reserved 约 13.92 GB。
|
||||
|
||||
它们与 Round 05 单进程约 109 ms 不可直接比较，因为本轮两进程并行竞争同一 RTX 5090，
|
||||
GPU 利用率 100%。训练 step 本身走父 runner exact 路径，新增诊断只发生在冻结时点。
|
||||
|
||||
因此：
|
||||
|
||||
- 计时证明实验真实执行；
|
||||
- 计时不用于评价新诊断 overhead；
|
||||
- 更不能外推到 K3 训练系统。
|
||||
|
||||
## 15. 工件与哈希
|
||||
|
||||
核心源码：
|
||||
|
||||
| 工件 | SHA-256 |
|
||||
|---|---|
|
||||
| manifest | `d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef` |
|
||||
| protocol | `6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16` |
|
||||
| scoping | `590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27` |
|
||||
| runner | `77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084` |
|
||||
| analyzer | `b9dd7f723e8c5765ecffd9ae3050ff7a95643f95853adaeeaa2c2c323d0ca057` |
|
||||
|
||||
raw physical hashes：
|
||||
|
||||
| 文件 | SHA-256 |
|
||||
|---|---|
|
||||
| formal seed-1 | `e39e93b7a7fce3c56f5f14f95cfdc04afdce53628affee1202fe62bd1bdb7f71` |
|
||||
| formal seed-2 | `1c6f6c731030ec0adb2a8e7a4d586e0c4005cc3319568a7ac83c08c2a4b8eaf8` |
|
||||
| formal seed-3 | `115f8245577ece6dfaaa8ada68445c186e6523a7f3b26efcc3eb4c0c4ce82406` |
|
||||
| replay seed-1 | `4f91c7a2698662ea69af9aa3425e144107834fd9b6b717fb82b84f153b867dc5` |
|
||||
|
||||
聚合工件：
|
||||
|
||||
| 工件 | physical SHA-256 | canonical SHA-256 |
|
||||
|---|---|---|
|
||||
| full aggregate | `75058f9e…c1e90` | `1d0faef2…fc8b4` |
|
||||
| compact | `7b3a3749…8ef6c` | `a430dd26…26cd` |
|
||||
| reproduction | `86ac20ee…879a` | `7c522314…3515` |
|
||||
|
||||
完整值保存在文件自身和 `reproduction.json`，缩写只用于阅读。
|
||||
|
||||
## 16. 允许与禁止的最终表述
|
||||
|
||||
### 允许
|
||||
|
||||
> 固定 layer 21–25 峰在四种预注册 Token-gradient reductions 中 12 / 12 过闸，
|
||||
> 所以它在本轮 reduction family 内稳健。
|
||||
|
||||
> 在同一个 learned forward 上，全局 value-backward coefficient uniformization 使
|
||||
> contrast / peak 在 3 / 3 seed 超过 20% 下降阈值。
|
||||
|
||||
> 全局移除 softmax/key source-gradient path 没有降低峰，反而在 3 / 3 seed 略增。
|
||||
|
||||
> 峰从训练中发展，且 seed-3 的形成更晚、更弱。
|
||||
|
||||
### 禁止
|
||||
|
||||
- “找到了梯度尖峰的唯一来源”；
|
||||
- “pre-attention 是尖峰生成点”；
|
||||
- “论文 Figure 5(c) 已复现”；
|
||||
- “Kimi K3 checkpoint 有 layer 21 尖峰”；
|
||||
- “uniform backward 会提高训练质量”；
|
||||
- “learned mixer 的 70.2% 梯度是坏的”；
|
||||
- “softmax/key path 没有任何作用”；
|
||||
- 从 3 seeds 推断总体显著性；
|
||||
- 从 width 192 / depth 32 外推到 2.8T 参数。
|
||||
|
||||
## 17. 下一轮最有价值的实验
|
||||
|
||||
1. 把全局 intervention 拆成 output mixer、group 6、group 7 和其他 mixers 四组；
|
||||
2. 对 completed-group value path 与 current-partial value path 分开做 same-forward backward
|
||||
ablation；
|
||||
3. 预先固定 local intervention grid，不根据本轮最大的单层事后选择；
|
||||
4. 比较“只改诊断 backward”与“从初始化按替代 backward 训练”的身份差异；
|
||||
5. 若 Attention Residuals 官方发布 Figure 5 telemetry，再开独立 protocol 对齐其 tensor、
|
||||
norm、reduction、batch 与 AMP 时点。
|
||||
|
||||
真实 K3 forward 仍受 `A_log [128]→[96]` 官方转换合同缺失阻断；本轮不绕过该边界。
|
||||
@@ -0,0 +1,87 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
|
||||
const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
||||
const read = (path) => {
|
||||
const bytes = readFileSync(new URL(path, import.meta.url));
|
||||
return { bytes, json: JSON.parse(bytes), sha256: hash(bytes) };
|
||||
};
|
||||
const aggregate = read("../src/data/k3-attnres-spike.json");
|
||||
const compact = read("../src/data/k3-attnres-spike-compact.json");
|
||||
const reproduction = read("../experiments/k3/attnres_spike/reproduction.json");
|
||||
const manifest = read("../experiments/k3/attnres_spike/manifest.json");
|
||||
const rawDirectory = new URL("../experiments/k3/attnres_spike/results/raw/", import.meta.url);
|
||||
const failures = [];
|
||||
const expect = (condition, message) => {
|
||||
if (!condition) failures.push(message);
|
||||
};
|
||||
|
||||
expect(aggregate.sha256 === "75058f9ef1f89593436960428378ab920569017f1758f9c15a2a0b1e081c1e90", "aggregate physical SHA-256 changed");
|
||||
expect(compact.sha256 === "7b3a3749d1344e56fb61039d2678a4e2aefe11a0a1e279516d6f27ff3bb8ef6c", "compact physical SHA-256 changed");
|
||||
expect(reproduction.sha256 === "86ac20ee06189a924eed2b2614e535660b536447f5ad019f4c1fd99cf4aa879a", "reproduction physical SHA-256 changed");
|
||||
expect(manifest.sha256 === "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "manifest physical SHA-256 changed");
|
||||
|
||||
expect(aggregate.json.canonical_sha256_without_self === "1d0faef297682d61d35ce2ab54da3868e18a84e9ebc6e44c617d08016e4fc8b4", "aggregate canonical SHA-256 changed");
|
||||
expect(compact.json.canonical_sha256_without_self === "a430dd26f512db7d64c933df65d9a97e6899028099333fe0a1212728f13126cd", "compact canonical SHA-256 changed");
|
||||
expect(reproduction.json.canonical_sha256_without_self === "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515", "reproduction canonical SHA-256 changed");
|
||||
|
||||
expect(compact.json.protocol_id === "llm-atlas-k3-attnres-spike-path-v1", "protocol identity mismatch");
|
||||
expect(compact.json.study.seeds.length === 3, "formal seed count changed");
|
||||
expect(compact.json.study.steps === 8000, "formal step budget changed");
|
||||
expect(compact.json.study.spike_layers.join(",") === "21,22,23,24,25", "fixed spike set changed");
|
||||
expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 4, "raw run count is not four");
|
||||
|
||||
for (const [name, expected] of Object.entries(reproduction.json.raw_files)) {
|
||||
const raw = read(`../experiments/k3/attnres_spike/results/raw/${name}`);
|
||||
expect(raw.sha256 === expected.file_sha256, `${name} physical hash mismatch`);
|
||||
expect(raw.json.canonical_sha256_without_self === expected.canonical_sha256, `${name} canonical hash mismatch`);
|
||||
}
|
||||
|
||||
expect(compact.json.hashes.aggregate_canonical_sha256 === aggregate.json.canonical_sha256_without_self, "compact→aggregate canonical link mismatch");
|
||||
expect(compact.json.hashes.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "compact→reproduction canonical link mismatch");
|
||||
expect(aggregate.json.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "aggregate→reproduction canonical link mismatch");
|
||||
|
||||
expect(reproduction.json.replay_gate.passed, "full replay is not exact");
|
||||
expect(reproduction.json.replay_gate.frozen_compare_sha256 === "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", "replay compare hash changed");
|
||||
expect(compact.json.gates.round05_equivalence.every((item) => item.passed), "Round 05 equivalence gate failed");
|
||||
expect(compact.json.gates.forward_identity.every((item) => item.passed), "forward identity gate failed");
|
||||
expect(compact.json.gates.initialization_negative_control.every((item) => item.passed), "initialization negative control failed");
|
||||
expect(compact.json.gates.loss_scale.every((item) => item.passed), "loss-scale gate failed");
|
||||
|
||||
const verdicts = compact.json.verdicts;
|
||||
expect(verdicts.reduction_robustness.verdict === "robust within the preregistered reduction family", "reduction robustness verdict changed");
|
||||
expect(verdicts.reduction_robustness.passed_cells === 12, "reduction pass count changed");
|
||||
expect(verdicts.visible_positions.visible_positions.length === 6, "visible-position count changed");
|
||||
expect(verdicts.visible_positions.earliest_observed_tensor === "pre_attention_input", "earliest observed tensor changed");
|
||||
expect(verdicts.interventions.softmax_key_path.opposite_direction_3_of_3, "softmax/key opposite-direction result changed");
|
||||
expect(!verdicts.interventions.softmax_key_path.passed_3_of_3, "softmax/key path unexpectedly passed material threshold");
|
||||
expect(verdicts.interventions.value_coefficients.passed_3_of_3, "value-coefficient material threshold failed");
|
||||
expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_contrast - 0.701998806119879) < 1e-15, "value contrast drop changed");
|
||||
expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_peak - 0.3702754647109951) < 1e-15, "value peak drop changed");
|
||||
|
||||
for (const run of compact.json.final_arrays) {
|
||||
expect(Object.keys(run.position_element_rms).length === 6, `seed ${run.seed} position count changed`);
|
||||
expect(Object.keys(run.post_mlp_reductions).length === 7, `seed ${run.seed} reduction count changed`);
|
||||
expect(Object.keys(run.interventions).length === 3, `seed ${run.seed} intervention count changed`);
|
||||
}
|
||||
|
||||
if (failures.length) {
|
||||
console.error(`FAIL K3 AttnRes spike data\n- ${failures.join("\n- ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
protocol: compact.json.protocol_id,
|
||||
formalRuns: compact.json.study.seeds.length,
|
||||
replayExact: reproduction.json.replay_gate.passed,
|
||||
reduction: verdicts.reduction_robustness.verdict,
|
||||
visiblePositions: verdicts.visible_positions.visible_positions.length,
|
||||
softmaxKey: verdicts.interventions.softmax_key_path.verdict,
|
||||
valueCoefficients: verdicts.interventions.value_coefficients.verdict,
|
||||
hashes: {
|
||||
aggregate: aggregate.sha256,
|
||||
compact: compact.sha256,
|
||||
reproduction: reproduction.sha256,
|
||||
},
|
||||
}, null, 2));
|
||||
console.log("PASS K3 AttnRes spike frozen data");
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user