research: audit AttnRes local path matrix
This commit is contained in:
wuyang
@@ -0,0 +1,274 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Package frozen Round 07 outputs without recomputing any result gate."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
PROTOCOL_ID = "llm-atlas-k3-attnres-local-path-v1"
|
||||||
|
SEEDS = (2026073001, 2026073002, 2026073003)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--raw-dir", type=Path, required=True)
|
||||||
|
parser.add_argument("--manifest", type=Path, required=True)
|
||||||
|
parser.add_argument("--aggregate", type=Path, required=True)
|
||||||
|
parser.add_argument("--reproduction-output", type=Path, required=True)
|
||||||
|
parser.add_argument("--compact-output", type=Path, required=True)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def file_sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_sha256(value: Any) -> str:
|
||||||
|
payload = json.dumps(
|
||||||
|
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||||
|
).encode()
|
||||||
|
return hashlib.sha256(payload).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def load_canonical(path: Path) -> dict[str, Any]:
|
||||||
|
value = json.loads(path.read_text())
|
||||||
|
expected = value["canonical_sha256_without_self"]
|
||||||
|
payload = {
|
||||||
|
key: item
|
||||||
|
for key, item in value.items()
|
||||||
|
if key != "canonical_sha256_without_self"
|
||||||
|
}
|
||||||
|
if canonical_sha256(payload) != expected:
|
||||||
|
raise RuntimeError(f"canonical hash mismatch: {path}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def write_canonical(path: Path, value: dict[str, Any]) -> None:
|
||||||
|
value["canonical_sha256_without_self"] = canonical_sha256(value)
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||||
|
temporary.write_text(
|
||||||
|
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True)
|
||||||
|
+ "\n"
|
||||||
|
)
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def replay_payload(value: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
cleaned = copy.deepcopy(value)
|
||||||
|
for key in ("run_kind", "timing", "canonical_sha256_without_self"):
|
||||||
|
cleaned.pop(key)
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def artifact_hashes(repo_root: Path) -> dict[str, str]:
|
||||||
|
paths = {
|
||||||
|
"runner": "experiments/k3/attnres_local_path/train.py",
|
||||||
|
"analyzer": "experiments/k3/attnres_local_path/analyze.py",
|
||||||
|
"packager": "experiments/k3/attnres_local_path/package.py",
|
||||||
|
"manifest": "experiments/k3/attnres_local_path/manifest.json",
|
||||||
|
"protocol": "research/K3_ATTNRES_LOCAL_PATH_PROTOCOL.md",
|
||||||
|
"scoping": "research/K3_ATTNRES_LOCAL_PATH_SCOPING.md",
|
||||||
|
"preresult_grok_review": (
|
||||||
|
"research/K3_ATTNRES_LOCAL_PATH_GROK_REVIEW.md"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
name: file_sha256(repo_root / path) for name, path in paths.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
args = parse_args()
|
||||||
|
repo_root = Path(__file__).resolve().parents[3]
|
||||||
|
manifest = json.loads(args.manifest.read_text())
|
||||||
|
aggregate = load_canonical(args.aggregate)
|
||||||
|
if (
|
||||||
|
manifest["protocol_id"] != PROTOCOL_ID
|
||||||
|
or aggregate["protocol_id"] != PROTOCOL_ID
|
||||||
|
):
|
||||||
|
raise RuntimeError("protocol mismatch")
|
||||||
|
|
||||||
|
formal = {}
|
||||||
|
raw_files = {}
|
||||||
|
for seed in SEEDS:
|
||||||
|
name = f"formal-seed-{seed}.json"
|
||||||
|
path = args.raw_dir / name
|
||||||
|
run = load_canonical(path)
|
||||||
|
if (
|
||||||
|
run["run_kind"] != "formal"
|
||||||
|
or run["seed"] != seed
|
||||||
|
or not run["round06_equivalence"]["passed"]
|
||||||
|
):
|
||||||
|
raise RuntimeError(f"invalid formal run: {name}")
|
||||||
|
formal[seed] = run
|
||||||
|
raw_files[name] = {
|
||||||
|
"file_sha256": file_sha256(path),
|
||||||
|
"canonical_sha256": run["canonical_sha256_without_self"],
|
||||||
|
}
|
||||||
|
|
||||||
|
replay_name = f"replay-seed-{SEEDS[0]}.json"
|
||||||
|
replay_path = args.raw_dir / replay_name
|
||||||
|
replay = load_canonical(replay_path)
|
||||||
|
if replay["run_kind"] != "replay" or replay["seed"] != SEEDS[0]:
|
||||||
|
raise RuntimeError("invalid replay")
|
||||||
|
raw_files[replay_name] = {
|
||||||
|
"file_sha256": file_sha256(replay_path),
|
||||||
|
"canonical_sha256": replay["canonical_sha256_without_self"],
|
||||||
|
}
|
||||||
|
compare_payload = replay_payload(formal[SEEDS[0]])
|
||||||
|
replay_exact = compare_payload == replay_payload(replay)
|
||||||
|
if not replay_exact or not aggregate["replay"][
|
||||||
|
"formal_seed1_exact_excluding_run_kind_and_timing"
|
||||||
|
]:
|
||||||
|
raise RuntimeError("replay exactness failed")
|
||||||
|
replay_gate = {
|
||||||
|
"passed": True,
|
||||||
|
"excluded_fields": [
|
||||||
|
"run_kind",
|
||||||
|
"timing",
|
||||||
|
"canonical_sha256_without_self",
|
||||||
|
],
|
||||||
|
"frozen_compare_sha256": canonical_sha256(compare_payload),
|
||||||
|
}
|
||||||
|
|
||||||
|
reproduction = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"protocol_id": PROTOCOL_ID,
|
||||||
|
"raw_files": raw_files,
|
||||||
|
"replay_gate": replay_gate,
|
||||||
|
"artifacts": artifact_hashes(repo_root),
|
||||||
|
"aggregate": {
|
||||||
|
"file_sha256": file_sha256(args.aggregate),
|
||||||
|
"canonical_sha256": aggregate[
|
||||||
|
"canonical_sha256_without_self"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"post_result_grok_review": {
|
||||||
|
"session": "019fb19d-94a3-7231-9a63-3a1ef33a9892",
|
||||||
|
"role": "read-only adversarial implementation audit; not an evidence source",
|
||||||
|
"blocking_errors": 0,
|
||||||
|
"localization_status_confirmed": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
write_canonical(args.reproduction_output, reproduction)
|
||||||
|
|
||||||
|
final_spectra = []
|
||||||
|
for seed in SEEDS:
|
||||||
|
final = formal[seed]["diagnostics"][-1]["local_matrix"]
|
||||||
|
final_spectra.append(
|
||||||
|
{
|
||||||
|
"seed": seed,
|
||||||
|
"modes": {
|
||||||
|
mode: {
|
||||||
|
"normalized": final[mode]["positions"][
|
||||||
|
"post_mlp_state"
|
||||||
|
]["reductions"]["element_rms"]["statistics"][
|
||||||
|
"normalized"
|
||||||
|
],
|
||||||
|
"spike_contrast": final[mode]["positions"][
|
||||||
|
"post_mlp_state"
|
||||||
|
]["reductions"]["element_rms"]["statistics"][
|
||||||
|
"spike_contrast"
|
||||||
|
],
|
||||||
|
"peak_normalized": final[mode]["positions"][
|
||||||
|
"post_mlp_state"
|
||||||
|
]["reductions"]["element_rms"]["statistics"][
|
||||||
|
"peak_normalized"
|
||||||
|
],
|
||||||
|
"peak_layer": final[mode]["positions"][
|
||||||
|
"post_mlp_state"
|
||||||
|
]["reductions"]["element_rms"]["statistics"][
|
||||||
|
"peak_layer"
|
||||||
|
],
|
||||||
|
"uniform_count": final[mode]["selector"][
|
||||||
|
"uniform_count"
|
||||||
|
],
|
||||||
|
}
|
||||||
|
for mode in manifest["matrix_modes"]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
compact = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"protocol_id": PROTOCOL_ID,
|
||||||
|
"study": {
|
||||||
|
"identity": manifest["study_identity"],
|
||||||
|
"seeds": list(SEEDS),
|
||||||
|
"steps": manifest["training"]["steps"],
|
||||||
|
"formal_target_bytes": (
|
||||||
|
len(SEEDS)
|
||||||
|
* manifest["training"]["target_bytes_per_cell"]
|
||||||
|
),
|
||||||
|
"total_target_bytes_with_replay": (
|
||||||
|
(len(SEEDS) + 1)
|
||||||
|
* manifest["training"]["target_bytes_per_cell"]
|
||||||
|
),
|
||||||
|
"modes": manifest["matrix_modes"],
|
||||||
|
"spike_layers": manifest["primary_object"][
|
||||||
|
"spike_layers_one_based"
|
||||||
|
],
|
||||||
|
"metrics": manifest["primary_object"]["metrics"],
|
||||||
|
},
|
||||||
|
"thresholds": manifest["thresholds"],
|
||||||
|
"formulas": manifest["formulas"],
|
||||||
|
"formal_cells": aggregate["formal_cells"],
|
||||||
|
"scores": aggregate["scores"],
|
||||||
|
"means": aggregate["means"],
|
||||||
|
"gates": aggregate["gates"],
|
||||||
|
"replay": {
|
||||||
|
**aggregate["replay"],
|
||||||
|
"frozen_compare_sha256": replay_gate[
|
||||||
|
"frozen_compare_sha256"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"final_spectra": final_spectra,
|
||||||
|
"limitations": aggregate["limitations"],
|
||||||
|
"hashes": {
|
||||||
|
"aggregate_file_sha256": file_sha256(args.aggregate),
|
||||||
|
"aggregate_canonical_sha256": aggregate[
|
||||||
|
"canonical_sha256_without_self"
|
||||||
|
],
|
||||||
|
"reproduction_file_sha256": file_sha256(
|
||||||
|
args.reproduction_output
|
||||||
|
),
|
||||||
|
"reproduction_canonical_sha256": reproduction[
|
||||||
|
"canonical_sha256_without_self"
|
||||||
|
],
|
||||||
|
"manifest_file_sha256": file_sha256(args.manifest),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
write_canonical(args.compact_output, compact)
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"reproduction": str(args.reproduction_output),
|
||||||
|
"compact": str(args.compact_output),
|
||||||
|
"raw_files": len(raw_files),
|
||||||
|
"replay_exact": replay_exact,
|
||||||
|
"localization": aggregate["gates"]["localization"][
|
||||||
|
"status"
|
||||||
|
],
|
||||||
|
"compact_canonical_sha256": compact[
|
||||||
|
"canonical_sha256_without_self"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
{
|
||||||
|
"aggregate": {
|
||||||
|
"canonical_sha256": "b86d119cd2f106e2cbee8a35760ed3244336a2fcfeb9178ea1e7dab13fc6f215",
|
||||||
|
"file_sha256": "bb0ec9fce5b30d50ad5c50c4b95af7a892d614f205a2c20cfc2662125e10160e"
|
||||||
|
},
|
||||||
|
"artifacts": {
|
||||||
|
"analyzer": "e0921562463e43d1ba6d47e4d23015107eb2df23579921088550b69afd47d02b",
|
||||||
|
"manifest": "db01e92ef2cf0896212fcd529429bd94a344de0e1195db7f87b9a56dc3449139",
|
||||||
|
"packager": "6a9ada0bc35b40475f45d7aca82877a93aeca667ed117c8e4417d924416c9196",
|
||||||
|
"preresult_grok_review": "2da1b6bf1f455c4121a7a2c5cfe40e102327dabafc7e24dccc23ed0d00ac6d71",
|
||||||
|
"protocol": "5ecc7ca92314ddb50aecf0cb50e115814c8983aa8bffb30e3634f7b3ce6dca1d",
|
||||||
|
"runner": "b42879e242a2f2d54aa6a87a718aeac4cf4509eae42da2b14403656675a8b03d",
|
||||||
|
"scoping": "670ca4edf31a4be1f54937d9c7a760dba7a96e1e820c38c6b10405e22b078fc8"
|
||||||
|
},
|
||||||
|
"canonical_sha256_without_self": "6f5d98fce6446fecc966dd2675f272f2c4f0c9a39a5741fabc4ffad6852ca7f4",
|
||||||
|
"post_result_grok_review": {
|
||||||
|
"blocking_errors": 0,
|
||||||
|
"localization_status_confirmed": true,
|
||||||
|
"role": "read-only adversarial implementation audit; not an evidence source",
|
||||||
|
"session": "019fb19d-94a3-7231-9a63-3a1ef33a9892"
|
||||||
|
},
|
||||||
|
"protocol_id": "llm-atlas-k3-attnres-local-path-v1",
|
||||||
|
"raw_files": {
|
||||||
|
"formal-seed-2026073001.json": {
|
||||||
|
"canonical_sha256": "f0a44f119836ed632c15880c3c2bb225173c0a05c50ea07abbe0e464ff407592",
|
||||||
|
"file_sha256": "73d46ae443d3e5ae3fe839c1656cda758f5f41aaee5f220c971c3c39b8a8cc3f"
|
||||||
|
},
|
||||||
|
"formal-seed-2026073002.json": {
|
||||||
|
"canonical_sha256": "b4629672b7d3b88a6be5525d2839e63e34fc9e4603ba1e7b98e957558da6da05",
|
||||||
|
"file_sha256": "bca4674c746e35a035acde7d2094a9c3bd59a988b052cd3feb30eb66eb0ca60a"
|
||||||
|
},
|
||||||
|
"formal-seed-2026073003.json": {
|
||||||
|
"canonical_sha256": "190b3deb06ae06caba287fce047b55cee613af6f1ebeb1661fcb53dd245abab0",
|
||||||
|
"file_sha256": "712f349e7715fee71f8e4678dcde0619d01b8d6c3b5c34825c88b9af0bf2abbb"
|
||||||
|
},
|
||||||
|
"replay-seed-2026073001.json": {
|
||||||
|
"canonical_sha256": "378df53ed9c89b2a4e0f3045b4c1e72754a7108d87fa9436e1db7aefa442e5eb",
|
||||||
|
"file_sha256": "872aabd9285ac346b4016c23de10769ab83c4dc29e62d8bc8b3156af98228d4e"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"replay_gate": {
|
||||||
|
"excluded_fields": [
|
||||||
|
"run_kind",
|
||||||
|
"timing",
|
||||||
|
"canonical_sha256_without_self"
|
||||||
|
],
|
||||||
|
"frozen_compare_sha256": "7dbd15ad03fbd357c5d91e159706d63b24703722f76c492ed1dc733535d6b9cf",
|
||||||
|
"passed": true
|
||||||
|
},
|
||||||
|
"schema_version": 1
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,302 @@
|
|||||||
|
# K3 Attention Residuals 局部 mixer 路径：Round 07 结果审计
|
||||||
|
|
||||||
|
研究日期：2026-07-30
|
||||||
|
协议：`llm-atlas-k3-attnres-local-path-v1`
|
||||||
|
预注册 commit：`6911efc`
|
||||||
|
结果前 runner / analyzer commit：`39a9ad6`
|
||||||
|
研究身份：**受 Round 05 / 06 启发的定向 reduced-model mechanism probe**
|
||||||
|
|
||||||
|
## 0. 一句话结论
|
||||||
|
|
||||||
|
> group 6+7 的 16 个 depth mixers 在 learned 背景上的局部 uniform intervention，
|
||||||
|
> 足以复现全局 log-gap reduction 的至少一半；但从 all-uniform 背景只恢复这 16 个
|
||||||
|
> mixer 时，contrast 恢复超过一半，peak 只恢复约 35.5%–41.3%。因此本轮得到强的
|
||||||
|
> **one-sided evidence**，但没有通过预注册的双向 localization 门。
|
||||||
|
|
||||||
|
这不是一句保守套话，而是协议第 13 节的直接判定：
|
||||||
|
|
||||||
|
```text
|
||||||
|
groups 6+7 sufficiency = PASS 6 / 6
|
||||||
|
groups 6+7 restoration = FAIL 3 / 6
|
||||||
|
localization = NOT ESTABLISHED
|
||||||
|
```
|
||||||
|
|
||||||
|
## 1. 运行与输入闸门
|
||||||
|
|
||||||
|
正式网格：
|
||||||
|
|
||||||
|
| run | seed | steps | target bytes | final validation BPC |
|
||||||
|
|---|---:|---:|---:|---:|
|
||||||
|
| formal | 2026073001 | 8,000 | 65,536,000 | 1.7123525941 |
|
||||||
|
| formal | 2026073002 | 8,000 | 65,536,000 | 1.7093240656 |
|
||||||
|
| formal | 2026073003 | 8,000 | 65,536,000 | 1.7030966813 |
|
||||||
|
| replay | 2026073001 | 8,000 | 65,536,000 | 1.7123525941 |
|
||||||
|
|
||||||
|
正式三格合计 196,608,000 target bytes，含 replay 为 262,144,000。
|
||||||
|
|
||||||
|
全部通过：
|
||||||
|
|
||||||
|
- 三 seed final model-state 与 Round 06 exact；
|
||||||
|
- 三 seed final optimizer-state 与 Round 06 exact；
|
||||||
|
- 六个 validation BPC、training history、六个 parent `learned` diagnostics exact；
|
||||||
|
- step 0 / 8,000 的 `detached_learned` 与 Round 06 同名 endpoint exact；
|
||||||
|
- step 0 / 8,000 的 `uniform_all` 与 Round 06
|
||||||
|
`uniform_value_backward` endpoint exact；
|
||||||
|
- 14 modes 的 logits、loss、六位置 activations、父 mixer summaries exact；
|
||||||
|
- 65 次 selector visits 的 identity、顺序、唯一性、exact mask 与 census 全部通过；
|
||||||
|
- step-0 14-mode negative control、parent learned-vs-detached control、
|
||||||
|
`loss ×1 / ×2` scale gate 全部通过；
|
||||||
|
- seed-1 从初始化完整 replay exact。
|
||||||
|
|
||||||
|
因此后续差异来自同一 forward state 上的预注册 backward coefficient masks，不是不同训练
|
||||||
|
状态、batch、loss、activation 或 selector 漂移。
|
||||||
|
|
||||||
|
## 2. 全局端点先复现
|
||||||
|
|
||||||
|
最终 `post_mlp_state / element_rms`：
|
||||||
|
|
||||||
|
| metric | detached learned（3-seed mean） | uniform all（3-seed mean） | Round 06 per-seed mean relative drop |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| spike contrast | 2.8065 | 0.7859 | 70.2% |
|
||||||
|
| peak / layer mean | 3.0651 | 1.8964 | 37.0% |
|
||||||
|
|
||||||
|
每个 seed、两个指标的 `G_X = ln(X_ref / X_uniform_all)` 都严格为正，raw relative
|
||||||
|
drop 6 / 6 超过 20%。global gap gate 完整成立。
|
||||||
|
|
||||||
|
这一步重要，因为局部 score 的分母不是任意“改善空间”，而是同一 seed、同一指标的
|
||||||
|
实测 global log gap。若任一 global gap 不成立，本轮主判定就必须停止；实际没有触发该
|
||||||
|
停止规则。
|
||||||
|
|
||||||
|
## 3. groups 6+7：充分性很强
|
||||||
|
|
||||||
|
只把 mixer indices 40–55 改成 uniform，其他 49 个 mixer 保持 detached learned：
|
||||||
|
|
||||||
|
| seed | `S_contrast` | `S_peak` | 50% × two metrics |
|
||||||
|
|---:|---:|---:|---:|
|
||||||
|
| 2026073001 | 0.697 | 1.817 | PASS |
|
||||||
|
| 2026073002 | 0.676 | 1.783 | PASS |
|
||||||
|
| 2026073003 | 0.658 | 1.501 | PASS |
|
||||||
|
| **mean** | **0.677** | **1.700** | **6 / 6** |
|
||||||
|
|
||||||
|
raw metric 的三 seed mean 也从 reference 的 `2.8065 / 3.0651` 变为
|
||||||
|
`1.1737 / 1.3450`。
|
||||||
|
|
||||||
|
`S_peak > 1` 不是 170% 因果贡献。它只表示在 log ratio 上，局部 uniform groups 6+7
|
||||||
|
把 peak 推得比 all-65 uniform endpoint 还低。这是很直接的 non-additivity / interaction
|
||||||
|
信号，也是协议坚持“不裁剪 score 到 [0,1]”的原因。
|
||||||
|
|
||||||
|
允许结论：
|
||||||
|
|
||||||
|
> groups 6+7 在本 diagnostic 中足以复现至少一半 global log-gap reduction。
|
||||||
|
|
||||||
|
禁止结论：
|
||||||
|
|
||||||
|
- “这 16 个 mixer 解释了 67.7% / 170.0% 的尖峰”；
|
||||||
|
- “剩下 49 个 mixer 只贡献 32.3% / −70.0%”；
|
||||||
|
- “group 6+7 是唯一原因”。
|
||||||
|
|
||||||
|
## 4. 反向 restoration 没有给出同样答案
|
||||||
|
|
||||||
|
从 all-uniform 背景出发，只把 groups 6+7 恢复为 detached-learned coefficients；
|
||||||
|
其他 49 个 mixer 仍为 uniform：
|
||||||
|
|
||||||
|
| seed | `R_contrast` | `R_peak` | 50% × two metrics |
|
||||||
|
|---:|---:|---:|---:|
|
||||||
|
| 2026073001 | 0.649 PASS | 0.372 FAIL | FAIL |
|
||||||
|
| 2026073002 | 0.621 PASS | 0.355 FAIL | FAIL |
|
||||||
|
| 2026073003 | 0.680 PASS | 0.413 FAIL | FAIL |
|
||||||
|
| **mean** | **0.650** | **0.380** | **3 / 6** |
|
||||||
|
|
||||||
|
raw metric 的三 seed mean 从 all-uniform 的 `0.7859 / 1.8964` 恢复为
|
||||||
|
`1.7693 / 2.2658`。contrast 明显朝 reference 回升，但 peak 的 log-gap recovery
|
||||||
|
没有一个 seed 达到 50%。
|
||||||
|
|
||||||
|
这说明同一个 scope 的作用强烈依赖其他 mixer 处于 learned 还是 uniform 背景：
|
||||||
|
|
||||||
|
- learned 背景中 uniformize groups 6+7，足以大幅压低 contrast 与 peak；
|
||||||
|
- uniform 背景中 restore groups 6+7，足以恢复 contrast，却不足以恢复 peak；
|
||||||
|
- 两个方向不对称，不能用单侧 sufficiency 替代双向 localization。
|
||||||
|
|
||||||
|
因此正式 verdict 是：
|
||||||
|
|
||||||
|
```text
|
||||||
|
one_sided_evidence_localization_not_established
|
||||||
|
```
|
||||||
|
|
||||||
|
不是 “almost passed”，也不因 `R_peak` mean 约 0.38 而软化 0.50 阈值。
|
||||||
|
|
||||||
|
## 5. 单 group 结果：同样显示交互
|
||||||
|
|
||||||
|
### 5.1 sufficiency
|
||||||
|
|
||||||
|
| scope | mean `S_contrast` | mean `S_peak` | 20% gate |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| group 6 | 0.281 | 1.044 | PASS 6 / 6 |
|
||||||
|
| group 7 | 0.438 | 0.843 | PASS 6 / 6 |
|
||||||
|
|
||||||
|
两个单 group 都在两个指标、三个 seed 通过 material local sufficiency。
|
||||||
|
|
||||||
|
但：
|
||||||
|
|
||||||
|
```text
|
||||||
|
S(group6) + S(group7) ≠ S(groups6+7)
|
||||||
|
```
|
||||||
|
|
||||||
|
尤其 peak 上，两个单 group 与联合 scope 都可能超过 global endpoint，不能按 mixer
|
||||||
|
数量或 score 相加做贡献账。
|
||||||
|
|
||||||
|
### 5.2 restoration
|
||||||
|
|
||||||
|
| restored scope | mean `R_contrast` | mean `R_peak` | 20% gate |
|
||||||
|
|---|---:|---:|---:|
|
||||||
|
| group 6 | +0.248 | −0.146 | MIXED / FAIL 3 / 6 |
|
||||||
|
| group 7 | +0.409 | −0.191 | MIXED / FAIL 3 / 6 |
|
||||||
|
|
||||||
|
恢复单个 group 时，contrast 在三 seed 都超过 20%，peak 却在三 seed 全为负：相对
|
||||||
|
all-uniform，恢复一个 group 的 learned coefficients 反而让最高层 / 均值更低。
|
||||||
|
|
||||||
|
这不是 “group 没作用”，而是 effect direction 随 metric 与背景改变。它进一步反对
|
||||||
|
简单、可加的局部归因故事。
|
||||||
|
|
||||||
|
## 6. attention vs MLP：只有 group 7 过闸
|
||||||
|
|
||||||
|
### group 6
|
||||||
|
|
||||||
|
MLP-only 在 6 个 branch cells 中赢 5 个；seed 2026073003 的 contrast
|
||||||
|
`S=0.177 < 0.20`。因此：
|
||||||
|
|
||||||
|
```text
|
||||||
|
group 6 branch dominance = NOT ESTABLISHED
|
||||||
|
```
|
||||||
|
|
||||||
|
不能因为 margin 大、均值高，忽略 material threshold 的单格失败。
|
||||||
|
|
||||||
|
### group 7
|
||||||
|
|
||||||
|
| branch | mean `S_contrast` | mean `S_peak` |
|
||||||
|
|---|---:|---:|
|
||||||
|
| attention-only | 0.015 | 0.026 |
|
||||||
|
| MLP-only | 0.426 | 0.823 |
|
||||||
|
|
||||||
|
MLP-only 自身 material，且在两个指标、三个 seed 都比 attention-only 高至少
|
||||||
|
15 percentage points，因此：
|
||||||
|
|
||||||
|
```text
|
||||||
|
group 7 MLP branch-dominant at the preregistered margin
|
||||||
|
```
|
||||||
|
|
||||||
|
这是协议第 14 节的**次级、sufficiency-only、探索性**判定；没有 branch-level
|
||||||
|
restoration，不得升级为第 13 节的双向 localization。
|
||||||
|
|
||||||
|
## 7. output / depth controls
|
||||||
|
|
||||||
|
| control | mean `S_contrast` | mean `S_peak` | verdict |
|
||||||
|
|---|---:|---:|---|
|
||||||
|
| output-only（1 mixer） | 0.131 | 0.187 | 0 / 6 at 50% |
|
||||||
|
| all-depth（64 mixers） | 0.949 | 1.145 | near / beyond global endpoint |
|
||||||
|
|
||||||
|
output mixer 单独无法解释 global gap 的一半。all-depth 已复现绝大多数 contrast gap，
|
||||||
|
peak 甚至超过 all-65 endpoint；把 output 与 depth scores 相加会产生负
|
||||||
|
`interaction_residual`。该 residual 只是 bookkeeping，不预期为 0，不是统计交互检验。
|
||||||
|
|
||||||
|
## 8. 32-layer 谱的直观变化
|
||||||
|
|
||||||
|
三个 seed 的 reference peak 都在 layer 21；all-uniform peak 都迁到 layer 2。
|
||||||
|
|
||||||
|
groups 6+7 only：
|
||||||
|
|
||||||
|
- seed 1 peak → layer 5；
|
||||||
|
- seed 2 peak → layer 25；
|
||||||
|
- seed 3 peak → layer 6。
|
||||||
|
|
||||||
|
restore groups 6+7 on uniform background：
|
||||||
|
|
||||||
|
- 三 seed peak 都回到 layer 21；
|
||||||
|
- 但 peak / mean 的恢复比例仍只有 0.355–0.413。
|
||||||
|
|
||||||
|
“peak layer 回来了”与“peak 强度恢复超过一半”不是同一判据。网站会同时展示谱与
|
||||||
|
预注册 score，避免只凭最高点位置讲故事。
|
||||||
|
|
||||||
|
## 9. replay 与 artifact 链
|
||||||
|
|
||||||
|
seed 2026073001 从初始化完整重跑。排除 `run_kind`、timing 与 self canonical hash 后：
|
||||||
|
|
||||||
|
```text
|
||||||
|
formal seed1 == replay
|
||||||
|
compare SHA-256 = 7dbd15ad03fbd357c5d91e159706d63b24703722f76c492ed1dc733535d6b9cf
|
||||||
|
```
|
||||||
|
|
||||||
|
它覆盖训练状态、六 checkpoints、14-mode 两端矩阵、六位置 gradient reductions、
|
||||||
|
selector visits 与所有 gates，不只是 final BPC。
|
||||||
|
|
||||||
|
冻结物理 hashes：
|
||||||
|
|
||||||
|
| artifact | SHA-256 |
|
||||||
|
|---|---|
|
||||||
|
| manifest | `db01e92e…9139` |
|
||||||
|
| runner | `b42879e2…b03d` |
|
||||||
|
| analyzer | `e0921562…d02b` |
|
||||||
|
| packager | `6a9ada0b…9196` |
|
||||||
|
| aggregate | `bb0ec9fc…160e` |
|
||||||
|
| compact | `3bb6c158…9c08` |
|
||||||
|
| reproduction | `524a6883…5db` |
|
||||||
|
|
||||||
|
canonical hashes：
|
||||||
|
|
||||||
|
```text
|
||||||
|
aggregate b86d119cd2f106e2cbee8a35760ed3244336a2fcfeb9178ea1e7dab13fc6f215
|
||||||
|
compact 2aff9288f52d3d41bb1f59c64d9a07518ad3e2120c61615478087b24aaabd835
|
||||||
|
reproduction 6f5d98fce6446fecc966dd2675f272f2c4f0c9a39a5741fabc4ffad6852ca7f4
|
||||||
|
```
|
||||||
|
|
||||||
|
## 10. 两次 Grok Headless 审阅
|
||||||
|
|
||||||
|
### 结果前
|
||||||
|
|
||||||
|
session `019fb151-9627-76c1-b7d7-53012874f85c` 找出 selector API、output identity、
|
||||||
|
“restore to learned”歧义、parent learned 调度、20% 分母和 negative control 六个硬问题。
|
||||||
|
全部在预注册 freeze 前修正并留档。
|
||||||
|
|
||||||
|
### 结果后
|
||||||
|
|
||||||
|
session `019fb19d-94a3-7231-9a63-3a1ef33a9892` 只读对照 protocol、manifest、
|
||||||
|
analyzer 与 aggregate，独立复算代表性 `G/S/R` cells：
|
||||||
|
|
||||||
|
- 阻断实现错误：0；
|
||||||
|
- one-sided verdict：确认正确；
|
||||||
|
- 不允许修改阈值；
|
||||||
|
- 指出单 group restoration 的 `score_sign_split` reason 是跨指标汇总，因此 reason
|
||||||
|
wording 略宽；`mixed / fail` 本身仍由 C/P pass split 独立成立，主 verdict 不受影响。
|
||||||
|
|
||||||
|
Grok 是方法学审稿人，不是论文或实验事实来源；正式证据仍是冻结代码与 raw outputs。
|
||||||
|
|
||||||
|
## 11. `A_log` 工件边界同步更新
|
||||||
|
|
||||||
|
本轮仍不是 K3 2.8T checkpoint forward。截止 2026-07-30 12:35 CST：
|
||||||
|
|
||||||
|
- official main 仍是 `9f62e4e9fffbd0a83ddd60e1c209d828994b3569`，96 vs 128
|
||||||
|
mismatch 未修；
|
||||||
|
- community PR #144 把 parameter 改成 128，但没有独立 forward 验证；
|
||||||
|
- community PR #150 保留 96，并在加载时验证 / 裁掉 32 个全零尾项；提交者报告检查
|
||||||
|
69 层并完成 disk-offloaded end-to-end generation；
|
||||||
|
- 两个 PR 都未合并，Moonshot 尚未给出官方裁决。
|
||||||
|
|
||||||
|
因此“没有任何公开候选解释”已经过时；“官方 contract 已解决”同样不成立。
|
||||||
|
|
||||||
|
## 12. 最强允许结论
|
||||||
|
|
||||||
|
可以说：
|
||||||
|
|
||||||
|
> 在本缩小 Block AttnRes 模型的同前向 diagnostic backward 中，global learned-value
|
||||||
|
> coefficient sensitivity 对 groups 6+7 的局部 uniformization 具有强 sufficiency；
|
||||||
|
> restoration 只在 contrast 上超过一半，在 peak 上稳定不足一半，因此预注册的双向
|
||||||
|
> localization 未建立。group 7 的 MLP mixer 在次级 sufficiency-only branch 判定中占优。
|
||||||
|
|
||||||
|
不能说：
|
||||||
|
|
||||||
|
- “证明 K3 的尖峰来自 group 6 / 7”；
|
||||||
|
- “groups 6+7 解释了 67.7% / 170.0%”；
|
||||||
|
- “MLP 是唯一原因”；
|
||||||
|
- “只要把这些 mixer 训练成 uniform 就会更稳定”；
|
||||||
|
- “复现了 K3 Figure 5(c)”；
|
||||||
|
- “community PR #150 已经是官方 `A_log` 修复”。
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { readdirSync, readFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
||||||
|
const read = (path) => {
|
||||||
|
const bytes = readFileSync(new URL(path, import.meta.url));
|
||||||
|
return { bytes, json: JSON.parse(bytes), sha256: hash(bytes) };
|
||||||
|
};
|
||||||
|
const aggregate = read("../src/data/k3-attnres-local-path.json");
|
||||||
|
const compact = read("../src/data/k3-attnres-local-path-compact.json");
|
||||||
|
const reproduction = read("../experiments/k3/attnres_local_path/reproduction.json");
|
||||||
|
const manifest = read("../experiments/k3/attnres_local_path/manifest.json");
|
||||||
|
const rawDirectory = new URL("../experiments/k3/attnres_local_path/results/raw/", import.meta.url);
|
||||||
|
const failures = [];
|
||||||
|
const expect = (condition, message) => {
|
||||||
|
if (!condition) failures.push(message);
|
||||||
|
};
|
||||||
|
const close = (actual, expected, tolerance = 1e-15) =>
|
||||||
|
Math.abs(actual - expected) <= tolerance;
|
||||||
|
|
||||||
|
expect(aggregate.sha256 === "bb0ec9fce5b30d50ad5c50c4b95af7a892d614f205a2c20cfc2662125e10160e", "aggregate physical SHA-256 changed");
|
||||||
|
expect(compact.sha256 === "3bb6c15815f37d2109386fb87f9642246d41e636242ca3a9ba3c17ae8f419c08", "compact physical SHA-256 changed");
|
||||||
|
expect(reproduction.sha256 === "524a6883c08941c1be908834d7bcf915eb32f0bdb501aae09f3cec761fdd65db", "reproduction physical SHA-256 changed");
|
||||||
|
expect(manifest.sha256 === "db01e92ef2cf0896212fcd529429bd94a344de0e1195db7f87b9a56dc3449139", "manifest physical SHA-256 changed");
|
||||||
|
|
||||||
|
expect(aggregate.json.canonical_sha256_without_self === "b86d119cd2f106e2cbee8a35760ed3244336a2fcfeb9178ea1e7dab13fc6f215", "aggregate canonical SHA-256 changed");
|
||||||
|
expect(compact.json.canonical_sha256_without_self === "2aff9288f52d3d41bb1f59c64d9a07518ad3e2120c61615478087b24aaabd835", "compact canonical SHA-256 changed");
|
||||||
|
expect(reproduction.json.canonical_sha256_without_self === "6f5d98fce6446fecc966dd2675f272f2c4f0c9a39a5741fabc4ffad6852ca7f4", "reproduction canonical SHA-256 changed");
|
||||||
|
|
||||||
|
expect(compact.json.protocol_id === "llm-atlas-k3-attnres-local-path-v1", "protocol identity mismatch");
|
||||||
|
expect(compact.json.study.seeds.length === 3, "formal seed count changed");
|
||||||
|
expect(compact.json.study.steps === 8000, "formal step budget changed");
|
||||||
|
expect(compact.json.study.modes.length === 14, "matrix mode count changed");
|
||||||
|
expect(compact.json.study.spike_layers.join(",") === "21,22,23,24,25", "fixed spike set changed");
|
||||||
|
expect(compact.json.study.formal_target_bytes === 196608000, "formal target-byte count changed");
|
||||||
|
expect(compact.json.study.total_target_bytes_with_replay === 262144000, "total target-byte count changed");
|
||||||
|
expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 4, "raw run count is not four");
|
||||||
|
|
||||||
|
for (const [name, expected] of Object.entries(reproduction.json.raw_files)) {
|
||||||
|
const raw = read(`../experiments/k3/attnres_local_path/results/raw/${name}`);
|
||||||
|
expect(raw.sha256 === expected.file_sha256, `${name} physical hash mismatch`);
|
||||||
|
expect(raw.json.canonical_sha256_without_self === expected.canonical_sha256, `${name} canonical hash mismatch`);
|
||||||
|
expect(raw.json.round06_equivalence.passed, `${name} Round 06 equivalence failed`);
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(compact.json.hashes.aggregate_canonical_sha256 === aggregate.json.canonical_sha256_without_self, "compact→aggregate canonical link mismatch");
|
||||||
|
expect(compact.json.hashes.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "compact→reproduction canonical link mismatch");
|
||||||
|
expect(reproduction.json.aggregate.canonical_sha256 === aggregate.json.canonical_sha256_without_self, "reproduction→aggregate canonical link mismatch");
|
||||||
|
expect(reproduction.json.replay_gate.passed, "full replay is not exact");
|
||||||
|
expect(reproduction.json.replay_gate.frozen_compare_sha256 === "7dbd15ad03fbd357c5d91e159706d63b24703722f76c492ed1dc733535d6b9cf", "replay compare hash changed");
|
||||||
|
expect(reproduction.json.post_result_grok_review.blocking_errors === 0, "post-result audit reports a blocking error");
|
||||||
|
expect(reproduction.json.post_result_grok_review.localization_status_confirmed, "post-result audit did not confirm the status");
|
||||||
|
|
||||||
|
const gates = compact.json.gates;
|
||||||
|
expect(gates.all_input_and_parent_gates_passed, "input or parent gate failed");
|
||||||
|
expect(gates.global_gap.passed && gates.global_gap.passed_cells === 6, "global gap gate changed");
|
||||||
|
expect(gates.sufficiency.groups_6_7.passed && gates.sufficiency.groups_6_7.passed_cells === 6, "groups 6+7 sufficiency gate failed");
|
||||||
|
expect(!gates.restoration.groups_6_7.passed && gates.restoration.groups_6_7.passed_cells === 3, "groups 6+7 restoration verdict changed");
|
||||||
|
expect(!gates.localization.passed, "localization unexpectedly passed");
|
||||||
|
expect(gates.localization.status === "one_sided_evidence_localization_not_established", "localization status changed");
|
||||||
|
expect(gates.sufficiency.group_6.passed && gates.sufficiency.group_7.passed, "single-group sufficiency gate changed");
|
||||||
|
expect(!gates.restoration.group_6.passed && !gates.restoration.group_7.passed, "single-group restoration unexpectedly passed");
|
||||||
|
expect(!gates.sufficiency.output_half_gap.passed && gates.sufficiency.output_half_gap.passed_cells === 0, "output half-gap control changed");
|
||||||
|
expect(!gates.branch_dominance.group_6.passed, "group 6 branch dominance unexpectedly passed");
|
||||||
|
expect(gates.branch_dominance.group_7.passed && gates.branch_dominance.group_7.dominant_branch === "mlp", "group 7 MLP dominance changed");
|
||||||
|
|
||||||
|
expect(close(compact.json.means.sufficiency.uniform_groups_6_7_only.spike_contrast, 0.6769080107044138), "groups 6+7 mean contrast sufficiency changed");
|
||||||
|
expect(close(compact.json.means.sufficiency.uniform_groups_6_7_only.peak_normalized, 1.7003398291502192), "groups 6+7 mean peak sufficiency changed");
|
||||||
|
expect(close(compact.json.means.restoration.uniform_except_groups_6_7.spike_contrast, 0.6499896714884171), "groups 6+7 mean contrast restoration changed");
|
||||||
|
expect(close(compact.json.means.restoration.uniform_except_groups_6_7.peak_normalized, 0.3801045652660336), "groups 6+7 mean peak restoration changed");
|
||||||
|
expect(close(compact.json.means.sufficiency.uniform_output_only.spike_contrast, 0.13066777032743535), "output-only mean contrast score changed");
|
||||||
|
expect(close(compact.json.means.sufficiency.uniform_output_only.peak_normalized, 0.18724852586461135), "output-only mean peak score changed");
|
||||||
|
|
||||||
|
for (const spectrum of compact.json.final_spectra) {
|
||||||
|
expect(Object.keys(spectrum.modes).length === 14, `seed ${spectrum.seed} spectrum mode count changed`);
|
||||||
|
expect(spectrum.modes.detached_learned.uniform_count === 0, `seed ${spectrum.seed} reference census changed`);
|
||||||
|
expect(spectrum.modes.uniform_groups_6_7_only.uniform_count === 16, `seed ${spectrum.seed} groups 6+7 census changed`);
|
||||||
|
expect(spectrum.modes.uniform_except_groups_6_7.uniform_count === 49, `seed ${spectrum.seed} restoration census changed`);
|
||||||
|
expect(spectrum.modes.uniform_all.uniform_count === 65, `seed ${spectrum.seed} all-uniform census changed`);
|
||||||
|
for (const mode of Object.values(spectrum.modes)) {
|
||||||
|
expect(mode.normalized.length === 32, `seed ${spectrum.seed} normalized spectrum length changed`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failures.length) {
|
||||||
|
console.error(`FAIL K3 AttnRes local-path data\n- ${failures.join("\n- ")}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
protocol: compact.json.protocol_id,
|
||||||
|
modes: compact.json.study.modes.length,
|
||||||
|
formalRuns: compact.json.study.seeds.length,
|
||||||
|
replayExact: reproduction.json.replay_gate.passed,
|
||||||
|
globalGap: gates.global_gap,
|
||||||
|
sufficiency: gates.sufficiency.groups_6_7,
|
||||||
|
restoration: gates.restoration.groups_6_7,
|
||||||
|
localization: gates.localization,
|
||||||
|
branch: gates.branch_dominance,
|
||||||
|
hashes: {
|
||||||
|
aggregate: aggregate.sha256,
|
||||||
|
compact: compact.sha256,
|
||||||
|
reproduction: reproduction.sha256,
|
||||||
|
},
|
||||||
|
}, null, 2));
|
||||||
|
console.log("PASS K3 AttnRes local-path frozen data");
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user