wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: publish AttnRes forward training study

Browse Source
This commit is contained in:
wuyang
2026-07-30 18:41:18 +08:00
parent 7ea91caabb
commit f177fa676d
47 changed files with 192164 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/k3/attnres_forward/README.md
+12
View File
@@ -6,6 +6,8 @@ This directory implements preregistered protocol
- `research/K3_ATTNRES_FORWARD_TRAINING_SCOPING.md`
- `research/K3_ATTNRES_FORWARD_TRAINING_PROTOCOL.md`
- `research/K3_ATTNRES_FORWARD_TRAINING_GROK_REVIEW.md`
- `research/K3_ATTNRES_FORWARD_TRAINING_IMPLEMENTATION_REVIEW.md`
- `research/K3_ATTNRES_FORWARD_TRAINING_AUDIT.md`
It is a depth-32 reduced Block AttnRes architecture ablation. It is not a
Kimi-K3 checkpoint forward pass and does not claim to recover unpublished
@@ -56,3 +58,13 @@ This runs 12 formal cells and one full replay. The analyzer reads all cells,
the frozen historical paired references, and generates the only authoritative
status, interaction map, and website compact artifact.
The checked-in Round 08 release contains:
- 13 raw results under `results/raw/`;
- `reproduction.json` with the exact primary scientific-payload hash;
- aggregate / compact website data under `src/data/`;
- a frozen-data checker and real-Chrome five-view regression in `scripts/`.
The established status is deliberately scoped to this reduced protocol. It is
not a real Kimi-K3 checkpoint result or a reproduction of unpublished Figure
5(c) telemetry.
experiments/k3/attnres_forward/analyze.py
+15
View File
@@ -425,6 +425,19 @@ def main() -> None:
== environment_metadata(references[seed])
),
}
metadata_warnings = [
{
"cell": cell,
"message": (
"GPU/version metadata differs from the historical paired "
"reference; frozen scientific-environment fields still match"
),
"run_environment": item["run_environment"],
"reference_environment": item["reference_environment"],
}
for cell, item in pairing.items()
if not item["metadata_equal"]
]
replay = read_result(args.replay, PROTOCOL_ID)
replay_contract = manifest["replay"]
@@ -593,6 +606,7 @@ def main() -> None:
"thresholds": manifest["thresholds"],
"input_files": input_files,
"historical_pairing": pairing,
"metadata_warnings": metadata_warnings,
"replay": {
"passed": replay_exact,
"scientific_payload_sha256": canonical_sha256(formal_payload),
@@ -636,6 +650,7 @@ def main() -> None:
"trajectories": trajectories,
"final_spectra": final_spectra,
"replay": aggregate["replay"],
"metadata_warnings": metadata_warnings,
"processed_target_bytes": manifest["new_target_bytes"],
"reporting_boundary": aggregate["reporting_boundary"],
"aggregate_sha256": aggregate["canonical_sha256_without_self"],
experiments/k3/attnres_forward/reproduction.json
+25
View File
@@ -0,0 +1,25 @@
{
"canonical_sha256_without_self": "57346df80c0d76bd1d306d5fa213feed74ac2094237c22ebcb49f16ea16437e0",
"excluded_fields": [
"run_kind",
"timing",
"self hashes",
"manifest path strings",
"GPU/version metadata"
],
"formal_file_sha256": "0962ebd1a00a11e61ac795282bfa99412166c8752f2a3731f137030d7f134dc1",
"formal_variant": "uniform_groups_6_7_forward",
"passed": true,
"post_result_grok_review": {
"blocking_errors": 0,
"claim_boundary_confirmed": true,
"replay_confirmed": true,
"session_id": "019fb28b-a9e1-7643-8e43-06f5e16a2077",
"status_confirmed": true
},
"protocol_id": "llm-atlas-k3-attnres-forward-training-v1",
"replay_file_sha256": "b85ac8062b2b0b8b3f2305d4b22a7c0212466fbfb28a9d63099cadd0a6917c8e",
"schema_version": 1,
"scientific_payload_sha256": "b85563ca5cb53e60b39c3801d372376206105b8a089a8633b3e81973a7f0c051",
"seed": 2026073001
}
experiments/k3/attnres_forward/results/gates/step-zero.json
+21 -11
View File
@@ -1,5 +1,5 @@
{
"canonical_sha256_without_self": "00db0569d2fd00599383ecdc50578c52df784a937a96dd69576f1366e88b0dcb",
"canonical_sha256_without_self": "9a0716bc496d39b9447b79aba0aec42ca4c953cbabe30e3aa349c1bba0701c5f",
"environment": {
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
@@ -21,7 +21,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -1254,7 +1255,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -1267,7 +1269,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -2500,7 +2503,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -2562,7 +2566,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -3795,7 +3800,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -3857,7 +3863,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -5090,7 +5097,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
@@ -5128,7 +5136,8 @@
"initial_mixer_hash": true,
"initial_public_hash": true,
"logits_sha256": true,
"loss_nats": true
"loss_nats": true,
"loss_tensor_sha256": true
},
"passed": true,
"payload": {
@@ -6361,7 +6370,8 @@
"initial_mixer_hash": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_hash": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58",
"logits_sha256": "6b44486d93c29a13dc0b961117cc2d22f26c81dc519346ac45b1bb0fefa29939",
"loss_nats": 5.516995429992676
"loss_nats": 5.516995429992676,
"loss_tensor_sha256": "c4e1d45c137a73ad6249c2db3ff5e29c849310d4f71f3182c41d27ff6b9fcb6a"
},
"pre_reduction_uniform_weight_gate": true,
"pre_reduction_uniform_weight_max_abs_error": 0.0,
experiments/k3/attnres_forward/results/gates/wrapper-smoke.json
+259 -131
View File
@@ -1,7 +1,7 @@
{
"architecture": "block",
"batch_size": 32,
"canonical_sha256_without_self": "a467cc1c05e1b860abbf44086273071bcd2a891db4354cc6e79b319e44d1cce5",
"canonical_sha256_without_self": "81b011e2ab02c49abb6b2686b3e0701d71ab6e7f46cbb7c426eec91702836b7f",
"depth": 32,
"diagnostics": [
{
@@ -2479,74 +2479,74 @@
],
"forward_intervention": {
"depth_visit_counts": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39,
39
],
"forward_calls": 0,
"forward_calls": 39,
"output_mixer_selected": false,
"output_visit_count": 0,
"output_visit_count": 39,
"passed": true,
"selected_depth_indices": [],
"selected_parameter_reachability_gate": true,
@@ -2555,70 +2555,198 @@
"selector_gate": true,
"semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel",
"source_counts_by_depth_index": {
"0": [],
"1": [],
"10": [],
"11": [],
"12": [],
"13": [],
"14": [],
"15": [],
"16": [],
"17": [],
"18": [],
"19": [],
"2": [],
"20": [],
"21": [],
"22": [],
"23": [],
"24": [],
"25": [],
"26": [],
"27": [],
"28": [],
"29": [],
"3": [],
"30": [],
"31": [],
"32": [],
"33": [],
"34": [],
"35": [],
"36": [],
"37": [],
"38": [],
"39": [],
"4": [],
"40": [],
"41": [],
"42": [],
"43": [],
"44": [],
"45": [],
"46": [],
"47": [],
"48": [],
"49": [],
"5": [],
"50": [],
"51": [],
"52": [],
"53": [],
"54": [],
"55": [],
"56": [],
"57": [],
"58": [],
"59": [],
"6": [],
"60": [],
"61": [],
"62": [],
"63": [],
"7": [],
"8": [],
"9": []
"0": [
1
],
"1": [
2
],
"10": [
3
],
"11": [
3
],
"12": [
3
],
"13": [
3
],
"14": [
3
],
"15": [
3
],
"16": [
3
],
"17": [
4
],
"18": [
4
],
"19": [
4
],
"2": [
2
],
"20": [
4
],
"21": [
4
],
"22": [
4
],
"23": [
4
],
"24": [
4
],
"25": [
5
],
"26": [
5
],
"27": [
5
],
"28": [
5
],
"29": [
5
],
"3": [
2
],
"30": [
5
],
"31": [
5
],
"32": [
5
],
"33": [
6
],
"34": [
6
],
"35": [
6
],
"36": [
6
],
"37": [
6
],
"38": [
6
],
"39": [
6
],
"4": [
2
],
"40": [
6
],
"41": [
7
],
"42": [
7
],
"43": [
7
],
"44": [
7
],
"45": [
7
],
"46": [
7
],
"47": [
7
],
"48": [
7
],
"49": [
8
],
"5": [
2
],
"50": [
8
],
"51": [
8
],
"52": [
8
],
"53": [
8
],
"54": [
8
],
"55": [
8
],
"56": [
8
],
"57": [
9
],
"58": [
9
],
"59": [
9
],
"6": [
2
],
"60": [
9
],
"61": [
9
],
"62": [
9
],
"63": [
9
],
"7": [
2
],
"8": [
2
],
"9": [
3
]
},
"uniform_weight_gate": true,
"uniform_weight_max_abs_error": 0.0,
experiments/k3/attnres_forward/results/raw/formal-uniform_group_6_forward-seed-2026073001.json
+14045
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_6_forward-seed-2026073002.json
+14045
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_6_forward-seed-2026073003.json
+14045
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_7_forward-seed-2026073001.json
+14045
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_7_forward-seed-2026073002.json
+14045
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_7_forward-seed-2026073003.json
+14045
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_7_mlp_forward-seed-2026073001.json
+14029
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_7_mlp_forward-seed-2026073002.json
+14029
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_group_7_mlp_forward-seed-2026073003.json
+14029
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_groups_6_7_forward-seed-2026073001.json
+14077
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_groups_6_7_forward-seed-2026073002.json
+14077
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/formal-uniform_groups_6_7_forward-seed-2026073003.json
+14077
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/results/raw/replay-uniform_groups_6_7_forward-seed-2026073001.json
+14077
View File
File diff suppressed because it is too large Load Diff
experiments/k3/attnres_forward/run_matrix.py
+36 -6
View File
@@ -84,6 +84,34 @@ def validate_manifest(args: argparse.Namespace) -> None:
raise ValueError("the frozen protocol permits one or two processes")
def stop_processes(items: list[dict[str, Any]]) -> None:
for item in items:
if item["process"].poll() is None:
item["process"].terminate()
for item in items:
process = item["process"]
if process.poll() is not None:
continue
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
def quarantine_failed_output(
output_dir: Path, variant: str, seed: int, run_kind: str
) -> str | None:
output = cell_output(output_dir, variant, seed, run_kind)
if not output.exists():
return None
failed = output.with_suffix(".failed.json")
if failed.exists():
failed = output.with_suffix(f".failed-{time.time_ns()}.json")
output.replace(failed)
return str(failed)
def run_cells(
args: argparse.Namespace,
cells: list[tuple[str, int, str]],
@@ -138,13 +166,15 @@ def run_cells(
variant, seed, run_kind = item["identity"]
elapsed = time.monotonic() - item["started"]
if return_code != 0:
for survivor in survivors:
survivor["process"].terminate()
for survivor in running:
if survivor is not item and survivor not in survivors:
survivor["process"].terminate()
stop_processes(
[candidate for candidate in running if candidate is not item]
)
quarantined = quarantine_failed_output(
args.output_dir, variant, seed, run_kind
)
raise RuntimeError(
f"cell failed: {variant}/{seed}/{run_kind}: {return_code}"
f"cell failed: {variant}/{seed}/{run_kind}: {return_code}; "
f"quarantined_output={quarantined}"
)
completed += 1
print(
experiments/k3/attnres_forward/train.py
+11 -4
View File
@@ -170,9 +170,6 @@ class ForwardInterventionLanguageModel(parent.GradientLanguageModel):
def forward(
self, input_ids: torch.Tensor, capture: bool = False
) -> tuple[torch.Tensor, parent.ActivationTrace | None]:
if not self.selected_indices:
return super().forward(input_ids, capture)
self.forward_calls += 1
embedded = self.embed(input_ids)
trace = parent.ActivationTrace([], [], [], [], []) if capture else None
@@ -276,11 +273,13 @@ def build_intervention_audit(
str(index): (
source_counts[str(index)]
== [study_manifest["selected_source_counts"][str(index)]]
== [EXPECTED_SOURCE_COUNTS[index]]
)
for index in selected
}
visit_gate = (
all(value == model.forward_calls for value in model.depth_visits)
model.forward_calls > 0
and all(value == model.forward_calls for value in model.depth_visits)
and model.output_visits == model.forward_calls
)
selected_parameter_gate = all(
@@ -415,6 +414,14 @@ def main() -> None:
)
if expected != VARIANTS[variant]:
raise ValueError("study manifest selector drift")
for index, source_count in EXPECTED_SOURCE_COUNTS.items():
if (
study_manifest["selected_source_counts"].get(str(index))
!= source_count
):
raise ValueError(
f"study manifest source-count drift at depth index {index}"
)
output_value = argument_value("--output")
if output_value is None:
experiments/k3/attnres_forward/verify.py
+9 -4
View File
@@ -92,10 +92,14 @@ def smoke_compare(args: argparse.Namespace) -> None:
"gradient_gate",
"environment",
)
checks = {
field: parent_result[field] == wrapper_result[field]
for field in fields
}
checks = {}
for field in fields:
parent_value = exact_structure(parent_result[field])
wrapper_value = exact_structure(wrapper_result[field])
if field == "manifest":
parent_value.pop("path", None)
wrapper_value.pop("path", None)
checks[field] = parent_value == wrapper_value
wrapper_identity = {
"protocol": wrapper_result["protocol_id"] == runner.PROTOCOL_ID,
"parent_protocol": (
@@ -185,6 +189,7 @@ def step_zero(args: argparse.Namespace) -> None:
"initial_mixer_hash": initial_mixer,
"logits_sha256": tensor_sha256(logits),
"loss_nats": loss.detach().cpu().item(),
"loss_tensor_sha256": tensor_sha256(loss),
"evaluation": exact_structure(evaluation),
"diagnostic": exact_structure(diagnostic),
}