research: audit AttnRes spike path study

scripts/check-k3-attnres-spike-data.mjs

@@ -0,0 +1,87 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync } from "node:fs";
+
+const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
+const read = (path) => {
+  const bytes = readFileSync(new URL(path, import.meta.url));
+  return { bytes, json: JSON.parse(bytes), sha256: hash(bytes) };
+};
+const aggregate = read("../src/data/k3-attnres-spike.json");
+const compact = read("../src/data/k3-attnres-spike-compact.json");
+const reproduction = read("../experiments/k3/attnres_spike/reproduction.json");
+const manifest = read("../experiments/k3/attnres_spike/manifest.json");
+const rawDirectory = new URL("../experiments/k3/attnres_spike/results/raw/", import.meta.url);
+const failures = [];
+const expect = (condition, message) => {
+  if (!condition) failures.push(message);
+};
+
+expect(aggregate.sha256 === "75058f9ef1f89593436960428378ab920569017f1758f9c15a2a0b1e081c1e90", "aggregate physical SHA-256 changed");
+expect(compact.sha256 === "7b3a3749d1344e56fb61039d2678a4e2aefe11a0a1e279516d6f27ff3bb8ef6c", "compact physical SHA-256 changed");
+expect(reproduction.sha256 === "86ac20ee06189a924eed2b2614e535660b536447f5ad019f4c1fd99cf4aa879a", "reproduction physical SHA-256 changed");
+expect(manifest.sha256 === "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "manifest physical SHA-256 changed");
+
+expect(aggregate.json.canonical_sha256_without_self === "1d0faef297682d61d35ce2ab54da3868e18a84e9ebc6e44c617d08016e4fc8b4", "aggregate canonical SHA-256 changed");
+expect(compact.json.canonical_sha256_without_self === "a430dd26f512db7d64c933df65d9a97e6899028099333fe0a1212728f13126cd", "compact canonical SHA-256 changed");
+expect(reproduction.json.canonical_sha256_without_self === "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515", "reproduction canonical SHA-256 changed");
+
+expect(compact.json.protocol_id === "llm-atlas-k3-attnres-spike-path-v1", "protocol identity mismatch");
+expect(compact.json.study.seeds.length === 3, "formal seed count changed");
+expect(compact.json.study.steps === 8000, "formal step budget changed");
+expect(compact.json.study.spike_layers.join(",") === "21,22,23,24,25", "fixed spike set changed");
+expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 4, "raw run count is not four");
+
+for (const [name, expected] of Object.entries(reproduction.json.raw_files)) {
+  const raw = read(`../experiments/k3/attnres_spike/results/raw/${name}`);
+  expect(raw.sha256 === expected.file_sha256, `${name} physical hash mismatch`);
+  expect(raw.json.canonical_sha256_without_self === expected.canonical_sha256, `${name} canonical hash mismatch`);
+}
+
+expect(compact.json.hashes.aggregate_canonical_sha256 === aggregate.json.canonical_sha256_without_self, "compact→aggregate canonical link mismatch");
+expect(compact.json.hashes.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "compact→reproduction canonical link mismatch");
+expect(aggregate.json.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "aggregate→reproduction canonical link mismatch");
+
+expect(reproduction.json.replay_gate.passed, "full replay is not exact");
+expect(reproduction.json.replay_gate.frozen_compare_sha256 === "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", "replay compare hash changed");
+expect(compact.json.gates.round05_equivalence.every((item) => item.passed), "Round 05 equivalence gate failed");
+expect(compact.json.gates.forward_identity.every((item) => item.passed), "forward identity gate failed");
+expect(compact.json.gates.initialization_negative_control.every((item) => item.passed), "initialization negative control failed");
+expect(compact.json.gates.loss_scale.every((item) => item.passed), "loss-scale gate failed");
+
+const verdicts = compact.json.verdicts;
+expect(verdicts.reduction_robustness.verdict === "robust within the preregistered reduction family", "reduction robustness verdict changed");
+expect(verdicts.reduction_robustness.passed_cells === 12, "reduction pass count changed");
+expect(verdicts.visible_positions.visible_positions.length === 6, "visible-position count changed");
+expect(verdicts.visible_positions.earliest_observed_tensor === "pre_attention_input", "earliest observed tensor changed");
+expect(verdicts.interventions.softmax_key_path.opposite_direction_3_of_3, "softmax/key opposite-direction result changed");
+expect(!verdicts.interventions.softmax_key_path.passed_3_of_3, "softmax/key path unexpectedly passed material threshold");
+expect(verdicts.interventions.value_coefficients.passed_3_of_3, "value-coefficient material threshold failed");
+expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_contrast - 0.701998806119879) < 1e-15, "value contrast drop changed");
+expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_peak - 0.3702754647109951) < 1e-15, "value peak drop changed");
+
+for (const run of compact.json.final_arrays) {
+  expect(Object.keys(run.position_element_rms).length === 6, `seed ${run.seed} position count changed`);
+  expect(Object.keys(run.post_mlp_reductions).length === 7, `seed ${run.seed} reduction count changed`);
+  expect(Object.keys(run.interventions).length === 3, `seed ${run.seed} intervention count changed`);
+}
+
+if (failures.length) {
+  console.error(`FAIL K3 AttnRes spike data\n- ${failures.join("\n- ")}`);
+  process.exit(1);
+}
+
+console.log(JSON.stringify({
+  protocol: compact.json.protocol_id,
+  formalRuns: compact.json.study.seeds.length,
+  replayExact: reproduction.json.replay_gate.passed,
+  reduction: verdicts.reduction_robustness.verdict,
+  visiblePositions: verdicts.visible_positions.visible_positions.length,
+  softmaxKey: verdicts.interventions.softmax_key_path.verdict,
+  valueCoefficients: verdicts.interventions.value_coefficients.verdict,
+  hashes: {
+    aggregate: aggregate.sha256,
+    compact: compact.sha256,
+    reproduction: reproduction.sha256,
+  },
+}, null, 2));
+console.log("PASS K3 AttnRes spike frozen data");