research: audit AttnRes local path matrix

scripts/check-k3-attnres-local-path-data.mjs

@@ -0,0 +1,106 @@
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync } from "node:fs";
+
+const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
+const read = (path) => {
+  const bytes = readFileSync(new URL(path, import.meta.url));
+  return { bytes, json: JSON.parse(bytes), sha256: hash(bytes) };
+};
+const aggregate = read("../src/data/k3-attnres-local-path.json");
+const compact = read("../src/data/k3-attnres-local-path-compact.json");
+const reproduction = read("../experiments/k3/attnres_local_path/reproduction.json");
+const manifest = read("../experiments/k3/attnres_local_path/manifest.json");
+const rawDirectory = new URL("../experiments/k3/attnres_local_path/results/raw/", import.meta.url);
+const failures = [];
+const expect = (condition, message) => {
+  if (!condition) failures.push(message);
+};
+const close = (actual, expected, tolerance = 1e-15) =>
+  Math.abs(actual - expected) <= tolerance;
+
+expect(aggregate.sha256 === "bb0ec9fce5b30d50ad5c50c4b95af7a892d614f205a2c20cfc2662125e10160e", "aggregate physical SHA-256 changed");
+expect(compact.sha256 === "3bb6c15815f37d2109386fb87f9642246d41e636242ca3a9ba3c17ae8f419c08", "compact physical SHA-256 changed");
+expect(reproduction.sha256 === "524a6883c08941c1be908834d7bcf915eb32f0bdb501aae09f3cec761fdd65db", "reproduction physical SHA-256 changed");
+expect(manifest.sha256 === "db01e92ef2cf0896212fcd529429bd94a344de0e1195db7f87b9a56dc3449139", "manifest physical SHA-256 changed");
+
+expect(aggregate.json.canonical_sha256_without_self === "b86d119cd2f106e2cbee8a35760ed3244336a2fcfeb9178ea1e7dab13fc6f215", "aggregate canonical SHA-256 changed");
+expect(compact.json.canonical_sha256_without_self === "2aff9288f52d3d41bb1f59c64d9a07518ad3e2120c61615478087b24aaabd835", "compact canonical SHA-256 changed");
+expect(reproduction.json.canonical_sha256_without_self === "6f5d98fce6446fecc966dd2675f272f2c4f0c9a39a5741fabc4ffad6852ca7f4", "reproduction canonical SHA-256 changed");
+
+expect(compact.json.protocol_id === "llm-atlas-k3-attnres-local-path-v1", "protocol identity mismatch");
+expect(compact.json.study.seeds.length === 3, "formal seed count changed");
+expect(compact.json.study.steps === 8000, "formal step budget changed");
+expect(compact.json.study.modes.length === 14, "matrix mode count changed");
+expect(compact.json.study.spike_layers.join(",") === "21,22,23,24,25", "fixed spike set changed");
+expect(compact.json.study.formal_target_bytes === 196608000, "formal target-byte count changed");
+expect(compact.json.study.total_target_bytes_with_replay === 262144000, "total target-byte count changed");
+expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 4, "raw run count is not four");
+
+for (const [name, expected] of Object.entries(reproduction.json.raw_files)) {
+  const raw = read(`../experiments/k3/attnres_local_path/results/raw/${name}`);
+  expect(raw.sha256 === expected.file_sha256, `${name} physical hash mismatch`);
+  expect(raw.json.canonical_sha256_without_self === expected.canonical_sha256, `${name} canonical hash mismatch`);
+  expect(raw.json.round06_equivalence.passed, `${name} Round 06 equivalence failed`);
+}
+
+expect(compact.json.hashes.aggregate_canonical_sha256 === aggregate.json.canonical_sha256_without_self, "compact→aggregate canonical link mismatch");
+expect(compact.json.hashes.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "compact→reproduction canonical link mismatch");
+expect(reproduction.json.aggregate.canonical_sha256 === aggregate.json.canonical_sha256_without_self, "reproduction→aggregate canonical link mismatch");
+expect(reproduction.json.replay_gate.passed, "full replay is not exact");
+expect(reproduction.json.replay_gate.frozen_compare_sha256 === "7dbd15ad03fbd357c5d91e159706d63b24703722f76c492ed1dc733535d6b9cf", "replay compare hash changed");
+expect(reproduction.json.post_result_grok_review.blocking_errors === 0, "post-result audit reports a blocking error");
+expect(reproduction.json.post_result_grok_review.localization_status_confirmed, "post-result audit did not confirm the status");
+
+const gates = compact.json.gates;
+expect(gates.all_input_and_parent_gates_passed, "input or parent gate failed");
+expect(gates.global_gap.passed && gates.global_gap.passed_cells === 6, "global gap gate changed");
+expect(gates.sufficiency.groups_6_7.passed && gates.sufficiency.groups_6_7.passed_cells === 6, "groups 6+7 sufficiency gate failed");
+expect(!gates.restoration.groups_6_7.passed && gates.restoration.groups_6_7.passed_cells === 3, "groups 6+7 restoration verdict changed");
+expect(!gates.localization.passed, "localization unexpectedly passed");
+expect(gates.localization.status === "one_sided_evidence_localization_not_established", "localization status changed");
+expect(gates.sufficiency.group_6.passed && gates.sufficiency.group_7.passed, "single-group sufficiency gate changed");
+expect(!gates.restoration.group_6.passed && !gates.restoration.group_7.passed, "single-group restoration unexpectedly passed");
+expect(!gates.sufficiency.output_half_gap.passed && gates.sufficiency.output_half_gap.passed_cells === 0, "output half-gap control changed");
+expect(!gates.branch_dominance.group_6.passed, "group 6 branch dominance unexpectedly passed");
+expect(gates.branch_dominance.group_7.passed && gates.branch_dominance.group_7.dominant_branch === "mlp", "group 7 MLP dominance changed");
+
+expect(close(compact.json.means.sufficiency.uniform_groups_6_7_only.spike_contrast, 0.6769080107044138), "groups 6+7 mean contrast sufficiency changed");
+expect(close(compact.json.means.sufficiency.uniform_groups_6_7_only.peak_normalized, 1.7003398291502192), "groups 6+7 mean peak sufficiency changed");
+expect(close(compact.json.means.restoration.uniform_except_groups_6_7.spike_contrast, 0.6499896714884171), "groups 6+7 mean contrast restoration changed");
+expect(close(compact.json.means.restoration.uniform_except_groups_6_7.peak_normalized, 0.3801045652660336), "groups 6+7 mean peak restoration changed");
+expect(close(compact.json.means.sufficiency.uniform_output_only.spike_contrast, 0.13066777032743535), "output-only mean contrast score changed");
+expect(close(compact.json.means.sufficiency.uniform_output_only.peak_normalized, 0.18724852586461135), "output-only mean peak score changed");
+
+for (const spectrum of compact.json.final_spectra) {
+  expect(Object.keys(spectrum.modes).length === 14, `seed ${spectrum.seed} spectrum mode count changed`);
+  expect(spectrum.modes.detached_learned.uniform_count === 0, `seed ${spectrum.seed} reference census changed`);
+  expect(spectrum.modes.uniform_groups_6_7_only.uniform_count === 16, `seed ${spectrum.seed} groups 6+7 census changed`);
+  expect(spectrum.modes.uniform_except_groups_6_7.uniform_count === 49, `seed ${spectrum.seed} restoration census changed`);
+  expect(spectrum.modes.uniform_all.uniform_count === 65, `seed ${spectrum.seed} all-uniform census changed`);
+  for (const mode of Object.values(spectrum.modes)) {
+    expect(mode.normalized.length === 32, `seed ${spectrum.seed} normalized spectrum length changed`);
+  }
+}
+
+if (failures.length) {
+  console.error(`FAIL K3 AttnRes local-path data\n- ${failures.join("\n- ")}`);
+  process.exit(1);
+}
+
+console.log(JSON.stringify({
+  protocol: compact.json.protocol_id,
+  modes: compact.json.study.modes.length,
+  formalRuns: compact.json.study.seeds.length,
+  replayExact: reproduction.json.replay_gate.passed,
+  globalGap: gates.global_gap,
+  sufficiency: gates.sufficiency.groups_6_7,
+  restoration: gates.restoration.groups_6_7,
+  localization: gates.localization,
+  branch: gates.branch_dominance,
+  hashes: {
+    aggregate: aggregate.sha256,
+    compact: compact.sha256,
+    reproduction: reproduction.sha256,
+  },
+}, null, 2));
+console.log("PASS K3 AttnRes local-path frozen data");