feat: add reduced AttnRes trace lab

2026-07-30 07:30:01 +08:00
parent d1d9d22bf3
commit 4ce780dcc9
11 changed files with 1070 additions and 26 deletions
@@ -0,0 +1,61 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+
+const read = (path) => {
+  const bytes = readFileSync(new URL(path, import.meta.url));
+  return {
+    bytes,
+    json: JSON.parse(bytes),
+    sha256: createHash("sha256").update(bytes).digest("hex"),
+  };
+};
+
+const raw = read("../src/data/k3-attnres-reduced.json");
+const compact = read("../src/data/k3-attnres-reduced-compact.json");
+const reproduction = read("../experiments/k3/attnres/reproduction.json");
+const failures = [];
+
+const expect = (condition, message) => {
+  if (!condition) failures.push(message);
+};
+
+expect(raw.sha256 === "44f8622654d32485f8d6e698c02ba1365ddffb10cbd73db0294136d0bd93ce88", "raw payload SHA-256 changed");
+expect(compact.sha256 === "44864d48eddb2ae5887fba4b74f63b5a3d6a23497886ee307decf5b4f45d9faf", "compact payload SHA-256 changed");
+expect(reproduction.sha256 === "545543b7e4a970ca3bc0e6246610a32fb53ec3d9546a98e0f17f38d7121918a2", "reproduction payload SHA-256 changed");
+expect(compact.json.protocol_id === "llm-atlas-k3-attnres-reduced-v1", "protocol identity mismatch");
+expect(compact.json.grid.runs === 9, "formal grid is not 9 cells");
+expect(compact.json.grid.target_bytes_total === 147_456_000, "formal target-byte budget mismatch");
+expect(raw.json.formal_runs.length === 9, "raw formal run count mismatch");
+expect(Object.values(reproduction.json.common_initial_parameters).every((row) => row.exact), "common initial parameters are not exact within seed");
+expect(Object.values(reproduction.json.smoke).every((row) => row.all_exact), "paired smoke replay mismatch");
+expect(reproduction.json.formal_replay.all_numeric_and_hash_fields_exact, "formal fresh-process replay mismatch");
+expect(!reproduction.json.formal_replay.timing_exact_required, "timing must not be an exact replay requirement");
+
+const full = compact.json.final_validation.full_contrast;
+const block = compact.json.final_validation.block_contrast;
+expect(full.paired_deltas_bpc.length === 3 && full.paired_deltas_bpc.every((value) => value < 0), "Full paired direction mismatch");
+expect(block.paired_deltas_bpc.length === 3 && block.paired_deltas_bpc.every((value) => value < 0), "Block paired direction mismatch");
+expect(Math.abs(full.mean_delta_bpc - (-0.014567152672337214)) < 1e-15, "Full mean delta changed");
+expect(Math.abs(block.mean_delta_bpc - (-0.04246557635602392)) < 1e-15, "Block mean delta changed");
+expect(compact.json.gradients.baseline.mean_cv < compact.json.gradients.full.mean_cv, "gradient counterevidence order baseline/full changed");
+expect(compact.json.gradients.full.mean_cv < compact.json.gradients.block.mean_cv, "gradient counterevidence order full/block changed");
+expect(compact.json.posthoc.label.startsWith("post-hoc"), "post-hoc callout lost its evidence label");
+
+if (failures.length) {
+  console.error(`FAIL K3 AttnRes data\n- ${failures.join("\n- ")}`);
+  process.exit(1);
+}
+
+console.log(JSON.stringify({
+  protocol: compact.json.protocol_id,
+  formalRuns: compact.json.grid.runs,
+  fullMeanDeltaBpc: full.mean_delta_bpc,
+  blockMeanDeltaBpc: block.mean_delta_bpc,
+  formalReplayExact: reproduction.json.formal_replay.all_numeric_and_hash_fields_exact,
+  hashes: {
+    raw: raw.sha256,
+    compact: compact.sha256,
+    reproduction: reproduction.sha256,
+  },
+}, null, 2));
+console.log("PASS K3 AttnRes frozen data");