import { createHash } from "node:crypto"; import { readdirSync, readFileSync } from "node:fs"; const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); const read = (path) => { const bytes = readFileSync(new URL(path, import.meta.url)); return { bytes, json: JSON.parse(bytes), sha256: hash(bytes) }; }; const aggregate = read("../src/data/k3-attnres-spike.json"); const compact = read("../src/data/k3-attnres-spike-compact.json"); const reproduction = read("../experiments/k3/attnres_spike/reproduction.json"); const manifest = read("../experiments/k3/attnres_spike/manifest.json"); const rawDirectory = new URL("../experiments/k3/attnres_spike/results/raw/", import.meta.url); const failures = []; const expect = (condition, message) => { if (!condition) failures.push(message); }; expect(aggregate.sha256 === "75058f9ef1f89593436960428378ab920569017f1758f9c15a2a0b1e081c1e90", "aggregate physical SHA-256 changed"); expect(compact.sha256 === "7b3a3749d1344e56fb61039d2678a4e2aefe11a0a1e279516d6f27ff3bb8ef6c", "compact physical SHA-256 changed"); expect(reproduction.sha256 === "86ac20ee06189a924eed2b2614e535660b536447f5ad019f4c1fd99cf4aa879a", "reproduction physical SHA-256 changed"); expect(manifest.sha256 === "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "manifest physical SHA-256 changed"); expect(aggregate.json.canonical_sha256_without_self === "1d0faef297682d61d35ce2ab54da3868e18a84e9ebc6e44c617d08016e4fc8b4", "aggregate canonical SHA-256 changed"); expect(compact.json.canonical_sha256_without_self === "a430dd26f512db7d64c933df65d9a97e6899028099333fe0a1212728f13126cd", "compact canonical SHA-256 changed"); expect(reproduction.json.canonical_sha256_without_self === "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515", "reproduction canonical SHA-256 changed"); expect(compact.json.protocol_id === "llm-atlas-k3-attnres-spike-path-v1", "protocol identity mismatch"); expect(compact.json.study.seeds.length === 3, "formal seed count changed"); expect(compact.json.study.steps === 8000, "formal step budget changed"); expect(compact.json.study.spike_layers.join(",") === "21,22,23,24,25", "fixed spike set changed"); expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 4, "raw run count is not four"); for (const [name, expected] of Object.entries(reproduction.json.raw_files)) { const raw = read(`../experiments/k3/attnres_spike/results/raw/${name}`); expect(raw.sha256 === expected.file_sha256, `${name} physical hash mismatch`); expect(raw.json.canonical_sha256_without_self === expected.canonical_sha256, `${name} canonical hash mismatch`); } expect(compact.json.hashes.aggregate_canonical_sha256 === aggregate.json.canonical_sha256_without_self, "compact→aggregate canonical link mismatch"); expect(compact.json.hashes.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "compact→reproduction canonical link mismatch"); expect(aggregate.json.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "aggregate→reproduction canonical link mismatch"); expect(reproduction.json.replay_gate.passed, "full replay is not exact"); expect(reproduction.json.replay_gate.frozen_compare_sha256 === "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", "replay compare hash changed"); expect(compact.json.gates.round05_equivalence.every((item) => item.passed), "Round 05 equivalence gate failed"); expect(compact.json.gates.forward_identity.every((item) => item.passed), "forward identity gate failed"); expect(compact.json.gates.initialization_negative_control.every((item) => item.passed), "initialization negative control failed"); expect(compact.json.gates.loss_scale.every((item) => item.passed), "loss-scale gate failed"); const verdicts = compact.json.verdicts; expect(verdicts.reduction_robustness.verdict === "robust within the preregistered reduction family", "reduction robustness verdict changed"); expect(verdicts.reduction_robustness.passed_cells === 12, "reduction pass count changed"); expect(verdicts.visible_positions.visible_positions.length === 6, "visible-position count changed"); expect(verdicts.visible_positions.earliest_observed_tensor === "pre_attention_input", "earliest observed tensor changed"); expect(verdicts.interventions.softmax_key_path.opposite_direction_3_of_3, "softmax/key opposite-direction result changed"); expect(!verdicts.interventions.softmax_key_path.passed_3_of_3, "softmax/key path unexpectedly passed material threshold"); expect(verdicts.interventions.value_coefficients.passed_3_of_3, "value-coefficient material threshold failed"); expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_contrast - 0.701998806119879) < 1e-15, "value contrast drop changed"); expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_peak - 0.3702754647109951) < 1e-15, "value peak drop changed"); for (const run of compact.json.final_arrays) { expect(Object.keys(run.position_element_rms).length === 6, `seed ${run.seed} position count changed`); expect(Object.keys(run.post_mlp_reductions).length === 7, `seed ${run.seed} reduction count changed`); expect(Object.keys(run.interventions).length === 3, `seed ${run.seed} intervention count changed`); } if (failures.length) { console.error(`FAIL K3 AttnRes spike data\n- ${failures.join("\n- ")}`); process.exit(1); } console.log(JSON.stringify({ protocol: compact.json.protocol_id, formalRuns: compact.json.study.seeds.length, replayExact: reproduction.json.replay_gate.passed, reduction: verdicts.reduction_robustness.verdict, visiblePositions: verdicts.visible_positions.visible_positions.length, softmaxKey: verdicts.interventions.softmax_key_path.verdict, valueCoefficients: verdicts.interventions.value_coefficients.verdict, hashes: { aggregate: aggregate.sha256, compact: compact.sha256, reproduction: reproduction.sha256, }, }, null, 2)); console.log("PASS K3 AttnRes spike frozen data");