wuyang
69 lines
3.9 KiB
JavaScript
69 lines
3.9 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import { readdirSync, readFileSync } from "node:fs";
|
|
|
|
const read = (path) => {
|
|
const bytes = readFileSync(new URL(path, import.meta.url));
|
|
return {
|
|
bytes,
|
|
json: JSON.parse(bytes),
|
|
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
};
|
|
};
|
|
|
|
const raw = read("../src/data/k3-attnres-gradient.json");
|
|
const compact = read("../src/data/k3-attnres-gradient-compact.json");
|
|
const reproduction = read("../experiments/k3/attnres_gradient/reproduction.json");
|
|
const rawDirectory = new URL("../experiments/k3/attnres_gradient/results/raw/", import.meta.url);
|
|
const failures = [];
|
|
const expect = (condition, message) => {
|
|
if (!condition) failures.push(message);
|
|
};
|
|
|
|
expect(raw.sha256 === "ad461cbe74fc671f356288d37c9b628618814915116e4fbe03aaaa48367e6a8d", "raw aggregate SHA-256 changed");
|
|
expect(compact.sha256 === "5377a5e731db2fdc85a0327f05d43f1cd067d3fc34633619d3004f3fc31968e3", "compact payload SHA-256 changed");
|
|
expect(reproduction.sha256 === "aedcde6accc6eb1c24b122ffb55706ef620062e848d9fe4c5f622b084a4fdea6", "reproduction payload SHA-256 changed");
|
|
expect(compact.json.protocol_id === "llm-atlas-k3-attnres-gradient-scale-v1", "protocol identity mismatch");
|
|
expect(compact.json.study.formal_runs === 12, "formal grid is not 12 cells");
|
|
expect(compact.json.study.formal_target_bytes === 786_432_000, "formal target-byte budget mismatch");
|
|
expect(compact.json.study.replay_target_bytes === 65_536_000, "replay byte budget mismatch");
|
|
expect(compact.json.cells.length === 12, "compact cell count mismatch");
|
|
expect(Object.keys(raw.json.runs).length === 12, "raw formal cell count mismatch");
|
|
expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 21, "public raw output count is not 21");
|
|
expect(reproduction.json.replay_exact.exact, "full formal replay is not exact");
|
|
expect(Object.values(reproduction.json.smoke_exact).every((row) => row.exact && row.gradient_gate.passed), "paired smoke or loss-scale gate failed");
|
|
|
|
for (const depth of ["16", "32"]) {
|
|
const summary = compact.json.depth_summaries[depth];
|
|
expect(summary.verdict.label === "mixed / inconclusive at this depth", `depth ${depth} verdict changed`);
|
|
expect(summary.by_seed.length === 3, `depth ${depth} seed count changed`);
|
|
expect(summary.by_seed.every((row) => row.block_minus_baseline_bpc < 0), `depth ${depth} BPC pairing changed`);
|
|
expect(summary.by_seed.every((row) => row.relative_cv_reduction < 0), `depth ${depth} CV counterevidence changed`);
|
|
expect(summary.by_seed.every((row) => row.relative_imbalance_reduction > 0), `depth ${depth} first/last improvement changed`);
|
|
}
|
|
|
|
expect(Math.abs(compact.json.depth_summaries["16"].means.relative_cv_reduction - (-0.10264135379685868)) < 1e-15, "depth-16 CV contrast changed");
|
|
expect(Math.abs(compact.json.depth_summaries["32"].means.relative_cv_reduction - (-0.600330100169428)) < 1e-15, "depth-32 CV contrast changed");
|
|
expect(Math.abs(compact.json.depth_summaries["16"].means.relative_imbalance_reduction - 0.6099652484299795) < 1e-15, "depth-16 imbalance contrast changed");
|
|
expect(Math.abs(compact.json.depth_summaries["32"].means.relative_imbalance_reduction - 0.7200517407719272) < 1e-15, "depth-32 imbalance contrast changed");
|
|
expect(compact.json.overall_verdict === "depth-dependent or inconclusive", "overall preregistered verdict changed");
|
|
|
|
if (failures.length) {
|
|
console.error(`FAIL K3 AttnRes gradient data\n- ${failures.join("\n- ")}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(JSON.stringify({
|
|
protocol: compact.json.protocol_id,
|
|
formalRuns: compact.json.study.formal_runs,
|
|
formalTargetBytes: compact.json.study.formal_target_bytes,
|
|
depth16: compact.json.depth_summaries["16"].verdict.label,
|
|
depth32: compact.json.depth_summaries["32"].verdict.label,
|
|
replayExact: reproduction.json.replay_exact.exact,
|
|
hashes: {
|
|
raw: raw.sha256,
|
|
compact: compact.sha256,
|
|
reproduction: reproduction.sha256,
|
|
},
|
|
}, null, 2));
|
|
console.log("PASS K3 AttnRes gradient frozen data");
|