research: audit task bootstrap CRN experiment
This commit is contained in:
wuyang
@@ -0,0 +1,347 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import process from "node:process";
|
||||
|
||||
const CONDITIONS = ["s0_eos", "s1_eos", "s0_period", "s1_period"];
|
||||
const DOMAINS = ["code", "math"];
|
||||
const CONTRASTS = [
|
||||
"period_at_s0",
|
||||
"period_at_s1",
|
||||
"system_at_eos",
|
||||
"system_at_period",
|
||||
];
|
||||
const PATHS = {
|
||||
sampling: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn.json",
|
||||
evaluation: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-eval.json",
|
||||
reproduction: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-reproduction.json",
|
||||
analysis: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-analysis.json",
|
||||
output: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-compact.json",
|
||||
};
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = { ...PATHS };
|
||||
for (let index = 2; index < argv.length; index += 2) {
|
||||
const key = argv[index]?.replace(/^--/, "");
|
||||
const value = argv[index + 1];
|
||||
if (!(key in result) || value === undefined) {
|
||||
throw new Error(`Unknown or incomplete argument: ${argv[index]}`);
|
||||
}
|
||||
result[key] = value;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function success(row) {
|
||||
const evaluation = row.task_evaluation;
|
||||
return row.domain === "code"
|
||||
? evaluation.fixed_budget_tests_pass
|
||||
: evaluation.fixed_budget_numeric_exact;
|
||||
}
|
||||
|
||||
const paths = parseArgs(process.argv);
|
||||
const inputBytes = {};
|
||||
const input = {};
|
||||
for (const name of ["sampling", "evaluation", "reproduction", "analysis"]) {
|
||||
inputBytes[name] = await readFile(paths[name]);
|
||||
input[name] = JSON.parse(inputBytes[name]);
|
||||
}
|
||||
const { sampling, evaluation, reproduction, analysis } = input;
|
||||
const evaluationIndex = new Map(
|
||||
evaluation.rows.map((row) => [
|
||||
`${row.source_id}\0${row.tape_label}\0${row.condition}`,
|
||||
row,
|
||||
]),
|
||||
);
|
||||
const sampleIndex = new Map(
|
||||
sampling.sources.flatMap((source) =>
|
||||
source.runs.flatMap((run) =>
|
||||
run.outputs.map((output) => [
|
||||
`${source.id}\0${run.tape_label}\0${output.condition}`,
|
||||
output,
|
||||
]),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const conditionTable = Object.fromEntries(
|
||||
DOMAINS.map((domain) => [
|
||||
domain,
|
||||
CONDITIONS.map((condition) => {
|
||||
const summary =
|
||||
evaluation.summary.main_t0_by_domain_condition[domain][condition];
|
||||
return {
|
||||
condition,
|
||||
outputs: summary.outputs,
|
||||
success: summary.fixed_budget_success,
|
||||
successRate: summary.fixed_budget_success / summary.outputs,
|
||||
naturalEos: summary.natural_eos,
|
||||
naturalEosRate: summary.natural_eos / summary.outputs,
|
||||
meanTokens: summary.mean_generated_tokens,
|
||||
outcomes: summary.task_outcomes,
|
||||
};
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const contrasts = Object.fromEntries(
|
||||
DOMAINS.map((domain) => [
|
||||
domain,
|
||||
Object.fromEntries(
|
||||
CONTRASTS.map((name) => {
|
||||
const row =
|
||||
analysis.main_t0_selected_task_analysis[domain].contrasts[name];
|
||||
const metrics = Object.fromEntries(
|
||||
Object.entries(row.metrics).map(([metric, value]) => [
|
||||
metric,
|
||||
{
|
||||
point: value.right_minus_left_point,
|
||||
band: value.selected_task_resampling_band,
|
||||
directions: value.direction_counts,
|
||||
transition: value.transition ?? null,
|
||||
},
|
||||
]),
|
||||
);
|
||||
return [
|
||||
name,
|
||||
{
|
||||
left: row.left,
|
||||
right: row.right,
|
||||
metrics,
|
||||
trajectory: {
|
||||
sources: row.trajectory.sources,
|
||||
sharedUniformExact:
|
||||
row.trajectory.shared_uniform_prefix_exact,
|
||||
exactTrajectories: row.trajectory.exact_trajectories,
|
||||
commonPrefixTokens: row.trajectory.common_prefix_tokens,
|
||||
},
|
||||
},
|
||||
];
|
||||
}),
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
const diagnostic = Object.fromEntries(
|
||||
DOMAINS.map((domain) => [
|
||||
domain,
|
||||
{
|
||||
sourceIds: analysis.multi_tape_diagnostic[domain].source_ids,
|
||||
tapes: analysis.multi_tape_diagnostic[domain].tapes,
|
||||
contrasts: Object.fromEntries(
|
||||
CONTRASTS.map((name) => {
|
||||
const row =
|
||||
analysis.multi_tape_diagnostic[domain].contrasts[name];
|
||||
return [
|
||||
name,
|
||||
{
|
||||
left: row.left,
|
||||
right: row.right,
|
||||
success: {
|
||||
matrix:
|
||||
row.metrics.fixed_budget_success.matrix_task_by_tape,
|
||||
tapeMeans:
|
||||
row.metrics.fixed_budget_success.tape_means,
|
||||
taskRangeWithinTape:
|
||||
row.metrics.fixed_budget_success.task_range_within_tape,
|
||||
tapeRangeWithinTask:
|
||||
row.metrics.fixed_budget_success.tape_range_within_task,
|
||||
},
|
||||
tokens: {
|
||||
matrix: row.metrics.generated_tokens.matrix_task_by_tape,
|
||||
tapeMeans: row.metrics.generated_tokens.tape_means,
|
||||
taskRangeWithinTape:
|
||||
row.metrics.generated_tokens.task_range_within_tape,
|
||||
tapeRangeWithinTask:
|
||||
row.metrics.generated_tokens.tape_range_within_task,
|
||||
},
|
||||
},
|
||||
];
|
||||
}),
|
||||
),
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const tasks = Object.fromEntries(
|
||||
DOMAINS.map((domain) => {
|
||||
const domainSources = sampling.sources
|
||||
.filter((source) => source.domain === domain)
|
||||
.sort((left, right) => left.within_domain_index - right.within_domain_index);
|
||||
return [
|
||||
domain,
|
||||
domainSources.map((source) => {
|
||||
const conditions = Object.fromEntries(
|
||||
CONDITIONS.map((condition) => {
|
||||
const evaluationRow = evaluationIndex.get(
|
||||
`${source.id}\0T0\0${condition}`,
|
||||
);
|
||||
const samplingRow = sampleIndex.get(
|
||||
`${source.id}\0T0\0${condition}`,
|
||||
);
|
||||
return [
|
||||
condition,
|
||||
{
|
||||
success: success(evaluationRow),
|
||||
outcome: evaluationRow.task_outcome,
|
||||
tokens: evaluationRow.generated_tokens,
|
||||
naturalEos: evaluationRow.hit_eos,
|
||||
truncated: evaluationRow.stopped_at_max_new_tokens,
|
||||
trajectoryHash:
|
||||
evaluationRow.generated_token_ids_sha256.slice(0, 12),
|
||||
firstTokenIds: samplingRow.generated_token_ids.slice(0, 8),
|
||||
},
|
||||
];
|
||||
}),
|
||||
);
|
||||
const taskContrasts = Object.fromEntries(
|
||||
CONTRASTS.map((name) => {
|
||||
const sourceRow =
|
||||
analysis.main_t0_selected_task_analysis[domain].contrasts[name];
|
||||
const successRow =
|
||||
sourceRow.metrics.fixed_budget_success.by_source.find(
|
||||
(row) => row.source_id === source.id,
|
||||
);
|
||||
const tokenRow = sourceRow.metrics.generated_tokens.by_source.find(
|
||||
(row) => row.source_id === source.id,
|
||||
);
|
||||
const trajectoryRow = sourceRow.trajectory.rows.find(
|
||||
(row) => row.source_id === source.id,
|
||||
);
|
||||
return [
|
||||
name,
|
||||
{
|
||||
successDelta: successRow.right_minus_left,
|
||||
tokenDelta: tokenRow.right_minus_left,
|
||||
commonPrefixTokens: trajectoryRow.common_prefix_tokens,
|
||||
exactTrajectory: trajectoryRow.token_ids_exact,
|
||||
},
|
||||
];
|
||||
}),
|
||||
);
|
||||
return {
|
||||
id: source.id,
|
||||
index: source.within_domain_index,
|
||||
conditions,
|
||||
contrasts: taskContrasts,
|
||||
diagnosticTapes: source.runs.map((run) => run.tape_label),
|
||||
};
|
||||
}),
|
||||
];
|
||||
}),
|
||||
);
|
||||
|
||||
const exampleSource = sampling.sources.find(
|
||||
(source) => source.id === "HumanEval/31",
|
||||
);
|
||||
const exampleRun = exampleSource.runs.find((run) => run.tape_label === "T0");
|
||||
const compact = {
|
||||
schemaVersion: 1,
|
||||
protocolId: sampling.protocol_id,
|
||||
capturedAt: sampling.captured_at,
|
||||
model: {
|
||||
repo: sampling.model.repo,
|
||||
revision: sampling.model.revision,
|
||||
checkpointIdentity: sampling.model.checkpoint_identity,
|
||||
dtype: sampling.model.dtype,
|
||||
},
|
||||
artifactHashes: Object.fromEntries(
|
||||
Object.entries(inputBytes).map(([name, bytes]) => [name, sha256(bytes)]),
|
||||
),
|
||||
grid: {
|
||||
formalSources: sampling.summary.sources,
|
||||
formalRuns: sampling.summary.runs,
|
||||
formalOutputs: sampling.summary.outputs,
|
||||
mainT0Outputs: evaluation.summary.main_t0.outputs,
|
||||
diagnosticAdditionalOutputs:
|
||||
evaluation.summary.diagnostic_additional_t1_t3.outputs,
|
||||
naturalEos: sampling.summary.natural_eos,
|
||||
budgetTruncated: sampling.summary.budget_truncated,
|
||||
uniqueTrajectories: sampling.summary.unique_generated_token_hashes,
|
||||
promptHashesExact: sampling.source_contract.prompt_hash_audit.exact,
|
||||
torchRngUnchangedRuns: sampling.summary.torch_rng_unchanged_runs,
|
||||
},
|
||||
sampler: {
|
||||
name: sampling.generation_contract.decode,
|
||||
temperature: sampling.generation_contract.distribution_temperature,
|
||||
topP: sampling.generation_contract.distribution_top_p,
|
||||
softmaxDtype: sampling.generation_contract.softmax_dtype,
|
||||
cdfDtype: sampling.generation_contract.cdf_dtype,
|
||||
uniformDtype: sampling.generation_contract.uniform_dtype,
|
||||
transformersGenerateCalled:
|
||||
sampling.generation_contract.transformers_generate_called,
|
||||
torchMultinomialCalled:
|
||||
sampling.generation_contract.torch_multinomial_called,
|
||||
commonRandomNumbers:
|
||||
sampling.seed_contract.explicit_common_random_numbers,
|
||||
},
|
||||
conditionTable,
|
||||
contrasts,
|
||||
diagnostic,
|
||||
tasks,
|
||||
outcomes: {
|
||||
code: evaluation.summary.main_t0.by_domain.code.task_outcomes,
|
||||
math: evaluation.summary.main_t0.by_domain.math.task_outcomes,
|
||||
},
|
||||
reproduction: reproduction.summary,
|
||||
bootstrap: analysis.bootstrap_contract,
|
||||
uniformExample: {
|
||||
sourceId: exampleSource.id,
|
||||
tape: exampleRun.tape_label,
|
||||
uniformUint64FirstEightHex: exampleRun.uniform_uint64_first_eight_hex,
|
||||
uniformFloat32FirstEight: exampleRun.uniform_float32_first_eight,
|
||||
conditions: Object.fromEntries(
|
||||
exampleRun.outputs.map((output) => [
|
||||
output.condition,
|
||||
{
|
||||
generatedTokenIds: output.generated_token_ids.slice(0, 8),
|
||||
generatedTokens: output.generated_tokens,
|
||||
naturalEos: output.hit_eos,
|
||||
},
|
||||
]),
|
||||
),
|
||||
},
|
||||
deviations: [
|
||||
{
|
||||
id: "prompt-hash-correction",
|
||||
severity: "corrected-before-output",
|
||||
summary:
|
||||
"The first manifest mislabeled routing-probe hashes as Chat prompt hashes; all 256 Chat hashes were corrected before model output.",
|
||||
},
|
||||
{
|
||||
id: "gold-loaded-in-runner",
|
||||
severity: "reported-process-deviation",
|
||||
summary:
|
||||
"The reused runner loaded gold for a post-decode narrow task_score before generation finished. Gold never entered prompts, logits, sampling, selection, or the authoritative evaluator.",
|
||||
},
|
||||
],
|
||||
evidenceBoundary: [
|
||||
"HumanEval and GSM8K remain separate.",
|
||||
"Selected-task bands cover only the frozen 32-task frame under T0.",
|
||||
"T1-T3 are sensitivity diagnostics, not extra independent tasks.",
|
||||
"The explicit sampler uses the official .3/.95 distribution but is not a torch.multinomial trajectory.",
|
||||
"Period prompts are counterfactual and not official-valid chats.",
|
||||
],
|
||||
};
|
||||
|
||||
await writeFile(paths.output, `${JSON.stringify(compact, null, 2)}\n`);
|
||||
const outputBytes = await readFile(paths.output);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
output: paths.output,
|
||||
bytes: outputBytes.length,
|
||||
sha256: sha256(outputBytes),
|
||||
tasks: Object.values(tasks).reduce((sum, rows) => sum + rows.length, 0),
|
||||
formalOutputs: compact.grid.formalOutputs,
|
||||
reproduction: compact.reproduction,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import process from "node:process";
|
||||
|
||||
const PROTOCOL_ID = "llm-atlas-deepseek-chat-task-bootstrap-crn-v1";
|
||||
const CONDITIONS = ["s0_eos", "s1_eos", "s0_period", "s1_period"];
|
||||
const EXPECTED_HASHES = {
|
||||
sampling: "ea0607f2b197fac3f794655c1538ce1f9a1cb072d637eb31d35573682e311809",
|
||||
evaluation: "82b2fc5d1f854a7e0cd7aba7c70ff74d733d24221522a4f92b962478c76177ab",
|
||||
replay: "6519947e2fa4327c1ba2cc506b0861f172a6bdbd4f2d6787447edaf8fbcac508",
|
||||
reproduction: "63ed39e5dcdbc2a30e516172f3657dfa453ff3b23d5bd5c49c734939242a70f5",
|
||||
analysis: "9ab17561ced930a668c082141f7c6e013cbda70e42b09de63d41f1b82c01a6ae",
|
||||
manifest: "6313e70536c464fe598a93035576752418f08016dfd60ac246437c3b43bf2ae1",
|
||||
};
|
||||
const DEFAULT_PATHS = {
|
||||
sampling: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn.json",
|
||||
evaluation: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-eval.json",
|
||||
replay: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-replay.json",
|
||||
reproduction: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-reproduction.json",
|
||||
analysis: "src/data/deepseek-v2-lite-chat-task-bootstrap-crn-analysis.json",
|
||||
manifest: "research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json",
|
||||
};
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function canonicalIntegerArray(values) {
|
||||
return `[${values.map((value) => value.toString()).join(",")}]`;
|
||||
}
|
||||
|
||||
function tapeUint64(tape, sourceId, step) {
|
||||
const payload = `${PROTOCOL_ID}\0uniform\0${tape}\0${sourceId}\0${step}`;
|
||||
return createHash("sha256").update(payload).digest().readBigUInt64BE(0);
|
||||
}
|
||||
|
||||
function assert(value, message) {
|
||||
if (!value) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
const loaded = {};
|
||||
for (const [name, path] of Object.entries(DEFAULT_PATHS)) {
|
||||
const bytes = await readFile(path);
|
||||
assert(
|
||||
sha256(bytes) === EXPECTED_HASHES[name],
|
||||
`${name}: file SHA-256 differs`,
|
||||
);
|
||||
loaded[name] = JSON.parse(bytes);
|
||||
}
|
||||
const { sampling, evaluation, replay, reproduction, analysis, manifest } = loaded;
|
||||
for (const [name, payload] of Object.entries(loaded)) {
|
||||
assert(payload.protocol_id === PROTOCOL_ID, `${name}: protocol ID differs`);
|
||||
}
|
||||
|
||||
assert(sampling.execution_mode === "formal", "sampling: not formal mode");
|
||||
assert(sampling.sources.length === 64, "sampling: source count differs");
|
||||
assert(sampling.summary.runs === 88, "sampling: run count differs");
|
||||
assert(sampling.summary.outputs === 352, "sampling: output count differs");
|
||||
assert(sampling.summary.natural_eos === 343, "sampling: EOS count differs");
|
||||
assert(sampling.summary.budget_truncated === 9, "sampling: truncation count differs");
|
||||
assert(
|
||||
sampling.summary.torch_rng_unchanged_runs === 88,
|
||||
"sampling: RNG nonconsumption count differs",
|
||||
);
|
||||
assert(
|
||||
sampling.source_contract.prompt_hash_audit.exact === 256,
|
||||
"sampling: prompt audit differs",
|
||||
);
|
||||
|
||||
const manifestBySource = new Map(
|
||||
manifest.sources.map((source) => [source.id, source]),
|
||||
);
|
||||
const samplingKeys = new Set();
|
||||
let uniformOutputHashesExact = 0;
|
||||
let diagnosticSources = 0;
|
||||
for (const source of sampling.sources) {
|
||||
const frozen = manifestBySource.get(source.id);
|
||||
assert(frozen, `${source.id}: absent from manifest`);
|
||||
assert(source.domain === frozen.domain, `${source.id}: domain differs`);
|
||||
assert(
|
||||
source.within_domain_index === frozen.domain_index,
|
||||
`${source.id}: domain index differs`,
|
||||
);
|
||||
const expectedTapes = [...frozen.main_tapes, ...frozen.diagnostic_tapes];
|
||||
const observedTapes = source.runs.map((run) => run.tape_label);
|
||||
assert(
|
||||
JSON.stringify(observedTapes) === JSON.stringify(expectedTapes),
|
||||
`${source.id}: tape assignment differs`,
|
||||
);
|
||||
diagnosticSources += source.runs.length === 4;
|
||||
for (const run of source.runs) {
|
||||
assert(run.torch_rng_unchanged, `${source.id}/${run.tape_label}: RNG changed`);
|
||||
assert(
|
||||
JSON.stringify(run.outputs.map((output) => output.condition))
|
||||
=== JSON.stringify(CONDITIONS),
|
||||
`${source.id}/${run.tape_label}: condition order differs`,
|
||||
);
|
||||
for (const output of run.outputs) {
|
||||
const key = `${source.id}\0${run.tape_label}\0${output.condition}`;
|
||||
assert(!samplingKeys.has(key), `${key}: duplicate sampling key`);
|
||||
samplingKeys.add(key);
|
||||
assert(
|
||||
output.prompt_token_ids_sha256
|
||||
=== frozen.chat_generation_prompt_token_ids_sha256[output.condition],
|
||||
`${key}: prompt hash differs`,
|
||||
);
|
||||
const uniforms = Array.from(
|
||||
{ length: output.uniform_steps_consumed },
|
||||
(_, step) => tapeUint64(run.tape_label, source.id, step),
|
||||
);
|
||||
assert(
|
||||
sha256(canonicalIntegerArray(uniforms))
|
||||
=== output.uniform_uint64_prefix_sha256,
|
||||
`${key}: uniform prefix hash differs`,
|
||||
);
|
||||
uniformOutputHashesExact += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(diagnosticSources === 8, "sampling: diagnostic source count differs");
|
||||
assert(samplingKeys.size === 352, "sampling: unique key count differs");
|
||||
assert(uniformOutputHashesExact === 352, "sampling: uniform audit differs");
|
||||
|
||||
assert(evaluation.rows.length === 352, "evaluation: row count differs");
|
||||
const evaluationKeys = new Set(
|
||||
evaluation.rows.map(
|
||||
(row) => `${row.source_id}\0${row.tape_label}\0${row.condition}`,
|
||||
),
|
||||
);
|
||||
assert(evaluationKeys.size === 352, "evaluation: duplicate keys");
|
||||
assert(
|
||||
[...evaluationKeys].every((key) => samplingKeys.has(key)),
|
||||
"evaluation: cell key absent from sampling",
|
||||
);
|
||||
assert(
|
||||
evaluation.summary.main_t0.outputs === 256,
|
||||
"evaluation: T0 output count differs",
|
||||
);
|
||||
assert(
|
||||
evaluation.summary.main_t0.by_domain.code.fixed_budget_success === 59,
|
||||
"evaluation: code success count differs",
|
||||
);
|
||||
assert(
|
||||
evaluation.summary.main_t0.by_domain.math.fixed_budget_success === 71,
|
||||
"evaluation: math success count differs",
|
||||
);
|
||||
|
||||
assert(replay.execution_mode === "replay", "replay: mode differs");
|
||||
assert(replay.sources.length === 16, "replay: source count differs");
|
||||
assert(replay.summary.outputs === 64, "replay: output count differs");
|
||||
assert(
|
||||
replay.summary.torch_rng_unchanged_runs === 16,
|
||||
"replay: RNG nonconsumption count differs",
|
||||
);
|
||||
assert(
|
||||
reproduction.summary.cells === 64
|
||||
&& reproduction.summary.all_preregistered_fields_exact === 64,
|
||||
"reproduction: exact cell count differs",
|
||||
);
|
||||
assert(
|
||||
Object.values(reproduction.summary.by_field).every((count) => count === 64),
|
||||
"reproduction: a field is not 64/64 exact",
|
||||
);
|
||||
|
||||
assert(
|
||||
analysis.bootstrap_contract.resamples === 10000
|
||||
&& analysis.bootstrap_contract.seed === 1364512825,
|
||||
"analysis: bootstrap contract differs",
|
||||
);
|
||||
let crnContrastChecks = 0;
|
||||
for (const domain of ["code", "math"]) {
|
||||
const domainAnalysis = analysis.main_t0_selected_task_analysis[domain];
|
||||
assert(domainAnalysis.tasks === 32, `${domain}: task count differs`);
|
||||
for (const contrast of Object.values(domainAnalysis.contrasts)) {
|
||||
assert(
|
||||
contrast.trajectory.shared_uniform_prefix_exact === 32,
|
||||
`${domain}: contrast CRN audit differs`,
|
||||
);
|
||||
crnContrastChecks += contrast.trajectory.shared_uniform_prefix_exact;
|
||||
}
|
||||
}
|
||||
assert(crnContrastChecks === 256, "analysis: CRN contrast total differs");
|
||||
|
||||
process.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
passed: true,
|
||||
protocol_id: PROTOCOL_ID,
|
||||
files: EXPECTED_HASHES,
|
||||
formal: {
|
||||
sources: sampling.sources.length,
|
||||
runs: sampling.summary.runs,
|
||||
outputs: sampling.summary.outputs,
|
||||
prompt_hashes_exact: sampling.source_contract.prompt_hash_audit.exact,
|
||||
uniform_output_hashes_exact: uniformOutputHashesExact,
|
||||
torch_rng_unchanged_runs: sampling.summary.torch_rng_unchanged_runs,
|
||||
},
|
||||
evaluation: {
|
||||
rows: evaluation.rows.length,
|
||||
main_t0_outputs: evaluation.summary.main_t0.outputs,
|
||||
code_success: evaluation.summary.main_t0.by_domain.code.fixed_budget_success,
|
||||
math_success: evaluation.summary.main_t0.by_domain.math.fixed_budget_success,
|
||||
},
|
||||
reproduction: reproduction.summary,
|
||||
analysis: {
|
||||
bootstrap_resamples: analysis.bootstrap_contract.resamples,
|
||||
crn_contrast_checks: crnContrastChecks,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
Reference in New Issue
Block a user