feat: map DeepSeek Chat sampling robustness
This commit is contained in:
wuyang
@@ -672,3 +672,85 @@ See `research/DEEPSEEK_V2_LITE_CHAT_COMPLETION_PROTOCOL.md` and
|
||||
`research/DEEPSEEK_V2_LITE_CHAT_COMPLETION_DEPTH_AUDIT.md` for the OOM
|
||||
amendment, task evaluators, completion table, depth curves, reproduction
|
||||
audit, artifact hashes, and claim boundaries.
|
||||
|
||||
## Preregistered multi-seed Chat sampling
|
||||
|
||||
`v2_lite_chat_sampling_probe.py` keeps the same four first-ranked sources,
|
||||
eight prompt conditions, 512-token cap, BF16 checkpoint, and offload contract.
|
||||
It enables the pinned official sampling values and explicitly disables top-k:
|
||||
|
||||
```text
|
||||
do_sample=true · temperature=.3 · top_p=.95 · top_k=0
|
||||
4 sources × 8 SHA-256-derived seeds × 8 conditions = 256 outputs
|
||||
```
|
||||
|
||||
The eight rows of one source remain in a fixed batch. They are batch-seed
|
||||
aligned, not common-random-number pairs: Transformers samples the whole batch
|
||||
with one `torch.multinomial` call.
|
||||
|
||||
```bash
|
||||
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
||||
PYTHONPATH=/path/to/transformers-4.41.2-deps \
|
||||
python -B experiments/deepseek/v2_lite_chat_sampling_probe.py \
|
||||
--artifact-dir /path/to/deepseek-v2-lite-chat \
|
||||
--reference-routing-json \
|
||||
src/data/deepseek-v2-lite-routing-special-token-family-control.json \
|
||||
--greedy-json src/data/deepseek-v2-lite-chat-completion-512.json \
|
||||
--human-eval /path/to/HumanEval.jsonl.gz \
|
||||
--gsm8k /path/to/gsm8k/test.jsonl \
|
||||
--tnews /path/to/tnews/test.json \
|
||||
--tnews-archive /path/to/tnews_public.zip \
|
||||
--wikitext /path/to/wikitext-validation.parquet \
|
||||
--base-seeds \
|
||||
19683830 1560062173 3978401375 1280933274 \
|
||||
1467459869 1297359489 2722953988 3330978061 \
|
||||
--max-new-tokens 512 \
|
||||
--gpu-memory 28GiB \
|
||||
--cpu-memory 80GiB \
|
||||
--output src/data/deepseek-v2-lite-chat-sampling.json
|
||||
```
|
||||
|
||||
The independent evaluator preserves stopping, semantic terminal state,
|
||||
coverage, and correctness as separate ledgers. Exact duplicate code candidates
|
||||
reuse a cache keyed by candidate, task tests, and sandbox-harness hashes:
|
||||
|
||||
```bash
|
||||
python -B experiments/deepseek/v2_lite_chat_sampling_evaluator.py \
|
||||
--sampling-json src/data/deepseek-v2-lite-chat-sampling.json \
|
||||
--human-eval /path/to/HumanEval.jsonl.gz \
|
||||
--gsm8k /path/to/gsm8k/test.jsonl \
|
||||
--sandbox-image \
|
||||
python:3.11-alpine@sha256:25976e9d34a0fab1f278cae931f34c8303d97bf0c0d7f85b6b4dcf641d7702a4 \
|
||||
--output src/data/deepseek-v2-lite-chat-sampling-eval.json
|
||||
```
|
||||
|
||||
A fresh Python process reruns R0/R1. The reproduction comparer checks run
|
||||
seeds, prompt hashes, complete token IDs, text, stopping state, and CPU/CUDA
|
||||
RNG pre-state hashes:
|
||||
|
||||
```bash
|
||||
python -B experiments/deepseek/v2_lite_chat_sampling_reproduction.py \
|
||||
--formal-json src/data/deepseek-v2-lite-chat-sampling.json \
|
||||
--rerun-json \
|
||||
src/data/deepseek-v2-lite-chat-sampling-repro-r0r1.json \
|
||||
--output \
|
||||
src/data/deepseek-v2-lite-chat-sampling-reproduction.json
|
||||
```
|
||||
|
||||
Formal results are 251/256 natural EOS and 242/256 globally unique complete
|
||||
token trajectories. Strict GSM8K exact is 62/64; HumanEval official-test pass
|
||||
is 63/64. These repeat one task per domain and are not benchmark estimates.
|
||||
The R0/R1 fresh-process rerun reproduces all eight preregistered fields in
|
||||
64/64 cells.
|
||||
|
||||
```text
|
||||
formal 46c7edfce1409e798d6b0f06e905dd3a9d4ab6911acd8e42e2ec9d06d65345af
|
||||
eval 078f486e021ffd0e4af8ad942d001a9f2429933b2fb0fb3523b7d1da5b24c4d8
|
||||
rerun 72d050eace531bb38fb2592e39dc7f1b9c9003306c92c568dcd4acaaa691f6a0
|
||||
compare 4d59a775943459604ea2d9976bb3be01ff7b3b08d71eb0ca22c376122ef82d15
|
||||
```
|
||||
|
||||
See `research/DEEPSEEK_V2_LITE_CHAT_SAMPLING_PROTOCOL.md` and
|
||||
`research/DEEPSEEK_V2_LITE_CHAT_SAMPLING_AUDIT.md` for seed derivation,
|
||||
smoke gates, source-condition diversity tables, edge-set comparisons, task
|
||||
failure cases, exact reproduction scope, primary sources, and non-claims.
|
||||
|
||||
@@ -0,0 +1,560 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate sampled DeepSeek-V2-Lite-Chat completions safely.
|
||||
|
||||
Stopping, task-terminal state, evaluator coverage, and correctness remain
|
||||
separate ledgers. HumanEval candidates execute only in fresh networkless,
|
||||
read-only Docker containers. An exact candidate+task+harness cache avoids
|
||||
re-executing duplicate code while preserving one result row per sampled cell.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from typing import Any
|
||||
|
||||
import v2_lite_chat_completion_evaluator as completion
|
||||
|
||||
|
||||
COMPARISONS = (
|
||||
("system_eos", "s0_eos", "s1_eos"),
|
||||
("system_bos", "s0_bos", "s1_bos"),
|
||||
("system_x", "s0_x", "s1_x"),
|
||||
("system_period", "s0_period", "s1_period"),
|
||||
("bos_at_s0", "s0_eos", "s0_bos"),
|
||||
("bos_at_s1", "s1_eos", "s1_bos"),
|
||||
("x_at_s0", "s0_eos", "s0_x"),
|
||||
("x_at_s1", "s1_eos", "s1_x"),
|
||||
("period_at_s0", "s0_eos", "s0_period"),
|
||||
("period_at_s1", "s1_eos", "s1_period"),
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--sampling-json", type=Path, required=True)
|
||||
parser.add_argument("--human-eval", type=Path, required=True)
|
||||
parser.add_argument("--gsm8k", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--sandbox-image",
|
||||
required=True,
|
||||
help="Pinned image reference including @sha256 digest.",
|
||||
)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=5.0)
|
||||
parser.add_argument("--skip-code-execution", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def evaluate_code_cached(
|
||||
*,
|
||||
text: str,
|
||||
hit_eos: bool,
|
||||
task: dict[str, Any],
|
||||
image: str,
|
||||
timeout_seconds: float,
|
||||
skip_execution: bool,
|
||||
cache: dict[tuple[str, str, str], dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
extracted = completion.extract_code(
|
||||
text,
|
||||
task["prompt"],
|
||||
task["entry_point"],
|
||||
)
|
||||
harness = completion.sandbox_harness(
|
||||
extracted["candidate"],
|
||||
task["test"],
|
||||
task["entry_point"],
|
||||
)
|
||||
cache_key = (
|
||||
extracted["candidate_sha256"],
|
||||
hashlib.sha256(task["test"].encode()).hexdigest(),
|
||||
hashlib.sha256(harness.encode()).hexdigest(),
|
||||
)
|
||||
cache_hit = cache_key in cache
|
||||
if cache_hit:
|
||||
execution = copy.deepcopy(cache[cache_key])
|
||||
else:
|
||||
execution = {
|
||||
"status": "not_run",
|
||||
"return_code": None,
|
||||
"runtime_ms": None,
|
||||
"harness_sha256": cache_key[2],
|
||||
}
|
||||
if extracted["python_ast_parse"] and not skip_execution:
|
||||
execution = completion.execute_code(
|
||||
extracted["candidate"],
|
||||
task,
|
||||
image,
|
||||
timeout_seconds,
|
||||
)
|
||||
cache[cache_key] = copy.deepcopy(execution)
|
||||
passed = execution["status"] == "passed"
|
||||
semantic_terminal = bool(
|
||||
hit_eos
|
||||
or extracted["closed_code_fence"]
|
||||
or passed
|
||||
)
|
||||
result = {
|
||||
key: value
|
||||
for key, value in extracted.items()
|
||||
if key != "candidate"
|
||||
} | {
|
||||
"semantic_terminal": semantic_terminal,
|
||||
"evaluator_covered": (
|
||||
extracted["python_ast_parse"]
|
||||
and execution["status"] != "not_run"
|
||||
),
|
||||
"execution": execution,
|
||||
"fixed_budget_tests_pass": passed,
|
||||
"strict_complete_tests_pass": (
|
||||
passed and semantic_terminal
|
||||
),
|
||||
"execution_cache_hit": cache_hit,
|
||||
}
|
||||
return result, cache_hit
|
||||
|
||||
|
||||
def output_success(row: dict[str, Any]) -> dict[str, int | None]:
|
||||
evaluation = row["task_evaluation"]
|
||||
if row["domain"] == "math":
|
||||
return {
|
||||
"fixed_budget": int(
|
||||
evaluation["fixed_budget_numeric_exact"]
|
||||
),
|
||||
"strict_complete": int(
|
||||
evaluation["strict_complete_numeric_exact"]
|
||||
),
|
||||
}
|
||||
if row["domain"] == "code":
|
||||
return {
|
||||
"fixed_budget": int(
|
||||
evaluation["fixed_budget_tests_pass"]
|
||||
),
|
||||
"strict_complete": int(
|
||||
evaluation["strict_complete_tests_pass"]
|
||||
),
|
||||
}
|
||||
return {
|
||||
"fixed_budget": None,
|
||||
"strict_complete": None,
|
||||
}
|
||||
|
||||
|
||||
def summarize_group(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
math_rows = [
|
||||
row for row in rows if row["domain"] == "math"
|
||||
]
|
||||
code_rows = [
|
||||
row for row in rows if row["domain"] == "code"
|
||||
]
|
||||
answers = Counter(
|
||||
row["task_evaluation"]["predicted_final"]
|
||||
for row in math_rows
|
||||
if row["task_evaluation"]["predicted_final"] is not None
|
||||
)
|
||||
max_count = max(answers.values(), default=0)
|
||||
modes = sorted(
|
||||
answer
|
||||
for answer, count in answers.items()
|
||||
if count == max_count
|
||||
)
|
||||
gold = (
|
||||
math_rows[0]["task_evaluation"]["gold_final"]
|
||||
if math_rows
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"outputs": len(rows),
|
||||
"natural_eos": sum(row["hit_eos"] for row in rows),
|
||||
"budget_truncated": sum(
|
||||
row["stopped_at_max_new_tokens"] for row in rows
|
||||
),
|
||||
"mean_generated_tokens": (
|
||||
mean(row["generated_tokens"] for row in rows)
|
||||
if rows
|
||||
else None
|
||||
),
|
||||
"completion_classes": dict(
|
||||
Counter(row["completion_class"] for row in rows)
|
||||
),
|
||||
"math": {
|
||||
"outputs": len(math_rows),
|
||||
"evaluator_covered": sum(
|
||||
row["task_evaluation"]["evaluator_covered"]
|
||||
for row in math_rows
|
||||
),
|
||||
"fixed_budget_exact": sum(
|
||||
row["task_evaluation"][
|
||||
"fixed_budget_numeric_exact"
|
||||
]
|
||||
for row in math_rows
|
||||
),
|
||||
"strict_complete_exact": sum(
|
||||
row["task_evaluation"][
|
||||
"strict_complete_numeric_exact"
|
||||
]
|
||||
for row in math_rows
|
||||
),
|
||||
"answer_frequencies": dict(answers),
|
||||
"modal_answers": modes,
|
||||
"modal_count": max_count,
|
||||
"absolute_majority_exists": (
|
||||
max_count > len(math_rows) / 2
|
||||
if math_rows
|
||||
else None
|
||||
),
|
||||
"unique_absolute_majority": (
|
||||
modes[0]
|
||||
if (
|
||||
math_rows
|
||||
and len(modes) == 1
|
||||
and max_count > len(math_rows) / 2
|
||||
)
|
||||
else None
|
||||
),
|
||||
"gold": gold,
|
||||
"unique_absolute_majority_matches_gold": (
|
||||
len(modes) == 1
|
||||
and max_count > len(math_rows) / 2
|
||||
and modes[0] == gold
|
||||
if math_rows
|
||||
else None
|
||||
),
|
||||
},
|
||||
"code": {
|
||||
"outputs": len(code_rows),
|
||||
"ast_parse": sum(
|
||||
row["task_evaluation"]["python_ast_parse"]
|
||||
for row in code_rows
|
||||
),
|
||||
"executed": sum(
|
||||
row["task_evaluation"]["execution"]["status"]
|
||||
!= "not_run"
|
||||
for row in code_rows
|
||||
),
|
||||
"execution_cache_hits": sum(
|
||||
row["task_evaluation"]["execution_cache_hit"]
|
||||
for row in code_rows
|
||||
),
|
||||
"tests_pass": sum(
|
||||
row["task_evaluation"][
|
||||
"fixed_budget_tests_pass"
|
||||
]
|
||||
for row in code_rows
|
||||
),
|
||||
"strict_complete_tests_pass": sum(
|
||||
row["task_evaluation"][
|
||||
"strict_complete_tests_pass"
|
||||
]
|
||||
for row in code_rows
|
||||
),
|
||||
"execution_statuses": dict(
|
||||
Counter(
|
||||
row["task_evaluation"]["execution"]["status"]
|
||||
for row in code_rows
|
||||
)
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def source_condition_summary(
|
||||
rows: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
source_ids = sorted({row["source_id"] for row in rows})
|
||||
conditions = tuple(
|
||||
condition
|
||||
for _, left, right in COMPARISONS
|
||||
for condition in (left, right)
|
||||
)
|
||||
ordered_conditions = tuple(dict.fromkeys(conditions))
|
||||
for source_id in source_ids:
|
||||
result[source_id] = {}
|
||||
for condition in ordered_conditions:
|
||||
subset = [
|
||||
row
|
||||
for row in rows
|
||||
if row["source_id"] == source_id
|
||||
and row["condition"] == condition
|
||||
]
|
||||
result[source_id][condition] = summarize_group(subset)
|
||||
return result
|
||||
|
||||
|
||||
def edge_summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
source_ids = sorted({row["source_id"] for row in rows})
|
||||
for source_id in source_ids:
|
||||
result[source_id] = {}
|
||||
source_rows = [
|
||||
row for row in rows if row["source_id"] == source_id
|
||||
]
|
||||
for name, left, right in COMPARISONS:
|
||||
left_rows = [
|
||||
row
|
||||
for row in source_rows
|
||||
if row["condition"] == left
|
||||
]
|
||||
right_rows = [
|
||||
row
|
||||
for row in source_rows
|
||||
if row["condition"] == right
|
||||
]
|
||||
left_success = [
|
||||
output_success(row) for row in left_rows
|
||||
]
|
||||
right_success = [
|
||||
output_success(row) for row in right_rows
|
||||
]
|
||||
result[source_id][name] = {
|
||||
"left": left,
|
||||
"right": right,
|
||||
"samples_per_side": len(left_rows),
|
||||
"completion_classes": {
|
||||
"left": dict(
|
||||
Counter(
|
||||
row["completion_class"]
|
||||
for row in left_rows
|
||||
)
|
||||
),
|
||||
"right": dict(
|
||||
Counter(
|
||||
row["completion_class"]
|
||||
for row in right_rows
|
||||
)
|
||||
),
|
||||
},
|
||||
"natural_eos_difference_right_minus_left": (
|
||||
sum(row["hit_eos"] for row in right_rows)
|
||||
- sum(row["hit_eos"] for row in left_rows)
|
||||
),
|
||||
"fixed_budget_success_difference_right_minus_left": (
|
||||
sum(
|
||||
row["fixed_budget"]
|
||||
for row in right_success
|
||||
if row["fixed_budget"] is not None
|
||||
)
|
||||
- sum(
|
||||
row["fixed_budget"]
|
||||
for row in left_success
|
||||
if row["fixed_budget"] is not None
|
||||
)
|
||||
if any(
|
||||
row["fixed_budget"] is not None
|
||||
for row in [*left_success, *right_success]
|
||||
)
|
||||
else None
|
||||
),
|
||||
"strict_success_difference_right_minus_left": (
|
||||
sum(
|
||||
row["strict_complete"]
|
||||
for row in right_success
|
||||
if row["strict_complete"] is not None
|
||||
)
|
||||
- sum(
|
||||
row["strict_complete"]
|
||||
for row in left_success
|
||||
if row["strict_complete"] is not None
|
||||
)
|
||||
if any(
|
||||
row["strict_complete"] is not None
|
||||
for row in [*left_success, *right_success]
|
||||
)
|
||||
else None
|
||||
),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
for path in (
|
||||
args.sampling_json,
|
||||
args.human_eval,
|
||||
args.gsm8k,
|
||||
):
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
if "@sha256:" not in args.sandbox_image:
|
||||
raise ValueError("--sandbox-image must include an immutable digest")
|
||||
if args.timeout_seconds <= 0:
|
||||
raise ValueError("--timeout-seconds must be positive")
|
||||
|
||||
sampling = json.loads(
|
||||
args.sampling_json.read_text(encoding="utf-8")
|
||||
)
|
||||
human_eval, gsm8k = completion.load_tasks(
|
||||
args.human_eval,
|
||||
args.gsm8k,
|
||||
)
|
||||
code_cache: dict[
|
||||
tuple[str, str, str],
|
||||
dict[str, Any],
|
||||
] = {}
|
||||
rows = []
|
||||
for source in sampling["sources"]:
|
||||
for run in source["runs"]:
|
||||
for output in run["outputs"]:
|
||||
evaluation = None
|
||||
cache_hit = False
|
||||
if source["domain"] == "math":
|
||||
evaluation = completion.evaluate_math(
|
||||
output["text"],
|
||||
output["hit_eos"],
|
||||
gsm8k[source["id"]]["answer"],
|
||||
)
|
||||
elif source["domain"] == "code":
|
||||
evaluation, cache_hit = evaluate_code_cached(
|
||||
text=output["text"],
|
||||
hit_eos=output["hit_eos"],
|
||||
task=human_eval[source["id"]],
|
||||
image=args.sandbox_image,
|
||||
timeout_seconds=args.timeout_seconds,
|
||||
skip_execution=args.skip_code_execution,
|
||||
cache=code_cache,
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"source_id": source["id"],
|
||||
"domain": source["domain"],
|
||||
"replicate_index": run["replicate_index"],
|
||||
"replicate_label": run["replicate_label"],
|
||||
"base_seed": run["base_seed"],
|
||||
"run_seed": run["run_seed"],
|
||||
"condition": output["condition"],
|
||||
"generated_tokens": output[
|
||||
"generated_tokens"
|
||||
],
|
||||
"hit_eos": output["hit_eos"],
|
||||
"stopped_at_max_new_tokens": output[
|
||||
"stopped_at_max_new_tokens"
|
||||
],
|
||||
"prompt_token_ids_sha256": output[
|
||||
"prompt_token_ids_sha256"
|
||||
],
|
||||
"generated_token_ids_sha256": output[
|
||||
"generated_token_ids_sha256"
|
||||
],
|
||||
"text_sha256": output["text_sha256"],
|
||||
"task_evaluation": evaluation,
|
||||
"completion_class": (
|
||||
completion.completion_class(
|
||||
output,
|
||||
evaluation,
|
||||
)
|
||||
),
|
||||
"code_execution_cache_hit": cache_hit,
|
||||
}
|
||||
)
|
||||
|
||||
by_condition = {
|
||||
condition: summarize_group(
|
||||
[
|
||||
row
|
||||
for row in rows
|
||||
if row["condition"] == condition
|
||||
]
|
||||
)
|
||||
for condition in sampling["seed_contract"][
|
||||
"condition_row_order"
|
||||
]
|
||||
}
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"protocol_id": sampling["protocol_id"],
|
||||
"input": {
|
||||
"sampling_path": str(args.sampling_json),
|
||||
"sampling_sha256": completion.sha256_file(
|
||||
args.sampling_json
|
||||
),
|
||||
"sampling_content_hash": sampling["content_hash"],
|
||||
"human_eval_sha256": completion.sha256_file(
|
||||
args.human_eval
|
||||
),
|
||||
"gsm8k_sha256": completion.sha256_file(args.gsm8k),
|
||||
"model_revision": sampling["model"]["revision"],
|
||||
"base_seeds": sampling["seed_contract"][
|
||||
"executed_base_seeds"
|
||||
],
|
||||
"max_new_tokens": sampling["generation_contract"][
|
||||
"max_new_tokens"
|
||||
],
|
||||
},
|
||||
"sandbox": {
|
||||
"image": args.sandbox_image,
|
||||
"timeout_seconds": args.timeout_seconds,
|
||||
"code_execution_skipped": args.skip_code_execution,
|
||||
"network": "none",
|
||||
"filesystem": "read-only",
|
||||
"user": "65534:65534",
|
||||
"capabilities": "ALL dropped",
|
||||
"no_new_privileges": True,
|
||||
"memory": "256m",
|
||||
"memory_swap": "256m",
|
||||
"pids_limit": 64,
|
||||
"cpus": 0.5,
|
||||
"tmpfs": "/tmp:rw,noexec,nosuid,size=16m",
|
||||
"host_mounts": 0,
|
||||
"cache_key": (
|
||||
"candidate SHA-256 + task-test SHA-256 + "
|
||||
"sandbox-harness SHA-256"
|
||||
),
|
||||
"unique_code_cache_entries": len(code_cache),
|
||||
},
|
||||
"rows": rows,
|
||||
"summary": {
|
||||
**summarize_group(rows),
|
||||
"by_condition": by_condition,
|
||||
"by_source_condition": source_condition_summary(rows),
|
||||
"by_source_edge": edge_summary(rows),
|
||||
},
|
||||
"claim_boundary": [
|
||||
"Four sources and eight seeds are not benchmark estimates.",
|
||||
"Completion-conditioned metrics are selection-biased diagnostics.",
|
||||
"A passing HumanEval test is functional evidence, not code-safety evidence.",
|
||||
"A modal sampled math answer is not standard self-consistency.",
|
||||
"Counterfactual token sequences are not official-valid chats.",
|
||||
"Repeated samples from one source are not independent tasks.",
|
||||
],
|
||||
}
|
||||
result["content_hash"] = completion.canonical_hash(rows)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
payload = args.output.read_bytes()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"sha256": completion.sha256_bytes(payload),
|
||||
"bytes": len(payload),
|
||||
"outputs": len(rows),
|
||||
"summary": {
|
||||
key: result["summary"][key]
|
||||
for key in (
|
||||
"natural_eos",
|
||||
"budget_truncated",
|
||||
"completion_classes",
|
||||
"math",
|
||||
"code",
|
||||
)
|
||||
},
|
||||
"unique_code_cache_entries": len(code_cache),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare a fresh sampling rerun with matching formal-grid cells."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--formal-json", type=Path, required=True)
|
||||
parser.add_argument("--rerun-json", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(16 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def canonical_hash(value: Any) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def run_index(
|
||||
payload: dict[str, Any],
|
||||
) -> dict[tuple[str, int, str], tuple[dict[str, Any], dict[str, Any]]]:
|
||||
return {
|
||||
(
|
||||
source["id"],
|
||||
run["base_seed"],
|
||||
output["condition"],
|
||||
): (run, output)
|
||||
for source in payload["sources"]
|
||||
for run in source["runs"]
|
||||
for output in run["outputs"]
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
for path in (args.formal_json, args.rerun_json):
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
formal = json.loads(
|
||||
args.formal_json.read_text(encoding="utf-8")
|
||||
)
|
||||
rerun = json.loads(
|
||||
args.rerun_json.read_text(encoding="utf-8")
|
||||
)
|
||||
if formal["protocol_id"] != rerun["protocol_id"]:
|
||||
raise RuntimeError("protocol IDs differ")
|
||||
if formal["model"]["revision"] != rerun["model"]["revision"]:
|
||||
raise RuntimeError("model revisions differ")
|
||||
if (
|
||||
formal["generation_contract"]
|
||||
!= rerun["generation_contract"]
|
||||
):
|
||||
raise RuntimeError("generation contracts differ")
|
||||
formal_rows = run_index(formal)
|
||||
rerun_rows = run_index(rerun)
|
||||
missing = sorted(set(rerun_rows) - set(formal_rows))
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"{len(missing)} rerun cells are absent from formal grid"
|
||||
)
|
||||
|
||||
rows = []
|
||||
for key in sorted(rerun_rows):
|
||||
formal_run, formal_output = formal_rows[key]
|
||||
rerun_run, rerun_output = rerun_rows[key]
|
||||
checks = {
|
||||
"run_seed_exact": (
|
||||
formal_run["run_seed"] == rerun_run["run_seed"]
|
||||
),
|
||||
"prompt_hash_exact": (
|
||||
formal_output["prompt_token_ids_sha256"]
|
||||
== rerun_output["prompt_token_ids_sha256"]
|
||||
),
|
||||
"generated_token_ids_exact": (
|
||||
formal_output["generated_token_ids"]
|
||||
== rerun_output["generated_token_ids"]
|
||||
),
|
||||
"decoded_text_exact": (
|
||||
formal_output["text"] == rerun_output["text"]
|
||||
),
|
||||
"eos_state_exact": (
|
||||
formal_output["hit_eos"]
|
||||
== rerun_output["hit_eos"]
|
||||
),
|
||||
"truncation_state_exact": (
|
||||
formal_output["stopped_at_max_new_tokens"]
|
||||
== rerun_output["stopped_at_max_new_tokens"]
|
||||
),
|
||||
"cpu_rng_pre_state_exact": (
|
||||
formal_run["rng_state_before"]["cpu_sha256"]
|
||||
== rerun_run["rng_state_before"]["cpu_sha256"]
|
||||
),
|
||||
"cuda_rng_pre_state_exact": (
|
||||
formal_run["rng_state_before"][
|
||||
"cuda_combined_sha256"
|
||||
]
|
||||
== rerun_run["rng_state_before"][
|
||||
"cuda_combined_sha256"
|
||||
]
|
||||
),
|
||||
}
|
||||
rows.append(
|
||||
{
|
||||
"source_id": key[0],
|
||||
"base_seed": key[1],
|
||||
"condition": key[2],
|
||||
**checks,
|
||||
"all_preregistered_fields_exact": all(
|
||||
checks.values()
|
||||
),
|
||||
}
|
||||
)
|
||||
check_names = [
|
||||
key
|
||||
for key in rows[0]
|
||||
if key.endswith("_exact")
|
||||
and key != "all_preregistered_fields_exact"
|
||||
]
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"protocol_id": formal["protocol_id"],
|
||||
"formal": {
|
||||
"path": str(args.formal_json),
|
||||
"sha256": sha256_file(args.formal_json),
|
||||
"content_hash": formal["content_hash"],
|
||||
"base_seeds": formal["seed_contract"][
|
||||
"executed_base_seeds"
|
||||
],
|
||||
},
|
||||
"rerun": {
|
||||
"path": str(args.rerun_json),
|
||||
"sha256": sha256_file(args.rerun_json),
|
||||
"content_hash": rerun["content_hash"],
|
||||
"base_seeds": rerun["seed_contract"][
|
||||
"executed_base_seeds"
|
||||
],
|
||||
},
|
||||
"rows": rows,
|
||||
"summary": {
|
||||
"cells": len(rows),
|
||||
"all_preregistered_fields_exact": sum(
|
||||
row["all_preregistered_fields_exact"]
|
||||
for row in rows
|
||||
),
|
||||
"by_field": {
|
||||
name: sum(row[name] for row in rows)
|
||||
for name in check_names
|
||||
},
|
||||
},
|
||||
"claim_boundary": [
|
||||
"Only the rerun seed subset is independently reproduced.",
|
||||
"Exact replay is scoped to the pinned software and hardware contract.",
|
||||
"Reproduction does not imply trajectories are seed-invariant.",
|
||||
],
|
||||
}
|
||||
result["content_hash"] = canonical_hash(rows)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
payload = args.output.read_bytes()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
"bytes": len(payload),
|
||||
"summary": result["summary"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user