feat: map DeepSeek Chat sampling robustness
This commit is contained in:
wuyang
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare a fresh sampling rerun with matching formal-grid cells."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--formal-json", type=Path, required=True)
|
||||
parser.add_argument("--rerun-json", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(16 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def canonical_hash(value: Any) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def run_index(
|
||||
payload: dict[str, Any],
|
||||
) -> dict[tuple[str, int, str], tuple[dict[str, Any], dict[str, Any]]]:
|
||||
return {
|
||||
(
|
||||
source["id"],
|
||||
run["base_seed"],
|
||||
output["condition"],
|
||||
): (run, output)
|
||||
for source in payload["sources"]
|
||||
for run in source["runs"]
|
||||
for output in run["outputs"]
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
for path in (args.formal_json, args.rerun_json):
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
formal = json.loads(
|
||||
args.formal_json.read_text(encoding="utf-8")
|
||||
)
|
||||
rerun = json.loads(
|
||||
args.rerun_json.read_text(encoding="utf-8")
|
||||
)
|
||||
if formal["protocol_id"] != rerun["protocol_id"]:
|
||||
raise RuntimeError("protocol IDs differ")
|
||||
if formal["model"]["revision"] != rerun["model"]["revision"]:
|
||||
raise RuntimeError("model revisions differ")
|
||||
if (
|
||||
formal["generation_contract"]
|
||||
!= rerun["generation_contract"]
|
||||
):
|
||||
raise RuntimeError("generation contracts differ")
|
||||
formal_rows = run_index(formal)
|
||||
rerun_rows = run_index(rerun)
|
||||
missing = sorted(set(rerun_rows) - set(formal_rows))
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"{len(missing)} rerun cells are absent from formal grid"
|
||||
)
|
||||
|
||||
rows = []
|
||||
for key in sorted(rerun_rows):
|
||||
formal_run, formal_output = formal_rows[key]
|
||||
rerun_run, rerun_output = rerun_rows[key]
|
||||
checks = {
|
||||
"run_seed_exact": (
|
||||
formal_run["run_seed"] == rerun_run["run_seed"]
|
||||
),
|
||||
"prompt_hash_exact": (
|
||||
formal_output["prompt_token_ids_sha256"]
|
||||
== rerun_output["prompt_token_ids_sha256"]
|
||||
),
|
||||
"generated_token_ids_exact": (
|
||||
formal_output["generated_token_ids"]
|
||||
== rerun_output["generated_token_ids"]
|
||||
),
|
||||
"decoded_text_exact": (
|
||||
formal_output["text"] == rerun_output["text"]
|
||||
),
|
||||
"eos_state_exact": (
|
||||
formal_output["hit_eos"]
|
||||
== rerun_output["hit_eos"]
|
||||
),
|
||||
"truncation_state_exact": (
|
||||
formal_output["stopped_at_max_new_tokens"]
|
||||
== rerun_output["stopped_at_max_new_tokens"]
|
||||
),
|
||||
"cpu_rng_pre_state_exact": (
|
||||
formal_run["rng_state_before"]["cpu_sha256"]
|
||||
== rerun_run["rng_state_before"]["cpu_sha256"]
|
||||
),
|
||||
"cuda_rng_pre_state_exact": (
|
||||
formal_run["rng_state_before"][
|
||||
"cuda_combined_sha256"
|
||||
]
|
||||
== rerun_run["rng_state_before"][
|
||||
"cuda_combined_sha256"
|
||||
]
|
||||
),
|
||||
}
|
||||
rows.append(
|
||||
{
|
||||
"source_id": key[0],
|
||||
"base_seed": key[1],
|
||||
"condition": key[2],
|
||||
**checks,
|
||||
"all_preregistered_fields_exact": all(
|
||||
checks.values()
|
||||
),
|
||||
}
|
||||
)
|
||||
check_names = [
|
||||
key
|
||||
for key in rows[0]
|
||||
if key.endswith("_exact")
|
||||
and key != "all_preregistered_fields_exact"
|
||||
]
|
||||
result = {
|
||||
"schema_version": 1,
|
||||
"protocol_id": formal["protocol_id"],
|
||||
"formal": {
|
||||
"path": str(args.formal_json),
|
||||
"sha256": sha256_file(args.formal_json),
|
||||
"content_hash": formal["content_hash"],
|
||||
"base_seeds": formal["seed_contract"][
|
||||
"executed_base_seeds"
|
||||
],
|
||||
},
|
||||
"rerun": {
|
||||
"path": str(args.rerun_json),
|
||||
"sha256": sha256_file(args.rerun_json),
|
||||
"content_hash": rerun["content_hash"],
|
||||
"base_seeds": rerun["seed_contract"][
|
||||
"executed_base_seeds"
|
||||
],
|
||||
},
|
||||
"rows": rows,
|
||||
"summary": {
|
||||
"cells": len(rows),
|
||||
"all_preregistered_fields_exact": sum(
|
||||
row["all_preregistered_fields_exact"]
|
||||
for row in rows
|
||||
),
|
||||
"by_field": {
|
||||
name: sum(row[name] for row in rows)
|
||||
for name in check_names
|
||||
},
|
||||
},
|
||||
"claim_boundary": [
|
||||
"Only the rerun seed subset is independently reproduced.",
|
||||
"Exact replay is scoped to the pinned software and hardware contract.",
|
||||
"Reproduction does not imply trajectories are seed-invariant.",
|
||||
],
|
||||
}
|
||||
result["content_hash"] = canonical_hash(rows)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
payload = args.output.read_bytes()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
"bytes": len(payload),
|
||||
"summary": result["summary"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user