diff --git a/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_analysis.py b/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_analysis.py new file mode 100644 index 0000000..226f275 --- /dev/null +++ b/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_analysis.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Build task-paired bootstrap and multi-tape diagnostics for Round 08.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path +from statistics import mean, median +from typing import Any + +import numpy as np + + +PROTOCOL_ID = "llm-atlas-deepseek-chat-task-bootstrap-crn-v1" +CONDITIONS = ( + "s0_eos", + "s1_eos", + "s0_period", + "s1_period", +) +DOMAINS = ("code", "math") +BOOTSTRAP_SEED = 1364512825 +BOOTSTRAP_RESAMPLES = 10000 +CONTRASTS = { + "period_at_s0": ("s0_eos", "s0_period"), + "period_at_s1": ("s1_eos", "s1_period"), + "system_at_eos": ("s0_eos", "s1_eos"), + "system_at_period": ("s0_period", "s1_period"), +} +METRICS = ( + "fixed_budget_success", + "strict_complete_success", + "natural_eos", + "generated_tokens", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--sampling-json", type=Path, required=True) + parser.add_argument("--evaluation-json", type=Path, required=True) + parser.add_argument("--reproduction-json", type=Path) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_hash(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + + +def metric_value(row: dict[str, Any], metric: str) -> float: + evaluation = row["task_evaluation"] + if metric == "fixed_budget_success": + key = ( + "fixed_budget_tests_pass" + if row["domain"] == "code" + else "fixed_budget_numeric_exact" + ) + return float(evaluation[key]) + if metric == "strict_complete_success": + key = ( + "strict_complete_tests_pass" + if row["domain"] == "code" + else "strict_complete_numeric_exact" + ) + return float(evaluation[key]) + if metric == "natural_eos": + return float(row["hit_eos"]) + if metric == "generated_tokens": + return float(row["generated_tokens"]) + raise KeyError(metric) + + +def summarize(values: list[float]) -> dict[str, Any]: + return { + "count": len(values), + "mean": mean(values) if values else None, + "median": median(values) if values else None, + "min": min(values) if values else None, + "max": max(values) if values else None, + } + + +def percentile_band(values: np.ndarray) -> dict[str, float]: + quantiles = np.percentile(values, [2.5, 50.0, 97.5]) + return { + "p2_5": float(quantiles[0]), + "p50": float(quantiles[1]), + "p97_5": float(quantiles[2]), + } + + +def direction_counts(values: list[float]) -> dict[str, int]: + epsilon = 1e-12 + return { + "positive": sum(value > epsilon for value in values), + "zero": sum(abs(value) <= epsilon for value in values), + "negative": sum(value < -epsilon for value in values), + } + + +def success_transition( + left: list[float], + right: list[float], +) -> dict[str, int]: + pairs = [(int(a), int(b)) for a, b in zip(left, right, strict=True)] + return { + "fail_to_fail": sum(a == 0 and b == 0 for a, b in pairs), + "fail_to_pass": sum(a == 0 and b == 1 for a, b in pairs), + "pass_to_fail": sum(a == 1 and b == 0 for a, b in pairs), + "pass_to_pass": sum(a == 1 and b == 1 for a, b in pairs), + } + + +def common_prefix(left: list[int], right: list[int]) -> int: + count = 0 + for left_id, right_id in zip(left, right): + if left_id != right_id: + break + count += 1 + return count + + +def tape_uint64(tape: str, source_id: str, step: int) -> int: + payload = ( + f"{PROTOCOL_ID}\0uniform\0" + f"{tape}\0{source_id}\0{step}" + ).encode() + return int.from_bytes(hashlib.sha256(payload).digest()[:8], "big") + + +def build_indices( + sampling: dict[str, Any], + evaluation: dict[str, Any], +) -> tuple[ + dict[tuple[str, str, str], dict[str, Any]], + dict[tuple[str, str, str], dict[str, Any]], +]: + sample_index = { + ( + source["id"], + run["tape_label"], + output["condition"], + ): output + for source in sampling["sources"] + for run in source["runs"] + for output in run["outputs"] + } + eval_index = { + ( + row["source_id"], + row["tape_label"], + row["condition"], + ): row + for row in evaluation["rows"] + } + if set(sample_index) != set(eval_index): + raise RuntimeError("sampling/evaluation cell keys differ") + return sample_index, eval_index + + +def main_analysis( + *, + sampling: dict[str, Any], + sample_index: dict[tuple[str, str, str], dict[str, Any]], + eval_index: dict[tuple[str, str, str], dict[str, Any]], +) -> dict[str, Any]: + result = {} + for domain in DOMAINS: + sources = [ + source + for source in sampling["sources"] + if source["domain"] == domain + ] + sources.sort(key=lambda row: row["within_domain_index"]) + if len(sources) != 32: + raise RuntimeError(f"{domain}: main frame must have 32 tasks") + source_ids = [source["id"] for source in sources] + rng = np.random.default_rng(BOOTSTRAP_SEED) + sampled_indices = rng.integers( + 0, + len(source_ids), + size=(BOOTSTRAP_RESAMPLES, len(source_ids)), + endpoint=False, + ) + conditions = {} + for condition in CONDITIONS: + rows = [ + eval_index[(source_id, "T0", condition)] + for source_id in source_ids + ] + conditions[condition] = { + metric: summarize( + [metric_value(row, metric) for row in rows] + ) + for metric in METRICS + } | { + "task_outcomes": dict( + sorted( + Counter(row["task_outcome"] for row in rows).items() + ) + ) + } + contrasts = {} + for name, (left_condition, right_condition) in CONTRASTS.items(): + metrics = {} + for metric in METRICS: + left = [ + metric_value( + eval_index[(source_id, "T0", left_condition)], + metric, + ) + for source_id in source_ids + ] + right = [ + metric_value( + eval_index[(source_id, "T0", right_condition)], + metric, + ) + for source_id in source_ids + ] + differences = np.asarray(right) - np.asarray(left) + bootstrap_means = differences[ + sampled_indices + ].mean(axis=1) + metric_result = { + "right_minus_left_point": float(differences.mean()), + "selected_task_resampling_band": percentile_band( + bootstrap_means + ), + "task_differences": summarize( + differences.tolist() + ), + "direction_counts": direction_counts( + differences.tolist() + ), + "by_source": [ + { + "source_id": source_id, + "domain_index": index, + "left": float(left[index]), + "right": float(right[index]), + "right_minus_left": float( + differences[index] + ), + } + for index, source_id in enumerate(source_ids) + ], + } + if metric in { + "fixed_budget_success", + "strict_complete_success", + }: + metric_result["transition"] = success_transition( + left, + right, + ) + metrics[metric] = metric_result + + trajectory_rows = [] + crn_exact = 0 + for source_id in source_ids: + left_output = sample_index[ + (source_id, "T0", left_condition) + ] + right_output = sample_index[ + (source_id, "T0", right_condition) + ] + left_ids = left_output["generated_token_ids"] + right_ids = right_output["generated_token_ids"] + shared_steps = min(len(left_ids), len(right_ids)) + expected_left_uniforms = [ + tape_uint64("T0", source_id, step) + for step in range( + left_output["uniform_steps_consumed"] + ) + ] + expected_right_uniforms = [ + tape_uint64("T0", source_id, step) + for step in range( + right_output["uniform_steps_consumed"] + ) + ] + left_uniform_exact = ( + canonical_hash(expected_left_uniforms) + == left_output["uniform_uint64_prefix_sha256"] + ) + right_uniform_exact = ( + canonical_hash(expected_right_uniforms) + == right_output["uniform_uint64_prefix_sha256"] + ) + crn_exact += left_uniform_exact and right_uniform_exact + trajectory_rows.append( + { + "source_id": source_id, + "shared_active_steps": shared_steps, + "common_prefix_tokens": common_prefix( + left_ids, + right_ids, + ), + "token_ids_exact": left_ids == right_ids, + "shared_uniform_prefix_exact": ( + left_uniform_exact and right_uniform_exact + ), + } + ) + contrasts[name] = { + "left": left_condition, + "right": right_condition, + "metrics": metrics, + "trajectory": { + "sources": len(trajectory_rows), + "shared_uniform_prefix_exact": crn_exact, + "exact_trajectories": sum( + row["token_ids_exact"] + for row in trajectory_rows + ), + "common_prefix_tokens": summarize( + [ + float(row["common_prefix_tokens"]) + for row in trajectory_rows + ] + ), + "rows": trajectory_rows, + }, + } + result[domain] = { + "tasks": len(source_ids), + "source_ids": source_ids, + "tape": "T0", + "conditions": conditions, + "contrasts": contrasts, + } + return result + + +def tape_diagnostic( + *, + sampling: dict[str, Any], + eval_index: dict[tuple[str, str, str], dict[str, Any]], +) -> dict[str, Any]: + result = {} + for domain in DOMAINS: + sources = [ + source + for source in sampling["sources"] + if source["domain"] == domain and len(source["runs"]) == 4 + ] + sources.sort(key=lambda row: row["within_domain_index"]) + if [row["within_domain_index"] for row in sources] != [ + 0, + 8, + 16, + 24, + ]: + raise RuntimeError( + f"{domain}: diagnostic task indices differ" + ) + source_ids = [row["id"] for row in sources] + tapes = ["T0", "T1", "T2", "T3"] + by_tape_condition = { + tape: { + condition: { + metric: summarize( + [ + metric_value( + eval_index[ + (source_id, tape, condition) + ], + metric, + ) + for source_id in source_ids + ] + ) + for metric in METRICS + } + for condition in CONDITIONS + } + for tape in tapes + } + contrasts = {} + for name, (left_condition, right_condition) in CONTRASTS.items(): + metric_results = {} + for metric in METRICS: + matrix = np.asarray( + [ + [ + metric_value( + eval_index[ + ( + source_id, + tape, + right_condition, + ) + ], + metric, + ) + - metric_value( + eval_index[ + ( + source_id, + tape, + left_condition, + ) + ], + metric, + ) + for tape in tapes + ] + for source_id in source_ids + ], + dtype=float, + ) + tape_means = matrix.mean(axis=0) + task_means = matrix.mean(axis=1) + task_ranges_within_tape = ( + matrix.max(axis=0) - matrix.min(axis=0) + ) + tape_ranges_within_task = ( + matrix.max(axis=1) - matrix.min(axis=1) + ) + metric_results[metric] = { + "matrix_task_by_tape": matrix.tolist(), + "tape_means": { + tape: float(tape_means[index]) + for index, tape in enumerate(tapes) + }, + "task_means": { + source_id: float(task_means[index]) + for index, source_id in enumerate(source_ids) + }, + "direction_by_tape": { + tape: direction_counts( + matrix[:, index].tolist() + ) + for index, tape in enumerate(tapes) + }, + "task_range_within_tape": summarize( + task_ranges_within_tape.tolist() + ), + "tape_range_within_task": summarize( + tape_ranges_within_task.tolist() + ), + "grand_mean_descriptive": float(matrix.mean()), + } + contrasts[name] = { + "left": left_condition, + "right": right_condition, + "metrics": metric_results, + } + result[domain] = { + "tasks": len(source_ids), + "source_ids": source_ids, + "tapes": tapes, + "by_tape_condition": by_tape_condition, + "contrasts": contrasts, + "independence_warning": ( + "The 4 tasks x 4 tapes are crossed repeated measures, " + "not 16 independent tasks." + ), + } + return result + + +def main() -> None: + args = parse_args() + for path in (args.sampling_json, args.evaluation_json): + if not path.is_file(): + raise FileNotFoundError(path) + sampling = json.loads( + args.sampling_json.read_text(encoding="utf-8") + ) + evaluation = json.loads( + args.evaluation_json.read_text(encoding="utf-8") + ) + if ( + sampling["protocol_id"] != PROTOCOL_ID + or evaluation["protocol_id"] != PROTOCOL_ID + ): + raise RuntimeError("protocol ID differs") + if sampling["summary"]["outputs"] != 352: + raise RuntimeError("sampling formal output count differs") + if len(evaluation["rows"]) != 352: + raise RuntimeError("evaluation row count differs") + sample_index, eval_index = build_indices(sampling, evaluation) + reproduction = None + if args.reproduction_json is not None: + if not args.reproduction_json.is_file(): + raise FileNotFoundError(args.reproduction_json) + reproduction_payload = json.loads( + args.reproduction_json.read_text(encoding="utf-8") + ) + if reproduction_payload["protocol_id"] != PROTOCOL_ID: + raise RuntimeError("reproduction protocol ID differs") + reproduction = { + "path": str(args.reproduction_json), + "sha256": sha256_file(args.reproduction_json), + "summary": reproduction_payload["summary"], + } + result = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "inputs": { + "sampling": { + "path": str(args.sampling_json), + "sha256": sha256_file(args.sampling_json), + "content_hash": sampling["content_hash"], + }, + "evaluation": { + "path": str(args.evaluation_json), + "sha256": sha256_file(args.evaluation_json), + "content_hash": evaluation["content_hash"], + }, + "reproduction": reproduction, + }, + "bootstrap_contract": { + "resamples": BOOTSTRAP_RESAMPLES, + "seed": BOOTSTRAP_SEED, + "rng": "numpy.random.default_rng reset per domain", + "unit": "selected task", + "paired_conditions": True, + "tape": "T0", + "interval_label": ( + "selected-task resampling band for the fixed " + "32-task frame and T0" + ), + "not": [ + "benchmark-population confidence interval", + "model-ability confidence interval", + "generation-seed uncertainty interval", + "causal-effect confidence interval", + ], + }, + "main_t0_selected_task_analysis": main_analysis( + sampling=sampling, + sample_index=sample_index, + eval_index=eval_index, + ), + "multi_tape_diagnostic": tape_diagnostic( + sampling=sampling, + eval_index=eval_index, + ), + "claim_boundary": [ + "HumanEval and GSM8K are analyzed separately.", + ( + "T0 selected-task bands describe only this frozen " + "32-task frame." + ), + ( + "T1-T3 are sensitivity diagnostics and are not pooled " + "into the T0 primary success rates." + ), + ( + "Common random numbers align probability quantiles; " + "they do not force identical sampled tokens." + ), + "Period prompts are counterfactual, not official-valid chats.", + ], + } + result["content_hash"] = canonical_hash( + { + "bootstrap_contract": result["bootstrap_contract"], + "main": result["main_t0_selected_task_analysis"], + "diagnostic": result["multi_tape_diagnostic"], + } + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + payload = args.output.read_bytes() + compact = { + domain: { + name: { + metric: result[ + "main_t0_selected_task_analysis" + ][domain]["contrasts"][name]["metrics"][metric][ + "right_minus_left_point" + ] + for metric in METRICS + } + for name in CONTRASTS + } + for domain in DOMAINS + } + print( + json.dumps( + { + "output": str(args.output), + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + "main_contrast_points": compact, + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_evaluator.py b/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_evaluator.py new file mode 100644 index 0000000..5522bd8 --- /dev/null +++ b/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_evaluator.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Evaluate Round 08 while keeping T0 and multi-tape diagnostics separate.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path +from statistics import mean +from typing import Any + +import v2_lite_chat_completion_evaluator as completion +import v2_lite_chat_sampling_evaluator as shared + + +PROTOCOL_ID = "llm-atlas-deepseek-chat-task-bootstrap-crn-v1" +CONDITIONS = ( + "s0_eos", + "s1_eos", + "s0_period", + "s1_period", +) +DOMAINS = ("code", "math") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--sampling-json", type=Path, required=True) + parser.add_argument("--human-eval", type=Path, required=True) + parser.add_argument("--gsm8k", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--sandbox-image", + required=True, + help="Pinned image reference including @sha256 digest.", + ) + parser.add_argument("--timeout-seconds", type=float, default=5.0) + parser.add_argument("--skip-code-execution", action="store_true") + return parser.parse_args() + + +def task_outcome( + domain: str, + evaluation: dict[str, Any], +) -> str: + if domain == "math": + if evaluation["fixed_budget_numeric_exact"]: + return "passed" + if evaluation["predicted_final"] is None: + return "no_numeric_answer" + return "wrong_numeric_answer" + if not evaluation["contains_entry_point_definition"]: + return "extract_failed" + if not evaluation["python_ast_parse"]: + return "syntax_error" + status = evaluation["execution"]["status"] + return { + "passed": "passed", + "timeout": "timeout", + "runtime_error": "runtime_error", + "assertion_failed": "assertion_failed", + "not_run": "not_run", + }.get(status, f"execution_{status}") + + +def success(row: dict[str, Any], *, strict: bool = False) -> int: + evaluation = row["task_evaluation"] + if row["domain"] == "code": + key = ( + "strict_complete_tests_pass" + if strict + else "fixed_budget_tests_pass" + ) + else: + key = ( + "strict_complete_numeric_exact" + if strict + else "fixed_budget_numeric_exact" + ) + return int(evaluation[key]) + + +def compact_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: + result = { + "outputs": len(rows), + "sources": len({row["source_id"] for row in rows}), + "tapes": sorted({row["tape_label"] for row in rows}), + "natural_eos": sum(row["hit_eos"] for row in rows), + "budget_truncated": sum( + row["stopped_at_max_new_tokens"] for row in rows + ), + "mean_generated_tokens": ( + mean(row["generated_tokens"] for row in rows) + if rows + else None + ), + "fixed_budget_success": sum(success(row) for row in rows), + "strict_complete_success": sum( + success(row, strict=True) for row in rows + ), + "task_outcomes": dict( + sorted(Counter(row["task_outcome"] for row in rows).items()) + ), + "completion_classes": dict( + sorted( + Counter( + row["completion_class"] for row in rows + ).items() + ) + ), + } + result["by_domain"] = { + domain: compact_summary_no_recursion( + [row for row in rows if row["domain"] == domain] + ) + for domain in DOMAINS + } + return result + + +def compact_summary_no_recursion( + rows: list[dict[str, Any]], +) -> dict[str, Any]: + return { + "outputs": len(rows), + "sources": len({row["source_id"] for row in rows}), + "tapes": sorted({row["tape_label"] for row in rows}), + "natural_eos": sum(row["hit_eos"] for row in rows), + "budget_truncated": sum( + row["stopped_at_max_new_tokens"] for row in rows + ), + "mean_generated_tokens": ( + mean(row["generated_tokens"] for row in rows) + if rows + else None + ), + "fixed_budget_success": sum(success(row) for row in rows), + "strict_complete_success": sum( + success(row, strict=True) for row in rows + ), + "task_outcomes": dict( + sorted(Counter(row["task_outcome"] for row in rows).items()) + ), + } + + +def main() -> None: + args = parse_args() + for path in ( + args.sampling_json, + args.human_eval, + args.gsm8k, + ): + if not path.is_file(): + raise FileNotFoundError(path) + if "@sha256:" not in args.sandbox_image: + raise ValueError("--sandbox-image must include immutable digest") + if args.timeout_seconds <= 0: + raise ValueError("--timeout-seconds must be positive") + sampling = json.loads( + args.sampling_json.read_text(encoding="utf-8") + ) + if sampling["protocol_id"] != PROTOCOL_ID: + raise RuntimeError("sampling protocol ID differs") + if tuple( + sampling["seed_contract"]["condition_row_order"] + ) != CONDITIONS: + raise RuntimeError("sampling condition order differs") + if sampling["summary"]["outputs"] != 352: + raise RuntimeError("formal grid must contain 352 outputs") + + human_eval, gsm8k = completion.load_tasks( + args.human_eval, + args.gsm8k, + ) + code_cache: dict[ + tuple[str, str, str], + dict[str, Any], + ] = {} + rows = [] + for source in sampling["sources"]: + for run in source["runs"]: + for output in run["outputs"]: + if source["domain"] == "math": + evaluation = completion.evaluate_math( + output["text"], + output["hit_eos"], + gsm8k[source["id"]]["answer"], + ) + cache_hit = False + else: + evaluation, cache_hit = ( + shared.evaluate_code_cached( + text=output["text"], + hit_eos=output["hit_eos"], + task=human_eval[source["id"]], + image=args.sandbox_image, + timeout_seconds=args.timeout_seconds, + skip_execution=args.skip_code_execution, + cache=code_cache, + ) + ) + row = { + "source_id": source["id"], + "domain": source["domain"], + "domain_index": source["within_domain_index"], + "replicate_index": run["replicate_index"], + "replicate_label": run["replicate_label"], + "tape_label": run["tape_label"], + "base_seed": run["base_seed"], + "run_seed": run["run_seed"], + "condition": output["condition"], + "generated_tokens": output["generated_tokens"], + "hit_eos": output["hit_eos"], + "stopped_at_max_new_tokens": output[ + "stopped_at_max_new_tokens" + ], + "prompt_token_ids_sha256": output[ + "prompt_token_ids_sha256" + ], + "generated_token_ids_sha256": output[ + "generated_token_ids_sha256" + ], + "text_sha256": output["text_sha256"], + "uniform_uint64_prefix_sha256": output[ + "uniform_uint64_prefix_sha256" + ], + "task_evaluation": evaluation, + "completion_class": completion.completion_class( + output, + evaluation, + ), + "code_execution_cache_hit": cache_hit, + } + row["task_outcome"] = task_outcome( + source["domain"], + evaluation, + ) + rows.append(row) + + main_rows = [ + row for row in rows if row["tape_label"] == "T0" + ] + diagnostic_sources = { + source["id"] + for source in sampling["sources"] + if len(source["runs"]) == 4 + } + diagnostic_rows = [ + row + for row in rows + if row["source_id"] in diagnostic_sources + ] + additional_rows = [ + row for row in rows if row["tape_label"] != "T0" + ] + if ( + len(main_rows) != 256 + or len(diagnostic_rows) != 128 + or len(additional_rows) != 96 + ): + raise RuntimeError( + "T0 / diagnostic grid counts differ from protocol" + ) + result = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "input": { + "sampling_path": str(args.sampling_json), + "sampling_sha256": completion.sha256_file( + args.sampling_json + ), + "sampling_content_hash": sampling["content_hash"], + "human_eval_sha256": completion.sha256_file( + args.human_eval + ), + "gsm8k_sha256": completion.sha256_file(args.gsm8k), + "model_revision": sampling["model"]["revision"], + "max_new_tokens": sampling["generation_contract"][ + "max_new_tokens" + ], + }, + "sandbox": { + "image": args.sandbox_image, + "timeout_seconds": args.timeout_seconds, + "code_execution_skipped": args.skip_code_execution, + "network": "none", + "filesystem": "read-only", + "user": "65534:65534", + "capabilities": "ALL dropped", + "no_new_privileges": True, + "memory": "256m", + "memory_swap": "256m", + "pids_limit": 64, + "cpus": 0.5, + "tmpfs": "/tmp:rw,noexec,nosuid,size=16m", + "host_mounts": 0, + "unique_code_cache_entries": len(code_cache), + }, + "rows": rows, + "summary": { + "formal_all_outputs": compact_summary(rows), + "main_t0": compact_summary(main_rows), + "diagnostic_all_four_tapes": compact_summary( + diagnostic_rows + ), + "diagnostic_additional_t1_t3": compact_summary( + additional_rows + ), + "main_t0_by_domain_condition": { + domain: { + condition: compact_summary_no_recursion( + [ + row + for row in main_rows + if row["domain"] == domain + and row["condition"] == condition + ] + ) + for condition in CONDITIONS + } + for domain in DOMAINS + }, + }, + "claim_boundary": [ + ( + "Primary summaries use T0 only; T1-T3 are isolated " + "multi-tape sensitivity diagnostics." + ), + "HumanEval and GSM8K are never pooled into one ability rate.", + ( + "The frozen 32 tasks per domain are a selected task " + "frame, not a full benchmark sample." + ), + "Passing HumanEval tests is functional, not safety, evidence.", + "Counterfactual period prompts are not official-valid chats.", + ], + } + result["content_hash"] = completion.canonical_hash(rows) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + payload = args.output.read_bytes() + print( + json.dumps( + { + "output": str(args.output), + "sha256": completion.sha256_bytes(payload), + "bytes": len(payload), + "rows": len(rows), + "main_t0": result["summary"]["main_t0"], + "unique_code_cache_entries": len(code_cache), + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_reproduction.py b/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_reproduction.py new file mode 100644 index 0000000..6df2df5 --- /dev/null +++ b/experiments/deepseek/v2_lite_chat_task_bootstrap_crn_reproduction.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Compare the preregistered 64-cell Round 08 replay with the formal grid.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +PROTOCOL_ID = "llm-atlas-deepseek-chat-task-bootstrap-crn-v1" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--formal-json", type=Path, required=True) + parser.add_argument("--rerun-json", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(16 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_hash(value: Any) -> str: + return hashlib.sha256( + json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + + +def run_index( + payload: dict[str, Any], +) -> dict[ + tuple[str, str, str], + tuple[dict[str, Any], dict[str, Any]], +]: + return { + ( + source["id"], + run["tape_label"], + output["condition"], + ): (run, output) + for source in payload["sources"] + for run in source["runs"] + for output in run["outputs"] + } + + +def main() -> None: + args = parse_args() + for path in (args.formal_json, args.rerun_json): + if not path.is_file(): + raise FileNotFoundError(path) + formal = json.loads( + args.formal_json.read_text(encoding="utf-8") + ) + rerun = json.loads( + args.rerun_json.read_text(encoding="utf-8") + ) + if ( + formal["protocol_id"] != PROTOCOL_ID + or rerun["protocol_id"] != PROTOCOL_ID + ): + raise RuntimeError("protocol ID differs") + if formal["model"]["revision"] != rerun["model"]["revision"]: + raise RuntimeError("model revision differs") + if formal["generation_contract"] != rerun["generation_contract"]: + raise RuntimeError("generation contract differs") + if rerun["summary"]["outputs"] != 64: + raise RuntimeError("replay grid must contain 64 outputs") + formal_rows = run_index(formal) + rerun_rows = run_index(rerun) + if any(key[1] != "T0" for key in rerun_rows): + raise RuntimeError("replay grid must use T0 only") + missing = sorted(set(rerun_rows) - set(formal_rows)) + if missing: + raise RuntimeError( + f"{len(missing)} replay cells are absent from formal grid" + ) + + rows = [] + for key in sorted(rerun_rows): + formal_run, formal_output = formal_rows[key] + rerun_run, rerun_output = rerun_rows[key] + checks = { + "run_seed_exact": ( + formal_run["run_seed"] == rerun_run["run_seed"] + ), + "prompt_hash_exact": ( + formal_output["prompt_token_ids_sha256"] + == rerun_output["prompt_token_ids_sha256"] + ), + "uniform_run_hash_exact": ( + formal_run["uniform_uint64_sha256"] + == rerun_run["uniform_uint64_sha256"] + ), + "uniform_output_prefix_hash_exact": ( + formal_output["uniform_uint64_prefix_sha256"] + == rerun_output["uniform_uint64_prefix_sha256"] + ), + "uniform_steps_exact": ( + formal_output["uniform_steps_consumed"] + == rerun_output["uniform_steps_consumed"] + ), + "generated_token_ids_exact": ( + formal_output["generated_token_ids"] + == rerun_output["generated_token_ids"] + ), + "decoded_text_exact": ( + formal_output["text"] == rerun_output["text"] + ), + "eos_state_exact": ( + formal_output["hit_eos"] + == rerun_output["hit_eos"] + ), + "truncation_state_exact": ( + formal_output["stopped_at_max_new_tokens"] + == rerun_output["stopped_at_max_new_tokens"] + ), + "cpu_rng_pre_state_exact": ( + formal_run["rng_state_before"]["cpu_sha256"] + == rerun_run["rng_state_before"]["cpu_sha256"] + ), + "cuda_rng_pre_state_exact": ( + formal_run["rng_state_before"][ + "cuda_combined_sha256" + ] + == rerun_run["rng_state_before"][ + "cuda_combined_sha256" + ] + ), + "torch_rng_unchanged_exact": ( + formal_run["torch_rng_unchanged"] + and rerun_run["torch_rng_unchanged"] + ), + } + rows.append( + { + "source_id": key[0], + "tape_label": key[1], + "condition": key[2], + **checks, + "all_preregistered_fields_exact": all( + checks.values() + ), + } + ) + check_names = [ + key + for key in rows[0] + if key.endswith("_exact") + and key != "all_preregistered_fields_exact" + ] + result = { + "schema_version": 1, + "protocol_id": PROTOCOL_ID, + "formal": { + "path": str(args.formal_json), + "sha256": sha256_file(args.formal_json), + "content_hash": formal["content_hash"], + }, + "rerun": { + "path": str(args.rerun_json), + "sha256": sha256_file(args.rerun_json), + "content_hash": rerun["content_hash"], + }, + "rows": rows, + "summary": { + "cells": len(rows), + "all_preregistered_fields_exact": sum( + row["all_preregistered_fields_exact"] + for row in rows + ), + "by_field": { + name: sum(row[name] for row in rows) + for name in check_names + }, + }, + "claim_boundary": [ + "Only the preregistered 16-source T0 subset is replayed.", + ( + "Exact replay is scoped to the pinned checkpoint, " + "software, sampler, precision, and hardware contract." + ), + "Reproduction does not imply tape-invariant trajectories.", + ], + } + if result["summary"]["cells"] != 64: + raise RuntimeError("reproduction comparison must contain 64 cells") + result["content_hash"] = canonical_hash(rows) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + payload = args.output.read_bytes() + print( + json.dumps( + { + "output": str(args.output), + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + "summary": result["summary"], + }, + ensure_ascii=False, + indent=2, + ) + ) + + +if __name__ == "__main__": + main()