research: freeze Chat prompts and CRN runner
This commit is contained in:
wuyang
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,6 +2,14 @@
|
||||
|
||||
> 状态：**已预注册，尚未运行本协议的任何生成输出**
|
||||
>
|
||||
> 预运行修订 1：实现审查后把“四条 trajectory 逐条执行”改为固定四行 batch；取样仍由
|
||||
> 每步显式共享的 `u_t` 完成。修订发生在任何 smoke / 正式输出之前，只为避免四倍重复
|
||||
> forward；finished row 的后续 PAD=EOS 与 mask=0 合同同时冻结在下文。
|
||||
>
|
||||
> 预运行修订 2：tokenizer-only 闸门发现首版 manifest 误把路由探针的短输入 hash 标成
|
||||
> Chat 生成 prompt hash。此时模型尚未加载、协议输出仍为 0。清单保留路由 hash，并新增
|
||||
> 由正式 Chat renderer 产生的 256 个 prompt hash；runner 只核验后者。
|
||||
>
|
||||
> 注册日期：2026-07-30
|
||||
>
|
||||
> 协议 ID：`llm-atlas-deepseek-chat-task-bootstrap-crn-v1`
|
||||
@@ -10,10 +18,13 @@
|
||||
> `research/DEEPSEEK_V2_LITE_CHAT_TASK_BOOTSTRAP_MANIFEST.json`
|
||||
>
|
||||
> 清单文件 SHA-256：
|
||||
> `7f5ef75f8e4bade35875e9132ea16f25f7fdb54450490e07cf1f6b2f701fceac`
|
||||
> `6313e70536c464fe598a93035576752418f08016dfd60ac246437c3b43bf2ae1`
|
||||
>
|
||||
> 清单规范内容 SHA-256：
|
||||
> `42ec988585d34ce9c2a193dd59a30a7f549d8a67b768c17098aaf3f30ca73804`
|
||||
> `d15303e18345f6dec2aaf891ba812c0a0232b86f416f8f685eabbe818249e838`
|
||||
>
|
||||
> 256 个 Chat prompt 合同 hash：
|
||||
> `7f766be54463c7f513948dd6d63a55e12b85b9220ee177361c743cd9997f6c3b`
|
||||
|
||||
## 0. 一句话说明这一轮
|
||||
|
||||
@@ -93,8 +104,9 @@ s0_eos, s1_eos, s0_period, s1_period
|
||||
- 其余 prompt token 与前序冻结模板相同；
|
||||
- 每条 prompt token hash 必须与冻结清单逐格 exact。
|
||||
|
||||
正式生成不是四行 batch。四条 trajectory 逐条执行，以避免一个条件先 EOS 后 batch
|
||||
padding / RNG 调用对其他条件产生隐式影响。
|
||||
正式生成使用固定四行 batch，行序即上述条件序。batch 内没有跨行 attention；取样不调用
|
||||
PyTorch RNG。同一行结束后，后续步追加 PAD=EOS 且该位置 attention mask 为 0，其他仍活跃
|
||||
行继续 forward。这个合同避免四倍重复 forward，也把执行形状固定下来。
|
||||
|
||||
---
|
||||
|
||||
@@ -196,7 +208,9 @@ u_t = (z_t + 0.5) / 2^64
|
||||
4. 按 token ID `0..vocab-1` 做 `torch.float32` cumulative sum；
|
||||
5. 强制最后一个 CDF 元素为 `1.0`；
|
||||
6. 将 `u_t` cast 为 `torch.float32`；
|
||||
7. `torch.searchsorted(cdf, u_t, right=False)` 取得 token ID。
|
||||
7. float32 转换若落在端点，则 clamp 到
|
||||
`torch.nextafter(0,1)` / `torch.nextafter(1,0)`；
|
||||
8. `torch.searchsorted(cdf, u_t, right=False)` 取得 token ID。
|
||||
|
||||
因此：
|
||||
|
||||
@@ -249,6 +263,7 @@ smoke 不进入正式统计。
|
||||
- 每题结束后写独立 journal；再次启动时必须核验协议、source、prompt 与输出内容 hash 后才
|
||||
能 resume；
|
||||
- journal 只用于可恢复执行，正式 JSON 由全部合格 journal 组装。
|
||||
- 四格固定同 batch；finished row 在后续步追加 PAD=EOS、对应 attention mask 为 0。
|
||||
|
||||
### 5.3 跨进程独立重放
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ const sources = DOMAINS.flatMap((domain) => {
|
||||
domain_index: domainIndex,
|
||||
selection_rank: row.selection_rank,
|
||||
source_text_sha256: row.text_sha256,
|
||||
prompt_token_ids_sha256: promptHashes,
|
||||
routing_probe_prompt_token_ids_sha256: promptHashes,
|
||||
main_tapes: ["T0"],
|
||||
diagnostic_tapes: DIAGNOSTIC_INDICES.has(domainIndex)
|
||||
? ["T1", "T2", "T3"]
|
||||
@@ -175,6 +175,11 @@ const manifest = {
|
||||
estimand_boundary:
|
||||
"Selected-task resampling bands for this fixed 32-task frame and one frozen tape; not benchmark-population or generation-seed confidence intervals.",
|
||||
},
|
||||
prompt_contract: {
|
||||
routing_probe_hashes_are_not_chat_generation_hashes: true,
|
||||
chat_generation_hashes:
|
||||
"Added by freeze-deepseek-chat-task-bootstrap-prompts.py before any protocol output.",
|
||||
},
|
||||
};
|
||||
|
||||
manifest.canonical_content_sha256 = sha256(canonical(manifest));
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze all 256 Chat-generation prompt hashes into the Round 08 manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEEPSEEK_EXPERIMENTS = ROOT / "experiments" / "deepseek"
|
||||
sys.path.insert(0, str(DEEPSEEK_EXPERIMENTS))
|
||||
|
||||
from transformers import AutoTokenizer # noqa: E402
|
||||
|
||||
import v2_lite_chat_special_token_behavior_probe as behavior # noqa: E402
|
||||
import v2_lite_routing_special_token_family_control as special # noqa: E402
|
||||
|
||||
|
||||
PROTOCOL_ID = "llm-atlas-deepseek-chat-task-bootstrap-crn-v1"
|
||||
CONDITIONS = (
|
||||
"s0_eos",
|
||||
"s1_eos",
|
||||
"s0_period",
|
||||
"s1_period",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--artifact-dir", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--reference-routing-json",
|
||||
type=Path,
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument("--human-eval", type=Path, required=True)
|
||||
parser.add_argument("--gsm8k", type=Path, required=True)
|
||||
parser.add_argument("--tnews", type=Path, required=True)
|
||||
parser.add_argument("--tnews-archive", type=Path, required=True)
|
||||
parser.add_argument("--wikitext", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def canonical_hash(value: Any) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def install_condition_contract() -> None:
|
||||
factors = {
|
||||
condition: special.FACTORS[condition]
|
||||
for condition in CONDITIONS
|
||||
}
|
||||
special.BOUNDARY_LEVELS = ("eos", "period")
|
||||
special.CONDITIONS = CONDITIONS
|
||||
special.FACTORS = factors
|
||||
special.SYSTEM_CELLS = {
|
||||
"eos": ("s0_eos", "s1_eos"),
|
||||
"period": ("s0_period", "s1_period"),
|
||||
}
|
||||
special.SYSTEM_EDGE_CONTRASTS = {
|
||||
"period_minus_eos": ("eos", "period"),
|
||||
}
|
||||
special.install_control_contract()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
|
||||
if manifest["protocol_id"] != PROTOCOL_ID:
|
||||
raise RuntimeError("manifest protocol ID differs")
|
||||
install_condition_contract()
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
args.artifact_dir,
|
||||
trust_remote_code=True,
|
||||
local_files_only=True,
|
||||
use_fast=True,
|
||||
)
|
||||
if tokenizer.pad_token_id is None:
|
||||
tokenizer.pad_token_id = tokenizer.eos_token_id
|
||||
args.domains = ["code", "math"]
|
||||
args.per_domain = 32
|
||||
source_rows, _ = behavior.selected_sources(args, tokenizer)
|
||||
frozen_rows = {
|
||||
row["id"]: row for row in manifest["sources"]
|
||||
}
|
||||
audit = []
|
||||
for source in source_rows:
|
||||
frozen = frozen_rows.get(source["id"])
|
||||
if frozen is None:
|
||||
raise RuntimeError(f"{source['id']} is absent from manifest")
|
||||
hashes = {}
|
||||
tokens = {}
|
||||
for condition in CONDITIONS:
|
||||
rendered = special.prior.render_boundary_variant(
|
||||
tokenizer,
|
||||
source["content"],
|
||||
condition,
|
||||
)
|
||||
hashes[condition] = rendered["token_ids_sha256"]
|
||||
tokens[condition] = rendered["tokens"]
|
||||
audit.append(
|
||||
{
|
||||
"source_id": source["id"],
|
||||
"condition": condition,
|
||||
"tokens": rendered["tokens"],
|
||||
"sha256": rendered["token_ids_sha256"],
|
||||
}
|
||||
)
|
||||
frozen["chat_generation_prompt_token_ids_sha256"] = hashes
|
||||
frozen["chat_generation_prompt_tokens"] = tokens
|
||||
if len(audit) != 256 or len(source_rows) != 64:
|
||||
raise RuntimeError(
|
||||
f"expected 64 sources / 256 cells, got "
|
||||
f"{len(source_rows)} / {len(audit)}"
|
||||
)
|
||||
manifest["status"] = (
|
||||
"preregistered_and_chat_prompt_hashes_corrected_before_"
|
||||
"any_protocol_output"
|
||||
)
|
||||
manifest["prompt_contract"] = {
|
||||
"routing_probe_hashes_are_not_chat_generation_hashes": True,
|
||||
"chat_generation_hashes_frozen": True,
|
||||
"chat_generation_prompt_cells": len(audit),
|
||||
"chat_generation_prompt_hashes_sha256": canonical_hash(audit),
|
||||
"renderer": (
|
||||
"official apply_chat_template followed by the frozen "
|
||||
"single boundary-ID edit for period cells"
|
||||
),
|
||||
}
|
||||
manifest.pop("canonical_content_sha256", None)
|
||||
manifest["canonical_content_sha256"] = canonical_hash(manifest)
|
||||
args.manifest.write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
payload = args.manifest.read_bytes()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"manifest": str(args.manifest),
|
||||
"bytes": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
"canonical_content_sha256": manifest[
|
||||
"canonical_content_sha256"
|
||||
],
|
||||
"sources": len(source_rows),
|
||||
"prompt_cells": len(audit),
|
||||
"prompt_hashes_sha256": manifest["prompt_contract"][
|
||||
"chat_generation_prompt_hashes_sha256"
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user