From b0acc8b47b196d065c2a6ff5f726dde91ca7f672 Mon Sep 17 00:00:00 2001 From: wuyang <5700876+banisherwy@user.noreply.gitee.com> Date: Wed, 29 Jul 2026 17:23:19 +0800 Subject: [PATCH] feat: add DeepSeek chat-template routing probe --- PROGRESS.md | 14 +- README.md | 13 +- experiments/deepseek/README.md | 52 + .../v2_lite_routing_template_probe.py | 1120 + research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md | 438 + scripts/check-deepseek-browser.mjs | 86 +- src/components/DeepSeekArtifactLab.astro | 594 +- ...epseek-v2-lite-routing-template-repro.json | 555633 +++++++++++++++ .../deepseek-v2-lite-routing-template.json | 555633 +++++++++++++++ src/pages/deepseek/index.astro | 12 +- src/pages/progress/index.astro | 10 +- 11 files changed, 1113573 insertions(+), 32 deletions(-) create mode 100644 experiments/deepseek/v2_lite_routing_template_probe.py create mode 100644 research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md create mode 100644 src/data/deepseek-v2-lite-routing-template-repro.json create mode 100644 src/data/deepseek-v2-lite-routing-template.json diff --git a/PROGRESS.md b/PROGRESS.md index 4947fa6..a564c4f 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -14,7 +14,7 @@ | 表示、位置与残差高速公路 | 完成首版 | 81% | 真实 hidden-state / norm traces、长上下文位置外推与深层稳定性消融 | | Scaling Laws | 完成首版 | 74% | 真实拟合复现、置信区间与更多模型族对照 | | 数据工程与预训练配方 | 完成首版 | 73% | FineWeb / DCLM 逐图精读、真实去重误伤与 mixture traces | -| DeepSeek 专题 | 三轮实证进行中 | 92% | SM90 FlashMLA kernel、完整 27 层、tokenization 扰动、FP8/pipeline 与 R1-like RL 复现 | +| DeepSeek 专题 | 三轮实证进行中 | 93% | SM90 FlashMLA kernel、完整 27 层、词元边界 / system / few-shot 正交扰动、FP8/pipeline 与 R1-like RL 复现 | | 指令微调与人类偏好 | 完成首版 | 75% | 真实偏好分歧、RM 长度偏置与 PPO/DPO 小模型复现 | | 推理与测试时扩展 | 完成首版 | 76% | 真实模型采样曲线、PRM 案例与逐篇图表精读 | | 工具使用与长程 Agent | 完成首版 | 74% | 真实环境 traces、cross-harness 对照、Agent RL 曲线与安全案例 | @@ -41,7 +41,7 @@ - [x] 完成 486 篇关键论文索引，覆盖 16 个标签专题与 Kimi/DeepSeek 聚光主线。 - [x] 完成可检索、可按专题筛选的论文库页面。 - [x] 完成 K3、语言模型前史、Transformer 基础、表示/位置/残差、DeepSeek 谱系、Scaling Laws、数据工程、长上下文、MoE、指令微调与人类偏好、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全十七篇首版长文。 -- [x] 完成 K3 三轴架构、八联报告实验与四联开放工件实验、语言模型前史四联实验、Transformer 四联实验、表示深度四联实验、DeepSeek 十联实验、长上下文、MoE 路由、推理三页签，以及训练系统、推理服务、Scaling、数据工程、数值、Alignment、Agent、原生多模态与评测安全专题各四页签等七十七个原创交互视图。 +- [x] 完成 K3 三轴架构、八联报告实验与四联开放工件实验、语言模型前史四联实验、Transformer 四联实验、表示深度四联实验、DeepSeek 十一联实验、长上下文、MoE 路由、推理三页签，以及训练系统、推理服务、Scaling、数据工程、数值、Alignment、Agent、原生多模态与评测安全专题各四页签等七十八个原创交互视图。 - [x] 完成长上下文首版：五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。 - [x] 核验 FlashAttention、DeepSeek-V2/V3.2/V4、Kimi Linear/K3 等六份论文原文，并建立长上下文研究账本。 - [x] 核验 Switch、ST-MoE、DeepSeekMoE、Loss-Free、V3、LatentMoE 与 K3 原文，并建立 MoE 研究账本。 @@ -190,11 +190,16 @@ - [x] 中文↔代码 JSD 在六层均下降但没有消失；Layer 2 `0.091→0.065`、Δ `-0.026 [-0.036,-0.018]`。结论限定为固定前缀长度敏感性，不推出因果内容效应、专家语义或训练/线上总体。 - [x] 自然长度与等长 16 / 24 三 cohort 合计 13,580 个有效 token、488,880 次真实路由；机器可读比较结果第二次生成 SHA-256 均为 `00bdc7fe…61a1`，网站加入 cohort、层、聚合口径与 paired delta 联动。 - [x] DeepSeek 长度敏感性版本以源提交 `9ca0850`、不可变镜像 `20260729T081849Z-9ca0850` 发布；OCI digest `sha256:c2146735…3d641`，NAS / VPS / NPM / DNS / TLS / HTTP2 / gzip / 门户与十六套生产 Chrome 回归全通过；保留 `20260729T073342Z-5bcfd58` 回滚。 +- [x] 官方 chat-template 扰动固定同一批 128 条 source prompt 与 23 个 canonical content tokens，在同一 padded batch 运行 raw / user / generation 384 个变体；用相对字符跨度与 token ID 的交集精确对齐 2,874 个内容 token。 +- [x] RTX 5090 新增 10,612 个输入 token、382,032 次真实 top-6 路由，使公开语料累计达到 870,912 次；完整 JSON 独立复跑 SHA-256 均为 `da1f1033…bc1b9`，byte-exact。 +- [x] causal suffix 负对照闭环：USER→GENERATION 的 3,642-token 共享前缀在每层全部 ordered top-6 exact，六层合计 `21,852 / 21,852`；对齐内容的 CV Δ / TV / JSD 全为零，而完整输入因新增 `Assistant:` token 保持非零分布变化。 +- [x] RAW→USER 的模板敏感性不写成单向规律：L1 英文/中文与 L2 中文/代码更平，L3 中文、L5 中文/代码与 L6 四域更集中；网站第七个真实工件页签联动 layer、scope、aggregation，并展示 paired 95% 区间与逐 token top-6 稳定性。 +- [x] 模板扰动里程碑本地闸门通过：69 个 Astro 文件零诊断、21 个页面、1,151 个站内引用、12 个跨页锚点零失败，十六套真实 Chrome 回归全部通过，桌面与 390px 移动端无文档级横向溢出。 ## 正在进行 - [ ] K3 三轮下一闸门：获得真实 token hidden states、expert load 与 cache traces，解释或修订 `A_log [128]` 工件冲突，再做 Figure 3/4/5 数值重绘和独立小模型复现。 -- [ ] DeepSeek 三轮下一闸门：在官方支持的 SM90 环境执行 FlashMLA 优化 kernel；扩到完整 27 层并补 tokenizer / prompt-template 扰动对照，再推进 FP8 / pipeline traces 与 R1-like RL 小模型复现。 +- [ ] DeepSeek 三轮下一闸门：在官方支持的 SM90 环境执行 FlashMLA 优化 kernel；扩到完整 27 层并补词元边界 / system / few-shot 正交扰动，再推进 FP8 / pipeline traces 与 R1-like RL 小模型复现。 - [ ] 表示、位置与残差二轮：真实 hidden-state / norm traces、长上下文位置外推复现与 mHC / AttnRes 深层稳定性消融。 - [ ] 评测安全二轮：真实 cross-harness / pass@k 复跑、Judge 元评测、动态污染与过拒案例。 - [ ] 推理服务二轮：真实 GPU kernel / workload traces、功耗与成本、跨 vLLM / SGLang / TensorRT-LLM 复现。 @@ -325,6 +330,9 @@ | 2026-07-29 | 长度效应必须用 paired prompt bootstrap | 16-token 输入是 24-token 输入前缀，2,000 次重采样共用 prompt indices；区间描述固定 cohort 的敏感性，不升级为内容因果或总体显著性 | | 2026-07-29 | 自然长度、matched-16 与 matched-24 永久分开呈现 | 自然 cohort 回答“本批原始样本如何路由”；matched cohort 回答“同一前缀多看 8 tokens 后如何变化”，不能互相替代 | | 2026-07-29 | DeepSeek 长度敏感性里程碑以 `20260729T081849Z-9ca0850` 发布 | OCI digest `sha256:c2146735…3d641`；复用 NAS 12010→8080、NPM 31 / cert 41、门户 order 180；十六套生产 Chrome 回归通过，保留上一不可变镜像回滚 | +| 2026-07-29 | 模板扰动只比较同一 canonical content | raw / official user / generation 三条件同 batch；字符跨度与 token ID 同时相同才进入 2,874-token 内容交集，wrapper 与边界重切分 token 不混入内容效应 | +| 2026-07-29 | `Assistant:` 追加条件承担 causal-mask 负对照 | 21,852 个共享前缀 ordered top-6 全部 exact；证明未来 suffix 不改写过去，但不推出 suffix 自身没有路由作用 | +| 2026-07-29 | 模板效应永久分 scope、layer 与 domain 报告 | 对齐内容回答上下文条件化，完整输入回答真实协议流量；RAW→USER 的 CV 方向跨层翻转，不压成“角色模板更均衡/更集中” | | 2026-07-29 | K3 二轮按 32 张对象账与完整报告顺序重建 | total/active、2.5×、KDA state、深度来源、专家路由、视觉目标、轨迹、缓存与评测协议不再压成一页组件摘要 | | 2026-07-29 | K3 原生视觉事实回到 §2.4 / §3.3 核验 | 删除“先冻结语言模型再解冻”旧表述；明确 MoonViT-V2 从头训练，视觉/文本从开始共同 NTP | | 2026-07-29 | K3 Figure 1–16 / Table 1–5 全部建立课程视觉契约 | 每张图同时写支持范围与不可外推项；作者报告、论文、推导与 toy model 使用 R/P/D/T 标签 | diff --git a/README.md b/README.md index 863d33e..ba70394 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ 当前里程碑包含 17 专题学习地图、486 篇关键论文索引、Kimi K3 完整导读， 语言模型前史、Transformer 基础、表示/位置/残差、DeepSeek 技术谱系、Scaling Laws、数据工程、长上下文、MoE、指令微调与人类偏好、推理、工具使用与长程 Agent、原生多模态、训练系统、推理服务、数值优化，以及评测与安全深度专题， -以及 77 个覆盖核心机制的原创交互视图。K3 二轮导读以 32 张问题账、16 图 / 5 表审计、 +以及 78 个覆盖核心机制的原创交互视图。K3 二轮导读以 32 张问题账、16 图 / 5 表审计、 8 个交互实验和 100 个一手/官方节点，完整覆盖架构、预训练、后训练、系统、评测、案例与附录。 第三轮已完成开放工件与首个真实 kernel 里程碑：固定官方模型与 FlashKDA revisions，审计 96 个 checkpoint shards、 497,220 个 tensor entries、真实 KDA / MLA / MoE / MoonViT shapes 与小范围参数统计，并用 4 个新视图 @@ -28,7 +28,7 @@ [K3_ARTIFACT_AUDIT.md](./research/K3_ARTIFACT_AUDIT.md) 与 [checkpoint_probe.py](./experiments/k3/checkpoint_probe.py)、[FlashKDA probe](./experiments/k3/flashkda/)。 DeepSeek 三轮专题以 24 张问题账、10 次技术转向、 -10 个交互实验和 60 个一手/官方节点，串起 Dense、MoE、MLA、V3 协同、R1 与 V4； +11 个交互实验和 60 个一手/官方节点，串起 Dense、MoE、MLA、V3 协同、R1 与 V4； 并固定官方 V2-Lite revision，在 RTX 5090 上连续执行 7/27 层，记录 3,240 次真实专家选择、 MLA/HF eager cache shapes 与 `31/31` exact 独立复跑；进一步用真实 layer-1 权重执行官方 V3 naive/absorb 路径，实际写入 576 元素 latent cache，并以 FP32 将两种结合顺序的最大误差压到 @@ -37,11 +37,16 @@ naive/absorb 路径，实际写入 576 元素 latent cache，并以 FP32 将两 整份 JSON byte-exact。随后又对同一批 128 条、源长度至少 24 tokens 的 prompt 执行 16 / 24-token 嵌套前缀对照，新增 184,320 次真实路由；paired bootstrap 显示延长前缀通常 降低 CV，尤其 Layer 2 代码为 `−0.251 [−0.283, −0.215]`，但六层中文↔代码 JSD 仍未消失。 -当前累计 488,880 次公开语料路由。FlashMLA 的 SM90/SM100 官方支持矩阵与本机 SM120 边界单独记账。详见 +最新一轮再用官方 chat template 对同一段内容构造 raw / user / generation 三个条件， +新增 382,032 次真实路由：精确对齐 2,874 个内容 token 后，RAW→USER 的方向随层与域改变， +而 USER→GENERATION 的 21,852 个共享前缀 ordered top-6 全部 exact，验证未来 suffix +不能改写过去路由。当前累计 870,912 次公开语料路由。FlashMLA 的 SM90/SM100 官方支持矩阵 +与本机 SM120 边界单独记账。详见 [DEEPSEEK_V2_LITE_TRACE.md](./research/DEEPSEEK_V2_LITE_TRACE.md) 与 [DEEPSEEK_MLA_ABSORB_AUDIT.md](./research/DEEPSEEK_MLA_ABSORB_AUDIT.md)、 [DEEPSEEK_ROUTING_CORPUS_AUDIT.md](./research/DEEPSEEK_ROUTING_CORPUS_AUDIT.md)、 -[DEEPSEEK_ROUTING_LENGTH_CONTROL_AUDIT.md](./research/DEEPSEEK_ROUTING_LENGTH_CONTROL_AUDIT.md)。 +[DEEPSEEK_ROUTING_LENGTH_CONTROL_AUDIT.md](./research/DEEPSEEK_ROUTING_LENGTH_CONTROL_AUDIT.md) 与 +[DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md](./research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md)。 其余专题按进度账本持续扩建。 ## 本地开发 diff --git a/experiments/deepseek/README.md b/experiments/deepseek/README.md index 0ceca53..f0bd529 100644 --- a/experiments/deepseek/README.md +++ b/experiments/deepseek/README.md @@ -153,3 +153,55 @@ python -B experiments/deepseek/compare_routing_length_control.py \ See `research/DEEPSEEK_ROUTING_LENGTH_CONTROL_AUDIT.md` for the sampling bias audit, paired CV/JSD deltas, total-variation accounting, and interpretation boundaries. + +## Official chat-template sensitivity + +`v2_lite_routing_template_probe.py` renders three variants of one fixed +23-content-token prefix: + +```text +raw BOS + content +user BOS + "User: " + content + "\n\n" +generation user prefix + "Assistant:" +``` + +The latter two use the pinned official `chat_template` through +`apply_chat_template`. All three variants of one source prompt execute in the +same padded batch. Statistics are split between: + +- the full operational input, including wrapper tokens; +- the exact intersection of `(relative character span, token ID)` inside the + source content across all three variants. + +The `user → generation` comparison is a causal negative control: the appended +suffix must not change routes on their shared prefix. + +```bash +PYTHONPATH=/path/to/transformers-4.41.2-deps:/usr/lib/python3/dist-packages \ +python -B experiments/deepseek/v2_lite_routing_template_probe.py \ + --artifact-dir /path/to/deepseek-v2-lite \ + --human-eval /path/to/HumanEval.jsonl.gz \ + --gsm8k /path/to/gsm8k/test.jsonl \ + --tnews /path/to/tnews/test.json \ + --tnews-archive /path/to/tnews_public.zip \ + --wikitext /path/to/wikitext-validation.parquet \ + --output src/data/deepseek-v2-lite-routing-template.json \ + --per-domain 32 \ + --content-tokens 23 \ + --batch-prompts 8 \ + --layers 7 \ + --bootstrap 2000 \ + --seed 20260729 \ + --captured-at 2026-07-29T08:30:00+00:00 +``` + +The three variants add 382,032 real top-6 route selections. Across six MoE +layers, all 21,852 `user → generation` shared-prefix token routes are +ordered-top-6 exact. The committed run and independent rerun are byte-exact: + +```text +da1f10333b2fa269e64f9716a0ca6c1a656d23d3e70b8a25d6e59f8a5b3bc1b9 +``` + +See `research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md` for the aligned-content +contract, paired intervals, per-token route stability, and claim boundaries. diff --git a/experiments/deepseek/v2_lite_routing_template_probe.py b/experiments/deepseek/v2_lite_routing_template_probe.py new file mode 100644 index 0000000..eb2d8f9 --- /dev/null +++ b/experiments/deepseek/v2_lite_routing_template_probe.py @@ -0,0 +1,1120 @@ +#!/usr/bin/env python3 +"""Probe DeepSeek-V2-Lite routing sensitivity to its official chat template. + +The experiment keeps one source-addressable public cohort fixed and renders +three inputs per prompt: + +1. regular tokenizer input (BOS + content); +2. the official one-user chat template; +3. the same official template with the generation prompt appended. + +All three variants execute together through official BF16 layers 0--6. The +output separates full-input loads from content-only loads, pairs bootstrap +indices by source prompt, aligns content tokens by character span, and checks +the causal invariant that appending a suffix cannot change a shared prefix. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import json +import math +import platform +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from safetensors import safe_open +from transformers import AutoTokenizer + +from v2_lite_routing_corpus import ( + DOMAIN_LABELS, + DOMAIN_ORDER, + bootstrap_domain, + distribution, + git_revision, + gpu_identity, + interval, + js_divergence, + load_candidates, + load_official_modules, + metric_vector, + scoped_seed, + sha256, + text_sha256, +) + + +CONDITIONS = ("raw", "user", "generation") +CONDITION_LABELS = { + "raw": "BOS + content", + "user": "official user template", + "generation": "official user template + Assistant:", +} +COMPARISONS = ( + ("raw_to_user", "raw", "user"), + ("user_to_generation", "user", "generation"), +) +SAMPLE_SALT = "llm-atlas-deepseek-routing-template-control-v1" +CHAT_TEMPLATE_REVISION = "604d5664dddd88a0433dbae533b7fe9472482de0" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--artifact-dir", type=Path, required=True) + parser.add_argument("--human-eval", type=Path, required=True) + parser.add_argument("--gsm8k", type=Path, required=True) + parser.add_argument("--tnews", type=Path, required=True) + parser.add_argument("--tnews-archive", type=Path, required=True) + parser.add_argument("--wikitext", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--per-domain", type=int, default=32) + parser.add_argument("--content-tokens", type=int, default=23) + parser.add_argument("--batch-prompts", type=int, default=8) + parser.add_argument("--layers", type=int, default=7) + parser.add_argument("--bootstrap", type=int, default=2000) + parser.add_argument("--seed", type=int, default=20260729) + parser.add_argument("--sample-salt", default=SAMPLE_SALT) + parser.add_argument("--device", default="cuda") + parser.add_argument("--captured-at", default=None) + return parser.parse_args() + + +def canonical_hash(value: Any) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def tokenize_with_offsets( + tokenizer: Any, + text: str, + *, + add_special_tokens: bool, +) -> tuple[list[int], list[tuple[int, int]]]: + encoded = tokenizer( + text, + add_special_tokens=add_special_tokens, + return_offsets_mapping=True, + padding=False, + truncation=False, + ) + return list(encoded.input_ids), [tuple(pair) for pair in encoded.offset_mapping] + + +def content_positions( + token_ids: list[int], + offsets: list[tuple[int, int]], + content_start: int, + content_end: int, +) -> tuple[list[int], list[dict[str, int]], int]: + positions = [] + records = [] + crossing = 0 + for index, (token_id, (start, end)) in enumerate(zip(token_ids, offsets, strict=True)): + if end <= start: + continue + overlaps = start < content_end and end > content_start + inside = start >= content_start and end <= content_end + if inside: + positions.append(index) + records.append( + { + "position": index, + "token_id": token_id, + "start": start - content_start, + "end": end - content_start, + } + ) + elif overlaps: + crossing += 1 + return positions, records, crossing + + +def common_prefix_length(left: list[int], right: list[int]) -> int: + length = 0 + for left_id, right_id in zip(left, right): + if left_id != right_id: + break + length += 1 + return length + + +def render_variant(tokenizer: Any, content: str, condition: str) -> dict[str, Any]: + if condition == "raw": + rendered = content + token_ids, offsets = tokenize_with_offsets( + tokenizer, + rendered, + add_special_tokens=True, + ) + content_start = 0 + else: + messages = [{"role": "user", "content": content}] + add_generation_prompt = condition == "generation" + rendered = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=add_generation_prompt, + ) + official_ids = list( + tokenizer.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=add_generation_prompt, + ) + ) + token_ids, offsets = tokenize_with_offsets( + tokenizer, + rendered, + add_special_tokens=False, + ) + if token_ids != official_ids: + raise RuntimeError(f"{condition} rendered token IDs differ from apply_chat_template") + content_start = rendered.index(content) + + content_end = content_start + len(content) + positions, records, crossing = content_positions( + token_ids, + offsets, + content_start, + content_end, + ) + if not positions: + raise RuntimeError(f"{condition} has no content tokens") + return { + "condition": condition, + "rendered_sha256": text_sha256(rendered), + "token_ids": token_ids, + "tokens": len(token_ids), + "token_ids_sha256": canonical_hash(token_ids), + "content_positions": positions, + "content_records": records, + "content_tokens": len(positions), + "wrapper_tokens": len(token_ids) - len(positions), + "boundary_crossing_tokens": crossing, + } + + +def select_samples( + tokenizer: Any, + candidates: dict[str, list[dict[str, str]]], + per_domain: int, + content_tokens: int, + sample_salt: str, +) -> tuple[list[dict[str, Any]], dict[str, dict[str, int]]]: + def fixed_token_prefix( + text: str, + full_offsets: list[tuple[int, int]], + ) -> tuple[str, list[int]]: + lower_index = max(0, content_tokens - 5) + upper_index = min(len(full_offsets) - 1, content_tokens + 7) + lower = max(1, full_offsets[lower_index][0]) + upper = max(lower, full_offsets[upper_index][1]) + matches = [] + for end in range(lower, upper + 1): + prefix = text[:end] + ids = list(tokenizer(prefix, add_special_tokens=False).input_ids) + if len(ids) == content_tokens: + matches.append((end, ids)) + if not matches: + raise RuntimeError("could not construct a fixed-token character prefix") + end, ids = matches[-1] + return text[:end], ids + + selected = [] + counts = {} + for domain in DOMAIN_ORDER: + ranked = [] + for row in candidates[domain]: + full_ids, full_offsets = tokenize_with_offsets( + tokenizer, + row["text"], + add_special_tokens=False, + ) + if len(full_ids) < content_tokens: + continue + rank = hashlib.sha256( + f"{sample_salt}|{domain}|{row['id']}".encode() + ).hexdigest() + ranked.append((rank, row, len(full_ids), full_offsets)) + ranked.sort(key=lambda item: (item[0], item[1]["id"])) + + eligible = [] + canonicalization_failures = 0 + for rank, row, source_tokens, full_offsets in ranked: + try: + content, prefix_ids = fixed_token_prefix(row["text"], full_offsets) + except RuntimeError: + canonicalization_failures += 1 + continue + variants = { + condition: render_variant(tokenizer, content, condition) + for condition in CONDITIONS + } + shared_content_keys = set.intersection( + *( + { + (record["start"], record["end"], record["token_id"]) + for record in variants[condition]["content_records"] + } + for condition in CONDITIONS + ) + ) + for condition in CONDITIONS: + variants[condition]["aligned_content_positions"] = [ + record["position"] + for record in variants[condition]["content_records"] + if (record["start"], record["end"], record["token_id"]) + in shared_content_keys + ] + variants[condition]["aligned_content_tokens"] = len( + variants[condition]["aligned_content_positions"] + ) + eligible.append( + { + "id": row["id"], + "domain": domain, + "label": DOMAIN_LABELS[domain], + "text_sha256": text_sha256(row["text"]), + "source_characters": len(row["text"]), + "source_tokens": source_tokens, + "content": content, + "content_sha256": text_sha256(content), + "content_characters": len(content), + "canonical_content_token_ids_sha256": canonical_hash(prefix_ids), + "selection_rank": rank, + "variants": variants, + } + ) + if len(eligible) == per_domain: + break + if len(eligible) < per_domain: + raise RuntimeError( + f"{domain} has only {len(eligible)} eligible prompts; need {per_domain}" + ) + domain_rows = eligible[:per_domain] + for index, row in enumerate(domain_rows): + row["within_domain_index"] = index + selected.extend(domain_rows) + counts[domain] = { + "candidate_records_after_text_filter": len(candidates[domain]), + "eligible_records": len(ranked), + "canonicalization_failures_before_selection_complete": ( + canonicalization_failures + ), + "selected_records": len(domain_rows), + "source_tokens_min": min(row["source_tokens"] for row in domain_rows), + "source_tokens_mean": float( + np.mean([row["source_tokens"] for row in domain_rows]) + ), + "source_tokens_max": max(row["source_tokens"] for row in domain_rows), + } + return selected, counts + + +def make_batches( + samples: list[dict[str, Any]], + batch_prompts: int, + pad_token_id: int, +) -> list[dict[str, Any]]: + ordered = sorted( + samples, + key=lambda row: ( + max(row["variants"][condition]["tokens"] for condition in CONDITIONS), + DOMAIN_ORDER.index(row["domain"]), + row["id"], + ), + ) + batches = [] + for start in range(0, len(ordered), batch_prompts): + prompt_rows = ordered[start : start + batch_prompts] + variants = [ + { + "sample": sample, + "condition": condition, + **sample["variants"][condition], + } + for sample in prompt_rows + for condition in CONDITIONS + ] + sequence = max(row["tokens"] for row in variants) + input_ids = torch.full( + (len(variants), sequence), + pad_token_id, + dtype=torch.long, + ) + attention_mask = torch.zeros((len(variants), sequence), dtype=torch.long) + for index, row in enumerate(variants): + length = row["tokens"] + input_ids[index, :length] = torch.tensor(row["token_ids"]) + attention_mask[index, :length] = 1 + batches.append( + { + "prompt_rows": prompt_rows, + "variants": variants, + "input_ids": input_ids, + "attention_mask": attention_mask, + "padded_sequence": sequence, + } + ) + return batches + + +def bootstrap_distributions( + loads: np.ndarray, + mode: str, + sampled: np.ndarray, +) -> np.ndarray: + if mode == "token_weighted": + values = loads[sampled].sum(axis=1, dtype=np.float64) + return values / values.sum(axis=1, keepdims=True) + prompt_distributions = loads / loads.sum(axis=1, keepdims=True) + values = prompt_distributions[sampled].mean(axis=1) + return values / values.sum(axis=1, keepdims=True) + + +def paired_domain( + before: np.ndarray, + after: np.ndarray, + mode: str, + replicates: int, + seed: int, + scope: str, +) -> dict[str, Any]: + if before.shape != after.shape: + raise ValueError(f"paired shape mismatch: {before.shape} != {after.shape}") + rng = np.random.default_rng(scoped_seed(seed, scope)) + sampled = rng.integers( + 0, + before.shape[0], + size=(replicates, before.shape[0]), + endpoint=False, + ) + before_point = distribution(before, mode) + after_point = distribution(after, mode) + before_boot = bootstrap_distributions(before, mode, sampled) + after_boot = bootstrap_distributions(after, mode, sampled) + before_metrics = metric_vector(before_point) + after_metrics = metric_vector(after_point) + before_boot_metrics = metric_vector(before_boot) + after_boot_metrics = metric_vector(after_boot) + metrics = {} + for name in before_metrics: + delta_boot = after_boot_metrics[name] - before_boot_metrics[name] + metrics[name] = { + "before": float(before_metrics[name][0]), + "after": float(after_metrics[name][0]), + "delta_after_minus_before": float( + after_metrics[name][0] - before_metrics[name][0] + ), + "delta_ci95": interval(delta_boot), + } + tv_boot = 0.5 * np.abs(after_boot - before_boot).sum(axis=1) + jsd_boot = js_divergence(before_boot, after_boot) + share_delta = after_point - before_point + return { + "metrics": metrics, + "total_variation": { + "point": float(0.5 * np.abs(share_delta).sum()), + "ci95": interval(tv_boot), + }, + "js_divergence": { + "point": float(js_divergence(before_point, after_point)[0]), + "ci95": interval(jsd_boot), + "unit": "nats", + "upper_bound": math.log(2), + }, + "expert_share_delta": share_delta.tolist(), + "expert_share_delta_ci95": interval(after_boot - before_boot), + } + + +def aligned_pairs( + left: dict[str, Any], + right: dict[str, Any], +) -> list[tuple[int, int]]: + right_by_key = { + (row["start"], row["end"], row["token_id"]): row["position"] + for row in right["content_records"] + } + return [ + ( + row["position"], + right_by_key[(row["start"], row["end"], row["token_id"])], + ) + for row in left["content_records"] + if (row["start"], row["end"], row["token_id"]) in right_by_key + ] + + +def route_alignment( + left_routes: torch.Tensor, + right_routes: torch.Tensor, + pairs: list[tuple[int, int]], +) -> dict[str, Any]: + if not pairs: + return { + "aligned_tokens": 0, + "ordered_topk_exact": 0, + "set_topk_exact": 0, + "mean_topk_overlap": None, + "mean_jaccard": None, + } + ordered_exact = 0 + set_exact = 0 + overlaps = [] + jaccards = [] + for left_position, right_position in pairs: + left = left_routes[left_position].tolist() + right = right_routes[right_position].tolist() + ordered_exact += int(left == right) + left_set = set(left) + right_set = set(right) + intersection = len(left_set & right_set) + union = len(left_set | right_set) + set_exact += int(left_set == right_set) + overlaps.append(intersection) + jaccards.append(intersection / union) + return { + "aligned_tokens": len(pairs), + "ordered_topk_exact": ordered_exact, + "set_topk_exact": set_exact, + "ordered_topk_exact_rate": ordered_exact / len(pairs), + "set_topk_exact_rate": set_exact / len(pairs), + "mean_topk_overlap": float(np.mean(overlaps)), + "mean_jaccard": float(np.mean(jaccards)), + } + + +def layer_statistics( + prompt_rows: list[dict[str, Any]], + replicates: int, + seed: int, + layer_index: int, +) -> dict[str, Any]: + scopes = {} + for load_scope, load_key in ( + ("full_input", "full_load"), + ("content_only", "content_load"), + ): + modes = {} + for mode in ("token_weighted", "prompt_balanced"): + conditions = {} + for condition in CONDITIONS: + conditions[condition] = {} + for domain in DOMAIN_ORDER: + loads = np.asarray( + [ + row["conditions"][condition][load_key] + for row in prompt_rows + if row["domain"] == domain + ], + dtype=np.int64, + ) + conditions[condition][domain] = bootstrap_domain( + loads, + mode, + replicates, + seed, + ( + f"layer={layer_index}|scope={load_scope}|mode={mode}|" + f"condition={condition}|domain={domain}" + ), + ) + + comparisons = {} + for comparison, before_condition, after_condition in COMPARISONS: + comparisons[comparison] = {} + for domain in DOMAIN_ORDER: + rows = [row for row in prompt_rows if row["domain"] == domain] + before = np.asarray( + [ + row["conditions"][before_condition][load_key] + for row in rows + ], + dtype=np.int64, + ) + after = np.asarray( + [ + row["conditions"][after_condition][load_key] + for row in rows + ], + dtype=np.int64, + ) + comparisons[comparison][domain] = paired_domain( + before, + after, + mode, + replicates, + seed, + ( + f"layer={layer_index}|scope={load_scope}|mode={mode}|" + f"comparison={comparison}|domain={domain}" + ), + ) + modes[mode] = { + "conditions": conditions, + "comparisons": comparisons, + } + scopes[load_scope] = {"modes": modes} + return scopes + + +def main() -> None: + args = parse_args() + root = args.artifact_dir.resolve() + shard = root / "model-00001-of-000004.safetensors" + required = [ + root / "config.json", + root / "configuration_deepseek.py", + root / "modeling_deepseek.py", + root / "model.safetensors.index.json", + root / "tokenizer.json", + root / "tokenizer_config.json", + shard, + args.human_eval, + args.gsm8k, + args.tnews, + args.tnews_archive, + args.wikitext, + ] + missing = [str(path) for path in required if not path.exists()] + if missing: + raise FileNotFoundError(f"missing artifacts: {missing}") + if args.device.startswith("cuda") and not torch.cuda.is_available(): + raise RuntimeError("CUDA requested but unavailable") + if not 2 <= args.layers <= 7: + raise ValueError("need layer 0 plus at least one MoE layer; shard ends at layer 6") + if args.per_domain < 2: + raise ValueError("per-domain sample must be at least two") + if args.content_tokens < 4: + raise ValueError("content prefix must contain at least four tokens") + if args.batch_prompts < 1: + raise ValueError("batch-prompts must be positive") + if args.bootstrap < 100: + raise ValueError("bootstrap replicates must be at least 100") + if not args.sample_salt.strip(): + raise ValueError("sample salt must not be empty") + + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + torch.backends.cuda.matmul.allow_tf32 = False + + configuration, modeling = load_official_modules(root) + config = configuration.DeepseekV2Config.from_pretrained(root) + config._attn_implementation = "eager" + tokenizer = AutoTokenizer.from_pretrained( + root, + trust_remote_code=True, + local_files_only=True, + ) + if not tokenizer.is_fast: + raise RuntimeError("offset alignment requires a fast tokenizer") + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + tokenizer.padding_side = "right" + tokenizer_config = json.loads((root / "tokenizer_config.json").read_text()) + if tokenizer.chat_template != tokenizer_config.get("chat_template"): + raise RuntimeError("loaded chat template differs from tokenizer_config.json") + + candidates = load_candidates(args) + samples, corpus_counts = select_samples( + tokenizer, + candidates, + args.per_domain, + args.content_tokens, + args.sample_salt, + ) + batches = make_batches(samples, args.batch_prompts, tokenizer.pad_token_id) + device = torch.device(args.device) + + with safe_open(shard, framework="pt", device="cpu") as handle: + embedding = handle.get_tensor("model.embed_tokens.weight") + for batch in batches: + batch["hidden"] = F.embedding(batch["input_ids"], embedding) + del embedding + + layer_results = [] + for layer_index in range(args.layers): + prefix = f"model.layers.{layer_index}." + with safe_open(shard, framework="pt", device="cpu") as handle: + state = { + key[len(prefix) :]: handle.get_tensor(key) + for key in handle.keys() + if key.startswith(prefix) + } + state_numel = sum(value.numel() for value in state.values()) + state_bytes = sum(value.numel() * value.element_size() for value in state.values()) + with torch.device("meta"): + layer = modeling.DeepseekV2DecoderLayer(config, layer_index) + layer.to_empty(device="cpu") + layer.load_state_dict(state, strict=True, assign=True) + del state + layer = layer.to(device=device, dtype=torch.bfloat16).eval() + + prompt_rows = [] + next_hidden = [] + causal_invariant = { + "shared_prefix_tokens": 0, + "ordered_topk_exact": 0, + "violations": 0, + } + for batch in batches: + hidden = batch["hidden"].to(device) + attention_mask = batch["attention_mask"].to(device) + sequence = batch["padded_sequence"] + position_ids = torch.arange(sequence, device=device).unsqueeze(0) + causal_mask = modeling._prepare_4d_causal_attention_mask( + attention_mask, + batch["input_ids"].shape, + hidden, + 0, + ) + captures: dict[str, torch.Tensor] = {} + hook = None + if layer_index > 0: + + def capture_gate(_module: Any, _inputs: Any, output: Any) -> None: + captures["topk_ids"] = output[0].detach().cpu() + + hook = layer.mlp.gate.register_forward_hook(capture_gate) + + with torch.inference_mode(): + outputs = layer( + hidden, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=None, + use_cache=False, + ) + next_hidden.append(outputs[0].cpu()) + if hook is not None: + hook.remove() + topk = captures["topk_ids"].view( + len(batch["variants"]), + sequence, + config.num_experts_per_tok, + ) + by_prompt: dict[str, dict[str, tuple[dict[str, Any], torch.Tensor]]] = {} + for variant_index, variant in enumerate(batch["variants"]): + sample = variant["sample"] + condition = variant["condition"] + routes = topk[variant_index, : variant["tokens"]] + by_prompt.setdefault(sample["id"], {})[condition] = (variant, routes) + + for sample in batch["prompt_rows"]: + variants = by_prompt[sample["id"]] + condition_rows = {} + for condition in CONDITIONS: + variant, routes = variants[condition] + full_load = torch.bincount( + routes.flatten(), + minlength=config.n_routed_experts, + ) + content_routes = routes[ + variant["aligned_content_positions"] + ] + content_load = torch.bincount( + content_routes.flatten(), + minlength=config.n_routed_experts, + ) + condition_rows[condition] = { + "input_tokens": variant["tokens"], + "content_tokens": variant["content_tokens"], + "aligned_content_tokens": variant[ + "aligned_content_tokens" + ], + "wrapper_tokens": variant["wrapper_tokens"], + "boundary_crossing_tokens": variant[ + "boundary_crossing_tokens" + ], + "routes": int(full_load.sum()), + "content_routes": int(content_load.sum()), + "full_load": full_load.tolist(), + "content_load": content_load.tolist(), + "topk_sha256": canonical_hash(routes.tolist()), + "content_topk_sha256": canonical_hash( + content_routes.tolist() + ), + } + + raw_variant, raw_routes = variants["raw"] + user_variant, user_routes = variants["user"] + generation_variant, generation_routes = variants["generation"] + raw_user_pairs = aligned_pairs(raw_variant, user_variant) + raw_user = route_alignment( + raw_routes, + user_routes, + raw_user_pairs, + ) + prefix_length = common_prefix_length( + user_variant["token_ids"], + generation_variant["token_ids"], + ) + prefix_left = user_routes[:prefix_length] + prefix_right = generation_routes[:prefix_length] + exact_prefix = int( + torch.equal(prefix_left, prefix_right) + ) + causal_invariant["shared_prefix_tokens"] += prefix_length + causal_invariant["ordered_topk_exact"] += ( + prefix_length if exact_prefix else int( + (prefix_left == prefix_right).all(dim=1).sum() + ) + ) + causal_invariant["violations"] += int(not exact_prefix) + prompt_rows.append( + { + "id": sample["id"], + "domain": sample["domain"], + "conditions": condition_rows, + "alignments": { + "raw_to_user_content": { + **raw_user, + "raw_content_tokens": raw_variant[ + "content_tokens" + ], + "user_content_tokens": user_variant[ + "content_tokens" + ], + "raw_alignment_coverage": ( + raw_user["aligned_tokens"] + / raw_variant["content_tokens"] + ), + "user_alignment_coverage": ( + raw_user["aligned_tokens"] + / user_variant["content_tokens"] + ), + }, + "user_to_generation_prefix": { + "shared_prefix_tokens": prefix_length, + "user_tokens": user_variant["tokens"], + "generation_tokens": generation_variant["tokens"], + "ordered_topk_exact": exact_prefix == 1, + "ordered_topk_exact_tokens": ( + prefix_length if exact_prefix else int( + (prefix_left == prefix_right) + .all(dim=1) + .sum() + ) + ), + }, + }, + } + ) + del hidden, attention_mask, causal_mask, outputs + + for batch, hidden in zip(batches, next_hidden, strict=True): + batch["hidden"] = hidden + + result: dict[str, Any] = { + "layer": layer_index, + "ffn": "dense" if layer_index == 0 else "moe", + "state_numel": state_numel, + "state_bytes": state_bytes, + } + if layer_index > 0: + prompt_rows.sort( + key=lambda row: ( + DOMAIN_ORDER.index(row["domain"]), + row["id"], + ) + ) + causal_invariant["exact_rate"] = ( + causal_invariant["ordered_topk_exact"] + / causal_invariant["shared_prefix_tokens"] + ) + result["prompts"] = prompt_rows + result["causal_suffix_invariant"] = causal_invariant + result["statistics"] = layer_statistics( + prompt_rows, + args.bootstrap, + args.seed, + layer_index, + ) + layer_results.append(result) + + del layer, next_hidden + gc.collect() + if device.type == "cuda": + torch.cuda.empty_cache() + + captured_at = args.captured_at or datetime.now(timezone.utc).isoformat() + model_index = json.loads((root / "model.safetensors.index.json").read_text()) + selected_identity = [] + for sample in samples: + selected_identity.append( + { + "id": sample["id"], + "domain": sample["domain"], + "within_domain_index": sample["within_domain_index"], + "selection_rank": sample["selection_rank"], + "text_sha256": sample["text_sha256"], + "source_characters": sample["source_characters"], + "source_tokens": sample["source_tokens"], + "content_sha256": sample["content_sha256"], + "content_characters": sample["content_characters"], + "canonical_content_tokens": args.content_tokens, + "canonical_content_token_ids_sha256": sample[ + "canonical_content_token_ids_sha256" + ], + "conditions": { + condition: { + key: sample["variants"][condition][key] + for key in ( + "rendered_sha256", + "tokens", + "token_ids_sha256", + "content_tokens", + "aligned_content_tokens", + "wrapper_tokens", + "boundary_crossing_tokens", + ) + } + for condition in CONDITIONS + }, + "user_generation_common_prefix_tokens": common_prefix_length( + sample["variants"]["user"]["token_ids"], + sample["variants"]["generation"]["token_ids"], + ), + "raw_user_aligned_content_tokens": len( + aligned_pairs( + sample["variants"]["raw"], + sample["variants"]["user"], + ) + ), + } + ) + + result = { + "schema_version": 1, + "captured_at": captured_at, + "evidence_identity": ( + "X / official BF16 weights, official tokenizer chat template, " + "paired local truncated forward" + ), + "boundary": { + "model": "DeepSeek-V2-Lite base", + "executed_layers": list(range(args.layers)), + "measured_moe_layers": list(range(1, args.layers)), + "total_model_layers": config.num_hidden_layers, + "full_model_generation": False, + "task_performance": False, + "training_or_online_load": False, + "expert_semantics_inferred": False, + "causal_claim": ( + "the user-to-generation shared-prefix equality is a causal-mask " + "implementation invariant; raw-to-user differences are descriptive " + "protocol sensitivity, not a capability effect" + ), + "population": ( + f"{len(samples)} fixed public prompts across four domains; " + "not training data, online traffic, or a task benchmark" + ), + "code_execution": False, + "answers_used": False, + }, + "provenance": { + "model": { + "huggingface_model": "deepseek-ai/DeepSeek-V2-Lite", + "huggingface_revision": CHAT_TEMPLATE_REVISION, + "sha256": { + "config": sha256(root / "config.json"), + "modeling_code": sha256(root / "modeling_deepseek.py"), + "tokenizer": sha256(root / "tokenizer.json"), + "tokenizer_config": sha256(root / "tokenizer_config.json"), + "index": sha256(root / "model.safetensors.index.json"), + "shard_1": sha256(shard), + }, + "checkpoint_tensor_bytes": model_index["metadata"]["total_size"], + "shard_1_bytes": shard.stat().st_size, + }, + "corpora": { + "english": { + "name": "WikiText-2 raw validation", + "url": "https://huggingface.co/datasets/Salesforce/wikitext", + "revision": "b08601e04326c79dfdd32d625aee71d232d685c3", + "file_sha256": sha256(args.wikitext), + "field_used": "text", + }, + "chinese": { + "name": "CLUE TNEWS public test", + "url": "https://github.com/CLUEbenchmark/CLUE", + "download_url": "https://storage.googleapis.com/cluebenchmark/tasks/tnews_public.zip", + "revision": "9e61ddd3659ddb57ed82b4d0ba0a8613dfb55a2e", + "archive_sha256": sha256(args.tnews_archive), + "file_sha256": sha256(args.tnews), + "field_used": "sentence", + }, + "code": { + "name": "OpenAI HumanEval", + "url": "https://github.com/openai/human-eval", + "revision": git_revision(args.human_eval), + "file_sha256": sha256(args.human_eval), + "field_used": "prompt", + }, + "math": { + "name": "OpenAI GSM8K test", + "url": "https://github.com/openai/grade-school-math", + "revision": git_revision(args.gsm8k), + "file_sha256": sha256(args.gsm8k), + "field_used": "question", + }, + }, + }, + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "transformers": __import__("transformers").__version__, + "jinja2": __import__("jinja2").__version__, + "safetensors": __import__("safetensors").__version__, + "numpy": np.__version__, + "pyarrow": __import__("pyarrow").__version__, + "device": str(device), + "gpu": gpu_identity(device), + "matmul_allow_tf32": torch.backends.cuda.matmul.allow_tf32, + }, + "configuration": { + "layers": config.num_hidden_layers, + "hidden": config.hidden_size, + "routed_experts": config.n_routed_experts, + "active_routed_experts": config.num_experts_per_tok, + "shared_experts": config.n_shared_experts, + "first_dense_layers": config.first_k_dense_replace, + "router_scoring": config.scoring_func, + "router_topk_method": config.topk_method, + "normalize_selected_weights": config.norm_topk_prob, + }, + "template_contract": { + "chat_template_revision": CHAT_TEMPLATE_REVISION, + "chat_template": tokenizer.chat_template, + "chat_template_sha256": text_sha256(tokenizer.chat_template), + "bos_token": tokenizer.bos_token, + "bos_token_id": tokenizer.bos_token_id, + "eos_token": tokenizer.eos_token, + "eos_token_id": tokenizer.eos_token_id, + "conditions": { + condition: CONDITION_LABELS[condition] + for condition in CONDITIONS + }, + "comparisons": [ + { + "name": name, + "before": before, + "after": after, + } + for name, before, after in COMPARISONS + ], + "scope_split": { + "full_input": "all BOS, wrapper, content, newline, and generation-prompt tokens", + "content_only": ( + "the exact intersection of (relative character span, token ID) " + "inside source content across all three conditions; boundary-" + "crossing and unaligned tokens are excluded from every condition" + ), + }, + }, + "corpus_contract": { + "domains": list(DOMAIN_ORDER), + "domain_labels": DOMAIN_LABELS, + "sample_salt": args.sample_salt, + "selection": "ascending SHA256(salt|domain|source_id), then source_id", + "per_domain": args.per_domain, + "canonical_content_tokens": args.content_tokens, + "content_prefix": ( + "source text cut at the end offset of the fixed regular-tokenizer " + "content-token prefix" + ), + "counts": corpus_counts, + "selected": selected_identity, + }, + "inference_contract": { + "batch_prompts": args.batch_prompts, + "variants_per_prompt": len(CONDITIONS), + "rows_per_full_batch": args.batch_prompts * len(CONDITIONS), + "batches": len(batches), + "batch_grouping": ( + "all raw/user/generation variants of one source prompt execute " + "in the same padded batch" + ), + "attention": "official eager causal mask", + "dtype": "BF16", + "total_source_prompts": len(samples), + "total_prompt_variants": len(samples) * len(CONDITIONS), + "input_tokens_by_condition": { + condition: sum( + sample["variants"][condition]["tokens"] for sample in samples + ) + for condition in CONDITIONS + }, + "content_span_tokens_by_condition": { + condition: sum( + sample["variants"][condition]["content_tokens"] + for sample in samples + ) + for condition in CONDITIONS + }, + "aligned_content_tokens_by_condition": { + condition: sum( + sample["variants"][condition]["aligned_content_tokens"] + for sample in samples + ) + for condition in CONDITIONS + }, + "routes_per_condition_all_moe_layers": { + condition: sum( + sample["variants"][condition]["tokens"] for sample in samples + ) + * config.num_experts_per_tok + * (args.layers - 1) + for condition in CONDITIONS + }, + "total_routes_all_conditions_all_moe_layers": sum( + sum(sample["variants"][condition]["tokens"] for sample in samples) + * config.num_experts_per_tok + * (args.layers - 1) + for condition in CONDITIONS + ), + }, + "statistical_contract": { + "unit": "source prompt", + "bootstrap_replicates": args.bootstrap, + "seed": args.seed, + "paired_indices": ( + "the same resampled source-prompt indices are used for before and " + "after within each domain/layer/scope/mode" + ), + "modes": { + "token_weighted": "sum selected routes, then normalize", + "prompt_balanced": ( + "normalize each prompt load, then average prompts equally" + ), + }, + "interval": "2.5th and 97.5th percentiles", + "multiple_comparison_correction": False, + "hypothesis_test": False, + }, + "layers": layer_results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md b/research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md new file mode 100644 index 0000000..66bddc0 --- /dev/null +++ b/research/DEEPSEEK_ROUTING_TEMPLATE_AUDIT.md @@ -0,0 +1,438 @@ +# DeepSeek-V2-Lite 路由的官方模板敏感性审计 + +> 状态：真实官方 BF16 权重、本机 RTX 5090、layer 0–6 连续 forward。 +> 固定模型 revision：`604d5664dddd88a0433dbae533b7fe9472482de0`。 +> 研究单位：同一批 128 条公开 source prompt 的三种输入协议。 +> 结论身份：**协议敏感性探针**，不是能力评测、训练分布、线上流量或专家语义。 + +## 1. 这轮究竟要排除什么混杂因素 + +上一轮已经证明：自然长度不同会改变路由集中度，因此我们不能把所有域间差异都 +解释成“内容语义”。 + +但即使内容和长度都固定，输入模型之前还有一层协议： + +```text +纯文本 + ↓ tokenizer +BOS + content + +单轮对话 + ↓ official chat template + tokenizer +BOS + "User: " + content + "\n\n" + optional "Assistant:" +``` + +角色前缀会： + +1. 增加 wrapper token； +2. 改变内容 token 的绝对位置； +3. 为每个内容 token 增加可见的左侧上下文； +4. 可能让第一个内容 token 在 BPE 边界处重新切分； +5. 改变“整段输入统计”的分母。 + +所以本轮不是问“chat 模型强不强”，而是问： + +> 同一段内容送入同一个 base checkpoint，仅改变官方输入包装时，前六个 MoE +> 层的路由统计会怎样变化？ + +## 2. 官方模板不是自行编写的教学字符串 + +固定 revision 的 `tokenizer_config.json` 给出： + +```jinja +{{ bos_token }} +{% for message in messages %} + user -> "User: " + content + "\n\n" + assistant -> "Assistant: " + content + eos_token + system -> content + "\n\n" +{% endfor %} +{% if add_generation_prompt %}"Assistant:"{% endif %} +``` + +完整模板字符串 SHA-256： + +```text +8aeba567270fa9a8d5372caf4b04affea947c4421d96adc01e3ff94021b0ae8e +``` + +特殊 token： + +| 角色 | 文本 | ID | +| --- | --- | ---: | +| BOS | `<｜begin▁of▁sentence｜>` | 100000 | +| EOS / PAD | `<｜end▁of▁sentence｜>` | 100001 | + +实验通过 Transformers 4.41.2 的 `apply_chat_template` 生成字符串和 token IDs； +随后再次对渲染字符串编码，并要求两条路径的 token IDs 完全一致。 + +## 3. 三个条件如何构成“实验 + 负对照” + +每条 source prompt 都渲染三遍： + +```text +RAW +[BOS] [content] + +USER +[BOS] [User:] [content] [\n\n] + +GENERATION +[BOS] [User:] [content] [\n\n] [Assistant:] +``` + +比较分两种： + +### 3.1 `RAW → USER` + +这是主要敏感性实验。 + +- 内容字符前缀相同； +- checkpoint、权重、batch、层、dtype 相同； +- 改变的是左侧角色前缀和尾部换行； +- 尾部换行不能反向影响更早的内容 token； +- `User:` 前缀可以通过因果注意力影响后续内容 token。 + +它能描述“输入协议敏感性”，但不能单独证明训练中的角色语义或能力变化。 + +### 3.2 `USER → GENERATION` + +这是因果负对照。 + +`GENERATION` 只在 `USER` 末尾追加 `Assistant:`。在严格因果 attention 中， +未来 suffix 不应改变此前共享前缀的 hidden state 或路由。 + +因此它同时验证： + +- causal mask； +- batching； +- token prefix； +- route 捕获； +- 分析管线。 + +若这个负对照失败，`RAW → USER` 的任何解释都不可信。 + +## 4. 固定 cohort 与内容长度合同 + +四个公开域仍为： + +| 域 | 来源 | 使用字段 | 数量 | +| --- | --- | --- | ---: | +| 英文百科 | WikiText-2 raw validation | `text` | 32 | +| 中文新闻 | CLUE TNEWS public test | `sentence` | 32 | +| Python 代码 | OpenAI HumanEval | `prompt` | 32 | +| 小学数学 | OpenAI GSM8K test | `question` | 32 | + +答案不输入，HumanEval 代码不执行。 + +### 4.1 为什么固定 23 个 content tokens + +上一轮 24-token matched cohort 的总长包括 BOS： + +```text +1 BOS + 23 source-content tokens = 24 raw input tokens +``` + +本轮直接把内容合同写清楚为 **23 个 regular-tokenizer content tokens**。 +这样 RAW 条件固定为 24 tokens，同时保留足够多的 TNEWS 候选。 + +### 4.2 字符前缀如何重建 + +BPE 对“完整字符串”和“截断后的字符串”可能在最后一个 token 发生不同合并。 +因此不能粗暴地取完整编码的前 23 个 IDs 再假设可逆。 + +脚本采用： + +1. 读取 fast tokenizer offset mapping； +2. 在目标 token 边界附近搜索字符终点； +3. 重新编码字符前缀； +4. 只接受恰好得到 23 tokens 的最长前缀。 + +选样排序固定为： + +```text +ascending SHA256( + "llm-atlas-deepseek-routing-template-control-v1" + | domain + | source_id +) +``` + +候选与选中来源长度： + +| 域 | 可用候选 | 选中 | source tokens min / mean / max | +| --- | ---: | ---: | ---: | +| 英文百科 | 1,655 | 32 | 29 / 142.84 / 280 | +| 中文新闻 | 1,609 | 32 | 23 / 24.66 / 30 | +| Python 代码 | 164 | 32 | 42 / 141.78 / 352 | +| 小学数学 | 1,319 | 32 | 31 / 59.88 / 131 | + +TNEWS 在完成选中前有 1 条无法构造精确 23-token 字符前缀，脚本显式记录并继续按 +哈希顺序选择下一条；其余域为 0。 + +## 5. 为什么必须分“整段输入”和“对齐内容” + +若直接把 wrapper token 与 content token 全部求和，得到的是部署协议问题： + +> 这一整段实际输入会把 token 路由到哪里？ + +若只研究内容，需要保证比较的是同一 token，而不是“都在内容字符范围内”就算相同。 + +本轮为三种条件中的每个 token 记录： + +```text +(相对内容字符起点, 相对内容字符终点, token ID) +``` + +然后只保留三个条件的精确交集。 + +### 5.1 Token 账 + +| 条件 | 输入 tokens | 对齐内容 tokens | 六层真实 top-6 routes | +| --- | ---: | ---: | ---: | +| RAW | 3,072 | 2,874 | 110,592 | +| USER | 3,642 | 2,874 | 131,112 | +| GENERATION | 3,898 | 2,874 | 140,328 | +| 合计 | 10,612 | — | **382,032** | + +RAW 原始内容范围共有 2,944 tokens；由于 `User: ` 后的 BPE 边界，70 个首 token +不能以相同 `(span, token ID)` 对齐，因此从三个条件中同时剔除。 + +按域的对齐覆盖： + +| 域 | aligned / RAW content | 覆盖率 | +| --- | ---: | ---: | +| 英文百科 | 706 / 736 | 95.9% | +| 中文新闻 | 735 / 736 | 99.9% | +| Python 代码 | 733 / 736 | 99.6% | +| 小学数学 | 700 / 736 | 95.1% | + +这意味着 `content_only` 比较中的每一条 route 都对应相同字符段与相同 token ID。 + +## 6. 统计合同 + +与长度实验相同，抽样单位仍是 source prompt，而不是 token。 + +每个 layer × scope × mode × domain： + +1. 32 条 prompt； +2. 有放回重采样 2,000 次； +3. before / after 使用完全相同的 prompt indices； +4. 固定 seed `20260729`； +5. 报告 percentile 95% 区间； +6. 不进行假设检验； +7. 不做多重比较校正。 + +聚合口径： + +- `prompt_balanced`：每条 prompt 先归一，再让 32 条 prompt 等权； +- `token_weighted`：先汇总选中 token routes，再整体归一。 + +主要正文使用 `prompt_balanced`，避免少量 tokenization 长度差异改变 prompt 权重。 + +## 7. 负对照：21,852 个共享前缀 token 全部 exact + +每个 MoE 层： + +| Layer | USER 共享前缀 tokens | ordered top-6 exact | 违规 prompt | +| --- | ---: | ---: | ---: | +| L1 | 3,642 | 3,642 | 0 | +| L2 | 3,642 | 3,642 | 0 | +| L3 | 3,642 | 3,642 | 0 | +| L4 | 3,642 | 3,642 | 0 | +| L5 | 3,642 | 3,642 | 0 | +| L6 | 3,642 | 3,642 | 0 | +| 合计 | **21,852** | **21,852** | **0** | + +`content_only` 的 USER→GENERATION 在 6 层 × 4 域共 24 个比较中： + +```text +CV Δ = 0 +total variation = 0 +JSD = 0 +``` + +但若看 `full_input`，因为 GENERATION 确实新增了 `Assistant:` token，prompt-balanced +total variation 为 0.037–0.051，平均 0.0445。 + +这组结果正好说明： + +> suffix 没有改写过去；整段统计改变，是因为统计对象加入了新 token。 + +## 8. 主结果：同一个 `User:` 前缀在不同深度方向不同 + +以下使用： + +- scope：三个条件精确对齐的 content tokens； +- mode：prompt-balanced； +- Δ：`CV(USER) − CV(RAW)`。 + +| Layer | English | Chinese | Code | Math | +|---|---:|---:|---:|---:| +| L1 | 0.633→0.593, **−0.040** [−0.063, −0.017] | 0.670→0.621, **−0.049** [−0.068, −0.028] | 0.901→0.882, −0.019 [−0.039, 0.001] | 0.817→0.796, −0.021 [−0.049, 0.006] | +| L2 | 0.404→0.405, 0.001 [−0.014, 0.016] | 0.383→0.353, **−0.030** [−0.041, −0.016] | 0.684→0.617, **−0.067** [−0.084, −0.047] | 0.453→0.455, 0.002 [−0.016, 0.016] | +| L3 | 0.405→0.422, 0.017 [−0.002, 0.033] | 0.415→0.471, **0.057** [0.033, 0.075] | 0.824→0.812, −0.011 [−0.023, 0.001] | 0.541→0.547, 0.006 [−0.011, 0.023] | +| L4 | 0.530→0.555, 0.025 [−0.002, 0.049] | 0.897→0.848, **−0.048** [−0.067, −0.030] | 1.064→1.038, **−0.027** [−0.039, −0.016] | 0.668→0.649, −0.020 [−0.040, 0.000] | +| L5 | 0.446→0.451, 0.005 [−0.015, 0.021] | 0.478→0.504, **0.026** [0.006, 0.042] | 0.808→0.850, **0.043** [0.032, 0.055] | 0.770→0.779, 0.008 [−0.005, 0.021] | +| L6 | 0.502→0.562, **0.060** [0.036, 0.076] | 0.509→0.539, **0.030** [0.009, 0.043] | 0.851→0.909, **0.058** [0.046, 0.068] | 0.738→0.775, **0.037** [0.013, 0.058] | + +### 8.1 初学者应该怎样读 + +CV 越高，64 个 routed experts 的份额越不平。 + +- L1：英文、中文的对齐内容变得更平； +- L2：中文、代码变得更平； +- L3：中文反而更集中； +- L4：中文、代码又变平； +- L5：中文、代码更集中； +- L6：四域全部更集中。 + +因此不应写成： + +> Chat template 会让路由更均衡。 + +也不应写成： + +> Chat template 会让路由更集中。 + +更准确的说法是： + +> 在这个 23-token 固定探针中，角色前缀对路由集中度的影响随层和域改变方向； +> 低层与高层不能用一个单调结论概括。 + +## 9. “整段输入”会给出另一幅图 + +若把 BOS、`User:`、换行与 content 全部计算，Δ CV 为： + +| Layer | English | Chinese | Code | Math | +|---|---:|---:|---:|---:| +| L1 | 0.061 [0.041, 0.077] | 0.093 [0.062, 0.112] | 0.014 [−0.009, 0.035] | 0.058 [0.029, 0.083] | +| L2 | 0.093 [0.069, 0.106] | 0.138 [0.106, 0.151] | 0.038 [0.020, 0.052] | 0.098 [0.065, 0.122] | +| L3 | 0.100 [0.072, 0.110] | 0.140 [0.096, 0.159] | 0.078 [0.058, 0.092] | 0.058 [0.031, 0.076] | +| L4 | 0.131 [0.093, 0.152] | −0.012 [−0.033, 0.003] | 0.051 [0.035, 0.063] | 0.066 [0.042, 0.083] | +| L5 | 0.083 [0.050, 0.094] | 0.036 [−0.003, 0.054] | 0.092 [0.070, 0.112] | −0.053 [−0.075, −0.036] | +| L6 | 0.134 [0.090, 0.155] | 0.046 [0.000, 0.069] | 0.103 [0.079, 0.118] | 0.027 [−0.004, 0.047] | + +多数格子是正值，与对齐内容的混合方向明显不同。 + +原因不是统计错误，而是问题不同： + +- `full_input` 包含重复出现的 wrapper token； +- `content_only` 只比较相同字符段和 token ID； +- wrapper 自己可能有很强、很稳定的路由偏好； +- wrapper 数量占短输入的比例不小。 + +这也是为什么评测协议、chat template 和 tokenizer 必须进入实验账本，而不能只写模型名。 + +## 10. 逐 token top-6 有多稳定 + +对 RAW 与 USER 中能按 `(relative span, token ID)` 对齐的内容 token，报告： + +- `set`：top-6 专家集合完全相同的比例； +- `J`：top-6 专家集合的平均 Jaccard。 + +| Layer | English | Chinese | Code | Math | +|---|---:|---:|---:|---:| +| L1 | set 0.552 · J 0.854 | set 0.600 · J 0.867 | set 0.625 · J 0.867 | set 0.459 · J 0.822 | +| L2 | set 0.606 · J 0.874 | set 0.604 · J 0.869 | set 0.562 · J 0.846 | set 0.577 · J 0.860 | +| L3 | set 0.625 · J 0.881 | set 0.634 · J 0.875 | set 0.570 · J 0.864 | set 0.644 · J 0.890 | +| L4 | set 0.572 · J 0.865 | set 0.590 · J 0.869 | set 0.562 · J 0.831 | set 0.593 · J 0.873 | +| L5 | set 0.540 · J 0.849 | set 0.550 · J 0.841 | set 0.551 · J 0.824 | set 0.610 · J 0.872 | +| L6 | set 0.581 · J 0.859 | set 0.558 · J 0.839 | set 0.529 · J 0.814 | set 0.544 · J 0.846 | + +两个数字可以同时成立： + +- 只有约 46%–64% 的 token 保持完全相同的 top-6 集合； +- 但平均 Jaccard 仍为 0.814–0.890。 + +直觉上，这说明很多变化不是“六个专家全部换掉”，而是 top-6 边缘的一两个专家发生替换。 +它仍不能告诉我们专家“负责什么语义”。 + +## 11. 分布距离不大，不等于逐 token 没变化 + +RAW→USER 的对齐内容、prompt-balanced： + +- total variation 点估计约为 0.024–0.067； +- JSD 点估计约为 0.001–0.008 nats。 + +聚合分布看起来接近，但上一节仍有大量 token 的 top-6 集合不完全相同。 + +这是一个重要的测量层级区别： + +```text +aggregate distribution close + ≠ +every token took the same route +``` + +相反，也不能从 token route 变化推出输出答案或任务分数必然变化；本实验没有执行完整模型生成。 + +## 12. 独立复跑 + +正式输出： + +```text +src/data/deepseek-v2-lite-routing-template.json +src/data/deepseek-v2-lite-routing-template-repro.json +``` + +两份完整 JSON： + +```text +SHA-256 +da1f10333b2fa269e64f9716a0ca6c1a656d23d3e70b8a25d6e59f8a5b3bc1b9 +``` + +`cmp` byte-exact。 + +复现命令： + +```bash +PYTHONPATH=/path/to/transformers-4.41.2-deps:/usr/lib/python3/dist-packages \ +python -B experiments/deepseek/v2_lite_routing_template_probe.py \ + --artifact-dir /path/to/deepseek-v2-lite \ + --human-eval /path/to/HumanEval.jsonl.gz \ + --gsm8k /path/to/gsm8k/test.jsonl \ + --tnews /path/to/tnews/test.json \ + --tnews-archive /path/to/tnews_public.zip \ + --wikitext /path/to/wikitext-validation.parquet \ + --output /path/to/template-probe.json \ + --per-domain 32 \ + --content-tokens 23 \ + --batch-prompts 8 \ + --layers 7 \ + --bootstrap 2000 \ + --seed 20260729 \ + --captured-at 2026-07-29T08:30:00+00:00 +``` + +Jinja2 是 `apply_chat_template` 的运行依赖；正式环境记录为 3.1.6。 + +## 13. 可以说什么，不能说什么 + +### 可以说 + +- 固定官方 revision 的 chat template 会改变输入 token 序列； +- 相同字符段、相同 token ID 的内容路由对角色前缀敏感； +- 敏感性随层和域改变方向； +- 追加在未来的 `Assistant:` 不改变此前共享前缀，六层 21,852 / 21,852 exact； +- wrapper-inclusive 与 aligned-content 统计回答不同问题； +- 正式运行与独立复跑 byte-exact。 + +### 不能说 + +- `User:` 专门激活某类“对话专家”； +- top-6 ID 的变化等于专家语义变化； +- 路由更平或更集中等于模型能力更强； +- 这个 base checkpoint 的 route probe 等于 chat model 的线上流量； +- 前六层等于完整 27 层； +- 23-token 前缀代表长对话； +- 不生成答案的实验可以给出任务准确率结论； +- 24 个 layer×domain 区间是已校正的显著性检验。 + +## 14. 下一道闸门 + +1. 下载并执行完整 layer 0–26，检查“低层变平、高层变集中”的模式是否延续； +2. 加入 system role、few-shot turn 与换行边界的正交扰动； +3. 把模板敏感性接到完整生成与输出指标，仍保持 route / output 两张账； +4. 在支持环境执行 FlashMLA optimized kernel； +5. 继续 FP8 / pipeline traces 与 R1-like 小模型复现。 diff --git a/scripts/check-deepseek-browser.mjs b/scripts/check-deepseek-browser.mjs index cd8754d..90a5ed2 100644 --- a/scripts/check-deepseek-browser.mjs +++ b/scripts/check-deepseek-browser.mjs @@ -80,6 +80,9 @@ const overview = await evaluate(`(() => ({ followups: document.querySelectorAll(".lineage-row.followup").length, navLinks: document.querySelectorAll(".top-nav a").length, activeNav: document.querySelector('.top-nav a[aria-current="page"]')?.textContent.trim(), + heroLabs: [...document.querySelectorAll(".page-facts > div")] + .find((node) => node.querySelector("dt")?.textContent.trim() === "LABS") + ?.querySelector("dd")?.textContent.trim(), documentOverflow: document.documentElement.scrollWidth - document.documentElement.clientWidth, }))()`); @@ -344,6 +347,57 @@ await evaluate(`(() => { await pause(120); await screenshot("/tmp/llm-atlas-deepseek-length-sensitivity-desktop.png"); +const artifactTemplate = await evaluate(`(() => { + const root = document.querySelector("[data-dsv2-lab]"); + root.querySelector('[data-artifact-tab="template"]').click(); + const read = () => ({ + panel: root.querySelector("[data-artifact-panel]:not([hidden])").dataset.artifactPanel, + domainCards: root.querySelectorAll("[data-template-domain-grid] > article").length, + domains: [...root.querySelectorAll("[data-template-domain-grid] > article")].map((node) => ({ + label: node.querySelector("span").textContent.trim(), + values: node.querySelector("b").textContent.trim(), + delta: node.querySelector("strong").textContent.trim(), + className: node.querySelector("strong").className, + ci: node.querySelector("p").textContent.trim(), + distance: node.querySelector("small").textContent.trim(), + stability: node.querySelector("em").textContent.trim(), + })), + prefixExact: root.querySelector("[data-template-prefix-exact]").textContent.trim(), + contentZero: root.querySelector("[data-template-content-zero]").textContent.trim(), + suffixTv: root.querySelector("[data-template-suffix-tv]").textContent.trim(), + note: root.querySelector("[data-template-note]").textContent.trim(), + depthRows: root.querySelectorAll("[data-template-depth-map] > div").length, + depthCells: root.querySelectorAll("[data-template-depth-map] > div > span").length, + exact: root.querySelector(".template-ledger .exact b").textContent.trim(), + activeLayer: root.querySelector("[data-template-layer].active").textContent.trim(), + activeScope: root.querySelector('[data-template-scope][aria-pressed="true"]').dataset.templateScope, + activeMode: root.querySelector('[data-template-mode][aria-pressed="true"]').dataset.templateMode, + }); + const layer1Content = read(); + root.querySelector('[data-template-layer="6"]').click(); + const layer6Content = read(); + root.querySelector('[data-template-scope="full_input"]').click(); + const layer6Full = read(); + root.querySelector('[data-template-mode="token_weighted"]').click(); + const layer6FullToken = read(); + root.querySelector('[data-template-scope="content_only"]').click(); + root.querySelector('[data-template-mode="prompt_balanced"]').click(); + root.querySelector('[data-template-layer="2"]').click(); + const layer2Content = read(); + return { layer1Content, layer6Content, layer6Full, layer6FullToken, layer2Content }; +})()`); +await evaluate(`(() => { + document.querySelector("[data-dsv2-lab]").scrollIntoView({ block: "start", behavior: "instant" }); + window.scrollBy(0, -82); +})()`); +await pause(180); +await screenshot("/tmp/llm-atlas-deepseek-template-desktop.png"); +await evaluate(`(() => { + document.querySelector(".template-domain-grid").scrollIntoView({ block: "center", behavior: "instant" }); +})()`); +await pause(120); +await screenshot("/tmp/llm-atlas-deepseek-template-results-desktop.png"); + const artifactEvidence = await evaluate(`(() => { const root = document.querySelector("[data-dsv2-lab]"); root.querySelector('[data-artifact-tab="evidence"]').click(); @@ -417,6 +471,11 @@ const mobile = await evaluate(`(() => { artifactHeatCells: artifact.querySelectorAll("[data-route-heatmap] > span").length, corpusCohorts: artifact.querySelectorAll("[data-corpus-cohort]").length, lengthDeltaCards: artifact.querySelectorAll("[data-length-delta-grid] > article").length, + templateLayers: artifact.querySelectorAll("[data-template-layer]").length, + templateScopes: artifact.querySelectorAll("[data-template-scope]").length, + templateModes: artifact.querySelectorAll("[data-template-mode]").length, + templateDomainCards: artifact.querySelectorAll("[data-template-domain-grid] > article").length, + templateDepthCells: artifact.querySelectorAll("[data-template-depth-map] > div > span").length, offenders: [...document.querySelectorAll("body *")] .filter((node) => !node.closest(".paper-chain, .advantage-table, .precision-table, .mapping-table, [data-deepseek-lab], [data-dsv2-lab]")) .filter((node) => node.getBoundingClientRect().right > document.documentElement.clientWidth + 1) @@ -444,8 +503,22 @@ await evaluate(`(() => { })()`); await pause(120); await screenshot("/tmp/llm-atlas-deepseek-length-sensitivity-mobile.png"); +await evaluate(`(() => { + const artifact = document.querySelector("[data-dsv2-lab]"); + artifact.querySelector('[data-artifact-tab="template"]').click(); + artifact.scrollIntoView({ block: "start", behavior: "instant" }); + window.scrollBy(0, -70); +})()`); +await pause(180); +await screenshot("/tmp/llm-atlas-deepseek-template-mobile.png"); +await evaluate(`(() => { + document.querySelector(".template-domain-grid").scrollIntoView({ block: "start", behavior: "instant" }); + window.scrollBy(0, -72); +})()`); +await pause(120); +await screenshot("/tmp/llm-atlas-deepseek-template-results-mobile.png"); -const report = { overview, capacity, cache, codesign, rl, artifactRoute, artifactLoad, artifactCache, artifactAbsorb, artifactCorpus, artifactEvidence, home, papers, mobile, exceptions }; +const report = { overview, capacity, cache, codesign, rl, artifactRoute, artifactLoad, artifactCache, artifactAbsorb, artifactCorpus, artifactTemplate, artifactEvidence, home, papers, mobile, exceptions }; console.log(JSON.stringify(report, null, 2)); const numeric = (text) => Number.parseFloat(text.replaceAll(",", "")); @@ -455,7 +528,8 @@ if (overview.sections !== 26 || overview.tocLinks !== 26) failures.push("二十 if (overview.ledgers !== 24 || overview.waves !== 10) failures.push("二十四张问题账或十次转向结构异常"); if (overview.paperLinks !== 60 || overview.branches !== 5 || overview.followups !== 1) failures.push("论文链、旁支或公开后续标记异常"); if (overview.labTabs !== 4 || overview.labPanels !== 4) failures.push("四联实验结构异常"); -if (overview.artifactTabs !== 6 || overview.artifactPanels !== 6 || overview.artifactLayers !== 27) failures.push("真实权重六联实验结构异常"); +if (overview.artifactTabs !== 7 || overview.artifactPanels !== 7 || overview.artifactLayers !== 27) failures.push("真实权重七联实验结构异常"); +if (overview.heroLabs !== "11 个可操作实验") failures.push("DeepSeek 实验总数账异常"); if (overview.navLinks !== 20 || home.navLinks !== 20 || mobile.mobileLinks !== 20 || overview.activeNav !== "DeepSeek") failures.push("全站导航未同步 DeepSeek"); if (overview.documentOverflow > 1 || mobile.documentOverflow > 1) failures.push("桌面或移动端存在文档级横向溢出"); if (capacity.initial.panel !== "capacity" || capacity.initial.total !== "32.1× FFN" || capacity.initial.active !== "1.13× FFN") failures.push("V3 稀疏容量初始账异常"); @@ -489,12 +563,18 @@ if (!artifactCorpus.matched16.highest.includes("Python 代码 · 0.969") || arti if (!artifactCorpus.matched24.highest.includes("Python 代码 · 0.718") || artifactCorpus.matched24.cohortTitle !== "同样本 · 24 tokens" || !artifactCorpus.matched24.tokens.every((value) => value.includes("768 tokens"))) failures.push("24-token 同源 cohort 切换异常"); if (artifactCorpus.matched24.deltaCards !== 4 || artifactCorpus.matched24.lengthLargest !== "Python 代码 · Δ -0.251" || artifactCorpus.matched24.lengthJsd !== "0.091 → 0.065 · Δ -0.026") failures.push("16→24 token 成对敏感性结论异常"); if (!artifactCorpus.tokenWeighted.heatTitle.includes("token 加权") || !artifactCorpus.tokenWeighted.modeNote.includes("理论上重合")) failures.push("等长 cohort 聚合口径切换异常"); +if (artifactTemplate.layer1Content.panel !== "template" || artifactTemplate.layer1Content.domainCards !== 4 || artifactTemplate.layer1Content.depthRows !== 4 || artifactTemplate.layer1Content.depthCells !== 24 || artifactTemplate.layer1Content.exact !== "BYTE-EXACT") failures.push("官方模板扰动结构或独立复跑闸门异常"); +if (artifactTemplate.layer1Content.domains[0].values !== "0.633 → 0.593" || artifactTemplate.layer1Content.domains[0].delta !== "Δ -0.040" || artifactTemplate.layer1Content.domains[1].delta !== "Δ -0.049") failures.push("L1 对齐内容模板敏感性统计异常"); +if (artifactTemplate.layer1Content.prefixExact !== "3,642 / 3,642 EXACT · L1" || artifactTemplate.layer1Content.contentZero !== "4 / 4 DOMAINS · Δ 0") failures.push("L1 causal suffix 负对照异常"); +if (artifactTemplate.layer6Content.domains.some((domain) => domain.className !== "up") || artifactTemplate.layer6Content.domains[0].delta !== "Δ +0.060" || artifactTemplate.layer6Content.domains[2].delta !== "Δ +0.058") failures.push("L6 对齐内容跨域方向异常"); +if (artifactTemplate.layer6Full.domains[0].delta !== "Δ +0.134" || !artifactTemplate.layer6Full.note.includes("完整输入") || artifactTemplate.layer6Full.activeScope !== "full_input") failures.push("模板完整输入 scope 切换异常"); +if (artifactTemplate.layer6FullToken.activeMode !== "token_weighted" || artifactTemplate.layer2Content.activeLayer !== "L2" || artifactTemplate.layer2Content.domains[2].delta !== "Δ -0.067") failures.push("模板层或聚合口径切换异常"); if (artifactEvidence.panel !== "evidence" || artifactEvidence.layers !== 27 || artifactEvidence.executed !== 7 || artifactEvidence.split !== 1 || artifactEvidence.unloaded !== 19 || artifactEvidence.exact !== "31 / 31") failures.push("真实工件执行边界或复跑闸门异常"); if (!artifactEvidence.dependency.includes("Transformers 5.5") || !artifactEvidence.dependency.includes("4.41.2") || !artifactEvidence.boundary.includes("完整 27 层生成")) failures.push("依赖版本或未覆盖边界异常"); if (artifactEvidence.keyboardSelected !== "load" || artifactEvidence.keyboardVisible !== "load") failures.push("真实工件实验键盘 tab 导航异常"); if (home.releaseCards !== 17 || !home.firstRelease.includes("47 页不再压成摘要") || home.firstHref !== "/k3/" || home.paperCount !== "486") failures.push("首页 DeepSeek 首发入口或论文数异常"); if (papers.total !== 486 || !papers.hasFilter || papers.visible < 20 || !papers.hasCoder || !papers.hasEngram) failures.push("论文库 DeepSeek 聚光异常"); -if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4 || mobile.artifactTabs !== 6 || mobile.artifactHeatCells !== 64 || mobile.corpusCohorts !== 3 || mobile.lengthDeltaCards !== 4) failures.push("移动端导航或实验异常"); +if (!mobile.menuVisible || mobile.menuOpen !== "true" || mobile.tabs !== 4 || mobile.artifactTabs !== 7 || mobile.artifactHeatCells !== 64 || mobile.corpusCohorts !== 3 || mobile.lengthDeltaCards !== 4 || mobile.templateLayers !== 6 || mobile.templateScopes !== 2 || mobile.templateModes !== 2 || mobile.templateDomainCards !== 4 || mobile.templateDepthCells !== 24) failures.push("移动端导航或实验异常"); if (mobile.offenders.length) failures.push(`移动端越界元素：${JSON.stringify(mobile.offenders)}`); if (exceptions.length) failures.push(`浏览器异常：${exceptions.join(" | ")}`); diff --git a/src/components/DeepSeekArtifactLab.astro b/src/components/DeepSeekArtifactLab.astro index d6ed95d..4ad30f9 100644 --- a/src/components/DeepSeekArtifactLab.astro +++ b/src/components/DeepSeekArtifactLab.astro @@ -10,6 +10,8 @@ import rawMatched16Repro from "@/data/deepseek-v2-lite-routing-matched16-repro.j import rawMatched24 from "@/data/deepseek-v2-lite-routing-matched24.json"; import rawMatched24Repro from "@/data/deepseek-v2-lite-routing-matched24-repro.json"; import rawLengthSensitivity from "@/data/deepseek-v2-lite-routing-length-sensitivity.json"; +import rawTemplate from "@/data/deepseek-v2-lite-routing-template.json"; +import rawTemplateRepro from "@/data/deepseek-v2-lite-routing-template-repro.json"; const trace = rawTrace as any; const repro = rawRepro as any; @@ -22,10 +24,13 @@ const matched16Repro = rawMatched16Repro as any; const matched24 = rawMatched24 as any; const matched24Repro = rawMatched24Repro as any; const lengthSensitivity = rawLengthSensitivity as any; +const template = rawTemplate as any; +const templateRepro = rawTemplateRepro as any; const absorbExact = JSON.stringify(absorb) === JSON.stringify(absorbRepro); const corpusExact = JSON.stringify(corpus) === JSON.stringify(corpusRepro); const matched16Exact = JSON.stringify(matched16) === JSON.stringify(matched16Repro); const matched24Exact = JSON.stringify(matched24) === JSON.stringify(matched24Repro); +const templateExact = JSON.stringify(template) === JSON.stringify(templateRepro); const bytes = (value: number) => value >= 1024 ? `${(value / 1024).toFixed(2)} KiB` : `${value.toLocaleString()} B`; @@ -82,6 +87,63 @@ const corpusCompact = { lengthSensitivity, }; const corpusCompactJson = JSON.stringify(corpusCompact).replaceAll("<", "\\u003c"); +const aggregateTemplateAlignment = (layer: any, domain: string) => { + const rows = layer.prompts + .filter((prompt: any) => prompt.domain === domain) + .map((prompt: any) => prompt.alignments.raw_to_user_content); + const aligned = rows.reduce((sum: number, row: any) => sum + row.aligned_tokens, 0); + const setExact = rows.reduce((sum: number, row: any) => sum + row.set_topk_exact, 0); + const orderedExact = rows.reduce((sum: number, row: any) => sum + row.ordered_topk_exact, 0); + const weightedJaccard = rows.reduce( + (sum: number, row: any) => sum + row.mean_jaccard * row.aligned_tokens, + 0, + ); + return { + aligned, + setExactRate: setExact / aligned, + orderedExactRate: orderedExact / aligned, + meanJaccard: weightedJaccard / aligned, + }; +}; +const templateCompact = { + domains: template.corpus_contract.domains, + labels: template.corpus_contract.domain_labels, + inference: template.inference_contract, + template: { + sha256: template.template_contract.chat_template_sha256, + bos: template.template_contract.bos_token_id, + }, + exact: templateExact, + layers: template.layers.slice(1).map((layer: any) => ({ + layer: layer.layer, + invariant: layer.causal_suffix_invariant, + alignment: Object.fromEntries( + template.corpus_contract.domains.map((domain: string) => [ + domain, + aggregateTemplateAlignment(layer, domain), + ]), + ), + scopes: Object.fromEntries( + ["content_only", "full_input"].map((scope) => [ + scope, + { + modes: Object.fromEntries( + ["prompt_balanced", "token_weighted"].map((mode) => [ + mode, + { + rawToUser: layer.statistics[scope].modes[mode] + .comparisons.raw_to_user, + userToGeneration: layer.statistics[scope].modes[mode] + .comparisons.user_to_generation, + }, + ]), + ), + }, + ]), + ), + })), +}; +const templateCompactJson = JSON.stringify(templateCompact).replaceAll("<", "\\u003c"); const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoint_tensor_bytes; --- @@ -93,7 +155,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi

固定官方 revision、tokenizer、模型代码和 BF16 第一分片；RTX 5090 连续执行 layer 0–6， - 从 3,240 次 token 显微轨迹扩到 488,880 次公开语料路由，并让 layer-1 权重继续走入官方吸收式 cache。 + 从 3,240 次 token 显微轨迹扩到 870,912 次公开语料路由，并让 layer-1 权重继续走入官方吸收式 cache。 所有结论都带证据身份与停止线。

@@ -121,8 +183,11 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi + @@ -541,6 +606,117 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi + + diff --git a/src/pages/progress/index.astro b/src/pages/progress/index.astro index 7423705..14f2391 100644 --- a/src/pages/progress/index.astro +++ b/src/pages/progress/index.astro @@ -15,7 +15,7 @@ const workstreams = [ { label: "表示、位置与残差高速公路", value: 81, next: "加入真实 hidden-state / norm traces、长上下文位置外推复现与更多深层稳定性消融" }, { label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" }, { label: "数据工程与预训练配方", value: 73, next: "逐图精读 FineWeb / DCLM，加入真实去重与 mixture traces" }, - { label: "DeepSeek 专题", value: 92, next: "SM90 FlashMLA kernel、完整 27 层、tokenization 扰动、FP8/pipeline 与 R1-like RL 复现" }, + { label: "DeepSeek 专题", value: 93, next: "SM90 FlashMLA kernel、完整 27 层、词元边界 / system / few-shot 正交扰动、FP8/pipeline 与 R1-like RL 复现" }, { label: "指令微调与人类偏好", value: 75, next: "加入真实偏好分歧样本、RM 长度偏置与 PPO/DPO 小模型复现" }, { label: "推理与测试时扩展", value: 76, next: "真实模型采样曲线、PRM 案例与逐篇图表精读" }, { label: "工具使用与长程 Agent", value: 74, next: "补真实环境 traces、cross-harness 对照、Agent RL 训练曲线与安全案例" }, @@ -50,7 +50,7 @@ const workstreams = [
OVERALL
专题平均 {average}%
READABLE
{published} 个首版可读专题
ACTIVE
{researching} 个研究/写作中
-
UPDATED
2026-07-29 16:03 CST
+
UPDATED
2026-07-29 17:17 CST
MODE
持续迭代，不锁死版本
@@ -97,12 +97,12 @@ const workstreams = [
✓

K3 报告已结构化拆解

47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。

✓

17 专题知识图

从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。

✓

编辑式网站系统

响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。

-
✓

七十七个原创交互视图

K3 三轴图、八联报告实验与四联开放工件实验，DeepSeek 四联公式实验与六联真实权重实验，以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。

+
✓

七十八个原创交互视图

K3 三轴图、八联报告实验与四联开放工件实验，DeepSeek 四联公式实验与七联真实权重实验，以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。

✓

十七篇首版长文

K3、语言模型前史、Transformer、表示/位置/残差、DeepSeek、Scaling、数据工程、长上下文、MoE、后训练、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全专题。

✓

语言模型前史深度专题

八张独立问题账、33 个正式节点、20 段长文与概率—向量—记忆—对齐四联实验。

✓

Transformer 深度专题

十张独立问题账、40 个正式节点、21 段正文与 QKV—Mask—多头位置—Block 成本四联实验。

✓

表示、位置与残差高速公路深度专题

二十张问题账、66 个一手节点、DeepSeek/Kimi 双谱系，以及 Token—位置—Norm—Residual/FFN 四联实验。

-
✓

DeepSeek 三轮真实权重里程碑

在二十四张问题账、十次转向与四联公式实验上，新增 V2-Lite 7/27 层连续 forward、官方 V3 absorb 的 576 元素真实缓存，以及自然长度 / 同源 16 / 同源 24 三个 cohort、488,880 次真实路由、2,000 次成对 prompt bootstrap 与 3/3 byte-exact 独立重跑的六联实验。

+
✓

DeepSeek 三轮真实权重里程碑

在二十四张问题账、十次转向与四联公式实验上，新增 V2-Lite 7/27 层连续 forward、官方 V3 absorb 的 576 元素真实缓存、自然长度 / 同源长度对照与官方模板三条件实验；累计 870,912 次真实路由，模板共享前缀 21,852 / 21,852 ordered top-6 exact，四份运行结果均 byte-exact 独立复跑。

✓

Kimi K3 技术报告二轮深读

三十二张问题账、Figure 1–16 / Table 1–5 审计、100 节点阅读链，以及 Delta—Decay—AttnRes—LatentMoE—SiTU—QB—MOPD—Cache 八联实验。

✓

Kimi K3 三轮开放工件里程碑

固定官方 revisions，审计 96 个 shards、497,220 个 tensor entries 与真实 KDA / MLA / MoE / MoonViT shapes；四联实验分开显示层型、tensor anatomy、参数范围和复现边界。

✓

FlashKDA RTX 5090 执行闸门

隔离 CUDA 13.0 / glibc 2.39 编译 sm_120a wheel；6/6 官方参考逐元素相等，并完成 fixed / varlen、三种 state mode 的 1,800 个 CUDA Event samples。

@@ -134,7 +134,7 @@ const workstreams = [
优先级专题本轮交付完成闸门
P0K3 三轮

开放权重 traces → FlashKDA / AttnRes / MoE 真实行为 → Figure 1–16 数值重绘与独立复现

运行证据 + 逐图复现
-
P0DeepSeek 三轮

SM90 FlashMLA kernel / 完整 27 层 / tokenizer 与 prompt-template 扰动 → FP8 / pipeline traces → R1-like RL 小模型复现

运行证据 + 独立复现
+
P0DeepSeek 三轮

SM90 FlashMLA kernel / 完整 27 层 / 词元边界与 system / few-shot 正交扰动 → FP8 / pipeline traces → R1-like RL 小模型复现

运行证据 + 独立复现
P0Transformer 二轮

多头电路逐图 → Pre/Post-LN 真实 traces → Flash/KV 配置与 kernel 对照

逐图笔记 + 实测边界
P0表示、位置与残差二轮

真实 hidden-state / norm traces → 长上下文位置外推 → mHC / AttnRes 深层稳定性消融

可复现实验 + 逐图笔记
P0语言模型前史二轮

Kneser–Ney / LSTM / Bahdanau 逐图 → 真实小语料复现 → tokenizer 公平性

可复现实验 + 逐图笔记