feat: isolate DeepSeek history boundary token

This commit is contained in:
wuyang
2026-07-29 20:00:37 +08:00
parent f5a69170c7
commit f4d4a05dfa
14 changed files with 3676654 additions and 33 deletions
+627 -12
View File
@@ -16,6 +16,7 @@ import rawHistory from "@/data/deepseek-v2-lite-routing-history-factorial.json";
import rawHistoryRepro from "@/data/deepseek-v2-lite-routing-history-factorial-repro.json";
import rawDistance from "@/data/deepseek-v2-lite-routing-history-distance-control.json";
import rawDistanceRepro from "@/data/deepseek-v2-lite-routing-history-distance-control-repro.json";
import rawBoundaryCompact from "@/data/deepseek-v2-lite-routing-history-boundary-token-control-compact.json";
const trace = rawTrace as any;
const repro = rawRepro as any;
@@ -34,6 +35,7 @@ const history = rawHistory as any;
const historyRepro = rawHistoryRepro as any;
const distance = rawDistance as any;
const distanceRepro = rawDistanceRepro as any;
const boundaryCompact = rawBoundaryCompact as any;
const absorbExact = JSON.stringify(absorb) === JSON.stringify(absorbRepro);
const corpusExact = JSON.stringify(corpus) === JSON.stringify(corpusRepro);
const matched16Exact = JSON.stringify(matched16) === JSON.stringify(matched16Repro);
@@ -41,6 +43,7 @@ const matched24Exact = JSON.stringify(matched24) === JSON.stringify(matched24Rep
const templateExact = JSON.stringify(template) === JSON.stringify(templateRepro);
const historyExact = JSON.stringify(history) === JSON.stringify(historyRepro);
const distanceExact = JSON.stringify(distance) === JSON.stringify(distanceRepro);
const boundaryExact = boundaryCompact.source.exact;
const bytes = (value: number) => value >= 1024
? `${(value / 1024).toFixed(2)} KiB`
: `${value.toLocaleString()} B`;
@@ -329,6 +332,7 @@ const distanceCompact = {
})),
};
const distanceCompactJson = JSON.stringify(distanceCompact).replaceAll("<", "\\u003c");
const boundaryCompactJson = JSON.stringify(boundaryCompact).replaceAll("<", "\\u003c");
const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoint_tensor_bytes;
---
@@ -340,7 +344,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
</div>
<p>
固定官方 revision、tokenizer、模型代码和 BF16 第一分片;RTX 5090 连续执行 layer 0–6,
从 3,240 次 token 显微轨迹扩到 3,112,848 次公开语料路由,并让 layer-1 权重继续走入官方吸收式 cache。
从 3,240 次 token 显微轨迹扩到 5,157,072 次公开语料路由,并让 layer-1 权重继续走入官方吸收式 cache。
所有结论都带证据身份与停止线。
</p>
</header>
@@ -377,8 +381,11 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
<button type="button" role="tab" data-artifact-tab="distance" aria-selected="false" tabindex="-1">
<span>08</span><b>等长历史控制</b><small>none → filler → demo</small>
</button>
<button type="button" role="tab" data-artifact-tab="boundary" aria-selected="false" tabindex="-1">
<span>09</span><b>边界单词元控制</b><small>EOS ↔ x / . / ↵</small>
</button>
<button type="button" role="tab" data-artifact-tab="evidence" aria-selected="false" tabindex="-1">
<span>09</span><b>证据断面</b><small>revision · shards · rerun</small>
<span>10</span><b>证据断面</b><small>revision · shards · rerun</small>
</button>
</div>
@@ -1164,6 +1171,147 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
</div>
</section>
<section class="artifact-panel" data-artifact-panel="boundary" hidden>
<div class="panel-lead">
<div><span>X / SINGLE-ID BOUNDARY CONTROL</span><h4>只换 assistant 后面的一个 ID:EOS 不是普通占位符</h4></div>
<p>
重复词元历史、角色标记、长度、目标绝对位置与 32-row batch 全部不动;
只把官方 EOS `100001` 分别换成单 token 的 x、句点或换行。
</p>
</div>
<div class="history-ledger boundary-ledger">
<article><span>SOURCE PROMPTS</span><b>128</b><p>四个公开域各 32 条;同一 cohort</p></article>
<article><span>2×4 VARIANTS</span><b>1,024</b><p>system 0/1 × 四种边界 ID</p></article>
<article><span>INPUT TOKENS</span><b>56,784</b><p>八格逐组完全同长度</p></article>
<article><span>REAL ROUTES</span><b>2,044,224</b><p>八格 × 前六个 MoE 层</p></article>
<article><span>ALIGNED TARGET</span><b>2,874 × 8</b><p>相同字符跨度、位置与 token ID</p></article>
<article class="exact"><span>INDEPENDENT RERUN</span><b>{boundaryExact ? "BYTE-EXACT" : "MISMATCH"}</b><p>完整 JSON SHA-256 9bb93834…b9c37</p></article>
</div>
<div class="boundary-protocol" aria-label="历史 assistant 边界的单 token ID 控制">
<div class="boundary-track">
<span>FIXED PREFIX</span>
<b>Assistant:</b>
<i>x</i>
<mark data-boundary-track-token>EOS</mark>
<b>User:</b>
<i>TARGET</i>
<span>FIXED SUFFIX</span>
</div>
<div class="boundary-token-grid">
<article class="official">
<span>OFFICIAL</span><b>EOS</b><code>ID 100001</code>
<p>官方 template 自然产生;唯一的合法序列格。</p>
</article>
<article>
<span>COUNTERFACTUAL A</span><b>x</b><code>ID 87</code>
<p>普通内容 token;相对官方序列只改一个 ID。</p>
</article>
<article>
<span>COUNTERFACTUAL B</span><b>.</b><code>ID 13</code>
<p>普通标点 token;长度与后续位置完全不变。</p>
</article>
<article>
<span>COUNTERFACTUAL C</span><b>↵</b><code>ID 185</code>
<p>普通换行 token;保留紧随其后的 `User:` 标记。</p>
</article>
</div>
<p>
每个 source 在 S0 / S1 内都通过 4 / 4 同长度、4 / 4 同目标位置;768 / 768
反事实格相对官方 token 序列恰好只改一个 ID。
</p>
</div>
<div class="boundary-controls">
<div>
<span>MOE LAYER</span>
<div class="layer-switch boundary-layer-switch" role="group" aria-label="选择边界 token 控制层">
{[1, 2, 3, 4, 5, 6].map((layer) => (
<button type="button" data-boundary-layer={layer} class={layer === 1 ? "active" : ""}>L{layer}</button>
))}
</div>
</div>
<div>
<span>MEASUREMENT SCOPE</span>
<div class="boundary-scope-switch" role="group" aria-label="选择边界 token 统计范围">
<button type="button" data-boundary-scope="target_content" aria-pressed="true">目标内容</button>
<button type="button" data-boundary-scope="full_input" aria-pressed="false">完整输入</button>
</div>
</div>
<div>
<span>AGGREGATION</span>
<div class="boundary-mode-switch" role="group" aria-label="选择边界 token 聚合口径">
<button type="button" data-boundary-mode="prompt_balanced" aria-pressed="true">prompt 等权</button>
<button type="button" data-boundary-mode="token_weighted" aria-pressed="false">token 加权</button>
</div>
</div>
<div>
<span>REPLACEMENT − EOS</span>
<div class="boundary-contrast-switch" role="group" aria-label="选择 EOS 的单 token 替换">
<button type="button" data-boundary-contrast="x_minus_eos" aria-pressed="true">x − EOS</button>
<button type="button" data-boundary-contrast="period_minus_eos" aria-pressed="false">. − EOS</button>
<button type="button" data-boundary-contrast="newline_minus_eos" aria-pressed="false">↵ − EOS</button>
</div>
</div>
<p data-boundary-note>
目标内容:八格只比较完全相同的后续内容 token;正 ΔTV 表示替换 EOS 后 system edge 更大,不表示能力更差。
</p>
</div>
<div class="boundary-domain-grid" data-boundary-domain-grid></div>
<div class="history-buffer-summary boundary-summary">
<article><span>X CONTROL</span><b>24 / 24 ↑</b><p>system-edge TV 全部高于 EOS;23 / 24 配对区间完全高于零。</p></article>
<article><span>PERIOD CONTROL</span><b>24 / 24 ↑</b><p>点估计全部高于 EOS;同样 23 / 24 区间完全高于零。</p></article>
<article><span>NEWLINE CONTROL</span><b>23 / 24 ↑</b><p>16 / 24 区间完全高于零;比另两个对照更依赖层与域。</p></article>
<article><span>MEAN TARGET TV</span><b>.037 → .054 / .055 / .049</b><p>EOS / x / 句点 / 换行;不是准确率或优劣排名。</p></article>
</div>
<div class="boundary-depth">
<div>
<span>DEPTH MAP / REPLACEMENT − EOS</span>
<h5 data-boundary-depth-title>x − EOS:只替换历史边界的一个 input ID</h5>
<p>红色为替换后 system-edge TV 更大,绿色为更小;每格使用八格共享 source-bootstrap。</p>
</div>
<div data-boundary-depth-map></div>
</div>
<div class="boundary-interpretation">
<article>
<span>WHAT IS CAUSAL</span>
<b>一个历史 input ID</b>
<p>在固定模型、目标、位置、mask 与 batch 内,EOS→control 的路由差异来自这一个输入干预。</p>
</article>
<article>
<span>WHAT REMAINS</span>
<b>`User:` 边界仍在</b>
<p>实验没有删除全部回合结构,只识别 EOS token identity;三个反事实也不是合法官方 chat。</p>
</article>
<article>
<span>CHECKPOINT BOUNDARY</span>
<b>BASE ≠ CHAT / SFT</b>
<p>不能把较小 TV 命名为“理解回合结束”;仍需 V2-Lite-Chat 与行为生成对照。</p>
</article>
</div>
<div class="evidence-links">
<a href="https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite/blob/604d5664dddd88a0433dbae533b7fe9472482de0/tokenizer_config.json" rel="noreferrer">固定官方 tokenizer_config ↗</a>
<a href="https://huggingface.co/docs/transformers/en/chat_templating" rel="noreferrer">HF Chat Templates ↗</a>
<a href="https://arxiv.org/abs/2305.11206" rel="noreferrer">LIMA · EOT supervision ↗</a>
<a href="https://aclanthology.org/2020.blackboxnlp-1.26/" rel="noreferrer">EOS Decision ↗</a>
<a href="https://arxiv.org/abs/2309.16042" rel="noreferrer">Activation Patching Limits ↗</a>
</div>
<div class="artifact-boundary">
<b>SINGLE-ID ROUTING CAUSALITY, NOT TURN-SEMANTIC OR CAPABILITY PROOF</b>
<p>
EOS 条件下后续目标路由对 system 开关更稳定,但本实验既未生成答案,也未覆盖 Chat 权重;
更小 TV 不等于更正确。下一步要拆 `User:` 角色标记、special-token 家族与行为指标。
</p>
</div>
</section>
<section class="artifact-panel" data-artifact-panel="evidence" hidden>
<div class="panel-lead">
<div><span>O + X / EVIDENCE SLICE</span><h4>为什么执行到 layer 6 就停,而不是把“部分下载”写成“完整复现”</h4></div>
@@ -1261,7 +1409,9 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
<code>experiments/deepseek/v2_lite_routing_history_factorial_probe.py</code> ·
<code>research/DEEPSEEK_ROUTING_HISTORY_FACTORIAL_AUDIT.md</code> ·
<code>experiments/deepseek/v2_lite_routing_history_distance_control.py</code> ·
<code>research/DEEPSEEK_ROUTING_HISTORY_DISTANCE_CONTROL_AUDIT.md</code>
<code>research/DEEPSEEK_ROUTING_HISTORY_DISTANCE_CONTROL_AUDIT.md</code> ·
<code>experiments/deepseek/v2_lite_routing_history_boundary_token_control.py</code> ·
<code>research/DEEPSEEK_ROUTING_HISTORY_BOUNDARY_TOKEN_AUDIT.md</code>
</figcaption>
<script is:inline type="application/json" data-dsv2-trace set:html={compactJson}></script>
@@ -1269,6 +1419,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
<script is:inline type="application/json" data-dsv2-template set:html={templateCompactJson}></script>
<script is:inline type="application/json" data-dsv2-history set:html={historyCompactJson}></script>
<script is:inline type="application/json" data-dsv2-distance set:html={distanceCompactJson}></script>
<script is:inline type="application/json" data-dsv2-boundary set:html={boundaryCompactJson}></script>
</figure>
<script>
@@ -1284,18 +1435,21 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
const templateNode = one<HTMLScriptElement>("[data-dsv2-template]");
const historyNode = one<HTMLScriptElement>("[data-dsv2-history]");
const distanceNode = one<HTMLScriptElement>("[data-dsv2-distance]");
const boundaryNode = one<HTMLScriptElement>("[data-dsv2-boundary]");
if (
!payloadNode?.textContent
|| !corpusNode?.textContent
|| !templateNode?.textContent
|| !historyNode?.textContent
|| !distanceNode?.textContent
|| !boundaryNode?.textContent
) return;
const data = JSON.parse(payloadNode.textContent);
const corpusData = JSON.parse(corpusNode.textContent);
const templateData = JSON.parse(templateNode.textContent);
const historyData = JSON.parse(historyNode.textContent);
const distanceData = JSON.parse(distanceNode.textContent);
const boundaryData = JSON.parse(boundaryNode.textContent);
const tabs = all<HTMLButtonElement>("[data-artifact-tab]");
const panels = all<HTMLElement>("[data-artifact-panel]");
@@ -2188,6 +2342,172 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
});
});
renderDistance();
let boundaryLayerNumber = 1;
let boundaryScope = "target_content";
let boundaryMode = "prompt_balanced";
let boundaryContrast = "x_minus_eos";
const boundaryContrastLabels: Record<string, string> = {
x_minus_eos: "X − EOS",
period_minus_eos: "PERIOD − EOS",
newline_minus_eos: "NEWLINE − EOS",
};
const boundaryLevelLabels: Record<string, string> = {
eos: "EOS",
x: "X",
period: ".",
newline: "↵",
};
const boundaryReplacement = () => boundaryContrast.replace("_minus_eos", "");
const renderBoundary = () => {
all<HTMLButtonElement>("[data-boundary-layer]").forEach((button) => {
button.classList.toggle(
"active",
Number(button.dataset.boundaryLayer) === boundaryLayerNumber,
);
});
all<HTMLButtonElement>("[data-boundary-scope]").forEach((button) => {
button.setAttribute(
"aria-pressed",
String(button.dataset.boundaryScope === boundaryScope),
);
});
all<HTMLButtonElement>("[data-boundary-mode]").forEach((button) => {
button.setAttribute(
"aria-pressed",
String(button.dataset.boundaryMode === boundaryMode),
);
});
all<HTMLButtonElement>("[data-boundary-contrast]").forEach((button) => {
button.setAttribute(
"aria-pressed",
String(button.dataset.boundaryContrast === boundaryContrast),
);
});
set(
"[data-boundary-note]",
boundaryScope === "target_content"
? "目标内容:八格只比较完全相同的后续内容 token;正 ΔTV 表示替换 EOS 后 system edge 更大,不表示能力更差。"
: "完整输入:system、历史、被替换边界、角色包装与目标全部计入;一个 ID 的后续效应会被整段协议流量稀释。",
);
const replacement = boundaryReplacement();
set(
"[data-boundary-track-token]",
boundaryLevelLabels[replacement],
);
const currentLayer = boundaryData.layers.find(
(item: any) => item.layer === boundaryLayerNumber,
);
const view = currentLayer.scopes[boundaryScope].modes[boundaryMode];
const grid = one<HTMLElement>("[data-boundary-domain-grid]");
if (grid) {
grid.replaceChildren(...boundaryData.domains.map((domain: string) => {
const data = view[domain];
const contrast = data.contrasts[boundaryContrast].total_variation_delta;
const direct = data.direct[replacement];
const interaction = direct.s1_minus_s0.total_variation_delta;
const eosAlignment = currentLayer.alignment[domain].system_eos;
const replacementAlignment = currentLayer.alignment[domain][`system_${replacement}`];
const card = document.createElement("article");
const label = document.createElement("span");
const ladder = document.createElement("div");
const primary = document.createElement("strong");
const ci = document.createElement("p");
const stability = document.createElement("small");
const substitution = document.createElement("em");
const interactionLine = document.createElement("u");
const cv = document.createElement("i");
label.textContent = corpusLabels[domain];
ladder.className = "boundary-tv-ladder";
["eos", "x", "period", "newline"].forEach((boundary) => {
const cell = document.createElement("b");
const name = document.createElement("small");
const value = document.createElement("strong");
name.textContent = boundaryLevelLabels[boundary];
value.textContent = data.distances[boundary].total_variation.point.toFixed(3);
cell.classList.toggle("selected", boundary === replacement);
cell.classList.toggle("official", boundary === "eos");
cell.append(name, value);
ladder.append(cell);
});
primary.textContent = `${boundaryContrastLabels[boundaryContrast]} · ΔTV ${signed(contrast.point)}`;
primary.className = deltaClass(contrast.ci95);
ci.textContent = `source-paired 95% ${formatSignedCi(contrast.ci95)}`;
stability.textContent = `system top-6 set exact EOS ${(eosAlignment.setExactRate * 100).toFixed(1)}% · ${boundaryLevelLabels[replacement]} ${(replacementAlignment.setExactRate * 100).toFixed(1)}% · J ${eosAlignment.meanJaccard.toFixed(3)} → ${replacementAlignment.meanJaccard.toFixed(3)}`;
substitution.textContent = `direct EOS↔${boundaryLevelLabels[replacement]} TV · S0 ${direct.at_s0.total_variation.point.toFixed(3)} · S1 ${direct.at_s1.total_variation.point.toFixed(3)}`;
interactionLine.textContent = `direct S1−S0 ${signed(interaction.point)} · 95% ${formatSignedCi(interaction.ci95)}`;
const cvContrast = data.cvContrasts[boundaryContrast];
cv.textContent = `system-edge ΔCV ${signed(cvContrast.point)} · 95% ${formatSignedCi(cvContrast.ci95)}`;
card.append(
label,
ladder,
primary,
ci,
stability,
substitution,
interactionLine,
cv,
);
return card;
}));
}
const titles: Record<string, string> = {
x_minus_eos: "x − EOS:普通内容 token 替换官方边界",
period_minus_eos: "句点 − EOS:普通标点 token 替换官方边界",
newline_minus_eos: "换行 − EOS:普通格式 token 替换官方边界",
};
set("[data-boundary-depth-title]", titles[boundaryContrast]);
const depth = one<HTMLElement>("[data-boundary-depth-map]");
if (depth) {
depth.replaceChildren(...boundaryData.domains.map((domain: string) => {
const row = document.createElement("div");
const label = document.createElement("b");
label.textContent = corpusLabels[domain];
row.append(label);
boundaryData.layers.forEach((layer: any) => {
const contrast = layer.scopes[boundaryScope].modes[boundaryMode]
[domain].contrasts[boundaryContrast].total_variation_delta;
const cell = document.createElement("span");
cell.className = deltaClass(contrast.ci95);
cell.style.setProperty(
"--strength",
String(Math.min(1, Math.abs(contrast.point) / 0.04)),
);
cell.textContent = `L${layer.layer} ${signed(contrast.point)}`;
cell.title = `${corpusLabels[domain]} · L${layer.layer} · ${boundaryContrastLabels[boundaryContrast]} Δ system-edge TV ${signed(contrast.point)} · paired 95% ${formatSignedCi(contrast.ci95)}`;
row.append(cell);
});
return row;
}));
}
};
all<HTMLButtonElement>("[data-boundary-layer]").forEach((button) => {
button.addEventListener("click", () => {
boundaryLayerNumber = Number(button.dataset.boundaryLayer);
renderBoundary();
});
});
all<HTMLButtonElement>("[data-boundary-scope]").forEach((button) => {
button.addEventListener("click", () => {
boundaryScope = button.dataset.boundaryScope ?? "target_content";
renderBoundary();
});
});
all<HTMLButtonElement>("[data-boundary-mode]").forEach((button) => {
button.addEventListener("click", () => {
boundaryMode = button.dataset.boundaryMode ?? "prompt_balanced";
renderBoundary();
});
});
all<HTMLButtonElement>("[data-boundary-contrast]").forEach((button) => {
button.addEventListener("click", () => {
boundaryContrast = button.dataset.boundaryContrast ?? "x_minus_eos";
renderBoundary();
});
});
renderBoundary();
});
</script>
@@ -2309,7 +2629,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.artifact-status b { color: var(--ink); font-size: .72rem; }
.artifact-tabs {
display: grid;
grid-template-columns: repeat(9, 1fr);
grid-template-columns: repeat(10, 1fr);
background: var(--ink);
}
.artifact-tabs button {
@@ -3691,6 +4011,281 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
font-size: .62rem;
line-height: 1.45;
}
.boundary-protocol {
margin-top: .8rem;
padding: 1rem;
border: 1px solid rgba(32,32,39,.15);
background:
linear-gradient(115deg, rgba(98,105,155,.09), transparent 50%),
#fffdf8;
}
.boundary-track {
display: flex;
align-items: center;
justify-content: center;
flex-wrap: wrap;
gap: .25rem;
padding: .8rem;
background: var(--ink);
color: white;
}
.boundary-track > * {
padding: .36rem .48rem;
font: 650 .61rem/1 var(--font-mono);
font-style: normal;
}
.boundary-track span {
color: rgba(255,255,255,.48);
font-size: .52rem;
}
.boundary-track b { background: rgba(255,255,255,.1); }
.boundary-track i { color: #d5d8ef; }
.boundary-track mark {
min-width: 3.6rem;
background: var(--amber);
color: white;
text-align: center;
}
.boundary-token-grid {
display: grid;
grid-template-columns: repeat(4, 1fr);
margin-top: .55rem;
border: 1px solid rgba(32,32,39,.12);
}
.boundary-token-grid article {
display: grid;
gap: .34rem;
padding: .75rem;
border-right: 1px solid rgba(32,32,39,.12);
background: #f3eee5;
}
.boundary-token-grid article:last-child { border-right: 0; }
.boundary-token-grid article.official {
background: rgba(57,120,110,.11);
}
.boundary-token-grid span {
color: var(--blue);
font: 700 .52rem/1 var(--font-mono);
}
.boundary-token-grid b {
font: 760 1.05rem/1 var(--font-display);
}
.boundary-token-grid code {
width: max-content;
padding: .22rem .3rem;
background: rgba(32,32,39,.08);
font-size: .57rem;
}
.boundary-token-grid p,
.boundary-protocol > p {
margin: 0;
color: rgba(32,32,39,.58);
font-size: .61rem;
line-height: 1.45;
}
.boundary-protocol > p {
margin-top: .7rem;
text-align: center;
}
.boundary-controls {
display: grid;
grid-template-columns: auto .9fr .9fr 1.55fr;
gap: .8rem;
align-items: end;
margin-top: .8rem;
padding: .85rem;
border: 1px solid rgba(32,32,39,.14);
background: #e8e2d7;
}
.boundary-controls > div { display: grid; gap: .45rem; }
.boundary-controls .layer-switch { margin: 0; }
.boundary-scope-switch,
.boundary-mode-switch,
.boundary-contrast-switch { display: flex; }
.boundary-scope-switch button,
.boundary-mode-switch button,
.boundary-contrast-switch button {
padding: .58rem .66rem;
border: 1px solid rgba(32,32,39,.22);
background: #fffdf8;
color: var(--ink);
font: 650 .6rem/1 var(--font-mono);
cursor: pointer;
}
.boundary-scope-switch button + button,
.boundary-mode-switch button + button,
.boundary-contrast-switch button + button { border-left: 0; }
.boundary-scope-switch button[aria-pressed="true"],
.boundary-mode-switch button[aria-pressed="true"],
.boundary-contrast-switch button[aria-pressed="true"] {
border-color: var(--blue);
background: var(--blue);
color: white;
}
.boundary-controls > p {
grid-column: 1 / -1;
margin: 0;
padding-top: .75rem;
border-top: 1px solid rgba(32,32,39,.12);
color: rgba(32,32,39,.62);
font-size: .69rem;
line-height: 1.5;
}
.boundary-domain-grid {
display: grid;
grid-template-columns: repeat(4, 1fr);
margin-top: .8rem;
border: 1px solid rgba(32,32,39,.14);
background: #fffdf8;
}
.boundary-domain-grid > :global(article) {
min-width: 0;
padding: .85rem;
border-right: 1px solid rgba(32,32,39,.12);
}
.boundary-domain-grid > :global(article:last-child) { border-right: 0; }
.boundary-domain-grid :global(.boundary-tv-ladder) {
display: grid;
grid-template-columns: repeat(4, 1fr);
gap: .18rem;
margin-top: .55rem;
}
.boundary-domain-grid :global(.boundary-tv-ladder > b) {
display: grid;
gap: .18rem;
min-width: 0;
padding: .34rem;
background: #e8e2d7;
}
.boundary-domain-grid :global(.boundary-tv-ladder > b.official) {
background: rgba(57,120,110,.12);
}
.boundary-domain-grid :global(.boundary-tv-ladder > b.selected) {
outline: 1px solid var(--blue);
outline-offset: -1px;
}
.boundary-domain-grid :global(.boundary-tv-ladder small) {
color: rgba(32,32,39,.5);
font: 650 .46rem/1 var(--font-mono);
}
.boundary-domain-grid :global(.boundary-tv-ladder strong) {
white-space: nowrap;
font: 720 .58rem/1 var(--font-mono);
letter-spacing: -.025em;
}
.boundary-domain-grid > :global(article > strong) {
display: inline-block;
margin-top: .48rem;
padding: .26rem .38rem;
font: 750 .62rem/1 var(--font-mono);
}
.boundary-domain-grid > :global(article > strong.down),
.boundary-depth :global(span.down) {
background: rgba(57,120,110,.13);
color: var(--teal);
}
.boundary-domain-grid > :global(article > strong.up),
.boundary-depth :global(span.up) {
background: rgba(161,77,77,.12);
color: var(--red);
}
.boundary-domain-grid > :global(article > strong.neutral),
.boundary-depth :global(span.neutral) {
background: rgba(186,118,44,.12);
color: var(--amber);
}
.boundary-domain-grid > :global(article > p),
.boundary-domain-grid > :global(article > small),
.boundary-domain-grid > :global(article > em),
.boundary-domain-grid > :global(article > u),
.boundary-domain-grid > :global(article > i) {
display: block;
margin: .38rem 0 0;
color: rgba(32,32,39,.57);
overflow-wrap: anywhere;
font: .54rem/1.4 var(--font-mono);
font-style: normal;
text-decoration: none;
}
.boundary-domain-grid > :global(article > em),
.boundary-domain-grid > :global(article > u),
.boundary-domain-grid > :global(article > i) {
padding-top: .34rem;
border-top: 1px solid rgba(32,32,39,.1);
}
.boundary-depth {
display: grid;
grid-template-columns: .52fr 1.48fr;
gap: 1rem;
margin-top: .8rem;
padding: 1rem;
border: 1px solid rgba(32,32,39,.14);
}
.boundary-depth h5 {
margin: .4rem 0;
font: 720 1rem/1.15 var(--font-display);
}
.boundary-depth p {
margin: 0;
color: rgba(32,32,39,.58);
font-size: .66rem;
line-height: 1.5;
}
.boundary-depth > :global([data-boundary-depth-map]) {
display: grid;
gap: .35rem;
}
.boundary-depth :global([data-boundary-depth-map] > div) {
display: grid;
grid-template-columns: 5.5rem repeat(6, 1fr);
gap: .25rem;
}
.boundary-depth :global([data-boundary-depth-map] > div > b),
.boundary-depth :global([data-boundary-depth-map] > div > span) {
display: grid;
align-items: center;
min-height: 2.2rem;
padding: .35rem;
font: 650 .55rem/1.2 var(--font-mono);
}
.boundary-depth :global([data-boundary-depth-map] > div > b) {
color: var(--blue);
}
.boundary-depth :global([data-boundary-depth-map] > div > span.down) {
background: color-mix(in srgb, var(--teal) calc(var(--strength) * 55%), #eef0e9);
color: var(--ink);
}
.boundary-depth :global([data-boundary-depth-map] > div > span.up) {
background: color-mix(in srgb, var(--red) calc(var(--strength) * 48%), #f3ebe6);
color: var(--ink);
}
.boundary-depth :global([data-boundary-depth-map] > div > span.neutral) {
background: rgba(186,118,44,.1);
color: var(--ink);
}
.boundary-interpretation {
display: grid;
grid-template-columns: repeat(3, 1fr);
margin-top: .8rem;
border: 1px solid rgba(32,32,39,.14);
background: #e8e2d7;
}
.boundary-interpretation article {
padding: .9rem;
border-right: 1px solid rgba(32,32,39,.12);
}
.boundary-interpretation article:last-child { border-right: 0; }
.boundary-interpretation b {
display: block;
margin-top: .4rem;
font: 730 .78rem/1.2 var(--font-display);
}
.boundary-interpretation p {
margin: .4rem 0 0;
color: rgba(32,32,39,.58);
font-size: .62rem;
line-height: 1.45;
}
.observed-cache {
display: grid;
grid-template-columns: 1fr auto 1.25fr;
@@ -3934,7 +4529,9 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-buffer,
.history-depth,
.distance-controls,
.distance-depth { grid-template-columns: 1fr; }
.distance-depth,
.boundary-controls,
.boundary-depth { grid-template-columns: 1fr; }
.artifact-status { grid-template-columns: 1fr 1fr; }
.artifact-tabs { grid-template-columns: 1fr 1fr; }
.route-controls { grid-template-columns: 1fr 1fr; }
@@ -3951,9 +4548,13 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-ledger { grid-template-columns: repeat(3, 1fr); }
.history-domain-grid,
.history-buffer-summary,
.distance-domain-grid { grid-template-columns: 1fr 1fr; }
.distance-interpretation { grid-template-columns: 1fr; }
.distance-interpretation article { border-right: 0; border-bottom: 1px solid rgba(32,32,39,.12); }
.distance-domain-grid,
.boundary-domain-grid,
.boundary-token-grid { grid-template-columns: 1fr 1fr; }
.distance-interpretation,
.boundary-interpretation { grid-template-columns: 1fr; }
.distance-interpretation article,
.boundary-interpretation article { border-right: 0; border-bottom: 1px solid rgba(32,32,39,.12); }
.template-protocol > i { transform: rotate(90deg); justify-self: center; }
.length-delta-grid { grid-template-columns: 1fr 1fr; }
.corpus-heat-head p { text-align: left; }
@@ -3994,7 +4595,10 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-domain-grid,
.history-buffer-summary,
.distance-domain-grid,
.distance-interpretation { grid-template-columns: 1fr; }
.distance-interpretation,
.boundary-domain-grid,
.boundary-token-grid,
.boundary-interpretation { grid-template-columns: 1fr; }
.route-metrics article,
.cache-ratio article,
.load-lessons article,
@@ -4010,7 +4614,10 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-domain-grid > :global(article),
.history-buffer-summary article,
.distance-domain-grid > :global(article),
.distance-interpretation article { border-right: 0; border-bottom: 1px solid rgba(32,32,39,.12); }
.distance-interpretation article,
.boundary-domain-grid > :global(article),
.boundary-token-grid article,
.boundary-interpretation article { border-right: 0; border-bottom: 1px solid rgba(32,32,39,.12); }
.corpus-mode-switch,
.corpus-cohort-switch,
.template-scope-switch,
@@ -4020,7 +4627,10 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-effect-switch,
.distance-scope-switch,
.distance-mode-switch,
.distance-contrast-switch { display: grid; grid-template-columns: 1fr; }
.distance-contrast-switch,
.boundary-scope-switch,
.boundary-mode-switch,
.boundary-contrast-switch { display: grid; grid-template-columns: 1fr; }
.corpus-mode-switch button + button,
.corpus-cohort-switch button + button,
.template-scope-switch button + button,
@@ -4030,7 +4640,10 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-effect-switch button + button,
.distance-scope-switch button + button,
.distance-mode-switch button + button,
.distance-contrast-switch button + button { border-left: 1px solid rgba(32,32,39,.22); border-top: 0; }
.distance-contrast-switch button + button,
.boundary-scope-switch button + button,
.boundary-mode-switch button + button,
.boundary-contrast-switch button + button { border-left: 1px solid rgba(32,32,39,.22); border-top: 0; }
.length-delta-grid > :global(article),
.length-pair-summary article { border-right: 0; border-bottom: 1px solid rgba(32,32,39,.11); }
.artifact-boundary { grid-template-columns: 1fr; }
@@ -4050,6 +4663,8 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
.history-buffer > :global([data-history-buffer-grid]) { grid-template-columns: 1fr; }
.history-depth { overflow-x: auto; }
.history-depth > :global([data-history-depth-map]) { min-width: 620px; }
.boundary-depth { overflow-x: auto; }
.boundary-depth > :global([data-boundary-depth-map]) { min-width: 620px; }
.layer-evidence { grid-template-columns: repeat(7, 1fr); }
.repro-gate { grid-template-columns: 1fr; }
.repro-gate > p { grid-column: auto; }
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+5 -5
View File
@@ -37,7 +37,7 @@ const toc = [
<BaseLayout
title="DeepSeek 技术谱系与真实权重深读:从 Dense、MoE、MLA 到 R1 与 V4"
description="用二十四张问题账、十次技术转向、十三个交互实验、真实 V2-Lite 权重、公开语料路由区间、官方模板、消息历史因子与等长 filler 控制、吸收式缓存 trace 和六十个一手节点,完整理解 DeepSeek 的 MoE、MLA、FP8、DualPipe、GRPO、R1、V3.2 与 V4。"
description="用二十四张问题账、十次技术转向、十四个交互实验、真实 V2-Lite 权重、公开语料路由区间、官方模板、消息历史、等长 filler 与单 EOS-ID 边界控制、吸收式缓存 trace 和六十个一手节点,完整理解 DeepSeek 的 MoE、MLA、FP8、DualPipe、GRPO、R1、V3.2 与 V4。"
section="deepseek"
>
<header class="page-hero deepseek-hero">
@@ -55,7 +55,7 @@ const toc = [
<div><dt>SPAN</dt><dd>2024.01 → 2026.06</dd></div>
<div><dt>LEDGERS</dt><dd>24 张问题账</dd></div>
<div><dt>LINEAGE</dt><dd>10 次技术转向</dd></div>
<div><dt>LABS</dt><dd>13 个可操作实验</dd></div>
<div><dt>LABS</dt><dd>14 个可操作实验</dd></div>
<div><dt>EVIDENCE</dt><dd>60 个一手 / 官方节点</dd></div>
<div><dt>STATUS</dt><dd>三轮 · 真实权重执行</dd></div>
</dl>
@@ -768,15 +768,15 @@ const toc = [
<p class="eyebrow"><span>22</span> OFFICIAL WEIGHTS / EXECUTED</p>
<h2>从“MLA 与 MoE 的概念”再往前一步:让官方 V2-Lite 权重真的跑起来</h2>
<p class="lede">
前面的四联实验负责建立公式与角色合同;下面的九联工件实验固定官方 revision、tokenizer、
前面的四联实验负责建立公式与角色合同;下面的十联工件实验固定官方 revision、tokenizer、
模型代码和 checkpoint 第一分片,在 RTX 5090 上连续执行 layer 0–6。它把真实观测、shape 推导、
吸收式 latent cache、长度对照、官方 chat-template 扰动、实现差距和未覆盖范围放在同一张证据图里。
</p>
<div class="artifact-callout">
<article><span>X / FORWARD</span><b>7 / 27 layers</b><p>1 个 dense 层 + 6 个 MoE 层;layer 7 因跨分片停止。</p></article>
<article><span>X / ROUTES</span><b>3,112,848</b><p>三档长度、模板、system × one-shot 与 none/filler/demo 六格的真实 top-6 选择。</p></article>
<article><span>X / ROUTES</span><b>5,157,072</b><p>三档长度、模板、消息历史、等长 filler 与单 EOS-ID 八格控制的真实 top-6 选择。</p></article>
<article><span>X / ABSORB CACHE</span><b>266,240 → 29,952 B</b><p>同一真实 layer-1 权重的 naive / absorb active buffers。</p></article>
<article><span>X / RERUN</span><b>6 / 6 EXACT</b><p>三档长度、官方模板、历史因子与等长 filler 控制均 byte-exact;比较使用 paired prompt bootstrap。</p></article>
<article><span>X / RERUN</span><b>7 / 7 EXACT</b><p>三档长度、官方模板、历史因子、等长 filler 与边界 ID 控制均 byte-exact;比较使用 paired prompt bootstrap。</p></article>
</div>
<DeepSeekArtifactLab />
</section>
+7 -5
View File
@@ -15,7 +15,7 @@ const workstreams = [
{ label: "表示、位置与残差高速公路", value: 81, next: "加入真实 hidden-state / norm traces、长上下文位置外推复现与更多深层稳定性消融" },
{ label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" },
{ label: "数据工程与预训练配方", value: 73, next: "逐图精读 FineWeb / DCLM,加入真实去重与 mixture traces" },
{ label: "DeepSeek 专题", value: 95, next: "SM90 FlashMLA kernel、完整 27 层、EOS / 角色 / 多 filler / 内容与 batch-shape 控制、FP8/pipeline 与 R1-like RL 复现" },
{ label: "DeepSeek 专题", value: 96, next: "角色标记与 special-token family、V2-Lite-Chat 行为、完整 27 层与固定 batch shape,再推进 SM90 FlashMLA、FP8/pipeline 与 R1-like RL" },
{ label: "指令微调与人类偏好", value: 75, next: "加入真实偏好分歧样本、RM 长度偏置与 PPO/DPO 小模型复现" },
{ label: "推理与测试时扩展", value: 76, next: "真实模型采样曲线、PRM 案例与逐篇图表精读" },
{ label: "工具使用与长程 Agent", value: 74, next: "补真实环境 traces、cross-harness 对照、Agent RL 训练曲线与安全案例" },
@@ -50,7 +50,7 @@ const workstreams = [
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
<div><dt>UPDATED</dt><dd>2026-07-29 19:05 CST</dd></div>
<div><dt>UPDATED</dt><dd>2026-07-29 19:55 CST</dd></div>
<div><dt>MODE</dt><dd>持续迭代,不锁死版本</dd></div>
</dl>
</div>
@@ -97,12 +97,12 @@ const workstreams = [
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
<article><span>✓</span><h3>17 专题知识图</h3><p>从语言模型基础到评测安全,包含先修依赖和三条贯穿案例。</p></article>
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
<article><span>✓</span><h3>八十个原创交互视图</h3><p>K3 三轴图、八联报告实验与四联开放工件实验,DeepSeek 四联公式实验与九联真实权重实验,以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
<article><span>✓</span><h3>八十一个原创交互视图</h3><p>K3 三轴图、八联报告实验与四联开放工件实验,DeepSeek 四联公式实验与十联真实权重实验,以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
<article><span>✓</span><h3>十七篇首版长文</h3><p>K3、语言模型前史、Transformer、表示/位置/残差、DeepSeek、Scaling、数据工程、长上下文、MoE、后训练、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全专题。</p></article>
<article><span>✓</span><h3>语言模型前史深度专题</h3><p>八张独立问题账、33 个正式节点、20 段长文与概率—向量—记忆—对齐四联实验。</p></article>
<article><span>✓</span><h3>Transformer 深度专题</h3><p>十张独立问题账、40 个正式节点、21 段正文与 QKV—Mask—多头位置—Block 成本四联实验。</p></article>
<article><span>✓</span><h3>表示、位置与残差高速公路深度专题</h3><p>二十张问题账、66 个一手节点、DeepSeek/Kimi 双谱系,以及 Token—位置—Norm—Residual/FFN 四联实验。</p></article>
<article><span>✓</span><h3>DeepSeek 三轮真实权重里程碑</h3><p>在二十四张问题账、十次转向与四联公式实验上,新增 V2-Lite 7/27 层连续 forward、官方 V3 absorb、长度/模板、system × one-shot 与等长 filler 控制;累计 3,112,848 次真实路由,none→filler→demo 的两个 TV 台阶均在 24 / 24 格下降,六份运行结果均 byte-exact 独立复跑。</p></article>
<article><span>✓</span><h3>DeepSeek 三轮真实权重里程碑</h3><p>在二十四张问题账、十次转向与四联公式实验上,新增 V2-Lite 7/27 层连续 forward、官方 V3 absorb、长度/模板、system × one-shot、等长 filler 与 EOS 单 token 边界控制;累计 5,157,072 次真实路由。最新八格实验固定长度、位置、角色与 batch,仅替换一个 input ID,两份 54,254,445-byte JSON 的 SHA-256 同为 9bb93834…b9c37。</p></article>
<article><span>✓</span><h3>Kimi K3 技术报告二轮深读</h3><p>三十二张问题账、Figure 1–16 / Table 1–5 审计、100 节点阅读链,以及 Delta—Decay—AttnRes—LatentMoE—SiTU—QB—MOPD—Cache 八联实验。</p></article>
<article><span>✓</span><h3>Kimi K3 三轮开放工件里程碑</h3><p>固定官方 revisions,审计 96 个 shards、497,220 个 tensor entries 与真实 KDA / MLA / MoE / MoonViT shapes;四联实验分开显示层型、tensor anatomy、参数范围和复现边界。</p></article>
<article><span>✓</span><h3>FlashKDA RTX 5090 执行闸门</h3><p>隔离 CUDA 13.0 / glibc 2.39 编译 sm_120a wheel;6/6 官方参考逐元素相等,并完成 fixed / varlen、三种 state mode 的 1,800 个 CUDA Event samples。</p></article>
@@ -134,7 +134,7 @@ const workstreams = [
<div class="queue-table">
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
<div><span>P0</span><strong>K3 三轮</strong><p>开放权重 traces → FlashKDA / AttnRes / MoE 真实行为 → Figure 1–16 数值重绘与独立复现</p><em>运行证据 + 逐图复现</em></div>
<div><span>P0</span><strong>DeepSeek 三轮</strong><p>SM90 FlashMLA kernel / 完整 27 层 / EOS、角色、多 filler、示例内容与 batch shape 控制 → FP8 / pipeline traces → R1-like RL 小模型复现</p><em>运行证据 + 独立复现</em></div>
<div><span>P0</span><strong>DeepSeek 三轮</strong><p>角色标记 / special-token family / V2-Lite-Chat 行为 → 完整 27 层与固定 batch shape → SM90 FlashMLA / FP8 / pipeline traces → R1-like RL 小模型复现</p><em>运行证据 + 独立复现</em></div>
<div><span>P0</span><strong>Transformer 二轮</strong><p>多头电路逐图 → Pre/Post-LN 真实 traces → Flash/KV 配置与 kernel 对照</p><em>逐图笔记 + 实测边界</em></div>
<div><span>P0</span><strong>表示、位置与残差二轮</strong><p>真实 hidden-state / norm traces → 长上下文位置外推 → mHC / AttnRes 深层稳定性消融</p><em>可复现实验 + 逐图笔记</em></div>
<div><span>P0</span><strong>语言模型前史二轮</strong><p>Kneser–Ney / LSTM / Bahdanau 逐图 → 真实小语料复现 → tokenizer 公平性</p><em>可复现实验 + 逐图笔记</em></div>
@@ -218,6 +218,8 @@ const workstreams = [
<div><time>2026-07-29</time><b>32-token 对照改为同源 16→24</b><p>TNEWS 只有 105/10,000 条达到 32 tokens,强行统一会落入约 1% 极端长尾;24-token eligibility 仍保留 1,609 条中文候选。</p></div>
<div><time>2026-07-29</time><b>长度敏感性必须成对重采样</b><p>16-token 输入严格是 24-token 输入前缀,2,000 次 bootstrap 共用 prompt indices;结果只描述固定 cohort 的长度敏感性。</p></div>
<div><time>2026-07-29</time><b>三类 cohort 永久分身份</b><p>自然长度回答本批样本如何路由;matched-16 / 24 回答同一 prompt 多看 8 tokens 后如何变化,不把二者混成内容因果。</p></div>
<div><time>2026-07-29</time><b>历史边界用单 ID 替换而非删除</b><p>EOS→x / 句点 / 换行保持长度、目标位置、角色标记、mask 与同一 batch;识别一个输入 ID 的干预,不冒充移除了全部回合结构。</p></div>
<div><time>2026-07-29</time><b>base、Chat 与行为永久分层</b><p>V2-Lite base 的路由 TV 不是回合理解或能力指标;`User:` 仍在,反事实不是官方合法 chat,后续另跑 Chat checkpoint 与生成指标。</p></div>
</div>
</section>