feat: execute DeepSeek absorbed MLA cache
This commit is contained in:
wuyang
@@ -1,9 +1,18 @@
|
||||
---
|
||||
import rawTrace from "@/data/deepseek-v2-lite-trace.json";
|
||||
import rawRepro from "@/data/deepseek-v2-lite-trace-repro.json";
|
||||
import rawAbsorb from "@/data/deepseek-v2-lite-absorb.json";
|
||||
import rawAbsorbRepro from "@/data/deepseek-v2-lite-absorb-repro.json";
|
||||
|
||||
const trace = rawTrace as any;
|
||||
const repro = rawRepro as any;
|
||||
const absorb = rawAbsorb as any;
|
||||
const absorbRepro = rawAbsorbRepro as any;
|
||||
const absorbExact = JSON.stringify(absorb) === JSON.stringify(absorbRepro);
|
||||
const bytes = (value: number) => value >= 1024
|
||||
? `${(value / 1024).toFixed(2)} KiB`
|
||||
: `${value.toLocaleString()} B`;
|
||||
const scientific = (value: number) => value.toExponential(2);
|
||||
const compact = {
|
||||
prompts: trace.prompts.map((prompt: any) => ({
|
||||
id: prompt.id,
|
||||
@@ -46,7 +55,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
</div>
|
||||
<p>
|
||||
固定官方 revision、tokenizer、模型代码和 BF16 第一分片；RTX 5090 连续执行 layer 0–6，
|
||||
捕获真实 MLA 状态与 3,240 次 routed-expert 选择。所有结论都带证据身份与停止线。
|
||||
捕获 3,240 次真实路由，并让 layer-1 权重继续走入官方吸收式 cache。所有结论都带证据身份与停止线。
|
||||
</p>
|
||||
</header>
|
||||
|
||||
@@ -67,8 +76,11 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
<button type="button" role="tab" data-artifact-tab="cache" aria-selected="false" tabindex="-1">
|
||||
<span>03</span><b>缓存实现账</b><small>latent vs HF eager</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-artifact-tab="absorb" aria-selected="false" tabindex="-1">
|
||||
<span>04</span><b>吸收式执行</b><small>real cache · SM120</small>
|
||||
</button>
|
||||
<button type="button" role="tab" data-artifact-tab="evidence" aria-selected="false" tabindex="-1">
|
||||
<span>04</span><b>证据断面</b><small>revision · shards · rerun</small>
|
||||
<span>05</span><b>证据断面</b><small>revision · shards · rerun</small>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@@ -243,7 +255,114 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
|
||||
<div class="artifact-boundary">
|
||||
<b>IMPLEMENTATION CONTRACT</b>
|
||||
<p>算法上“可以缓存 576 元素”与本次框架“实际缓存 5,120 元素”同时为真；生产吞吐还需要真实 latent-cache kernel 与 serving runtime。</p>
|
||||
<p>算法上“可以缓存 576 元素”与 HF eager“实际缓存 5,120 元素”同时为真；下一页继续执行官方 absorb 参考路径，生产吞吐仍需要受支持的优化 kernel 与 serving runtime。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="artifact-panel" data-artifact-panel="absorb" hidden>
|
||||
<div class="panel-lead">
|
||||
<div><span>X + O / WEIGHT ABSORPTION</span><h4>这一次 576 元素真的进入了缓存</h4></div>
|
||||
<p>
|
||||
同一组真实 V2-Lite layer-1 权重、同一条 26-token hidden-state 轨迹：
|
||||
先用 V3 官方 naive 路径展开 K/V，再用官方 absorb 路径做 25-token prefill + 1-token decode。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="absorb-algebra" aria-label="MLA 权重吸收的两条代数变换">
|
||||
<article>
|
||||
<span>SCORE / 把 Wᴷ 搬到 query</span>
|
||||
<code>qᵀ(Wᴷc) = (Wᴷᵀq)ᵀc</code>
|
||||
<p>不再为每个历史 token、每个 head 存展开后的 no-RoPE key；query 先变成 512 维，再与 latent <i>c</i> 相乘。</p>
|
||||
</article>
|
||||
<article>
|
||||
<span>VALUE / 把 Wⱽ 搬到 attention 后</span>
|
||||
<code>Σ pₜ(Wⱽcₜ) = Wⱽ(Σ pₜcₜ)</code>
|
||||
<p>先对 512 维 latent 做加权和，再展开成各 head 的 value。含位置的 64 维 RoPE key 不能这样吸收，必须单独缓存。</p>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="absorb-cache-flow">
|
||||
<article class="naive">
|
||||
<header><span>X / V3 NAIVE · BF16</span><b>{bytes(absorb.cache_accounting.naive_active_bytes)}</b></header>
|
||||
<div>
|
||||
<code>K [1, 26, 16, 192]</code>
|
||||
<code>V [1, 26, 16, 128]</code>
|
||||
</div>
|
||||
<p>26 × 5,120 元素；与 HF eager 是同一种展开状态。</p>
|
||||
</article>
|
||||
<i aria-hidden="true">→</i>
|
||||
<article class="absorbed">
|
||||
<header><span>X / V3 ABSORB · BF16</span><b>{bytes(absorb.cache_accounting.absorb_active_bytes)}</b></header>
|
||||
<div>
|
||||
<code>latent [1, 26, 512]</code>
|
||||
<code>RoPE [1, 26, 64]</code>
|
||||
</div>
|
||||
<p>26 × 576 元素；两个官方 buffer 的真实 active slice。</p>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<div class="absorb-metrics">
|
||||
<article><span>REAL CACHE RATIO</span><b>{absorb.cache_accounting.naive_over_absorb_ratio.toFixed(4)}×</b><p>展开 K/V ÷ latent + RoPE</p></article>
|
||||
<article><span>BF16 MAX |Δ|</span><b>{absorb.correctness.v3_naive_vs_absorb_bfloat16.max_abs}</b><p>官方 V3 naive ↔ absorb decode</p></article>
|
||||
<article><span>FP32 MAX |Δ|</span><b>{scientific(absorb.correctness.v3_naive_vs_absorb_float32.max_abs)}</b><p>同一 BF16 权重转 FP32 做代数审计</p></article>
|
||||
<article class="exact"><span>INDEPENDENT RERUN</span><b>{absorbExact ? "BYTE-EXACT" : "MISMATCH"}</b><p>完整 JSON SHA-256 6b4c714a…e63d</p></article>
|
||||
</div>
|
||||
|
||||
<div class="precision-lens">
|
||||
<div>
|
||||
<span>HF EAGER ↔ V3 NAIVE / BF16</span>
|
||||
<i><em style={`--error:${absorb.correctness.hf_eager_vs_v3_naive_bfloat16.max_abs / .004}`}></em></i>
|
||||
<b>max {absorb.correctness.hf_eager_vs_v3_naive_bfloat16.max_abs}</b>
|
||||
</div>
|
||||
<div>
|
||||
<span>V3 NAIVE ↔ ABSORB / BF16</span>
|
||||
<i><em style={`--error:${absorb.correctness.v3_naive_vs_absorb_bfloat16.max_abs / .004}`}></em></i>
|
||||
<b>max {absorb.correctness.v3_naive_vs_absorb_bfloat16.max_abs}</b>
|
||||
</div>
|
||||
<div>
|
||||
<span>V3 NAIVE ↔ ABSORB / FP32</span>
|
||||
<i><em style={`--error:${absorb.correctness.v3_naive_vs_absorb_float32.max_abs / .004}`}></em></i>
|
||||
<b>max {scientific(absorb.correctness.v3_naive_vs_absorb_float32.max_abs)}</b>
|
||||
</div>
|
||||
<p>
|
||||
三组输出均 finite。FP32 不是新的 checkpoint 精度，而是把同一组 BF16 权重提升后，
|
||||
隔离矩阵乘法顺序造成的舍入误差。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="kernel-contract">
|
||||
<div>
|
||||
<span>OFFICIAL FLASHMLA / PINNED {absorb.flashmla_boundary.official_revision.slice(0, 8)}</span>
|
||||
<h5>算法路径已执行，不等于优化 kernel 已执行</h5>
|
||||
<p>官方支持矩阵与编译目标只覆盖 SM90 / SM100；本机 RTX 5090 是 SM120。</p>
|
||||
</div>
|
||||
<div class="kernel-matrix" role="table" aria-label="FlashMLA 官方架构支持矩阵">
|
||||
<div class="head" role="row"><b>KERNEL</b><b>SM90</b><b>SM100</b><b>LOCAL SM120</b></div>
|
||||
<div role="row"><span>dense decode</span><i class="yes">支持</i><i>—</i><i class="no">未支持</i></div>
|
||||
<div role="row"><span>sparse decode</span><i class="yes">支持</i><i class="yes">支持</i><i class="no">未支持</i></div>
|
||||
<div role="row"><span>dense prefill</span><i>—</i><i class="yes">支持</i><i class="no">未支持</i></div>
|
||||
<div role="row"><span>sparse prefill</span><i class="yes">支持</i><i class="yes">支持</i><i class="no">未支持</i></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="execution-split">
|
||||
<article><span>EXECUTED</span><b>官方 V3 pure-PyTorch absorb</b><p>真实权重、真实 latent / RoPE buffers、真实 incremental decode。</p></article>
|
||||
<article><span>BUILD ATTEMPTS</span><b>2 个环境，均未产出 wheel</b><p>host 编译器边界；隔离 CUDA 13 环境缺少 <code>cuda/std/utility</code>。</p></article>
|
||||
<article class="blocked"><span>NOT EXECUTED</span><b>FlashMLA optimized kernel</b><p>源码只生成 <code>sm_90a</code> / <code>sm_100f</code>，dense decode 运行时还拒绝非 SM90a。</p></article>
|
||||
</div>
|
||||
|
||||
<div class="evidence-links">
|
||||
<a href="https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/model.py" rel="noreferrer">官方 V3 absorb 参考实现 ↗</a>
|
||||
<a href="https://github.com/deepseek-ai/FlashMLA" rel="noreferrer">官方 FlashMLA 支持矩阵 ↗</a>
|
||||
<a href="https://arxiv.org/abs/2405.04434" rel="noreferrer">DeepSeek-V2 MLA 报告 ↗</a>
|
||||
</div>
|
||||
|
||||
<div class="artifact-boundary">
|
||||
<b>CORRECT CLAIM</b>
|
||||
<p>
|
||||
本实验复现的是“真实权重 + 官方参考实现中的压缩缓存与 decode 等价性”，不是 FlashMLA 性能、
|
||||
生产 serving 吞吐或 V2 论文完整模型相对 MHA 的 93.3% 缓存降幅。
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -328,15 +447,16 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
|
||||
<div class="artifact-boundary">
|
||||
<b>U / STILL OPEN</b>
|
||||
<p>完整 27 层生成、真实 latent-cache kernel、生产服务、训练负载、FP8/pipeline 与 R1-like 训练 trace 仍未覆盖。</p>
|
||||
<p>完整 27 层生成、受支持硬件上的 FlashMLA 优化 kernel、生产服务、训练负载、FP8/pipeline 与 R1-like 训练 trace 仍未覆盖。</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<figcaption>
|
||||
<span>可复现入口</span>
|
||||
<code>experiments/deepseek/v2_lite_trace.py</code> ·
|
||||
<code>experiments/deepseek/v2_lite_absorb_probe.py</code> ·
|
||||
<code>src/data/deepseek-v2-lite-trace.json</code> ·
|
||||
<code>research/DEEPSEEK_V2_LITE_TRACE.md</code>
|
||||
<code>research/DEEPSEEK_MLA_ABSORB_AUDIT.md</code>
|
||||
</figcaption>
|
||||
|
||||
<script is:inline type="application/json" data-dsv2-trace set:html={compactJson}></script>
|
||||
@@ -575,6 +695,11 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.jaccard-block span,
|
||||
.observed-cache span,
|
||||
.artifact-identity span,
|
||||
.absorb-algebra span,
|
||||
.absorb-cache-flow span,
|
||||
.absorb-metrics span,
|
||||
.kernel-contract span,
|
||||
.execution-split span,
|
||||
.repro-gate span,
|
||||
.dependency-split span,
|
||||
.checksum-grid span {
|
||||
@@ -631,7 +756,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.artifact-status b { color: var(--ink); font-size: .72rem; }
|
||||
.artifact-tabs {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, 1fr);
|
||||
grid-template-columns: repeat(5, 1fr);
|
||||
background: var(--ink);
|
||||
}
|
||||
.artifact-tabs button {
|
||||
@@ -792,6 +917,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.route-metrics,
|
||||
.cache-ratio,
|
||||
.load-lessons,
|
||||
.absorb-metrics,
|
||||
.artifact-identity,
|
||||
.checksum-grid {
|
||||
display: grid;
|
||||
@@ -802,6 +928,7 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.route-metrics article,
|
||||
.cache-ratio article,
|
||||
.load-lessons article,
|
||||
.absorb-metrics article,
|
||||
.artifact-identity article,
|
||||
.checksum-grid article {
|
||||
padding: .8rem;
|
||||
@@ -810,20 +937,24 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.route-metrics article:last-child,
|
||||
.cache-ratio article:last-child,
|
||||
.load-lessons article:last-child,
|
||||
.absorb-metrics article:last-child,
|
||||
.artifact-identity article:last-child,
|
||||
.checksum-grid article:last-child { border-right: 0; }
|
||||
.route-metrics span,
|
||||
.cache-ratio span,
|
||||
.load-lessons span {
|
||||
.load-lessons span,
|
||||
.absorb-metrics span {
|
||||
display: block;
|
||||
color: var(--blue);
|
||||
font: 700 .61rem/1 var(--font-mono);
|
||||
}
|
||||
.route-metrics b,
|
||||
.cache-ratio b { display: block; margin-top: .35rem; font: 750 1.25rem/1 var(--font-display); }
|
||||
.cache-ratio b,
|
||||
.absorb-metrics b { display: block; margin-top: .35rem; font: 750 1.25rem/1 var(--font-display); }
|
||||
.route-metrics p,
|
||||
.cache-ratio p,
|
||||
.load-lessons p { margin: .3rem 0 0; color: rgba(32,32,39,.56); font-size: .65rem; line-height: 1.45; }
|
||||
.load-lessons p,
|
||||
.absorb-metrics p { margin: .3rem 0 0; color: rgba(32,32,39,.56); font-size: .65rem; line-height: 1.45; }
|
||||
.artifact-boundary {
|
||||
display: grid;
|
||||
grid-template-columns: 11rem 1fr;
|
||||
@@ -950,6 +1081,127 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.cache-ledger p { margin: .35rem 0 0; color: rgba(32,32,39,.58); font-size: .68rem; line-height: 1.45; }
|
||||
.cache-ratio { grid-template-columns: repeat(3, 1fr); }
|
||||
.cache-ratio .warning { background: rgba(186,118,44,.09); }
|
||||
.absorb-algebra {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: .8rem;
|
||||
}
|
||||
.absorb-algebra article {
|
||||
padding: 1rem;
|
||||
border: 1px solid rgba(32,32,39,.15);
|
||||
background: #fffdf8;
|
||||
}
|
||||
.absorb-algebra code {
|
||||
display: block;
|
||||
margin: .75rem 0;
|
||||
color: var(--ink);
|
||||
font: 750 clamp(.88rem, 1.7vw, 1.2rem)/1.2 var(--font-mono);
|
||||
}
|
||||
.absorb-algebra p,
|
||||
.absorb-cache-flow p,
|
||||
.kernel-contract p,
|
||||
.execution-split p {
|
||||
margin: 0;
|
||||
color: rgba(32,32,39,.62);
|
||||
font-size: .69rem;
|
||||
line-height: 1.55;
|
||||
}
|
||||
.absorb-cache-flow {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr auto 1fr;
|
||||
gap: .8rem;
|
||||
align-items: center;
|
||||
margin-top: .8rem;
|
||||
}
|
||||
.absorb-cache-flow > i { color: var(--teal); font: 800 1.5rem/1 var(--font-display); }
|
||||
.absorb-cache-flow article {
|
||||
padding: 1rem;
|
||||
border: 1px solid rgba(32,32,39,.15);
|
||||
}
|
||||
.absorb-cache-flow article.naive { background: rgba(161,77,77,.07); }
|
||||
.absorb-cache-flow article.absorbed { background: rgba(57,120,110,.09); }
|
||||
.absorb-cache-flow header { display: flex; justify-content: space-between; gap: 1rem; align-items: center; }
|
||||
.absorb-cache-flow header b { font: 760 1.25rem/1 var(--font-display); }
|
||||
.absorb-cache-flow div { display: flex; flex-wrap: wrap; gap: .4rem; margin: .75rem 0; }
|
||||
.absorb-cache-flow code {
|
||||
padding: .35rem .45rem;
|
||||
background: rgba(32,32,39,.08);
|
||||
font-size: .66rem;
|
||||
}
|
||||
.absorb-metrics { grid-template-columns: repeat(4, 1fr); }
|
||||
.absorb-metrics article.exact { background: rgba(57,120,110,.1); }
|
||||
.precision-lens {
|
||||
display: grid;
|
||||
gap: .55rem;
|
||||
margin-top: .8rem;
|
||||
padding: 1rem;
|
||||
background: var(--ink);
|
||||
color: white;
|
||||
}
|
||||
.precision-lens > div {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(13rem, 1.3fr) 1fr minmax(8rem, auto);
|
||||
gap: .8rem;
|
||||
align-items: center;
|
||||
}
|
||||
.precision-lens span,
|
||||
.precision-lens b { font: 650 .62rem/1.3 var(--font-mono); }
|
||||
.precision-lens b { text-align: right; }
|
||||
.precision-lens i { height: 7px; overflow: hidden; background: rgba(255,255,255,.13); }
|
||||
.precision-lens em {
|
||||
display: block;
|
||||
width: max(calc(var(--error) * 100%), 2px);
|
||||
max-width: 100%;
|
||||
height: 100%;
|
||||
background: #b7bce5;
|
||||
}
|
||||
.precision-lens > div:last-of-type em { background: #91cfc1; }
|
||||
.precision-lens > p {
|
||||
margin: .25rem 0 0;
|
||||
padding-top: .7rem;
|
||||
border-top: 1px solid rgba(255,255,255,.15);
|
||||
color: rgba(255,255,255,.58);
|
||||
font-size: .66rem;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.kernel-contract {
|
||||
display: grid;
|
||||
grid-template-columns: .7fr 1.3fr;
|
||||
gap: 1rem;
|
||||
margin-top: .8rem;
|
||||
padding: 1rem;
|
||||
border: 1px solid rgba(32,32,39,.15);
|
||||
background: #e6dfd3;
|
||||
}
|
||||
.kernel-contract h5 { margin: .45rem 0; font: 720 1rem/1.15 var(--font-display); }
|
||||
.kernel-matrix { overflow-x: auto; border: 1px solid rgba(32,32,39,.14); background: #fffdf8; }
|
||||
.kernel-matrix > div {
|
||||
display: grid;
|
||||
grid-template-columns: 1.4fr repeat(3, minmax(5rem, .7fr));
|
||||
min-width: 520px;
|
||||
}
|
||||
.kernel-matrix b,
|
||||
.kernel-matrix span,
|
||||
.kernel-matrix i {
|
||||
padding: .55rem;
|
||||
border-right: 1px solid rgba(32,32,39,.1);
|
||||
border-bottom: 1px solid rgba(32,32,39,.1);
|
||||
font: 650 .61rem/1.2 var(--font-mono);
|
||||
}
|
||||
.kernel-matrix .head { background: var(--ink); color: white; }
|
||||
.kernel-matrix i { color: rgba(32,32,39,.45); font-style: normal; text-align: center; }
|
||||
.kernel-matrix i.yes { color: var(--teal); background: rgba(57,120,110,.08); }
|
||||
.kernel-matrix i.no { color: var(--red); background: rgba(161,77,77,.08); }
|
||||
.execution-split {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
margin-top: .8rem;
|
||||
border: 1px solid rgba(32,32,39,.14);
|
||||
}
|
||||
.execution-split article { padding: .9rem; border-right: 1px solid rgba(32,32,39,.12); }
|
||||
.execution-split article:last-child { border-right: 0; }
|
||||
.execution-split article.blocked { background: rgba(161,77,77,.08); }
|
||||
.execution-split b { display: block; margin: .4rem 0; font-size: .77rem; }
|
||||
.artifact-identity { grid-template-columns: repeat(3, 1fr); }
|
||||
.artifact-identity article { display: grid; gap: .35rem; }
|
||||
.artifact-identity b { font-size: .78rem; }
|
||||
@@ -1030,7 +1282,8 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.panel-lead,
|
||||
.heat-head,
|
||||
.aggregate-card,
|
||||
.jaccard-block { grid-template-columns: 1fr; }
|
||||
.jaccard-block,
|
||||
.kernel-contract { grid-template-columns: 1fr; }
|
||||
.artifact-status { grid-template-columns: 1fr 1fr; }
|
||||
.artifact-tabs { grid-template-columns: 1fr 1fr; }
|
||||
.route-controls { grid-template-columns: 1fr 1fr; }
|
||||
@@ -1039,7 +1292,8 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.route-stage > i { transform: rotate(90deg); justify-self: center; }
|
||||
.cache-controls { grid-template-columns: 1fr 1fr; }
|
||||
.route-metrics,
|
||||
.checksum-grid { grid-template-columns: 1fr 1fr; }
|
||||
.checksum-grid,
|
||||
.absorb-metrics { grid-template-columns: 1fr 1fr; }
|
||||
.layer-evidence { grid-template-columns: repeat(9, 1fr); }
|
||||
.repro-gate { grid-template-columns: 1fr 1fr; }
|
||||
.repro-gate > p { grid-column: 1 / -1; padding: .8rem 0 0; border-left: 0; border-top: 1px solid rgba(255,255,255,.18); }
|
||||
@@ -1062,20 +1316,28 @@ const shardFraction = trace.provenance.shard_1_bytes / trace.provenance.checkpoi
|
||||
.route-metrics,
|
||||
.cache-ratio,
|
||||
.load-lessons,
|
||||
.absorb-metrics,
|
||||
.artifact-identity,
|
||||
.checksum-grid,
|
||||
.cache-ledger { grid-template-columns: 1fr; }
|
||||
.route-metrics article,
|
||||
.cache-ratio article,
|
||||
.load-lessons article,
|
||||
.absorb-metrics article,
|
||||
.artifact-identity article,
|
||||
.checksum-grid article { border-right: 0; border-bottom: 1px solid rgba(32,32,39,.12); }
|
||||
.artifact-boundary { grid-template-columns: 1fr; }
|
||||
.load-dials { grid-template-columns: 1fr; }
|
||||
.observed-cache,
|
||||
.dependency-split { grid-template-columns: 1fr; }
|
||||
.dependency-split,
|
||||
.absorb-algebra,
|
||||
.absorb-cache-flow,
|
||||
.execution-split { grid-template-columns: 1fr; }
|
||||
.observed-cache > i,
|
||||
.dependency-split > i { transform: rotate(90deg); justify-self: center; }
|
||||
.absorb-cache-flow > i { transform: rotate(90deg); justify-self: center; }
|
||||
.precision-lens > div { grid-template-columns: 1fr; }
|
||||
.precision-lens b { text-align: left; }
|
||||
.layer-evidence { grid-template-columns: repeat(7, 1fr); }
|
||||
.repro-gate { grid-template-columns: 1fr; }
|
||||
.repro-gate > p { grid-column: auto; }
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"captured_at": "2026-07-29T06:35:00Z",
|
||||
"evidence_identity": "X / official V2-Lite weights executed through official HF eager and V3 naive/absorb reference paths",
|
||||
"boundary": {
|
||||
"correctness_probe": true,
|
||||
"serving_benchmark": false,
|
||||
"full_model_generation": false,
|
||||
"optimized_flashmla_kernel_executed": false,
|
||||
"scope": "one real layer-1 attention input; 25-token prefill plus one-token decode"
|
||||
},
|
||||
"provenance": {
|
||||
"huggingface_model": "deepseek-ai/DeepSeek-V2-Lite",
|
||||
"huggingface_revision": "604d5664dddd88a0433dbae533b7fe9472482de0",
|
||||
"deepseek_v3_repository": "deepseek-ai/DeepSeek-V3",
|
||||
"deepseek_v3_revision": "9b4e9788e4a3a731f7567338ed15d3ec549ce03b",
|
||||
"flashmla_repository": "deepseek-ai/FlashMLA",
|
||||
"flashmla_revision": "15f13e5030374295491c5ce31b02d7e63a7772c6",
|
||||
"sha256": {
|
||||
"v2_config": "f346286b0f1c8b044252fd54cb4fa78b9fab6472a6e8bebb9edfe03d414ea03d",
|
||||
"v2_modeling": "7d8e5221095286eea991137760893fd7ba52727c0b4ebf48ec09e8bc56b45b9c",
|
||||
"v2_shard_1": "0d7e9f39bde40111a4c0f390b87497dce4565cf578d916395e6b2c7851f1e8da",
|
||||
"v3_model": "fdb7995bc234badbf0011b5af8cf391be94343abc51305428cb9a04452116f02"
|
||||
}
|
||||
},
|
||||
"environment": {
|
||||
"python": "3.10.14",
|
||||
"platform": "Linux-7.0.0-28-generic-x86_64-with-glibc2.43",
|
||||
"libc": [
|
||||
"glibc",
|
||||
"2.43"
|
||||
],
|
||||
"torch": "2.11.0+cu128",
|
||||
"torch_cuda": "12.8",
|
||||
"transformers": "4.41.2",
|
||||
"safetensors": "0.8.0",
|
||||
"device": "cuda",
|
||||
"cuda_capability": [
|
||||
12,
|
||||
0
|
||||
],
|
||||
"nvidia_smi": {
|
||||
"name": "NVIDIA GeForce RTX 5090",
|
||||
"driver_version": "595.84",
|
||||
"memory.total": "32607",
|
||||
"compute_cap": "12.0"
|
||||
}
|
||||
},
|
||||
"input": {
|
||||
"prompt": "用通俗的语言解释，为什么稀疏专家模型可以拥有很多参数，但每个 token 只使用其中一小部分。",
|
||||
"token_ids": [
|
||||
100000,
|
||||
1082,
|
||||
83983,
|
||||
91476,
|
||||
17882,
|
||||
19304,
|
||||
10285,
|
||||
21844,
|
||||
20923,
|
||||
10900,
|
||||
39322,
|
||||
1876,
|
||||
9476,
|
||||
3599,
|
||||
21273,
|
||||
19304,
|
||||
1705,
|
||||
13712,
|
||||
10728,
|
||||
207,
|
||||
2141,
|
||||
5118,
|
||||
6802,
|
||||
51905,
|
||||
5781,
|
||||
398
|
||||
],
|
||||
"tokens": [
|
||||
"<｜begin▁of▁sentence｜>",
|
||||
"çĶ¨",
|
||||
"éĢļä¿Ĺ",
|
||||
"çļĦè¯Ńè¨Ģ",
|
||||
"解éĩĬ",
|
||||
"ï¼Į",
|
||||
"为ä»Ģä¹Ī",
|
||||
"ç¨Ģ",
|
||||
"çĸı",
|
||||
"ä¸ĵ家",
|
||||
"模åŀĭ",
|
||||
"åı¯ä»¥",
|
||||
"æĭ¥æľī",
|
||||
"å¾Īå¤ļ",
|
||||
"åıĤæķ°",
|
||||
"ï¼Į",
|
||||
"ä½Ĩ",
|
||||
"æ¯ı个",
|
||||
"Ġtoken",
|
||||
"Ġ",
|
||||
"åıª",
|
||||
"使çĶ¨",
|
||||
"åħ¶ä¸Ń",
|
||||
"ä¸Ģå°ı",
|
||||
"éĥ¨åĪĨ",
|
||||
"ãĢĤ"
|
||||
],
|
||||
"sequence_tokens": 26,
|
||||
"prefill_tokens": 25,
|
||||
"decode_tokens": 1,
|
||||
"layer0_executed": true,
|
||||
"layer1_input_normalized": true
|
||||
},
|
||||
"configuration": {
|
||||
"hidden": 2048,
|
||||
"attention_heads": 16,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"kv_lora_rank": 512,
|
||||
"cache_slots": 32
|
||||
},
|
||||
"execution": {
|
||||
"huggingface_eager": {
|
||||
"implementation": "huggingface_eager",
|
||||
"dtype": "bfloat16",
|
||||
"key_shape": [
|
||||
1,
|
||||
16,
|
||||
26,
|
||||
192
|
||||
],
|
||||
"value_shape": [
|
||||
1,
|
||||
16,
|
||||
26,
|
||||
128
|
||||
],
|
||||
"active_cache_bytes": 266240,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_naive_bfloat16": {
|
||||
"implementation": "naive",
|
||||
"dtype": "bfloat16",
|
||||
"active_cache": {
|
||||
"key": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"bytes": 159744
|
||||
},
|
||||
"value": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"bytes": 106496
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 266240,
|
||||
"allocated_cache": {
|
||||
"key_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"key_bytes": 196608,
|
||||
"value_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"value_bytes": 131072
|
||||
},
|
||||
"allocated_cache_bytes": 327680,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_absorb_bfloat16": {
|
||||
"implementation": "absorb",
|
||||
"dtype": "bfloat16",
|
||||
"active_cache": {
|
||||
"latent": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
512
|
||||
],
|
||||
"bytes": 26624
|
||||
},
|
||||
"rope": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
64
|
||||
],
|
||||
"bytes": 3328
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 29952,
|
||||
"allocated_cache": {
|
||||
"latent_shape": [
|
||||
1,
|
||||
32,
|
||||
512
|
||||
],
|
||||
"latent_bytes": 32768,
|
||||
"rope_shape": [
|
||||
1,
|
||||
32,
|
||||
64
|
||||
],
|
||||
"rope_bytes": 4096
|
||||
},
|
||||
"allocated_cache_bytes": 36864,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_naive_float32": {
|
||||
"implementation": "naive",
|
||||
"dtype": "float32",
|
||||
"active_cache": {
|
||||
"key": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"bytes": 319488
|
||||
},
|
||||
"value": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"bytes": 212992
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 532480,
|
||||
"allocated_cache": {
|
||||
"key_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"key_bytes": 393216,
|
||||
"value_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"value_bytes": 262144
|
||||
},
|
||||
"allocated_cache_bytes": 655360,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_absorb_float32": {
|
||||
"implementation": "absorb",
|
||||
"dtype": "float32",
|
||||
"active_cache": {
|
||||
"latent": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
512
|
||||
],
|
||||
"bytes": 53248
|
||||
},
|
||||
"rope": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
64
|
||||
],
|
||||
"bytes": 6656
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 59904,
|
||||
"allocated_cache": {
|
||||
"latent_shape": [
|
||||
1,
|
||||
32,
|
||||
512
|
||||
],
|
||||
"latent_bytes": 65536,
|
||||
"rope_shape": [
|
||||
1,
|
||||
32,
|
||||
64
|
||||
],
|
||||
"rope_bytes": 8192
|
||||
},
|
||||
"allocated_cache_bytes": 73728,
|
||||
"decode_finite": true
|
||||
}
|
||||
},
|
||||
"correctness": {
|
||||
"hf_eager_vs_v3_naive_bfloat16": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.28466796875,
|
||||
"max_abs": 0.001220703125,
|
||||
"mean_abs": 8.882676775101572e-05,
|
||||
"p50_abs": 6.103515625e-05,
|
||||
"p90_abs": 0.000244140625,
|
||||
"p99_abs": 0.00038238614797592163
|
||||
},
|
||||
"v3_naive_vs_absorb_bfloat16": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.10009765625,
|
||||
"max_abs": 0.00390625,
|
||||
"mean_abs": 0.00027031131321564317,
|
||||
"p50_abs": 0.000244140625,
|
||||
"p90_abs": 0.00054931640625,
|
||||
"p99_abs": 0.0009236149489879608
|
||||
},
|
||||
"hf_eager_vs_v3_absorb_bfloat16": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.1044921875,
|
||||
"max_abs": 0.00390625,
|
||||
"mean_abs": 0.00027483145822770894,
|
||||
"p50_abs": 0.000244140625,
|
||||
"p90_abs": 0.00054931640625,
|
||||
"p99_abs": 0.0009765625
|
||||
},
|
||||
"v3_naive_vs_absorb_float32": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.0556640625,
|
||||
"max_abs": 1.1920928955078125e-07,
|
||||
"mean_abs": 9.464486083743395e-09,
|
||||
"p50_abs": 7.450580596923828e-09,
|
||||
"p90_abs": 1.955777406692505e-08,
|
||||
"p99_abs": 3.144612037431216e-08
|
||||
}
|
||||
},
|
||||
"cache_accounting": {
|
||||
"active_tokens": 26,
|
||||
"dtype": "BF16",
|
||||
"naive_active_bytes": 266240,
|
||||
"absorb_active_bytes": 29952,
|
||||
"naive_elements_per_token": 5120,
|
||||
"absorb_elements_per_token": 576,
|
||||
"naive_over_absorb_ratio": 8.88888888888889,
|
||||
"absorb_reduction_vs_naive": 0.8875
|
||||
},
|
||||
"flashmla_boundary": {
|
||||
"official_revision": "15f13e5030374295491c5ce31b02d7e63a7772c6",
|
||||
"cutlass_revision": "147f5673d0c1c3dcf66f78d677fd647e4a020219",
|
||||
"official_support_matrix": [
|
||||
{
|
||||
"kernel": "dense_decode",
|
||||
"architectures": [
|
||||
"SM90"
|
||||
],
|
||||
"mode": "MQA",
|
||||
"cache": "BF16"
|
||||
},
|
||||
{
|
||||
"kernel": "sparse_decode",
|
||||
"architectures": [
|
||||
"SM90",
|
||||
"SM100"
|
||||
],
|
||||
"mode": "MQA",
|
||||
"cache": "FP8"
|
||||
},
|
||||
{
|
||||
"kernel": "dense_prefill",
|
||||
"architectures": [
|
||||
"SM100"
|
||||
],
|
||||
"mode": "MHA",
|
||||
"cache": null
|
||||
},
|
||||
{
|
||||
"kernel": "sparse_prefill",
|
||||
"architectures": [
|
||||
"SM90",
|
||||
"SM100"
|
||||
],
|
||||
"mode": "MQA",
|
||||
"cache": null
|
||||
}
|
||||
],
|
||||
"compiled_gencode": [
|
||||
"sm_100f",
|
||||
"sm_90a"
|
||||
],
|
||||
"dense_decode_runtime_guard": "Dense decode MLA is only supported on SM90a architecture",
|
||||
"local_architecture": "SM120",
|
||||
"optimized_kernel_executed": false,
|
||||
"why_not": "The pinned official source does not list or generate SM120 kernels; dense decode additionally rejects non-SM90a devices.",
|
||||
"build_attempts": [
|
||||
{
|
||||
"environment": "host CUDA 12.8 / g++ 15.2",
|
||||
"result": "failed before compilation",
|
||||
"evidence": "CUDA 12.8 rejects host compiler versions newer than g++ 13; g++-13 is not installed."
|
||||
},
|
||||
{
|
||||
"environment": "isolated CUDA 13.0.2 / torch 2.11.0+cu130 / g++ 13.3",
|
||||
"result": "failed during official extension compilation",
|
||||
"evidence": "csrc/api.cpp could not find cuda/std/utility; no wheel or runtime kernel was produced."
|
||||
}
|
||||
],
|
||||
"source_sha256": {
|
||||
"readme": "46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a",
|
||||
"setup": "b493b648e4f718e50dd323775834ab53703b155aee6f639e7c812b9e8a2ce48d",
|
||||
"dense_decode_header": "f3b012556eca4c8469bc99f6e299785793e6e6fe4aef090789501696e68e860f"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"captured_at": "2026-07-29T06:35:00Z",
|
||||
"evidence_identity": "X / official V2-Lite weights executed through official HF eager and V3 naive/absorb reference paths",
|
||||
"boundary": {
|
||||
"correctness_probe": true,
|
||||
"serving_benchmark": false,
|
||||
"full_model_generation": false,
|
||||
"optimized_flashmla_kernel_executed": false,
|
||||
"scope": "one real layer-1 attention input; 25-token prefill plus one-token decode"
|
||||
},
|
||||
"provenance": {
|
||||
"huggingface_model": "deepseek-ai/DeepSeek-V2-Lite",
|
||||
"huggingface_revision": "604d5664dddd88a0433dbae533b7fe9472482de0",
|
||||
"deepseek_v3_repository": "deepseek-ai/DeepSeek-V3",
|
||||
"deepseek_v3_revision": "9b4e9788e4a3a731f7567338ed15d3ec549ce03b",
|
||||
"flashmla_repository": "deepseek-ai/FlashMLA",
|
||||
"flashmla_revision": "15f13e5030374295491c5ce31b02d7e63a7772c6",
|
||||
"sha256": {
|
||||
"v2_config": "f346286b0f1c8b044252fd54cb4fa78b9fab6472a6e8bebb9edfe03d414ea03d",
|
||||
"v2_modeling": "7d8e5221095286eea991137760893fd7ba52727c0b4ebf48ec09e8bc56b45b9c",
|
||||
"v2_shard_1": "0d7e9f39bde40111a4c0f390b87497dce4565cf578d916395e6b2c7851f1e8da",
|
||||
"v3_model": "fdb7995bc234badbf0011b5af8cf391be94343abc51305428cb9a04452116f02"
|
||||
}
|
||||
},
|
||||
"environment": {
|
||||
"python": "3.10.14",
|
||||
"platform": "Linux-7.0.0-28-generic-x86_64-with-glibc2.43",
|
||||
"libc": [
|
||||
"glibc",
|
||||
"2.43"
|
||||
],
|
||||
"torch": "2.11.0+cu128",
|
||||
"torch_cuda": "12.8",
|
||||
"transformers": "4.41.2",
|
||||
"safetensors": "0.8.0",
|
||||
"device": "cuda",
|
||||
"cuda_capability": [
|
||||
12,
|
||||
0
|
||||
],
|
||||
"nvidia_smi": {
|
||||
"name": "NVIDIA GeForce RTX 5090",
|
||||
"driver_version": "595.84",
|
||||
"memory.total": "32607",
|
||||
"compute_cap": "12.0"
|
||||
}
|
||||
},
|
||||
"input": {
|
||||
"prompt": "用通俗的语言解释，为什么稀疏专家模型可以拥有很多参数，但每个 token 只使用其中一小部分。",
|
||||
"token_ids": [
|
||||
100000,
|
||||
1082,
|
||||
83983,
|
||||
91476,
|
||||
17882,
|
||||
19304,
|
||||
10285,
|
||||
21844,
|
||||
20923,
|
||||
10900,
|
||||
39322,
|
||||
1876,
|
||||
9476,
|
||||
3599,
|
||||
21273,
|
||||
19304,
|
||||
1705,
|
||||
13712,
|
||||
10728,
|
||||
207,
|
||||
2141,
|
||||
5118,
|
||||
6802,
|
||||
51905,
|
||||
5781,
|
||||
398
|
||||
],
|
||||
"tokens": [
|
||||
"<｜begin▁of▁sentence｜>",
|
||||
"çĶ¨",
|
||||
"éĢļä¿Ĺ",
|
||||
"çļĦè¯Ńè¨Ģ",
|
||||
"解éĩĬ",
|
||||
"ï¼Į",
|
||||
"为ä»Ģä¹Ī",
|
||||
"ç¨Ģ",
|
||||
"çĸı",
|
||||
"ä¸ĵ家",
|
||||
"模åŀĭ",
|
||||
"åı¯ä»¥",
|
||||
"æĭ¥æľī",
|
||||
"å¾Īå¤ļ",
|
||||
"åıĤæķ°",
|
||||
"ï¼Į",
|
||||
"ä½Ĩ",
|
||||
"æ¯ı个",
|
||||
"Ġtoken",
|
||||
"Ġ",
|
||||
"åıª",
|
||||
"使çĶ¨",
|
||||
"åħ¶ä¸Ń",
|
||||
"ä¸Ģå°ı",
|
||||
"éĥ¨åĪĨ",
|
||||
"ãĢĤ"
|
||||
],
|
||||
"sequence_tokens": 26,
|
||||
"prefill_tokens": 25,
|
||||
"decode_tokens": 1,
|
||||
"layer0_executed": true,
|
||||
"layer1_input_normalized": true
|
||||
},
|
||||
"configuration": {
|
||||
"hidden": 2048,
|
||||
"attention_heads": 16,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"kv_lora_rank": 512,
|
||||
"cache_slots": 32
|
||||
},
|
||||
"execution": {
|
||||
"huggingface_eager": {
|
||||
"implementation": "huggingface_eager",
|
||||
"dtype": "bfloat16",
|
||||
"key_shape": [
|
||||
1,
|
||||
16,
|
||||
26,
|
||||
192
|
||||
],
|
||||
"value_shape": [
|
||||
1,
|
||||
16,
|
||||
26,
|
||||
128
|
||||
],
|
||||
"active_cache_bytes": 266240,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_naive_bfloat16": {
|
||||
"implementation": "naive",
|
||||
"dtype": "bfloat16",
|
||||
"active_cache": {
|
||||
"key": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"bytes": 159744
|
||||
},
|
||||
"value": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"bytes": 106496
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 266240,
|
||||
"allocated_cache": {
|
||||
"key_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"key_bytes": 196608,
|
||||
"value_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"value_bytes": 131072
|
||||
},
|
||||
"allocated_cache_bytes": 327680,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_absorb_bfloat16": {
|
||||
"implementation": "absorb",
|
||||
"dtype": "bfloat16",
|
||||
"active_cache": {
|
||||
"latent": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
512
|
||||
],
|
||||
"bytes": 26624
|
||||
},
|
||||
"rope": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
64
|
||||
],
|
||||
"bytes": 3328
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 29952,
|
||||
"allocated_cache": {
|
||||
"latent_shape": [
|
||||
1,
|
||||
32,
|
||||
512
|
||||
],
|
||||
"latent_bytes": 32768,
|
||||
"rope_shape": [
|
||||
1,
|
||||
32,
|
||||
64
|
||||
],
|
||||
"rope_bytes": 4096
|
||||
},
|
||||
"allocated_cache_bytes": 36864,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_naive_float32": {
|
||||
"implementation": "naive",
|
||||
"dtype": "float32",
|
||||
"active_cache": {
|
||||
"key": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"bytes": 319488
|
||||
},
|
||||
"value": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"bytes": 212992
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 532480,
|
||||
"allocated_cache": {
|
||||
"key_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
192
|
||||
],
|
||||
"key_bytes": 393216,
|
||||
"value_shape": [
|
||||
1,
|
||||
32,
|
||||
16,
|
||||
128
|
||||
],
|
||||
"value_bytes": 262144
|
||||
},
|
||||
"allocated_cache_bytes": 655360,
|
||||
"decode_finite": true
|
||||
},
|
||||
"v3_absorb_float32": {
|
||||
"implementation": "absorb",
|
||||
"dtype": "float32",
|
||||
"active_cache": {
|
||||
"latent": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
512
|
||||
],
|
||||
"bytes": 53248
|
||||
},
|
||||
"rope": {
|
||||
"shape": [
|
||||
1,
|
||||
26,
|
||||
64
|
||||
],
|
||||
"bytes": 6656
|
||||
}
|
||||
},
|
||||
"active_cache_bytes": 59904,
|
||||
"allocated_cache": {
|
||||
"latent_shape": [
|
||||
1,
|
||||
32,
|
||||
512
|
||||
],
|
||||
"latent_bytes": 65536,
|
||||
"rope_shape": [
|
||||
1,
|
||||
32,
|
||||
64
|
||||
],
|
||||
"rope_bytes": 8192
|
||||
},
|
||||
"allocated_cache_bytes": 73728,
|
||||
"decode_finite": true
|
||||
}
|
||||
},
|
||||
"correctness": {
|
||||
"hf_eager_vs_v3_naive_bfloat16": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.28466796875,
|
||||
"max_abs": 0.001220703125,
|
||||
"mean_abs": 8.882676775101572e-05,
|
||||
"p50_abs": 6.103515625e-05,
|
||||
"p90_abs": 0.000244140625,
|
||||
"p99_abs": 0.00038238614797592163
|
||||
},
|
||||
"v3_naive_vs_absorb_bfloat16": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.10009765625,
|
||||
"max_abs": 0.00390625,
|
||||
"mean_abs": 0.00027031131321564317,
|
||||
"p50_abs": 0.000244140625,
|
||||
"p90_abs": 0.00054931640625,
|
||||
"p99_abs": 0.0009236149489879608
|
||||
},
|
||||
"hf_eager_vs_v3_absorb_bfloat16": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.1044921875,
|
||||
"max_abs": 0.00390625,
|
||||
"mean_abs": 0.00027483145822770894,
|
||||
"p50_abs": 0.000244140625,
|
||||
"p90_abs": 0.00054931640625,
|
||||
"p99_abs": 0.0009765625
|
||||
},
|
||||
"v3_naive_vs_absorb_float32": {
|
||||
"shape": [
|
||||
1,
|
||||
1,
|
||||
2048
|
||||
],
|
||||
"finite": true,
|
||||
"exact_fraction": 0.0556640625,
|
||||
"max_abs": 1.1920928955078125e-07,
|
||||
"mean_abs": 9.464486083743395e-09,
|
||||
"p50_abs": 7.450580596923828e-09,
|
||||
"p90_abs": 1.955777406692505e-08,
|
||||
"p99_abs": 3.144612037431216e-08
|
||||
}
|
||||
},
|
||||
"cache_accounting": {
|
||||
"active_tokens": 26,
|
||||
"dtype": "BF16",
|
||||
"naive_active_bytes": 266240,
|
||||
"absorb_active_bytes": 29952,
|
||||
"naive_elements_per_token": 5120,
|
||||
"absorb_elements_per_token": 576,
|
||||
"naive_over_absorb_ratio": 8.88888888888889,
|
||||
"absorb_reduction_vs_naive": 0.8875
|
||||
},
|
||||
"flashmla_boundary": {
|
||||
"official_revision": "15f13e5030374295491c5ce31b02d7e63a7772c6",
|
||||
"cutlass_revision": "147f5673d0c1c3dcf66f78d677fd647e4a020219",
|
||||
"official_support_matrix": [
|
||||
{
|
||||
"kernel": "dense_decode",
|
||||
"architectures": [
|
||||
"SM90"
|
||||
],
|
||||
"mode": "MQA",
|
||||
"cache": "BF16"
|
||||
},
|
||||
{
|
||||
"kernel": "sparse_decode",
|
||||
"architectures": [
|
||||
"SM90",
|
||||
"SM100"
|
||||
],
|
||||
"mode": "MQA",
|
||||
"cache": "FP8"
|
||||
},
|
||||
{
|
||||
"kernel": "dense_prefill",
|
||||
"architectures": [
|
||||
"SM100"
|
||||
],
|
||||
"mode": "MHA",
|
||||
"cache": null
|
||||
},
|
||||
{
|
||||
"kernel": "sparse_prefill",
|
||||
"architectures": [
|
||||
"SM90",
|
||||
"SM100"
|
||||
],
|
||||
"mode": "MQA",
|
||||
"cache": null
|
||||
}
|
||||
],
|
||||
"compiled_gencode": [
|
||||
"sm_100f",
|
||||
"sm_90a"
|
||||
],
|
||||
"dense_decode_runtime_guard": "Dense decode MLA is only supported on SM90a architecture",
|
||||
"local_architecture": "SM120",
|
||||
"optimized_kernel_executed": false,
|
||||
"why_not": "The pinned official source does not list or generate SM120 kernels; dense decode additionally rejects non-SM90a devices.",
|
||||
"build_attempts": [
|
||||
{
|
||||
"environment": "host CUDA 12.8 / g++ 15.2",
|
||||
"result": "failed before compilation",
|
||||
"evidence": "CUDA 12.8 rejects host compiler versions newer than g++ 13; g++-13 is not installed."
|
||||
},
|
||||
{
|
||||
"environment": "isolated CUDA 13.0.2 / torch 2.11.0+cu130 / g++ 13.3",
|
||||
"result": "failed during official extension compilation",
|
||||
"evidence": "csrc/api.cpp could not find cuda/std/utility; no wheel or runtime kernel was produced."
|
||||
}
|
||||
],
|
||||
"source_sha256": {
|
||||
"readme": "46c001c53800f47148918f4fc2969102492dd5164f710fc015db1622f6f9929a",
|
||||
"setup": "b493b648e4f718e50dd323775834ab53703b155aee6f639e7c812b9e8a2ce48d",
|
||||
"dense_decode_header": "f3b012556eca4c8469bc99f6e299785793e6e6fe4aef090789501696e68e860f"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ const toc = [
|
||||
|
||||
<BaseLayout
|
||||
title="DeepSeek 技术谱系与真实权重深读：从 Dense、MoE、MLA 到 R1 与 V4"
|
||||
description="用二十四张问题账、十次技术转向、八个交互实验、真实 V2-Lite 权重 trace 和六十个一手节点，完整理解 DeepSeek 的 MoE、MLA、FP8、DualPipe、GRPO、R1、V3.2 与 V4。"
|
||||
description="用二十四张问题账、十次技术转向、九个交互实验、真实 V2-Lite 权重与吸收式缓存 trace 和六十个一手节点，完整理解 DeepSeek 的 MoE、MLA、FP8、DualPipe、GRPO、R1、V3.2 与 V4。"
|
||||
section="deepseek"
|
||||
>
|
||||
<header class="page-hero deepseek-hero">
|
||||
@@ -55,7 +55,7 @@ const toc = [
|
||||
<div><dt>SPAN</dt><dd>2024.01 → 2026.06</dd></div>
|
||||
<div><dt>LEDGERS</dt><dd>24 张问题账</dd></div>
|
||||
<div><dt>LINEAGE</dt><dd>10 次技术转向</dd></div>
|
||||
<div><dt>LABS</dt><dd>8 个可操作实验</dd></div>
|
||||
<div><dt>LABS</dt><dd>9 个可操作实验</dd></div>
|
||||
<div><dt>EVIDENCE</dt><dd>60 个一手 / 官方节点</dd></div>
|
||||
<div><dt>STATUS</dt><dd>三轮 · 真实权重执行</dd></div>
|
||||
</dl>
|
||||
@@ -768,14 +768,14 @@ const toc = [
|
||||
<p class="eyebrow"><span>22</span> OFFICIAL WEIGHTS / EXECUTED</p>
|
||||
<h2>从“MLA 与 MoE 的概念”再往前一步：让官方 V2-Lite 权重真的跑起来</h2>
|
||||
<p class="lede">
|
||||
前面的四联实验负责建立公式与角色合同；下面的四联工件实验固定官方 revision、tokenizer、
|
||||
前面的四联实验负责建立公式与角色合同；下面的五联工件实验固定官方 revision、tokenizer、
|
||||
模型代码和 checkpoint 第一分片，在 RTX 5090 上连续执行 layer 0–6。它把真实观测、shape 推导、
|
||||
实现差距和未覆盖范围放在同一张证据图里。
|
||||
吸收式 latent cache、实现差距和未覆盖范围放在同一张证据图里。
|
||||
</p>
|
||||
<div class="artifact-callout">
|
||||
<article><span>X / FORWARD</span><b>7 / 27 layers</b><p>1 个 dense 层 + 6 个 MoE 层；layer 7 因跨分片停止。</p></article>
|
||||
<article><span>X / ROUTES</span><b>3,240</b><p>90 个有效 token × 6 层 × top-6 routed experts。</p></article>
|
||||
<article><span>X + D / CACHE</span><b>576 ↔ 5,120</b><p>latent 合同与 HF eager 实际展开元素，两张账同时保留。</p></article>
|
||||
<article><span>X / ABSORB CACHE</span><b>266,240 → 29,952 B</b><p>同一真实 layer-1 权重的 naive / absorb active buffers。</p></article>
|
||||
<article><span>X / RERUN</span><b>31 / 31 exact</b><p>hidden hashes、MLA shapes、loads 与全部 token routes。</p></article>
|
||||
</div>
|
||||
<DeepSeekArtifactLab />
|
||||
|
||||
@@ -15,7 +15,7 @@ const workstreams = [
|
||||
{ label: "表示、位置与残差高速公路", value: 81, next: "加入真实 hidden-state / norm traces、长上下文位置外推复现与更多深层稳定性消融" },
|
||||
{ label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" },
|
||||
{ label: "数据工程与预训练配方", value: 73, next: "逐图精读 FineWeb / DCLM，加入真实去重与 mixture traces" },
|
||||
{ label: "DeepSeek 专题", value: 87, next: "真实 latent-cache kernel、更大样本负载、FP8/pipeline 与 R1-like RL 复现" },
|
||||
{ label: "DeepSeek 专题", value: 89, next: "SM90 FlashMLA 优化 kernel、更大样本负载、FP8/pipeline 与 R1-like RL 复现" },
|
||||
{ label: "指令微调与人类偏好", value: 75, next: "加入真实偏好分歧样本、RM 长度偏置与 PPO/DPO 小模型复现" },
|
||||
{ label: "推理与测试时扩展", value: 76, next: "真实模型采样曲线、PRM 案例与逐篇图表精读" },
|
||||
{ label: "工具使用与长程 Agent", value: 74, next: "补真实环境 traces、cross-harness 对照、Agent RL 训练曲线与安全案例" },
|
||||
@@ -97,12 +97,12 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
|
||||
<article><span>✓</span><h3>17 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
|
||||
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
|
||||
<article><span>✓</span><h3>七十五个原创交互视图</h3><p>K3 三轴图、八联报告实验与四联开放工件实验，DeepSeek 四联公式实验与四联真实权重实验，以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>七十六个原创交互视图</h3><p>K3 三轴图、八联报告实验与四联开放工件实验，DeepSeek 四联公式实验与五联真实权重实验，以及语言模型前史、Transformer、表示深度、长上下文、MoE、推理、Agent、多模态、训练系统、推理服务、Scaling、数据工程、数值、Alignment 与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>十七篇首版长文</h3><p>K3、语言模型前史、Transformer、表示/位置/残差、DeepSeek、Scaling、数据工程、长上下文、MoE、后训练、推理、Agent、原生多模态、训练系统、推理服务、数值优化与评测安全专题。</p></article>
|
||||
<article><span>✓</span><h3>语言模型前史深度专题</h3><p>八张独立问题账、33 个正式节点、20 段长文与概率—向量—记忆—对齐四联实验。</p></article>
|
||||
<article><span>✓</span><h3>Transformer 深度专题</h3><p>十张独立问题账、40 个正式节点、21 段正文与 QKV—Mask—多头位置—Block 成本四联实验。</p></article>
|
||||
<article><span>✓</span><h3>表示、位置与残差高速公路深度专题</h3><p>二十张问题账、66 个一手节点、DeepSeek/Kimi 双谱系，以及 Token—位置—Norm—Residual/FFN 四联实验。</p></article>
|
||||
<article><span>✓</span><h3>DeepSeek 三轮真实权重里程碑</h3><p>在二十四张问题账、十次转向与四联公式实验上，新增 V2-Lite 7/27 层连续 forward、3,240 次真实专家选择、MLA/eager cache 实现账与 31/31 exact 复跑四联实验。</p></article>
|
||||
<article><span>✓</span><h3>DeepSeek 三轮真实权重里程碑</h3><p>在二十四张问题账、十次转向与四联公式实验上，新增 V2-Lite 7/27 层连续 forward、3,240 次真实专家选择、MLA/eager cache 实现账、31/31 exact 复跑，以及官方 V3 absorb 的 576 元素真实缓存五联实验。</p></article>
|
||||
<article><span>✓</span><h3>Kimi K3 技术报告二轮深读</h3><p>三十二张问题账、Figure 1–16 / Table 1–5 审计、100 节点阅读链，以及 Delta—Decay—AttnRes—LatentMoE—SiTU—QB—MOPD—Cache 八联实验。</p></article>
|
||||
<article><span>✓</span><h3>Kimi K3 三轮开放工件里程碑</h3><p>固定官方 revisions，审计 96 个 shards、497,220 个 tensor entries 与真实 KDA / MLA / MoE / MoonViT shapes；四联实验分开显示层型、tensor anatomy、参数范围和复现边界。</p></article>
|
||||
<article><span>✓</span><h3>FlashKDA RTX 5090 执行闸门</h3><p>隔离 CUDA 13.0 / glibc 2.39 编译 sm_120a wheel；6/6 官方参考逐元素相等，并完成 fixed / varlen、三种 state mode 的 1,800 个 CUDA Event samples。</p></article>
|
||||
@@ -134,7 +134,7 @@ const workstreams = [
|
||||
<div class="queue-table">
|
||||
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
|
||||
<div><span>P0</span><strong>K3 三轮</strong><p>开放权重 traces → FlashKDA / AttnRes / MoE 真实行为 → Figure 1–16 数值重绘与独立复现</p><em>运行证据 + 逐图复现</em></div>
|
||||
<div><span>P0</span><strong>DeepSeek 三轮</strong><p>真实 latent-cache kernel / 更大负载样本 → FP8 / pipeline traces → R1-like RL 小模型复现</p><em>运行证据 + 独立复现</em></div>
|
||||
<div><span>P0</span><strong>DeepSeek 三轮</strong><p>SM90 FlashMLA 优化 kernel / 更大负载样本 → FP8 / pipeline traces → R1-like RL 小模型复现</p><em>运行证据 + 独立复现</em></div>
|
||||
<div><span>P0</span><strong>Transformer 二轮</strong><p>多头电路逐图 → Pre/Post-LN 真实 traces → Flash/KV 配置与 kernel 对照</p><em>逐图笔记 + 实测边界</em></div>
|
||||
<div><span>P0</span><strong>表示、位置与残差二轮</strong><p>真实 hidden-state / norm traces → 长上下文位置外推 → mHC / AttnRes 深层稳定性消融</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P0</span><strong>语言模型前史二轮</strong><p>Kneser–Ney / LSTM / Bahdanau 逐图 → 真实小语料复现 → tokenizer 公平性</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
|
||||
Reference in New Issue
Block a user