wuyang
1336 lines
82 KiB
Plaintext
1336 lines
82 KiB
Plaintext
---
|
||
import BaseLayout from "@/layouts/BaseLayout.astro";
|
||
import ArchitectureExplorer from "@/components/ArchitectureExplorer.astro";
|
||
import K3ArtifactLab from "@/components/K3ArtifactLab.astro";
|
||
import K3AttnResGradientLab from "@/components/K3AttnResGradientLab.astro";
|
||
import K3AttnResTraceLab from "@/components/K3AttnResTraceLab.astro";
|
||
import K3ReportLab from "@/components/K3ReportLab.astro";
|
||
import { k3FigureAtlas, k3Ledgers, k3PaperChain, k3ReportMap } from "@/data/k3";
|
||
|
||
const toc = [
|
||
["00", "compass", "三十二张问题账"],
|
||
["01", "report-map", "47 页报告地图"],
|
||
["02", "architecture", "三维信息流"],
|
||
["03", "specs", "K2 → K3 规格"],
|
||
["04", "kda", "KDA 状态与 Delta Rule"],
|
||
["05", "bounded-decay", "Bounded decay"],
|
||
["06", "mla", "Gated MLA 与 NoPE"],
|
||
["07", "attnres", "跨深度 Attention"],
|
||
["08", "latentmoe", "Stable LatentMoE"],
|
||
["09", "stability", "SiTU 与 QB"],
|
||
["10", "vision", "原生视觉"],
|
||
["11", "data", "数据工程"],
|
||
["12", "scaling", "Scaling 与训练配方"],
|
||
["13", "context", "8K → 1M"],
|
||
["14", "sft", "SFT 与 XTML 冷启动"],
|
||
["15", "rl", "九专家与 effort"],
|
||
["16", "mopd", "MOPD 与 partial rollout"],
|
||
["17", "deployment", "QAT 与 draft model"],
|
||
["18", "harness", "White-box harness"],
|
||
["19", "environments", "环境与任务合成"],
|
||
["20", "pretrain-systems", "预训练系统"],
|
||
["21", "moonep", "MoonEP"],
|
||
["22", "rl-systems", "RL 状态与 AgentENV"],
|
||
["23", "cache", "混合 prefix cache"],
|
||
["24", "serving", "Kernel 与 fleet"],
|
||
["25", "evaluation", "评测协议"],
|
||
["26", "cases", "案例边界"],
|
||
["27", "xtml", "XTML 协议"],
|
||
["28", "lab", "八联交互实验"],
|
||
["29", "artifacts", "开放权重工件审计"],
|
||
["30", "attnres-reduced", "AttnRes 缩小机制实验"],
|
||
["31", "attnres-gradient", "梯度定义与深度扩展"],
|
||
["32", "audit", "21 张图表审计"],
|
||
["↳", "papers", "100 节点阅读链"],
|
||
];
|
||
|
||
const specs = [
|
||
["Transformer layers", "61", "93", "+32"],
|
||
["Total parameters", "1.04T", "2.78T", "容量约 2.67×"],
|
||
["Activated parameters", "32.6B", "104.2B", "路径约 3.20×"],
|
||
["Hidden dimension", "7168", "7168", "主干不变"],
|
||
["Routed latent dimension", "—", "3584", "主干的 0.5×"],
|
||
["MoE expert hidden", "2048", "3072", "专家内部更宽"],
|
||
["Routed experts", "384", "896", "池更大"],
|
||
["Activated routed", "8", "16", "每 Token 多选"],
|
||
["Shared experts", "1", "2", "完整宽度公共路径"],
|
||
["Attention heads", "64", "96", "+32"],
|
||
["Dense layers", "1", "1", "不变"],
|
||
["Vocabulary", "160K", "160K", "不变"],
|
||
["Context", "128K", "1M", "四阶段扩展"],
|
||
["Attention", "61 MLA", "69 KDA + 24 MLA", "3:1 hybrid"],
|
||
["Activation", "SwiGLU", "SiTU-GLU", "有界乘积分支"],
|
||
["MTP layers", "1", "1", "后调为 draft"],
|
||
["Vision encoder", "—", "401M / 27L / p14 / 12H", "从头联合训练"],
|
||
];
|
||
|
||
const dataDomains = [
|
||
["WEB TEXT", "网页正文与长文档", "规则、质量分类、exact/fuzzy dedup；长文档单独清洗与上采样。"],
|
||
["CODE", "代码、仓库与可执行材料", "不仅生成文本，还支撑 kernel、web、软件工程和 programmatic vision。"],
|
||
["MATHEMATICS", "数学推理与重写材料", "K2 参与多风格、多视角 rephrasing，并做 fidelity verification。"],
|
||
["KNOWLEDGE", "知识密集语料", "chunk-wise 自回归改写，尽量在扩展表达的同时保留事实。"],
|
||
["VISION", "caption / OCR / video / visual coding", "图文交错、绝对与归一坐标、SVG/3D/Web/Game/CAD 等程序化视觉。"],
|
||
];
|
||
|
||
const environments = [
|
||
["01", "Verifiable search & professional work", "多步搜索、投行、法律、数据分析与办公交付物；证据和最终产物都能检查。"],
|
||
["02", "Vision reasoning", "在隔离 Python 环境裁剪、放大、计算，再把新图像作为 observation 回到同一轨迹。"],
|
||
["03", "GPU kernel", "先过数值正确性，再比较专家实现和硬件 roofline，同时检测缓存/降精度作弊。"],
|
||
["04", "Long-term assistant", "Gmail、Notion、Slack 等 mock app 跨多日演进，单任务可含数千次工具调用。"],
|
||
["05", "Autonomous Execution", "只给初始状态、目标、约束、工具和 verifier，不给参考轨迹；奖励最终环境状态。"],
|
||
["06", "Web development", "容器内构建网页、游戏、3D 与可视化，由功能、结构/像素和模型检查共同评分。"],
|
||
];
|
||
|
||
const stateStack = [
|
||
["PRETRAIN", "参数 / optimizer / activation / KDA state / vision tokens", "FlashKDA · KCP · MoonEP · pipeline/offload"],
|
||
["AGENTIC RL", "policy / reference / KV / trajectory / sandbox / files", "partial rollout · throttling · AgentENV"],
|
||
["SERVING ENGINE", "KDA recurrent state / MLA KV / prefix hash", "unified page pool · sparse checkpoints · COW"],
|
||
["DEVICE", "projected replay input / AttnRes block / expert weights", "fused KDA · SP · WarpDecode-like kernel"],
|
||
["FLEET", "session affinity / request-class budget", "primary+secondary hash · budget admission"],
|
||
];
|
||
|
||
const paperGroups = [
|
||
["FOUNDATION", "基础"],
|
||
["STATE", "序列状态"],
|
||
["DEPTH", "深度"],
|
||
["MOE", "稀疏宽度"],
|
||
["NUMERICS", "数值"],
|
||
["SCALING", "Scaling"],
|
||
["CONTEXT", "长上下文"],
|
||
["KIMI", "Kimi 谱系"],
|
||
["DEEPSEEK", "DeepSeek 交叉线"],
|
||
["RL", "后训练"],
|
||
["AGENT", "Agent 环境"],
|
||
["DEPLOY", "部署"],
|
||
["SYSTEM", "系统"],
|
||
["EVAL", "评测"],
|
||
] as const;
|
||
---
|
||
|
||
<BaseLayout
|
||
title="Kimi K3 技术报告完整深读：架构、训练、RL、系统与评测"
|
||
description="用三十二张问题账、二十一张图表审计、八个机制实验、四个开放工件视图、两轮十个 AttnRes 独立实验视图与一百个一手阅读节点，逐节读懂 Kimi K3。"
|
||
section="k3"
|
||
>
|
||
<header class="page-hero k3-hero">
|
||
<div class="page-hero-inner">
|
||
<div>
|
||
<p class="eyebrow"><span>ANCHOR REPORT / ROUND 05</span> KIMI K3 · REPORT → ARTIFACTS → INDEPENDENT PROBE</p>
|
||
<h1>不把报告压成摘要<br />把每个因果环节<br />重新展开</h1>
|
||
<p class="lead">
|
||
K3 同时扩展序列、深度、宽度、视觉与 Agent 轨迹。真正值得读的不是 2.8T 这个最大数字，
|
||
而是每种状态为什么出现、放在哪里、怎样被训练，以及作者证据究竟支持到哪一步。
|
||
</p>
|
||
</div>
|
||
<dl class="page-facts">
|
||
<div><dt>QUESTIONS</dt><dd>32 张问题账</dd></div>
|
||
<div><dt>REPORT</dt><dd>16 Figures · 5 Tables</dd></div>
|
||
<div><dt>LABS</dt><dd>8 + 4 + 5 + 5 个交互视图</dd></div>
|
||
<div><dt>READING</dt><dd>100 个一手 / 官方节点</dd></div>
|
||
<div><dt>MODEL</dt><dd>2.78T total / 104.2B active</dd></div>
|
||
<div><dt>ARTIFACTS</dt><dd>96 shards · 497,220 tensors</dd></div>
|
||
<div><dt>STATUS</dt><dd>K3 五轮 · 梯度定义闭环</dd></div>
|
||
</dl>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="report-shell">
|
||
<aside class="side-rail" aria-label="本页目录">
|
||
<p>CONTENTS</p>
|
||
<ol>
|
||
{toc.map(([number, id, label]) => (
|
||
<li><a href={`#${id}`}><span>{number}</span>{label}</a></li>
|
||
))}
|
||
</ol>
|
||
<div class="rail-note">
|
||
<b>证据约定</b>
|
||
报告层用 R / P / D / T 区分报告、原始来源、推导与教学模型；
|
||
工件层用 O / D / X / S / U 区分观测、推导、本站执行、合成探针与未决矛盾。
|
||
</div>
|
||
</aside>
|
||
|
||
<article class="article">
|
||
<section class="article-section" id="compass">
|
||
<p class="eyebrow"><span>00</span> THIRTY-TWO QUESTION LEDGERS</p>
|
||
<h2>先把三十二个对象拆开，才不会把“规模、效率、能力”揉成一句话</h2>
|
||
<p class="lede">
|
||
技术报告最容易制造一种错觉：看完所有术语，却没有建立可追问的对象。
|
||
下面每张账只回答一个问题，同时写出答案不能被扩张到哪里。
|
||
</p>
|
||
<div class="ledger-grid">
|
||
{k3Ledgers.map(([id, object, question, answer]) => (
|
||
<article class="ledger-card">
|
||
<header><span>{id}</span><b>{object}</b></header>
|
||
<h3>{question}</h3>
|
||
<p>{answer}</p>
|
||
</article>
|
||
))}
|
||
</div>
|
||
<div class="reading-rule">
|
||
<span>贯穿全页的读法</span>
|
||
<p><b>先认状态</b><i>→</i><b>找到瓶颈</b><i>→</i><b>看机制怎样改写状态</b><i>→</i><b>核对系统代价</b><i>→</i><b>最后读评测</b></p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="report-map">
|
||
<p class="eyebrow"><span>01</span> REPORT MAP</p>
|
||
<h2>47 页不是一条直线：先知道每章在回答哪一层问题</h2>
|
||
<div class="report-map">
|
||
{k3ReportMap.map(([section, title, question, evidence]) => (
|
||
<article>
|
||
<span>{section}</span>
|
||
<div><b>{title}</b><p>{question}</p></div>
|
||
<small>{evidence}</small>
|
||
</article>
|
||
))}
|
||
</div>
|
||
<div class="thesis">
|
||
<span>报告结构里的隐藏主线</span>
|
||
<p>
|
||
§2 并不独立于 §5：KDA 的 recurrent state 决定 context parallelism 与 prefix cache；
|
||
极稀疏 LatentMoE 决定 expert communication；长 Agent 轨迹又决定 partial rollout 与可恢复 sandbox。
|
||
架构、训练、系统是同一条状态链的不同截面。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="architecture">
|
||
<p class="eyebrow"><span>02</span> THREE-DIMENSIONAL INFORMATION FLOW</p>
|
||
<h2>K3 的统一设计语言：Token、Layer、Channel 三个方向的信息流</h2>
|
||
<ArchitectureExplorer />
|
||
<div class="axis-grid">
|
||
<article><span>SEQUENCE / TOKEN</span><h3>KDA + Gated MLA</h3><p>低成本持续状态与周期性全局回看分工，不要求单一 attention 同时擅长所有时间尺度。</p></article>
|
||
<article><span>DEPTH / LAYER</span><h3>Block AttnRes</h3><p>当前层不只接收统一 residual sum，而能选择早期 block 的表示。</p></article>
|
||
<article><span>WIDTH / CHANNEL</span><h3>Stable LatentMoE</h3><p>shared path 保留完整宽度，routed path 压到 latent space 后从 896 个专家选 16 个。</p></article>
|
||
<article><span>MODALITY / OBSERVATION</span><h3>MoonViT-V2</h3><p>视觉与文本从训练开始共享 backbone 与 NTP objective，使 screenshot 能成为 Agent 轨迹里的连续 observation。</p></article>
|
||
</div>
|
||
<div class="hybrid-stack" aria-label="K3 3:1 hybrid attention 教学条带">
|
||
{Array.from({ length: 6 }, (_, block) => (
|
||
<div>
|
||
<span>KDA</span><span>KDA</span><span>KDA</span><b>MLA</b><small>B{block + 1}</small>
|
||
</div>
|
||
))}
|
||
<i>… 69 KDA + 24 MLA · 最后一层全局 attention</i>
|
||
</div>
|
||
<div class="warning-note">
|
||
<b>不要把 2.5× 贴到其中一个方块上</b>
|
||
<p>
|
||
报告把约 2.5× overall scaling efficiency 归因于 KDA、AttnRes、Stable LatentMoE、
|
||
训练 recipe 和数据的组合。它不是 KDA 单项消融，也不是线上推理 2.5×。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="specs">
|
||
<p class="eyebrow"><span>03</span> TABLE 1 / EXACT SPECIFICATION</p>
|
||
<h2>从 K2 到 K3：每个数字改变的是哪一本账</h2>
|
||
<div class="spec-table-wrap">
|
||
<table class="spec-table">
|
||
<thead><tr><th>FIELD</th><th>KIMI K2</th><th>KIMI K3</th><th>HOW TO READ</th></tr></thead>
|
||
<tbody>
|
||
{specs.map(([field, k2, k3, note]) => (
|
||
<tr><th>{field}</th><td>{k2}</td><td>{k3}</td><td>{note}</td></tr>
|
||
))}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<p>
|
||
2.78T 是总容量，104.2B 是激活路径规模；部署仍需放置大量 expert weights，
|
||
推理仍需 attention、router、shared experts、dispatch/combine 与 kernel。把 active parameters
|
||
直接当成 dense 等价成本，会漏掉系统账。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="kda">
|
||
<p class="eyebrow"><span>04</span> KIMI DELTA ATTENTION</p>
|
||
<h2>KDA 不保存每个历史配对，而是维护一份会遗忘、会纠错的工作记忆</h2>
|
||
<p class="lede">
|
||
标准 attention 让新 Query 直接查看许多历史 Key/Value；KDA 将历史压入固定形状矩阵状态
|
||
S<sub>t</sub>∈R<sup>dₖ×dᵥ</sup>。状态更便宜，但压缩也意味着它不是无损数据库。
|
||
</p>
|
||
<div class="formula-card">
|
||
<span>REPORT EQ. 1 / SINGLE HEAD</span>
|
||
<code>Sₜ = (I − βₜkₜkₜᵀ) Diag(αₜ) Sₜ₋₁ + βₜkₜvₜᵀ</code>
|
||
<code>õₜ = Sₜᵀqₜ</code>
|
||
<p>α∈(0,1)<sup>dₖ</sup> 逐 key channel 保留旧状态；β∈(0,1) 控制写入；Query 从更新后的 state 读取。</p>
|
||
</div>
|
||
<div class="mechanism-steps">
|
||
<article><span>01</span><h3>先遗忘</h3><p>Diag(α) 让不同 key channel 有不同记忆时长。</p></article>
|
||
<article><span>02</span><h3>再擦除旧预测</h3><p>(I−βkkᵀ) 抑制当前 key 在旧 state 中已经对应的内容。</p></article>
|
||
<article><span>03</span><h3>写入新 value</h3><p>βkvᵀ 把新关联写入；等价直觉是只补预测误差。</p></article>
|
||
<article><span>04</span><h3>按 Query 读取</h3><p>Sᵀq 从固定状态返回 value-space 表示，再经 RMSNorm 与 full-rank gate。</p></article>
|
||
</div>
|
||
<h3>为什么 chunkwise form 是工程关键</h3>
|
||
<p>
|
||
纯递归适合 decode，却会让训练沿序列串行。KDA 让 chunk 之间递归传 S，
|
||
chunk 内把 inter-chunk 与 causal intra-chunk 两部分改写成矩阵乘。
|
||
于是训练/prefill 能吃到并行硬件，decode 仍只更新固定 state。
|
||
</p>
|
||
<div class="state-flow">
|
||
<b>S[t]</b><i>→</i><div><span>CHUNK t</span><strong>parallel Q/K/V + causal tile</strong></div><i>→</i><b>S[t+1]</b><i>→</i><div><span>CHUNK t+1</span><strong>parallel inside / recurrent across</strong></div>
|
||
</div>
|
||
<p class="cross-link">继续读：<a href="https://arxiv.org/abs/2510.26692">Kimi Linear</a> 给出 KDA 前身与完整 UT transform；<a href="https://github.com/MoonshotAI/FlashKDA">FlashKDA</a> 是官方 kernel 入口。</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="bounded-decay">
|
||
<p class="eyebrow"><span>05</span> FIGURE 3 / LOWER-BOUNDED DECAY</p>
|
||
<h2>一个看似细小的函数改动，为什么会改变 diagonal tile 能否上 Tensor Core</h2>
|
||
<p>
|
||
chunkwise 公式需要用累计 retention Γ 缩放 Key；因为 Γ 是许多 (0,1) 数的乘积，1/Γ 可能爆大。
|
||
Kimi Linear 用负 Softplus，log-decay 下界为 −∞；K3 改成 scaled sigmoid：
|
||
</p>
|
||
<div class="formula-card two">
|
||
<div><span>KIMI LINEAR</span><code>g = −exp(A) Softplus(z) ∈ (−∞, 0)</code></div>
|
||
<div><span>KIMI K3 / EQ. 5</span><code>g = gmin Sigmoid(exp(A)z) ∈ (−5, 0)</code><code>α = exp(g) ∈ (e⁻⁵, 1)</code></div>
|
||
</div>
|
||
<div class="number-proof">
|
||
<article><span>ONE STEP</span><b>α > e⁻⁵ ≈ 6.7×10⁻³</b></article>
|
||
<i>× 16-token tile</i>
|
||
<article><span>CUMULATIVE LOG</span><b>Σg > −80</b></article>
|
||
<i>take reciprocal</i>
|
||
<article><span>RESCALE</span><b>1/Γ < e⁸⁰ ≈ 5.54×10³⁴</b></article>
|
||
</div>
|
||
<p>
|
||
BF16 最大有限量级约 3.39×10³⁸，因此报告默认最坏 rescale 仍在动态范围内。
|
||
这让 diagonal 和 off-diagonal causal tiles 都可使用稠密 Tensor Core 矩阵乘，
|
||
消除显式 position-pair diagonal path。动态范围内不等于无舍入误差，二者要分开。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="mla">
|
||
<p class="eyebrow"><span>06</span> GATED MLA / GLOBAL LOOKBACK</p>
|
||
<h2>工作记忆之外，K3 仍然周期性把全局历史摊开来看</h2>
|
||
<p>
|
||
固定状态擅长持续压缩，弱点是精确回看遥远 Token。K3 因而保留周期性 MLA：
|
||
每个历史 Token 的多头 K/V 先压成 latent c<sub>t</sub>，服务时缓存 latent，再恢复 attention 所需内容。
|
||
这条机制来自 <a href="https://arxiv.org/abs/2405.04434">DeepSeek‑V2</a>，是 K3 与 DeepSeek 最明确的继承边。
|
||
</p>
|
||
<div class="comparison-grid">
|
||
<article><span>KDA / RECURRENT</span><h3>固定状态</h3><p>成本随 decode step 近似固定；历史被压缩，位置与近因通过递归门控保留。</p></article>
|
||
<article><span>MLA / GLOBAL</span><h3>随 Token 增长的 latent cache</h3><p>仍做全局 token-to-token 内容交互；缓存比 MHA 小，但不固定。</p></article>
|
||
<article><span>HYBRID</span><h3>3:1 分工</h3><p>KDA 负责低成本连续混合，MLA 定期校正和全局检索，最后一层保证全局 attention。</p></article>
|
||
</div>
|
||
<h3>NoPE 不是“没有位置”</h3>
|
||
<p>
|
||
K3 的 MLA Query/Key 不加显式位置编码；穿插的 KDA 提供位置敏感、近因敏感的序列混合。
|
||
MLA 因此专注全局内容匹配，也避免扩展到 1M 时调整 RoPE base 或 YaRN。
|
||
这是一种跨模块分工，不是证明纯 MLA 可以忽略顺序。
|
||
</p>
|
||
<h3>FP32 attention output 是独立的数值合同</h3>
|
||
<p>
|
||
报告为纠正 FlashAttention 中有偏舍入误差，在训练中保留 FP32 attention output。
|
||
它会把输出 tile 的片上 footprint 加倍，所以 kernel 改为与 KV staging buffers 重叠，
|
||
腾出更深 KV pipeline 的共享内存。算法精度和 tile 排布在这里直接耦合。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="attnres">
|
||
<p class="eyebrow"><span>07</span> ATTENTION RESIDUALS</p>
|
||
<h2>把 attention 从时间轴旋转到深度轴：当前层选择自己要读的早层</h2>
|
||
<p class="lede">
|
||
普通 residual 把所有历史层持续压进一个 h<sub>l</sub>；这很利于梯度传播，却也让早期表示统一累加。
|
||
AttnRes 把 embedding 与早层输出当成一组 depth sources。
|
||
</p>
|
||
<div class="formula-card">
|
||
<span>REPORT EQ. 8–9 / FULL ATTNRES</span>
|
||
<code>αᵢ→ₗ = exp(qₗᵀ RMSNorm(kᵢ)) / Σⱼ exp(qₗᵀ RMSNorm(kⱼ))</code>
|
||
<code>hₗ = Σᵢ αᵢ→ₗvᵢ</code>
|
||
<p>qₗ 是 layer-specific learnable pseudo-query，不随当前 Token 内容改变；RMSNorm 防止大幅值来源垄断权重。</p>
|
||
</div>
|
||
<div class="depth-diagram">
|
||
<div><span>EMBED</span><b>b₀</b></div>
|
||
{Array.from({ length: 8 }, (_, index) => <div><span>12 LAYERS</span><b>b{index + 1}</b></div>)}
|
||
<i>→ final layer attends over 9 block-level sources</i>
|
||
</div>
|
||
<p>
|
||
Full 版的 O(L²d) 算术在 L<100 时尚可，实际压力是所有层输出必须存活，以及 pipeline 跨 stage 通信。
|
||
Block 版在 12 层内部求 partial sum，跨 block 才做完整深度 attention：
|
||
K3 有 8 个 layer blocks，加 embedding 共 9 个来源。报告 §7 芯片 nano 模型的 block size=2
|
||
只是案例配置，不能写回 2.78T 主模型。
|
||
</p>
|
||
<div class="evidence-note">
|
||
<b>证据边界</b>
|
||
<p>
|
||
<a href="https://arxiv.org/abs/2603.15031">Attention Residuals</a> 独立预印本在
|
||
48B-total / 3B-active、1.4T Token 等设置上做 scaling 与消融；K3 报告证明它进入了更大系统。
|
||
作为 2026 新方法，跨团队复现仍有限。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="latentmoe">
|
||
<span class="anchor-alias" id="moe" aria-hidden="true"></span>
|
||
<p class="eyebrow"><span>08</span> STABLE LATENTMOE</p>
|
||
<h2>896 选 16 的真正难点，不只在稀疏 FLOPs，而在 payload、权重流和激活稳定</h2>
|
||
<p>
|
||
conventional MoE 把完整 d 维 Token 发给每个选中专家；激活专家数变多时，
|
||
dispatch payload 与 expert-weight traffic 一起增长。LatentMoE 把公共与专业路径拆开：
|
||
</p>
|
||
<div class="moe-flow">
|
||
<div><span>FULL 7168</span><b>Token x</b><small>共享表示</small></div><i>↗</i>
|
||
<div class="shared"><span>ALWAYS ON</span><b>2 shared experts</b><small>full-width common path</small></div>
|
||
<div class="down"><span>W↓</span><b>3584 latent</b><small>routed payload</small></div><i>→</i>
|
||
<div class="experts"><span>TOP-K</span><b>896 choose 16</b><small>sparsity 56</small></div><i>→</i>
|
||
<div><span>RMSNORM + W↑</span><b>back to 7168</b><small>combine with shared</small></div>
|
||
</div>
|
||
<div class="formula-card">
|
||
<span>REPORT EQ. 11 / ROUTED + SHARED</span>
|
||
<code>u = Σᵢ∈Tk(x) pᵢ Eᵢʳᵒᵘᵗᵉᵈ(W↓x)</code>
|
||
<code>y = Σⱼ₌₁² Eⱼˢʰᵃʳᵉᵈ(x) + W↑ RMSNorm(u)</code>
|
||
<p>新增 RMSNorm 位于 routed aggregate 与 up-projection 之间，控制不同专家组合产生的尺度变化。</p>
|
||
</div>
|
||
<p>
|
||
routed branch 接近四次连续矩阵乘，2.8T 规模与极端稀疏会放大内部 activation explosion。
|
||
Stable LatentMoE 不是一个技巧，而是三件事的组合：Normalized LatentMoE、SiTU‑GLU、Quantile Balancing。
|
||
</p>
|
||
<a class="button primary" href="/moe/#k3">进入 MoE 专题：比较 DeepSeekMoE、LatentMoE、QB 与系统通信 →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="stability">
|
||
<p class="eyebrow"><span>09</span> FIGURE 4–5 / ACTIVATION × ROUTING</p>
|
||
<h2>SiTU 管数值幅值，QB 管专家负载；两者不能互相替代</h2>
|
||
<div class="two-column">
|
||
<article>
|
||
<span>SITU-GLU / EQ. 12</span>
|
||
<h3>给两个乘积分支分别加 smooth cap</h3>
|
||
<code>[β₁ tanh(Wg x/β₁) ⊙ Sigmoid(Wg x)] ⊙ [β₂ tanh(Wu x/β₂)]</code>
|
||
<p>报告用 β₁=4、β₂=25，标量切片满足 |f(x)|≤100；近原点保持近似线性与 SwiGLU 局部形状。</p>
|
||
</article>
|
||
<article>
|
||
<span>QUANTILE BALANCING / EQ. 14</span>
|
||
<h3>一次 forward 估计每个 expert 的进入门槛</h3>
|
||
<code>b̃ⱼ(t+1) ← −quantile₁₋ₖ/ₙ(s:,j − α(t))</code>
|
||
<p>Top-(k+1) 的最后一项给 token cutoff α；新 bias 下一 step 生效，最终 bias 冻结用于 inference。</p>
|
||
</article>
|
||
</div>
|
||
<p>
|
||
QB 的 bias 进入 Top-k 选择，但从最终 mixture weight p 中移除，因此调 dispatch 而不直接改 mixture 权重。
|
||
全局 batch 的数百万 margin 不宜 gather，实际用每 expert histogram：各 rank 只 all-reduce bin counts，
|
||
quantile 误差受 bin width 限制。
|
||
</p>
|
||
<div class="layer-separation">
|
||
<article><span>MODEL / QB</span><b>谁被选中</b><p>router score、cutoff、quantile、bias。</p></article>
|
||
<article><span>RUNTIME / MOONEP</span><b>怎样平衡执行</b><p>static shape、expert execution、zero-copy communication。</p></article>
|
||
<article><span>FLEET / ADMISSION</span><b>谁先进入系统</b><p>按 request class 给资源预算，避免 1M burst 饿死短请求。</p></article>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="vision">
|
||
<p class="eyebrow"><span>10</span> NATIVE VISION / FIGURE 6</p>
|
||
<h2>关键纠正：K3 不是“冻结语言模型、接上视觉塔、再逐步解冻”</h2>
|
||
<div class="correction">
|
||
<article class="wrong"><span>错误心智模型</span><b>pretrained LLM + attached vision encoder</b><p>先做 post-hoc modality alignment，再逐步解冻。</p></article>
|
||
<i>≠</i>
|
||
<article class="right"><span>K3 REPORT §2.4 / §3.3</span><b>vision + language jointly optimized from start</b><p>MoonViT‑V2 从头训练；视觉/文本 Token 交错在同一个 next-token prediction objective。</p></article>
|
||
</div>
|
||
<p>
|
||
MoonViT‑V2 约 401M、27 层、patch 14、12 heads；使用 RMSNorm，linear/attention projections 无 bias。
|
||
图像和视频共享参数，以 spatial/temporal factorized attention 处理时空关系，
|
||
temporal pooling 压视频长度，2×2 pixel shuffle 减少送入主干的视觉 Token，输入最高 3584×3584。
|
||
</p>
|
||
<div class="vision-flow">
|
||
<div><span>IMAGE / VIDEO</span><b>dynamic resolution</b></div><i>→</i>
|
||
<div><span>MOONVIT-V2</span><b>space × time factorized</b></div><i>→</i>
|
||
<div><span>POOL / SHUFFLE</span><b>temporal + 2×2 pixels</b></div><i>→</i>
|
||
<div><span>PROJECTOR</span><b>shared embedding</b></div><i>→</i>
|
||
<div><span>K3 BACKBONE</span><b>one NTP stream</b></div>
|
||
</div>
|
||
<div class="evidence-note">
|
||
<b>Figure 6 能支持到哪里</b>
|
||
<p>
|
||
作者消融中，从头训练的 MoonViT‑V2 比 SigLIP 初始化的 MoonViT‑3D 有更低 gradient norm 与更少 spike，
|
||
并在其视觉评测中匹配 baseline。这是特定大规模联合训练配置的结果，不是“视觉预训练普遍无用”。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="data">
|
||
<p class="eyebrow"><span>11</span> PRE-TRAINING DATA</p>
|
||
<h2>数据不是一个总 Token 数，而是五个域、清洗、重写与采样实验共同组成的配方</h2>
|
||
<div class="domain-grid">
|
||
{dataDomains.map(([name, title, detail]) => (
|
||
<article><span>{name}</span><h3>{title}</h3><p>{detail}</p></article>
|
||
))}
|
||
</div>
|
||
<p>
|
||
团队先用规则、质量分类器和去重筛数据，再用小模型 ablation 调不同 domain sampling。
|
||
Knowledge / Mathematics 使用 K2 生成多风格、多视角改写：长材料采用 chunk-wise 自回归生成，
|
||
最后做 fidelity verification。视觉坐标同时提供绝对值与 [0,1] 归一值，
|
||
programmatic data 覆盖 SVG、3D、Webpage、Game、CAD。
|
||
</p>
|
||
<div class="warning-note">
|
||
<b>报告没有公开的，网站不会补写</b>
|
||
<p>完整语料来源、精确域配比、总训练 Token、版权构成与完整过滤阈值均不足以独立复现；参数量不能填补这些空白。</p>
|
||
</div>
|
||
<a class="button" href="/pretraining/data/">进入数据专题：从去重、质量过滤到合成数据与污染审计 →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="scaling">
|
||
<p class="eyebrow"><span>12</span> FIGURE 7 / SCALING & RECIPE</p>
|
||
<h2>Scaling law 在 K3 中是实验导航器，不是“参数越大越好”的口号</h2>
|
||
<p>
|
||
团队在小规模模型上共同重调 batch size、learning rate、tokens per parameter 与 model shape，
|
||
并在 held-out OOD data 上比较候选。cosine 与 WSD 各自独立搜索最优超参，
|
||
作者设置中 cosine 最终 loss 更低。约 2.5× 的含义是：相对 K2，
|
||
K3 的架构、数据与 recipe 组合在 compute–loss scaling 上更有效。
|
||
</p>
|
||
<div class="efficiency-grammar">
|
||
<article class="yes"><span>可以说</span><b>同等 compute 下经验 loss 更低，或同等 loss 所需 compute 更少</b></article>
|
||
<article class="no"><span>不可以说</span><b>推理快 2.5×</b></article>
|
||
<article class="no"><span>不可以说</span><b>KDA 单独贡献 2.5×</b></article>
|
||
<article class="no"><span>不可以说</span><b>所有下游分数提升 2.5×</b></article>
|
||
</div>
|
||
<h3>完整训练 recipe 还有哪些角色</h3>
|
||
<p>
|
||
Per-Head Muon + K2 weight clipping；QB 负责 MoE load balance；cosine schedule、1% linear warmup、
|
||
weight decay=0.1。Per-Head 表示 Q/K/V 等多头矩阵按 head 分组应用 Muon，
|
||
不能把整体 scaling 收益单独归到 optimizer。
|
||
</p>
|
||
<a class="button" href="/scaling/">进入 Scaling 专题：读懂 loss、compute、数据与外推误差 →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="context">
|
||
<p class="eyebrow"><span>13</span> FOUR-STAGE CONTEXT CURRICULUM</p>
|
||
<h2>1M 不是配置文件里的一行数字，而是数据、算法和系统共同完成的四阶段课程</h2>
|
||
<div class="curriculum">
|
||
<article><span>PRETRAIN / 01</span><b>8K</b><p>低成本建立基础能力。</p></article><i>→</i>
|
||
<article><span>PRETRAIN / 02</span><b>64K</b><p>在预训练后段延长。</p></article><i>→</i>
|
||
<article><span>COOLDOWN / 03</span><b>256K</b><p>集中投入昂贵长序列。</p></article><i>→</i>
|
||
<article><span>COOLDOWN / 04</span><b>1M</b><p>适应目标窗口。</p></article>
|
||
</div>
|
||
<p>
|
||
自然长文档/视频要做 exact/fuzzy dedup、frame perceptual hashing、heuristic/classifier filtering
|
||
与 structural validation，并因稀缺而上采样。长度本身不训练跨远距离依赖，所以又把多模态文档/子任务
|
||
置换、拼接，要求答案依赖散落在整段 1M 中的证据。
|
||
</p>
|
||
<div class="plain-language">
|
||
<b>窗口容量 ≠ 使用能力</b>
|
||
<p>
|
||
一百万 Token 像允许把整间资料室搬进考场；能否跨文档找到证据、维持任务状态、在恰当时机压缩，
|
||
才是模型和 harness 是否真正会用它。
|
||
</p>
|
||
</div>
|
||
<a class="button" href="/long-context/">进入长上下文专题：比较 KDA、MLA、CSA/HCA 与 context management →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="sft">
|
||
<span class="anchor-alias" id="posttrain" aria-hidden="true"></span>
|
||
<p class="eyebrow"><span>14</span> SFT COLD START</p>
|
||
<h2>SFT 的任务不是教完所有能力，而是给长程 RL 一个可读、可调用工具的起点</h2>
|
||
<p>
|
||
K3 扩展前代 Kimi SFT pipeline，由 domain-specialized models 合成长 agent trajectory，
|
||
经多阶段验证与 human-in-the-loop annotation。所有复杂轨迹用 XTML 统一序列化，
|
||
使 reasoning、response、tool call 与动态工具边界保持一致。
|
||
</p>
|
||
<div class="process-line">
|
||
<div><span>PRIOR KIMI EXPERTS</span><b>trajectory synthesis</b></div><i>→</i>
|
||
<div><span>VERIFY</span><b>multi-stage checks</b></div><i>→</i>
|
||
<div><span>HITL</span><b>annotation</b></div><i>→</i>
|
||
<div><span>XTML</span><b>serialize channels</b></div><i>→</i>
|
||
<div><span>SFT</span><b>cold-start policy + QAT</b></div>
|
||
</div>
|
||
<p>MXFP4 expert weights / MXFP8 expert activations 的 QAT 从 SFT 开始，而不是在全部 RL 完成后临时量化。</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="rl">
|
||
<p class="eyebrow"><span>15</span> FIGURE 8 / NINE RL EXPERTS</p>
|
||
<h2>三类领域 × 三种思考强度：九位老师，最后进入一个学生</h2>
|
||
<div class="teacher-grid">
|
||
<div class="corner">DOMAIN × EFFORT</div><b>LOW</b><b>HIGH</b><b>MAX</b>
|
||
<strong>GENERAL TASKS</strong><span>reasoning · vision · search</span><span>knowledge · faithfulness</span><span>max-budget general</span>
|
||
<strong>GENERAL AGENTS</strong><span>assistant</span><span>deep research</span><span>long-horizon agent</span>
|
||
<strong>CODING AGENTS</strong><span>coding experience</span><span>SWE · web</span><span>kernel · hardest coding</span>
|
||
</div>
|
||
<p>
|
||
Figure 8 中，随着 RL FLOPs 增长，多个公开/内部评测分数与平均 assistant steps 同时上升。
|
||
这支持作者在其训练过程里的 scaling 观察，但不能直接推出“任何任务多走几步都会更好”。
|
||
</p>
|
||
<h3>Reasoning effort 是每题预算，不是三个固定长度</h3>
|
||
<div class="formula-card">
|
||
<span>PER-PROBLEM BUDGET</span>
|
||
<code>if T(y) > τ · b₀(x), override task reward with −1</code>
|
||
<p>
|
||
b₀(x) 由 cold-start model 估计；general task 主要计 thinking tokens，
|
||
agentic task 计 reasoning trace、tool-call arguments 等累计输出。τ 按 domain 由人工指导退火。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="mopd">
|
||
<p class="eyebrow"><span>16</span> PARTIAL ROLLOUT × MOPD</p>
|
||
<h2>一个机制解决长尾等待，另一个解决九位老师怎样教同一个学生</h2>
|
||
<h3>Partial rollout：不等最慢轨迹</h3>
|
||
<p>
|
||
每轮 N 个 prompts、每题 K 条 completion，保持 N×K 条 active trajectories。
|
||
当 λN K 条完成就暂停 generation，先进行 policy optimization；未完成轨迹进队列，
|
||
下一轮优先恢复。好处是减少 straggler，代价是同一长轨迹跨多个 iteration，
|
||
数据变 stale/off-policy；报告用 per-token regularization 把更新限制在局部邻域。
|
||
</p>
|
||
<h3>MOPD：学生走自己的 prefix，老师在那个位置给 dense reward</h3>
|
||
<div class="formula-card">
|
||
<span>REPORT EQ. 15 / TOKEN REWARD</span>
|
||
<code>rᵈₒₚd(yₜ|e,x,y<t) = clip(sg log[πteacherᵈ,ᵉ(yₜ|x,y<t) / πθ(yₜ|e,x,y<t)], −Rmax, Rmax)</code>
|
||
<p>domain d 与 effort e 选择对应 teacher；stop-gradient 防 teacher ratio 被学生优化反向改变。</p>
|
||
</div>
|
||
<div class="mopd-flow">
|
||
<div><span>STUDENT</span><b>generate current prefix</b></div><i>→</i>
|
||
<div><span>SELECT</span><b>domain d × effort e</b></div><i>→</i>
|
||
<div><span>TEACHER</span><b>score next-token distribution</b></div><i>→</i>
|
||
<div><span>DENSE REWARD</span><b>clipped log-ratio</b></div><i>→</i>
|
||
<div><span>ONE MODEL</span><b>consolidated policy</b></div>
|
||
</div>
|
||
<p>
|
||
九个 expert models 是训练老师，不是推理时投票。on-policy 的意义是教师覆盖学生真正访问的 prefix，
|
||
避免只模仿固定离线答案；它仍受 teacher 质量、domain mapping 与 compute 约束。
|
||
</p>
|
||
<a class="button primary" href="/reasoning/">进入推理专题：比较 GRPO、R1、effort control 与蒸馏 →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="deployment">
|
||
<p class="eyebrow"><span>17</span> DEPLOYMENT-AWARE POST-TRAINING</p>
|
||
<h2>部署精度和 speculative acceptance 在训练中就成为目标</h2>
|
||
<div class="precision-contract">
|
||
<article><span>ROUTED EXPERT WEIGHTS</span><b>MXFP4</b><p>占大多数参数内存。</p></article>
|
||
<article><span>EXPERT ACTIVATIONS</span><b>MXFP8</b><p>rollout 与 training 同方案。</p></article>
|
||
<article><span>NON-EXPERT</span><b>higher precision</b><p>attention、latent projections、shared experts、router。</p></article>
|
||
<article><span>LIFECYCLE</span><b>SFT + RL</b><p>全 post-training QAT，减少 train–inference mismatch。</p></article>
|
||
</div>
|
||
<p>
|
||
预训练 MTP 层结构与 EAGLE‑3 的单 decoder draft 相近，因此冻结 target model，
|
||
只调 draft layer 和 feature-fusion projection。输入融合第 1、第 4、最后 AttnRes blocks 的低/中/高层特征，
|
||
训练展开 7 steps，逼近真实 recurrent drafting。
|
||
</p>
|
||
<div class="formula-card">
|
||
<span>REPORT EQ. 16 / LK LOSS</span>
|
||
<code>Lᴸᴷ = −log Σₓ∈V min(p(x), q(x))</code>
|
||
<p>p/q 是 target/draft next-token distribution；直接优化无损 speculative sampling 的接受率，而不是只用 KL surrogate。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="harness">
|
||
<p class="eyebrow"><span>18</span> UNIFIED WHITE-BOX RL ENVIRONMENT</p>
|
||
<h2>Agent harness 被拆成可组合模块，避免模型只会一种工具口音</h2>
|
||
<p>
|
||
固定 harness 会让模型过拟合 tool schema、system prompt、context management 与 interaction protocol。
|
||
K3 把 harness 表示为配置化模块集合，训练时按 task group 动态组合，
|
||
可实例化 Kimi Code、Claude Code、Codex、OpenClaw、Hermes 等风格或全新组合。
|
||
</p>
|
||
<div class="harness-parts">
|
||
{["TOOLS", "SYSTEM PROMPT", "CONTEXT", "SKILLS", "MEMORY", "SUBAGENTS", "PROTOCOL", "VERIFIER"].map((item, index) => (
|
||
<div><span>{String(index + 1).padStart(2, "0")}</span><b>{item}</b></div>
|
||
))}
|
||
</div>
|
||
<div class="plain-language">
|
||
<b>评测时也要记住这件事</b>
|
||
<p>代码/Agent 分数是 model × harness × environment × verifier 的系统结果。模型名不是完整实验条件。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="environments">
|
||
<p class="eyebrow"><span>19</span> FIGURE 9–10 / TASKS & ENVIRONMENTS</p>
|
||
<h2>可靠 Agent 的学习单位，是状态、动作与独立验证组成的一整段轨迹</h2>
|
||
<div class="environment-list">
|
||
{environments.map(([number, title, detail]) => (
|
||
<article><span>{number}</span><h3>{title}</h3><p>{detail}</p></article>
|
||
))}
|
||
</div>
|
||
<h3>Knowledge-graph-guided task synthesis</h3>
|
||
<div class="process-line four">
|
||
<div><span>CONCEPT GRAPH</span><b>细粒度关键词与层级</b></div><i>→</i>
|
||
<div><span>MATERIAL</span><b>论文 · 博客 · 代码仓库</b></div><i>→</i>
|
||
<div><span>TASK TYPE</span><b>knowledge · coding · vision</b></div><i>→</i>
|
||
<div><span>VERIFY</span><b>规则 / 测试 / 独立 judge</b></div>
|
||
</div>
|
||
<h3>Autonomous Execution Tasks</h3>
|
||
<p>
|
||
AET 只给初始环境、目标、约束、工具和 verifier，不提供 reference trajectory。
|
||
public verifier 给诊断，hidden verifier 检查保留场景，并限制提交预算降低 reward hacking。
|
||
奖励落在独立读取的环境结果，而不是 Agent 自己说“完成了”。
|
||
</p>
|
||
<a class="button" href="/agents/#k3-whitebox">进入 Agent 专题：完整拆解 white-box harness、AET 与 verifier →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="pretrain-systems">
|
||
<p class="eyebrow"><span>20</span> PRE-TRAINING INFRASTRUCTURE</p>
|
||
<h2>从算子到 pipeline：先问每种 state 放在哪里、什么时候移动</h2>
|
||
<div class="state-stack">
|
||
{stateStack.map(([layer, state, mechanism]) => (
|
||
<article><span>{layer}</span><h3>{state}</h3><p>{mechanism}</p></article>
|
||
))}
|
||
</div>
|
||
<h3>FlashKDA 与 KDA Context Parallelism</h3>
|
||
<p>
|
||
bounded decay 让全部 causal tiles 使用 dense Tensor Core path；
|
||
KCP 则沿 sequence dimension 分片 1M Token。不同 sequence regime 需要不同 fused kernel，
|
||
算法公式、secondary tile 与设备内存共同决定实现。
|
||
</p>
|
||
<h3>多模态 pipeline 不是均匀 decoder stack</h3>
|
||
<p>
|
||
vision encoder 的计算形状和 decoder 不同，会制造 pipeline bubble。
|
||
§5.2 / Figure 11 的重点是重新安排 encoder、pipeline 与 offload，让参数/activation 的放置不把 3T 训练拖成等待链。
|
||
报告没有公开完整 GPU 数和总训练成本，页面不从示意图反推。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="moonep">
|
||
<p class="eyebrow"><span>21</span> MOONEP / EXPERT PARALLEL</p>
|
||
<h2>模型路由“尽量均衡”以后，系统还要把不规则 Token 变成可执行的静态形状</h2>
|
||
<p>
|
||
MoE 的理论 FLOPs 不包含 all-to-all、专家权重读取、padding 与慢 expert 长尾。
|
||
MoonEP 追求 balanced expert execution，以 static computation shape 与 zero-copy communication
|
||
处理 dispatch/combine，并让通信与 shared-expert 等计算重叠。
|
||
</p>
|
||
<div class="moonep-flow">
|
||
<div><span>ROUTER</span><b>token → expert IDs</b></div><i>→</i>
|
||
<div><span>DISPATCH</span><b>zero-copy communication</b></div><i>→</i>
|
||
<div><span>STATIC SHAPE</span><b>balanced execution</b></div><i>→</i>
|
||
<div><span>COMBINE</span><b>overlap communication</b></div>
|
||
</div>
|
||
<div class="warning-note">
|
||
<b>三种“平衡”必须分开</b>
|
||
<p>QB 让 router 负载接近目标；MoonEP 让设备执行可预测；fleet admission 让不同请求类不互相饿死。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="rl-systems">
|
||
<p class="eyebrow"><span>22</span> LONG-HORIZON RL STATE</p>
|
||
<h2>一条轨迹跨多个训练 step 时，语言上下文和外部世界都必须能够暂停与恢复</h2>
|
||
<p>
|
||
重新生成已经调用数百次工具、积累几十万 Token 的轨迹极其浪费。K3 将 KV retention 外置，
|
||
partial rollout 可暂停/续接；AgentENV 用 Firecracker microVM 保存代码、文件、应用与系统状态。
|
||
</p>
|
||
<div class="agentenv-grid">
|
||
<article><span>PAUSE / RESUME</span><b>等待 inference 时释放 CPU / memory</b><p>报告称等待可占 sandbox 生命周期 98%。</p></article>
|
||
<article><span>FORK</span><b>从完全相同状态分叉</b><p>可让 reward judge 检查而不污染原环境。</p></article>
|
||
<article><span>SNAPSHOT</span><b>定期恢复点</b><p>错误后不必从任务最初重来。</p></article>
|
||
<article><span>INCREMENTAL</span><b>只保存 dirty pages</b><p>报告最低 checkpoint / resume 133ms / 49ms。</p></article>
|
||
</div>
|
||
<div class="number-band">
|
||
<div><span>SANDBOXES</span><b>51,219,741</b></div>
|
||
<div><span>IMAGES</span><b>1,505,678</b></div>
|
||
<div><span>MEMORY OVERCOMMIT</span><b>up to 6.5×</b></div>
|
||
<p>均为 K3 报告的训练/评测基础设施数字，本站未独立复跑。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="cache">
|
||
<p class="eyebrow"><span>23</span> FIGURE 12 / KDA-AWARE PREFIX CACHE</p>
|
||
<h2>MLA KV 随 Token 增长，KDA state 固定却很大；一个 prefix 只有两者同时恢复才可复用</h2>
|
||
<p>
|
||
分开 manager 会重复 allocation、eviction、transfer。K3 把 KDA states 与 MLA KV 放进同字节大小的统一 page pool，
|
||
共享 allocation、reference counting、eviction；KDA 各 head byte stream 连续，跨 prefill/decode 不同 TP
|
||
时在 transfer path re-layout。
|
||
</p>
|
||
<div class="cache-anatomy">
|
||
<header><span>PHYSICAL BLOCK / 6144 TOKENS</span><b>12 × 512-token hash blocks</b></header>
|
||
<div class="hash-blocks">
|
||
{Array.from({ length: 12 }, (_, index) => (
|
||
<div class={index < 5 ? "hit" : ""}><span>{index + 1}</span><i></i><b>{(index + 1) * 512}</b>{index === 4 ? <em>● KDA</em> : <em>○</em>}</div>
|
||
))}
|
||
</div>
|
||
<p>Figure 12 示例：请求匹配到 2800，最长 joint hit 在 B=2560；复用五个 MLA hash blocks 与该处 KDA checkpoint。</p>
|
||
</div>
|
||
<h3>为什么 hash granularity 与 physical allocation 要解耦</h3>
|
||
<p>
|
||
KDA checkpoint 大，只能稀疏保存；若把 hash 也绑到 1024–6144 的物理块，短请求几乎无法命中，
|
||
chunked prefill 也要等完整块。K3 允许 512-token hash endpoint 落在粗 physical block 内，
|
||
KDA checkpoint 只存可查询 endpoint 的稀疏子集。
|
||
</p>
|
||
<h3>并发一致性还需要三条约束</h3>
|
||
<ol>
|
||
<li>所有 cache group 分配前先 pin 全部 hit blocks，避免一个 group 的 COW 驱逐另一个刚命中的块；</li>
|
||
<li>当前 scheduling step 新分配/注册的块在 GPU copy 落地前不可匹配；</li>
|
||
<li>一个 KDA group checkpoint 被驱逐时，siblings 原子失效：要么每组都有，要么全部不可命中。</li>
|
||
</ol>
|
||
</section>
|
||
|
||
<section class="article-section" id="serving">
|
||
<p class="eyebrow"><span>24</span> DEVICE KERNELS × FLEET SCHEDULING</p>
|
||
<h2>服务端的目标从“平均快”变成：状态正确、单 Token 低延迟、长短请求互不拖垮</h2>
|
||
<div class="serving-grid">
|
||
<article>
|
||
<span>KDA DECODE</span><h3>缓存 projected inputs，不缓存每个 draft state</h3>
|
||
<p>speculative rejection 后 state 难回滚；重新在片上 replay accepted prefix，再写 verified/bonus states。</p>
|
||
</article>
|
||
<article>
|
||
<span>BLOCK ATTNRES</span><h3>prefill 用 SP，decode 用 side stream + fusion</h3>
|
||
<p>避免每个 TP rank 复制 block representations；inter-block overlap，intra-block merge/RMSNorm fusion。</p>
|
||
</article>
|
||
<article>
|
||
<span>LATENTMOE</span><h3>融合 down-projection/router 与通信</h3>
|
||
<p>latent weight 分片，output all-gather 进 GEMM epilogue；小 batch routed experts 用 token-centric kernel。</p>
|
||
</article>
|
||
<article>
|
||
<span>FLEET</span><h3>cache affinity + budget admission</h3>
|
||
<p>session 绑定 primary/secondary clusters；长请求只消耗自己的 class budget，不饿死短请求。</p>
|
||
</article>
|
||
</div>
|
||
<div class="plain-language">
|
||
<b>为什么 prefix affinity 如此重要</b>
|
||
<p>
|
||
报告给出 typical coding input：400K prefix、只增加 4K。cache hit 可避免重做整个 prefill；
|
||
而生产请求从 <2K 到 1M，单请求成本跨约三个数量级，按“平均请求”规划会失效。
|
||
</p>
|
||
</div>
|
||
<a class="button" href="/systems/inference/">进入推理系统专题：KV、prefix cache、speculative 与调度 →</a>
|
||
</section>
|
||
|
||
<section class="article-section" id="evaluation">
|
||
<p class="eyebrow"><span>25</span> FIGURE 1 & 13 / TABLE 2–5</p>
|
||
<h2>先读协议，再读分数：K3 报告自己的总体结论比单榜截图更克制</h2>
|
||
<p class="lede">
|
||
报告结论是：K3 在其套件中领先被比较的其他开放与部分闭源模型，
|
||
但整体仍落后 Claude Fable 5 与 GPT‑5.6 Sol。任何单项第一都不能覆盖这句总体判断。
|
||
</p>
|
||
<div class="eval-axes">
|
||
<article><span>REASONING & KNOWLEDGE</span><b>GPQA · HLE · AA-LCR · CritPt</b></article>
|
||
<article><span>CODING</span><b>DeepSWE · Terminal-Bench · SciCode · FrontierSWE</b></article>
|
||
<article><span>AGENTIC</span><b>BrowseComp · DeepSearchQA · OSWorld · Office · SaaS</b></article>
|
||
<article><span>VISION</span><b>WorldVQA · OmniDoc · Video-MME · MMMU-Pro · Math-Vision</b></article>
|
||
</div>
|
||
<div class="protocol-grid">
|
||
<article><span>01</span><h3>Reasoning effort</h3><p>K3 主结果多用 max；比较模型尽量用 max/xhigh。成本与延迟不相等。</p></article>
|
||
<article><span>02</span><h3>Harness</h3><p>代码/Agent 结果携带 Codex、Kimi Code、Claude Code 等脚手架。</p></article>
|
||
<article><span>03</span><h3>Tool augmentation</h3><p>HLE、视觉数学等有/无工具必须分列，不可把工具增强写成纯模型能力。</p></article>
|
||
<article><span>04</span><h3>Fallback / guard</h3><p>闭源 fallback、拒答或 cyber guard 会改变特定任务结果。</p></article>
|
||
<article><span>05</span><h3>Date / service</h3><p>模型、价格和服务行为会变，第三方比较必须附评测日期。</p></article>
|
||
<article><span>06</span><h3>Cost / output</h3><p>Figure 13 同时画 score 与 token/cost，能力不该脱离预算阅读。</p></article>
|
||
</div>
|
||
<div class="case-callout">
|
||
<span>BROWSECOMP / CONTEXT MANAGEMENT</span>
|
||
<b>300K + compression: 91.2 vs full 1M without management: 90.4</b>
|
||
<p>这是报告设置里的工程信号：更大窗口不自动消灭 context management，适时压缩可能更有效。</p>
|
||
</div>
|
||
<div class="warning-note">
|
||
<b>局限必须与亮点同页</b>
|
||
<p>报告与外部材料还提示 research reasoning、cyber 等能力边界；新架构缺少广泛第三方复现，完整训练数据、集群与成本未披露。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="cases">
|
||
<p class="eyebrow"><span>26</span> §7 CASE STUDIES</p>
|
||
<h2>案例展示“Agent 能走多远”，但每个数字都必须带着硬件、时间和任务边界</h2>
|
||
<div class="case-grid">
|
||
<article><span>KERNEL / 24H PER TASK</span><h3>AttnRes 283.6 → 114.4 ms</h3><p>报告还给出 DSA/KDA reductions 55.1%/73.6%，MLA 超过一半 peak；均为作者个案。</p></article>
|
||
<article><span>MINITRITON / L20</span><h3>从 compiler 到 distributed stack</h3><p>模型构建 tensor library、autograd、compiler 与 distributed；roofline 结果只限报告硬件/任务。</p></article>
|
||
<article><span>CHIP / 48H</span><h3>nano-kpu 与 RTL simulation</h3><p>nano model 同型但 AttnRes block size=2；4mm²、100MHz、>8700 tok/s 等不是主模型推理规格。</p></article>
|
||
<article><span>KNOWLEDGE / RESEARCH / VIDEO</span><h3>长程交付物与视觉闭环</h3><p>展示工具、证据、迭代与产物；属于作者选择的 case study，不是平均成功率。</p></article>
|
||
</div>
|
||
<div class="evidence-note">
|
||
<b>案例的正确用途</b>
|
||
<p>用来研究 trajectory、工具与 verifier 怎样协作；不能用一个成功案例推断所有真实任务的可靠性。</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="xtml">
|
||
<p class="eyebrow"><span>27</span> APPENDIX F / XTML</p>
|
||
<h2>消息格式不是装饰：它决定选项放在哪里、哪些 prefix 可以复用、工具何时动态出现</h2>
|
||
<div class="xtml-stage">
|
||
<div class="global"><span>GLOBAL OPTIONS</span><b>history 之前</b><p>tool declarations · reasoning effort</p></div>
|
||
<i>[open]</i>
|
||
<div><span>HISTORY</span><b>messages + channels</b><p>think · response · tools</p></div>
|
||
<i>[sep]</i>
|
||
<div class="oneshot"><span>ONE-SHOT OPTIONS</span><b>history 之后</b><p>避免破坏已有 KV prefix</p></div>
|
||
<i>[close]</i>
|
||
<div><span>NEW INPUT</span><b>dynamic tools</b><p>会话中可加入 input options</p></div>
|
||
<i>[end_of_msg]</i>
|
||
</div>
|
||
<p>
|
||
XTML 用 `[open]`、`[sep]`、`[close]`、`[end_of_msg]` 标消息边界；
|
||
think / response / tools 分 channel；thinking / instruct 由 prefix 选择。
|
||
global options 放历史前，one-shot options 放历史后，是为了既传控制又保留 KV cache。
|
||
</p>
|
||
</section>
|
||
|
||
<section class="article-section" id="lab">
|
||
<p class="eyebrow"><span>28</span> EIGHT INTERACTIVE WORKBENCHES</p>
|
||
<h2>现在动手：从标量递推一路算到混合 prefix cache</h2>
|
||
<p class="lede">
|
||
八张实验台各自声明“精确计算”和“教学模型”的边界。
|
||
重点不是玩滑条，而是看哪个量发生变化、哪个结论仍然不能推出。
|
||
</p>
|
||
<K3ReportLab />
|
||
</section>
|
||
|
||
<section class="article-section" id="artifacts">
|
||
<p class="eyebrow"><span>29</span> OPEN ARTIFACT FORENSICS</p>
|
||
<h2>从“报告说了什么”走到“公开 checkpoint 实际长什么样”</h2>
|
||
<p class="lede">
|
||
第三轮固定到官方 Hugging Face revision，读取 config、remote code、60 MB tensor index、
|
||
四个 safetensors headers 和两个小范围参数切片。原始权重不进入本站仓库；
|
||
结构、shape、计数、参数统计、隔离 wheel 与 RTX 5090 执行结果都可以从公开脚本重复生成。
|
||
</p>
|
||
<div class="artifact-callout">
|
||
<article><span>O / OBSERVED</span><b>1.4196 TiB tensor data</b><p>96 shards、497,220 entries；不是运行显存，也不是参数量口径。</p></article>
|
||
<article><span>D / CLOSED LOOP</span><b>69 KDA · 24 MLA · 92 MoE</b><p>配置、tensor names 与 header shape 三方闭合。</p></article>
|
||
<article><span>X / RTX 5090</span><b>exact 6/6 · max error 0</b><p>官方 torch reference；fixed BF16 mean 2.6210 ms。</p></article>
|
||
<article class="warning"><span>U / UNRESOLVED</span><b>A_log [128] ≠ expected [96]</b><p>checkpoint 与公开代码 / kernel API 的形状冲突保留在主视区，不擅自解释。</p></article>
|
||
</div>
|
||
<K3ArtifactLab />
|
||
<div class="hero-actions">
|
||
<a class="button primary" href="https://huggingface.co/moonshotai/Kimi-K3">打开官方开放权重</a>
|
||
<a class="button" href="https://github.com/MoonshotAI/FlashKDA">打开 FlashKDA 官方实现</a>
|
||
<a class="button" href="https://github.com/MoonshotAI/FlashKDA/blob/master/BENCHMARK_GB200.md">核对作者 GB200 benchmark</a>
|
||
<a class="button" href="https://git.k1412.top/wuyang/llm-atlas/src/branch/main/experiments/k3/flashkda">复跑本站 RTX 5090 探针</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="attnres-reduced">
|
||
<p class="eyebrow"><span>30</span> REDUCED ATTENTION RESIDUALS STUDY</p>
|
||
<h2>真实 K3 权重还不能诚实 forward；先把一个可证伪的 AttnRes 问题完整做完</h2>
|
||
<p class="lede">
|
||
checkpoint 的 <code>A_log [128]</code> 与 config、remote code、FlashKDA、vLLM 和 SGLang
|
||
期望的 96 heads 仍没有公开转换合同。本轮不裁剪权重冒充 K3，而是预注册一个从零训练的缩小实验：
|
||
相同 16-block Transformer、相同数据窗口与相同初始化，只改变 residual source 的读取拓扑。
|
||
</p>
|
||
<div class="artifact-callout">
|
||
<article><span>F / FROZEN</span><b>3 structures × 3 seeds</b><p>9 格各 2,000 steps；每格 16,384,000 target bytes。</p></article>
|
||
<article><span>X / OBSERVED</span><b>Full −0.01457 BPC</b><p>三个 paired seed 同为负，达到预注册 −0.010 判据。</p></article>
|
||
<article><span>X / OBSERVED</span><b>Block −0.04247 BPC</b><p>同样满足本 reduced protocol 内的方向支持规则。</p></article>
|
||
<article class="warning"><span>B / BOUNDARY</span><b>gradient CV 未复现</b><p>Baseline 0.3447；Full 0.5087；Block 0.6306。</p></article>
|
||
</div>
|
||
<K3AttnResTraceLab />
|
||
<div class="hero-actions">
|
||
<a class="button primary" href="https://git.k1412.top/wuyang/llm-atlas/src/branch/main/research/K3_ATTNRES_REDUCED_AUDIT.md">阅读完整研究审计</a>
|
||
<a class="button" href="https://git.k1412.top/wuyang/llm-atlas/src/branch/main/experiments/k3/attnres">复跑公开实验代码</a>
|
||
<a class="button" href="https://arxiv.org/abs/2603.15031">Attention Residuals 原论文</a>
|
||
<a class="button" href="https://github.com/MoonshotAI/Attention-Residuals">官方论文工件</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="attnres-gradient">
|
||
<p class="eyebrow"><span>31</span> GRADIENT DEFINITION × DEPTH SCALE</p>
|
||
<h2>“论文说梯度更均匀”，和上一轮参数梯度反结果，测的是同一件事吗？</h2>
|
||
<p class="lede">
|
||
第五轮先审计 Attention Residuals 官方论文与仓库：Figure 5(c) 没有公开 gradient tensor、
|
||
norm、reduction、diagnostic batch、AMP / clipping 时点或统计代码。本站因此冻结一个可复现的
|
||
post-MLP output activation-gradient 定义，把深度扩到 16 / 32 blocks、预算扩到 8,000 steps，
|
||
再用三 seed 检查“首尾平衡”和“全层离散度”是否真的同方向。
|
||
</p>
|
||
<div class="artifact-callout">
|
||
<article><span>F / FROZEN</span><b>12 × 8,000 steps</b><p>786,432,000 formal target bytes；两深度、两结构、三 seed。</p></article>
|
||
<article><span>X / OBSERVED</span><b>first/last 6 / 6 改善</b><p>depth-16 平均 61.0%；depth-32 平均 72.0%。</p></article>
|
||
<article class="warning"><span>X / COUNTEREVIDENCE</span><b>CV 6 / 6 恶化</b><p>局部尖峰让 depth-16 / 32 平均相对恶化 10.3% / 60.0%。</p></article>
|
||
<article><span>R / REPLAY</span><b>model + optimizer exact</b><p>指定 32-layer Block 格从零重训 8,000 steps，冻结字段逐项一致。</p></article>
|
||
</div>
|
||
<K3AttnResGradientLab />
|
||
<div class="hero-actions">
|
||
<a class="button primary" href="https://git.k1412.top/wuyang/llm-atlas/src/branch/main/research/K3_ATTNRES_GRADIENT_SCALE_AUDIT.md">阅读完整结果审计</a>
|
||
<a class="button" href="https://git.k1412.top/wuyang/llm-atlas/src/branch/main/research/K3_ATTNRES_GRADIENT_DEFINITION_AUDIT.md">核对论文定义边界</a>
|
||
<a class="button" href="https://git.k1412.top/wuyang/llm-atlas/src/branch/main/experiments/k3/attnres_gradient">复跑 12 格实验</a>
|
||
<a class="button" href="https://github.com/MoonshotAI/Attention-Residuals">官方一手工件</a>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="audit">
|
||
<p class="eyebrow"><span>32</span> FIGURE & TABLE AUDIT</p>
|
||
<h2>Figure 1–16、Table 1–5：每张图究竟支持什么，不能支持什么</h2>
|
||
<div class="figure-atlas">
|
||
{k3FigureAtlas.map(([id, report, title, contract]) => (
|
||
<article>
|
||
<header><span>{id}</span><b>{report}</b></header>
|
||
<h3>{title}</h3>
|
||
<p>{contract}</p>
|
||
</article>
|
||
))}
|
||
</div>
|
||
<div class="audit-legend">
|
||
<div><span>R</span><b>REPORT</b><p>作者直接陈述或报告数字。</p></div>
|
||
<div><span>P</span><b>PRIMARY</b><p>原论文、官方代码或 benchmark。</p></div>
|
||
<div><span>D</span><b>DERIVATION</b><p>由公开公式确定性计算。</p></div>
|
||
<div><span>T</span><b>TOY</b><p>帮助理解方向，不是实测。</p></div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="papers">
|
||
<p class="eyebrow"><span>↳</span> ONE HUNDRED PRIMARY NODES</p>
|
||
<h2>不要一次读完：沿着正在困惑的那条机制往回走</h2>
|
||
<p class="lede">
|
||
这 100 个节点经过标题、年份、作用与 URL 核对；优先采用论文原页、官方仓库与 benchmark 官方入口。
|
||
它们不是“引用越多越好”，而是把 K3 放回 attention、MoE、数值、RL、系统与评测的历史脉络。
|
||
</p>
|
||
{paperGroups.map(([group, label]) => {
|
||
const items = k3PaperChain.filter((paper) => paper[5] === group);
|
||
if (!items.length) return null;
|
||
return (
|
||
<div class="paper-group">
|
||
<header><span>{group}</span><h3>{label}</h3><b>{String(items.length).padStart(2, "0")} NODES</b></header>
|
||
<div class="paper-chain">
|
||
{items.map(([id, year, title, href, reason]) => (
|
||
<a class="paper-row" href={href}>
|
||
<time>{id} / {year}</time><b>{title}</b><p>{reason}</p>
|
||
</a>
|
||
))}
|
||
</div>
|
||
</div>
|
||
);
|
||
})}
|
||
<div class="hero-actions">
|
||
<a class="button primary" href="https://arxiv.org/abs/2607.24653">K3 论文原页</a>
|
||
<a class="button" href="https://github.com/MoonshotAI/Kimi-K3">官方仓库与权重</a>
|
||
<a class="button" href="/deepseek/">重点深读 DeepSeek 技术谱系</a>
|
||
<a class="button" href="/roadmap/">回到完整课程地图</a>
|
||
</div>
|
||
</section>
|
||
</article>
|
||
</div>
|
||
|
||
<style>
|
||
.k3-hero { border-bottom-color: var(--copper); }
|
||
.anchor-alias { position: relative; top: -88px; display: block; visibility: hidden; }
|
||
.artifact-callout {
|
||
display: grid;
|
||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||
max-width: 1080px;
|
||
margin: 32px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
.artifact-callout article {
|
||
min-height: 145px;
|
||
padding: 20px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
background: var(--paper-raised);
|
||
}
|
||
.artifact-callout article.warning {
|
||
border-color: var(--signal);
|
||
background: color-mix(in srgb, var(--signal) 7%, var(--paper));
|
||
}
|
||
.artifact-callout span { color: var(--copper); font: .58rem/1.2 var(--mono); letter-spacing: .08em; }
|
||
.artifact-callout b { display: block; margin-top: 13px; font: 800 .85rem/1.3 var(--mono); }
|
||
.artifact-callout p { margin: 12px 0 0; color: var(--muted); font-size: .68rem; line-height: 1.6; }
|
||
.ledger-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||
max-width: 1080px;
|
||
margin: 34px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
.ledger-card {
|
||
min-height: 220px;
|
||
padding: 20px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
background: var(--paper-raised);
|
||
}
|
||
.ledger-card header { display: flex; justify-content: space-between; gap: 12px; }
|
||
.ledger-card header span, .ledger-card header b { font: .58rem/1 var(--mono); letter-spacing: .08em; }
|
||
.ledger-card header span { color: var(--copper); }
|
||
.ledger-card header b { color: var(--muted); }
|
||
.ledger-card h3 { margin: 32px 0 13px; font-size: .94rem; line-height: 1.45; }
|
||
.ledger-card p { color: var(--muted); font-size: .72rem; line-height: 1.65; }
|
||
.reading-rule {
|
||
max-width: 1080px;
|
||
margin: 30px 0;
|
||
padding: 23px;
|
||
color: var(--paper);
|
||
background: var(--ink);
|
||
}
|
||
.reading-rule > span { color: var(--copper); font: .58rem/1 var(--mono); letter-spacing: .1em; }
|
||
.reading-rule p { display: flex; flex-wrap: wrap; gap: 13px; align-items: center; margin: 18px 0 0; }
|
||
.reading-rule b { font-size: .77rem; }
|
||
.reading-rule i { color: var(--copper); font-style: normal; }
|
||
.report-map { max-width: 960px; margin: 34px 0; border-top: 1px solid var(--line); }
|
||
.report-map article {
|
||
display: grid;
|
||
grid-template-columns: 62px minmax(0, 1fr) 180px;
|
||
gap: 20px;
|
||
align-items: center;
|
||
padding: 18px 0;
|
||
border-bottom: 1px solid var(--line);
|
||
}
|
||
.report-map > article > span { color: var(--copper); font: 700 .75rem var(--mono); }
|
||
.report-map b { font-size: .84rem; }
|
||
.report-map p { margin: 5px 0 0; color: var(--muted); font-size: .71rem; }
|
||
.report-map small { color: var(--muted); font: .58rem/1.4 var(--mono); text-align: right; }
|
||
.thesis, .case-callout {
|
||
max-width: 930px;
|
||
margin: 30px 0;
|
||
padding: 24px;
|
||
border-left: 4px solid var(--copper);
|
||
background: var(--copper-pale);
|
||
}
|
||
.thesis span, .case-callout span { color: var(--copper); font: .58rem/1 var(--mono); letter-spacing: .1em; }
|
||
.thesis p, .case-callout p { margin: 12px 0 0; }
|
||
.case-callout b { display: block; margin-top: 18px; font: 700 .91rem/1.5 var(--mono); }
|
||
.axis-grid, .domain-grid, .agentenv-grid, .serving-grid, .eval-axes, .case-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
.axis-grid article, .domain-grid article, .agentenv-grid article, .serving-grid article, .eval-axes article, .case-grid article {
|
||
min-height: 170px;
|
||
padding: 22px;
|
||
border-right: 1px solid var(--line);
|
||
border-bottom: 1px solid var(--line);
|
||
background: var(--paper-raised);
|
||
}
|
||
.axis-grid span, .domain-grid span, .agentenv-grid span, .serving-grid span, .eval-axes span, .case-grid span {
|
||
color: var(--copper);
|
||
font: .57rem/1 var(--mono);
|
||
letter-spacing: .09em;
|
||
}
|
||
.axis-grid h3, .domain-grid h3, .serving-grid h3, .case-grid h3 { margin: 25px 0 10px; font-size: .94rem; }
|
||
.axis-grid p, .domain-grid p, .agentenv-grid p, .serving-grid p, .case-grid p { color: var(--muted); font-size: .72rem; line-height: 1.65; }
|
||
.hybrid-stack { display: flex; max-width: 960px; margin: 35px 0; gap: 6px; align-items: stretch; }
|
||
.hybrid-stack div { position: relative; display: grid; flex: 1; grid-template-columns: repeat(4, 1fr); min-width: 0; border: 1px solid var(--line); }
|
||
.hybrid-stack div > span, .hybrid-stack div > b { padding: 18px 3px; border-right: 1px solid var(--line); font: .52rem/1 var(--mono); text-align: center; }
|
||
.hybrid-stack div > b { border: 0; color: var(--paper); background: var(--copper); }
|
||
.hybrid-stack small { position: absolute; left: 4px; bottom: 3px; color: var(--muted); font: .45rem var(--mono); }
|
||
.hybrid-stack > i { align-self: center; flex: 0 0 180px; padding-left: 12px; color: var(--muted); font: .55rem/1.5 var(--mono); }
|
||
.spec-table-wrap { max-width: 980px; margin: 34px 0; overflow-x: auto; }
|
||
.spec-table { width: 100%; border-collapse: collapse; font-size: .72rem; }
|
||
.spec-table th, .spec-table td { padding: 13px 15px; border: 1px solid var(--line); text-align: left; }
|
||
.spec-table thead th { color: var(--paper); background: var(--ink); font: .58rem var(--mono); }
|
||
.spec-table tbody th { min-width: 170px; font: 700 .66rem var(--mono); }
|
||
.spec-table td:nth-child(3) { color: var(--copper); font-weight: 700; }
|
||
.spec-table td:last-child { color: var(--muted); }
|
||
.formula-card {
|
||
max-width: 930px;
|
||
margin: 32px 0;
|
||
padding: 24px;
|
||
overflow-x: auto;
|
||
border: 1px solid var(--line);
|
||
background: var(--paper-raised);
|
||
}
|
||
.formula-card > span, .formula-card > div > span { color: var(--copper); font: .57rem/1 var(--mono); letter-spacing: .08em; }
|
||
.formula-card code { display: block; margin-top: 17px; white-space: normal; color: var(--ink); font: .78rem/1.7 var(--mono); }
|
||
.formula-card p { margin: 18px 0 0; color: var(--muted); font-size: .69rem; line-height: 1.6; }
|
||
.formula-card.two { display: grid; grid-template-columns: repeat(2, 1fr); gap: 0; padding: 0; }
|
||
.formula-card.two > div { padding: 24px; }
|
||
.formula-card.two > div + div { border-left: 1px solid var(--line); background: var(--copper-pale); }
|
||
.mechanism-steps, .protocol-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(4, 1fr);
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
.mechanism-steps article, .protocol-grid article { min-height: 170px; padding: 20px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
|
||
.mechanism-steps span, .protocol-grid span { color: var(--copper); font: .6rem var(--mono); }
|
||
.mechanism-steps h3, .protocol-grid h3 { margin: 30px 0 10px; font-size: .88rem; }
|
||
.mechanism-steps p, .protocol-grid p { color: var(--muted); font-size: .7rem; line-height: 1.6; }
|
||
.state-flow, .process-line, .mopd-flow, .moonep-flow, .vision-flow {
|
||
display: flex;
|
||
max-width: 1030px;
|
||
margin: 34px 0;
|
||
gap: 9px;
|
||
align-items: center;
|
||
}
|
||
.state-flow > div, .process-line > div, .mopd-flow > div, .moonep-flow > div, .vision-flow > div {
|
||
flex: 1;
|
||
min-width: 0;
|
||
padding: 17px;
|
||
border: 1px solid var(--line);
|
||
background: var(--paper-raised);
|
||
}
|
||
.state-flow > b { padding: 17px; border: 1px solid var(--copper); color: var(--copper); font: .7rem var(--mono); }
|
||
.state-flow i, .process-line i, .mopd-flow i, .moonep-flow i, .vision-flow i { color: var(--muted-light); font-style: normal; }
|
||
.state-flow span, .process-line span, .mopd-flow span, .moonep-flow span, .vision-flow span { display: block; color: var(--muted); font: .53rem var(--mono); }
|
||
.state-flow strong, .process-line b, .mopd-flow b, .moonep-flow b, .vision-flow b { display: block; margin-top: 14px; font-size: .68rem; line-height: 1.45; }
|
||
.cross-link { max-width: 930px; padding: 17px; background: var(--sage-pale); }
|
||
.number-proof, .curriculum {
|
||
display: flex;
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
gap: 9px;
|
||
align-items: center;
|
||
}
|
||
.number-proof article, .curriculum article { flex: 1; min-width: 0; padding: 19px; border: 1px solid var(--line); background: var(--paper-raised); }
|
||
.number-proof span, .curriculum span { color: var(--muted); font: .54rem var(--mono); }
|
||
.number-proof b, .curriculum b { display: block; margin-top: 17px; color: var(--copper); font: 700 .78rem/1.4 var(--mono); }
|
||
.number-proof i, .curriculum i { color: var(--muted); font: .53rem var(--mono); }
|
||
.curriculum p { margin: 9px 0 0; color: var(--muted); font-size: .65rem; }
|
||
.comparison-grid, .layer-separation, .precision-contract {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
.comparison-grid article, .layer-separation article, .precision-contract article { min-height: 155px; padding: 21px; border-right: 1px solid var(--line); }
|
||
.comparison-grid article:last-child, .layer-separation article:last-child, .precision-contract article:last-child { border-right: 0; }
|
||
.comparison-grid span, .layer-separation span, .precision-contract span { color: var(--copper); font: .56rem var(--mono); }
|
||
.comparison-grid h3 { margin: 25px 0 10px; font-size: .88rem; }
|
||
.comparison-grid p, .layer-separation p, .precision-contract p { color: var(--muted); font-size: .7rem; line-height: 1.6; }
|
||
.layer-separation b, .precision-contract b { display: block; margin-top: 25px; font-size: .82rem; }
|
||
.depth-diagram { display: flex; max-width: 960px; margin: 34px 0; gap: 5px; align-items: stretch; }
|
||
.depth-diagram div { flex: 1; min-width: 0; padding: 14px 5px; border: 1px solid var(--line); background: var(--sage-pale); text-align: center; }
|
||
.depth-diagram span, .depth-diagram b { display: block; font: .5rem var(--mono); }
|
||
.depth-diagram b { margin-top: 14px; color: var(--sage); font-size: .67rem; }
|
||
.depth-diagram > i { flex: 0 0 165px; align-self: center; color: var(--muted); font: .53rem/1.5 var(--mono); }
|
||
.moe-flow {
|
||
display: grid;
|
||
grid-template-columns: 1fr 22px 1.2fr;
|
||
gap: 10px;
|
||
align-items: center;
|
||
max-width: 930px;
|
||
margin: 34px 0;
|
||
}
|
||
.moe-flow > div { min-height: 108px; padding: 17px; border: 1px solid var(--line); background: var(--paper-raised); }
|
||
.moe-flow .down { grid-column: 1; }
|
||
.moe-flow .shared { color: var(--sage); background: var(--sage-pale); }
|
||
.moe-flow .experts { color: var(--copper); background: var(--copper-pale); }
|
||
.moe-flow span, .moe-flow b, .moe-flow small { display: block; }
|
||
.moe-flow span { color: var(--muted); font: .53rem var(--mono); }
|
||
.moe-flow b { margin-top: 20px; font-size: .82rem; }
|
||
.moe-flow small { margin-top: 8px; color: var(--muted); font-size: .61rem; }
|
||
.moe-flow > i { color: var(--muted-light); font-style: normal; text-align: center; }
|
||
.two-column { display: grid; grid-template-columns: repeat(2, 1fr); max-width: 960px; margin: 34px 0; border: 1px solid var(--line); }
|
||
.two-column article { min-height: 260px; padding: 24px; }
|
||
.two-column article + article { border-left: 1px solid var(--line); background: var(--copper-pale); }
|
||
.two-column span { color: var(--copper); font: .57rem var(--mono); }
|
||
.two-column h3 { margin: 28px 0 18px; font-size: .96rem; }
|
||
.two-column code { display: block; padding: 12px; white-space: normal; background: rgba(255,255,255,.6); font-size: .68rem; line-height: 1.6; }
|
||
.two-column p { color: var(--muted); font-size: .71rem; line-height: 1.65; }
|
||
.correction {
|
||
display: grid;
|
||
grid-template-columns: 1fr 30px 1fr;
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
gap: 10px;
|
||
align-items: center;
|
||
}
|
||
.correction article { min-height: 180px; padding: 22px; border: 1px solid var(--line); }
|
||
.correction .wrong { opacity: .72; background: #f2ece8; }
|
||
.correction .right { border-color: var(--copper); background: var(--copper-pale); }
|
||
.correction span { color: var(--muted); font: .56rem var(--mono); }
|
||
.correction b { display: block; margin-top: 25px; font: 700 .79rem/1.5 var(--mono); }
|
||
.correction p { color: var(--muted); font-size: .7rem; line-height: 1.6; }
|
||
.correction > i { color: var(--copper); font: 700 1.1rem var(--mono); text-align: center; }
|
||
.efficiency-grammar { display: grid; grid-template-columns: repeat(2, 1fr); max-width: 960px; margin: 34px 0; gap: 10px; }
|
||
.efficiency-grammar article { min-height: 130px; padding: 20px; border: 1px solid var(--line); }
|
||
.efficiency-grammar .yes { border-color: var(--sage); background: var(--sage-pale); }
|
||
.efficiency-grammar .no { background: #f4efec; }
|
||
.efficiency-grammar span { color: var(--muted); font: .55rem var(--mono); }
|
||
.efficiency-grammar b { display: block; margin-top: 23px; font-size: .74rem; line-height: 1.55; }
|
||
.teacher-grid {
|
||
display: grid;
|
||
grid-template-columns: 1.25fr repeat(3, 1fr);
|
||
max-width: 960px;
|
||
margin: 34px 0;
|
||
border-top: 1px solid var(--line);
|
||
border-left: 1px solid var(--line);
|
||
}
|
||
.teacher-grid > * { min-height: 72px; padding: 15px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
|
||
.teacher-grid .corner, .teacher-grid > b { color: var(--paper); background: var(--ink); font: .59rem var(--mono); }
|
||
.teacher-grid strong { font-size: .67rem; background: var(--sage-pale); }
|
||
.teacher-grid span { color: var(--muted); font-size: .64rem; line-height: 1.4; }
|
||
.precision-contract { grid-template-columns: repeat(4, 1fr); }
|
||
.harness-parts { display: grid; grid-template-columns: repeat(4, 1fr); max-width: 960px; margin: 34px 0; border-top: 1px solid var(--line); border-left: 1px solid var(--line); }
|
||
.harness-parts div { min-height: 100px; padding: 16px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
|
||
.harness-parts span { color: var(--copper); font: .54rem var(--mono); }
|
||
.harness-parts b { display: block; margin-top: 24px; font: .7rem var(--mono); }
|
||
.environment-list { display: grid; grid-template-columns: repeat(2, 1fr); max-width: 960px; margin: 34px 0; border-top: 1px solid var(--line); border-left: 1px solid var(--line); }
|
||
.environment-list article { min-height: 175px; padding: 22px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
|
||
.environment-list span { color: var(--copper); font: .58rem var(--mono); }
|
||
.environment-list h3 { margin: 28px 0 10px; font-size: .92rem; }
|
||
.environment-list p { color: var(--muted); font-size: .71rem; line-height: 1.65; }
|
||
.state-stack { max-width: 960px; margin: 34px 0; border-top: 1px solid var(--line); }
|
||
.state-stack article { display: grid; grid-template-columns: 140px 1fr 1fr; gap: 20px; align-items: center; padding: 18px 0; border-bottom: 1px solid var(--line); }
|
||
.state-stack span { color: var(--copper); font: .57rem var(--mono); }
|
||
.state-stack h3 { margin: 0; font-size: .76rem; line-height: 1.45; }
|
||
.state-stack p { margin: 0; color: var(--muted); font: .61rem/1.5 var(--mono); }
|
||
.number-band { display: grid; grid-template-columns: repeat(3, 1fr); max-width: 960px; margin: 34px 0; border: 1px solid var(--ink); color: var(--paper); background: var(--ink); }
|
||
.number-band div { min-height: 120px; padding: 20px; border-right: 1px solid rgba(255,255,255,.18); }
|
||
.number-band span { color: rgba(255,255,255,.55); font: .55rem var(--mono); }
|
||
.number-band b { display: block; margin-top: 28px; color: var(--copper); font: 700 1rem var(--mono); }
|
||
.number-band > p { grid-column: 1/-1; margin: 0; padding: 14px 20px; border-top: 1px solid rgba(255,255,255,.18); color: rgba(255,255,255,.55); font-size: .65rem; }
|
||
.cache-anatomy { max-width: 960px; margin: 34px 0; border: 1px solid var(--line); background: var(--paper-raised); }
|
||
.cache-anatomy header { display: flex; justify-content: space-between; padding: 16px; color: var(--paper); background: var(--ink); }
|
||
.cache-anatomy header span, .cache-anatomy header b { font: .58rem var(--mono); }
|
||
.hash-blocks { display: grid; grid-template-columns: repeat(12, 1fr); gap: 3px; padding: 16px; }
|
||
.hash-blocks div { position: relative; min-height: 82px; padding: 7px 2px; border: 1px solid var(--line); text-align: center; }
|
||
.hash-blocks div.hit { border-color: var(--copper); background: var(--copper-pale); }
|
||
.hash-blocks span, .hash-blocks b { display: block; color: var(--muted); font: .48rem var(--mono); }
|
||
.hash-blocks i { display: block; height: 20px; margin: 10px 0; background: var(--line); }
|
||
.hash-blocks .hit i { background: var(--copper); }
|
||
.hash-blocks em { color: var(--copper); font: .46rem var(--mono); }
|
||
.cache-anatomy > p { margin: 0; padding: 0 16px 16px; color: var(--muted); font-size: .67rem; }
|
||
.eval-axes { grid-template-columns: repeat(4, 1fr); }
|
||
.eval-axes article { min-height: 140px; }
|
||
.eval-axes b { display: block; margin-top: 25px; font-size: .7rem; line-height: 1.6; }
|
||
.protocol-grid { grid-template-columns: repeat(3, 1fr); }
|
||
.xtml-stage {
|
||
display: grid;
|
||
grid-template-columns: 1fr 72px 1.2fr 72px 1fr 72px 1fr 100px;
|
||
max-width: 1080px;
|
||
margin: 34px 0;
|
||
gap: 5px;
|
||
align-items: center;
|
||
}
|
||
.xtml-stage > div { min-height: 150px; padding: 18px; border: 1px solid var(--line); background: var(--paper-raised); }
|
||
.xtml-stage .global { background: var(--sage-pale); }
|
||
.xtml-stage .oneshot { background: var(--copper-pale); }
|
||
.xtml-stage > i { color: var(--copper); font: .5rem/1.5 var(--mono); text-align: center; }
|
||
.xtml-stage span { color: var(--muted); font: .54rem var(--mono); }
|
||
.xtml-stage b { display: block; margin-top: 24px; font-size: .72rem; }
|
||
.xtml-stage p { color: var(--muted); font-size: .63rem; line-height: 1.5; }
|
||
.figure-atlas { display: grid; grid-template-columns: repeat(3, 1fr); max-width: 1080px; margin: 34px 0; border-top: 1px solid var(--line); border-left: 1px solid var(--line); }
|
||
.figure-atlas article { min-height: 185px; padding: 20px; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); }
|
||
.figure-atlas header { display: flex; justify-content: space-between; }
|
||
.figure-atlas header span { color: var(--copper); font: .58rem var(--mono); }
|
||
.figure-atlas header b { color: var(--muted); font: .55rem var(--mono); }
|
||
.figure-atlas h3 { margin: 30px 0 10px; font-size: .86rem; }
|
||
.figure-atlas p { color: var(--muted); font-size: .69rem; line-height: 1.6; }
|
||
.audit-legend { display: grid; grid-template-columns: repeat(4, 1fr); max-width: 960px; margin: 34px 0; border: 1px solid var(--line); }
|
||
.audit-legend div { min-height: 135px; padding: 18px; border-right: 1px solid var(--line); }
|
||
.audit-legend div:last-child { border: 0; }
|
||
.audit-legend span { display: inline-grid; width: 28px; height: 28px; place-items: center; color: var(--paper); background: var(--copper); font: 700 .7rem var(--mono); }
|
||
.audit-legend b { display: block; margin-top: 17px; font: .61rem var(--mono); }
|
||
.audit-legend p { color: var(--muted); font-size: .65rem; line-height: 1.5; }
|
||
.paper-group { max-width: 1030px; margin: 42px 0; }
|
||
.paper-group > header { display: grid; grid-template-columns: 130px 1fr 110px; gap: 20px; align-items: baseline; padding: 0 0 13px; border-bottom: 2px solid var(--ink); }
|
||
.paper-group > header span, .paper-group > header b { color: var(--muted); font: .58rem var(--mono); }
|
||
.paper-group > header h3 { margin: 0; font-size: 1.05rem; }
|
||
.paper-group > header b { text-align: right; }
|
||
@media (max-width: 1000px) {
|
||
.ledger-grid, .figure-atlas { grid-template-columns: repeat(2, 1fr); }
|
||
.hybrid-stack { flex-wrap: wrap; }
|
||
.hybrid-stack div { flex-basis: 30%; }
|
||
.hybrid-stack > i { flex: 1 0 100%; }
|
||
.protocol-grid { grid-template-columns: repeat(2, 1fr); }
|
||
.xtml-stage { grid-template-columns: 1fr 45px 1fr; }
|
||
.xtml-stage > i:nth-of-type(n+3) { display: none; }
|
||
}
|
||
@media (max-width: 720px) {
|
||
.artifact-callout, .ledger-grid, .axis-grid, .domain-grid, .agentenv-grid, .serving-grid, .eval-axes, .case-grid,
|
||
.mechanism-steps, .protocol-grid, .comparison-grid, .layer-separation, .precision-contract,
|
||
.two-column, .environment-list, .figure-atlas, .audit-legend, .harness-parts {
|
||
grid-template-columns: 1fr;
|
||
}
|
||
.report-map article, .state-stack article { grid-template-columns: 1fr; gap: 8px; }
|
||
.report-map small { text-align: left; }
|
||
.formula-card.two { grid-template-columns: 1fr; }
|
||
.formula-card.two > div + div { border-left: 0; border-top: 1px solid var(--line); }
|
||
.number-proof, .curriculum, .state-flow, .process-line, .mopd-flow, .moonep-flow, .vision-flow {
|
||
flex-direction: column;
|
||
align-items: stretch;
|
||
}
|
||
.number-proof i, .curriculum i, .state-flow i, .process-line i, .mopd-flow i, .moonep-flow i, .vision-flow i { transform: rotate(90deg); text-align: center; }
|
||
.correction { grid-template-columns: 1fr; }
|
||
.correction > i { transform: rotate(90deg); }
|
||
.depth-diagram { flex-wrap: wrap; }
|
||
.depth-diagram div { flex-basis: 19%; }
|
||
.depth-diagram > i { flex: 1 0 100%; }
|
||
.moe-flow { grid-template-columns: 1fr; }
|
||
.moe-flow .down { grid-column: auto; }
|
||
.moe-flow > i { transform: rotate(90deg); }
|
||
.teacher-grid { grid-template-columns: 1fr repeat(3, .8fr); overflow-x: auto; }
|
||
.number-band { grid-template-columns: 1fr; }
|
||
.number-band > p { grid-column: 1; }
|
||
.hash-blocks { grid-template-columns: repeat(6, 1fr); }
|
||
.xtml-stage { grid-template-columns: 1fr; }
|
||
.xtml-stage > i { display: block !important; }
|
||
.paper-group > header { grid-template-columns: 1fr; gap: 6px; }
|
||
.paper-group > header b { text-align: left; }
|
||
}
|
||
</style>
|
||
</BaseLayout>
|