wuyang
1266 lines
64 KiB
Plaintext
1266 lines
64 KiB
Plaintext
---
|
||
import BaseLayout from "@/layouts/BaseLayout.astro";
|
||
import TrainingSystemsLab from "@/components/TrainingSystemsLab.astro";
|
||
|
||
const toc = [
|
||
["00", "map", "先拆成九张账"],
|
||
["01", "one-step", "一步训练发生什么"],
|
||
["02", "state", "模型状态与 ZeRO"],
|
||
["03", "activation", "激活：存、算、压、搬"],
|
||
["04", "parallel", "DP / TP / PP"],
|
||
["05", "collectives", "Collective 通信"],
|
||
["06", "pipeline", "流水线与气泡"],
|
||
["07", "expert", "Expert Parallel"],
|
||
["08", "context", "长上下文并行"],
|
||
["09", "precision", "低精度与优化器"],
|
||
["10", "deepseek", "DeepSeek 系统谱系"],
|
||
["11", "kimi", "Kimi K2 → K3"],
|
||
["12", "lab", "四合一互动实验室"],
|
||
["13", "agentic", "百万 Token Agentic RL"],
|
||
["14", "decisions", "怎样选择配置"],
|
||
["↳", "papers", "37 个一手节点"],
|
||
];
|
||
|
||
const ledgers = [
|
||
["L1 / MODEL STATE", "模型状态", "参数、梯度、master weights、optimizer states 各占多少？"],
|
||
["L2 / ACTIVATION", "激活", "Backward 需要的中间量留在 HBM、重算、压缩还是卸载？"],
|
||
["L3 / COMPUTE", "计算划分", "切 batch、矩阵、layer、expert 还是 sequence？"],
|
||
["L4 / PIPELINE", "流水线气泡", "哪些 GPU 在等依赖，哪些计算可以挪进空隙？"],
|
||
["L5 / COLLECTIVE", "集合通信", "搬多少字节、调用多频繁、走哪一级网络？"],
|
||
["L6 / EXPERT", "专家派发", "动态路由怎样变成两次 All-to-All 与最慢 rank？"],
|
||
["L7 / CONTEXT", "长上下文状态", "Q/K/V、KV cache 或 recurrent state 怎样跨 rank？"],
|
||
["L8 / NUMERICS", "数值与优化器", "存储、GEMM、累加、归约和更新各用什么精度？"],
|
||
["L9 / RELIABILITY", "可靠性与环境", "checkpoint、rollout、KV、沙箱和故障怎样恢复？"],
|
||
];
|
||
|
||
const memoryRows = [
|
||
["BF16 / FP16 parameter", "2P", "Forward / backward 当前使用的权重"],
|
||
["BF16 / FP16 gradient", "2P", "未分片的 gradient buffer"],
|
||
["FP32 master parameter", "4P", "优化器更新的高精度副本"],
|
||
["FP32 first moment", "4P", "Adam momentum"],
|
||
["FP32 second moment", "4P", "Adam variance"],
|
||
["TOTAL", "16P", "只属于这套 mixed-precision Adam 配方"],
|
||
];
|
||
|
||
const zeroStages = [
|
||
["DP", "全部复制", "16P", "每步 gradient AllReduce"],
|
||
["ZeRO-1", "切 optimizer", "4P + 12P/N", "Reduce/更新后 AllGather"],
|
||
["ZeRO-2", "再切 gradient", "2P + 14P/N", "ReduceScatter + AllGather"],
|
||
["ZeRO-3", "再切 parameter", "16P/N", "每层参数 gather，梯度 scatter"],
|
||
];
|
||
|
||
const activationMoves = [
|
||
{
|
||
id: "store",
|
||
title: "留在 GPU",
|
||
tag: "CAPACITY",
|
||
body: "Forward 后原样保留，Backward 直接读取。速度最直接，容量最昂贵。",
|
||
debt: "HBM",
|
||
},
|
||
{
|
||
id: "recompute",
|
||
title: "丢掉再重算",
|
||
tag: "COMPUTE",
|
||
body: "只留 checkpoint；Backward 前重跑部分 forward。省容量，增加 FLOPs。",
|
||
debt: "额外 forward",
|
||
},
|
||
{
|
||
id: "compress",
|
||
title: "压成低精度",
|
||
tag: "NUMERICS",
|
||
body: "以 FP8 或其他表示保存，使用前恢复。省字节，引入 scale、cast 与精度验证。",
|
||
debt: "数值风险",
|
||
},
|
||
{
|
||
id: "offload",
|
||
title: "搬到别处",
|
||
tag: "BANDWIDTH",
|
||
body: "放进 CPU、远端 GPU 或 NVMe；Backward 前预取并尽量与计算重叠。",
|
||
debt: "PCIe / RDMA / I/O",
|
||
},
|
||
];
|
||
|
||
const parallelAxes = [
|
||
["DATA", "DP", "切 batch", "每卡完整模型；每 optimizer step 同步 gradient。", "低频大 collective"],
|
||
["TENSOR", "TP", "切一层矩阵", "同一 layer 的 GEMM 与 heads 分到多卡。", "每层高频 collective"],
|
||
["PIPELINE", "PP", "切 layer 深度", "相邻 stage 传 activation 与 gradient。", "P2P + bubble"],
|
||
["EXPERT", "EP", "切 routed experts", "Token 去往远端 expert，再把输出送回。", "每 MoE 层 2× A2A"],
|
||
["CONTEXT", "CP", "切一条长序列", "多卡共同完成同一样本的 attention / recurrent state。", "A2A 或 P2P ring"],
|
||
];
|
||
|
||
const collectives = [
|
||
["AllReduce", "完整 x", "聚合后的完整 Σx", "DP gradient；TP partial output", "ring: 2(N−1) rounds"],
|
||
["ReduceScatter", "完整 x", "聚合后的 1/N shard", "ZeRO/FSDP gradient", "约 (N−1)/N · M sent"],
|
||
["AllGather", "1/N shard", "完整 x", "ZeRO parameter；SP tensor", "约 (N−1)/N · M sent"],
|
||
["All-to-All", "给每个 peer 的不同 shard", "来自每个 peer 的不同 shard", "MoE dispatch；Ulysses", "对拓扑与负载敏感"],
|
||
["P2P", "一个邻居消息", "一个邻居消息", "PP；Ring Attention；Muon", "易 pipeline / overlap"],
|
||
];
|
||
|
||
const pipelineWaves = [
|
||
{
|
||
year: "2018–19",
|
||
title: "PipeDream / GPipe",
|
||
gain: "Micro-batch 让多个 stage 同时工作；GPipe 保持同步语义，PipeDream 探索异步 1F1B。",
|
||
debt: "flush bubble、activation residency、weight version。",
|
||
},
|
||
{
|
||
year: "2021",
|
||
title: "Interleaved 1F1B / Chimera",
|
||
gain: "virtual chunks 缩短 bubble 单元；双向 pipeline 从两端注入工作。",
|
||
debt: "更多消息、更多调度约束，stage 划分仍需平衡。",
|
||
},
|
||
{
|
||
year: "2023",
|
||
title: "Zero Bubble",
|
||
gain: "把 backward 拆成 input-gradient 与 weight-gradient，用可延后的 W 填空隙。",
|
||
debt: "能否零气泡取决于 F/B/W 比例、内存和 optimizer sync。",
|
||
},
|
||
{
|
||
year: "2024",
|
||
title: "DeepSeek DualPipe",
|
||
gain: "双向 pipeline 配对 F/B，并把 MoE A2A 与 compute 重排、重叠。",
|
||
debt: "两份参数；只有被覆盖的通信才不暴露在关键路径。",
|
||
},
|
||
{
|
||
year: "2025–26",
|
||
title: "K2 / K3 的不同取舍",
|
||
gain: "K2 选择省状态的 interleaved 1F1B；K3 继续把 ViT、offload 与 gradient reduce 填进不同相位。",
|
||
debt: "最佳 schedule 由模型容量、MoE 比例和多模态负载共同决定。",
|
||
},
|
||
];
|
||
|
||
const contextMethods = [
|
||
["MEGATRON SP", "切 element-wise activations", "AllGather + ReduceScatter", "与 TP 共组；主要省复制激活"],
|
||
["ULYSSES", "sequence shard ↔ head shard", "2× All-to-All", "并行度受 head / KV-head 可切分性约束"],
|
||
["RING ATTENTION", "固定 Q，轮转 K/V blocks", "P2P ring", "可重叠；block 太小会伤 kernel，causal 需均衡"],
|
||
["USP", "Ulysses × Ring 二维 mesh", "A2A + P2P", "把高带宽域与慢链路分别映射"],
|
||
["K3 KCP", "组合 fixed-size recurrent fragments", "AllGather + prefix scan", "只适用于 KDA 分支，不代表 MLA"],
|
||
];
|
||
|
||
const deepseekSteps = [
|
||
["V2", "2024", "16 PP · 8 EP · ZeRO-1", "少 activated parameters + 重计算使其不需 TP；shared expert compute 与 A2A overlap。"],
|
||
["V3", "2024", "16 PP · 64 EP · ZeRO-1", "DualPipe、node-limited routing、cross-node A2A kernels 与 FP8 协同。"],
|
||
["V4", "2026", "Million-token hybrid attention", "CP 传输对象扩展为压缩/稀疏 attention 与 recurrent state，不能套标准 MHA 单式。"],
|
||
];
|
||
|
||
const kimiSteps = [
|
||
["K2", "1T MoE", "选择 interleaved 1F1B", "不采用 DualPipe：两份 parameter+gradient 会迫使 PP/EP 扩大；用更多 warmup 和 WGrad overlap 代替。"],
|
||
["K2.5", "Native multimodal", "DEP 解耦 ViT", "视觉 forward 全局均衡、只留输出；backbone 完成后重算 ViT 并 backward。"],
|
||
["K3", "2.78T / 1M", "重构执行与存储层", "MoonEP、统一 activation manager、Pipeline ZeRO-2、remote activation、P2P Muon 与 external KV pool。"],
|
||
];
|
||
|
||
const paperChain = [
|
||
["2012", "Large Scale Distributed Deep Networks", "https://arxiv.org/abs/1206.5533", "DistBelief 与参数服务器。"],
|
||
["2014", "One Weird Trick for Parallelizing CNNs", "https://arxiv.org/abs/1404.5997", "数据/模型并行直觉。"],
|
||
["2016", "Training Deep Nets with Sublinear Memory Cost", "https://arxiv.org/abs/1604.06174", "activation checkpointing。"],
|
||
["2017", "Mixed Precision Training", "https://arxiv.org/abs/1710.03740", "master weights、loss scaling、FP32 accumulation。"],
|
||
["2018", "PipeDream", "https://arxiv.org/abs/1806.03377", "1F1B、异步 pipeline 与 weight stashing。"],
|
||
["2018", "Mesh-TensorFlow", "https://arxiv.org/abs/1811.02084", "layout 到 device mesh。"],
|
||
["2019", "GPipe", "https://arxiv.org/abs/1811.06965", "同步 micro-batch pipeline。"],
|
||
["2019", "Megatron-LM", "https://arxiv.org/abs/1909.08053", "Transformer tensor parallel。"],
|
||
["2019", "ZeRO", "https://arxiv.org/abs/1910.02054", "模型状态三阶段分片。"],
|
||
["2020", "GShard", "https://arxiv.org/abs/2006.16668", "MoE expert parallel 与自动分片。"],
|
||
["2021", "3D Megatron-LM", "https://arxiv.org/abs/2104.04473", "TP + PP + DP 与 interleaved schedule。"],
|
||
["2021", "ZeRO-Infinity", "https://arxiv.org/abs/2104.07857", "GPU / CPU / NVMe 异构内存。"],
|
||
["2021", "GSPMD", "https://arxiv.org/abs/2105.04663", "general sharding propagation。"],
|
||
["2021", "Chimera", "https://arxiv.org/abs/2107.06925", "bidirectional pipeline。"],
|
||
["2022", "DeepSpeed-MoE", "https://arxiv.org/abs/2201.05596", "多轴 MoE 系统。"],
|
||
["2022", "Alpa", "https://arxiv.org/abs/2201.12023", "自动 inter/intra-operator parallelism。"],
|
||
["2022", "Reducing Activation Recomputation", "https://arxiv.org/abs/2205.05198", "Megatron SP 与 selective recompute。"],
|
||
["2022", "FlashAttention", "https://arxiv.org/abs/2205.14135", "IO-aware exact attention。"],
|
||
["2022", "Tutel", "https://arxiv.org/abs/2206.03382", "adaptive MoE system。"],
|
||
["2022", "MegaBlocks", "https://arxiv.org/abs/2211.15841", "dropless block-sparse MoE。"],
|
||
["2023", "PyTorch FSDP", "https://arxiv.org/abs/2304.11277", "fully sharded 实现经验。"],
|
||
["2023", "FlashAttention-2", "https://arxiv.org/abs/2307.08691", "attention work partition。"],
|
||
["2023", "DeepSpeed-Ulysses", "https://arxiv.org/abs/2309.14509", "sequence ↔ head All-to-All。"],
|
||
["2023", "Ring Attention", "https://arxiv.org/abs/2310.01889", "blockwise P2P context parallel。"],
|
||
["2023", "Zero Bubble Pipeline Parallelism", "https://arxiv.org/abs/2401.10241", "B/W 拆分与 schedule search。"],
|
||
["2024", "MegaScale", "https://arxiv.org/abs/2402.15627", "10K+ GPU 训练与可靠性。"],
|
||
["2024", "DeepSeek-V2", "https://arxiv.org/abs/2405.04434", "zero-bubble PP + EP + ZeRO-1。"],
|
||
["2024", "USP", "https://arxiv.org/abs/2405.07719", "Ulysses × Ring 二维 SP。"],
|
||
["2024", "DeepSeek-V3", "https://arxiv.org/abs/2412.19437", "DualPipe、A2A、FP8。"],
|
||
["2025", "DeepEP", "https://github.com/deepseek-ai/DeepEP", "expert dispatch / combine kernels。"],
|
||
["2025", "Kimi K2", "https://arxiv.org/abs/2507.20534", "1T MoE 与 co-located RL。"],
|
||
["2026", "Kimi K2.5", "https://arxiv.org/abs/2602.02276", "DEP 与 100K agent tasks。"],
|
||
["2026", "DeepSeek-V4", "https://arxiv.org/abs/2606.19348", "百万 Token 系统约束。"],
|
||
["2026", "Kimi K3", "https://arxiv.org/abs/2607.24653", "2.8T pretraining + 1M agentic RL。"],
|
||
["2026", "MoonEP", "https://github.com/MoonshotAI/MoonEP", "完美 rank balance 与 static shape。"],
|
||
["2026", "AgentENV", "https://github.com/kvcache-ai/AgentENV", "resumable microVM agent environments。"],
|
||
["2026", "K2 Checkpoint Engine", "https://github.com/MoonshotAI/checkpoint-engine", "train / inference resharding。"],
|
||
];
|
||
---
|
||
|
||
<BaseLayout
|
||
title="大规模 LLM 训练系统：从 ZeRO、Megatron 到 DualPipe、MoonEP 与百万 Token RL"
|
||
description="用九张资源账系统讲解 LLM 大规模训练：显存、激活、DP/TP/PP、collective、pipeline、Expert/Context Parallel、DeepSeek 系统谱系与 Kimi K3 基础设施。"
|
||
section="training-systems"
|
||
>
|
||
<header class="page-hero systems-hero">
|
||
<div class="page-hero-inner">
|
||
<div>
|
||
<p class="eyebrow"><span>SYSTEMS / 12</span> LARGE-SCALE TRAINING</p>
|
||
<h1>一万张 GPU，<br />为什么仍可能有一半在等？</h1>
|
||
<p class="lead">
|
||
大模型训练不是“卡越多越快”。参数要有地方放，激活要活到反向，矩阵和层要正确切分，
|
||
数据必须穿过真实网络，气泡、负载不均与故障还会把理论算力变成等待。
|
||
</p>
|
||
</div>
|
||
<dl class="page-facts">
|
||
<div><dt>LEDGERS</dt><dd>9 张资源账</dd></div>
|
||
<div><dt>CORE SOURCES</dt><dd>37 个一手节点</dd></div>
|
||
<div><dt>LINEAGE</dt><dd>2012 → 2026</dd></div>
|
||
<div><dt>LAB</dt><dd>4 个独立实验</dd></div>
|
||
<div><dt>SPOTLIGHT</dt><dd>DeepSeek × Kimi</dd></div>
|
||
</dl>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="report-shell">
|
||
<aside class="side-rail" aria-label="本页目录">
|
||
<p>CONTENTS</p>
|
||
<ol>
|
||
{toc.map(([number, id, label]) => (
|
||
<li><a href={`#${id}`}><span>{number}</span>{label}</a></li>
|
||
))}
|
||
</ol>
|
||
<div class="rail-note">
|
||
<b>研究截止</b>
|
||
2026-07-29。公式均标明假设；性能数字只沿用论文原始设置。
|
||
</div>
|
||
</aside>
|
||
|
||
<article class="article">
|
||
<section class="article-section" id="map">
|
||
<p class="eyebrow"><span>00</span> NINE LEDGERS</p>
|
||
<h2>训练系统的核心：把等待、容量和数据移动分别记账</h2>
|
||
<p class="lede">
|
||
“模型放不下”可能是参数、Adam 状态、激活或临时通信 buffer；“扩展效率低”可能是 GEMM 太小、
|
||
网络太慢、最忙专家拖尾或 pipeline 没填满。只有先拆账，才知道优化是在消灭成本，还是把它搬到另一层。
|
||
</p>
|
||
|
||
<div class="ledger-grid system-ledgers">
|
||
{ledgers.map(([code, title, question]) => (
|
||
<article>
|
||
<span>{code}</span><b>{title}</b><p>{question}</p>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<div class="cost-stack" aria-label="训练系统四层成本示意">
|
||
<div>
|
||
<span>04</span><b>IDLE / FAILURE</b><p>气泡、长尾、故障恢复、环境等待</p>
|
||
</div>
|
||
<div>
|
||
<span>03</span><b>NETWORK</b><p>NVLink、IB/RoCE、PCIe、存储 I/O</p>
|
||
</div>
|
||
<div>
|
||
<span>02</span><b>MEMORY</b><p>HBM、CPU DRAM、remote GPU、NVMe</p>
|
||
</div>
|
||
<div>
|
||
<span>01</span><b>COMPUTE</b><p>Tensor Core、CUDA Core、kernel launch</p>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="plain-language">
|
||
<b>像一间超大型餐厅：食材、厨师、传菜和空桌是四张不同的账</b>
|
||
<p>
|
||
GPU 算力是厨师；HBM 是手边案台；跨卡网络是传菜通道；pipeline bubble 是厨师在等上一道工序。
|
||
增加厨师，如果案台太小、传菜口拥堵或订单全堵在某个专家窗口，仍然不会线性提速。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="one-step">
|
||
<p class="eyebrow"><span>01</span> ONE TRAINING STEP</p>
|
||
<h2>先看一步训练，所有系统优化才有落点</h2>
|
||
<p>
|
||
一个同步训练 step 并不是一次“模型运行”。它包含数据进入、forward、保存或处理 activation、
|
||
backward、梯度聚合、optimizer update，以及下一步开始前必须完成的同步。不同状态只在特定时间被需要。
|
||
</p>
|
||
|
||
<div class="step-flow">
|
||
<article><span>01</span><b>LOAD</b><p>读入并 pack Token / 图像 / 视频</p><small>storage → CPU → GPU</small></article>
|
||
<i>→</i>
|
||
<article><span>02</span><b>FORWARD</b><p>按 layer 产生 activation 与 loss</p><small>parameter + compute</small></article>
|
||
<i>→</i>
|
||
<article><span>03</span><b>BACKWARD</b><p>逆序消费 activation，产生 gradient</p><small>recompute / prefetch</small></article>
|
||
<i>→</i>
|
||
<article><span>04</span><b>REDUCE</b><p>跨 replica 聚合或分片 gradient</p><small>collective</small></article>
|
||
<i>→</i>
|
||
<article><span>05</span><b>UPDATE</b><p>用 optimizer states 更新参数</p><small>master weights</small></article>
|
||
</div>
|
||
|
||
<h3>“不是什么状态都要一直在 GPU”</h3>
|
||
<div class="lifetime-diagram">
|
||
<div><b>PARAMETER</b><i class="long"></i><span>forward</span><span>backward</span><span>update</span></div>
|
||
<div><b>ACTIVATION</b><i class="split"></i><span>产生</span><span>等待 / 卸载</span><span>消费后释放</span></div>
|
||
<div><b>GRADIENT</b><i class="late"></i><span></span><span>逐层产生</span><span>reduce / update</span></div>
|
||
<div><b>OPTIMIZER</b><i class="pulse"></i><span>可分片 / 卸载</span><span></span><span>短时使用</span></div>
|
||
</div>
|
||
|
||
<div class="insight-note">
|
||
<b>ZeRO、checkpointing 与 offload 共享同一个直觉</b>
|
||
<p>
|
||
如果一个状态当前不需要，就不必在每张 GPU 上完整常驻。区别在于：ZeRO 把状态放到其他 rank，
|
||
checkpointing 把中间量变成未来的重计算，offload 则把它放到更慢的存储层。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="state">
|
||
<p class="eyebrow"><span>02</span> MODEL-STATE MEMORY</p>
|
||
<h2>“BF16 参数 2P bytes”只是训练账单的第一行</h2>
|
||
<p>
|
||
<a href="https://arxiv.org/abs/1910.02054">ZeRO</a>
|
||
用 FP16/FP32 mixed-precision Adam 说明：前反向权重和梯度各 2 字节，更新还保留 FP32 master parameter、
|
||
momentum 与 variance，各 4 字节。于是基线是 16P，而不是 2P。
|
||
</p>
|
||
|
||
<div class="byte-ledger">
|
||
{memoryRows.map(([name, bytes, role], index) => (
|
||
<div class:list={{ total: index === memoryRows.length - 1 }}>
|
||
<span>{String(index + 1).padStart(2, "0")}</span><b>{name}</b><strong>{bytes}</strong><p>{role}</p>
|
||
</div>
|
||
))}
|
||
</div>
|
||
|
||
<div class="warning-note">
|
||
<b>16P 不是 Adam 的永久常数</b>
|
||
<p>
|
||
这条式子依赖具体 dtype 与状态实现。BF16 gradient、FP32 accumulation、EMA、FP8 scale、flat buffer、
|
||
Muon state 或 fused optimizer 都会改变账单。正确做法是逐张量列 byte ledger。
|
||
</p>
|
||
</div>
|
||
|
||
<h3>ZeRO：沿数据并行轴逐步取消复制</h3>
|
||
<div class="zero-ladder">
|
||
{zeroStages.map(([name, split, memory, traffic], index) => (
|
||
<article style={`--level:${index}`}>
|
||
<span>{String(index).padStart(2, "0")}</span>
|
||
<h4>{name}</h4>
|
||
<b>{split}</b>
|
||
<strong>{memory}</strong>
|
||
<p>{traffic}</p>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<div class="formula-note">
|
||
<span>COMMUNICATION CONVENTION</span>
|
||
<p>
|
||
ZeRO 论文把 ReduceScatter 与 AllGather 各近似为 P 的数据移动，所以普通 DP 与 ZeRO-2 都约为 2P，
|
||
ZeRO-3 约为 3P。严格 ring 每 rank 的单向发送量还要乘 `(N−1)/N`。不同口径不能直接相除。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="activation">
|
||
<p class="eyebrow"><span>03</span> ACTIVATION LIFECYCLE</p>
|
||
<h2>参数按模型大小增长，激活按 batch × sequence × hidden × layers 增长</h2>
|
||
<p>
|
||
增大 DP 只切 batch，不会切开一个超长样本的 activation。标准注意力如果显式保存 `S×S` matrix，
|
||
还会出现平方级中间量。因而 7B 长上下文训练可能比更大参数的短上下文配置更早撞上 HBM。
|
||
</p>
|
||
|
||
<div class="activation-moves">
|
||
{activationMoves.map((move, index) => (
|
||
<article class={move.id}>
|
||
<span>{String(index + 1).padStart(2, "0")} / {move.tag}</span>
|
||
<h3>{move.title}</h3>
|
||
<p>{move.body}</p>
|
||
<b>新账单：{move.debt}</b>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<div class="checkpoint-visual">
|
||
<div>
|
||
<span>STORE ALL</span>
|
||
<div>{Array.from({ length: 12 }, (_, i) => <i><b>A{i + 1}</b></i>)}</div>
|
||
<p>Backward 直接读取；activation memory 随层数线性增长。</p>
|
||
</div>
|
||
<div>
|
||
<span>CHECKPOINT + RECOMPUTE</span>
|
||
<div>{Array.from({ length: 12 }, (_, i) => <i class:list={{ keep: i % 4 === 0 }}><b>{i % 4 === 0 ? `C${i / 4 + 1}` : "重算"}</b></i>)}</div>
|
||
<p>只留边界 checkpoint；Backward 经过某段时重跑该段 forward。</p>
|
||
</div>
|
||
</div>
|
||
|
||
<h3>关键论文怎样逐步把重计算变细</h3>
|
||
<div class="evidence-cards">
|
||
<article>
|
||
<span>2016</span><b>Sublinear Memory</b>
|
||
<p>分段 checkpoint 把 n 层 feature-map memory 从 O(n) 降到 O(√n)；递归可继续换内存。</p>
|
||
</article>
|
||
<article>
|
||
<span>2022</span><b>Selective Recomputation</b>
|
||
<p>优先重算 memory-heavy、compute-light 中间量，并用 Megatron SP 分片原本复制的 element-wise activation。</p>
|
||
</article>
|
||
<article>
|
||
<span>2022</span><b>FlashAttention</b>
|
||
<p>用 SRAM tiling 与 online softmax 避免把完整 S×S attention matrix 写回 HBM；dense attention FLOPs 仍是平方级。</p>
|
||
</article>
|
||
<article>
|
||
<span>2026</span><b>K3 Unified Manager</b>
|
||
<p>把 recompute、FP8、local / remote offload 变成 tensor 粒度的可组合 storage policy。</p>
|
||
</article>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="parallel">
|
||
<p class="eyebrow"><span>04</span> PARALLEL AXES</p>
|
||
<h2>五种并行，切的是五个不同对象</h2>
|
||
<p>
|
||
如果只背缩写，很容易把“更多 GPU”误解成同一种扩展。判断任何配置时，先问：
|
||
每张 GPU 持有什么、每次通信发生在哪个频率、哪个维度真的独立。
|
||
</p>
|
||
|
||
<div class="axis-map">
|
||
{parallelAxes.map(([full, short, cut, body, comm], index) => (
|
||
<article>
|
||
<span>{String(index + 1).padStart(2, "0")} / {full}</span>
|
||
<strong>{short}</strong>
|
||
<h3>{cut}</h3>
|
||
<p>{body}</p>
|
||
<b>{comm}</b>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<div class="mesh-visual">
|
||
<div class="mesh-labels">
|
||
<span>NODE 0 · FAST DOMAIN</span><span>NODE 1</span><span>NODE 2</span><span>NODE 3</span>
|
||
</div>
|
||
<div class="mesh-nodes">
|
||
{Array.from({ length: 4 }, (_, node) => (
|
||
<div>
|
||
{Array.from({ length: 4 }, (_, gpu) => <i><b>{node * 4 + gpu}</b><small>TP{gpu}</small></i>)}
|
||
</div>
|
||
))}
|
||
</div>
|
||
<div class="mesh-links">
|
||
<span class="tp">TP · 节点内高频 AllReduce</span>
|
||
<span class="pp">PP · 相邻 stage P2P</span>
|
||
<span class="dp">DP · 每 step gradient collective</span>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="warning-note">
|
||
<b>不要机械写 `world = DP × TP × PP × EP × CP`</b>
|
||
<p>
|
||
只有互相正交的 device-mesh axes 才连乘。Megatron SP 与 TP 共组；EP 常只作用于 MoE layer；
|
||
dense attention、shared expert 和 context group 还可能用另一套 process groups。先画 rank membership。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="collectives">
|
||
<p class="eyebrow"><span>05</span> COLLECTIVE COMMUNICATION</p>
|
||
<h2>通信量必须同时写：消息大小、rank 口径、算法和调用频率</h2>
|
||
|
||
<div class="alpha-beta">
|
||
<div>
|
||
<span>LATENCY</span><b>α × rounds</b><p>小消息、同步、协议与 kernel launch 更敏感。</p>
|
||
</div>
|
||
<i>+</i>
|
||
<div>
|
||
<span>BANDWIDTH</span><b>β × bytes</b><p>大 tensor 搬运由链路吞吐主导。</p>
|
||
</div>
|
||
<i>−</i>
|
||
<div>
|
||
<span>OVERLAP</span><b>hidden time</b><p>只减关键路径时间，不减物理传输字节。</p>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="collective-table">
|
||
<div class="table-head"><span>OP</span><span>每 rank 输入</span><span>每 rank 输出</span><span>LLM 位置</span><span>环形直觉</span></div>
|
||
{collectives.map((row) => <div>{row.map((cell, index) => index === 0 ? <b>{cell}</b> : <span>{cell}</span>)}</div>)}
|
||
</div>
|
||
|
||
<h3>为什么相同字节数也可能完全不同</h3>
|
||
<div class="frequency-comparison">
|
||
<article><span>DP</span><b>一次 / optimizer step</b><p>消息大，但能被一整个 batch 的计算摊薄。</p></article>
|
||
<article><span>TP</span><b>多次 / layer / micro-batch</b><p>高频，强依赖节点内低延迟高带宽。</p></article>
|
||
<article><span>PP</span><b>邻接 / micro-batch</b><p>点对点；stage 越多，消息与 bubble 越复杂。</p></article>
|
||
<article><span>EP</span><b>两次 / MoE layer</b><p>All-to-All；payload 与 top-k、路由宽度和负载相关。</p></article>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="pipeline">
|
||
<p class="eyebrow"><span>06</span> PIPELINE BUBBLES</p>
|
||
<h2>流水线不改变模型数学，只改变谁在什么时候做哪一段</h2>
|
||
|
||
<div class="pipeline-formula">
|
||
<div><span>FLUSH BUBBLE</span><b>(p − 1)(F + B)</b><p>理想平衡 stage、忽略通信。</p></div>
|
||
<div><span>IDEAL WORK</span><b>m(F + B)</b><p>m 个 micro-batches 的有效计算。</p></div>
|
||
<div><span>UTILIZATION</span><b>m / (m + p − 1)</b><p>GPipe / non-interleaved flush 的简化式。</p></div>
|
||
</div>
|
||
|
||
<div class="pipeline-waves">
|
||
{pipelineWaves.map((wave, index) => (
|
||
<article>
|
||
<div>{String(index + 1).padStart(2, "0")}</div>
|
||
<time>{wave.year}</time>
|
||
<section><h3>{wave.title}</h3><p>{wave.gain}</p><small>留下的债：{wave.debt}</small></section>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<div class="bw-split">
|
||
<div>
|
||
<span>STANDARD BACKWARD</span>
|
||
<b>B = input-gradient + weight-gradient</b>
|
||
<p>两部分绑成一个调度单元，必须一起完成。</p>
|
||
</div>
|
||
<i>→</i>
|
||
<div>
|
||
<span>ZERO BUBBLE IDEA</span>
|
||
<b>B<sub>input</sub> 在依赖链上 · W 可以延后</b>
|
||
<p>先让前一 stage 继续反传，再把 W 放进之后的空隙。</p>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="deepseek-note">
|
||
<span>DEEPSEEK-V3 / DUALPIPE</span>
|
||
<h3>“近零 All-to-All 开销”不是“没有 All-to-All”</h3>
|
||
<p>
|
||
V3 把 forward/backward chunk 拆成 attention、dispatch、MLP、combine，再把 input-grad / weight-grad 分开；
|
||
双向注入 micro-batches，并手动划分通信与计算使用的 SM。通信字节仍真实经过 IB/NVLink，
|
||
只是多数传输在该配置下被相邻计算覆盖，不再暴露在关键路径。
|
||
</p>
|
||
<b>边界：DualPipe 需要两份参数，并比表中的 1F1B 多一个 stage-normalized activation 单位。</b>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="expert">
|
||
<p class="eyebrow"><span>07</span> EXPERT PARALLEL</p>
|
||
<h2>MoE 省的是每 Token 计算，不会自动省通信与等待</h2>
|
||
|
||
<div class="expert-flow">
|
||
<div><span>ROUTER</span><b>Token → top-k experts</b></div>
|
||
<i>→</i>
|
||
<div class="network"><span>DISPATCH</span><b>All-to-All #1</b></div>
|
||
<i>→</i>
|
||
<div><span>EXPERT GEMM</span><b>每 rank 本地执行</b></div>
|
||
<i>→</i>
|
||
<div class="network"><span>COMBINE</span><b>All-to-All #2</b></div>
|
||
<i>→</i>
|
||
<div><span>MIX</span><b>按 router weight 聚合</b></div>
|
||
</div>
|
||
|
||
<p>
|
||
逻辑 payload 近似随 `tokens × top-k × routed width × bytes` 增长；但 end-to-end time 还受 capacity、
|
||
padding、路由 metadata、跨节点比例和最忙 rank makespan 影响。<a href="/moe/">MoE 专题</a>
|
||
已详细解释模型路由，本章只聚焦执行系统。
|
||
</p>
|
||
|
||
<div class="moe-systems">
|
||
<article><span>GShard</span><b>把 expert parallel 带进大规模 Transformer</b><p>路由、自动分片与跨设备 expert execution 成为一体。</p></article>
|
||
<article><span>Tutel</span><b>自适应 parallelism 与 kernel</b><p>不同专家数、capacity、硬件下切换执行策略。</p></article>
|
||
<article><span>MegaBlocks</span><b>Dropless block-sparse compute</b><p>不靠固定 capacity padding / dropping，但真实不均衡工作仍存在。</p></article>
|
||
<article><span>DeepEP</span><b>优化 dispatch / combine 数据路径</b><p>高吞吐与低延迟 kernel；不单独保证每 rank token load 相同。</p></article>
|
||
</div>
|
||
|
||
<h3>MoonEP：先给 rank 级完美均衡一个上界保证</h3>
|
||
<div class="moonep-proof">
|
||
<div>
|
||
<span>INPUT</span><b>S×K×R assignments</b><p>每 rank 本地 S Tokens，每 Token 选择 K experts。</p>
|
||
</div>
|
||
<i>→</i>
|
||
<div>
|
||
<span>ONLINE PLAN</span><b>填满 underloaded rank</b><p>remote tokens 最多来自一个 source rank。</p>
|
||
</div>
|
||
<i>→</i>
|
||
<div>
|
||
<span>BOUND</span><b>≤ E/R redundant experts</b><p>每个 source rank 本地最多 E/R experts。</p>
|
||
</div>
|
||
<i>→</i>
|
||
<div>
|
||
<span>STATIC SHAPE</span><b>每 rank 恰好 S×K</b><p>固定 buffer，消除逐层 shape host sync。</p>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="warning-note">
|
||
<b>完美 rank balance 不是完美 expert-GEMM balance</b>
|
||
<p>
|
||
每 rank 总 assignments 相同以后，rank 内不同 experts 仍可一多一少。K3 还需要 workload-aware scheduler
|
||
平衡 SM makespan。MoonEP 的 `S×K` vs DeepEP `S×K×R` 也只适用于报告定义的 worst-case copy-free buffer 对照。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="context">
|
||
<p class="eyebrow"><span>08</span> CONTEXT PARALLEL</p>
|
||
<h2>长序列并行不是一种算法，而是一组不同的数据布局</h2>
|
||
|
||
<div class="context-table">
|
||
<div class="table-head"><span>方法</span><span>切什么</span><span>通信</span><span>关键边界</span></div>
|
||
{contextMethods.map((row) => <div>{row.map((cell, index) => index === 0 ? <b>{cell}</b> : <span>{cell}</span>)}</div>)}
|
||
</div>
|
||
|
||
<div class="context-visuals">
|
||
<article>
|
||
<span>ULYSSES</span>
|
||
<div class="ulysses-grid before"><i>S/4 · all heads</i><i>S/4 · all heads</i><i>S/4 · all heads</i><i>S/4 · all heads</i></div>
|
||
<b>All-to-All</b>
|
||
<div class="ulysses-grid after"><i>full S · H0</i><i>full S · H1</i><i>full S · H2</i><i>full S · H3</i></div>
|
||
<p>把 sequence shard 转成 head shard；attention 后再转回来。</p>
|
||
</article>
|
||
<article>
|
||
<span>RING ATTENTION</span>
|
||
<div class="ring-grid"><i>Q0<br />K/V0</i><i>Q1<br />K/V1</i><i>Q2<br />K/V2</i><i>Q3<br />K/V3</i></div>
|
||
<b>K/V blocks 轮转 →</b>
|
||
<p>每 rank 固定本地 Q，在线累积对全局 K/V 的精确 softmax。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<div class="kcp-note">
|
||
<span>K3 / KDA CONTEXT PARALLEL</span>
|
||
<p>
|
||
KDA 是 recurrent linear attention：每 rank 从本地 Token 计算 fixed-size transition/state fragments，
|
||
一次 AllGather 后用 prefix scan 恢复 incoming state。它不搬完整历史 KV，但只覆盖 KDA 分支；
|
||
K3 的 MLA 与视觉 encoder 仍有各自 CP。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="precision">
|
||
<p class="eyebrow"><span>09</span> NUMERICS & OPTIMIZER</p>
|
||
<h2>低精度训练要分别回答：存什么、算什么、在哪里累加</h2>
|
||
|
||
<div class="precision-pipeline">
|
||
<div><span>STORE</span><b>BF16 / FP8</b><p>权重与 activation 占多少 HBM。</p></div>
|
||
<i>→</i>
|
||
<div><span>GEMM INPUT</span><b>FP16 / BF16 / FP8</b><p>Tensor Core 使用何种格式。</p></div>
|
||
<i>→</i>
|
||
<div><span>ACCUMULATE</span><b>FP32 / mixed</b><p>大量乘加与 reduction 是否丢失小量。</p></div>
|
||
<i>→</i>
|
||
<div><span>UPDATE</span><b>FP32 master</b><p>优化器状态和参数更新精度。</p></div>
|
||
</div>
|
||
|
||
<div class="compare-columns">
|
||
<article>
|
||
<span>DEEPSEEK-V3</span>
|
||
<h3>FP8 compute framework</h3>
|
||
<p>
|
||
主要 compute-intensive GEMM 使用 FP8；敏感操作保留更高精度；tile/block scaling 管动态范围；
|
||
WGrad FP8 也让相关 activation 可按 FP8 保存。
|
||
</p>
|
||
</article>
|
||
<article>
|
||
<span>KIMI K2</span>
|
||
<h3>FP8 activation storage</h3>
|
||
<p>
|
||
部分 MoE / SwiGLU 输入压成 FP8-E4M3，scale 为 FP32；K2 明确没有用 FP8 compute，
|
||
因为前期研究观察到潜在性能退化风险。
|
||
</p>
|
||
</article>
|
||
<article>
|
||
<span>KIMI K3</span>
|
||
<h3>Muon 改写通信图</h3>
|
||
<p>
|
||
Newton–Schulz orthogonalization 需要完整矩阵；K3 不全量 AllGather，而让 owner rank
|
||
P2P 获取自己更新所需 shards，并按 model chunk pipeline。
|
||
</p>
|
||
</article>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="deepseek">
|
||
<p class="eyebrow"><span>10</span> DEEPSEEK SYSTEMS LINEAGE</p>
|
||
<h2>DeepSeek 的亮点不是单个 kernel，而是模型、路由、调度与网络共同设计</h2>
|
||
|
||
<div class="lineage-rail deepseek-rail">
|
||
{deepseekSteps.map(([model, year, config, body], index) => (
|
||
<article>
|
||
<div><span>{String(index + 1).padStart(2, "0")}</span><b>{model}</b><time>{year}</time></div>
|
||
<section><strong>{config}</strong><p>{body}</p></section>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<h3>V3 的四层闭环</h3>
|
||
<div class="v3-loop">
|
||
<article><span>MODEL</span><b>MLA + fine-grained MoE</b><p>少 activated parameters；但跨节点 expert traffic 重。</p></article>
|
||
<article><span>ROUTER</span><b>最多 4 个节点</b><p>限制 IB fan-out，再在节点内经 NVLink 转发。</p></article>
|
||
<article><span>KERNEL</span><b>20 SM communication</b><p>warp specialization 与动态任务分配；这是报告集群的实测配置。</p></article>
|
||
<article><span>SCHEDULE</span><b>DualPipe overlap</b><p>把 A2A 与 attention / MLP / backward 重排进同一时间轴。</p></article>
|
||
</div>
|
||
|
||
<div class="boundary-note">
|
||
<b>“V3 不用 TP”是配置事实，不是架构定律</b>
|
||
<p>
|
||
V3 借助大 EP、ZeRO-1、selective recomputation 与 FP8 把状态放下，避免高频 TP 通信。
|
||
换硬件、batch、专家布局或上下文长度后，最优并行配置可能改变。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="kimi">
|
||
<p class="eyebrow"><span>11</span> KIMI SYSTEMS LINEAGE</p>
|
||
<h2>K2 → K3：从“稳定复用一套并行配置”走向统一执行与存储系统</h2>
|
||
|
||
<div class="kimi-lineage">
|
||
{kimiSteps.map(([model, scale, mechanism, body], index) => (
|
||
<article>
|
||
<div><span>{String(index + 1).padStart(2, "0")}</span><b>{model}</b><small>{scale}</small></div>
|
||
<section><h3>{mechanism}</h3><p>{body}</p></section>
|
||
</article>
|
||
))}
|
||
</div>
|
||
|
||
<h3>K2 为什么主动不采用 DualPipe</h3>
|
||
<div class="decision-balance">
|
||
<div>
|
||
<span>DUALPIPE GAIN</span>
|
||
<b>更少 bubble + 重 A2A overlap</b>
|
||
</div>
|
||
<i>VS</i>
|
||
<div>
|
||
<span>K2 COST</span>
|
||
<b>2× parameter / gradient memory</b>
|
||
</div>
|
||
<i>→</i>
|
||
<div class="chosen">
|
||
<span>K2 CHOICE</span>
|
||
<b>Interleaved 1F1B + extra warmup + WGrad overlap</b>
|
||
</div>
|
||
</div>
|
||
|
||
<p>
|
||
K2 是 1T total-parameter MoE。报告指出，DualPipe 的额外状态会迫使系统增加 PP 或 EP：
|
||
更大 PP 增加 bubble，更大 EP 提高通信与负载成本。因此团队保留 16 PP / 16 EP / ZeRO-1，
|
||
用更多 warmup micro-batches 覆盖 EP communication，并让 WGrad 与 PP communication 并行。
|
||
</p>
|
||
|
||
<h3>K3 3T-class 预训练执行图</h3>
|
||
<div class="k3-stack">
|
||
<div><span>ROUTING</span><b>MoonEP</b><p>在线冗余 expert 规划 → rank perfect balance → static shape</p></div>
|
||
<div><span>ACTIVATION</span><b>Unified manager</b><p>recompute + block FP8 + local/remote offload</p></div>
|
||
<div><span>GRADIENT</span><b>Pipeline ZeRO-2</b><p>GPU double buffer → DP reduce → CPU shards</p></div>
|
||
<div><span>OPTIMIZER</span><b>P2P Muon</b><p>只取 locally owned matrices 的远端 shards</p></div>
|
||
<div><span>MULTIMODAL</span><b>Dynamic CP + bubble fill</b><p>大图切 patch；大部分 ViT compute 放入 text PP 空隙</p></div>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section lab-section" id="lab">
|
||
<p class="eyebrow"><span>12</span> INTERACTIVE LAB</p>
|
||
<h2>亲手改变配置，看瓶颈怎样从一张账移动到另一张账</h2>
|
||
<p>
|
||
推荐依次尝试：在显存账中把 70B 切到 ZeRO-3；在 device mesh 选择“故意冲突”；
|
||
在 pipeline 中比较 1F1B 与 DualPipe 的参数副本；最后把通信 overlap 拉到 95%，观察 bytes 不变、exposed time 下降。
|
||
</p>
|
||
<TrainingSystemsLab />
|
||
</section>
|
||
|
||
<section class="article-section" id="agentic">
|
||
<p class="eyebrow"><span>13</span> MILLION-TOKEN AGENTIC RL</p>
|
||
<h2>到 Agentic RL，训练系统还要管理 KV、环境和跨迭代长尾</h2>
|
||
|
||
<div class="rl-cycle">
|
||
<article><span>01</span><b>TRAIN</b><p>Policy update；model、optimizer 与 gradient 占 GPU/CPU。</p></article>
|
||
<i>→</i>
|
||
<article><span>02</span><b>RESHARD</b><p>训练布局转成 inference layout；K2 用 checkpoint engine。</p></article>
|
||
<i>→</i>
|
||
<article><span>03</span><b>ROLLOUT</b><p>百万 Token KV、tool latency、partial trajectory 与 sandbox。</p></article>
|
||
<i>→</i>
|
||
<article><span>04</span><b>REWARD</b><p>Reference / judge forward；权重可能无法常驻 GPU。</p></article>
|
||
<i>↺</i>
|
||
</div>
|
||
|
||
<h3>K3 的三次“生命周期复用”</h3>
|
||
<div class="reuse-cards">
|
||
<article>
|
||
<span>KV / WRITE-BACK</span><b>只在 GPU eviction 时写 CPU</b>
|
||
<p>active decode blocks 留在 GPU；idle reusable prefix 才进入 external pool，KDA state 与 MLA KV 一起管理。</p>
|
||
</article>
|
||
<article>
|
||
<span>HBM / GRAD BUFFER</span><b>Reference weights 借用 policy gradient storage</b>
|
||
<p>一个 VPP slot 当前 forward，另一个 prefetch 下一 chunk；真实 backward 前再被 gradient 覆盖。</p>
|
||
</article>
|
||
<article>
|
||
<span>ENV / PAUSE</span><b>推理等待时释放 sandbox 资源</b>
|
||
<p>AgentENV 用 Firecracker microVM，支持 incremental checkpoint、resume、fork 与 snapshot。</p>
|
||
</article>
|
||
</div>
|
||
|
||
<div class="agentenv-stats">
|
||
<div><span>CHECKPOINT</span><b>133 ms</b><p>报告最低延迟</p></div>
|
||
<div><span>RESUME</span><b>49 ms</b><p>报告最低延迟</p></div>
|
||
<div><span>MEMORY OVERCOMMIT</span><b>up to 6.5×</b><p>真实 workload 报告</p></div>
|
||
<div><span>SANDBOXES</span><b>51,219,741</b><p>K3 训练与评估累计</p></div>
|
||
</div>
|
||
|
||
<div class="warning-note">
|
||
<b>这些是 K3 报告的生产统计，不是通用 Firecracker benchmark</b>
|
||
<p>
|
||
系统数字高度依赖镜像、内存 dirty rate、存储层和并发形态。课程保留原始口径，不把最小延迟或最高 overcommit 外推到其他环境。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="decisions">
|
||
<p class="eyebrow"><span>14</span> CONFIGURATION PLAYBOOK</p>
|
||
<h2>没有“最好并行策略”，只有当前最先触顶的约束</h2>
|
||
|
||
<div class="decision-tree">
|
||
<article><span>01</span><b>单 replica 的模型状态放不下？</b><p>先算精确 byte ledger；考虑 ZeRO/FSDP、TP/PP/EP 或 optimizer offload。</p></article>
|
||
<article><span>02</span><b>参数放下，但 activation OOM？</b><p>减 micro-batch；selective recompute、FlashAttention、SP/CP、压缩或 offload。</p></article>
|
||
<article><span>03</span><b>GPU 忙但 MFU 低？</b><p>检查 GEMM shape、kernel fusion、micro-batch、低精度和 launch overhead。</p></article>
|
||
<article><span>04</span><b>GPU 在等网络？</b><p>先定位 collective、频率和拓扑；再谈减少体积、换 group、chunk 或 overlap。</p></article>
|
||
<article><span>05</span><b>PP 有大块空白？</b><p>增加 m、virtual chunks、B/W split 或双向 schedule，同时重算 activation 与参数副本。</p></article>
|
||
<article><span>06</span><b>MoE 最忙 rank 拖尾？</b><p>分开 router balance、rank placement、redundant experts 与 rank 内 GEMM scheduling。</p></article>
|
||
<article><span>07</span><b>长序列单样本放不下？</b><p>按 attention 类型选择 Ulysses、Ring、USP 或 recurrent-state CP；不要只增 DP。</p></article>
|
||
<article><span>08</span><b>训练能跑但经常失败？</b><p>计算 checkpoint 恢复时间、数据确定性、world-size reshard 与环境状态恢复。</p></article>
|
||
</div>
|
||
|
||
<div class="final-principle">
|
||
<span>ONE RULE TO KEEP</span>
|
||
<h3>每项优化都要写两句话：它省了什么；它把代价搬到了哪里。</h3>
|
||
</div>
|
||
</section>
|
||
|
||
<section class="article-section" id="papers">
|
||
<p class="eyebrow"><span>↳</span> PRIMARY-SOURCE CHAIN</p>
|
||
<h2>从分布式深度学习到 K3：37 个一手阅读节点</h2>
|
||
<p>
|
||
这不是按引用数排序的“必读榜”，而是一条问题链。建议先读带有当前瓶颈的节点：
|
||
容量读 ZeRO；层内切分读 Megatron；气泡读 GPipe / ZeroBubble；MoE 执行读 MegaBlocks / DeepEP / MoonEP；
|
||
长序列读 Ulysses / Ring / USP；最后回到 DeepSeek 与 Kimi 的整机协同。
|
||
</p>
|
||
|
||
<div class="paper-chain systems-papers">
|
||
{paperChain.map(([year, title, url, note], index) => (
|
||
<a href={url}>
|
||
<span>{String(index + 1).padStart(2, "0")}</span>
|
||
<time>{year}</time>
|
||
<b>{title}</b>
|
||
<p>{note}</p>
|
||
</a>
|
||
))}
|
||
</div>
|
||
|
||
<div class="source-note">
|
||
<b>证据规则</b>
|
||
<p>
|
||
本页机制和数字回查论文、官方技术报告或作者仓库；图均为课程原创简化示意。
|
||
性能提升只属于论文的模型、集群和基线，不作跨系统排行榜。
|
||
</p>
|
||
</div>
|
||
</section>
|
||
</article>
|
||
</div>
|
||
</BaseLayout>
|
||
|
||
<style>
|
||
.systems-hero {
|
||
background:
|
||
linear-gradient(105deg, rgba(23, 36, 33, 0.98) 0 54%, rgba(23, 36, 33, 0.76) 100%),
|
||
repeating-linear-gradient(90deg, transparent 0 76px, rgba(255,255,255,.045) 76px 77px),
|
||
#1c2d29;
|
||
}
|
||
|
||
.systems-hero::after {
|
||
content: "";
|
||
position: absolute;
|
||
right: 5%;
|
||
bottom: 12%;
|
||
width: min(38vw, 520px);
|
||
height: min(24vw, 330px);
|
||
opacity: 0.44;
|
||
background:
|
||
linear-gradient(90deg, transparent 47%, #b8794c 48% 52%, transparent 53%) 0 0 / 25% 100%,
|
||
linear-gradient(0deg, transparent 47%, #64857d 48% 52%, transparent 53%) 0 0 / 100% 25%;
|
||
mask-image: linear-gradient(120deg, transparent, #000 28% 78%, transparent);
|
||
pointer-events: none;
|
||
}
|
||
|
||
.systems-hero .page-hero-inner { z-index: 1; }
|
||
.systems-hero h1 { color: #f6f2e8; }
|
||
.systems-hero .lead { color: #b9c9c4; }
|
||
.systems-hero .eyebrow { color: #91aaa4; }
|
||
.systems-hero .eyebrow span { color: #d08352; }
|
||
.systems-hero .page-facts { border-color: rgba(219, 230, 226, 0.25); }
|
||
.systems-hero .page-facts div { border-color: rgba(219, 230, 226, 0.2); }
|
||
.systems-hero .page-facts dt { color: #91aaa4; }
|
||
.systems-hero .page-facts dd { color: #f6f2e8; }
|
||
|
||
.system-ledgers { grid-template-columns: repeat(3, 1fr); }
|
||
.system-ledgers article { min-height: 170px; }
|
||
.system-ledgers article:nth-child(5) { background: #263b37; color: #fff; }
|
||
.system-ledgers article:nth-child(5) b { color: #fff; }
|
||
.system-ledgers article:nth-child(5) p { color: #bfcac6; }
|
||
|
||
.cost-stack {
|
||
display: grid;
|
||
grid-template-columns: repeat(4, 1fr);
|
||
margin: 2rem 0;
|
||
border: 1px solid var(--line);
|
||
}
|
||
|
||
.cost-stack div {
|
||
min-height: 190px;
|
||
padding: 1rem;
|
||
border-right: 1px solid rgba(255,255,255,.17);
|
||
color: #fff;
|
||
}
|
||
|
||
.cost-stack div:last-child { border-right: 0; }
|
||
.cost-stack div:nth-child(1) { background: #784e3b; }
|
||
.cost-stack div:nth-child(2) { background: #566d69; }
|
||
.cost-stack div:nth-child(3) { background: #34534d; }
|
||
.cost-stack div:nth-child(4) { background: #243d38; }
|
||
.cost-stack span { font: 0.62rem var(--mono); opacity: 0.65; }
|
||
.cost-stack b { display: block; margin: 4rem 0 0.5rem; font: 700 0.72rem var(--mono); letter-spacing: .08em; }
|
||
.cost-stack p { margin: 0; color: rgba(255,255,255,.72); font-size: 0.69rem; line-height: 1.5; }
|
||
|
||
.step-flow,
|
||
.expert-flow,
|
||
.precision-pipeline,
|
||
.rl-cycle {
|
||
display: flex;
|
||
align-items: stretch;
|
||
gap: 0.55rem;
|
||
margin: 1.75rem 0;
|
||
}
|
||
|
||
.step-flow article,
|
||
.rl-cycle article { flex: 1; padding: 1rem; border: 1px solid var(--line); background: #f6f1e7; }
|
||
.step-flow > i,
|
||
.expert-flow > i,
|
||
.precision-pipeline > i,
|
||
.rl-cycle > i { align-self: center; color: #a45d36; font: normal 1.2rem var(--mono); }
|
||
.step-flow span,
|
||
.rl-cycle span { color: #a45d36; font: 0.58rem var(--mono); }
|
||
.step-flow b,
|
||
.rl-cycle b { display: block; margin: 1.7rem 0 .3rem; color: #263b37; font: 700 0.72rem var(--mono); }
|
||
.step-flow p,
|
||
.rl-cycle p { margin: 0; color: #65706c; font-size: .68rem; line-height: 1.5; }
|
||
.step-flow small { display: block; margin-top: .7rem; color: #8a785f; font: .55rem var(--mono); }
|
||
|
||
.lifetime-diagram { padding: 1.25rem; background: #263b37; }
|
||
.lifetime-diagram > div { display: grid; grid-template-columns: 110px 1fr repeat(3, .52fr); gap: .5rem; align-items: center; margin-bottom: .55rem; }
|
||
.lifetime-diagram > div:last-child { margin-bottom: 0; }
|
||
.lifetime-diagram b { color: #c5d0cc; font: .58rem var(--mono); }
|
||
.lifetime-diagram i { height: 12px; background: #568078; }
|
||
.lifetime-diagram i.split { background: linear-gradient(90deg, #a76440 0 18%, #3f5651 18% 74%, #a76440 74%); }
|
||
.lifetime-diagram i.late { background: linear-gradient(90deg, transparent 0 54%, #7a6a8c 54%); }
|
||
.lifetime-diagram i.pulse { background: linear-gradient(90deg, #3f5651 0 75%, #b18c59 75%); }
|
||
.lifetime-diagram span { color: #9bacA7; font-size: .58rem; text-align: center; }
|
||
|
||
.insight-note,
|
||
.formula-note,
|
||
.kcp-note,
|
||
.boundary-note {
|
||
margin-top: 1.25rem;
|
||
padding: 1rem 1.2rem;
|
||
border-left: 4px solid #5d8178;
|
||
background: #e0e9e5;
|
||
}
|
||
|
||
.insight-note b,
|
||
.boundary-note b { color: #27423c; }
|
||
.insight-note p,
|
||
.formula-note p,
|
||
.kcp-note p,
|
||
.boundary-note p { margin: .35rem 0 0; color: #596966; line-height: 1.65; }
|
||
|
||
.byte-ledger { margin: 1.5rem 0; border-top: 1px solid var(--line); }
|
||
.byte-ledger > div { display: grid; grid-template-columns: 42px 1.2fr .45fr 1.5fr; gap: 1rem; align-items: center; padding: .9rem; border: 1px solid var(--line); border-top: 0; }
|
||
.byte-ledger span { color: #9a7752; font: .58rem var(--mono); }
|
||
.byte-ledger b { color: #31423e; font: 650 .76rem var(--mono); }
|
||
.byte-ledger strong { color: #a45d36; font: 650 1rem var(--mono); }
|
||
.byte-ledger p { margin: 0; color: #6f7774; font-size: .69rem; }
|
||
.byte-ledger .total { color: #fff; background: #263b37; }
|
||
.byte-ledger .total b,
|
||
.byte-ledger .total strong { color: #fff; }
|
||
.byte-ledger .total p { color: #b7c4c0; }
|
||
|
||
.zero-ladder { display: grid; grid-template-columns: repeat(4, 1fr); gap: 1px; margin: 1rem 0; background: var(--line); border: 1px solid var(--line); }
|
||
.zero-ladder article { min-height: 230px; padding: 1rem; background: color-mix(in srgb, #f7f3ea calc(100% - var(--level) * 8%), #dce8e3); }
|
||
.zero-ladder span { color: #9b7955; font: .58rem var(--mono); }
|
||
.zero-ladder h4 { margin: 2.1rem 0 .3rem; color: #263b37; font: 500 1.35rem var(--serif); }
|
||
.zero-ladder b { display: block; color: #61716c; font-size: .68rem; }
|
||
.zero-ladder strong { display: block; margin: 1rem 0 .35rem; color: #a45d36; font: 650 .8rem var(--mono); }
|
||
.zero-ladder p { margin: 0; color: #747c79; font-size: .65rem; line-height: 1.5; }
|
||
.formula-note span,
|
||
.kcp-note span { color: #4f7269; font: .58rem var(--mono); letter-spacing: .08em; }
|
||
|
||
.activation-moves { display: grid; grid-template-columns: repeat(4, 1fr); gap: .65rem; margin: 1.5rem 0; }
|
||
.activation-moves article { min-height: 235px; padding: 1rem; border: 1px solid var(--line); background: #f7f3ea; }
|
||
.activation-moves span { color: #a45d36; font: .55rem var(--mono); }
|
||
.activation-moves h3 { margin: 2.4rem 0 .6rem; font-size: 1.15rem; }
|
||
.activation-moves p { color: #68716e; font-size: .7rem; line-height: 1.6; }
|
||
.activation-moves b { display: block; margin-top: 1rem; color: #7a6148; font-size: .67rem; }
|
||
|
||
.checkpoint-visual { display: grid; grid-template-columns: 1fr 1fr; gap: .8rem; margin: 1.5rem 0; }
|
||
.checkpoint-visual > div { padding: 1rem; border: 1px solid var(--line); }
|
||
.checkpoint-visual > div > span { color: #7a6b56; font: .58rem var(--mono); }
|
||
.checkpoint-visual > div > div { display: grid; grid-template-columns: repeat(6, 1fr); gap: 3px; margin: .8rem 0; }
|
||
.checkpoint-visual i { display: grid; height: 42px; place-items: center; background: #587b73; font-style: normal; }
|
||
.checkpoint-visual i.keep { background: #a45d36; }
|
||
.checkpoint-visual i:not(.keep) b { color: #c5d2ce; font-size: .45rem; }
|
||
.checkpoint-visual i b { color: #fff; font: .5rem var(--mono); }
|
||
.checkpoint-visual p { margin: 0; color: #6b7471; font-size: .67rem; }
|
||
|
||
.evidence-cards,
|
||
.moe-systems,
|
||
.v3-loop,
|
||
.reuse-cards,
|
||
.compare-columns { display: grid; grid-template-columns: repeat(4, 1fr); gap: .65rem; margin: 1rem 0; }
|
||
.evidence-cards article,
|
||
.moe-systems article,
|
||
.v3-loop article,
|
||
.reuse-cards article,
|
||
.compare-columns article { padding: 1rem; border-top: 3px solid #a45d36; background: #eee8dc; }
|
||
.evidence-cards span,
|
||
.moe-systems span,
|
||
.v3-loop span,
|
||
.reuse-cards span,
|
||
.compare-columns span { color: #9a704d; font: .56rem var(--mono); }
|
||
.evidence-cards b,
|
||
.moe-systems b,
|
||
.v3-loop b,
|
||
.reuse-cards b { display: block; margin: .7rem 0 .4rem; color: #2c403b; font-size: .76rem; }
|
||
.evidence-cards p,
|
||
.moe-systems p,
|
||
.v3-loop p,
|
||
.reuse-cards p,
|
||
.compare-columns p { margin: 0; color: #69716f; font-size: .67rem; line-height: 1.55; }
|
||
|
||
.axis-map { display: grid; grid-template-columns: repeat(5, 1fr); gap: 1px; margin: 1.5rem 0; border: 1px solid var(--line); background: var(--line); }
|
||
.axis-map article { min-height: 285px; padding: 1rem; background: #f6f1e7; }
|
||
.axis-map span { color: #96714f; font: .52rem var(--mono); }
|
||
.axis-map strong { display: block; margin: 2rem 0 .3rem; color: #a45d36; font: 500 2rem var(--serif); }
|
||
.axis-map h3 { margin: 0 0 .8rem; font-size: 1rem; }
|
||
.axis-map p { color: #69716e; font-size: .67rem; line-height: 1.55; }
|
||
.axis-map b { display: block; margin-top: 1rem; color: #536d67; font: .58rem var(--mono); }
|
||
|
||
.mesh-visual { padding: 1rem; background: #263b37; }
|
||
.mesh-labels,
|
||
.mesh-nodes { display: grid; grid-template-columns: repeat(4, 1fr); gap: .6rem; }
|
||
.mesh-labels span { color: #9fb1ac; font: .5rem var(--mono); }
|
||
.mesh-nodes > div { display: grid; grid-template-columns: repeat(4, 1fr); gap: 3px; padding: .5rem; border: 1px solid #547068; }
|
||
.mesh-nodes i { display: grid; min-height: 58px; place-content: center; color: #fff; background: #517a71; font-style: normal; text-align: center; }
|
||
.mesh-nodes i b { font: .62rem var(--mono); }
|
||
.mesh-nodes i small { margin-top: .15rem; color: #c1d0cc; font: .45rem var(--mono); }
|
||
.mesh-links { display: flex; gap: .5rem; margin-top: .8rem; }
|
||
.mesh-links span { flex: 1; padding: .5rem; color: #fff; font: .54rem var(--mono); }
|
||
.mesh-links .tp { background: #517a71; }
|
||
.mesh-links .pp { background: #a45d36; }
|
||
.mesh-links .dp { background: #756985; }
|
||
|
||
.alpha-beta { display: flex; gap: .7rem; align-items: center; margin: 1.4rem 0; }
|
||
.alpha-beta div { flex: 1; min-height: 145px; padding: 1rem; border: 1px solid var(--line); background: #f5f0e6; }
|
||
.alpha-beta > i { color: #a45d36; font: normal 1.4rem var(--mono); }
|
||
.alpha-beta span { color: #8a745b; font: .58rem var(--mono); }
|
||
.alpha-beta b { display: block; margin: 1.4rem 0 .4rem; color: #263b37; font: 650 1rem var(--mono); }
|
||
.alpha-beta p { margin: 0; color: #6b7471; font-size: .67rem; }
|
||
|
||
.collective-table,
|
||
.context-table { border: 1px solid var(--line); }
|
||
.collective-table > div,
|
||
.context-table > div { display: grid; grid-template-columns: .8fr 1fr 1.2fr 1.5fr 1.1fr; gap: 1rem; padding: .75rem .9rem; border-bottom: 1px solid var(--line); align-items: center; }
|
||
.context-table > div { grid-template-columns: .8fr 1.25fr 1fr 2fr; }
|
||
.collective-table > div:last-child,
|
||
.context-table > div:last-child { border-bottom: 0; }
|
||
.collective-table .table-head,
|
||
.context-table .table-head { color: #fff; background: #263b37; font: .54rem var(--mono); letter-spacing: .06em; }
|
||
.collective-table b,
|
||
.context-table b { color: #a45d36; font: .66rem var(--mono); }
|
||
.collective-table span,
|
||
.context-table span { color: #69716e; font-size: .64rem; }
|
||
|
||
.frequency-comparison { display: grid; grid-template-columns: repeat(4, 1fr); gap: 1px; border: 1px solid var(--line); background: var(--line); }
|
||
.frequency-comparison article { padding: 1rem; background: #f6f1e7; }
|
||
.frequency-comparison span { color: #a45d36; font: 500 1.4rem var(--serif); }
|
||
.frequency-comparison b { display: block; margin: 1rem 0 .4rem; color: #31433f; font-size: .69rem; }
|
||
.frequency-comparison p { margin: 0; color: #6f7774; font-size: .65rem; }
|
||
|
||
.pipeline-formula { display: grid; grid-template-columns: repeat(3, 1fr); gap: .7rem; margin: 1.3rem 0; }
|
||
.pipeline-formula div { padding: 1rem; color: #fff; background: #263b37; }
|
||
.pipeline-formula span { color: #b5c3bf; font: .55rem var(--mono); }
|
||
.pipeline-formula b { display: block; margin: 1.6rem 0 .4rem; color: #fff; font: 650 .92rem var(--mono); }
|
||
.pipeline-formula p { margin: 0; color: #aebcb8; font-size: .65rem; }
|
||
|
||
.pipeline-waves article,
|
||
.lineage-rail article,
|
||
.kimi-lineage article { display: grid; grid-template-columns: 54px 96px 1fr; gap: 1rem; padding: 1rem 0; border-top: 1px solid var(--line); }
|
||
.pipeline-waves article > div { color: #a45d36; font: .65rem var(--mono); }
|
||
.pipeline-waves time { color: #78807d; font: .6rem var(--mono); }
|
||
.pipeline-waves h3 { margin: 0 0 .35rem; font-size: 1rem; }
|
||
.pipeline-waves p { margin: 0 0 .35rem; color: #626c69; font-size: .72rem; }
|
||
.pipeline-waves small { color: #94694b; font-size: .65rem; }
|
||
|
||
.bw-split,
|
||
.decision-balance { display: flex; gap: .7rem; align-items: stretch; margin: 1.4rem 0; }
|
||
.bw-split div,
|
||
.decision-balance div { flex: 1; padding: 1rem; border: 1px solid var(--line); background: #f5f0e6; }
|
||
.bw-split > i,
|
||
.decision-balance > i { align-self: center; color: #a45d36; font: normal .8rem var(--mono); }
|
||
.bw-split span,
|
||
.decision-balance span { color: #8c6a4c; font: .55rem var(--mono); }
|
||
.bw-split b,
|
||
.decision-balance b { display: block; margin: 1rem 0 .4rem; color: #2e433e; font: 650 .78rem var(--mono); }
|
||
.bw-split p { margin: 0; color: #69716e; font-size: .67rem; }
|
||
.decision-balance .chosen { color: #fff; background: #263b37; }
|
||
.decision-balance .chosen b { color: #fff; }
|
||
|
||
.deepseek-note { margin-top: 1.3rem; padding: 1.3rem; color: #fff; background: #6f4c39; }
|
||
.deepseek-note span { color: #e2b98f; font: .58rem var(--mono); }
|
||
.deepseek-note h3 { color: #fff; }
|
||
.deepseek-note p { color: #eaded5; line-height: 1.65; }
|
||
.deepseek-note b { font-size: .68rem; }
|
||
|
||
.expert-flow div,
|
||
.precision-pipeline div { flex: 1; padding: 1rem; border: 1px solid var(--line); background: #f5f0e6; }
|
||
.expert-flow div.network { color: #fff; background: #a45d36; }
|
||
.expert-flow span,
|
||
.precision-pipeline span { color: #8c6a4c; font: .54rem var(--mono); }
|
||
.expert-flow .network span { color: #f1caa5; }
|
||
.expert-flow b,
|
||
.precision-pipeline b { display: block; margin: 1.3rem 0 .2rem; color: #2c403b; font-size: .7rem; }
|
||
.expert-flow .network b { color: #fff; }
|
||
.precision-pipeline p { margin: 0; color: #69716e; font-size: .64rem; }
|
||
|
||
.moonep-proof { display: flex; gap: .45rem; align-items: stretch; margin: 1.3rem 0; }
|
||
.moonep-proof div { flex: 1; padding: .9rem; border: 1px solid var(--line); background: #e5ede9; }
|
||
.moonep-proof > i { align-self: center; color: #a45d36; font-style: normal; }
|
||
.moonep-proof span { color: #55756d; font: .52rem var(--mono); }
|
||
.moonep-proof b { display: block; margin: 1rem 0 .35rem; color: #29463f; font: 650 .69rem var(--mono); }
|
||
.moonep-proof p { margin: 0; color: #67736f; font-size: .63rem; }
|
||
|
||
.context-visuals { display: grid; grid-template-columns: 1fr 1fr; gap: .8rem; margin: 1.3rem 0; }
|
||
.context-visuals article { padding: 1rem; border: 1px solid var(--line); background: #f5f0e6; }
|
||
.context-visuals article > span { color: #a45d36; font: .56rem var(--mono); }
|
||
.context-visuals article > b { display: block; margin: .55rem 0; color: #607a74; font: .62rem var(--mono); text-align: center; }
|
||
.context-visuals article > p { margin: .8rem 0 0; color: #68716e; font-size: .67rem; }
|
||
.ulysses-grid,
|
||
.ring-grid { display: grid; grid-template-columns: repeat(4, 1fr); gap: 3px; margin-top: .8rem; }
|
||
.ulysses-grid i,
|
||
.ring-grid i { display: grid; min-height: 58px; place-content: center; padding: .3rem; color: #fff; background: #587b73; font: normal .49rem var(--mono); text-align: center; }
|
||
.ulysses-grid.after i { background: #9b6645; }
|
||
|
||
.compare-columns { grid-template-columns: repeat(3, 1fr); }
|
||
.compare-columns h3 { font-size: 1rem; }
|
||
|
||
.lineage-rail article { grid-template-columns: 130px 1fr; align-items: stretch; }
|
||
.lineage-rail article > div { display: grid; grid-template-columns: auto 1fr; gap: .3rem .7rem; padding: .8rem; color: #fff; background: #263b37; }
|
||
.lineage-rail article > div span { font: .55rem var(--mono); }
|
||
.lineage-rail article > div b { font: 500 1.4rem var(--serif); }
|
||
.lineage-rail article > div time { grid-column: 2; color: #aebbb7; font: .55rem var(--mono); }
|
||
.lineage-rail section { padding: .8rem; background: #f5f0e6; }
|
||
.lineage-rail strong { color: #a45d36; font: .65rem var(--mono); }
|
||
.lineage-rail p { margin: .6rem 0 0; color: #68716e; font-size: .7rem; }
|
||
|
||
.kimi-lineage article { grid-template-columns: 125px 1fr; }
|
||
.kimi-lineage article > div { padding: 1rem; color: #fff; background: #a45d36; }
|
||
.kimi-lineage article > div span { font: .55rem var(--mono); }
|
||
.kimi-lineage article > div b { display: block; margin-top: .5rem; font: 500 1.5rem var(--serif); }
|
||
.kimi-lineage article > div small { color: #edd4c1; font: .54rem var(--mono); }
|
||
.kimi-lineage section { padding: 1rem; border: 1px solid var(--line); border-left: 0; }
|
||
.kimi-lineage h3 { margin: 0 0 .4rem; font-size: 1rem; }
|
||
.kimi-lineage p { margin: 0; color: #68716e; font-size: .7rem; }
|
||
|
||
.k3-stack { display: grid; grid-template-columns: repeat(5, 1fr); gap: 1px; border: 1px solid var(--line); background: var(--line); }
|
||
.k3-stack div { min-height: 190px; padding: 1rem; background: #263b37; }
|
||
.k3-stack div:nth-child(even) { background: #34534d; }
|
||
.k3-stack span { color: #c49466; font: .53rem var(--mono); }
|
||
.k3-stack b { display: block; margin: 2rem 0 .5rem; color: #fff; font-size: .76rem; }
|
||
.k3-stack p { margin: 0; color: #b8c6c2; font-size: .64rem; line-height: 1.5; }
|
||
|
||
.reuse-cards { grid-template-columns: repeat(3, 1fr); }
|
||
.agentenv-stats { display: grid; grid-template-columns: repeat(4, 1fr); gap: 1px; border: 1px solid var(--line); background: var(--line); }
|
||
.agentenv-stats div { padding: 1rem; background: #263b37; }
|
||
.agentenv-stats span { color: #b9c7c3; font: .52rem var(--mono); }
|
||
.agentenv-stats b { display: block; margin: 1.2rem 0 .3rem; color: #fff; font: 600 1.15rem var(--serif); }
|
||
.agentenv-stats p { margin: 0; color: #9fb0ab; font-size: .61rem; }
|
||
|
||
.decision-tree { display: grid; grid-template-columns: repeat(2, 1fr); gap: .6rem; }
|
||
.decision-tree article { display: grid; grid-template-columns: 38px 1fr; gap: .25rem .7rem; padding: 1rem; border: 1px solid var(--line); }
|
||
.decision-tree span { grid-row: 1 / 3; color: #a45d36; font: .6rem var(--mono); }
|
||
.decision-tree b { color: #2f423e; font-size: .76rem; }
|
||
.decision-tree p { margin: 0; color: #69716e; font-size: .68rem; }
|
||
.final-principle { margin-top: 1rem; padding: 2rem; color: #fff; background: #a45d36; }
|
||
.final-principle span { color: #efd6c1; font: .58rem var(--mono); }
|
||
.final-principle h3 { max-width: 760px; margin-bottom: 0; color: #fff; font-size: clamp(1.4rem, 3vw, 2.3rem); }
|
||
|
||
.systems-papers { grid-template-columns: repeat(2, 1fr); }
|
||
.systems-papers a { min-height: 118px; }
|
||
|
||
@media (max-width: 980px) {
|
||
.axis-map,
|
||
.k3-stack { grid-template-columns: repeat(3, 1fr); }
|
||
.activation-moves,
|
||
.evidence-cards,
|
||
.moe-systems,
|
||
.v3-loop { grid-template-columns: repeat(2, 1fr); }
|
||
.step-flow,
|
||
.expert-flow,
|
||
.precision-pipeline,
|
||
.rl-cycle { display: grid; grid-template-columns: repeat(2, 1fr); }
|
||
.step-flow > i,
|
||
.expert-flow > i,
|
||
.precision-pipeline > i,
|
||
.rl-cycle > i { display: none; }
|
||
}
|
||
|
||
@media (max-width: 720px) {
|
||
.system-ledgers,
|
||
.cost-stack,
|
||
.zero-ladder,
|
||
.axis-map,
|
||
.frequency-comparison,
|
||
.pipeline-formula,
|
||
.k3-stack,
|
||
.agentenv-stats,
|
||
.compare-columns,
|
||
.reuse-cards { grid-template-columns: 1fr 1fr; }
|
||
.byte-ledger > div { grid-template-columns: 30px 1fr auto; }
|
||
.byte-ledger p { grid-column: 2 / -1; }
|
||
.collective-table,
|
||
.context-table { overflow-x: auto; }
|
||
.collective-table > div { min-width: 760px; }
|
||
.context-table > div { min-width: 680px; }
|
||
.mesh-labels { display: none; }
|
||
.mesh-nodes { grid-template-columns: repeat(2, 1fr); }
|
||
.mesh-links { flex-direction: column; }
|
||
.moonep-proof,
|
||
.bw-split,
|
||
.decision-balance,
|
||
.alpha-beta { display: grid; grid-template-columns: 1fr; }
|
||
.moonep-proof > i,
|
||
.bw-split > i,
|
||
.decision-balance > i,
|
||
.alpha-beta > i { display: none; }
|
||
.context-visuals,
|
||
.checkpoint-visual,
|
||
.decision-tree { grid-template-columns: 1fr; }
|
||
}
|
||
|
||
@media (max-width: 520px) {
|
||
.system-ledgers,
|
||
.cost-stack,
|
||
.zero-ladder,
|
||
.axis-map,
|
||
.activation-moves,
|
||
.evidence-cards,
|
||
.moe-systems,
|
||
.v3-loop,
|
||
.k3-stack,
|
||
.agentenv-stats,
|
||
.compare-columns,
|
||
.reuse-cards,
|
||
.step-flow,
|
||
.expert-flow,
|
||
.precision-pipeline,
|
||
.rl-cycle,
|
||
.systems-papers { grid-template-columns: 1fr; }
|
||
.lifetime-diagram > div { grid-template-columns: 84px 1fr; }
|
||
.lifetime-diagram span { display: none; }
|
||
.pipeline-waves article { grid-template-columns: 38px 1fr; }
|
||
.pipeline-waves time { grid-column: 2; grid-row: 1; }
|
||
.pipeline-waves section { grid-column: 2; }
|
||
}
|
||
</style>
|