feat: publish training systems deep dive

2026-07-29 01:26:38 +08:00
parent c3e1e85c67
commit c6306fe260
14 changed files with 3053 additions and 41 deletions
@@ -121,16 +121,16 @@ export const chapters: Chapter[] = [
  },
  {
    number: "08",
-    slug: "systems/training",
+    slug: "training-systems",
    title: "大规模训练系统",
    kicker: "DISTRIBUTED TRAINING",
    question: "一个 2.8T 模型如何摊到成千上万张卡上？",
-    summary: "从 ZeRO、Megatron 和五类并行，走到 DeepSeek DualPipe/DeepEP 与 K3 MoonEP。",
-    status: "queued",
-    progress: 12,
-    papers: 18,
+    summary: "用九张资源账串起 ZeRO、Megatron、Expert/Context Parallel、DeepSeek DualPipe 与 K3 MoonEP/百万 Token RL。",
+    status: "published",
+    progress: 71,
+    papers: 37,
    prerequisites: ["04", "06"],
-    highlights: ["并行维度", "通信重叠", "MoonEP"],
+    highlights: ["显存字节账", "流水线与通信", "MoonEP"],
  },
  {
    number: "09",
@@ -590,6 +590,54 @@ export const papers: Paper[] = [
    contribution: "AdamW 将权重衰减与梯度更新正确解耦。",
    verified: true,
  },
+  {
+    year: 2012,
+    title: "Large Scale Distributed Deep Networks",
+    url: "https://arxiv.org/abs/1206.5533",
+    topics: ["训练系统"],
+    contribution: "DistBelief 与参数服务器奠定早期大规模分布式深度学习系统路线。",
+    verified: true,
+  },
+  {
+    year: 2014,
+    title: "One Weird Trick for Parallelizing Convolutional Neural Networks",
+    url: "https://arxiv.org/abs/1404.5997",
+    topics: ["训练系统"],
+    contribution: "系统比较数据并行、模型并行及其通信边界。",
+    verified: true,
+  },
+  {
+    year: 2016,
+    title: "Training Deep Nets with Sublinear Memory Cost",
+    url: "https://arxiv.org/abs/1604.06174",
+    topics: ["训练系统"],
+    contribution: "用 activation checkpointing 以额外计算换取 O(√n) feature-map memory。",
+    verified: true,
+  },
+  {
+    year: 2017,
+    title: "Mixed Precision Training",
+    url: "https://arxiv.org/abs/1710.03740",
+    topics: ["低精度", "训练系统"],
+    contribution: "建立 FP32 master weights、loss scaling 与 FP32 accumulation 配方。",
+    verified: true,
+  },
+  {
+    year: 2018,
+    title: "PipeDream: Fast and Efficient Pipeline Parallel DNN Training",
+    url: "https://arxiv.org/abs/1806.03377",
+    topics: ["训练系统"],
+    contribution: "以 1F1B、异步 pipeline 与 weight stashing 提高 stage 利用率。",
+    verified: true,
+  },
+  {
+    year: 2018,
+    title: "Mesh-TensorFlow: Deep Learning for Supercomputers",
+    url: "https://arxiv.org/abs/1811.02084",
+    topics: ["训练系统"],
+    contribution: "把 tensor dimension 的布局声明映射到逻辑 device mesh。",
+    verified: true,
+  },
  {
    year: 2019,
    title: "GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism",
@@ -614,6 +662,86 @@ export const papers: Paper[] = [
    contribution: "分片优化器、梯度与参数，消除数据并行状态冗余。",
    verified: true,
  },
+  {
+    year: 2021,
+    title: "Efficient Large-Scale Language Model Training on GPU Clusters Using Megatron-LM",
+    url: "https://arxiv.org/abs/2104.04473",
+    topics: ["训练系统"],
+    contribution: "组合 TP、PP、DP，并提出 interleaved pipeline schedule。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning",
+    url: "https://arxiv.org/abs/2104.07857",
+    topics: ["训练系统"],
+    contribution: "把 GPU、CPU 与 NVMe 组成可带宽感知的异构训练内存层。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "GSPMD: General and Scalable Parallelization for ML Computation Graphs",
+    url: "https://arxiv.org/abs/2105.04663",
+    topics: ["训练系统"],
+    contribution: "用统一 SPMD 表达和 sharding propagation 扩展模型并行。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "Chimera: Efficiently Training Large-Scale Neural Networks with Bidirectional Pipelines",
+    url: "https://arxiv.org/abs/2107.06925",
+    topics: ["训练系统"],
+    contribution: "从 pipeline 两端注入 micro-batches，降低同步流水线气泡。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "DeepSpeed-MoE: Advancing Mixture-of-Experts Inference and Training",
+    url: "https://arxiv.org/abs/2201.05596",
+    topics: ["MoE", "训练系统"],
+    contribution: "组合 data、expert 与 tensor parallel，系统优化 MoE 训练和推理。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Alpa: Automating Inter- and Intra-Operator Parallelism for Distributed Deep Learning",
+    url: "https://arxiv.org/abs/2201.12023",
+    topics: ["训练系统"],
+    contribution: "用代价模型联合搜索 operator 内和 operator 间并行。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Reducing Activation Recomputation in Large Transformer Models",
+    url: "https://arxiv.org/abs/2205.05198",
+    topics: ["训练系统"],
+    contribution: "以 Megatron sequence parallel 与 selective recomputation 降低激活内存和重算开销。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Tutel: Adaptive Mixture-of-Experts at Scale",
+    url: "https://arxiv.org/abs/2206.03382",
+    topics: ["MoE", "训练系统"],
+    contribution: "让 MoE parallelism、dispatch 和 kernels 自适应模型与硬件配置。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "MegaBlocks: Efficient Sparse Training with Mixture-of-Experts",
+    url: "https://arxiv.org/abs/2211.15841",
+    topics: ["MoE", "训练系统"],
+    contribution: "用 block-sparse kernels 支持 dropless 动态 expert workloads。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "PyTorch FSDP: Experiences on Scaling Fully Sharded Data Parallel",
+    url: "https://arxiv.org/abs/2304.11277",
+    topics: ["训练系统"],
+    contribution: "总结 fully sharded 参数、梯度、状态与 prefetch 的工程实践。",
+    verified: true,
+  },
  {
    year: 2023,
    title: "DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models",
@@ -622,6 +750,39 @@ export const papers: Paper[] = [
    contribution: "沿 attention head 做 sequence parallel，扩展极长序列训练。",
    verified: true,
  },
+  {
+    year: 2024,
+    title: "Zero Bubble Pipeline Parallelism",
+    url: "https://arxiv.org/abs/2401.10241",
+    topics: ["训练系统"],
+    contribution: "拆分 input-gradient 与 weight-gradient，并在内存约束下搜索同步零气泡调度。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "MegaScale: Scaling Large Language Model Training to More Than 10,000 GPUs",
+    url: "https://arxiv.org/abs/2402.15627",
+    topics: ["训练系统"],
+    contribution: "系统讨论 10K+ GPU 集群的并行、网络优化与可靠性。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "USP: A Unified Sequence Parallelism Approach for Long Context Generative AI",
+    url: "https://arxiv.org/abs/2405.07719",
+    topics: ["长上下文", "训练系统"],
+    contribution: "在二维 mesh 上组合 Ulysses All-to-All 与 Ring P2P。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "DeepEP: An Efficient Expert-Parallel Communication Library",
+    url: "https://github.com/deepseek-ai/DeepEP",
+    topics: ["MoE", "训练系统"],
+    contribution: "提供面向 NVLink/RDMA 的高吞吐与低延迟 expert dispatch/combine kernels。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
  {
    year: 2024,
    title: "Mooncake: A KVCache-centric Disaggregated Architecture for LLM Serving",
@@ -1055,7 +1216,7 @@ export const papers: Paper[] = [
    year: 2025,
    title: "Kimi K2: Open Agentic Intelligence",
    url: "https://arxiv.org/abs/2507.20534",
-    topics: ["MoE", "Agent", "后训练", "推理"],
+    topics: ["MoE", "训练系统", "Agent", "后训练", "推理"],
    contribution: "开放 1T MoE Agent 模型，以 verifiable gym 与 self-critique rubric 做 joint RL。",
    spotlight: "Kimi",
    verified: true,
@@ -1064,7 +1225,7 @@ export const papers: Paper[] = [
    year: 2026,
    title: "Kimi K2.5: Visual Agentic Intelligence",
    url: "https://arxiv.org/abs/2602.02276",
-    topics: ["Agent", "多模态", "推理"],
+    topics: ["训练系统", "Agent", "多模态", "推理"],
    contribution: "把视觉、工具使用和并行 Agent Swarm 纳入统一后训练。",
    spotlight: "Kimi",
    verified: true,