feat: add inference serving chapter

2026-07-29 08:39:32 +08:00
parent a7492e1a59
commit db2b7d3570
25 changed files with 4178 additions and 44 deletions
@@ -203,12 +203,12 @@ export const chapters: Chapter[] = [
    title: "推理服务与低成本部署",
    kicker: "INFERENCE",
    question: "模型训练完以后，怎样让千万人用得起？",
-    summary: "拆解 KV Cache、PagedAttention、批处理、推测解码、PD 解耦、前缀缓存与集群调度。",
-    status: "queued",
-    progress: 12,
-    papers: 20,
+    summary: "用十八本账拆开显存、KV Cache、PagedAttention、连续批处理、P/D 解耦、量化、推测解码与集群调度，并重点追踪 DeepSeek V2→V4 与 Mooncake→K3。",
+    status: "published",
+    progress: 78,
+    papers: 62,
    prerequisites: ["07", "08", "09"],
-    highlights: ["vLLM", "Mooncake", "KDA 缓存"],
+    highlights: ["十八本服务账", "DeepSeek / Kimi 谱系", "四联实验"],
  },
  {
    number: "15",
@@ -12,6 +12,7 @@ export type PaperTopic =
  | "推理"
  | "Agent"
  | "多模态"
+  | "推理服务"
  | "评测";

 export interface Paper {
@@ -2891,6 +2892,368 @@ export const papers: Paper[] = [
    spotlight: "Kimi",
    verified: true,
  },
+  {
+    year: 2020,
+    title: "Clockwork: Predictable and Efficient Deep Learning Inference",
+    url: "https://www.usenix.org/conference/osdi20/presentation/gujarati",
+    topics: ["推理服务", "训练系统"],
+    contribution: "用可预测执行、集中式调度和显式模型缓存控制 DNN 服务尾延迟。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "ZeRO-Inference: Democratizing Massive Model Inference",
+    url: "https://arxiv.org/abs/2207.00032",
+    topics: ["推理服务", "训练系统", "低精度"],
+    contribution: "把巨量权重卸载到 CPU / NVMe，并通过分层预取和并行减少 GPU 容量门槛。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Orca: A Distributed Serving System for Transformer-Based Generative Models",
+    url: "https://www.usenix.org/conference/osdi22/presentation/yu",
+    topics: ["推理服务", "训练系统"],
+    contribution: "以 iteration-level scheduling 和 selective batching 奠定连续批处理主线。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Petals: Collaborative Inference and Fine-tuning of Large Models",
+    url: "https://arxiv.org/abs/2209.01188",
+    topics: ["推理服务", "训练系统"],
+    contribution: "让参与者跨互联网协作托管模型层，暴露异构、不可靠网络下的服务边界。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Fast Inference from Transformers via Speculative Decoding",
+    url: "https://arxiv.org/abs/2211.17192",
+    topics: ["推理服务", "推理"],
+    contribution: "用便宜 draft model 提议多个 Token，再由目标模型并行验收且保持目标分布。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Accelerating Large Language Model Decoding with Speculative Sampling",
+    url: "https://arxiv.org/abs/2302.01318",
+    topics: ["推理服务", "推理"],
+    contribution: "独立建立保分布的推测采样算法，减少昂贵模型的串行调用轮数。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "FlexGen: High-Throughput Generative Inference of Large Language Models with a Single GPU",
+    url: "https://arxiv.org/abs/2303.06865",
+    topics: ["推理服务", "训练系统", "低精度"],
+    contribution: "联合 GPU、CPU 与磁盘 offload，并以搜索策略优化吞吐导向的单卡大模型推理。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving",
+    url: "https://arxiv.org/abs/2302.11665",
+    topics: ["推理服务", "训练系统"],
+    contribution: "把统计复用、模型并行和模型放置结合，用于突发式大模型工作负载。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "FastServe: Fast Distributed Inference Serving for Large Language Models",
+    url: "https://arxiv.org/abs/2305.05920",
+    topics: ["推理服务", "训练系统"],
+    contribution: "借鉴多级反馈队列和抢占，降低自回归请求的 head-of-line blocking 与尾延迟。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "SpecInfer: Accelerating Generative Large Language Model Serving with Tree-based Speculative Inference and Verification",
+    url: "https://arxiv.org/abs/2305.09781",
+    topics: ["推理服务", "推理"],
+    contribution: "用树状候选和并行验证扩展推测解码，使一次目标调用覆盖多条草稿分支。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Efficient Memory Management for Large Language Model Serving with PagedAttention",
+    url: "https://arxiv.org/abs/2309.06180",
+    topics: ["推理服务", "长上下文", "训练系统"],
+    contribution: "vLLM 把 KV Cache 分页，支持非连续分配、共享与 copy-on-write。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "CacheGen: KV Cache Compression and Streaming for Fast Large Language Model Serving",
+    url: "https://arxiv.org/abs/2310.07240",
+    topics: ["推理服务", "长上下文", "低精度"],
+    contribution: "压缩并流式传输 KV Cache，以降低远端上下文加载延迟。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "SGLang: Efficient Execution of Structured Language Model Programs",
+    url: "https://arxiv.org/abs/2312.07104",
+    topics: ["推理服务", "Agent", "长上下文"],
+    contribution: "用 RadixAttention 自动复用共享前缀，并为结构化生成提供语言与运行时。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Splitwise: Efficient Generative LLM Inference Using Phase Splitting",
+    url: "https://arxiv.org/abs/2311.18677",
+    topics: ["推理服务", "训练系统"],
+    contribution: "将 prompt processing 与 token generation 拆到适配的硬件池并独立调度。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Prompt Cache: Modular Attention Reuse for Low-Latency Inference",
+    url: "https://arxiv.org/abs/2311.04934",
+    topics: ["推理服务", "长上下文"],
+    contribution: "用模块化 prompt schema 标注可复用片段并缓存中间注意力状态。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving",
+    url: "https://arxiv.org/abs/2401.09670",
+    topics: ["推理服务", "训练系统"],
+    contribution: "以满足 TTFT / TPOT SLO 的 goodput 为目标，分离并独立扩容 prefill 与 decode。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Medusa: Simple LLM Inference Acceleration Framework with Multiple Decoding Heads",
+    url: "https://arxiv.org/abs/2401.10774",
+    topics: ["推理服务", "推理"],
+    contribution: "在目标模型上增加多个 decoding heads，构造树状候选并减少独立 draft 成本。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "EAGLE: Speculative Sampling Requires Rethinking Feature Uncertainty",
+    url: "https://arxiv.org/abs/2401.15077",
+    topics: ["推理服务", "推理"],
+    contribution: "在 feature 层自回归预测并结合已知 Token，提升草稿准确率与推测解码速度。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "KVQuant: Towards 10 Million Context Length LLM Inference with KV Cache Quantization",
+    url: "https://arxiv.org/abs/2401.18079",
+    topics: ["推理服务", "低精度", "长上下文"],
+    contribution: "按通道量化 key、按 token 量化 value，并分离 outlier 以压缩长上下文 KV。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache",
+    url: "https://arxiv.org/abs/2402.02750",
+    topics: ["推理服务", "低精度", "长上下文"],
+    contribution: "利用 key / value 不同分布采用非对称 2-bit 量化，并保留近期高精度 residual。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Hydragen: High-Throughput LLM Inference with Shared Prefixes",
+    url: "https://arxiv.org/abs/2402.05099",
+    topics: ["推理服务", "长上下文"],
+    contribution: "将共享前缀与独有后缀注意力拆开，跨请求批量复用前缀计算。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "ChunkAttention: Efficient Self-Attention with Prefix-Aware KV Cache and Two-Phase Partition",
+    url: "https://arxiv.org/abs/2402.15220",
+    topics: ["推理服务", "长上下文"],
+    contribution: "以前缀树组织共享 KV chunks，并用两阶段分区提高内存与计算复用。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Sarathi-Serve: Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve",
+    url: "https://arxiv.org/abs/2403.02310",
+    topics: ["推理服务", "训练系统"],
+    contribution: "用 chunked prefill 与 stall-free scheduling 缓和长 prefill 对 decode 的干扰。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving",
+    url: "https://arxiv.org/abs/2405.04532",
+    topics: ["推理服务", "低精度"],
+    contribution: "联合 W4A8KV4、数据布局和硬件友好 kernel 优化端到端服务。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "CacheBlend: Fast Large Language Model Serving for RAG with Cached Knowledge Fusion",
+    url: "https://arxiv.org/abs/2405.16444",
+    topics: ["推理服务", "长上下文"],
+    contribution: "融合独立缓存的文本块，并选择性重算跨块依赖以接近完整 prefill。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Llumnix: Dynamic Scheduling for Large Language Model Serving",
+    url: "https://arxiv.org/abs/2406.03243",
+    topics: ["推理服务", "训练系统"],
+    contribution: "用跨实例实时迁移重平衡请求、处理碎片并支持主动容错。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "MemServe: Context Caching for Disaggregated LLM Serving with Elastic Memory Pool",
+    url: "https://arxiv.org/abs/2406.17565",
+    topics: ["推理服务", "长上下文", "训练系统"],
+    contribution: "把独立 KV 缓存池连接到弹性 prefill / decode 服务，形成存算分离架构。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Preble: Efficient Distributed Prompt Scheduling for LLM Serving",
+    url: "https://arxiv.org/abs/2407.00023",
+    topics: ["推理服务", "长上下文", "训练系统"],
+    contribution: "在分布式 prompt 服务中联合前缀复用、实例负载与路由决策。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "FlashAttention-3: Fast and Accurate Attention with Asynchrony and Low-precision",
+    url: "https://arxiv.org/abs/2407.08608",
+    topics: ["推理服务", "低精度", "长上下文"],
+    contribution: "面向 Hopper 用异步、warp specialization 与低精度流水进一步优化精确注意力。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "EAGLE-2: Faster Inference of Language Models with Dynamic Draft Trees",
+    url: "https://arxiv.org/abs/2406.16858",
+    topics: ["推理服务", "推理"],
+    contribution: "根据上下文置信度动态调整草稿树，而不是固定候选拓扑。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "FlashInfer: Efficient and Customizable Attention Engine for LLM Inference Serving",
+    url: "https://arxiv.org/abs/2501.01005",
+    topics: ["推理服务", "长上下文"],
+    contribution: "提供面向多种 KV 布局、batch 形状与生成阶段的可组合 attention / sampling kernel。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "EAGLE-3: Scaling up Inference Acceleration of Large Language Models via Training-Time Test",
+    url: "https://arxiv.org/abs/2503.01840",
+    topics: ["推理服务", "推理"],
+    contribution: "直接预测 Token，并融合低、中、高层特征构造更强的推测草稿。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "DeepGEMM",
+    url: "https://github.com/deepseek-ai/DeepGEMM",
+    topics: ["推理服务", "低精度", "训练系统"],
+    contribution: "DeepSeek 官方开源 FP8 与低精度 GEMM kernel 库，覆盖 dense 与 MoE 形状。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "FlashMLA",
+    url: "https://github.com/deepseek-ai/FlashMLA",
+    topics: ["推理服务", "长上下文", "训练系统"],
+    contribution: "DeepSeek 官方面向 Multi-head Latent Attention prefill / decode 的高效 kernel。",
+    spotlight: "DeepSeek",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "REST: Retrieval-Based Speculative Decoding",
+    url: "https://arxiv.org/abs/2311.08252",
+    topics: ["推理服务", "推理"],
+    contribution: "从历史语料检索连续 Token 构造免训练草稿，再由目标模型并行验证。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Lookahead Decoding: Parallel Decoding without a Draft Model or Data Store",
+    url: "https://arxiv.org/abs/2402.02057",
+    topics: ["推理服务", "推理"],
+    contribution: "用 Jacobi 迭代并行发现并验证 n-gram，无需独立草稿模型。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "LayerSkip: Enabling Early Exit Inference and Self-Speculative Decoding",
+    url: "https://arxiv.org/abs/2404.16710",
+    topics: ["推理服务", "推理"],
+    contribution: "训练早退层作为同一模型的草稿，再由剩余层验证，形成自推测解码。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models",
+    url: "https://arxiv.org/abs/2306.14048",
+    topics: ["推理服务", "长上下文"],
+    contribution: "识别并保留 attention heavy hitters，在有限预算下淘汰其余 KV。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "StreamingLLM: Efficient Streaming Language Models with Attention Sinks",
+    url: "https://arxiv.org/abs/2309.17453",
+    topics: ["推理服务", "长上下文"],
+    contribution: "保留初始 attention sinks 与近期窗口，使有限缓存支持持续流式文本。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "SnapKV: LLM Knows What You are Looking for Before Generation",
+    url: "https://arxiv.org/abs/2404.14469",
+    topics: ["推理服务", "长上下文"],
+    contribution: "用 prompt 末端 observation window 选择各头重要位置，压缩长 prompt KV。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "PyramidKV: Dynamic KV Cache Compression based on Pyramidal Information Funneling",
+    url: "https://arxiv.org/abs/2406.02069",
+    topics: ["推理服务", "长上下文"],
+    contribution: "依据层间信息聚合现象，为不同层分配递减的 KV 容量预算。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "InfiniGen: Efficient Generative Inference of Large Language Models with Dynamic KV Cache Management",
+    url: "https://arxiv.org/abs/2406.19707",
+    topics: ["推理服务", "长上下文"],
+    contribution: "提前预测关键 KV，并从 CPU 内存选择性加载以减少 GPU 状态容量。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Quest: Query-Aware Sparsity for Efficient Long-Context LLM Inference",
+    url: "https://arxiv.org/abs/2406.10774",
+    topics: ["推理服务", "长上下文"],
+    contribution: "利用 page 级统计按 query 选择相关 KV pages，减少长上下文 decode 读取。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "BurstGPT: A Real-world Workload Dataset to Optimize LLM Serving Systems",
+    url: "https://arxiv.org/abs/2401.17644",
+    topics: ["推理服务", "训练系统", "评测"],
+    contribution: "公开真实服务 trace，揭示突发、长度和模型混合对容量规划的影响。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "TensorRT-LLM",
+    url: "https://github.com/NVIDIA/TensorRT-LLM",
+    topics: ["推理服务", "低精度", "训练系统"],
+    contribution: "NVIDIA 官方 LLM 推理运行时，整合低精度 kernel、并行、KV Cache 与动态批处理。",
+    verified: true,
+  },
 ];

 export const paperTopics: PaperTopic[] = [
@@ -2907,5 +3270,6 @@ export const paperTopics: PaperTopic[] = [
  "推理",
  "Agent",
  "多模态",
+  "推理服务",
  "评测",
 ];