feat: publish scaling laws chapter

2026-07-29 02:17:31 +08:00
parent 4c4b5b0c7c
commit eff0e4c942
17 changed files with 4228 additions and 33 deletions
@@ -73,12 +73,12 @@ export const chapters: Chapter[] = [
    title: "Scaling Laws：规模为什么有效",
    kicker: "SCALING",
    question: "多大模型、多少数据、多少计算才是划算的？",
-    summary: "区分参数量、激活参数、训练计算和推理预算；理解 Kaplan、Chinchilla 与 K3 的双轴扩展。",
-    status: "researching",
-    progress: 22,
-    papers: 11,
+    summary: "用九张账区分参数、数据、计算、配方、外推与部署经济；理解 Kaplan、Chinchilla、DeepSeek 与 K3。",
+    status: "published",
+    progress: 74,
+    papers: 29,
    prerequisites: ["01", "02"],
-    highlights: ["幂律", "计算最优", "测试时扩展"],
+    highlights: ["幂律曲面", "计算最优", "四联实验"],
  },
  {
    number: "05",
@@ -574,6 +574,134 @@ export const papers: Paper[] = [
    contribution: "开放 3T Token 语料与完整数据处理工具。",
    verified: true,
  },
+  {
+    year: 2017,
+    title: "Deep Learning Scaling is Predictable, Empirically",
+    url: "https://arxiv.org/abs/1712.00409",
+    topics: ["Scaling"],
+    contribution: "跨语言、视觉、语音与翻译任务观察数据规模和误差之间的可预测幂律。",
+    verified: true,
+  },
+  {
+    year: 2018,
+    title: "An Empirical Model of Large-Batch Training",
+    url: "https://arxiv.org/abs/1812.06162",
+    topics: ["Scaling", "训练系统"],
+    contribution: "以 gradient noise scale 连接关键 batch size、优化步数和训练计算效率。",
+    verified: true,
+  },
+  {
+    year: 2020,
+    title: "Scaling Laws for Autoregressive Generative Modeling",
+    url: "https://arxiv.org/abs/2010.14701",
+    topics: ["Scaling", "多模态"],
+    contribution: "把自回归幂律扩展到图像、视频、数学和语言，显示指数依赖数据模态。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "Scaling Language Models: Methods, Analysis & Insights from Training Gopher",
+    url: "https://arxiv.org/abs/2112.11446",
+    topics: ["Scaling", "评测"],
+    contribution: "以 280B Gopher 系列研究规模、任务表现、训练实践与能力风险的分化。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Broken Neural Scaling Laws",
+    url: "https://arxiv.org/abs/2210.14891",
+    topics: ["Scaling"],
+    contribution: "用平滑折断幂律刻画平台、拐点、double descent 与斜率改变。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Are Emergent Abilities of Large Language Models a Mirage?",
+    url: "https://arxiv.org/abs/2304.15004",
+    topics: ["Scaling", "评测"],
+    contribution: "展示不连续指标可把平滑的底层改进显示成突然涌现，要求同时检查评分函数。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Scaling Data-Constrained Language Models",
+    url: "https://arxiv.org/abs/2305.16264",
+    topics: ["Scaling"],
+    contribution: "区分唯一数据量与重复看到的 Token，系统研究数据受限时的多轮训练。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws",
+    url: "https://arxiv.org/abs/2401.00448",
+    topics: ["Scaling", "训练系统"],
+    contribution: "把推理需求加入训练目标，说明生命周期最优模型可能小于纯训练计算最优模型。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Language Models Scale Reliably With Over-Training and on Downstream Tasks",
+    url: "https://arxiv.org/abs/2403.08540",
+    topics: ["Scaling", "评测"],
+    contribution: "检验超过 Chinchilla 配比后的 loss 与下游趋势，并给出多项可预测缩放结果。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Chinchilla Scaling: A Replication Attempt",
+    url: "https://arxiv.org/abs/2404.10102",
+    topics: ["Scaling"],
+    contribution: "复查原论文三种估计方法，质疑部分精确参数与置信区间但未推翻近似等比分配结论。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "More Compute Is What You Need",
+    url: "https://arxiv.org/abs/2404.19484",
+    topics: ["Scaling"],
+    contribution: "提出训练计算分配会改变可迁移性的争议性假说；在课程中仅作为待检验观点。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Resolving Discrepancies in Compute-Optimal Scaling of Language Models",
+    url: "https://arxiv.org/abs/2406.19146",
+    topics: ["Scaling"],
+    contribution: "从 FLOPs 口径、warmup 与随规模调参解释 Kaplan—Chinchilla 配比差异。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Optimization Hyper-parameter Laws for Large Language Models",
+    url: "https://arxiv.org/abs/2409.04777",
+    topics: ["Scaling", "训练系统"],
+    contribution: "研究最优学习率等优化超参数怎样随模型规模和训练长度变化。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Establishing Task Scaling Laws via Compute-Efficient Model Ladders",
+    url: "https://arxiv.org/abs/2412.04403",
+    topics: ["Scaling", "评测"],
+    contribution: "先由模型/数据预测任务 loss，再由任务 loss 预测能力，以小模型梯队降低外推成本。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "Overtrained Language Models Are Harder to Fine-Tune",
+    url: "https://arxiv.org/abs/2503.19206",
+    topics: ["Scaling", "后训练"],
+    contribution: "显示更长预训练可能降低后续微调可塑性，分开 base loss 与最终产品目标。",
+    verified: true,
+  },
+  {
+    year: 2026,
+    title: "Data-Constrained Language Model Pretraining: Improved Regularization and Scaling Laws",
+    url: "https://arxiv.org/abs/2606.06888",
+    topics: ["Scaling"],
+    contribution: "以 SoftQ 和 masked-input regularization 更新有限唯一数据下的重复训练研究。",
+    verified: true,
+  },
  {
    year: 2014,
    title: "Adam: A Method for Stochastic Optimization",