feat: publish numerics and optimization chapter

2026-07-29 04:02:56 +08:00
parent 2a74cba527
commit c21af182fb
19 changed files with 4495 additions and 34 deletions
@@ -16,6 +16,7 @@ const items = [
  { id: "long-context", href: "/long-context/", label: "长上下文" },
  { id: "reasoning", href: "/reasoning/", label: "推理" },
  { id: "training-systems", href: "/training-systems/", label: "训练系统" },
+  { id: "numerics", href: "/systems/numerics/", label: "数值" },
  { id: "papers", href: "/papers/", label: "论文库" },
  { id: "progress", href: "/progress/", label: "进度" },
 ];
@@ -138,12 +138,12 @@ export const chapters: Chapter[] = [
    title: "数值精度、优化器与稳定性",
    kicker: "OPTIMIZATION",
    question: "为什么少几个比特能省巨资，也可能让训练瞬间崩掉？",
-    summary: "解释 AdamW、μP、Muon、BF16/FP8/MXFP4 与量化感知训练的数值直觉。",
-    status: "queued",
-    progress: 13,
-    papers: 16,
+    summary: "用十张数值账解释 AdamW、μP、Muon、BF16/FP8/MXFP4、QAT 与训练失稳反馈环。",
+    status: "published",
+    progress: 75,
+    papers: 36,
    prerequisites: ["02", "08"],
-    highlights: ["FP8", "Muon", "MXFP4"],
+    highlights: ["FP8", "MuonClip", "MXFP4", "稳定性控制室"],
  },
  {
    number: "10",
@@ -6,6 +6,7 @@ export type PaperTopic =
  | "Scaling"
  | "数据"
  | "训练系统"
+  | "优化器"
  | "低精度"
  | "后训练"
  | "推理"
@@ -867,7 +868,7 @@ export const papers: Paper[] = [
    year: 2014,
    title: "Adam: A Method for Stochastic Optimization",
    url: "https://arxiv.org/abs/1412.6980",
-    topics: ["训练系统"],
+    topics: ["训练系统", "优化器"],
    contribution: "自适应一阶/二阶矩估计，成为 LLM 优化器基线。",
    verified: true,
  },
@@ -875,7 +876,7 @@ export const papers: Paper[] = [
    year: 2017,
    title: "Decoupled Weight Decay Regularization",
    url: "https://arxiv.org/abs/1711.05101",
-    topics: ["训练系统"],
+    topics: ["训练系统", "优化器"],
    contribution: "AdamW 将权重衰减与梯度更新正确解耦。",
    verified: true,
  },
@@ -1081,11 +1082,19 @@ export const papers: Paper[] = [
    spotlight: "Kimi",
    verified: true,
  },
+  {
+    year: 2024,
+    title: "Muon: An Optimizer for Hidden Layers in Neural Networks",
+    url: "https://kellerjordan.github.io/posts/muon/",
+    topics: ["优化器"],
+    contribution: "把 momentum matrix 经 Newton–Schulz 近似正交化，给出 Muon 的原始定义与参考实现。",
+    verified: true,
+  },
  {
    year: 2025,
    title: "Muon is Scalable for LLM Training",
    url: "https://arxiv.org/abs/2502.16982",
-    topics: ["训练系统"],
+    topics: ["训练系统", "优化器"],
    contribution: "补足 Muon 在 LLM 规模的权重衰减与更新尺度配方。",
    spotlight: "Kimi",
    verified: true,
@@ -1130,6 +1139,166 @@ export const papers: Paper[] = [
    contribution: "基于近似二阶信息逐层量化大模型权重。",
    verified: true,
  },
+  {
+    year: 2015,
+    title: "Deep Learning with Limited Numerical Precision",
+    url: "https://arxiv.org/abs/1502.02551",
+    topics: ["低精度"],
+    contribution: "系统研究定点低精度训练，并用 stochastic rounding 保住微小更新的长期积累。",
+    verified: true,
+  },
+  {
+    year: 2017,
+    title: "Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference",
+    url: "https://arxiv.org/abs/1712.05877",
+    topics: ["低精度"],
+    contribution: "建立 fake quantization 与量化感知训练的经典整数推理配方。",
+    verified: true,
+  },
+  {
+    year: 2018,
+    title: "Shampoo: Preconditioned Stochastic Tensor Optimization",
+    url: "https://arxiv.org/abs/1802.09568",
+    topics: ["优化器"],
+    contribution: "沿 tensor 各 mode 构造结构化预条件器，在逐坐标自适应之外利用矩阵几何。",
+    verified: true,
+  },
+  {
+    year: 2018,
+    title: "Adafactor: Adaptive Learning Rates with Sublinear Memory Cost",
+    url: "https://arxiv.org/abs/1804.04235",
+    topics: ["优化器", "训练系统"],
+    contribution: "用行列因子近似二阶矩，并以 update clipping 降低优化器状态与失稳风险。",
+    verified: true,
+  },
+  {
+    year: 2019,
+    title: "A Study of BFLOAT16 for Deep Learning Training",
+    url: "https://arxiv.org/abs/1905.12322",
+    topics: ["低精度", "训练系统"],
+    contribution: "解释 BF16 保留 FP32 exponent、压缩 mantissa 的范围—精度取舍。",
+    verified: true,
+  },
+  {
+    year: 2020,
+    title: "Query-Key Normalization for Transformers",
+    url: "https://arxiv.org/abs/2010.04245",
+    topics: ["Transformer", "优化器"],
+    contribution: "归一化每个 attention head 的 Q/K 并学习缩放，直接约束 Softmax logit 尺度。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "8-bit Optimizers via Block-wise Quantization",
+    url: "https://arxiv.org/abs/2110.02861",
+    topics: ["优化器", "低精度"],
+    contribution: "用 block-wise dynamic quantization 压缩 optimizer state，同时隔离离群值影响。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "DeepNet: Scaling Transformers to 1,000 Layers",
+    url: "https://arxiv.org/abs/2203.00555",
+    topics: ["Transformer", "优化器"],
+    contribution: "用 DeepNorm 的残差缩放与初始化界约束深层 Transformer 更新。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Tensor Programs V: Tuning Large Neural Networks via Zero-Shot Hyperparameter Transfer",
+    url: "https://arxiv.org/abs/2203.03466",
+    topics: ["Scaling", "优化器"],
+    contribution: "μP 让小代理模型调出的超参数可随宽度迁移到更大网络。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Scaling Vision Transformers to 22 Billion Parameters",
+    url: "https://arxiv.org/abs/2302.05442",
+    topics: ["Transformer", "优化器", "多模态"],
+    contribution: "以 QK LayerNorm 等改动缓解超大 ViT 的 attention logit 爆炸。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Stable and Low-precision Training for Large-scale Vision-language Models",
+    url: "https://arxiv.org/abs/2304.13013",
+    topics: ["低精度", "优化器", "多模态"],
+    contribution: "用 SwitchBack 与稳定性诊断研究低精度 VLM 训练中的 outlier 和 Adam second-moment spike。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "QLoRA: Efficient Finetuning of Quantized LLMs",
+    url: "https://arxiv.org/abs/2305.14314",
+    topics: ["低精度", "后训练"],
+    contribution: "以 NF4、double quantization 和 paged optimizers 微调冻结的 4-bit 基座。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "LLM-QAT: Data-Free Quantization Aware Training for Large Language Models",
+    url: "https://arxiv.org/abs/2305.17888",
+    topics: ["低精度", "后训练"],
+    contribution: "用自生成数据进行 weight、activation 与 KV cache 的量化感知训练。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Small-scale Proxies for Large-scale Transformer Training Instabilities",
+    url: "https://arxiv.org/abs/2309.14322",
+    topics: ["优化器", "训练系统"],
+    contribution: "用高学习率小模型代理复现大模型失稳，降低稳定性机制的研究成本。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "FP8-LM: Training FP8 Large Language Models",
+    url: "https://arxiv.org/abs/2310.18313",
+    topics: ["低精度", "训练系统"],
+    contribution: "把 FP8 扩展到计算、优化器与分布式通信的端到端 LLM 训练。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Spike No More: Stabilizing the Pre-training of Large Language Models",
+    url: "https://arxiv.org/abs/2312.16903",
+    topics: ["优化器", "Transformer"],
+    contribution: "从“小 sublayer、大 shortcut”原则解释并抑制预训练早期 loss spike。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Deconstructing What Makes a Good Optimizer for Language Models",
+    url: "https://arxiv.org/abs/2407.07972",
+    topics: ["优化器"],
+    contribution: "拆解 Adam、Adafactor、Lion 等方法的更新尺度、方向与超参边界。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Scaling FP8 Training to Trillion-Token LLMs",
+    url: "https://arxiv.org/abs/2409.12517",
+    topics: ["低精度", "训练系统"],
+    contribution: "研究万亿 Token 长程 FP8 训练的 SwiGLU outlier，并提出 Smooth-SwiGLU。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "FP4 All the Way: Fully Quantized Training of LLMs",
+    url: "https://arxiv.org/abs/2505.19115",
+    topics: ["低精度", "训练系统"],
+    contribution: "探索权重、激活与梯度广泛采用 FP4 的 fully quantized 训练边界。",
+    verified: true,
+  },
+  {
+    year: 2026,
+    title: "PowLU: Power-Law Units for Stable and Efficient LLM Training",
+    url: "https://arxiv.org/abs/2605.25704",
+    topics: ["优化器", "低精度", "Transformer"],
+    contribution: "用有界幂律激活控制极值，连接 activation 设计与低精度稳定性。",
+    verified: true,
+  },
  {
    year: 2023,
    title: "AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration",
@@ -1669,6 +1838,7 @@ export const paperTopics: PaperTopic[] = [
  "Scaling",
  "数据",
  "训练系统",
+  "优化器",
  "低精度",
  "后训练",
  "推理",
@@ -13,6 +13,7 @@ const routes: Record<string, string> = {
  "long-context": "/long-context/",
  reasoning: "/reasoning/",
  "training-systems": "/training-systems/",
+  "systems/numerics": "/systems/numerics/",
 };

 const paths = [
@@ -86,6 +87,7 @@ const paths = [
          <a class="button" href="/long-context/">长上下文专题</a>
          <a class="button" href="/reasoning/">推理专题</a>
          <a class="button" href="/training-systems/">训练系统专题</a>
+          <a class="button" href="/systems/numerics/">数值与优化专题</a>
        </div>
      </div>
      <aside class="hero-aside" aria-label="项目统计">
@@ -95,20 +97,36 @@ const paths = [
        <div class="hero-stats">
          <div><b>16</b><span>核心专题</span></div>
          <div><b>151</b><span>K3 报告来源</span></div>
-          <div><b>202</b><span>关键论文索引</span></div>
+          <div><b>223</b><span>关键论文索引</span></div>
          <div><b>47p</b><span>K3 技术报告</span></div>
        </div>
      </aside>
    </div>
    <div class="scope-strip">
      <span>研究范围</span>
-      <p>表示学习 · Transformer · Scaling · 数据工程 · MoE · 长上下文 · 训练系统 · 后训练 · 推理 · Agent · 多模态 · 服务与评测</p>
+      <p>表示学习 · Transformer · Scaling · 数据工程 · MoE · 长上下文 · 训练系统 · 数值优化 · 后训练 · 推理 · Agent · 多模态 · 服务与评测</p>
      <a href="/progress/">查看实时进度 →</a>
    </div>
  </section>

  <section class="section compact release-section" id="new-chapters">
    <div class="release-grid">
+      <a class="release-card numerics-release" href="/systems/numerics/">
+        <div>
+          <p class="eyebrow"><span>NEW / CHAPTER 09</span> PRECISION · OPTIMIZATION · STABILITY</p>
+          <h2>少几个比特为什么省巨资，也可能让训练瞬间崩掉？</h2>
+          <p>
+            用表示、缩放、计算、累加、搬运、更新、参数化、观测、恢复与证据十张账，
+            从 mixed precision、BF16、FP8 和 MXFP4，一路走到 AdamW、Muon、Kimi K2/K3 与 DeepSeek-V3/V4。
+          </p>
+        </div>
+        <dl>
+          <div><dt>LINEAGE</dt><dd>2014 → 2026</dd></div>
+          <div><dt>PAPERS</dt><dd>36 个一手节点</dd></div>
+          <div><dt>LAB</dt><dd>格式 · 状态 · 更新 · 失稳</dd></div>
+        </dl>
+        <span class="release-arrow" aria-hidden="true">进入数值与优化专题 →</span>
+      </a>
      <a class="release-card data-release" href="/pretraining/data/">
        <div>
          <p class="eyebrow"><span>NEW / CHAPTER 05</span> DATA & PRE-TRAINING</p>
@@ -438,6 +456,7 @@ const paths = [
      transition: transform 180ms ease, border-color 180ms ease;
    }

+    .numerics-release,
    .data-release,
    .scaling-release,
    .training-release,
@@ -446,6 +465,14 @@ const paths = [
      min-height: 510px;
    }

+    .numerics-release {
+      background:
+        radial-gradient(circle at 82% 18%, rgba(113, 82, 137, 0.2), transparent 30%),
+        repeating-linear-gradient(90deg, transparent 0 64px, rgba(113, 82, 137, 0.045) 64px 65px),
+        repeating-linear-gradient(0deg, transparent 0 64px, rgba(76, 118, 112, 0.04) 64px 65px),
+        var(--paper-raised);
+    }
+
    .data-release {
      background:
        radial-gradient(circle at 82% 18%, rgba(76, 118, 112, 0.2), transparent 30%),
@@ -536,6 +563,7 @@ const paths = [
        padding-bottom: 76px;
      }

+      .numerics-release,
      .data-release,
      .scaling-release,
      .training-release,
@@ -18,6 +18,7 @@ const workstreams = [
  { label: "稀疏计算与 MoE", value: 74, next: "补充真实集群 traces 与专家特化案例" },
  { label: "长上下文专题", value: 72, next: "加入更多论文逐图笔记与真实模型配置对比" },
  { label: "大规模训练系统", value: 71, next: "补真实集群 traces、故障案例与精确 topology 配置" },
+  { label: "数值精度、优化器与稳定性", value: 75, next: "加入真实 kernel 吞吐、长程训练 traces 与逐图论文精读" },
  { label: "引用与事实检查", value: 57, next: "自动化外链复查与来源等级扩展" },
  { label: "开源与部署", value: 100, next: "每轮保留不可变镜像、提交与回滚点" },
 ];
@@ -42,7 +43,7 @@ const workstreams = [
        <div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
        <div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
        <div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
-        <div><dt>UPDATED</dt><dd>2026-07-29 03:01 CST</dd></div>
+        <div><dt>UPDATED</dt><dd>2026-07-29 03:55 CST</dd></div>
        <div><dt>MODE</dt><dd>持续迭代，不锁死版本</dd></div>
      </dl>
    </div>
@@ -52,7 +53,7 @@ const workstreams = [
    <div class="section-heading">
      <div>
        <p class="eyebrow"><span>01</span> WORKSTREAMS</p>
-        <h2>十三条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
+        <h2>十四条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
      </div>
      <p class="section-lead">
        内容首版优先打通全局脉络；随后每轮迭代选择一个专题推进到论文/工程层，并做独立事实复核。
@@ -89,15 +90,16 @@ const workstreams = [
      <article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
      <article><span>✓</span><h3>16 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
      <article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
-      <article><span>✓</span><h3>二十个原创交互视图</h3><p>K3、注意力、DeepSeek、长上下文、MoE、推理，以及训练系统、Scaling 与数据工程的多页签实验。</p></article>
-      <article><span>✓</span><h3>九篇首版长文</h3><p>K3、Transformer、DeepSeek、Scaling、数据工程、长上下文、MoE、推理与训练系统专题。</p></article>
+      <article><span>✓</span><h3>二十四个原创交互视图</h3><p>K3、注意力、DeepSeek、长上下文、MoE、推理，以及训练系统、Scaling、数据工程和数值专题的多页签实验。</p></article>
+      <article><span>✓</span><h3>十篇首版长文</h3><p>K3、Transformer、DeepSeek、Scaling、数据工程、长上下文、MoE、推理、训练系统与数值优化专题。</p></article>
      <article><span>✓</span><h3>Scaling Laws 深度专题</h3><p>九张账、29 个一手节点、DeepSeek/Kimi 双谱系与曲面—部署—复用—涌现四联实验。</p></article>
      <article><span>✓</span><h3>数据工程深度专题</h3><p>十二张账、31 个一手节点、DeepSeek/Kimi 双谱系与流水线—去重—混合—改写四联实验。</p></article>
      <article><span>✓</span><h3>长上下文深度专题</h3><p>五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。</p></article>
      <article><span>✓</span><h3>MoE 深度专题</h3><p>六张账、19 篇一手论文、DeepSeek/K3 主线与路由—容量—通信交互实验室。</p></article>
      <article><span>✓</span><h3>推理深度专题</h3><p>八张账、30 篇一手论文链、DeepSeek/Kimi 双主线与三页签互动实验室。</p></article>
      <article><span>✓</span><h3>训练系统深度专题</h3><p>九张账、37 个一手节点、DeepSeek/Kimi 双谱系与显存—网格—气泡—通信实验室。</p></article>
-      <article><span>✓</span><h3>202 篇关键论文索引</h3><p>新增数据专题标签和 20 个一手节点，继续支持全文搜索与 Kimi/DeepSeek 聚光主线。</p></article>
+      <article><span>✓</span><h3>数值、优化器与稳定性深度专题</h3><p>十张账、36 个一手节点、K2/K3 与 DeepSeek-V3/V4 双谱系，以及格式—状态—更新—失稳四联实验。</p></article>
+      <article><span>✓</span><h3>223 篇关键论文索引</h3><p>新增「优化器」标签和 21 个一手节点，继续支持全文搜索与 Kimi/DeepSeek 聚光主线。</p></article>
      <article><span>✓</span><h3>公开仓库与自托管发布</h3><p>源码公开到 git.k1412.top，网站由不可变镜像、Compose Manager 与 HTTPS 交付。</p></article>
    </div>
  </section>
@@ -115,6 +117,7 @@ const workstreams = [
      <div><span>P0</span><strong>Scaling Laws 二轮</strong><p>真实拟合复现 → 置信区间 → 更多模型族与下游任务外推</p><em>可复现实验 + 逐图笔记</em></div>
      <div><span>P1</span><strong>数据工程二轮</strong><p>FineWeb / DCLM 逐图 → 真实去重误伤 → mixture traces 与污染案例</p><em>逐图笔记 + 案例库</em></div>
      <div><span>P1</span><strong>大规模训练系统二轮</strong><p>真实集群 traces → 故障恢复 → 精确 topology / kernel 配置</p><em>案例库 + 实测边界</em></div>
+      <div><span>P1</span><strong>数值与稳定性二轮</strong><p>真实 kernel 吞吐 → 长程失稳 traces → optimizer / quantization 逐图复现</p><em>实测边界 + 逐图笔记</em></div>
      <div><span>P1</span><strong>长上下文二轮深化</strong><p>真实模型配置 → 内核细节 → 长上下文评测与失败案例</p><em>配置比较器 + 逐图论文笔记</em></div>
      <div><span>P1</span><strong>MoE 二轮深化</strong><p>真实负载 traces → 专家特化可解释性 → 共享专家语义</p><em>案例库 + 集群证据</em></div>
      <div><span>P1</span><strong>推理二轮深化</strong><p>真实 pass@k 曲线 → PRM 失败案例 → 逐篇图表精读</p><em>案例库 + 真实 traces</em></div>
@@ -162,6 +165,8 @@ const workstreams = [
      <div><time>2026-07-29</time><b>Scaling Laws 按九张账组织</b><p>把观测量、模型、数据、算术、配比、配方、外推、经济目标与能力阶段分开核算。</p></div>
      <div><time>2026-07-29</time><b>数据工程按十二张账组织</b><p>把来源、解析、语言、质量、唯一性、污染、混合、变换、Tokenizer、Packing、课程与价值分开核算。</p></div>
      <div><time>2026-07-29</time><b>Grok 数据线索与正式账本永久分离</b><p>1264 行正式研究账本只接受回查一手来源后的结论；203 行 Grok 产物保留为未核验发现队列。</p></div>
+      <div><time>2026-07-29</time><b>数值专题按十张账组织</b><p>把格式、缩放、计算、累加、搬运、更新、参数化、观测、恢复与证据经济分开核算。</p></div>
+      <div><time>2026-07-29</time><b>低精度结论必须写完整角色合同</b><p>对象、格式、scale 粒度、accumulator、输出与硬件不再被压缩成一个 dtype 标签。</p></div>
    </div>
  </section>