feat: publish reasoning deep dive

2026-07-29 00:40:09 +08:00
parent 3db826dd4a
commit dce94b283f
18 changed files with 3449 additions and 33 deletions
@@ -165,9 +165,9 @@ export const chapters: Chapter[] = [
    kicker: "REASONING",
    question: "模型如何学会多想一会儿，并检查自己的答案？",
    summary: "从 CoT、搜索与验证器，到 GRPO、DeepSeek-R1、Kimi k1.5 和 multi-effort RL。",
-    status: "researching",
-    progress: 25,
-    papers: 21,
+    status: "published",
+    progress: 76,
+    papers: 30,
    prerequisites: ["10"],
    highlights: ["GRPO", "R1-Zero", "On-policy 蒸馏"],
  },
@@ -760,6 +760,30 @@ export const papers: Paper[] = [
    contribution: "用中间推理示例显著提升大模型复杂任务表现。",
    verified: true,
  },
+  {
+    year: 2022,
+    title: "Large Language Models are Zero-Shot Reasoners",
+    url: "https://arxiv.org/abs/2205.11916",
+    topics: ["推理"],
+    contribution: "用统一的 step-by-step 触发语句，在不提供 few-shot rationale 时激发多任务零样本 CoT。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Solving Quantitative Reasoning Problems with Language Models",
+    url: "https://arxiv.org/abs/2206.14858",
+    topics: ["推理", "Scaling"],
+    contribution: "Minerva 以技术内容继续训练语言模型，系统推进数学与科学定量推理。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "Training Verifiers to Solve Math Word Problems",
+    url: "https://arxiv.org/abs/2110.14168",
+    topics: ["推理", "后训练"],
+    contribution: "建立 GSM8K，并系统展示多采样后由学习式 verifier 选择答案的收益。",
+    verified: true,
+  },
  {
    year: 2022,
    title: "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
@@ -768,6 +792,14 @@ export const papers: Paper[] = [
    contribution: "采样多条推理路径并对最终答案聚合。",
    verified: true,
  },
+  {
+    year: 2022,
+    title: "Least-to-Most Prompting Enables Complex Reasoning in Large Language Models",
+    url: "https://arxiv.org/abs/2205.10625",
+    topics: ["推理"],
+    contribution: "先把难题分解成子问题，再按顺序利用已解结果组合答案。",
+    verified: true,
+  },
  {
    year: 2022,
    title: "STaR: Bootstrapping Reasoning With Reasoning",
@@ -776,6 +808,30 @@ export const papers: Paper[] = [
    contribution: "迭代生成、筛选并训练成功 rationale。",
    verified: true,
  },
+  {
+    year: 2022,
+    title: "Solving Math Word Problems With Process- and Outcome-Based Feedback",
+    url: "https://arxiv.org/abs/2211.14275",
+    topics: ["推理", "后训练"],
+    contribution: "在 GSM8K 中比较过程与结果反馈，揭示最终正确率和推理轨迹错误率的不同需求。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Program of Thoughts Prompting: Disentangling Computation from Reasoning for Numerical Reasoning Tasks",
+    url: "https://arxiv.org/abs/2211.12588",
+    topics: ["推理", "Agent"],
+    contribution: "让模型用程序表达推理，把数值计算交给外部执行器。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Self-Refine: Iterative Refinement with Self-Feedback",
+    url: "https://arxiv.org/abs/2303.17651",
+    topics: ["推理"],
+    contribution: "由同一模型循环生成、反馈与修订，在不追加训练的情况下扩展串行测试时计算。",
+    verified: true,
+  },
  {
    year: 2023,
    title: "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
@@ -784,12 +840,20 @@ export const papers: Paper[] = [
    contribution: "显式搜索多个 thought 分支并评估中间状态。",
    verified: true,
  },
+  {
+    year: 2023,
+    title: "Language Models Don't Always Say What They Think: Unfaithful Explanations in Chain-of-Thought Prompting",
+    url: "https://arxiv.org/abs/2305.04388",
+    topics: ["推理", "评测"],
+    contribution: "用偏置提示实验表明，流畅的公开 CoT 可能合理化答案而不忠实披露影响因素。",
+    verified: true,
+  },
  {
    year: 2023,
    title: "Let's Verify Step by Step",
    url: "https://arxiv.org/abs/2305.20050",
    topics: ["推理", "后训练"],
-    contribution: "过程奖励模型在数学推理中优于只看最终答案。",
+    contribution: "发布 PRM800K；在 500 题 MATH 子集的 best-of-1860 选择中验证过程监督优势。",
    verified: true,
  },
  {
@@ -809,12 +873,36 @@ export const papers: Paper[] = [
    spotlight: "DeepSeek",
    verified: true,
  },
+  {
+    year: 2024,
+    title: "Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters",
+    url: "https://arxiv.org/abs/2408.03314",
+    topics: ["推理", "Scaling"],
+    contribution: "按模型与题目难度在修订、并行采样和 PRM 搜索之间分配测试时计算。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Tülu 3: Pushing Frontiers in Open Language Model Post-Training",
+    url: "https://arxiv.org/abs/2411.15124",
+    topics: ["后训练", "推理"],
+    contribution: "开放从数据策展、SFT、偏好学习到 RLVR 的完整 post-training 配方。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "OpenAI o1 System Card",
+    url: "https://arxiv.org/abs/2412.16720",
+    topics: ["推理", "评测"],
+    contribution: "记录 reasoning model 的能力、安全评测与测试时推理边界。",
+    verified: true,
+  },
  {
    year: 2025,
    title: "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
    url: "https://arxiv.org/abs/2501.12948",
    topics: ["推理", "后训练"],
-    contribution: "R1-Zero 纯 RL 涌现推理；R1 加冷启动、多阶段训练与蒸馏。",
+    contribution: "R1-Zero 从 base 直接做规则奖励 RL；R1 再加入冷启动、SFT、多阶段 RL 与蒸馏。",
    spotlight: "DeepSeek",
    verified: true,
  },
@@ -823,10 +911,50 @@ export const papers: Paper[] = [
    title: "Kimi k1.5: Scaling Reinforcement Learning with LLMs",
    url: "https://arxiv.org/abs/2501.12599",
    topics: ["推理", "后训练"],
-    contribution: "扩展长 CoT 强化学习和测试时计算。",
+    contribution: "用 128K RL context、partial rollout 与 long2short 扩展长 CoT 和测试时计算。",
    spotlight: "Kimi",
    verified: true,
  },
+  {
+    year: 2025,
+    title: "s1: Simple test-time scaling",
+    url: "https://arxiv.org/abs/2501.19393",
+    topics: ["推理", "后训练"],
+    contribution: "从强教师精选 1K 道推理题蒸馏，并用 budget forcing 控制思考长度。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "DAPO: An Open-Source LLM Reinforcement Learning System at Scale",
+    url: "https://arxiv.org/abs/2503.14476",
+    topics: ["推理", "后训练"],
+    contribution: "用 Clip-Higher、Dynamic Sampling、token-level loss 与 overlong shaping 稳定长 CoT RL。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "Understanding R1-Zero-Like Training: A Critical Perspective",
+    url: "https://arxiv.org/abs/2503.20783",
+    topics: ["推理", "后训练"],
+    contribution: "提出 Dr.GRPO，分析 response-length 与 question-difficulty 两种优化偏置。",
+    verified: true,
+  },
+  {
+    year: 2025,
+    title: "Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?",
+    url: "https://arxiv.org/abs/2504.13837",
+    topics: ["推理", "后训练", "评测"],
+    contribution: "用 pass@k 检查当前 RLVR 是扩展解法覆盖，还是主要重排已有正确路径。",
+    verified: true,
+  },
+  {
+    year: 2026,
+    title: "MOPD: Multi-Teacher On-Policy Distillation for Capability Integration in LLM Post-Training",
+    url: "https://arxiv.org/abs/2606.30406",
+    topics: ["推理", "后训练", "Agent"],
+    contribution: "让学生在自己的 rollout 上接收同源领域教师的稠密逐 Token 信号，整合多项 RL 能力。",
+    verified: true,
+  },
  {
    year: 2021,
    title: "WebGPT: Browser-assisted Question-Answering with Human Feedback",
@@ -927,8 +1055,8 @@ export const papers: Paper[] = [
    year: 2025,
    title: "Kimi K2: Open Agentic Intelligence",
    url: "https://arxiv.org/abs/2507.20534",
-    topics: ["MoE", "Agent", "后训练"],
-    contribution: "开放 1T MoE Agent 模型，强调工具调用数据合成与 joint RL。",
+    topics: ["MoE", "Agent", "后训练", "推理"],
+    contribution: "开放 1T MoE Agent 模型，以 verifiable gym 与 self-critique rubric 做 joint RL。",
    spotlight: "Kimi",
    verified: true,
  },