feat: publish alignment deep dive

src/data/chapters.ts

@@ -151,12 +151,12 @@ export const chapters: Chapter[] = [
     title: "指令微调与人类偏好",
     kicker: "ALIGNMENT",
     question: "会续写的 base model，怎样变成愿意协作的助手？",
-    summary: "从 instruction tuning、SFT、RLHF/PPO 到 DPO 与 RLAIF，区分能力学习和行为塑形。",
-    status: "queued",
-    progress: 10,
-    papers: 18,
+    summary: "用十二张账拆开 instruction tuning、SFT、偏好数据、奖励模型、RLHF/PPO、RLAIF、DPO 与 RLVR，并追到 DeepSeek 和 K3。",
+    status: "published",
+    progress: 75,
+    papers: 44,
     prerequisites: ["01", "02"],
-    highlights: ["SFT", "奖励模型", "偏好优化"],
+    highlights: ["十二张对齐账", "四联实验", "DeepSeek / K3"],
   },
   {
     number: "11",

src/data/papers.ts

@@ -1651,6 +1651,182 @@ export const papers: Paper[] = [
     contribution: "把偏好优化改写为直接分类目标，省去在线 RL loop。",
     verified: true,
   },
+  {
+    year: 2017,
+    title: "Deep Reinforcement Learning from Human Preferences",
+    url: "https://proceedings.neurips.cc/paper/2017/hash/d5e2c0adad503c91f91df240d0cd4e49-Abstract.html",
+    topics: ["后训练"],
+    contribution: "把人类对轨迹片段的成对比较拟合成奖励预测器，再用预测奖励优化策略。",
+    verified: true,
+  },
+  {
+    year: 2019,
+    title: "Fine-Tuning Language Models from Human Preferences",
+    url: "https://arxiv.org/abs/1909.08593",
+    topics: ["后训练"],
+    contribution: "把偏好奖励、reference KL 与 PPO 接到生成式语言模型，是现代语言模型 RLHF 的直接前身。",
+    verified: true,
+  },
+  {
+    year: 2021,
+    title: "Natural Instructions: Benchmarking Generalization to New Tasks from Natural Language Instructions",
+    url: "https://arxiv.org/abs/2104.08773",
+    topics: ["后训练", "评测"],
+    contribution: "把任务定义写成声明式自然语言说明，系统检验模型对未见任务的 instruction generalization。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback",
+    url: "https://arxiv.org/abs/2204.05862",
+    topics: ["后训练"],
+    contribution: "把 helpfulness 与 harmlessness 分开收集偏好并研究在线迭代，揭示通用助手对齐的多目标性质。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks",
+    url: "https://arxiv.org/abs/2204.07705",
+    topics: ["后训练", "评测"],
+    contribution: "把自然语言任务定义扩展到 1,600 余个任务，检验跨任务和跨类型泛化。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Scaling Instruction-Finetuned Language Models",
+    url: "https://arxiv.org/abs/2210.11416",
+    topics: ["后训练", "Scaling"],
+    contribution: "联合扩展 Flan 的任务混合、模型规模与 chain-of-thought 数据，研究 instruction tuning 的规模效应。",
+    verified: true,
+  },
+  {
+    year: 2022,
+    title: "Scaling Laws for Reward Model Overoptimization",
+    url: "https://arxiv.org/abs/2210.10760",
+    topics: ["后训练", "Scaling", "评测"],
+    contribution: "在 proxy reward 与 gold reward 的受控设置中量化 Goodhart 曲线，展示继续追高代理奖励何时反而伤害真实目标。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "OpenAssistant Conversations: Democratizing Large Language Model Alignment",
+    url: "https://arxiv.org/abs/2304.07327",
+    topics: ["后训练", "数据"],
+    contribution: "开放多轮对话树、候选排序和标注流程，为可复查的助手后训练数据提供公共基线。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "RRHF: Rank Responses to Align Language Models with Human Feedback",
+    url: "https://arxiv.org/abs/2304.05302",
+    topics: ["后训练"],
+    contribution: "用回答序列得分的排序约束让模型学习多来源候选的相对质量。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "SLiC-HF: Sequence Likelihood Calibration with Human Feedback",
+    url: "https://arxiv.org/abs/2305.10425",
+    topics: ["后训练"],
+    contribution: "直接校准 chosen 与 rejected 回答的序列似然，为不运行在线 RL 的偏好训练提供早期路线。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "LIMA: Less Is More for Alignment",
+    url: "https://arxiv.org/abs/2305.11206",
+    topics: ["后训练", "数据"],
+    contribution: "用约一千条精选示范检验强基座上少量高质量 SFT 的接口塑形能力，并明确其适用边界。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "AlpacaFarm: A Simulation Framework for Methods that Learn from Human Feedback",
+    url: "https://arxiv.org/abs/2305.14387",
+    topics: ["后训练", "评测"],
+    contribution: "提供模拟偏好、自动评测与多种 RLHF 基线，降低比较后训练方法的实验成本。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "RLAIF: Scaling Reinforcement Learning from Human Feedback with AI Feedback",
+    url: "https://arxiv.org/abs/2309.00267",
+    topics: ["后训练", "评测"],
+    contribution: "在摘要、helpful 与 harmless 场景直接比较 AI 和人工偏好标签，而不是把 AI feedback 当作无价值来源。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "UltraFeedback: Boosting Language Models with Scaled AI Feedback",
+    url: "https://arxiv.org/abs/2310.01377",
+    topics: ["后训练", "数据"],
+    contribution: "规模化生成多候选、多维度 AI 反馈，成为开放偏好训练与直接优化的重要数据来源。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "A General Theoretical Paradigm to Understand Learning from Human Preferences",
+    url: "https://arxiv.org/abs/2310.12036",
+    topics: ["后训练"],
+    contribution: "分析确定性偏好下 DPO 的过拟合边界，并以有限间隔回归提出 IPO。",
+    verified: true,
+  },
+  {
+    year: 2023,
+    title: "Zephyr: Direct Distillation of LM Alignment",
+    url: "https://arxiv.org/abs/2310.16944",
+    topics: ["后训练"],
+    contribution: "以蒸馏式 SFT、UltraFeedback AI 偏好与直接偏好优化展示开放模型的完整对齐配方。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "KTO: Model Alignment as Prospect Theoretic Optimization",
+    url: "https://arxiv.org/abs/2402.01306",
+    topics: ["后训练"],
+    contribution: "用前景理论式效用处理 desirable / undesirable 二元反馈，放宽必须提供成对偏好的数据合同。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs",
+    url: "https://arxiv.org/abs/2402.14740",
+    topics: ["后训练"],
+    contribution: "以 leave-one-out baseline 重新建立 critic-free REINFORCE 基线，说明 PPO 不是在线 RLHF 的唯一实现。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "ORPO: Monolithic Preference Optimization without Reference Model",
+    url: "https://arxiv.org/abs/2403.07691",
+    topics: ["后训练"],
+    contribution: "把 SFT 负对数似然与 odds-ratio 偏好惩罚合成单阶段、无独立 reference model 的目标。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "RewardBench: Evaluating Reward Models for Language Modeling",
+    url: "https://arxiv.org/abs/2403.13787",
+    topics: ["后训练", "评测"],
+    contribution: "按 chat、hard、safety、reasoning 与分布外样本直接评估奖励模型的成对排序能力。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "SimPO: Simple Preference Optimization with a Reference-Free Reward",
+    url: "https://arxiv.org/abs/2405.14734",
+    topics: ["后训练"],
+    contribution: "用长度归一的平均 log-prob 与目标 margin 构造 reference-free implicit reward。",
+    verified: true,
+  },
+  {
+    year: 2024,
+    title: "Unpacking DPO and PPO: Disentangling Best Practices for Learning from Preference Feedback",
+    url: "https://proceedings.neurips.cc/paper_files/paper/2024/hash/404df2480b6eef0486a1679e371894b0-Abstract-Conference.html",
+    topics: ["后训练", "评测"],
+    contribution: "把 preference data、优化算法、奖励模型和 policy prompts 四个变量拆开，避免把配方差异误判为算法胜负。",
+    verified: true,
+  },
   {
     year: 2022,
     title: "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",