feat: publish scaling laws chapter
This commit is contained in:
wuyang
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,7 @@ const items = [
|
||||
{ id: "k3", href: "/k3/", label: "K3 解剖" },
|
||||
{ id: "deepseek", href: "/deepseek/", label: "DeepSeek" },
|
||||
{ id: "foundations", href: "/foundations/", label: "基础原理" },
|
||||
{ id: "scaling", href: "/scaling/", label: "Scaling" },
|
||||
{ id: "moe", href: "/moe/", label: "MoE" },
|
||||
{ id: "long-context", href: "/long-context/", label: "长上下文" },
|
||||
{ id: "reasoning", href: "/reasoning/", label: "推理" },
|
||||
|
||||
@@ -73,12 +73,12 @@ export const chapters: Chapter[] = [
|
||||
title: "Scaling Laws：规模为什么有效",
|
||||
kicker: "SCALING",
|
||||
question: "多大模型、多少数据、多少计算才是划算的？",
|
||||
summary: "区分参数量、激活参数、训练计算和推理预算；理解 Kaplan、Chinchilla 与 K3 的双轴扩展。",
|
||||
status: "researching",
|
||||
progress: 22,
|
||||
papers: 11,
|
||||
summary: "用九张账区分参数、数据、计算、配方、外推与部署经济；理解 Kaplan、Chinchilla、DeepSeek 与 K3。",
|
||||
status: "published",
|
||||
progress: 74,
|
||||
papers: 29,
|
||||
prerequisites: ["01", "02"],
|
||||
highlights: ["幂律", "计算最优", "测试时扩展"],
|
||||
highlights: ["幂律曲面", "计算最优", "四联实验"],
|
||||
},
|
||||
{
|
||||
number: "05",
|
||||
|
||||
@@ -574,6 +574,134 @@ export const papers: Paper[] = [
|
||||
contribution: "开放 3T Token 语料与完整数据处理工具。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2017,
|
||||
title: "Deep Learning Scaling is Predictable, Empirically",
|
||||
url: "https://arxiv.org/abs/1712.00409",
|
||||
topics: ["Scaling"],
|
||||
contribution: "跨语言、视觉、语音与翻译任务观察数据规模和误差之间的可预测幂律。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2018,
|
||||
title: "An Empirical Model of Large-Batch Training",
|
||||
url: "https://arxiv.org/abs/1812.06162",
|
||||
topics: ["Scaling", "训练系统"],
|
||||
contribution: "以 gradient noise scale 连接关键 batch size、优化步数和训练计算效率。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2020,
|
||||
title: "Scaling Laws for Autoregressive Generative Modeling",
|
||||
url: "https://arxiv.org/abs/2010.14701",
|
||||
topics: ["Scaling", "多模态"],
|
||||
contribution: "把自回归幂律扩展到图像、视频、数学和语言，显示指数依赖数据模态。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2021,
|
||||
title: "Scaling Language Models: Methods, Analysis & Insights from Training Gopher",
|
||||
url: "https://arxiv.org/abs/2112.11446",
|
||||
topics: ["Scaling", "评测"],
|
||||
contribution: "以 280B Gopher 系列研究规模、任务表现、训练实践与能力风险的分化。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2022,
|
||||
title: "Broken Neural Scaling Laws",
|
||||
url: "https://arxiv.org/abs/2210.14891",
|
||||
topics: ["Scaling"],
|
||||
contribution: "用平滑折断幂律刻画平台、拐点、double descent 与斜率改变。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2023,
|
||||
title: "Are Emergent Abilities of Large Language Models a Mirage?",
|
||||
url: "https://arxiv.org/abs/2304.15004",
|
||||
topics: ["Scaling", "评测"],
|
||||
contribution: "展示不连续指标可把平滑的底层改进显示成突然涌现，要求同时检查评分函数。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2023,
|
||||
title: "Scaling Data-Constrained Language Models",
|
||||
url: "https://arxiv.org/abs/2305.16264",
|
||||
topics: ["Scaling"],
|
||||
contribution: "区分唯一数据量与重复看到的 Token，系统研究数据受限时的多轮训练。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws",
|
||||
url: "https://arxiv.org/abs/2401.00448",
|
||||
topics: ["Scaling", "训练系统"],
|
||||
contribution: "把推理需求加入训练目标，说明生命周期最优模型可能小于纯训练计算最优模型。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Language Models Scale Reliably With Over-Training and on Downstream Tasks",
|
||||
url: "https://arxiv.org/abs/2403.08540",
|
||||
topics: ["Scaling", "评测"],
|
||||
contribution: "检验超过 Chinchilla 配比后的 loss 与下游趋势，并给出多项可预测缩放结果。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Chinchilla Scaling: A Replication Attempt",
|
||||
url: "https://arxiv.org/abs/2404.10102",
|
||||
topics: ["Scaling"],
|
||||
contribution: "复查原论文三种估计方法，质疑部分精确参数与置信区间但未推翻近似等比分配结论。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "More Compute Is What You Need",
|
||||
url: "https://arxiv.org/abs/2404.19484",
|
||||
topics: ["Scaling"],
|
||||
contribution: "提出训练计算分配会改变可迁移性的争议性假说；在课程中仅作为待检验观点。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Resolving Discrepancies in Compute-Optimal Scaling of Language Models",
|
||||
url: "https://arxiv.org/abs/2406.19146",
|
||||
topics: ["Scaling"],
|
||||
contribution: "从 FLOPs 口径、warmup 与随规模调参解释 Kaplan—Chinchilla 配比差异。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Optimization Hyper-parameter Laws for Large Language Models",
|
||||
url: "https://arxiv.org/abs/2409.04777",
|
||||
topics: ["Scaling", "训练系统"],
|
||||
contribution: "研究最优学习率等优化超参数怎样随模型规模和训练长度变化。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2024,
|
||||
title: "Establishing Task Scaling Laws via Compute-Efficient Model Ladders",
|
||||
url: "https://arxiv.org/abs/2412.04403",
|
||||
topics: ["Scaling", "评测"],
|
||||
contribution: "先由模型/数据预测任务 loss，再由任务 loss 预测能力，以小模型梯队降低外推成本。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2025,
|
||||
title: "Overtrained Language Models Are Harder to Fine-Tune",
|
||||
url: "https://arxiv.org/abs/2503.19206",
|
||||
topics: ["Scaling", "后训练"],
|
||||
contribution: "显示更长预训练可能降低后续微调可塑性，分开 base loss 与最终产品目标。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2026,
|
||||
title: "Data-Constrained Language Model Pretraining: Improved Regularization and Scaling Laws",
|
||||
url: "https://arxiv.org/abs/2606.06888",
|
||||
topics: ["Scaling"],
|
||||
contribution: "以 SoftQ 和 masked-input regularization 更新有限唯一数据下的重复训练研究。",
|
||||
verified: true,
|
||||
},
|
||||
{
|
||||
year: 2014,
|
||||
title: "Adam: A Method for Stochastic Optimization",
|
||||
|
||||
+28
-1
@@ -7,6 +7,7 @@ import { chapters, statusLabel } from "@/data/chapters";
|
||||
const routes: Record<string, string> = {
|
||||
roadmap: "/roadmap/",
|
||||
foundations: "/foundations/",
|
||||
scaling: "/scaling/",
|
||||
moe: "/moe/",
|
||||
"long-context": "/long-context/",
|
||||
reasoning: "/reasoning/",
|
||||
@@ -78,6 +79,7 @@ const paths = [
|
||||
<a class="button primary" href="/roadmap/">选择学习路径 <span aria-hidden="true">↓</span></a>
|
||||
<a class="button" href="/k3/">直接解剖 K3</a>
|
||||
<a class="button" href="/deepseek/">DeepSeek 专题</a>
|
||||
<a class="button" href="/scaling/">Scaling Laws 专题</a>
|
||||
<a class="button" href="/moe/">MoE 专题</a>
|
||||
<a class="button" href="/long-context/">长上下文专题</a>
|
||||
<a class="button" href="/reasoning/">推理专题</a>
|
||||
@@ -91,7 +93,7 @@ const paths = [
|
||||
<div class="hero-stats">
|
||||
<div><b>16</b><span>核心专题</span></div>
|
||||
<div><b>151</b><span>K3 报告来源</span></div>
|
||||
<div><b>166</b><span>关键论文索引</span></div>
|
||||
<div><b>182</b><span>关键论文索引</span></div>
|
||||
<div><b>47p</b><span>K3 技术报告</span></div>
|
||||
</div>
|
||||
</aside>
|
||||
@@ -105,6 +107,22 @@ const paths = [
|
||||
|
||||
<section class="section compact release-section" id="new-chapters">
|
||||
<div class="release-grid">
|
||||
<a class="release-card scaling-release" href="/scaling/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / CHAPTER 04</span> SCALING LAWS</p>
|
||||
<h2>“大一点就会更强”，远远不够指导一次训练</h2>
|
||||
<p>
|
||||
用观测量、模型、数据、算术、配比、配方、外推、经济目标与能力阶段九张账，
|
||||
从早期经验幂律、Kaplan、Chinchilla 与复现争议，一路走到 DeepSeek scaling study 和 Kimi K3。
|
||||
</p>
|
||||
</div>
|
||||
<dl>
|
||||
<div><dt>LINEAGE</dt><dd>2017 → 2026</dd></div>
|
||||
<div><dt>PAPERS</dt><dd>29 个一手节点</dd></div>
|
||||
<div><dt>LAB</dt><dd>曲面 · 部署 · 重复 · 涌现</dd></div>
|
||||
</dl>
|
||||
<span class="release-arrow" aria-hidden="true">进入 Scaling Laws 专题 →</span>
|
||||
</a>
|
||||
<a class="release-card training-release" href="/training-systems/">
|
||||
<div>
|
||||
<p class="eyebrow"><span>NEW / CHAPTER 08</span> LARGE-SCALE TRAINING SYSTEMS</p>
|
||||
@@ -402,12 +420,20 @@ const paths = [
|
||||
transition: transform 180ms ease, border-color 180ms ease;
|
||||
}
|
||||
|
||||
.scaling-release,
|
||||
.training-release,
|
||||
.reasoning-release {
|
||||
grid-column: 1 / -1;
|
||||
min-height: 510px;
|
||||
}
|
||||
|
||||
.scaling-release {
|
||||
background:
|
||||
radial-gradient(circle at 82% 18%, rgba(159, 91, 52, 0.18), transparent 30%),
|
||||
repeating-linear-gradient(135deg, transparent 0 84px, rgba(159, 91, 52, 0.045) 84px 85px),
|
||||
var(--paper-raised);
|
||||
}
|
||||
|
||||
.training-release {
|
||||
background:
|
||||
radial-gradient(circle at 82% 18%, rgba(66, 111, 101, 0.19), transparent 30%),
|
||||
@@ -484,6 +510,7 @@ const paths = [
|
||||
padding-bottom: 76px;
|
||||
}
|
||||
|
||||
.scaling-release,
|
||||
.training-release,
|
||||
.reasoning-release {
|
||||
grid-column: auto;
|
||||
|
||||
@@ -683,7 +683,7 @@ const paperChain = [
|
||||
<div class="next-links">
|
||||
<a class="button primary" href="/k3/#moe">回到 K3：在整机架构中定位 Stable LatentMoE →</a>
|
||||
<a class="button" href="/deepseek/">DeepSeek 完整论文谱系</a>
|
||||
<a class="button" href="/papers/">搜索全部 166 篇论文</a>
|
||||
<a class="button" href="/papers/">搜索全部 182 篇论文</a>
|
||||
</div>
|
||||
</section>
|
||||
</article>
|
||||
|
||||
@@ -7,16 +7,17 @@ const published = chapters.filter((chapter) => chapter.status === "published").l
|
||||
const researching = chapters.filter((chapter) => ["researching", "drafting"].includes(chapter.status)).length;
|
||||
|
||||
const workstreams = [
|
||||
{ label: "研究框架与规范", value: 82, next: "给推理专题补逐篇图表/实验精读层级" },
|
||||
{ label: "研究框架与规范", value: 83, next: "给 Scaling 与推理专题补逐篇图表/实验精读层级" },
|
||||
{ label: "网站设计系统", value: 89, next: "打印样式与更多通用可视化组件" },
|
||||
{ label: "Kimi K3 深读", value: 64, next: "扩写 scaling / pre-training / infra 逐图笔记" },
|
||||
{ label: "Kimi K3 深读", value: 66, next: "扩写 pre-training / infra 逐图笔记" },
|
||||
{ label: "Transformer 基础", value: 52, next: "加入矩阵形状动画与手算练习" },
|
||||
{ label: "Scaling Laws", value: 74, next: "加入真实拟合复现、置信区间与更多模型族对照" },
|
||||
{ label: "DeepSeek 专题", value: 71, next: "补 R1 / DAPO 的逐图训练轨迹与复现对照" },
|
||||
{ label: "推理与测试时扩展", value: 76, next: "真实模型采样曲线、PRM 案例与逐篇图表精读" },
|
||||
{ label: "稀疏计算与 MoE", value: 74, next: "补充真实集群 traces 与专家特化案例" },
|
||||
{ label: "长上下文专题", value: 72, next: "加入更多论文逐图笔记与真实模型配置对比" },
|
||||
{ label: "大规模训练系统", value: 71, next: "补真实集群 traces、故障案例与精确 topology 配置" },
|
||||
{ label: "引用与事实检查", value: 54, next: "自动化外链复查与来源等级扩展" },
|
||||
{ label: "引用与事实检查", value: 57, next: "自动化外链复查与来源等级扩展" },
|
||||
{ label: "开源与部署", value: 100, next: "每轮保留不可变镜像、提交与回滚点" },
|
||||
];
|
||||
---
|
||||
@@ -40,7 +41,7 @@ const workstreams = [
|
||||
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
|
||||
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
|
||||
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 01:19 CST</dd></div>
|
||||
<div><dt>UPDATED</dt><dd>2026-07-29 02:08 CST</dd></div>
|
||||
<div><dt>MODE</dt><dd>持续迭代，不锁死版本</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
@@ -50,7 +51,7 @@ const workstreams = [
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<p class="eyebrow"><span>01</span> WORKSTREAMS</p>
|
||||
<h2>十一条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
|
||||
<h2>十二条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
|
||||
</div>
|
||||
<p class="section-lead">
|
||||
内容首版优先打通全局脉络；随后每轮迭代选择一个专题推进到论文/工程层，并做独立事实复核。
|
||||
@@ -87,13 +88,14 @@ const workstreams = [
|
||||
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
|
||||
<article><span>✓</span><h3>16 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
|
||||
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
|
||||
<article><span>✓</span><h3>十二个原创交互视图</h3><p>K3、注意力、DeepSeek、长上下文、MoE、推理三页签，以及训练系统四页签实验。</p></article>
|
||||
<article><span>✓</span><h3>七篇首版长文</h3><p>K3 导读、Transformer 基础、DeepSeek 谱系、长上下文、MoE、推理与训练系统专题。</p></article>
|
||||
<article><span>✓</span><h3>十六个原创交互视图</h3><p>K3、注意力、DeepSeek、长上下文、MoE、推理三页签，以及训练系统与 Scaling 各四页签实验。</p></article>
|
||||
<article><span>✓</span><h3>八篇首版长文</h3><p>K3 导读、Transformer 基础、DeepSeek 谱系、Scaling、长上下文、MoE、推理与训练系统专题。</p></article>
|
||||
<article><span>✓</span><h3>Scaling Laws 深度专题</h3><p>九张账、29 个一手节点、DeepSeek/Kimi 双谱系与曲面—部署—复用—涌现四联实验。</p></article>
|
||||
<article><span>✓</span><h3>长上下文深度专题</h3><p>五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。</p></article>
|
||||
<article><span>✓</span><h3>MoE 深度专题</h3><p>六张账、19 篇一手论文、DeepSeek/K3 主线与路由—容量—通信交互实验室。</p></article>
|
||||
<article><span>✓</span><h3>推理深度专题</h3><p>八张账、30 篇一手论文链、DeepSeek/Kimi 双主线与三页签互动实验室。</p></article>
|
||||
<article><span>✓</span><h3>训练系统深度专题</h3><p>九张账、37 个一手节点、DeepSeek/Kimi 双谱系与显存—网格—气泡—通信实验室。</p></article>
|
||||
<article><span>✓</span><h3>166 篇关键论文索引</h3><p>覆盖 12 个专题，支持全文搜索、标签筛选与 Kimi/DeepSeek 聚光主线。</p></article>
|
||||
<article><span>✓</span><h3>182 篇关键论文索引</h3><p>覆盖 12 个专题，支持全文搜索、标签筛选与 Kimi/DeepSeek 聚光主线。</p></article>
|
||||
<article><span>✓</span><h3>公开仓库与自托管发布</h3><p>源码公开到 git.k1412.top，网站由不可变镜像、Compose Manager 与 HTTPS 交付。</p></article>
|
||||
</div>
|
||||
</section>
|
||||
@@ -108,7 +110,8 @@ const workstreams = [
|
||||
</div>
|
||||
<div class="queue-table">
|
||||
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
|
||||
<div><span>P0</span><strong>大规模训练系统二轮</strong><p>真实集群 traces → 故障恢复 → 精确 topology / kernel 配置</p><em>案例库 + 实测边界</em></div>
|
||||
<div><span>P0</span><strong>Scaling Laws 二轮</strong><p>真实拟合复现 → 置信区间 → 更多模型族与下游任务外推</p><em>可复现实验 + 逐图笔记</em></div>
|
||||
<div><span>P1</span><strong>大规模训练系统二轮</strong><p>真实集群 traces → 故障恢复 → 精确 topology / kernel 配置</p><em>案例库 + 实测边界</em></div>
|
||||
<div><span>P1</span><strong>长上下文二轮深化</strong><p>真实模型配置 → 内核细节 → 长上下文评测与失败案例</p><em>配置比较器 + 逐图论文笔记</em></div>
|
||||
<div><span>P1</span><strong>MoE 二轮深化</strong><p>真实负载 traces → 专家特化可解释性 → 共享专家语义</p><em>案例库 + 集群证据</em></div>
|
||||
<div><span>P1</span><strong>推理二轮深化</strong><p>真实 pass@k 曲线 → PRM 失败案例 → 逐篇图表精读</p><em>案例库 + 真实 traces</em></div>
|
||||
@@ -153,6 +156,7 @@ const workstreams = [
|
||||
<div><time>2026-07-28</time><b>自托管交付</b><p>源码公开到 git.k1412.top，网站部署到 k1412 私有基础设施。</p></div>
|
||||
<div><time>2026-07-29</time><b>推理按八张账组织</b><p>把答案、覆盖、选择、过程、预算、优化、分布与系统证据分开核算。</p></div>
|
||||
<div><time>2026-07-29</time><b>训练系统按九张账组织</b><p>把模型状态、激活、并行、气泡、通信、专家、上下文、数值与可靠性分开核算。</p></div>
|
||||
<div><time>2026-07-29</time><b>Scaling Laws 按九张账组织</b><p>把观测量、模型、数据、算术、配比、配方、外推、经济目标与能力阶段分开核算。</p></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -699,7 +699,7 @@ const paperChain = [
|
||||
<h2>不要按模型榜单读，按“旧方法留下了哪堵墙”读</h2>
|
||||
<p>
|
||||
下面 30 篇构成本专题首版主干。课程中的机制、公式与数字优先回到这些一手来源；
|
||||
更完整的 166 篇跨专题索引位于论文库。
|
||||
更完整的 182 篇跨专题索引位于论文库。
|
||||
</p>
|
||||
<div class="paper-chain">
|
||||
{paperChain.map(([year, title, href, note], index) => (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -201,7 +201,7 @@
|
||||
.top-nav {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 34px;
|
||||
gap: clamp(12px, 1.55vw, 26px);
|
||||
}
|
||||
|
||||
.top-nav a {
|
||||
|
||||
Reference in New Issue
Block a user