wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: publish MoE deep dive

Browse Source
This commit is contained in:
wuyang
2026-07-28 23:32:14 +08:00
parent 761a75d5af
commit 3db826dd4a
17 changed files with 3533 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/MoERoutingLab.astro
+1298
View File
File diff suppressed because it is too large Load Diff
src/components/SiteFooter.astro
+1
View File
@@ -5,6 +5,7 @@
</div>
<div class="footer-links">
<a href="/roadmap/">学习地图</a>
<a href="/moe/">MoE 专题</a>
<a href="/long-context/">长上下文专题</a>
<a href="/progress/">研究进度</a>
<a href="https://git.k1412.top/wuyang/llm-atlas" rel="noreferrer">开放源码</a>
src/components/SiteHeader.astro
+1
View File
@@ -10,6 +10,7 @@ const items = [
{ id: "k3", href: "/k3/", label: "K3 解剖" },
{ id: "deepseek", href: "/deepseek/", label: "DeepSeek" },
{ id: "foundations", href: "/foundations/", label: "基础原理" },
{ id: "moe", href: "/moe/", label: "MoE" },
{ id: "long-context", href: "/long-context/", label: "长上下文" },
{ id: "papers", href: "/papers/", label: "论文库" },
{ id: "progress", href: "/progress/", label: "进度" },
src/data/chapters.ts
+4 -4
View File
@@ -95,14 +95,14 @@ export const chapters: Chapter[] = [
},
{
number: "06",
slug: "architecture/moe",
slug: "moe",
title: "稀疏计算与 MoE",
kicker: "SPARSE EXPERTS",
question: "怎样让模型装下更多知识，却不让每个 Token 都付全部算力？",
summary: "从条件计算到 DeepSeekMoE、LatentMoE 与 K3 Stable LatentMoE，解释路由、特化和负载均衡。",
status: "researching",
progress: 28,
papers: 17,
status: "published",
progress: 74,
papers: 19,
prerequisites: ["02", "04"],
highlights: ["专家路由", "细粒度专家", "896 选 16"],
},
src/data/papers.ts
+41
View File
@@ -71,6 +71,14 @@ export const papers: Paper[] = [
contribution: "把 BPE 引入神经机器翻译，形成现代子词分词主线。",
verified: true,
},
{
year: 1991,
title: "Adaptive Mixtures of Local Experts",
url: "https://doi.org/10.1162/neco.1991.3.1.79",
topics: ["MoE"],
contribution: "以门控网络让多个局部专家竞争分工，是现代 MoE 的概念起点。",
verified: true,
},
{
year: 2017,
title: "Attention Is All You Need",
@@ -401,6 +409,22 @@ export const papers: Paper[] = [
contribution: "系统化稀疏专家的稳定训练与迁移配方。",
verified: true,
},
{
year: 2022,
title: "Mixture-of-Experts with Expert Choice Routing",
url: "https://arxiv.org/abs/2202.09368",
topics: ["MoE"],
contribution: "把 token 选专家反转为专家选固定容量的 token，直接控制专家负载。",
verified: true,
},
{
year: 2023,
title: "From Sparse to Soft Mixtures of Experts",
url: "https://arxiv.org/abs/2308.00951",
topics: ["MoE"],
contribution: "用连续 slot 分配替代硬 top-k，探索完全可微的专家混合。",
verified: true,
},
{
year: 2024,
title: "Mixtral of Experts",
@@ -427,6 +451,23 @@ export const papers: Paper[] = [
spotlight: "DeepSeek",
verified: true,
},
{
year: 2024,
title: "Mixture-of-Depths: Dynamically Allocating Compute in Transformer-Based Language Models",
url: "https://arxiv.org/abs/2404.02258",
topics: ["MoE", "Transformer"],
contribution: "把条件计算从专家宽度扩到网络深度，让不同 Token 动态跳过或进入层。",
verified: true,
},
{
year: 2024,
title: "Auxiliary-Loss-Free Load Balancing Strategy for Mixture-of-Experts",
url: "https://arxiv.org/abs/2408.15664",
topics: ["MoE"],
contribution: "以只影响 top-k 选择的 expert bias 调负载，避免均衡辅助损失干扰主梯度。",
spotlight: "DeepSeek",
verified: true,
},
{
year: 2024,
title: "DeepSeek-V3 Technical Report",
src/pages/deepseek/index.astro
+5
View File
@@ -140,6 +140,7 @@ const toc = [
因此模型侧路由与系统侧 Expert Parallel 从来不能分开看。
</p>
</div>
<a class="button primary" href="/moe/#deepseek">进入 MoE 专题：比较粗专家、细粒度专家与共享专家 →</a>
</section>
<section class="article-section" id="mla">
@@ -212,6 +213,10 @@ const toc = [
根据近期负载上调冷门专家、下调热门专家；bias 只影响路由选择，不直接进入最终门控权重，
从而把“系统要均衡”和“模型要准确”更松地解耦。
</p>
<p>
<a href="/moe/#balance">专题版推导</a>会进一步区分 z-loss、辅助平衡损失、Loss-Free expert bias
与 K3 Quantile Balancing；“aux-loss-free”并不等于系统里不存在任何平衡约束。
</p>
<h3>FP8 训练真正难在哪</h3>
<p>
src/pages/index.astro
+57 -25
View File
@@ -7,6 +7,7 @@ import { chapters, statusLabel } from "@/data/chapters";
const routes: Record<string, string> = {
roadmap: "/roadmap/",
foundations: "/foundations/",
moe: "/moe/",
"long-context": "/long-context/",
};
@@ -75,6 +76,7 @@ const paths = [
<a class="button primary" href="/roadmap/">选择学习路径 <span aria-hidden="true">↓</span></a>
<a class="button" href="/k3/">直接解剖 K3</a>
<a class="button" href="/deepseek/">DeepSeek 专题</a>
<a class="button" href="/moe/">MoE 专题</a>
<a class="button" href="/long-context/">长上下文专题</a>
</div>
</div>
@@ -85,7 +87,7 @@ const paths = [
<div class="hero-stats">
<div><b>16</b><span>核心专题</span></div>
<div><b>151</b><span>K3 报告来源</span></div>
<div><b>125</b><span>关键论文索引</span></div>
<div><b>130</b><span>关键论文索引</span></div>
<div><b>47p</b><span>K3 技术报告</span></div>
</div>
</aside>
@@ -97,23 +99,41 @@ const paths = [
</div>
</section>
<section class="section compact release-section" id="new-long-context">
<a class="release-card" href="/long-context/">
<div>
<p class="eyebrow"><span>NEW / CHAPTER 07</span> LONG CONTEXT</p>
<h2>一百万 Token，不是一扇更大的窗</h2>
<p>
新专题把长上下文拆成计算、缓存、位置、状态容量与系统五张账单，
沿 26 篇一手论文走完 FlashAttention、MLA、Delta Rule、KDA、Kimi K3 与 DeepSeek-V4。
</p>
</div>
<dl>
<div><dt>LINEAGE</dt><dd>2019 → 2026</dd></div>
<div><dt>VISUALS</dt><dd>10+ 机制图</dd></div>
<div><dt>LAB</dt><dd>8 种策略 · 4 档长度</dd></div>
</dl>
<span class="release-arrow" aria-hidden="true">进入专题 →</span>
</a>
<section class="section compact release-section" id="new-chapters">
<div class="release-grid">
<a class="release-card moe-release" href="/moe/">
<div>
<p class="eyebrow"><span>NEW / CHAPTER 06</span> SPARSE EXPERTS</p>
<h2>2.8T 参数，不等于每个 Token 跑 2.8T</h2>
<p>
把 MoE 拆成容量、激活计算、路由、负载、通信与稳定性六张账，
从 Switch、DeepSeekMoE、Loss-Free、LatentMoE 一路走到 K3 Stable LatentMoE。
</p>
</div>
<dl>
<div><dt>LINEAGE</dt><dd>1991 → 2026</dd></div>
<div><dt>PAPERS</dt><dd>19 篇一手来源</dd></div>
<div><dt>LAB</dt><dd>8 架构 · 4 平衡策略</dd></div>
</dl>
<span class="release-arrow" aria-hidden="true">进入 MoE 专题 →</span>
</a>
<a class="release-card context-release" href="/long-context/">
<div>
<p class="eyebrow"><span>CHAPTER 07</span> LONG CONTEXT</p>
<h2>一百万 Token，不是一扇更大的窗</h2>
<p>
把长上下文拆成计算、缓存、位置、状态容量与系统五张账，
沿 26 篇一手论文走完 FlashAttention、MLA、Delta Rule、KDA、Kimi K3 与 DeepSeek-V4。
</p>
</div>
<dl>
<div><dt>LINEAGE</dt><dd>2019 → 2026</dd></div>
<div><dt>VISUALS</dt><dd>10+ 机制图</dd></div>
<div><dt>LAB</dt><dd>8 种策略 · 4 档长度</dd></div>
</dl>
<span class="release-arrow" aria-hidden="true">进入长上下文 →</span>
</a>
</div>
</section>
<section class="section" id="why">
@@ -323,12 +343,20 @@ const paths = [
padding-bottom: 0;
}
.release-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 22px;
}
.release-card {
position: relative;
display: grid;
grid-template-columns: minmax(0, 1.45fr) minmax(280px, 0.65fr);
gap: clamp(34px, 6vw, 90px);
padding: clamp(28px, 4vw, 58px);
grid-template-rows: 1fr auto;
gap: 28px;
min-height: 560px;
padding: clamp(28px, 3.6vw, 50px);
padding-bottom: clamp(74px, 7vw, 92px);
border: 1px solid var(--line-strong);
color: inherit;
text-decoration: none;
@@ -346,8 +374,8 @@ const paths = [
.release-card h2 {
max-width: 760px;
margin: 20px 0 18px;
font-size: clamp(2rem, 4vw, 4.2rem);
line-height: 1.03;
font-size: clamp(2rem, 3.2vw, 3.5rem);
line-height: 1.05;
}
.release-card p:not(.eyebrow) {
@@ -358,7 +386,7 @@ const paths = [
}
.release-card dl {
margin: 0 0 28px;
margin: 0;
border-top: 1px solid var(--line);
}
@@ -391,8 +419,12 @@ const paths = [
}
@media (max-width: 760px) {
.release-card {
.release-grid {
grid-template-columns: 1fr;
}
.release-card {
min-height: 0;
padding-bottom: 76px;
}
src/pages/k3/index.astro
+1
View File
@@ -286,6 +286,7 @@ const toc = [
这条技术线与 DeepSeekMoE 紧密相连：shared experts 保存公共知识，细粒度 routed experts 促进专业化。
K3 再借 LatentMoE 让“选 16 个专家”的通信和权重读取可承受，并为极端稀疏补上稳定性机制。
</p>
<a class="button primary" href="/moe/#k3">进入 MoE 专题：逐步推导 LatentMoE 与 Quantile Balancing →</a>
</section>
<section class="article-section" id="vision">
src/pages/moe/index.astro
+1680
View File
File diff suppressed because it is too large Load Diff
src/pages/progress/index.astro
+10 -8
View File
@@ -7,11 +7,12 @@ const published = chapters.filter((chapter) => chapter.status === "published").l
const researching = chapters.filter((chapter) => ["researching", "drafting"].includes(chapter.status)).length;
const workstreams = [
{ label: "研究框架与规范", value: 72, next: "给 125 篇索引补充逐篇精读层级" },
{ label: "研究框架与规范", value: 76, next: "给 130 篇索引补充逐篇精读层级" },
{ label: "网站设计系统", value: 89, next: "打印样式与更多通用可视化组件" },
{ label: "Kimi K3 深读", value: 55, next: "扩写 scaling / infra 逐图笔记" },
{ label: "Transformer 基础", value: 52, next: "加入矩阵形状动画与手算练习" },
{ label: "DeepSeek 专题", value: 61, next: "GRPO 完整公式与训练轨迹推导" },
{ label: "稀疏计算与 MoE", value: 74, next: "补充真实集群 traces 与专家特化案例" },
{ label: "长上下文专题", value: 72, next: "加入更多论文逐图笔记与真实模型配置对比" },
{ label: "引用与事实检查", value: 54, next: "自动化外链复查与来源等级扩展" },
{ label: "开源与部署", value: 100, next: "每轮保留不可变镜像、提交与回滚点" },
@@ -37,7 +38,7 @@ const workstreams = [
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
<div><dt>UPDATED</dt><dd>2026-07-28 22:47 CST</dd></div>
<div><dt>UPDATED</dt><dd>2026-07-28 23:24 CST</dd></div>
<div><dt>MODE</dt><dd>持续迭代，不锁死版本</dd></div>
</dl>
</div>
@@ -47,7 +48,7 @@ const workstreams = [
<div class="section-heading">
<div>
<p class="eyebrow"><span>01</span> WORKSTREAMS</p>
<h2>八条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
<h2>九条工作流同时推进，但不混淆“有页面”和“已核验”</h2>
</div>
<p class="section-lead">
内容首版优先打通全局脉络；随后每轮迭代选择一个专题推进到论文/工程层，并做独立事实复核。
@@ -84,10 +85,11 @@ const workstreams = [
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
<article><span>✓</span><h3>16 专题知识图</h3><p>从语言模型基础到评测安全，包含先修依赖和三条贯穿案例。</p></article>
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
<article><span>✓</span><h3>四张原创交互图</h3><p>K3 三轴架构、Self-Attention Query、DeepSeek 谱系与长上下文成本实验室。</p></article>
<article><span>✓</span><h3>四篇首版长文</h3><p>K3 完整导读、Transformer 基础、DeepSeek 论文谱系与长上下文专题。</p></article>
<article><span>✓</span><h3>五张原创交互图</h3><p>K3 三轴架构、Self-Attention Query、DeepSeek 谱系、长上下文成本与 MoE 路由实验室。</p></article>
<article><span>✓</span><h3>五篇首版长文</h3><p>K3 导读、Transformer 基础、DeepSeek 谱系、长上下文与 MoE 专题。</p></article>
<article><span>✓</span><h3>长上下文深度专题</h3><p>五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。</p></article>
<article><span>✓</span><h3>125 篇关键论文索引</h3><p>覆盖 12 个专题，支持全文搜索、标签筛选与 Kimi/DeepSeek 聚光主线。</p></article>
<article><span>✓</span><h3>MoE 深度专题</h3><p>六张账、19 篇一手论文、DeepSeek/K3 主线与路由—容量—通信交互实验室。</p></article>
<article><span>✓</span><h3>130 篇关键论文索引</h3><p>覆盖 12 个专题，支持全文搜索、标签筛选与 Kimi/DeepSeek 聚光主线。</p></article>
<article><span>✓</span><h3>公开仓库与自托管发布</h3><p>源码公开到 git.k1412.top，网站由不可变镜像、Compose Manager 与 HTTPS 交付。</p></article>
</div>
</section>
@@ -102,9 +104,9 @@ const workstreams = [
</div>
<div class="queue-table">
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
<div><span>P0</span><strong>稀疏计算与 MoE</strong><p>Switch → DeepSeekMoE → LatentMoE → Stable LatentMoE</p><em>路由模拟器 + 通信账本</em></div>
<div><span>P0</span><strong>推理模型与测试时扩展</strong><p>CoT → verifier → GRPO → R1 → k1.5 → K3 MOPD</p><em>奖励/预算交互图</em></div>
<div><span>P1</span><strong>长上下文二轮深化</strong><p>真实模型配置 → 内核细节 → 长上下文评测与失败案例</p><em>配置比较器 + 逐图论文笔记</em></div>
<div><span>P1</span><strong>推理模型与测试时扩展</strong><p>CoT → verifier → GRPO → R1 → k1.5 → K3 MOPD</p><em>奖励/预算交互图</em></div>
<div><span>P1</span><strong>MoE 二轮深化</strong><p>真实负载 traces → 专家特化可解释性 → 共享专家语义</p><em>案例库 + 集群证据</em></div>
<div><span>P1</span><strong>大规模训练系统</strong><p>ZeRO/Megatron → Expert/Context Parallel → DualPipe/MoonEP</p><em>显存与通信计算器</em></div>
<div><span>P2</span><strong>原生多模态</strong><p>ViT/CLIP → connector VLM → Kimi-VL/MoonViT-V2</p><em>视觉 Token 流程图</em></div>
</div>