feat: publish MoE deep dive

This commit is contained in:
wuyang
2026-07-28 23:32:14 +08:00
parent 761a75d5af
commit 3db826dd4a
17 changed files with 3533 additions and 48 deletions
+5
View File
@@ -140,6 +140,7 @@ const toc = [
因此模型侧路由与系统侧 Expert Parallel 从来不能分开看。
</p>
</div>
<a class="button primary" href="/moe/#deepseek">进入 MoE 专题:比较粗专家、细粒度专家与共享专家 →</a>
</section>
<section class="article-section" id="mla">
@@ -212,6 +213,10 @@ const toc = [
根据近期负载上调冷门专家、下调热门专家;bias 只影响路由选择,不直接进入最终门控权重,
从而把“系统要均衡”和“模型要准确”更松地解耦。
</p>
<p>
<a href="/moe/#balance">专题版推导</a>会进一步区分 z-loss、辅助平衡损失、Loss-Free expert bias
与 K3 Quantile Balancing;“aux-loss-free”并不等于系统里不存在任何平衡约束。
</p>
<h3>FP8 训练真正难在哪</h3>
<p>
+57 -25
View File
@@ -7,6 +7,7 @@ import { chapters, statusLabel } from "@/data/chapters";
const routes: Record<string, string> = {
roadmap: "/roadmap/",
foundations: "/foundations/",
moe: "/moe/",
"long-context": "/long-context/",
};
@@ -75,6 +76,7 @@ const paths = [
<a class="button primary" href="/roadmap/">选择学习路径 <span aria-hidden="true">↓</span></a>
<a class="button" href="/k3/">直接解剖 K3</a>
<a class="button" href="/deepseek/">DeepSeek 专题</a>
<a class="button" href="/moe/">MoE 专题</a>
<a class="button" href="/long-context/">长上下文专题</a>
</div>
</div>
@@ -85,7 +87,7 @@ const paths = [
<div class="hero-stats">
<div><b>16</b><span>核心专题</span></div>
<div><b>151</b><span>K3 报告来源</span></div>
<div><b>125</b><span>关键论文索引</span></div>
<div><b>130</b><span>关键论文索引</span></div>
<div><b>47p</b><span>K3 技术报告</span></div>
</div>
</aside>
@@ -97,23 +99,41 @@ const paths = [
</div>
</section>
<section class="section compact release-section" id="new-long-context">
<a class="release-card" href="/long-context/">
<div>
<p class="eyebrow"><span>NEW / CHAPTER 07</span> LONG CONTEXT</p>
<h2>一百万 Token,不是一扇更大的窗</h2>
<p>
新专题把长上下文拆成计算、缓存、位置、状态容量与系统五张账单,
沿 26 篇一手论文走完 FlashAttention、MLA、Delta Rule、KDA、Kimi K3 与 DeepSeek-V4。
</p>
</div>
<dl>
<div><dt>LINEAGE</dt><dd>2019 → 2026</dd></div>
<div><dt>VISUALS</dt><dd>10+ 机制图</dd></div>
<div><dt>LAB</dt><dd>8 种策略 · 4 档长度</dd></div>
</dl>
<span class="release-arrow" aria-hidden="true">进入专题 →</span>
</a>
<section class="section compact release-section" id="new-chapters">
<div class="release-grid">
<a class="release-card moe-release" href="/moe/">
<div>
<p class="eyebrow"><span>NEW / CHAPTER 06</span> SPARSE EXPERTS</p>
<h2>2.8T 参数,不等于每个 Token 跑 2.8T</h2>
<p>
把 MoE 拆成容量、激活计算、路由、负载、通信与稳定性六张账,
从 Switch、DeepSeekMoE、Loss-Free、LatentMoE 一路走到 K3 Stable LatentMoE。
</p>
</div>
<dl>
<div><dt>LINEAGE</dt><dd>1991 → 2026</dd></div>
<div><dt>PAPERS</dt><dd>19 篇一手来源</dd></div>
<div><dt>LAB</dt><dd>8 架构 · 4 平衡策略</dd></div>
</dl>
<span class="release-arrow" aria-hidden="true">进入 MoE 专题 →</span>
</a>
<a class="release-card context-release" href="/long-context/">
<div>
<p class="eyebrow"><span>CHAPTER 07</span> LONG CONTEXT</p>
<h2>一百万 Token,不是一扇更大的窗</h2>
<p>
把长上下文拆成计算、缓存、位置、状态容量与系统五张账,
沿 26 篇一手论文走完 FlashAttention、MLA、Delta Rule、KDA、Kimi K3 与 DeepSeek-V4。
</p>
</div>
<dl>
<div><dt>LINEAGE</dt><dd>2019 → 2026</dd></div>
<div><dt>VISUALS</dt><dd>10+ 机制图</dd></div>
<div><dt>LAB</dt><dd>8 种策略 · 4 档长度</dd></div>
</dl>
<span class="release-arrow" aria-hidden="true">进入长上下文 →</span>
</a>
</div>
</section>
<section class="section" id="why">
@@ -323,12 +343,20 @@ const paths = [
padding-bottom: 0;
}
.release-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 22px;
}
.release-card {
position: relative;
display: grid;
grid-template-columns: minmax(0, 1.45fr) minmax(280px, 0.65fr);
gap: clamp(34px, 6vw, 90px);
padding: clamp(28px, 4vw, 58px);
grid-template-rows: 1fr auto;
gap: 28px;
min-height: 560px;
padding: clamp(28px, 3.6vw, 50px);
padding-bottom: clamp(74px, 7vw, 92px);
border: 1px solid var(--line-strong);
color: inherit;
text-decoration: none;
@@ -346,8 +374,8 @@ const paths = [
.release-card h2 {
max-width: 760px;
margin: 20px 0 18px;
font-size: clamp(2rem, 4vw, 4.2rem);
line-height: 1.03;
font-size: clamp(2rem, 3.2vw, 3.5rem);
line-height: 1.05;
}
.release-card p:not(.eyebrow) {
@@ -358,7 +386,7 @@ const paths = [
}
.release-card dl {
margin: 0 0 28px;
margin: 0;
border-top: 1px solid var(--line);
}
@@ -391,8 +419,12 @@ const paths = [
}
@media (max-width: 760px) {
.release-card {
.release-grid {
grid-template-columns: 1fr;
}
.release-card {
min-height: 0;
padding-bottom: 76px;
}
+1
View File
@@ -286,6 +286,7 @@ const toc = [
这条技术线与 DeepSeekMoE 紧密相连:shared experts 保存公共知识,细粒度 routed experts 促进专业化。
K3 再借 LatentMoE 让“选 16 个专家”的通信和权重读取可承受,并为极端稀疏补上稳定性机制。
</p>
<a class="button primary" href="/moe/#k3">进入 MoE 专题:逐步推导 LatentMoE 与 Quantile Balancing →</a>
</section>
<section class="article-section" id="vision">
File diff suppressed because it is too large Load Diff
+10 -8
View File
@@ -7,11 +7,12 @@ const published = chapters.filter((chapter) => chapter.status === "published").l
const researching = chapters.filter((chapter) => ["researching", "drafting"].includes(chapter.status)).length;
const workstreams = [
{ label: "研究框架与规范", value: 72, next: "给 125 篇索引补充逐篇精读层级" },
{ label: "研究框架与规范", value: 76, next: "给 130 篇索引补充逐篇精读层级" },
{ label: "网站设计系统", value: 89, next: "打印样式与更多通用可视化组件" },
{ label: "Kimi K3 深读", value: 55, next: "扩写 scaling / infra 逐图笔记" },
{ label: "Transformer 基础", value: 52, next: "加入矩阵形状动画与手算练习" },
{ label: "DeepSeek 专题", value: 61, next: "GRPO 完整公式与训练轨迹推导" },
{ label: "稀疏计算与 MoE", value: 74, next: "补充真实集群 traces 与专家特化案例" },
{ label: "长上下文专题", value: 72, next: "加入更多论文逐图笔记与真实模型配置对比" },
{ label: "引用与事实检查", value: 54, next: "自动化外链复查与来源等级扩展" },
{ label: "开源与部署", value: 100, next: "每轮保留不可变镜像、提交与回滚点" },
@@ -37,7 +38,7 @@ const workstreams = [
<div><dt>OVERALL</dt><dd>专题平均 {average}%</dd></div>
<div><dt>READABLE</dt><dd>{published} 个首版可读专题</dd></div>
<div><dt>ACTIVE</dt><dd>{researching} 个研究/写作中</dd></div>
<div><dt>UPDATED</dt><dd>2026-07-28 22:47 CST</dd></div>
<div><dt>UPDATED</dt><dd>2026-07-28 23:24 CST</dd></div>
<div><dt>MODE</dt><dd>持续迭代,不锁死版本</dd></div>
</dl>
</div>
@@ -47,7 +48,7 @@ const workstreams = [
<div class="section-heading">
<div>
<p class="eyebrow"><span>01</span> WORKSTREAMS</p>
<h2>条工作流同时推进,但不混淆“有页面”和“已核验”</h2>
<h2>条工作流同时推进,但不混淆“有页面”和“已核验”</h2>
</div>
<p class="section-lead">
内容首版优先打通全局脉络;随后每轮迭代选择一个专题推进到论文/工程层,并做独立事实复核。
@@ -84,10 +85,11 @@ const workstreams = [
<article><span>✓</span><h3>K3 报告已结构化拆解</h3><p>47 页报告目录、151 条参考来源和架构/后训练/系统主线已经提取。</p></article>
<article><span>✓</span><h3>16 专题知识图</h3><p>从语言模型基础到评测安全,包含先修依赖和三条贯穿案例。</p></article>
<article><span>✓</span><h3>编辑式网站系统</h3><p>响应式导航、章节模板、侧栏、进度、论文链和证据提示组件。</p></article>
<article><span>✓</span><h3>张原创交互图</h3><p>K3 三轴架构、Self-Attention Query、DeepSeek 谱系长上下文成本实验室。</p></article>
<article><span>✓</span><h3>篇首版长文</h3><p>K3 完整导读、Transformer 基础、DeepSeek 论文谱系长上下文专题。</p></article>
<article><span>✓</span><h3>张原创交互图</h3><p>K3 三轴架构、Self-Attention Query、DeepSeek 谱系长上下文成本与 MoE 路由实验室。</p></article>
<article><span>✓</span><h3>篇首版长文</h3><p>K3 导读、Transformer 基础、DeepSeek 谱系长上下文与 MoE 专题。</p></article>
<article><span>✓</span><h3>长上下文深度专题</h3><p>五张成本账、26 篇一手论文、10+ 机制图与 8 策略交互实验室。</p></article>
<article><span>✓</span><h3>125 篇关键论文索引</h3><p>覆盖 12 个专题,支持全文搜索、标签筛选与 Kimi/DeepSeek 聚光主线。</p></article>
<article><span>✓</span><h3>MoE 深度专题</h3><p>六张账、19 篇一手论文、DeepSeek/K3 主线与路由—容量—通信交互实验室。</p></article>
<article><span>✓</span><h3>130 篇关键论文索引</h3><p>覆盖 12 个专题,支持全文搜索、标签筛选与 Kimi/DeepSeek 聚光主线。</p></article>
<article><span>✓</span><h3>公开仓库与自托管发布</h3><p>源码公开到 git.k1412.top,网站由不可变镜像、Compose Manager 与 HTTPS 交付。</p></article>
</div>
</section>
@@ -102,9 +104,9 @@ const workstreams = [
</div>
<div class="queue-table">
<div class="head"><b>优先级</b><b>专题</b><b>本轮交付</b><b>完成闸门</b></div>
<div><span>P0</span><strong>稀疏计算与 MoE</strong><p>Switch → DeepSeekMoE → LatentMoE → Stable LatentMoE</p><em>路由模拟器 + 通信账本</em></div>
<div><span>P0</span><strong>推理模型与测试时扩展</strong><p>CoT → verifier → GRPO → R1 → k1.5 → K3 MOPD</p><em>奖励/预算交互图</em></div>
<div><span>P1</span><strong>长上下文二轮深化</strong><p>真实模型配置 → 内核细节 → 长上下文评测与失败案例</p><em>配置比较器 + 逐图论文笔记</em></div>
<div><span>P1</span><strong>推理模型与测试时扩展</strong><p>CoT → verifier → GRPO → R1 → k1.5 → K3 MOPD</p><em>奖励/预算交互图</em></div>
<div><span>P1</span><strong>MoE 二轮深化</strong><p>真实负载 traces → 专家特化可解释性 → 共享专家语义</p><em>案例库 + 集群证据</em></div>
<div><span>P1</span><strong>大规模训练系统</strong><p>ZeRO/Megatron → Expert/Context Parallel → DualPipe/MoonEP</p><em>显存与通信计算器</em></div>
<div><span>P2</span><strong>原生多模态</strong><p>ViT/CLIP → connector VLM → Kimi-VL/MoonViT-V2</p><em>视觉 Token 流程图</em></div>
</div>