wuyang
111 lines
12 KiB
TypeScript
111 lines
12 KiB
TypeScript
export const deepseekLedgers = [
|
||
["Q01", "Dense 坐标系", "为什么 DeepSeek LLM 不是可跳过的序章？", "它固定 tokenizer、数据、架构和 scaling 试验的起点；并不单独证明后续所有设计。"],
|
||
["Q02", "参数角色", "671B / 37B 各表示什么？", "total 是装下的容量，activated 是每 Token 经过的专家参数子集；都不等于端到端 FLOPs。"],
|
||
["Q03", "专家粒度", "为什么切小专家还要多选？", "DeepSeekMoE 把每个专家缩成 1/m，总数和激活数同乘 m，近似保持专家计算。"],
|
||
["Q04", "Shared expert", "为什么把公共知识单独隔离？", "始终激活的 shared experts 减少 routed experts 重复；它们仍然要付激活计算。"],
|
||
["Q05", "通信税", "为什么稀疏 FLOPs 不等于便宜？", "路由会产生 dispatch/combine、跨节点 all-to-all、负载长尾和权重访问。"],
|
||
["Q06", "均衡", "aux-loss-free 到底去掉了什么？", "V3 的 expert bias 影响选择、不进入最终 gate weight；仍有 sequence-wise auxiliary loss 防极端失衡。"],
|
||
["Q07", "KV 状态", "为什么 V2 把服务状态当架构问题？", "权重只装一次，KV 随请求、层、Token 增长，直接限制并发和长上下文。"],
|
||
["Q08", "Attention 压缩", "MQA、GQA、MLA 的差别是什么？", "MQA/GQA 共享 K/V 头；MLA 联合低秩压缩 K/V 内容并在计算中恢复。"],
|
||
["Q09", "矩阵吸收", "MLA 为什么不必恢复完整 content K/V？", "无位置项时可利用矩阵乘结合律，把 K/V 上投影吸收到 query/output 投影。"],
|
||
["Q10", "位置分叉", "为什么要 decoupled RoPE？", "RoPE 会阻断固定权重吸收，所以 V2 另设小 RoPE query/key 分支，并缓存 key。"],
|
||
["Q11", "FP8 合同", "“FP8 训练”包含哪些角色？", "主要 GEMM 用 FP8，并配细粒度缩放、较高精度累加和高精度敏感算子；不是全路径 FP8。"],
|
||
["Q12", "Pipeline", "DualPipe 隐藏了什么？", "从两端注入 micro-batch，让成对前后向 chunk 与通信重叠；它减少而非清零 bubble。"],
|
||
["Q13", "MTP", "训练和推理各怎样使用 MTP？", "顺序模块增加未来 Token 监督；推理可丢弃，也可复用于 speculative draft。"],
|
||
["Q14", "GRPO", "去掉 critic 后还剩什么？", "policy/reference、同题多 rollout、reward/verifier、clip 和 KL；主要省掉 value model。"],
|
||
["Q15", "可验证奖励", "R1-Zero 的奖励能覆盖哪些任务？", "论文用数学、代码、逻辑等规则可验证域和格式奖励；开放任务仍是限制。"],
|
||
["Q16", "纯 RL 实验", "R1-Zero 究竟证明了什么？", "强 V3 Base 在无 reasoning SFT 时可被规则奖励继续塑造；不等于没有预训练先验。"],
|
||
["Q17", "R1 pipeline", "正式 R1 为什么不是纯 RL？", "cold start → reasoning RL → rejection/SFT mix → general RL，分别修可读性、广度和对齐。"],
|
||
["Q18", "蒸馏", "学生为什么不是“小号 R1-Zero”？", "1.5B–70B 学生主要对约 800K 教师样本做 SFT，没有重演同一 RL 探索。"],
|
||
["Q19", "复现反查", "DAPO / Dr.GRPO 修的是哪类问题？", "它们处理 clip、采样、聚合、截断、长度和难度偏差；是后续研究，不是已披露 R1 内部配方。"],
|
||
["Q20", "DSA", "可学习 indexer 为什么不是固定稀疏？", "indexer 对历史内容评分，主 attention 只读 top-k；它需要专门训练，也可能漏检。"],
|
||
["Q21", "Agent 数据", "V3.2 怎样把 reasoning 放进环境？", "specialist distillation + mixed RL；环境、工具、任务、解法和 verifier 构成数据闭环。"],
|
||
["Q22", "V4 Attention", "CSA 与 HCA 各压什么？", "CSA 先压缩再稀疏 top-k；HCA 更强压缩后保留全部 compressed entries。"],
|
||
["Q23", "V4 稳定化", "mHC、Muon、QK/RMSNorm、clamp 各管什么？", "它们分别管残差混合、矩阵更新、attention 尺度和 FFN 极值，不能合成一个技巧。"],
|
||
["Q24", "K3 对照", "哪些是祖先，哪些只是同题新解？", "DeepSeekMoE/MLA 有明确继承；QB、KDA、AttnRes、SiTU、MOPD 多是新解或同期路线。"],
|
||
] as const;
|
||
|
||
export const deepseekWaves = [
|
||
["W1", "2024.01", "Dense 坐标", "先固定数据、tokenizer、训练与 scaling 对照，后面的结构收益才有可比起点。"],
|
||
["W2", "2024.01", "稀疏容量", "细粒度 routed experts 加 shared experts，把总容量与单 Token 激活计算第一次清楚分开。"],
|
||
["W3", "2024.05", "服务状态", "MLA 不再只优化训练 FLOPs，而是直接改写随请求增长的 KV Cache。"],
|
||
["W4", "2024.12", "协同训练", "V3 把路由、FP8、MTP、pipeline 与通信写成同一套训练合同。"],
|
||
["W5", "2024.02 → 2025.01", "推理 RL", "DeepSeekMath 先减掉 critic；R1-Zero 再隔离规则奖励，R1 恢复可读性与通用性。"],
|
||
["W6", "2025.03", "复现显微镜", "DAPO 与 Dr.GRPO 暴露 clipping、采样、截断、长度归一和题目难度偏差。"],
|
||
["W7", "2025.12", "稀疏检索", "V3.2 用学习型 indexer 选历史，再让主 attention 读取 top-k。"],
|
||
["W8", "2025.12", "Agent 环境", "推理从静态题目进入含工具、状态转移和 verifier 的交互数据闭环。"],
|
||
["W9", "2026.06", "异构长状态", "V4 用 CSA 与 HCA 处理不同时间尺度，并联合 mHC、Muon 和数值约束。"],
|
||
["W10", "2026.07", "K3 对照", "继承图必须允许没有箭头：相同的百万上下文目标，可以有完全不同的状态机器。"],
|
||
] as const;
|
||
|
||
export const deepseekBranches = [
|
||
["代码与专家", "DeepSeek-Coder → Coder-V2 → ESFT", "代码数据配方、continued pretraining 与只微调相关专家，说明 MoE 的价值不只在通用模型参数量。", "https://arxiv.org/abs/2406.11931"],
|
||
["数学与证明", "DeepSeekMath → Prover-V1.5 → Prover-V2", "从数学语料、GRPO 走到 formal proof feedback、subgoal decomposition 与可验证证明搜索。", "https://arxiv.org/abs/2504.21801"],
|
||
["视觉与压缩", "DeepSeek-VL/VL2 → Janus → OCR", "理解、生成与光学压缩形成另一条主干；它们不应被挤进纯文本 V2→V4 时间线。", "https://arxiv.org/abs/2412.10302"],
|
||
["系统实现", "DeepEP → DualPipe → DeepGEMM / FlashMLA", "论文里的稀疏计算、流水线、FP8 与 MLA 最终必须落到可调用的通信和 kernel 实现。", "https://github.com/deepseek-ai/DeepEP"],
|
||
["条件记忆", "Engram", "把可查表的静态模式从动态网络中分离，增加一条不同于 MoE 与 attention 的稀疏轴。", "https://arxiv.org/abs/2601.07372"],
|
||
] as const;
|
||
|
||
export const deepseekPaperChain = [
|
||
["01", "1991", "Adaptive Mixtures of Local Experts", "https://proceedings.neurips.cc/paper/1991/hash/59b90e1005a220e2ebc542eb9d950b1e-Abstract.html", "专家门控前史"],
|
||
["02", "2000", "Conditional Computation", "https://arxiv.org/abs/cs/0008102", "条件计算"],
|
||
["03", "2003", "A Neural Probabilistic Language Model", "https://www.jmlr.org/papers/v3/bengio03a.html", "Dense LM 坐标"],
|
||
["04", "2017", "Attention Is All You Need", "https://arxiv.org/abs/1706.03762", "Transformer 主干"],
|
||
["05", "2017", "Outrageously Large Neural Networks", "https://arxiv.org/abs/1701.06538", "稀疏 MoE"],
|
||
["06", "2017", "Proximal Policy Optimization", "https://arxiv.org/abs/1707.06347", "GRPO 对照"],
|
||
["07", "2018", "GPipe", "https://arxiv.org/abs/1811.06965", "Pipeline 前史"],
|
||
["08", "2018", "PipeDream", "https://arxiv.org/abs/1806.03377", "Pipeline schedule"],
|
||
["09", "2019", "Fast Transformer Decoding / MQA", "https://arxiv.org/abs/1911.02150", "KV 共享"],
|
||
["10", "2019", "Megatron-LM", "https://arxiv.org/abs/1909.08053", "模型并行"],
|
||
["11", "2019", "ZeRO", "https://arxiv.org/abs/1910.02054", "状态分片"],
|
||
["12", "2019", "RMSNorm", "https://arxiv.org/abs/1910.07467", "尺度控制"],
|
||
["13", "2020", "GShard", "https://arxiv.org/abs/2006.16668", "大规模 MoE"],
|
||
["14", "2020", "QK-Normalization", "https://arxiv.org/abs/2010.04245", "attention logit 稳定"],
|
||
["15", "2021", "Switch Transformers", "https://arxiv.org/abs/2101.03961", "coarse top-1 MoE"],
|
||
["16", "2021", "RoFormer / RoPE", "https://arxiv.org/abs/2104.09864", "MLA 位置分叉"],
|
||
["17", "2022", "ST-MoE", "https://arxiv.org/abs/2202.08906", "MoE 稳定性"],
|
||
["18", "2022", "DeepNet", "https://arxiv.org/abs/2203.00555", "深层残差"],
|
||
["19", "2022", "InstructGPT", "https://arxiv.org/abs/2203.02155", "SFT/RM/PPO 合同"],
|
||
["20", "2022", "FlashAttention", "https://arxiv.org/abs/2205.14135", "IO-aware exact attention"],
|
||
["21", "2022", "Process and Outcome Feedback", "https://arxiv.org/abs/2211.14275", "reasoning reward 前史"],
|
||
["22", "2022", "Self-Consistency", "https://arxiv.org/abs/2203.11171", "多采样聚合"],
|
||
["23", "2023", "GQA", "https://arxiv.org/abs/2305.13245", "KV 分组"],
|
||
["24", "2023", "Let's Verify Step by Step", "https://arxiv.org/abs/2305.20050", "verifier / PRM"],
|
||
["25", "2023", "Direct Preference Optimization", "https://arxiv.org/abs/2305.18290", "RL 外偏好路线"],
|
||
["26", "2023", "FlashAttention-2", "https://arxiv.org/abs/2307.08691", "attention kernel"],
|
||
["27", "2023", "PagedAttention / vLLM", "https://arxiv.org/abs/2309.06180", "KV 服务状态"],
|
||
["28", "2024", "DeepSeek LLM", "https://arxiv.org/abs/2401.02954", "Dense / scaling 基线"],
|
||
["29", "2024", "DeepSeek-Coder", "https://arxiv.org/abs/2401.14196", "代码数据旁支"],
|
||
["30", "2024", "DeepSeekMoE", "https://arxiv.org/abs/2401.06066", "细粒度 + shared"],
|
||
["31", "2024", "DeepSeekMath", "https://arxiv.org/abs/2402.03300", "数学数据 + GRPO"],
|
||
["32", "2024", "RLOO", "https://arxiv.org/abs/2402.14740", "critic-free 对照"],
|
||
["33", "2024", "DeepSeek-V2", "https://arxiv.org/abs/2405.04434", "MLA + MoE"],
|
||
["34", "2024", "Better & Faster LLMs via MTP", "https://arxiv.org/abs/2404.19737", "MTP 祖先"],
|
||
["35", "2024", "DeepSeek-Coder-V2", "https://arxiv.org/abs/2406.11931", "V2 continued pretrain"],
|
||
["36", "2024", "ESFT", "https://arxiv.org/abs/2407.01906", "专家特化微调"],
|
||
["37", "2024", "DeepSeek-Prover-V1.5", "https://arxiv.org/abs/2408.08152", "proof feedback RL"],
|
||
["38", "2024", "Hyper-Connections", "https://arxiv.org/abs/2409.19606", "mHC 前身"],
|
||
["39", "2024", "DeepSeek-V3", "https://arxiv.org/abs/2412.19437", "FP8 / DualPipe / MTP"],
|
||
["40", "2025", "DeepSeek-R1", "https://arxiv.org/abs/2501.12948", "R1-Zero / R1 / 蒸馏"],
|
||
["41", "2025", "Muon is Scalable for LLM Training", "https://arxiv.org/abs/2502.16982", "V4 optimizer 前史"],
|
||
["42", "2025", "DAPO", "https://arxiv.org/abs/2503.14476", "GRPO 工程修正"],
|
||
["43", "2025", "Understanding R1-Zero-Like Training", "https://arxiv.org/abs/2503.20783", "Dr.GRPO / 偏差"],
|
||
["44", "2025", "DeepSeek-Prover-V2", "https://arxiv.org/abs/2504.21801", "subgoal + RL"],
|
||
["45", "2025", "DeepEP", "https://github.com/deepseek-ai/DeepEP", "Expert Parallel kernel"],
|
||
["46", "2025", "DualPipe", "https://github.com/deepseek-ai/DualPipe", "V3/R1 pipeline 实现"],
|
||
["47", "2025", "DeepGEMM", "https://github.com/deepseek-ai/DeepGEMM", "FP8 GEMM 实现"],
|
||
["48", "2025", "DeepSeek-VL2", "https://arxiv.org/abs/2412.10302", "多模态理解旁支"],
|
||
["49", "2025", "Janus-Pro", "https://arxiv.org/abs/2501.17811", "统一理解/生成旁支"],
|
||
["50", "2025", "Kimi k1.5", "https://arxiv.org/abs/2501.12599", "同期 reasoning RL"],
|
||
["51", "2025", "Kimi K2", "https://arxiv.org/abs/2507.20534", "MLA / MoE / Muon 对照"],
|
||
["52", "2025", "Kimi Linear", "https://arxiv.org/abs/2510.26692", "KDA 前身"],
|
||
["53", "2025", "DeepSeek-V3.2", "https://arxiv.org/abs/2512.02556", "DSA + Agent"],
|
||
["54", "2025", "mHC", "https://arxiv.org/abs/2512.24880", "受约束 residual"],
|
||
["55", "2026", "Engram", "https://arxiv.org/abs/2601.07372", "条件记忆新稀疏轴"],
|
||
["56", "2026", "LatentMoE", "https://arxiv.org/abs/2601.18089", "K3 routed latent 前身"],
|
||
["57", "2026", "Attention Residuals", "https://arxiv.org/abs/2603.15031", "K3 深度路由"],
|
||
["58", "2026", "DeepSeek-V4", "https://arxiv.org/abs/2606.19348", "CSA / HCA / mHC / Muon"],
|
||
["59", "2026", "Kimi K3", "https://arxiv.org/abs/2607.24653", "对照锚点"],
|
||
["60", "2026", "Kimi K3 official repository", "https://github.com/MoonshotAI/Kimi-K3", "开放实现边界"],
|
||
] as const;
|