From 2a6a7c2780da52cd6e99ae33face85d048a5d6bd Mon Sep 17 00:00:00 2001 From: wuyang6 Date: Tue, 7 Jul 2026 19:58:00 +0800 Subject: [PATCH] Enhance model training lite workflow --- .gitignore | 1 + skills/model-training-lite/SKILL.md | 122 ++++++++- .../jupyter_defaults.example.json | 5 + .../references/zk_sft_defaults.md | 58 +++++ .../scripts/render_sft_submission_plan.py | 76 +++++- .../scripts/summarize_eval_result.py | 236 ++++++++++++++++++ 6 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 skills/model-training-lite/jupyter_defaults.example.json mode change 100644 => 100755 skills/model-training-lite/scripts/render_sft_submission_plan.py create mode 100755 skills/model-training-lite/scripts/summarize_eval_result.py diff --git a/.gitignore b/.gitignore index 399604e..bd1fb74 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ router_session_parquet/ # Environment files .env .env.* +skills/*/.local/ # Local benchmark outputs benchmark_artifacts/ diff --git a/skills/model-training-lite/SKILL.md b/skills/model-training-lite/SKILL.md index c9ff0dc..1856171 100644 --- a/skills/model-training-lite/SKILL.md +++ b/skills/model-training-lite/SKILL.md @@ -1,8 +1,8 @@ --- name: model-training-lite name_zh: 轻量模型训练 -description: 从 Codex、Claude Code 或其他 Agent 通过用户提供的 Jupyter 环境和数据版本发起一次模型训练，聚焦“准备远端工作区、同步数据、提交 CML SFT、返回 JobID”，不包含完整自主迭代闭环。 -when_to_use: 当用户已经准备好训练数据，想让 Agent 使用 Jupyter/CloudML 发起一次模型训练、复用当前仓库的训练脚本和模板、或询问如何把模型训练流程沉淀成轻量 skill 时使用。 +description: 从 Codex、Claude Code 或其他 Agent 通过用户提供的 Jupyter 环境和数据版本发起一次模型训练，并可选发起评测和生成轻量结果摘要；不包含自动数据增强闭环。 +when_to_use: 当用户已经准备好训练数据，想让 Agent 使用 Jupyter/CloudML 基于某个数据分支或 commit 发起一次模型训练、训练后评测、汇总结果，或询问如何把模型训练流程沉淀成轻量 skill 时使用。 aliases: jupyter-model-training, sft-submit, 轻量训练, 模型训练提交 examples: - 用这个 Jupyter 和 ai-planning 当前分支重新发起训练 @@ -14,7 +14,17 @@ allowed_tools: read_file, write_file, grep_search, glob_search, ask_user_questio # 轻量模型训练 -本 skill 只负责一次训练提交： +本 skill 是人机协同的训练/评测执行器：人负责目标和边界裁决，AI 负责候选、统计、证据整理和确定性执行。 + +```text +人提出目标问题 + -> AI 整理候选、统计分布、生成 review 表 + -> 人确认边界、保留样本、数据是否入库 + -> AI 提交训练、发起评测、汇总结果 + -> 人判断是否继续下一轮 +``` + +训练评测执行链路： ```text Jupyter 授权环境 @@ -23,26 +33,53 @@ Jupyter 授权环境 -> 同步训练脚本与数据版本 -> 检查 CML / 数据 / zk_trainer -> 提交 SFT - -> 返回 JobID、产物路径、下一步评测入口 + -> 可选：SFT 成功后提交 CML 评测 + -> 可选：读取 workflow metric_diff 生成轻量结果摘要 + -> 返回 JobID、workflow ID、产物路径、摘要路径 ``` -不要把它扩展成 `model-iteration` 那种完整“baseline -> 分析 -> 增强 -> 训练 -> 评测 -> 再分析”的自主循环。 +不要把它扩展成 `model-iteration` 那种完整“baseline -> 分析 -> 增强 -> 训练 -> 评测 -> 再分析”的自主循环。本 skill 可以辅助人完成每一步决策前的信息准备，但不自动裁决边界、不自动修改数据、不自动生成增强样本、不自动进入下一轮训练。 ## 必要输入 开始前必须拿到： -- **Jupyter 地址和授权方式**：URL；密码、token、cookie/session，或说明当前环境已有可用登录态。 -- **数据版本**：git repo + branch + commit，或远端工作区中已存在的数据路径。 +- **Jupyter 地址和授权方式**：URL；密码、token、cookie/session，或使用本地默认配置。 +- **数据版本**：git repo + branch + commit，或 git repo + branch，或远端工作区中已存在的数据路径。 - **训练配方**：如果是当前 ZK 中控 SFT，默认复用 `skills/model-iteration/scripts/`；如果不是，必须让用户提供训练脚本、模板或命令。 通常用户只需要显式给 `Jupyter 地址` 和 `数据版本`。当前项目的默认训练配方可从本仓库继承，不必每次追问。 +本地默认 Jupyter 配置路径： + +```text +skills/model-training-lite/.local/jupyter_defaults.json +``` + +`.local/` 已加入 gitignore。真实密码只能放在本地配置或环境变量里，不要写入 `SKILL.md`、README、提交记录或最终回复。 + +配置格式可参考： + +```text +skills/model-training-lite/jupyter_defaults.example.json +``` + +脚本也支持环境变量： + +```text +MODEL_TRAINING_LITE_JUPYTER_URL +MODEL_TRAINING_LITE_JUPYTER_PASSWORD +MODEL_TRAINING_LITE_JUPYTER_TOKEN +MODEL_TRAINING_LITE_JUPYTER_COOKIE +MODEL_TRAINING_LITE_JUPYTER_AUTH_TYPE +``` + 可选但推荐确认： - `owner`：远端输出目录使用的用户前缀，例如 `wuyang6`。 - `run_name`：本次训练工作区名，例如 `manual_YYYYMMDD_zk_intent_xxx`。 -- 是否训练后立刻发起评测。默认只提交训练，不自动串接评测。 +- 是否训练后立刻发起评测。默认只提交训练；用户要求“训练后评测/看效果/分析结果”时启用。 +- `model_old`：评测工作流中的对照模型路径；未提供时使用默认线上基线。 ## 默认 ZK SFT 配方 @@ -62,6 +99,22 @@ skills/model-iteration/assets/config.yaml ## 推荐流程 +### 0. 人机协同边界 + +AI 应主动辅助人完成信息准备，但不能替人拍板： + +| 阶段 | AI 负责 | 人负责 | +|---|---|---| +| 目标定义 | 把问题拆成可评测口径，列出候选集合和风险 | 确认目标问题和主指标 | +| 数据准备 | 调用相关 skill 生成候选、统计、review 表 | 判断边界样本是否保留 | +| 数据构造 | 参考 `product-data` 生成训练/评测格式 | 确认数据是否入库 | +| 标签判断 | 参考 `label-master` 校验 tag / function / complex | 确认争议规则 | +| 线上挖掘 | 参考 `online-mining-v2` 获取 rid/session/prompt/output | 确认挖掘口径 | +| 批量打标 | 参考 `model-labeling` 请求线上模型，生成基线 | 确认宽口径和人工 override | +| 训练评测 | 本 skill 提交 SFT、评测、轻量摘要 | 决定是否继续下一轮 | + +需要全自动假设驱动迭代时，使用 `model-iteration`，不要把本 skill 临时扩展成自动闭环。 + ### 1. 先生成提交计划 优先执行 portable script： @@ -80,12 +133,16 @@ python skills/model-training-lite/scripts/render_sft_submission_plan.py --input "data_commit": "b92be709", "owner": "wuyang6", "run_name": "manual_20260528_zk_intent_clean_train", - "recipe": "zk_sft" + "recipe": "zk_sft", + "run_eval": true, + "model_old": "/mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface" } ``` 脚本只生成计划和命令草案，不连接 Jupyter，不提交训练。 +如果只提供 `data_branch` 不提供 `data_commit`，脚本会生成 `git reset --hard origin/`，并在结果中写 warning。正式训练报告里必须记录实际 `git rev-parse HEAD`，避免后续无法复现。 + ### 2. 连接 Jupyter 如果已经有 session/cookie，可直接调用 Jupyter REST API： @@ -117,6 +174,12 @@ zk_trainer/ 把默认 ZK SFT 配方里的 `scripts/` 和 `config.yaml` 同步到远端工作区。再 clone 或更新数据仓库，并 checkout 到用户指定 commit。 +除 `model-iteration` 默认脚本外，还要同步本 skill 的轻量分析脚本： + +```text +skills/model-training-lite/scripts/summarize_eval_result.py +``` + ### 4. 提交前检查 提交训练前必须输出： @@ -158,10 +221,49 @@ eval "$(./scripts/resolve_run_ids.sh)" 再查一次 `cml custom_train describe `，确认状态不是提交后立即失败。 +### 7. 可选：训练后提交评测 + +只有用户要求“训练后评测/看效果/分析结果”时执行。 + +训练成功标记存在后： + +```bash +source ~/.cloudml-cli/.profile +export AUTORESEARCH_CHAT_ROOT= +export AUTORESEARCH_ROOT="$AUTORESEARCH_CHAT_ROOT" +cd "$AUTORESEARCH_CHAT_ROOT" +eval "$(./scripts/resolve_run_ids.sh)" +./scripts/submit_cml_eval.sh "$EVAL_RUNDIC" "$AUTORESEARCH_CHAT_ROOT/sft_output" "" +``` + +提交后必须回报： + +- `EVAL_RUNDIC` +- CloudML workflow 提交输出 +- 评测产物目录：`/mnt/xiaoai-zk-model-train-tj5/workflow5/workflow/` +- 评测完成标记：`metric_diff/lark_template.json` + +### 8. 可选：轻量结果摘要 + +评测产物落盘后执行： + +```bash +python "$AUTORESEARCH_CHAT_ROOT/scripts/summarize_eval_result.py" \ + --workflow-root /mnt/xiaoai-zk-model-train-tj5/workflow5 \ + --run-dic "$EVAL_RUNDIC" \ + --output "$AUTORESEARCH_CHAT_ROOT/results/eval_summary_${EVAL_RUNDIC}.md" +``` + +摘要脚本只做轻量汇总： + +- 从 `metric_diff/lark_template.json` 尽力抽取指标。 +- 从 `metric_diff/specific_comparison.csv` 汇总错误分布和错误样例。 +- 不做数据增强建议，不做训练集修改，不替代 `model-iteration` 的深度归因。 + ## 边界 - 不自动修改训练数据。 -- 不自动做错误归因。 +- 不自动做深度错误归因；只允许生成轻量指标/错误分布摘要。 - 不自动生成增强样本。 - 不默认串接评测；训练成功后是否评测由用户或后续明确指令决定。 - 不把 Jupyter 密码、CloudML key、cookie 写进产物或最终回复。 diff --git a/skills/model-training-lite/jupyter_defaults.example.json b/skills/model-training-lite/jupyter_defaults.example.json new file mode 100644 index 0000000..6826b69 --- /dev/null +++ b/skills/model-training-lite/jupyter_defaults.example.json @@ -0,0 +1,5 @@ +{ + "jupyter_url": "https://example-jupyter/lab?", + "auth_type": "password", + "password": "DO_NOT_COMMIT_REAL_PASSWORD" +} diff --git a/skills/model-training-lite/references/zk_sft_defaults.md b/skills/model-training-lite/references/zk_sft_defaults.md index 990ec41..b00436f 100644 --- a/skills/model-training-lite/references/zk_sft_defaults.md +++ b/skills/model-training-lite/references/zk_sft_defaults.md @@ -11,6 +11,7 @@ skills/model-iteration/scripts/submit_sft.sh skills/model-iteration/scripts/submit_cml_eval.sh skills/model-iteration/scripts/sft_train_job.yaml.tpl skills/model-iteration/assets/config.yaml +skills/model-training-lite/scripts/summarize_eval_result.py ``` ## 默认基模 @@ -28,6 +29,14 @@ git@git.n.xiaomi.com:ai-service/ai-planning.git branch: autoresearch-v1 ``` +如果用户只提供数据分支、不提供 commit，训练前必须在远端工作区记录： + +```bash +git -C ai-planning rev-parse HEAD +``` + +正式报告使用该 commit 作为本次训练的数据版本。 + 训练脚本会扫描： ```text @@ -49,6 +58,22 @@ ai-planning/data/train_set/*/*.jsonl /sft_output/_SUCCESS ``` +## 默认 Jupyter 配置 + +本地默认配置文件： + +```text +skills/model-training-lite/.local/jupyter_defaults.json +``` + +`.local/` 不提交。配置优先级： + +```text +脚本输入 > 环境变量 > .local/jupyter_defaults.json +``` + +脚本输出只显示认证是否已配置，不回显密码、token 或 cookie。 + ## CML 环境 CloudML CLI 通常位于： @@ -71,6 +96,37 @@ cml config show cml custom_train describe ``` +## 默认评测 + +训练后评测复用： + +```text +skills/model-iteration/scripts/submit_cml_eval.sh +skills/model-iteration/assets/config.yaml +``` + +默认 workflow 配置从 `config.yaml` 读取： + +```text +cml_eval.workflow_id +cml_eval.version +``` + +评测产物目录： + +```text +/mnt/xiaoai-zk-model-train-tj5/workflow5/workflow/metric_diff/ +``` + +轻量摘要脚本： + +```bash +python scripts/summarize_eval_result.py \ + --workflow-root /mnt/xiaoai-zk-model-train-tj5/workflow5 \ + --run-dic "$EVAL_RUNDIC" \ + --output results/eval_summary_${EVAL_RUNDIC}.md +``` + ## 常见坑 - Jupyter POST 请求缺少 `_xsrf`：从 cookie 取 `_xsrf`，请求头带 `X-XSRFToken`。 @@ -78,3 +134,5 @@ cml custom_train describe - `which cml` 为空：先 `source ~/.cloudml-cli/.profile`。 - `resolve_run_ids.sh` 没有 `RUNDIC`：只使用 `SFT_RUNDIC` / `EVAL_RUNDIC`。 - git clone 看似卡住：先检查目标目录是否已经完整、是否存在残留进程，不要重复提交训练。 +- 只给 branch 不给 commit：可训练，但必须回填实际 commit，否则结果不可复现。 +- 训练和评测不要放在同一个长 bash 里串到底；SFT `_SUCCESS` 落盘后再单独提交评测。 diff --git a/skills/model-training-lite/scripts/render_sft_submission_plan.py b/skills/model-training-lite/scripts/render_sft_submission_plan.py old mode 100644 new mode 100755 index d57c2d6..e15f851 --- a/skills/model-training-lite/scripts/render_sft_submission_plan.py +++ b/skills/model-training-lite/scripts/render_sft_submission_plan.py @@ -3,8 +3,10 @@ from __future__ import annotations import argparse import json +import os import re import sys +from pathlib import Path from pathlib import PurePosixPath from typing import Any @@ -26,14 +28,18 @@ def main() -> int: def render_plan(payload: dict[str, Any]) -> dict[str, Any]: + defaults = load_local_defaults() missing: list[str] = [] - jupyter_url = clean(payload.get("jupyter_url")) + jupyter_url = clean(payload.get("jupyter_url")) or clean(defaults.get("jupyter_url")) + jupyter_auth_type = clean(payload.get("jupyter_auth_type") or payload.get("auth_type")) or clean(defaults.get("auth_type")) + jupyter_password = clean(payload.get("jupyter_password") or payload.get("password")) or clean(defaults.get("password")) + jupyter_token = clean(payload.get("jupyter_token") or payload.get("token")) or clean(defaults.get("token")) data_commit = clean(payload.get("data_commit") or payload.get("commit")) data_path = clean(payload.get("data_path")) if not jupyter_url: missing.append("jupyter_url") - if not data_commit and not data_path: - missing.append("data_commit or data_path") + if jupyter_url and not (jupyter_password or jupyter_token or clean(payload.get("jupyter_cookie")) or clean(defaults.get("cookie"))): + missing.append("jupyter auth") owner = clean(payload.get("owner")) or "wuyang6" run_name = clean(payload.get("run_name")) or default_run_name(data_commit or data_path or "manual") @@ -43,6 +49,12 @@ def render_plan(payload: dict[str, Any]) -> dict[str, Any]: data_repo = clean(payload.get("data_repo")) or DEFAULT_DATA_REPO data_branch = clean(payload.get("data_branch")) or DEFAULT_BRANCH recipe = clean(payload.get("recipe")) or DEFAULT_RECIPE + run_eval = bool(payload.get("run_eval") or payload.get("eval_after_train")) + model_old = clean(payload.get("model_old")) or "/mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface" + wf_root = clean(payload.get("workflow_root")) or "/mnt/xiaoai-zk-model-train-tj5/workflow5" + warnings: list[str] = [] + if not data_commit and not data_path: + warnings.append("data_commit 未提供：计划将使用 data_branch 当前 HEAD，复现性弱于固定 commit。") commands = { "prepare_workspace": [ @@ -53,7 +65,11 @@ def render_plan(payload: dict[str, Any]) -> dict[str, Any]: f"[ -d ai-planning/.git ] || git clone -b {data_branch} {data_repo} ai-planning", f"git -C ai-planning fetch origin {data_branch}", f"git -C ai-planning checkout {data_branch}", - f"git -C ai-planning reset --hard {data_commit}" if data_commit else f"# use existing data_path: {data_path}", + f"git -C ai-planning reset --hard {data_commit}" if data_commit else ( + f"# use existing data_path: {data_path}" if data_path else + f"git -C ai-planning reset --hard origin/{data_branch}" + ), + "git -C ai-planning rev-parse HEAD", ], "submit_sft": [ "source ~/.cloudml-cli/.profile", @@ -67,21 +83,42 @@ def render_plan(payload: dict[str, Any]) -> dict[str, Any]: "cml custom_train describe ", f"test -f {workspace}/sft_output/_SUCCESS", ], + "submit_eval": [ + "source ~/.cloudml-cli/.profile", + f"export AUTORESEARCH_CHAT_ROOT={workspace}", + 'export AUTORESEARCH_ROOT="$AUTORESEARCH_CHAT_ROOT"', + f"export WF_ROOT={wf_root}", + f"cd {workspace}", + 'eval "$(./scripts/resolve_run_ids.sh)"', + f'./scripts/submit_cml_eval.sh "$EVAL_RUNDIC" "{workspace}/sft_output" "{model_old}"', + ], + "analyze_eval": [ + f"python {workspace}/scripts/summarize_eval_result.py --workflow-root {wf_root} --run-dic --output {workspace}/results/eval_summary_.md", + ], } return { "ok": not missing, "missing": missing, "plan": { "jupyter_url": jupyter_url, + "jupyter_auth": { + "type": jupyter_auth_type or ("token" if jupyter_token else "password" if jupyter_password else ""), + "configured": bool(jupyter_password or jupyter_token or clean(payload.get("jupyter_cookie")) or clean(defaults.get("cookie"))), + "source": auth_source(payload, defaults), + }, "recipe": recipe, "workspace": workspace, "data_repo": data_repo, "data_branch": data_branch, "data_commit": data_commit, "data_path": data_path, + "run_eval": run_eval, + "model_old": model_old, + "workflow_root": wf_root, "success_marker": f"{workspace}/sft_output/_SUCCESS", }, "commands": commands, + "warnings": warnings, "required_outputs": [ "CloudML JobID", "CloudML task URL", @@ -89,6 +126,8 @@ def render_plan(payload: dict[str, Any]) -> dict[str, Any]: "EVAL_RUNDIC", "workspace", "success_marker", + "eval workflow id when run_eval=true", + "eval summary path when analysis is requested", ], } @@ -97,6 +136,35 @@ def clean(value: Any) -> str: return str(value or "").strip() +def load_local_defaults() -> dict[str, Any]: + defaults: dict[str, Any] = {} + env_defaults = { + "jupyter_url": os.environ.get("MODEL_TRAINING_LITE_JUPYTER_URL"), + "password": os.environ.get("MODEL_TRAINING_LITE_JUPYTER_PASSWORD"), + "token": os.environ.get("MODEL_TRAINING_LITE_JUPYTER_TOKEN"), + "cookie": os.environ.get("MODEL_TRAINING_LITE_JUPYTER_COOKIE"), + "auth_type": os.environ.get("MODEL_TRAINING_LITE_JUPYTER_AUTH_TYPE"), + } + defaults.update({key: value for key, value in env_defaults.items() if value}) + config_path = Path(__file__).resolve().parents[1] / ".local" / "jupyter_defaults.json" + if config_path.exists(): + try: + file_defaults = json.loads(config_path.read_text(encoding="utf-8")) + if isinstance(file_defaults, dict): + defaults.update({key: value for key, value in file_defaults.items() if value}) + except json.JSONDecodeError as exc: + raise SystemExit(f"invalid local defaults JSON: {config_path}: {exc}") from exc + return defaults + + +def auth_source(payload: dict[str, Any], defaults: dict[str, Any]) -> str: + if any(clean(payload.get(key)) for key in ("jupyter_password", "password", "jupyter_token", "token", "jupyter_cookie")): + return "input" + if any(clean(defaults.get(key)) for key in ("password", "token", "cookie")): + return "local_default" + return "" + + def default_run_name(seed: str) -> str: slug = re.sub(r"[^a-zA-Z0-9_-]+", "_", seed).strip("_")[:32] or "manual" return f"manual_sft_{slug}" diff --git a/skills/model-training-lite/scripts/summarize_eval_result.py b/skills/model-training-lite/scripts/summarize_eval_result.py new file mode 100755 index 0000000..5738a73 --- /dev/null +++ b/skills/model-training-lite/scripts/summarize_eval_result.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +from collections import Counter +from pathlib import Path +from typing import Any + + +def main() -> int: + parser = argparse.ArgumentParser(description="Summarize a CML eval workflow metric_diff directory.") + parser.add_argument("--workflow-root", default="/mnt/xiaoai-zk-model-train-tj5/workflow5") + parser.add_argument("--run-dic", required=True, help="workflow runDic, for example 18065") + parser.add_argument("--output", help="Markdown output path. Prints to stdout when omitted.") + parser.add_argument("--max-error-examples", type=int, default=30) + args = parser.parse_args() + + workflow_dir = Path(args.workflow_root) / f"workflow{args.run_dic}" + metric_dir = workflow_dir / "metric_diff" + result = summarize(metric_dir, args.max_error_examples) + text = render_markdown(args.run_dic, workflow_dir, result) + if args.output: + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text, encoding="utf-8") + else: + print(text) + return 0 if result["ok"] else 1 + + +def summarize(metric_dir: Path, max_error_examples: int) -> dict[str, Any]: + lark_path = metric_dir / "lark_template.json" + comparison_path = metric_dir / "specific_comparison.csv" + result: dict[str, Any] = { + "ok": True, + "metric_dir": str(metric_dir), + "missing": [], + "metrics": [], + "specific_rows": 0, + "error_summary": [], + "error_examples": [], + } + if not lark_path.exists(): + result["missing"].append(str(lark_path)) + else: + result["metrics"] = extract_lark_metrics(load_json(lark_path)) + if comparison_path.exists(): + rows = read_csv(comparison_path) + result["specific_rows"] = len(rows) + result["error_summary"] = summarize_comparison(rows) + result["error_examples"] = select_error_examples(rows, max_error_examples) + else: + result["missing"].append(str(comparison_path)) + result["ok"] = not result["missing"] + return result + + +def load_json(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def extract_lark_metrics(obj: Any) -> list[dict[str, Any]]: + metrics: list[dict[str, Any]] = [] + + def walk(value: Any, path: list[str]) -> None: + if isinstance(value, dict): + maybe_name = value.get("name") or value.get("title") or value.get("sub_cate") or value.get("cate") + maybe_acc = first_present(value, ["acc", "accuracy", "new_acc", "dev_acc", "rate", "pass_rate"]) + maybe_total = first_present(value, ["total", "count", "all"]) + maybe_right = first_present(value, ["right", "correct", "hit"]) + if maybe_acc is not None or (maybe_total is not None and maybe_right is not None): + metrics.append( + { + "path": " / ".join(path + ([str(maybe_name)] if maybe_name else [])), + "acc": maybe_acc, + "right": maybe_right, + "total": maybe_total, + } + ) + for key, child in value.items(): + if isinstance(child, (dict, list)): + walk(child, path + [str(key)]) + elif isinstance(value, list): + for idx, child in enumerate(value): + if isinstance(child, (dict, list)): + walk(child, path + [str(idx)]) + + walk(obj, []) + # De-duplicate while preserving order. + seen: set[tuple[str, str, str, str]] = set() + uniq: list[dict[str, Any]] = [] + for item in metrics: + key = tuple(str(item.get(k, "")) for k in ("path", "acc", "right", "total")) + if key not in seen: + seen.add(key) + uniq.append(item) + return uniq[:80] + + +def first_present(obj: dict[str, Any], keys: list[str]) -> Any: + for key in keys: + if key in obj and obj[key] not in ("", None): + return obj[key] + return None + + +def read_csv(path: Path) -> list[dict[str, str]]: + for encoding in ("utf-8-sig", "utf-8", "gb18030"): + try: + with path.open(encoding=encoding, newline="") as handle: + return list(csv.DictReader(handle)) + except UnicodeDecodeError: + continue + raise UnicodeDecodeError("csv", b"", 0, 1, f"cannot decode {path}") + + +def summarize_comparison(rows: list[dict[str, str]]) -> list[dict[str, Any]]: + if not rows: + return [] + fieldnames = set(rows[0]) + cate_field = pick_field(fieldnames, ["sub_cate", "子集", "cate", "类别"]) + gold_field = pick_field(fieldnames, ["code_label", "gold", "label", "类别真实标签", "code标签"]) + pred_field = pick_field(fieldnames, ["origin_predict_dev", "predict_dev", "prediction", "pred", "模型输出"]) + if not cate_field: + cate_field = "__all__" + counter: Counter[tuple[str, str, str]] = Counter() + for row in rows: + cate = row.get(cate_field, "ALL") if cate_field != "__all__" else "ALL" + gold = row.get(gold_field, "") if gold_field else "" + pred = row.get(pred_field, "") if pred_field else "" + is_error = detect_error(row) + if is_error: + counter[(cate, gold[:80], pred[:80])] += 1 + return [ + {"sub_cate": cate, "gold": gold, "pred": pred, "count": count} + for (cate, gold, pred), count in counter.most_common(30) + ] + + +def detect_error(row: dict[str, str]) -> bool: + for key in ("is_correct", "correct", "是否正确", "same", "is_same"): + value = row.get(key) + if value is None: + continue + text = str(value).strip().lower() + if text in {"false", "0", "错", "错误", "no", "n"}: + return True + if text in {"true", "1", "对", "正确", "yes", "y"}: + return False + base = row.get("origin_predict_base") or row.get("predict_base") or "" + dev = row.get("origin_predict_dev") or row.get("predict_dev") or row.get("prediction") or "" + gold = row.get("code_label") or row.get("gold") or row.get("code标签") or "" + return bool(gold and dev and normalize_label(gold) != normalize_label(dev)) or bool(base and dev and base != dev) + + +def normalize_label(text: str) -> str: + return "".join(str(text).split()).replace("'", '"') + + +def select_error_examples(rows: list[dict[str, str]], limit: int) -> list[dict[str, str]]: + examples: list[dict[str, str]] = [] + for row in rows: + if not detect_error(row): + continue + examples.append( + { + "query": row.get("query") or row.get("当前query") or row.get("current_query") or "", + "sub_cate": row.get("sub_cate") or row.get("子集") or "", + "gold": row.get("code_label") or row.get("gold") or row.get("code标签") or "", + "pred": row.get("origin_predict_dev") or row.get("predict_dev") or row.get("prediction") or "", + } + ) + if len(examples) >= limit: + break + return examples + + +def pick_field(fieldnames: set[str], candidates: list[str]) -> str: + for candidate in candidates: + if candidate in fieldnames: + return candidate + return "" + + +def render_markdown(run_dic: str, workflow_dir: Path, result: dict[str, Any]) -> str: + lines: list[str] = [ + f"# workflow{run_dic} 评测摘要", + "", + f"- workflow_dir: `{workflow_dir}`", + f"- metric_dir: `{result['metric_dir']}`", + ] + if result["missing"]: + lines.append(f"- missing: `{', '.join(result['missing'])}`") + lines += ["", "## 指标摘录", ""] + if result["metrics"]: + lines.append("| path | acc | right | total |") + lines.append("|---|---:|---:|---:|") + for item in result["metrics"]: + lines.append( + f"| {safe_cell(item.get('path'))} | {safe_cell(item.get('acc'))} | {safe_cell(item.get('right'))} | {safe_cell(item.get('total'))} |" + ) + else: + lines.append("未解析到 lark_template 指标。") + lines += ["", "## 错误分布 Top", ""] + if result["error_summary"]: + lines.append("| sub_cate | gold | pred | count |") + lines.append("|---|---|---|---:|") + for item in result["error_summary"]: + lines.append( + f"| {safe_cell(item['sub_cate'])} | {safe_cell(item['gold'])} | {safe_cell(item['pred'])} | {item['count']} |" + ) + else: + lines.append("未解析到错误分布。") + lines += ["", "## 错误样例", ""] + if result["error_examples"]: + lines.append("| sub_cate | query | gold | pred |") + lines.append("|---|---|---|---|") + for item in result["error_examples"]: + lines.append( + f"| {safe_cell(item['sub_cate'])} | {safe_cell(item['query'])} | {safe_cell(item['gold'])} | {safe_cell(item['pred'])} |" + ) + else: + lines.append("未抽到错误样例。") + lines.append("") + return "\n".join(lines) + + +def safe_cell(value: Any) -> str: + text = str(value or "").replace("\n", "
").replace("|", "\\|") + return text[:500] + + +if __name__ == "__main__": + raise SystemExit(main())