model-iteration: runDic 单一可信源 + label-master 逐条调用

- 新增 scripts/resolve_run_ids.sh：从 workflow5/ 解析出 SFT_RUNDIC / EVAL_RUNDIC，禁止 agent 心算 +1 - submit_sft_via_cml.sh 拆成 <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC] 三参，yaml 用 SFT_RUNDIC，cml workflow run 用 EVAL_RUNDIC - program.md §746 / §5.2 / §1.2 同步约束，并加 R2 落盘多 +1 的踩坑案例 - §4.0.1 Step A.5 / §4.5 label-master 复核改逐条调用（≤8 路并发），禁止批量塞多条 query Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-25 11:42:44 +08:00
parent 362321c460
commit 57f5b60a3e
6 changed files with 606 additions and 74 deletions
@@ -26,7 +26,8 @@ from pathlib import Path
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)

-AI_PLANNING = Path(__file__).resolve().parent / "ai-planning"
+CHAT_ROOT = Path(__file__).resolve().parent.parent
+AI_PLANNING = CHAT_ROOT / "ai-planning"
 TRAIN_SET_DIR = AI_PLANNING / "data" / "train_set"
 DATA_TRAIN_DIR = AI_PLANNING / "data_train"
 GENERATE_TRAIN_SCRIPT = DATA_TRAIN_DIR / "generate_train.py"
@@ -148,7 +149,7 @@ def run_sft_training(data_dir: str, model_path: str, model_type: str, train_outp
                     epochs: int = 3, lr: float = 1e-5, max_seq_length: int = 1024,
                     per_device_batch: int = 1, grad_accum: int = 1):
    """基于 zk_trainer 的 accelerate launch 方式启动 SFT 训练。"""
-    zk_trainer_dir = str(Path(__file__).resolve().parent / "zk_trainer")
+    zk_trainer_dir = str(CHAT_ROOT / "zk_trainer")
    if not os.path.isdir(zk_trainer_dir):
        log.error(f"zk_trainer 目录不存在: {zk_trainer_dir}")
        return None
@@ -0,0 +1,42 @@
+#!/bin/bash
+# resolve_run_ids.sh —— SFT/EVAL runDic 单一可信来源
+#
+# 规则（与 program.md §746 + §5.2 对齐）：
+#   SFT_RUNDIC  = workflow5/ 下「已落盘」（含 metric_diff/lark_template.json）的最大 runDic
+#                 即 R{n-1}.runDic —— augment 落盘、SFT yaml、modified_samples 归档全用这个值
+#   EVAL_RUNDIC = SFT_RUNDIC + 1
+#                 仅 cml workflow run 提交本轮评测时使用
+#
+# 用法：
+#   eval "$(./scripts/resolve_run_ids.sh)"
+#   echo "$SFT_RUNDIC $EVAL_RUNDIC"
+#
+# 失败行为：找不到任何已落盘 workflow 时打 ERR 到 stderr 并 exit 1，调用方 set -e 时直接终止。
+
+set -euo pipefail
+
+WF_ROOT=${WF_ROOT:-/mnt/xiaoai-zk-model-train-tj5/workflow5}
+
+if [ ! -d "$WF_ROOT" ]; then
+  echo "ERR: WF_ROOT=$WF_ROOT 不存在或不可读" >&2
+  exit 1
+fi
+
+# 只挑「已落盘」的 workflow：必须存在 metric_diff/lark_template.json
+# 这样可以避免 CML 创建了空目录就被算成最大值
+MAX=$(
+  for d in "$WF_ROOT"/workflow*; do
+    [ -d "$d" ] || continue
+    [ -f "$d/metric_diff/lark_template.json" ] || continue
+    name=$(basename "$d")
+    echo "${name#workflow}"
+  done | sort -n | tail -1
+)
+
+if [ -z "$MAX" ]; then
+  echo "ERR: $WF_ROOT 下没有任何已落盘的 workflow（含 metric_diff/lark_template.json）" >&2
+  exit 1
+fi
+
+echo "SFT_RUNDIC=$MAX"
+echo "EVAL_RUNDIC=$((MAX+1))"
@@ -0,0 +1,117 @@
+jobName: "sft-train-r{RUNDIC}"
+description: "AutoResearch R{RUNDIC} SFT training (40k+ samples, qwen3-4B base, 3 epochs FSDP2)"
+accessType: PUBLIC
+
+imageConfig:
+  imageUrl: micr.cloud.mioffice.cn/vllm-image/ai-arch-llm-prod:vllm-v0.12.0-f098b188
+  imageCommand: |-
+    set -ex
+    cd {AUTORESEARCH_ROOT}
+
+    # 0a. 验环境 + 装缺的包（vllm 镜像 py3.12 + torch 已含）
+    python3 --version
+    python3 -c "import torch; print(f'torch={torch.__version__}')"
+    pip install -q --no-deps "accelerate==1.7.0" 2>&1 | tail -3
+    pip install -q --ignore-installed blinker 2>&1 | tail -2
+    pip install -q peft 2>&1 | tail -3
+    pip install -q wandb bitsandbytes 2>&1 | tail -3
+    pip install -q luigi mlflow scikit-learn openpyxl pyyaml sentencepiece tiktoken protobuf pynvml datasets 2>&1 | tail -3
+    pip install -q "transformers>=4.45" 2>&1 | tail -3
+    python3 -c 'import torch, accelerate, transformers, peft; print(torch.__version__, accelerate.__version__, transformers.__version__, peft.__version__)'
+
+    # 0c. 把 HF cache 重定向到容器本地大盘（默认在 ~/.cache 可能是 juicefs，mmap 易 SIGBUS）
+    export HF_HOME=/tmp/hf_cache
+    export HF_DATASETS_CACHE=/tmp/hf_cache/datasets
+    export TRANSFORMERS_CACHE=/tmp/hf_cache/transformers
+    mkdir -p /tmp/hf_cache/datasets /tmp/hf_cache/transformers
+    df -h /tmp /dev/shm 2>/dev/null || true
+
+    # 强制 datasets 加载进内存而不是 arrow mmap（避免 /dev/shm 溢出 SIGBUS）
+    export HF_DATASETS_IN_MEMORY_MAX_SIZE=20000000000
+    export HF_DATASETS_NUM_PROC=1
+    export TOKENIZERS_PARALLELISM=false
+    export OMP_NUM_THREADS=1
+    export MKL_NUM_THREADS=1
+    # 关闭 NCCL 用 shm 通信（改 socket 通道）
+    export NCCL_SHM_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_DEBUG=WARN
+
+    # 0b. 把上一轮产出搬走（cml job 内幂等，本地搬过就跳过）
+    if [ -d sft_output ] && [ ! -d sft_output_r{PREV_RUNDIC} ]; then
+      mv sft_output sft_output_r{PREV_RUNDIC}
+    fi
+    rm -rf sft_output
+
+    # 1. 组装数据
+    python3 {AUTORESEARCH_ROOT}/prepare_and_train_sft.py prepare \
+      --output_dir {AUTORESEARCH_ROOT}/sft_data
+
+    # 2. 启动训练（绝对路径，避免相对路径在容器内找不到 zk_trainer）
+    python3 {AUTORESEARCH_ROOT}/prepare_and_train_sft.py train \
+      --data_dir {AUTORESEARCH_ROOT}/sft_data \
+      --model_path /mnt/wangsenhao/verl_zk/Qwen3-4B-Instruct-2507 \
+      --model_type qwen3 \
+      --train_output {AUTORESEARCH_ROOT}/sft_output \
+      --epochs 3 --lr 1e-5
+
+    # 3. 训练成功标记（被 watcher 检测）
+    [ -f {AUTORESEARCH_ROOT}/sft_output/config.json ] && \
+      touch {AUTORESEARCH_ROOT}/sft_output/_SUCCESS
+
+# 挂载 wangsenhao + xiaoai-zk-model-train-tj5 + verl_zk 所在卷
+juiceFsMountConfigs:
+  - volume: wangsenhao
+    juiceFsCluster: tj5-common
+    subPath: /
+    mountPath: /mnt/wangsenhao
+    readOnly: false
+  - volume: xiaoai-zk-model-train-tj5
+    juiceFsCluster: tj5-common
+    subPath: /
+    mountPath: /mnt/xiaoai-zk-model-train-tj5
+    readOnly: false
+
+envConfigs:
+  - key: PYTHONPATH
+    value: {AUTORESEARCH_ROOT}/zk_trainer
+  - key: WANDB_DISABLED
+    value: "true"
+  - key: WANDB_MODE
+    value: offline
+  - key: VOLUME_PREFIX
+    value: /mnt/xiaoai-zk-model-train-tj5
+  - key: HF_HOME
+    value: /mnt/wangsenhao/.hf_cache
+
+# h20-96g 8 卡 FSDP2
+queueId: "6052"
+priority: 5
+preemptible: false
+framework: pytorch
+resourceConfigs:
+  - nodeRole: worker
+    nodeNumber: 1
+    perNodeResourceSpec:
+      resourcePriority: GUARANTEED
+      resourceName: cloudml.ng2h20-8-8.20-199
+      resourceNumber: 8
+
+# 故障自动重试（节点级失败）
+retryConfig:
+  enableRetry: true
+  maxRetryTimes: 2
+  policySets:
+    - NodeFailure
+
+# 失败/完成飞书告警
+alertConfig:
+  enableAlert: true
+  alertItems:
+    - alertConditions:
+        - FAILED
+        - SUCCEED
+      alertLevel: P2
+      alertReceivers:
+        persons:
+          - wangsenhao
@@ -0,0 +1,124 @@
+#!/bin/bash
+# 用 cml custom_train submit 提交 SFT 训练，训练完成后自动起 cml workflow run 评测
+#
+# 用法：
+#   ./submit_sft_via_cml.sh <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC]
+#
+# 推荐调用方式（runDic 由 resolve_run_ids.sh 决定，不要手敲）：
+#   eval "$(./scripts/resolve_run_ids.sh)"
+#   ./scripts/submit_sft_via_cml.sh "$SFT_RUNDIC" "$EVAL_RUNDIC"
+#
+# 语义（与 program.md §746 + §5.2 对齐）：
+#   SFT_RUNDIC  = R{n-1}.runDic（augment / yaml / 归档全用这个，不 +1）
+#   EVAL_RUNDIC = R{n-1}.runDic + 1（仅本步起 eval 时使用，仅在这一处 +1）
+#   PREV_RUNDIC = SFT_RUNDIC - 1（默认；只用作上一轮 sft_output 改名）
+
+set -euo pipefail
+
+SFT_RUNDIC=${1:?usage: $0 <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC]}
+EVAL_RUNDIC=${2:?usage: $0 <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC]}
+PREV_RUNDIC=${3:-$((SFT_RUNDIC-1))}
+
+# 防御：EVAL_RUNDIC 必须严格大于 SFT_RUNDIC，否则一定是调用方算错
+if [ "$EVAL_RUNDIC" -le "$SFT_RUNDIC" ]; then
+  echo "[cml-sft] ❌ EVAL_RUNDIC ($EVAL_RUNDIC) 必须 > SFT_RUNDIC ($SFT_RUNDIC)" >&2
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT=${AUTORESEARCH_ROOT:-/mnt/wangsenhao/autoresearch-zk}
+MODEL_OLD=${MODEL_OLD:-/mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface}
+TPL=${SFT_TRAIN_JOB_TEMPLATE:-$SCRIPT_DIR/sft_train_job.yaml.tpl}
+YAML=/tmp/sft_train_job_r${SFT_RUNDIC}.yaml
+
+if [ -f ~/.cloudml-cli/.profile ]; then
+  source ~/.cloudml-cli/.profile
+else
+  echo "[cml-sft] ❌ 未找到 ~/.cloudml-cli/.profile，请先安装并初始化 cml" >&2
+  exit 1
+fi
+
+# 1. 渲染 yaml 模板（用 SFT_RUNDIC，不是 EVAL_RUNDIC）
+sed \
+  -e "s|{RUNDIC}|${SFT_RUNDIC}|g" \
+  -e "s|{PREV_RUNDIC}|${PREV_RUNDIC}|g" \
+  -e "s|{AUTORESEARCH_ROOT}|${ROOT}|g" \
+  "$TPL" > "$YAML"
+echo "[cml-sft] yaml: $YAML  (SFT_RUNDIC=$SFT_RUNDIC EVAL_RUNDIC=$EVAL_RUNDIC PREV_RUNDIC=$PREV_RUNDIC)"
+
+# 2. 提交训练任务
+SUBMIT_OUT=$(cml custom_train submit --filename "$YAML" 2>&1)
+echo "$SUBMIT_OUT"
+JOB_ID=$(echo "$SUBMIT_OUT" | grep -oE 't-[0-9]+-[a-z0-9]+' | head -1)
+if [ -z "$JOB_ID" ]; then
+  echo "[cml-sft] ❌ 提交失败" >&2
+  exit 1
+fi
+echo "[cml-sft] ✅ JobID: $JOB_ID"
+
+# 3. 后台 watcher：等训练成功 → 自动起评测（用 EVAL_RUNDIC）
+WATCHER_LOG=/tmp/r${SFT_RUNDIC}_logs/cml_watcher.log
+mkdir -p "$(dirname "$WATCHER_LOG")"
+
+nohup bash -c '
+JOB_ID='"$JOB_ID"'
+SFT_RUNDIC='"$SFT_RUNDIC"'
+EVAL_RUNDIC='"$EVAL_RUNDIC"'
+LOG='"$WATCHER_LOG"'
+ROOT='"$ROOT"'
+MODEL_NEW="$ROOT/sft_output"
+MODEL_OLD='"$MODEL_OLD"'
+EVAL_WORKFLOW_ID=f-20260408161444-wu3pz
+EVAL_VERSION=v28
+
+source ~/.cloudml-cli/.profile
+
+echo "[$(date)] 等待 cml job $JOB_ID 完成..." >> "$LOG"
+while true; do
+  STATE=$(cml custom_train describe "$JOB_ID" 2>/dev/null | grep -o "\"state\": \"[a-z]*\"" | head -1 | sed "s/.*: \"//;s/\"//")
+  case "$STATE" in
+    succeed)
+      echo "[$(date)] 训练成功" >> "$LOG"
+      break ;;
+    failed|killed)
+      echo "[$(date)] ❌ 训练 $STATE" >> "$LOG"
+      exit 1 ;;
+    *)
+      sleep 60 ;;
+  esac
+done
+
+# 验证产出
+[ ! -f "$MODEL_NEW/_SUCCESS" ] && [ ! -f "$MODEL_NEW/config.json" ] && {
+  echo "[$(date)] ❌ 产出缺失" >> "$LOG"; exit 1
+}
+
+# 起评测（这里且仅这里用 EVAL_RUNDIC）
+echo "[$(date)] 启动 CML 评测 EVAL_RUNDIC=$EVAL_RUNDIC" >> "$LOG"
+cml workflow run \
+  --workflow_id $EVAL_WORKFLOW_ID --version $EVAL_VERSION \
+  --global_inputs runDic=$EVAL_RUNDIC \
+  --global_inputs model_path_new=$MODEL_NEW \
+  --global_inputs model_path_old=$MODEL_OLD >> "$LOG" 2>&1
+echo "[$(date)] cml workflow run 提交完成 (workflow$EVAL_RUNDIC)" >> "$LOG"
+' > /tmp/r${SFT_RUNDIC}_logs/watcher_runner.log 2>&1 &
+
+WATCHER_PID=$!
+echo "[cml-sft] watcher PID: $WATCHER_PID（log: $WATCHER_LOG）"
+
+# 4. 立即输出可查看命令
+cat <<EOF
+
+[cml-sft] 任务提交完成，关键命令：
+  查看任务状态:  cml custom_train describe $JOB_ID
+  查看实时日志:  cml custom_train logs $JOB_ID --follow
+  停止任务:      cml custom_train kill $JOB_ID
+  watcher log:   tail -f $WATCHER_LOG
+
+训练完成后，evaluation workflow 会自动启动；评测产物在
+  /mnt/xiaoai-zk-model-train-tj5/workflow5/workflow${EVAL_RUNDIC}/
+
+JobID:        $JOB_ID
+SFT_RUNDIC:   $SFT_RUNDIC   (yaml / sft_output 命名)
+EVAL_RUNDIC:  $EVAL_RUNDIC  (workflow 评测目录)
+EOF