武阳
105 lines
3.1 KiB
Bash
105 lines
3.1 KiB
Bash
#!/bin/bash
|
||
# 用 cml custom_train submit 提交 SFT 训练，训练完成后自动起 cml workflow run 评测
|
||
# 用法: ./submit_sft_via_cml.sh <RUNDIC>
|
||
# 例: ./submit_sft_via_cml.sh 17756
|
||
|
||
set -euo pipefail
|
||
|
||
RUNDIC=${1:?usage: $0 <RUNDIC>}
|
||
PREV_RUNDIC=${2:-$((RUNDIC-1))}
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ROOT=${AUTORESEARCH_ROOT:-/mnt/wangsenhao/autoresearch-zk}
|
||
MODEL_OLD=${MODEL_OLD:-/mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface}
|
||
TPL=${SFT_TRAIN_JOB_TEMPLATE:-$SCRIPT_DIR/sft_train_job.yaml.tpl}
|
||
YAML=/tmp/sft_train_job_r${RUNDIC}.yaml
|
||
|
||
if [ -f ~/.cloudml-cli/.profile ]; then
|
||
source ~/.cloudml-cli/.profile
|
||
else
|
||
echo "[cml-sft] ❌ 未找到 ~/.cloudml-cli/.profile，请先安装并初始化 cml" >&2
|
||
exit 1
|
||
fi
|
||
|
||
# 1. 渲染 yaml 模板
|
||
sed \
|
||
-e "s|{RUNDIC}|${RUNDIC}|g" \
|
||
-e "s|{PREV_RUNDIC}|${PREV_RUNDIC}|g" \
|
||
-e "s|{AUTORESEARCH_ROOT}|${ROOT}|g" \
|
||
"$TPL" > "$YAML"
|
||
echo "[cml-sft] yaml: $YAML"
|
||
|
||
# 2. 提交训练任务
|
||
SUBMIT_OUT=$(cml custom_train submit --filename "$YAML" 2>&1)
|
||
echo "$SUBMIT_OUT"
|
||
JOB_ID=$(echo "$SUBMIT_OUT" | grep -oE 't-[0-9]+-[a-z0-9]+' | head -1)
|
||
if [ -z "$JOB_ID" ]; then
|
||
echo "[cml-sft] ❌ 提交失败" >&2
|
||
exit 1
|
||
fi
|
||
echo "[cml-sft] ✅ JobID: $JOB_ID"
|
||
|
||
# 3. 后台 watcher：等训练成功 → 自动起评测
|
||
WATCHER_LOG=/tmp/r${RUNDIC}_logs/cml_watcher.log
|
||
mkdir -p "$(dirname "$WATCHER_LOG")"
|
||
|
||
nohup bash -c '
|
||
JOB_ID='"$JOB_ID"'
|
||
RUNDIC='"$RUNDIC"'
|
||
LOG='"$WATCHER_LOG"'
|
||
ROOT='"$ROOT"'
|
||
MODEL_NEW="$ROOT/sft_output"
|
||
MODEL_OLD='"$MODEL_OLD"'
|
||
EVAL_WORKFLOW_ID=f-20260408161444-wu3pz
|
||
EVAL_VERSION=v28
|
||
|
||
source ~/.cloudml-cli/.profile
|
||
|
||
echo "[$(date)] 等待 cml job $JOB_ID 完成..." >> "$LOG"
|
||
while true; do
|
||
STATE=$(cml custom_train describe "$JOB_ID" 2>/dev/null | grep -o "\"state\": \"[a-z]*\"" | head -1 | sed "s/.*: \"//;s/\"//")
|
||
case "$STATE" in
|
||
succeed)
|
||
echo "[$(date)] 训练成功" >> "$LOG"
|
||
break ;;
|
||
failed|killed)
|
||
echo "[$(date)] ❌ 训练 $STATE" >> "$LOG"
|
||
exit 1 ;;
|
||
*)
|
||
sleep 60 ;;
|
||
esac
|
||
done
|
||
|
||
# 验证产出
|
||
[ ! -f "$MODEL_NEW/_SUCCESS" ] && [ ! -f "$MODEL_NEW/config.json" ] && {
|
||
echo "[$(date)] ❌ 产出缺失" >> "$LOG"; exit 1
|
||
}
|
||
|
||
# 起评测
|
||
echo "[$(date)] 启动 CML 评测 runDic=$RUNDIC" >> "$LOG"
|
||
cml workflow run \
|
||
--workflow_id $EVAL_WORKFLOW_ID --version $EVAL_VERSION \
|
||
--global_inputs runDic=$RUNDIC \
|
||
--global_inputs model_path_new=$MODEL_NEW \
|
||
--global_inputs model_path_old=$MODEL_OLD >> "$LOG" 2>&1
|
||
echo "[$(date)] cml workflow run 提交完成" >> "$LOG"
|
||
' > /tmp/r${RUNDIC}_logs/watcher_runner.log 2>&1 &
|
||
|
||
WATCHER_PID=$!
|
||
echo "[cml-sft] watcher PID: $WATCHER_PID（log: $WATCHER_LOG）"
|
||
|
||
# 4. 立即输出可查看命令
|
||
cat <<EOF
|
||
|
||
[cml-sft] 任务提交完成，关键命令：
|
||
查看任务状态: cml custom_train describe $JOB_ID
|
||
查看实时日志: cml custom_train logs $JOB_ID --follow
|
||
停止任务: cml custom_train kill $JOB_ID
|
||
watcher log: tail -f $WATCHER_LOG
|
||
|
||
训练完成后，evaluation workflow 会自动启动；评测产物在
|
||
/mnt/xiaoai-zk-model-train-tj5/workflow5/workflow${RUNDIC}/
|
||
|
||
JobID: $JOB_ID
|
||
RUNDIC: $RUNDIC
|
||
EOF
|