refactor: split submit_sft_via_cml.sh into submit_sft.sh + submit_cml_eval.sh
- submit_sft.sh: pure SFT submission, prints JobID and exits (no embedded watcher) - submit_cml_eval.sh: pure CML eval, reads workflow_id/version from config.yaml - jupyter_runtime write_text: rewrite to use Contents API instead of terminal websocket - SKILL.md: restrict trigger to explicit UI click only - program.md: update §5.2 docs to reflect two-script workflow Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
hupenglong1
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
# 起 CML evaluation workflow（**只做这一件事**）。
|
||||
# workflow_id / version 都从 skills/model-iteration/assets/config.yaml 读，不在这里硬编码。
|
||||
#
|
||||
# 用法：
|
||||
# ./submit_cml_eval.sh <EVAL_RUNDIC> [MODEL_NEW] [MODEL_OLD]
|
||||
#
|
||||
# 默认值：
|
||||
# MODEL_NEW = $AUTORESEARCH_ROOT/sft_output（最新一轮 SFT 产物）
|
||||
# MODEL_OLD = /mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
EVAL_RUNDIC=${1:?usage: $0 <EVAL_RUNDIC> [MODEL_NEW] [MODEL_OLD]}
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT=${AUTORESEARCH_ROOT:-/mnt/wangsenhao/autoresearch-zk}
|
||||
MODEL_NEW=${2:-${MODEL_NEW:-$ROOT/sft_output}}
|
||||
MODEL_OLD=${3:-${MODEL_OLD:-/mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface}}
|
||||
|
||||
# 定位 config.yaml：先看脚本旁边的 ../assets/，再看 skills/ 树
|
||||
find_config() {
|
||||
local candidates=(
|
||||
"$SCRIPT_DIR/../assets/config.yaml"
|
||||
"${ZK_AGENT_SKILLS_ROOT:-}/model-iteration/assets/config.yaml"
|
||||
"${AUTORESEARCH_CHAT_ROOT:-}/skills/model-iteration/assets/config.yaml"
|
||||
)
|
||||
local c
|
||||
for c in "${candidates[@]}"; do
|
||||
if [ -n "$c" ] && [ -f "$c" ]; then
|
||||
echo "$c"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
CONFIG=$(find_config) || {
|
||||
echo "[eval] ❌ 找不到 config.yaml（尝试过：$SCRIPT_DIR/../assets/、\$ZK_AGENT_SKILLS_ROOT/model-iteration/assets/、\$AUTORESEARCH_CHAT_ROOT/skills/...）" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 极简 yaml 解析：只提 cml_eval.workflow_id / cml_eval.version（与后端 _read_workflow_version 风格保持一致）
|
||||
read_cml_eval_field() {
|
||||
local field=$1
|
||||
awk -v field="$field" '
|
||||
/^[^[:space:]#]/ { in_cml = ($1 == "cml_eval:") }
|
||||
in_cml && $1 == field { gsub(/^["\x27]|["\x27]$/, "", $2); print $2; exit }
|
||||
' "$CONFIG"
|
||||
}
|
||||
|
||||
EVAL_WORKFLOW_ID=$(read_cml_eval_field "workflow_id:")
|
||||
EVAL_VERSION=$(read_cml_eval_field "version:")
|
||||
|
||||
if [ -z "$EVAL_WORKFLOW_ID" ] || [ -z "$EVAL_VERSION" ]; then
|
||||
echo "[eval] ❌ 从 $CONFIG 读到的 cml_eval 不完整：workflow_id='$EVAL_WORKFLOW_ID' version='$EVAL_VERSION'" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -f ~/.cloudml-cli/.profile ]; then
|
||||
source ~/.cloudml-cli/.profile
|
||||
else
|
||||
echo "[eval] ❌ 未找到 ~/.cloudml-cli/.profile，请先安装并初始化 cml" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[eval] EVAL_RUNDIC=$EVAL_RUNDIC workflow_id=$EVAL_WORKFLOW_ID version=$EVAL_VERSION (config: $CONFIG)"
|
||||
echo "[eval] MODEL_NEW=$MODEL_NEW"
|
||||
echo "[eval] MODEL_OLD=$MODEL_OLD"
|
||||
|
||||
cml workflow run \
|
||||
--workflow_id "$EVAL_WORKFLOW_ID" --version "$EVAL_VERSION" \
|
||||
--global_inputs runDic="$EVAL_RUNDIC" \
|
||||
--global_inputs model_path_new="$MODEL_NEW" \
|
||||
--global_inputs model_path_old="$MODEL_OLD"
|
||||
|
||||
echo "[eval] ✅ 已提交 workflow$EVAL_RUNDIC，产物在 /mnt/xiaoai-zk-model-train-tj5/workflow5/workflow${EVAL_RUNDIC}/"
|
||||
Executable
+63
@@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
# 用 cml custom_train submit 提交 SFT 训练（**只做这一件事**）。
|
||||
# 训练完成后请用 scripts/submit_cml_eval.sh 单独起评测，watcher 由 agent 自己挂。
|
||||
#
|
||||
# 用法：
|
||||
# ./submit_sft.sh <SFT_RUNDIC> [PREV_RUNDIC]
|
||||
#
|
||||
# 推荐调用方式（runDic 由 resolve_run_ids.sh 决定，不要手敲）：
|
||||
# eval "$(./scripts/resolve_run_ids.sh)"
|
||||
# ./scripts/submit_sft.sh "$SFT_RUNDIC"
|
||||
#
|
||||
# 语义（与 program.md §746 + §5.2 对齐）：
|
||||
# SFT_RUNDIC = R{n-1}.runDic（augment / yaml / 归档全用这个，不 +1）
|
||||
# PREV_RUNDIC = SFT_RUNDIC - 1（默认；只用作上一轮 sft_output 改名）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SFT_RUNDIC=${1:?usage: $0 <SFT_RUNDIC> [PREV_RUNDIC]}
|
||||
PREV_RUNDIC=${2:-$((SFT_RUNDIC-1))}
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT=${AUTORESEARCH_ROOT:-/mnt/wangsenhao/autoresearch-zk}
|
||||
TPL=${SFT_TRAIN_JOB_TEMPLATE:-$SCRIPT_DIR/sft_train_job.yaml.tpl}
|
||||
YAML=/tmp/sft_train_job_r${SFT_RUNDIC}.yaml
|
||||
|
||||
if [ -f ~/.cloudml-cli/.profile ]; then
|
||||
source ~/.cloudml-cli/.profile
|
||||
else
|
||||
echo "[sft] ❌ 未找到 ~/.cloudml-cli/.profile，请先安装并初始化 cml" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 1. 渲染 yaml 模板
|
||||
sed \
|
||||
-e "s|{RUNDIC}|${SFT_RUNDIC}|g" \
|
||||
-e "s|{PREV_RUNDIC}|${PREV_RUNDIC}|g" \
|
||||
-e "s|{AUTORESEARCH_ROOT}|${ROOT}|g" \
|
||||
"$TPL" > "$YAML"
|
||||
echo "[sft] yaml: $YAML (SFT_RUNDIC=$SFT_RUNDIC PREV_RUNDIC=$PREV_RUNDIC)"
|
||||
|
||||
# 2. 提交训练任务
|
||||
SUBMIT_OUT=$(cml custom_train submit --filename "$YAML" 2>&1)
|
||||
echo "$SUBMIT_OUT"
|
||||
JOB_ID=$(echo "$SUBMIT_OUT" | grep -oE 't-[0-9]+-[a-z0-9]+' | head -1)
|
||||
if [ -z "$JOB_ID" ]; then
|
||||
echo "[sft] ❌ 提交失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[sft] ✅ JobID: $JOB_ID"
|
||||
|
||||
cat <<EOF
|
||||
|
||||
[sft] 任务提交完成，关键命令：
|
||||
查看任务状态: cml custom_train describe $JOB_ID
|
||||
查看实时日志: cml custom_train logs $JOB_ID --follow
|
||||
停止任务: cml custom_train kill $JOB_ID
|
||||
|
||||
训练成功后产物在: $ROOT/sft_output/_SUCCESS
|
||||
随后用 scripts/submit_cml_eval.sh <EVAL_RUNDIC> 起评测（不要在本脚本里串接，避免跨 step bg 任务）。
|
||||
|
||||
JobID: $JOB_ID
|
||||
SFT_RUNDIC: $SFT_RUNDIC (yaml / sft_output 命名)
|
||||
EOF
|
||||
@@ -1,124 +0,0 @@
|
||||
#!/bin/bash
|
||||
# 用 cml custom_train submit 提交 SFT 训练，训练完成后自动起 cml workflow run 评测
|
||||
#
|
||||
# 用法：
|
||||
# ./submit_sft_via_cml.sh <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC]
|
||||
#
|
||||
# 推荐调用方式（runDic 由 resolve_run_ids.sh 决定，不要手敲）：
|
||||
# eval "$(./scripts/resolve_run_ids.sh)"
|
||||
# ./scripts/submit_sft_via_cml.sh "$SFT_RUNDIC" "$EVAL_RUNDIC"
|
||||
#
|
||||
# 语义（与 program.md §746 + §5.2 对齐）：
|
||||
# SFT_RUNDIC = R{n-1}.runDic（augment / yaml / 归档全用这个，不 +1）
|
||||
# EVAL_RUNDIC = R{n-1}.runDic + 1（仅本步起 eval 时使用，仅在这一处 +1）
|
||||
# PREV_RUNDIC = SFT_RUNDIC - 1（默认；只用作上一轮 sft_output 改名）
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SFT_RUNDIC=${1:?usage: $0 <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC]}
|
||||
EVAL_RUNDIC=${2:?usage: $0 <SFT_RUNDIC> <EVAL_RUNDIC> [PREV_RUNDIC]}
|
||||
PREV_RUNDIC=${3:-$((SFT_RUNDIC-1))}
|
||||
|
||||
# 防御：EVAL_RUNDIC 必须严格大于 SFT_RUNDIC，否则一定是调用方算错
|
||||
if [ "$EVAL_RUNDIC" -le "$SFT_RUNDIC" ]; then
|
||||
echo "[cml-sft] ❌ EVAL_RUNDIC ($EVAL_RUNDIC) 必须 > SFT_RUNDIC ($SFT_RUNDIC)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT=${AUTORESEARCH_ROOT:-/mnt/wangsenhao/autoresearch-zk}
|
||||
MODEL_OLD=${MODEL_OLD:-/mnt/wangsenhao/verl_zk/qwen4b_cispo_wokl_add_bvt_2/global_step_5/actor/huggingface}
|
||||
TPL=${SFT_TRAIN_JOB_TEMPLATE:-$SCRIPT_DIR/sft_train_job.yaml.tpl}
|
||||
YAML=/tmp/sft_train_job_r${SFT_RUNDIC}.yaml
|
||||
|
||||
if [ -f ~/.cloudml-cli/.profile ]; then
|
||||
source ~/.cloudml-cli/.profile
|
||||
else
|
||||
echo "[cml-sft] ❌ 未找到 ~/.cloudml-cli/.profile，请先安装并初始化 cml" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 1. 渲染 yaml 模板（用 SFT_RUNDIC，不是 EVAL_RUNDIC）
|
||||
sed \
|
||||
-e "s|{RUNDIC}|${SFT_RUNDIC}|g" \
|
||||
-e "s|{PREV_RUNDIC}|${PREV_RUNDIC}|g" \
|
||||
-e "s|{AUTORESEARCH_ROOT}|${ROOT}|g" \
|
||||
"$TPL" > "$YAML"
|
||||
echo "[cml-sft] yaml: $YAML (SFT_RUNDIC=$SFT_RUNDIC EVAL_RUNDIC=$EVAL_RUNDIC PREV_RUNDIC=$PREV_RUNDIC)"
|
||||
|
||||
# 2. 提交训练任务
|
||||
SUBMIT_OUT=$(cml custom_train submit --filename "$YAML" 2>&1)
|
||||
echo "$SUBMIT_OUT"
|
||||
JOB_ID=$(echo "$SUBMIT_OUT" | grep -oE 't-[0-9]+-[a-z0-9]+' | head -1)
|
||||
if [ -z "$JOB_ID" ]; then
|
||||
echo "[cml-sft] ❌ 提交失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[cml-sft] ✅ JobID: $JOB_ID"
|
||||
|
||||
# 3. 后台 watcher：等训练成功 → 自动起评测（用 EVAL_RUNDIC）
|
||||
WATCHER_LOG=/tmp/r${SFT_RUNDIC}_logs/cml_watcher.log
|
||||
mkdir -p "$(dirname "$WATCHER_LOG")"
|
||||
|
||||
nohup bash -c '
|
||||
JOB_ID='"$JOB_ID"'
|
||||
SFT_RUNDIC='"$SFT_RUNDIC"'
|
||||
EVAL_RUNDIC='"$EVAL_RUNDIC"'
|
||||
LOG='"$WATCHER_LOG"'
|
||||
ROOT='"$ROOT"'
|
||||
MODEL_NEW="$ROOT/sft_output"
|
||||
MODEL_OLD='"$MODEL_OLD"'
|
||||
EVAL_WORKFLOW_ID=f-20260408161444-wu3pz
|
||||
EVAL_VERSION=v28
|
||||
|
||||
source ~/.cloudml-cli/.profile
|
||||
|
||||
echo "[$(date)] 等待 cml job $JOB_ID 完成..." >> "$LOG"
|
||||
while true; do
|
||||
STATE=$(cml custom_train describe "$JOB_ID" 2>/dev/null | grep -o "\"state\": \"[a-z]*\"" | head -1 | sed "s/.*: \"//;s/\"//")
|
||||
case "$STATE" in
|
||||
succeed)
|
||||
echo "[$(date)] 训练成功" >> "$LOG"
|
||||
break ;;
|
||||
failed|killed)
|
||||
echo "[$(date)] ❌ 训练 $STATE" >> "$LOG"
|
||||
exit 1 ;;
|
||||
*)
|
||||
sleep 60 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# 验证产出
|
||||
[ ! -f "$MODEL_NEW/_SUCCESS" ] && [ ! -f "$MODEL_NEW/config.json" ] && {
|
||||
echo "[$(date)] ❌ 产出缺失" >> "$LOG"; exit 1
|
||||
}
|
||||
|
||||
# 起评测（这里且仅这里用 EVAL_RUNDIC）
|
||||
echo "[$(date)] 启动 CML 评测 EVAL_RUNDIC=$EVAL_RUNDIC" >> "$LOG"
|
||||
cml workflow run \
|
||||
--workflow_id $EVAL_WORKFLOW_ID --version $EVAL_VERSION \
|
||||
--global_inputs runDic=$EVAL_RUNDIC \
|
||||
--global_inputs model_path_new=$MODEL_NEW \
|
||||
--global_inputs model_path_old=$MODEL_OLD >> "$LOG" 2>&1
|
||||
echo "[$(date)] cml workflow run 提交完成 (workflow$EVAL_RUNDIC)" >> "$LOG"
|
||||
' > /tmp/r${SFT_RUNDIC}_logs/watcher_runner.log 2>&1 &
|
||||
|
||||
WATCHER_PID=$!
|
||||
echo "[cml-sft] watcher PID: $WATCHER_PID（log: $WATCHER_LOG）"
|
||||
|
||||
# 4. 立即输出可查看命令
|
||||
cat <<EOF
|
||||
|
||||
[cml-sft] 任务提交完成，关键命令：
|
||||
查看任务状态: cml custom_train describe $JOB_ID
|
||||
查看实时日志: cml custom_train logs $JOB_ID --follow
|
||||
停止任务: cml custom_train kill $JOB_ID
|
||||
watcher log: tail -f $WATCHER_LOG
|
||||
|
||||
训练完成后，evaluation workflow 会自动启动；评测产物在
|
||||
/mnt/xiaoai-zk-model-train-tj5/workflow5/workflow${EVAL_RUNDIC}/
|
||||
|
||||
JobID: $JOB_ID
|
||||
SFT_RUNDIC: $SFT_RUNDIC (yaml / sft_output 命名)
|
||||
EVAL_RUNDIC: $EVAL_RUNDIC (workflow 评测目录)
|
||||
EOF
|
||||
Reference in New Issue
Block a user