修改

2026-05-20 15:04:19 +08:00
parent c4b935692c
commit 1a94cec822
27 changed files with 4831 additions and 1693 deletions
@@ -1,117 +0,0 @@
-jobName: "sft-train-r{RUNDIC}"
-description: "AutoResearch R{RUNDIC} SFT training (40k+ samples, qwen3-4B base, 3 epochs FSDP2)"
-accessType: PUBLIC
-
-imageConfig:
-  imageUrl: micr.cloud.mioffice.cn/vllm-image/ai-arch-llm-prod:vllm-v0.12.0-f098b188
-  imageCommand: |-
-    set -ex
-    cd {AUTORESEARCH_ROOT}
-
-    # 0a. 验环境 + 装缺的包（vllm 镜像 py3.12 + torch 已含）
-    python3 --version
-    python3 -c "import torch; print(f'torch={torch.__version__}')"
-    pip install -q --no-deps "accelerate==1.7.0" 2>&1 | tail -3
-    pip install -q --ignore-installed blinker 2>&1 | tail -2
-    pip install -q peft 2>&1 | tail -3
-    pip install -q wandb bitsandbytes 2>&1 | tail -3
-    pip install -q luigi mlflow scikit-learn openpyxl pyyaml sentencepiece tiktoken protobuf pynvml datasets 2>&1 | tail -3
-    pip install -q "transformers>=4.45" 2>&1 | tail -3
-    python3 -c 'import torch, accelerate, transformers, peft; print(torch.__version__, accelerate.__version__, transformers.__version__, peft.__version__)'
-
-    # 0c. 把 HF cache 重定向到容器本地大盘（默认在 ~/.cache 可能是 juicefs，mmap 易 SIGBUS）
-    export HF_HOME=/tmp/hf_cache
-    export HF_DATASETS_CACHE=/tmp/hf_cache/datasets
-    export TRANSFORMERS_CACHE=/tmp/hf_cache/transformers
-    mkdir -p /tmp/hf_cache/datasets /tmp/hf_cache/transformers
-    df -h /tmp /dev/shm 2>/dev/null || true
-
-    # 强制 datasets 加载进内存而不是 arrow mmap（避免 /dev/shm 溢出 SIGBUS）
-    export HF_DATASETS_IN_MEMORY_MAX_SIZE=20000000000
-    export HF_DATASETS_NUM_PROC=1
-    export TOKENIZERS_PARALLELISM=false
-    export OMP_NUM_THREADS=1
-    export MKL_NUM_THREADS=1
-    # 关闭 NCCL 用 shm 通信（改 socket 通道）
-    export NCCL_SHM_DISABLE=1
-    export NCCL_P2P_DISABLE=0
-    export NCCL_DEBUG=WARN
-
-    # 0b. 把上一轮产出搬走（cml job 内幂等，本地搬过就跳过）
-    if [ -d sft_output ] && [ ! -d sft_output_r{PREV_RUNDIC} ]; then
-      mv sft_output sft_output_r{PREV_RUNDIC}
-    fi
-    rm -rf sft_output
-
-    # 1. 组装数据
-    python3 {AUTORESEARCH_ROOT}/prepare_and_train_sft.py prepare \
-      --output_dir {AUTORESEARCH_ROOT}/sft_data
-
-    # 2. 启动训练（绝对路径，避免相对路径在容器内找不到 zk_trainer）
-    python3 {AUTORESEARCH_ROOT}/prepare_and_train_sft.py train \
-      --data_dir {AUTORESEARCH_ROOT}/sft_data \
-      --model_path /mnt/wangsenhao/verl_zk/Qwen3-4B-Instruct-2507 \
-      --model_type qwen3 \
-      --train_output {AUTORESEARCH_ROOT}/sft_output \
-      --epochs 3 --lr 1e-5
-
-    # 3. 训练成功标记（被 watcher 检测）
-    [ -f {AUTORESEARCH_ROOT}/sft_output/config.json ] && \
-      touch {AUTORESEARCH_ROOT}/sft_output/_SUCCESS
-
-# 挂载 wangsenhao + xiaoai-zk-model-train-tj5 + verl_zk 所在卷
-juiceFsMountConfigs:
-  - volume: wangsenhao
-    juiceFsCluster: tj5-common
-    subPath: /
-    mountPath: /mnt/wangsenhao
-    readOnly: false
-  - volume: xiaoai-zk-model-train-tj5
-    juiceFsCluster: tj5-common
-    subPath: /
-    mountPath: /mnt/xiaoai-zk-model-train-tj5
-    readOnly: false
-
-envConfigs:
-  - key: PYTHONPATH
-    value: {AUTORESEARCH_ROOT}/zk_trainer
-  - key: WANDB_DISABLED
-    value: "true"
-  - key: WANDB_MODE
-    value: offline
-  - key: VOLUME_PREFIX
-    value: /mnt/xiaoai-zk-model-train-tj5
-  - key: HF_HOME
-    value: /mnt/wangsenhao/.hf_cache
-
-# h20-96g 8 卡 FSDP2
-queueId: "6052"
-priority: 5
-preemptible: false
-framework: pytorch
-resourceConfigs:
-  - nodeRole: worker
-    nodeNumber: 1
-    perNodeResourceSpec:
-      resourcePriority: GUARANTEED
-      resourceName: cloudml.ng2h20-8-8.20-199
-      resourceNumber: 8
-
-# 故障自动重试（节点级失败）
-retryConfig:
-  enableRetry: true
-  maxRetryTimes: 2
-  policySets:
-    - NodeFailure
-
-# 失败/完成飞书告警
-alertConfig:
-  enableAlert: true
-  alertItems:
-    - alertConditions:
-        - FAILED
-        - SUCCEED
-      alertLevel: P2
-      alertReceivers:
-        persons:
-          - wangsenhao