diff --git a/.logs/server-20260521-094247.log b/.logs/server-20260521-094247.log new file mode 100644 index 0000000..45d73f4 --- /dev/null +++ b/.logs/server-20260521-094247.log @@ -0,0 +1,14 @@ +INFO: Started server process [2637646] +INFO: Waiting for application startup. +INFO: Application startup complete. +ERROR: [Errno 98] error while attempting to bind on address ('127.0.0.1', 8975): address already in use +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +Claw Code GUI listening on http://127.0.0.1:8975 + cwd : /home/mi/zk-data-agent-wsh + model : xiaomi/mimo-v2-flash + base-url : http://model.mify.ai.srv/v1 + timeout : 3600s + sessions : /home/mi/zk-data-agent-wsh/.port_sessions/agent + shell : on + write : on diff --git a/backend/api/server.py b/backend/api/server.py index 64c4a95..618eba6 100644 --- a/backend/api/server.py +++ b/backend/api/server.py @@ -24,7 +24,7 @@ import venv from dataclasses import dataclass, field, replace from pathlib import Path from threading import Lock, RLock -from typing import Any +from typing import Any, Callable from urllib import error, request from urllib.parse import quote, unquote, urlparse from uuid import uuid4 @@ -57,6 +57,13 @@ from src.personal_memory import ( USER_MEMORY_FILENAME, PersonalMemoryManager, ) +from src.bash_bg_store import ( + ACTIVE_BG_STATUSES, + TERMINAL_BG_STATUSES, + BashBgStore, + BgTaskSpec, + BgTaskStatus, +) from src.run_state_store import ACTIVE_RUN_STATUSES, RunStateStore from src.session_store import ( DEFAULT_AGENT_SESSION_DIR, @@ -86,6 +93,7 @@ FEISHU_SPREADSHEET_MAX_CELL_CHARS = 5000 FEISHU_SPREADSHEET_WRITE_BATCH_ROWS = 500 FEISHU_ONLINE_DOCS_FILENAME = 'online-docs.json' JUPYTER_WORKSPACE_FILENAME = 'jupyter_workspace.json' +JUPYTER_WORKSPACES_REGISTRY_FILENAME = 'workspaces.json' JUPYTER_STREAM_CHUNK_BYTES = 1024 * 1024 FEISHU_REMOTE_DOC_MAX_BYTES = 50 * 1024 * 1024 FEISHU_REMOTE_SHEET_MAX_BYTES = 100 * 1024 * 1024 @@ -543,6 +551,8 @@ class AgentState: self._account_configs: dict[str, AgentInstanceConfig] = {} self.run_manager = RunManager() self.run_state_store = RunStateStore(self.session_directory.parent / 'run_state.db') + self.bash_bg_store = BashBgStore(self.session_directory.parent / 'bash_bg.db') + self.event_loop: 'asyncio.AbstractEventLoop | None' = None self.jupyter_runtime_manager = JupyterRuntimeManager() self._default_config = AgentInstanceConfig( cwd=cwd.resolve(), @@ -840,6 +850,7 @@ class AgentState: scratchpad_root=paths['scratchpad'], python_env_dir=paths['python_env'], enabled_skill_names=self.enabled_skill_names(account_id), + auto_compact_threshold_tokens=180_000, ) model_config = ModelConfig( model=config.model, @@ -989,6 +1000,11 @@ class JupyterWorkspaceBindRequest(BaseModel): account_id: str | None = None +class SavedJupyterWorkspaceBindRequest(BaseModel): + session_id: str = Field(min_length=1) + account_id: str | None = None + + class SkillPreferenceUpdate(BaseModel): skill: str | None = None enabled: bool @@ -1042,12 +1058,17 @@ def _append_runtime_context(current: str | None, addition: str) -> str: def create_app(state: AgentState) -> FastAPI: @asynccontextmanager async def lifespan(app: FastAPI): # noqa: ARG001 + state.event_loop = asyncio.get_running_loop() + _bash_bg_manager.set_chat_runner(_run_chat_payload) + await _bash_bg_manager.recover_from_store(state) scanner_task = asyncio.create_task(_watcher_scanner_loop(state)) try: yield finally: scanner_task.cancel() _watcher_manager.cancel_all() + _bash_bg_manager.cancel_all() + state.event_loop = None app = FastAPI(title='Claw Code GUI', version='1.0', lifespan=lifespan) @@ -1189,8 +1210,83 @@ def create_app(state: AgentState) -> FastAPI: state.account_paths(payload.account_id)['sessions'], runtime, ) + _register_jupyter_workspace( + state, + payload.account_id, + base_url=payload.base_url, + workspace_root=payload.workspace_root, + password=payload.password, + ) return runtime.to_dict() + @app.get('/api/jupyter/workspaces') + async def list_jupyter_workspaces( + account_id: str | None = None, + ) -> list[dict[str, Any]]: + entries = _load_jupyter_workspaces_registry(state, account_id) + return [_public_jupyter_workspace_entry(e) for e in entries] + + @app.post('/api/jupyter/workspaces/{workspace_id}/bind') + async def bind_saved_jupyter_workspace( + workspace_id: str, + payload: SavedJupyterWorkspaceBindRequest, + ) -> dict[str, Any]: + safe_session_id = _safe_session_id(payload.session_id) + if not safe_session_id: + raise HTTPException(status_code=400, detail='session_id is required') + entries = _load_jupyter_workspaces_registry(state, payload.account_id) + match = next((e for e in entries if e.get('id') == workspace_id), None) + if match is None: + raise HTTPException(status_code=404, detail='workspace not found') + password = match.get('password') + base_url = match.get('base_url') + workspace_root = match.get('workspace_root') or DEFAULT_JUPYTER_WORKSPACE_ROOT + if not (isinstance(password, str) and isinstance(base_url, str)): + raise HTTPException(status_code=400, detail='workspace entry is incomplete') + account_key = state._account_key(payload.account_id) + try: + runtime = await asyncio.to_thread( + state.jupyter_runtime_manager.bind_session, + account_id=account_key, + session_id=safe_session_id, + base_url=base_url, + password=password, + workspace_root=workspace_root, + project_root=state.config_for(payload.account_id).cwd, + ) + except JupyterRuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + status_code=502, + detail=f'Unable to connect Jupyter runtime: {exc}', + ) from exc + _save_jupyter_runtime_binding( + state.account_paths(payload.account_id)['sessions'], + runtime, + ) + _register_jupyter_workspace( + state, + payload.account_id, + base_url=base_url, + workspace_root=workspace_root, + password=password, + label=match.get('label') if isinstance(match.get('label'), str) else None, + ) + return runtime.to_dict() + + @app.delete('/api/jupyter/workspaces/{workspace_id}') + async def delete_saved_jupyter_workspace( + workspace_id: str, + account_id: str | None = None, + ) -> dict[str, Any]: + entries = _load_jupyter_workspaces_registry(state, account_id) + kept = [e for e in entries if e.get('id') != workspace_id] + if len(kept) == len(entries): + raise HTTPException(status_code=404, detail='workspace not found') + _save_jupyter_workspaces_registry(state, account_id, kept) + return {'ok': True} + @app.get('/api/slash-commands') async def list_slash_commands() -> list[dict[str, Any]]: commands: list[dict[str, Any]] = [] @@ -1607,7 +1703,7 @@ def create_app(state: AgentState) -> FastAPI: state_entries, _ = _read_program_state(state_path) run_dic = _resolve_run_dic_from_state(state_entries) runtime = _jupyter_runtime_for_session(state, account_id, session_id) - artifact_paths = _step_artifact_paths(step, run_dic) + artifact_paths = _step_artifact_paths(step, run_dic, runtime) if not artifact_paths: payload = { @@ -1625,6 +1721,152 @@ def create_app(state: AgentState) -> FastAPI: headers={'Cache-Control': 'no-store'}, ) + # 多分区表格视图：augment（H1 修改 / H2 仿写）与 dist-analysis（候选 + 报告）共用 + # path_indices 指向 _step_artifact_paths 返回的路径列表；多个候选按顺序尝试， + # 用第一个能读到的（让 dist-analysis 的 NFS 主路径 + jupyter pod 本地兜底 + # 路径合并成同一个 section，避免出现重复卡片）。 + multi_section_specs: dict[str, list[dict[str, Any]]] = { + 'augment': [ + {'title': 'H1 确认修改（complex 翻转 → 原文件 in-place 改标）', 'max_rows': None, 'path_indices': [0]}, + {'title': 'H2 仿写候选 raw（GPT 原始输出，未过 sanity）', 'max_rows': None, 'path_indices': [1]}, + {'title': 'H2 仿写最终（本轮新增训练样本，合入 augment.jsonl）', 'max_rows': None, 'path_indices': [2]}, + {'title': 'label-master 标签复核（H1+H2 全量 verdict）', 'max_rows': None, 'path_indices': [3]}, + ], + 'dist-analysis': [ + {'title': '问题分析报告（workflow.md）', 'max_rows': None, 'path_indices': [0]}, + {'title': 'H1 预计修改候选', 'max_rows': None, 'path_indices': [1, 2]}, + ], + } + if step == 'dist-analysis': + # 从 iteration_log.jsonl 读 H1 假设短标签，注入到候选集 section 标题 + h1_label = await asyncio.to_thread( + _read_iteration_log_h1_label, runtime, run_dic + ) + if h1_label: + for spec in multi_section_specs['dist-analysis']: + if spec['title'] == 'H1 预计修改候选': + spec['title'] = f'H1 预计修改候选（{h1_label}）' + break + if step in multi_section_specs: + section_specs = multi_section_specs[step] + table_sections: list[dict[str, Any]] = [] + tried: list[str] = [] + errors: list[str] = [] + for spec in section_specs: + title = spec['title'] + max_rows = spec.get('max_rows') + candidate_paths = [ + artifact_paths[i] + for i in spec.get('path_indices', []) + if 0 <= i < len(artifact_paths) + ] + if not candidate_paths: + continue + # 多候选按顺序尝试，用第一个能读到的；都读不到则用首个候选作为缺失占位 + content: str | None = None + path_str: str = candidate_paths[0] + last_err: str | None = None + for cand in candidate_paths: + tried.append(cand) + content_try, err = await asyncio.to_thread( + _read_artifact, runtime, cand + ) + if content_try is not None: + content = content_try + path_str = cand + break + last_err = err + if content is None: + if last_err: + errors.append(f'{title}: {last_err}') + table_sections.append( + { + 'title': title, + 'source_path': candidate_paths[0], + 'kind': 'missing', + 'columns': [], + 'rows': [], + 'total_rows': 0, + 'shown_rows': 0, + 'error': last_err or '产物文件不存在', + } + ) + continue + fmt = _detect_artifact_format(path_str) + try: + if fmt == 'markdown': + # markdown 直接以原文渲染，不走表格 + table_sections.append( + { + 'title': title, + 'source_path': path_str, + 'kind': 'markdown', + 'columns': [], + 'rows': [], + 'total_rows': 0, + 'shown_rows': 0, + 'body': content, + } + ) + continue + if fmt == 'csv': + cols, table_rows, total = _csv_to_table(content, max_rows) + kind = 'csv' + elif fmt in {'jsonl', 'json'}: + cols, table_rows, total = _jsonl_to_table( + content, max_rows + ) + kind = 'jsonl' + else: + lines = content.splitlines() + total = len(lines) + capped = lines if max_rows is None else lines[:max_rows] + cols = ['line'] + table_rows = [[_truncate_cell(line)] for line in capped] + kind = 'text' + except Exception as exc: # noqa: BLE001 + errors.append(f'{title}: parse failed: {exc}') + table_sections.append( + { + 'title': title, + 'source_path': path_str, + 'kind': 'error', + 'columns': [], + 'rows': [], + 'total_rows': 0, + 'shown_rows': 0, + 'error': f'解析失败：{exc}', + } + ) + continue + table_sections.append( + { + 'title': title, + 'source_path': path_str, + 'kind': kind, + 'columns': cols, + 'rows': table_rows, + 'total_rows': total, + 'shown_rows': len(table_rows), + } + ) + payload = { + 'step': step, + 'run_dic': run_dic, + 'format': 'tables', + 'sections': table_sections, + 'source_path': artifact_paths[0], + 'tried_paths': tried, + 'fetched_at_ms': int(time.time() * 1000), + } + if errors: + payload['error'] = '; '.join(errors) + return Response( + content=json.dumps(payload, ensure_ascii=False), + media_type='application/json', + headers={'Cache-Control': 'no-store'}, + ) + last_error: str | None = None for path_str in artifact_paths: content, err = await asyncio.to_thread( @@ -1675,13 +1917,35 @@ def create_app(state: AgentState) -> FastAPI: sessions_dir = state.account_paths(account_id)['sessions'] state_path = _session_state_path(sessions_dir, session_id) state_entries, _ = _read_program_state(state_path) - payload = _hardcoded_autoresearch_pipeline( + iter_count = await asyncio.to_thread( + _read_iteration_log_count, + session_id, + agent_state=state, + account_id=account_id, + ) + payload = await asyncio.to_thread( + _hardcoded_autoresearch_pipeline, session_id, sessions_dir=sessions_dir, state_entries=state_entries, model_config=state.model_config_for(account_id), + agent_state=state, + account_id=account_id, + ) + failure_status = await asyncio.to_thread( + _read_session_failure_status, + sessions_dir, + session_id, + ) + run_active = _is_run_active(state, account_id, session_id) + _apply_program_state( + payload, + state_entries, + iteration_log_count=iter_count or 0, + session_failure_status=failure_status, + run_active=run_active, + session_id=session_id, ) - _apply_program_state(payload, state_entries) return Response( content=json.dumps(payload, ensure_ascii=False), media_type='application/json', @@ -1704,12 +1968,34 @@ def create_app(state: AgentState) -> FastAPI: if await request.is_disconnected(): break state_entries, state_mtime = _read_program_state(state_path) - payload = _hardcoded_autoresearch_pipeline( + iter_count = await asyncio.to_thread( + _read_iteration_log_count, + session_id, + agent_state=state, + account_id=account_id, + ) + payload = await asyncio.to_thread( + _hardcoded_autoresearch_pipeline, session_id, sessions_dir=sessions_dir, state_entries=state_entries, + agent_state=state, + account_id=account_id, + ) + failure_status = await asyncio.to_thread( + _read_session_failure_status, + sessions_dir, + session_id, + ) + run_active = _is_run_active(state, account_id, session_id) + _apply_program_state( + payload, + state_entries, + iteration_log_count=iter_count or 0, + session_failure_status=failure_status, + run_active=run_active, + session_id=session_id, ) - _apply_program_state(payload, state_entries) payload_str = json.dumps(payload, ensure_ascii=False) signature = f'{state_mtime}|{hash(payload_str)}' if signature != last_signature: @@ -1983,11 +2269,44 @@ def create_app(state: AgentState) -> FastAPI: if stored_event is not None: state.run_state_store.record_event(run_record.run_id, stored_event) _emit_runtime_event(event_sink, started_event) + event_loop = state.event_loop + + def _bg_register(spec: BgTaskSpec) -> None: + if event_loop is None: + raise RuntimeError( + 'event loop unavailable; backend not started via lifespan' + ) + asyncio.run_coroutine_threadsafe( + _bash_bg_manager.register(state, spec), + event_loop, + ) + + def _bg_status_query(task_id: str) -> 'BgTaskStatus | None': + return _bash_bg_manager.status_query(state, task_id) + + def _bg_kill_request(task_id: str) -> bool: + if event_loop is None: + return False + future = asyncio.run_coroutine_threadsafe( + _bash_bg_manager.cancel(state, task_id), + event_loop, + ) + try: + return bool(future.result(timeout=15)) + except Exception: # noqa: BLE001 + return False + agent.tool_context = replace( agent.tool_context, cancel_event=run_record.cancel_event, process_registry=run_record.process_registry, jupyter_runtime=jupyter_runtime, + bg_register=_bg_register, + bg_status_query=_bg_status_query, + bg_kill_request=_bg_kill_request, + account_id=account_key, + session_id=requested_session_id, + run_id=run_record.run_id, ) started_at = time.perf_counter() try: @@ -2057,6 +2376,12 @@ def create_app(state: AgentState) -> FastAPI: cancel_event=None, process_registry=None, jupyter_runtime=None, + bg_register=None, + bg_status_query=None, + bg_kill_request=None, + account_id=None, + session_id=None, + run_id=None, ) elapsed_ms = max(0, int((time.perf_counter() - started_at) * 1000)) payload = _serialize_run_result(result) @@ -2276,6 +2601,53 @@ def _stored_session_has_incomplete_tail(stored: StoredAgentSession) -> bool: return budget_state.get('status') in {'queued', 'running'} +def _is_run_active(state: 'AgentState', account_id: str | None, session_id: str | None) -> bool | None: + """True iff `run_state_store` says the latest run for this session is in + `queued`/`running`. None when the lookup is impossible (no session id, or + no row yet). The pipeline endpoint uses this to detect orphan `running` + step entries — agent wrote `step:running`, finished its turn cleanly, and + never wrote the closing entry; from the user's POV the run is idle but + the panel was still spinning. + """ + if not session_id: + return None + try: + account_key = state._account_key(account_id) + snapshot = state.run_state_store.snapshot_latest(account_key, session_id) + except Exception: + return None + if snapshot is None: + return None + return snapshot.get('status') in ACTIVE_RUN_STATUSES + + +def _read_session_failure_status( + directory: Path, + session_id: str | None, +) -> str | None: + """Returns 'failed' / 'cancelled' / 'interrupted' iff the stored session + was explicitly tombstoned by `_mark_session_interrupted`. Returns None for + healthy sessions AND for live runs (budget_state.status='running'/'queued'). + + The pipeline panel uses this to paint a failed run red. We deliberately do + NOT use `_stored_session_has_incomplete_tail` here — that helper is for + detecting incomplete tails when LOADING a session for resume, where + budget_state='running' implies a dead writer. Polled live, 'running' just + means the agent is still chugging. + """ + if not session_id: + return None + try: + stored = load_agent_session(session_id, directory=directory) + except (FileNotFoundError, OSError, json.JSONDecodeError): + return None + bs = stored.budget_state if isinstance(stored.budget_state, dict) else {} + status = bs.get('status') + if status in ('failed', 'cancelled', 'interrupted'): + return str(status) + return None + + def _mark_session_interrupted( directory: Path, session_id: str, @@ -3053,6 +3425,107 @@ def _save_jupyter_runtime_binding( return +# ── Per-account remote-Jupyter workspace registry ───────────────────────────── +# Saved at accounts//workspaces.json. Each entry stores enough to re-bind a +# new chat without re-prompting (base_url + workspace_root + password). Cookies +# expire and aren't reusable across sessions, so for one-click re-bind we store +# the password — same plaintext-on-disk security posture as the per-session +# jupyter_workspace.json that already keeps login cookies. + +def _jupyter_workspaces_registry_path(state: 'AgentState', account_id: str | None) -> Path: + return state.account_paths(account_id)['base'] / JUPYTER_WORKSPACES_REGISTRY_FILENAME + + +def _load_jupyter_workspaces_registry( + state: 'AgentState', account_id: str | None +) -> list[dict[str, Any]]: + path = _jupyter_workspaces_registry_path(state, account_id) + try: + payload = json.loads(path.read_text(encoding='utf-8')) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(payload, list): + return [] + out: list[dict[str, Any]] = [] + for item in payload: + if isinstance(item, dict) and isinstance(item.get('id'), str): + out.append(item) + return out + + +def _save_jupyter_workspaces_registry( + state: 'AgentState', account_id: str | None, entries: list[dict[str, Any]] +) -> None: + path = _jupyter_workspaces_registry_path(state, account_id) + try: + path.parent.mkdir(parents=True, exist_ok=True) + _write_session_metadata(path, entries) + except OSError: + return + + +def _jupyter_workspace_registry_id(base_url: str, workspace_root: str) -> str: + """Deterministic ID = sha1(base_url|workspace_root). Same (host, root) tuple + de-duplicates rather than piling up entries on every successful bind.""" + digest = hashlib.sha1( + f'{base_url}|{workspace_root}'.encode('utf-8'), + ).hexdigest() + return digest[:16] + + +def _public_jupyter_workspace_entry(entry: dict[str, Any]) -> dict[str, Any]: + """Strip password before returning to the frontend.""" + return { + 'id': entry.get('id'), + 'label': entry.get('label'), + 'base_url': entry.get('base_url'), + 'workspace_root': entry.get('workspace_root'), + 'created_at': entry.get('created_at'), + 'last_used_at': entry.get('last_used_at'), + } + + +def _register_jupyter_workspace( + state: 'AgentState', + account_id: str | None, + *, + base_url: str, + workspace_root: str, + password: str, + label: str | None = None, +) -> dict[str, Any]: + entries = _load_jupyter_workspaces_registry(state, account_id) + workspace_id = _jupyter_workspace_registry_id(base_url, workspace_root) + now = time.time() + found: dict[str, Any] | None = None + rest: list[dict[str, Any]] = [] + for entry in entries: + if entry.get('id') == workspace_id: + found = entry + else: + rest.append(entry) + if found is None: + found = { + 'id': workspace_id, + 'label': label or base_url, + 'base_url': base_url, + 'workspace_root': workspace_root, + 'created_at': now, + } + else: + if label: + found['label'] = label + elif not found.get('label'): + found['label'] = base_url + found['base_url'] = base_url + found['workspace_root'] = workspace_root + found['password'] = password + found['last_used_at'] = now + rest.insert(0, found) # most-recently-used first + _save_jupyter_workspaces_registry(state, account_id, rest) + return found + + def _jupyter_runtime_for_session( state: AgentState, account_id: str | None, @@ -3950,6 +4423,28 @@ def _safe_account_id(account_id: str | None) -> str: return normalized[:80] or 'default' +_XIAOMI_EMAIL_RE = re.compile(r'^([\w.-]+)@xiaomi\.com$', re.IGNORECASE) +_EMAIL_ACCOUNT_ID_RE = re.compile(r'^[a-z0-9._-]{2,40}$') + + +def _account_id_from_email(email: str | None) -> str | None: + """Validate a Xiaomi email and return its prefix as account_id. + + The prefix is lowercased and must satisfy the platform account_id rule: + only lowercase letters / digits / dot / underscore / dash, length 2-40. + Returns None if the email or prefix is invalid (caller must reject). + """ + if not isinstance(email, str): + return None + match = _XIAOMI_EMAIL_RE.match(email.strip()) + if not match: + return None + prefix = match.group(1).lower() + if not _EMAIL_ACCOUNT_ID_RE.match(prefix): + return None + return prefix + + def _admin_token() -> str: return os.environ.get('ZK_ADMIN_TOKEN') or 'admin' @@ -4377,6 +4872,7 @@ class _WatcherManager: state_path, step=step, status='complete', + run_id=run_id, agent_state=agent_state, account_id=account_id, session_id=session_id, @@ -4395,6 +4891,7 @@ class _WatcherManager: state_path, step=step, status='failed', + run_id=run_id, error=f'watcher 超时 {int(timeout)}s, target 未出现: {target}', agent_state=agent_state, account_id=account_id, @@ -4430,6 +4927,7 @@ class _WatcherManager: *, step: str, status: str, + run_id: str | None = None, error: str | None = None, agent_state: 'AgentState | None' = None, account_id: str | None = None, @@ -4440,6 +4938,8 @@ class _WatcherManager: 'status': status, 'ts': _now_iso_local(), } + if run_id: + entry['run_id'] = run_id if error: entry['error'] = error await self._append_state_entry( @@ -4579,6 +5079,352 @@ class _WatcherManager: _watcher_manager = _WatcherManager() +class _BashBackgroundManager: + """Polls remote/local bg bash tasks; finalizes them and (optionally) fires + an auto-resume turn when the agent has paused with the previous run already + completed. + + Lifecycle mirrors `_WatcherManager`: one asyncio.Task per task_id, written + to/read from `BashBgStore` so polling can resume after a server restart. + """ + + POLL_INTERVAL_SECONDS = 5.0 + MAX_TASK_LIFETIME_SECONDS = 6 * 3600.0 + REMOTE_PROBE_TIMEOUT_SECONDS = 10.0 + OUTPUT_PREVIEW_BYTES = 4096 + + def __init__(self) -> None: + self._tasks: dict[str, asyncio.Task[None]] = {} + self._chat_runner: 'Callable[[ChatRequest], dict[str, Any]] | None' = None + + def set_chat_runner( + self, + runner: 'Callable[[ChatRequest], dict[str, Any]]', + ) -> None: + self._chat_runner = runner + + async def register(self, state: 'AgentState', spec: BgTaskSpec) -> None: + state.bash_bg_store.record_start(spec) + if spec.task_id in self._tasks: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._tasks[spec.task_id] = loop.create_task( + self._poll_alive(state, spec) + ) + + async def recover_from_store(self, state: 'AgentState') -> None: + for row in state.bash_bg_store.list_active(): + spec = self._spec_from_row(row) + await self.register(state, spec) + + async def cancel(self, state: 'AgentState', task_id: str) -> bool: + row = state.bash_bg_store.get(task_id) + if row is None or row['status'] != 'running': + return False + await self._kill_remote_or_local(state, row) + state.bash_bg_store.mark_killed(task_id) + existing = self._tasks.pop(task_id, None) + if existing is not None: + existing.cancel() + return True + + def status_query( + self, state: 'AgentState', task_id: str + ) -> 'BgTaskStatus | None': + row = state.bash_bg_store.get(task_id) + if row is None: + return None + preview = self._read_output_preview_sync(state, row) + return BgTaskStatus( + task_id=row['task_id'], + status=row['status'], + pid=row['pid'], + started_at=row['started_at'], + finished_at=row['finished_at'], + exit_code=row['exit_code'], + output_path=row['output_path'], + output_preview=preview, + auto_resumed=bool(row['auto_resumed']), + ) + + def cancel_all(self) -> None: + for task in list(self._tasks.values()): + task.cancel() + self._tasks.clear() + + async def _poll_alive(self, state: 'AgentState', spec: BgTaskSpec) -> None: + try: + start = time.monotonic() + while True: + alive = await self._check_pid_alive(state, spec) + if not alive: + await self._finalize(state, spec) + return + if time.monotonic() - start > self.MAX_TASK_LIFETIME_SECONDS: + state.bash_bg_store.mark_completed( + spec.task_id, exit_code=124, + ) + return + await asyncio.sleep(self.POLL_INTERVAL_SECONDS) + except asyncio.CancelledError: + return + except Exception as exc: # noqa: BLE001 + print( + f'[bg-bash] poll error task={spec.task_id}: {exc}', flush=True, + ) + finally: + self._tasks.pop(spec.task_id, None) + + async def _check_pid_alive( + self, state: 'AgentState', spec: BgTaskSpec + ) -> bool: + runtime = _jupyter_runtime_for_session( + state, spec.account_key or None, spec.session_id, + ) + if runtime is not None: + cmd = ( + f'kill -0 {spec.pid} 2>/dev/null && echo alive || echo dead' + ) + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=64, + cancel_event=None, + ) + except Exception: # noqa: BLE001 + # Treat probe failure as "still running" — don't finalize on + # transient runtime errors. The 6h lifetime cap is the backstop. + return True + return 'alive' in result.stdout + # Local fallback. Note: the spawned child is detached via + # start_new_session=True but the FastAPI process never reaps it, so + # after exit it becomes a zombie and kill(pid, 0) keeps reporting it + # alive. The MAX_TASK_LIFETIME_SECONDS cap is the eventual backstop; + # production runs use jupyter_runtime where the child belongs to a + # different process tree. + try: + os.kill(spec.pid, 0) + return True + except (ProcessLookupError, PermissionError): + return False + except OSError: + return True + + async def _finalize(self, state: 'AgentState', spec: BgTaskSpec) -> None: + exit_code = await self._read_exit_code(state, spec) + state.bash_bg_store.mark_completed(spec.task_id, exit_code=exit_code) + if not spec.wait_for_completion: + return + await self._maybe_auto_resume(state, spec, exit_code) + + async def _read_exit_code( + self, state: 'AgentState', spec: BgTaskSpec + ) -> int: + runtime = _jupyter_runtime_for_session( + state, spec.account_key or None, spec.session_id, + ) + if runtime is not None: + cmd = f'cat {shlex.quote(spec.exit_code_path)} 2>/dev/null || echo' + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=64, + cancel_event=None, + ) + text = result.stdout.strip() + return int(text) if text else -1 + except (ValueError, Exception): # noqa: BLE001 + return -1 + try: + text = Path(spec.exit_code_path).read_text( + encoding='utf-8', + ).strip() + return int(text) if text else -1 + except (OSError, ValueError): + return -1 + + async def _read_output_preview_async( + self, state: 'AgentState', spec_row: 'dict[str, Any] | BgTaskSpec' + ) -> str: + if isinstance(spec_row, BgTaskSpec): + account = spec_row.account_key + session = spec_row.session_id + output_path = spec_row.output_path + else: + account = spec_row['account_key'] + session = spec_row['session_id'] + output_path = spec_row['output_path'] + runtime = _jupyter_runtime_for_session(state, account or None, session) + if runtime is not None: + cmd = ( + f'tail -c {self.OUTPUT_PREVIEW_BYTES} ' + f'{shlex.quote(output_path)} 2>/dev/null || true' + ) + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=self.OUTPUT_PREVIEW_BYTES + 256, + cancel_event=None, + ) + return result.stdout.strip() + except Exception: # noqa: BLE001 + return '(读取远端输出失败)' + try: + text = Path(output_path).read_text( + encoding='utf-8', errors='replace', + ) + return text[-self.OUTPUT_PREVIEW_BYTES:].strip() + except OSError: + return '(无法读取输出文件)' + + def _read_output_preview_sync( + self, state: 'AgentState', row: dict[str, Any] + ) -> str: + runtime = _jupyter_runtime_for_session( + state, row['account_key'] or None, row['session_id'], + ) + if runtime is not None: + cmd = ( + f'tail -c {self.OUTPUT_PREVIEW_BYTES} ' + f'{shlex.quote(row["output_path"])} 2>/dev/null || true' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=self.OUTPUT_PREVIEW_BYTES + 256, + cancel_event=None, + ) + return result.stdout.strip() + except Exception: # noqa: BLE001 + return '(读取远端输出失败)' + try: + text = Path(row['output_path']).read_text( + encoding='utf-8', errors='replace', + ) + return text[-self.OUTPUT_PREVIEW_BYTES:].strip() + except OSError: + return '(无法读取输出文件)' + + async def _kill_remote_or_local( + self, state: 'AgentState', row: dict[str, Any] + ) -> None: + runtime = _jupyter_runtime_for_session( + state, row['account_key'] or None, row['session_id'], + ) + pid = int(row['pid']) + if runtime is not None: + cmd = ( + f'kill -- -$(ps -o pgid= -p {pid} 2>/dev/null ' + f'| tr -d " ") 2>/dev/null; ' + f'kill {pid} 2>/dev/null; true' + ) + try: + await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=64, + cancel_event=None, + ) + except Exception: # noqa: BLE001 + pass + return + try: + os.killpg(os.getpgid(pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + try: + os.kill(pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + pass + + async def _maybe_auto_resume( + self, state: 'AgentState', spec: BgTaskSpec, exit_code: int, + ) -> None: + runner = self._chat_runner + if runner is None: + return + account_key = spec.account_key or state._account_key(None) + session_id = spec.session_id + if not session_id: + return + snapshot = state.run_state_store.snapshot_latest( + account_key, session_id, + ) + if not snapshot or snapshot.get('status') != 'completed': + return + run_lock = state.run_lock_for(account_key, session_id) + if not run_lock.acquire(blocking=False): + return + run_lock.release() + if not state.bash_bg_store.mark_auto_resumed(spec.task_id): + return + output_preview = await self._read_output_preview_async(state, spec) + prompt = ( + f'[system] 后台任务 {spec.task_id} 已结束，' + f'exit_code={exit_code}\n' + f'命令：{spec.command}\n' + f'输出预览（最后 {self.OUTPUT_PREVIEW_BYTES} 字节）：\n' + f'{output_preview}' + ) + request = ChatRequest( + prompt=prompt, + account_id=spec.account_key or None, + session_id=session_id, + resume_session_id=session_id, + ) + loop = asyncio.get_running_loop() + loop.run_in_executor(None, _safe_run_chat_payload, runner, request) + + @staticmethod + def _spec_from_row(row: dict[str, Any]) -> BgTaskSpec: + return BgTaskSpec( + task_id=row['task_id'], + account_key=row['account_key'] or '', + session_id=row['session_id'] or '', + run_id=row['run_id'] or '', + pid=int(row['pid']), + task_dir=row['task_dir'], + output_path=row['output_path'], + pid_path=row['pid_path'], + exit_code_path=row['exit_code_path'], + command=row['command'], + started_at=float(row['started_at']), + wait_for_completion=bool(row['wait_for_completion']), + ) + + +_bash_bg_manager = _BashBackgroundManager() + + +def _safe_run_chat_payload( + runner: 'Callable[[ChatRequest], dict[str, Any]]', + request: ChatRequest, +) -> None: + """Fire-and-forget wrapper around the chat payload runner for auto-resume. + + Runs in the default thread pool. Any exception is logged but swallowed — + the bg task is already finalized in SQLite, so the user can still + bash_status / 继续 manually if the auto-resume fails. + """ + try: + runner(request) + except Exception as exc: # noqa: BLE001 + print( + f'[bg-bash] auto-resume failed session={request.session_id}: {exc}', + flush=True, + ) + + def _sync_remote_program_state( state: AgentState, account_id: str, @@ -4880,22 +5726,578 @@ def _step_matches_card(step: str, card: dict[str, Any]) -> bool: return False +# Round-section pipeline data model. +# +# Old: 3 fixed phases (eval / intervene / train) with cards dynamically injected. +# New: per-round numbered sections (R0·Baseline → R1·Train → R1·Analysis → ...), +# with optional HiTL gates inserted between rounds when agent writes +# step:"human-check" or step:"human-review". +# +# Each step belongs to a "section type" (baseline / train / analysis): +# - R0 has only one section: R0·Baseline (cml/gold-drift/dist-analysis/report/hypothesis/log) +# - R{n>=1} has two sections: R{n}·Train (augment/verify/sft) +# R{n}·Analysis (cml/gold-drift/dist-analysis/report/hypothesis/log) +# +# 'hypothesis' is classified as 'analysis' (not 'train'): forming the next-round +# hypothesis is the conclusion of the current round's analysis cycle, not the +# start of the next round's training. Putting it in analysis makes the gate +# (Human Check / Review) — which sorts after all sections of run_id=R{n} but +# before R{n+1}·Train — block training kickoff cleanly. +# +# 'next-round' is a boundary marker — does not produce a card. +# 'human-check' / 'human-review' produce gate items, not cards. + +# step → kind: 'analysis' | 'train' | 'gate' | 'boundary' +_STEP_KIND: dict[str, str] = { + 'cml': 'analysis', + 'gold-drift': 'analysis', + 'dist-analysis': 'analysis', + 'report': 'analysis', + 'hypothesis': 'analysis', + 'log': 'analysis', + 'augment': 'train', + 'verify': 'train', + 'sft': 'train', + 'human-check': 'gate', + 'human-review': 'gate', + 'next-round': 'boundary', +} + +_GATE_TITLES: dict[str, dict[str, str]] = { + 'human-check': { + 'title': '人工确认', + 'description': '已生成本轮假设与候选，等待你点头进入下一轮训练', + }, + 'human-review': { + 'title': '人工复核', + 'description': '触发了 §4.0.1 / Gold-drift 规则，需要你逐条审一下再继续', + }, +} + +# (step, section_type) → display metadata. section_type is 'baseline' for R0 +# steps, 'train'/'analysis' for R{n>=1}. +_CARD_META: dict[tuple[str, str], dict[str, Any]] = { + # R0·Baseline + ('cml', 'baseline'): {'icon': 'bar-chart', 'title': 'Baseline 评测', 'subtitle': 'CML workflow metric_diff'}, + ('gold-drift', 'baseline'): {'icon': 'alert-triangle', 'title': 'Gold Drift 检查', 'subtitle': 'drift_.json'}, + ('dist-analysis', 'baseline'): {'icon': 'trending-up', 'title': '分层结果分析', 'subtitle': '需求集合 / 车载 / specific'}, + ('report', 'baseline'): {'icon': 'file-text', 'title': '问题分析 & 报告', 'subtitle': 'workflow.md'}, + ('hypothesis', 'baseline'): {'icon': 'dna', 'title': '形成假设', 'subtitle': 'iteration_log hypothesis 字段'}, + ('log', 'baseline'): {'icon': 'clipboard-list', 'title': '记录迭代日志', 'subtitle': 'iteration_log.jsonl'}, + # R{n}·Train + ('augment', 'train'): {'icon': 'flask', 'title': '数据增强', 'subtitle': 'augment_.jsonl'}, + ('verify', 'train'): {'icon': 'shield', 'title': '修改返回验证', 'subtitle': 'sanity + 人审'}, + ('sft', 'train'): {'icon': 'graduation-cap','title': 'SFT 训练', 'subtitle': 'submit_sft_via_cml.sh'}, + # R{n}·Analysis + ('cml', 'analysis'): {'icon': 'bar-chart', 'title': 'CML 评测', 'subtitle': 'workflow metric_diff'}, + ('gold-drift', 'analysis'): {'icon': 'alert-triangle', 'title': 'Gold Drift 检查', 'subtitle': 'drift_.json'}, + ('dist-analysis', 'analysis'): {'icon': 'trending-up', 'title': '分层结果分析', 'subtitle': '需求集合 / 车载 / specific'}, + ('report', 'analysis'): {'icon': 'file-text', 'title': '问题分析 & 报告', 'subtitle': 'workflow.md'}, + ('hypothesis', 'analysis'): {'icon': 'dna', 'title': '形成假设', 'subtitle': 'iteration_log hypothesis 字段'}, + ('log', 'analysis'): {'icon': 'clipboard-list', 'title': '记录迭代日志', 'subtitle': 'iteration_log.jsonl'}, +} + +# Order within a section. +_CARD_ORDER: dict[tuple[str, str], int] = { + # baseline + ('cml', 'baseline'): 0, + ('gold-drift', 'baseline'): 1, + ('dist-analysis', 'baseline'): 2, + ('report', 'baseline'): 3, + ('hypothesis', 'baseline'): 4, + ('log', 'baseline'): 5, + # train + ('augment', 'train'): 0, + ('verify', 'train'): 1, + ('sft', 'train'): 2, + # analysis + ('cml', 'analysis'): 0, + ('gold-drift', 'analysis'): 1, + ('dist-analysis', 'analysis'): 2, + ('report', 'analysis'): 3, + ('hypothesis', 'analysis'): 4, + ('log', 'analysis'): 5, +} + +_SECTION_LABELS: dict[str, str] = { + 'baseline': 'Baseline', + 'train': 'Train', + 'analysis': 'Analysis', +} + +_SECTION_DESCRIPTIONS: dict[str, str] = { + 'baseline': 'Establish baseline performance and metrics', + 'train': 'Form hypothesis & retrain on augmented data', + 'analysis': 'Evaluate new checkpoint & diagnose regression', +} + + +def _parse_run_index(run_id: str | None) -> int | None: + """'R0' → 0, 'R12' → 12, anything else → None.""" + if not isinstance(run_id, str): + return None + m = re.match(r'^[Rr](\d+)$', run_id.strip()) + if not m: + return None + try: + return int(m.group(1)) + except ValueError: + return None + + +def _infer_run_id( + entry: dict[str, Any], + iteration_log_count: int, + *, + same_step_hint: str | None = None, +) -> str | None: + """Resolve which round this state entry belongs to. + + 1. explicit entry.run_id wins — UNLESS it's semantically invalid (a train + step tagged R0; R0 is baseline-only by design). In that case we treat + the explicit value as a typo and fall through to inference. + 2. else: same_step_hint (run_id from nearest same-step entry) — handles + stateless writes like the watcher's complete/failed entries that lack + run_id but should obviously inherit the round of the agent-tagged + cml/running line they accompany. + 3. else: derive from kind + iteration_log line count + - analysis steps: evaluating R{count}'s output → R{count} + - train steps: preparing R{count+1} → R{count+1} + - gates / boundary: cannot infer alone — caller back-fills from neighbor + """ + step = entry.get('step') + kind = _STEP_KIND.get(step) if isinstance(step, str) else None + + explicit = entry.get('run_id') + if isinstance(explicit, str) and explicit.strip(): + explicit_idx = _parse_run_index(explicit) + if explicit_idx is not None: + # R0+train is a common agent typo: hypothesis/augment/verify/sft are + # "preparing R1", but agents sometimes tag them with the round they + # see in the report. We rewrite to R1 specifically (NOT current-time + # inference) so retroactive iteration_log changes don't shift the + # cards into R{n>=2}. + if kind == 'train' and explicit_idx == 0: + return 'R1' + return explicit.strip() + + if not isinstance(step, str): + return None + if same_step_hint: + return same_step_hint + if kind == 'analysis': + return f'R{iteration_log_count}' + if kind == 'train': + return f'R{max(iteration_log_count, 0) + 1}' + return None + + +def _build_same_step_hints( + state_entries: list[dict[str, Any]], +) -> list[str | None]: + """For each entry, find the nearest same-step entry (by index distance) + that carries an explicit, semantically-valid run_id, and return its + run_id. Used as a fallback for stateless writes (e.g. watcher's complete + line lacks run_id but the agent's cml/running line right above has it). + + Returns a parallel list aligned to state_entries; None for entries with + no usable neighbor. + """ + explicit_by_step: dict[str, list[tuple[int, str]]] = {} + for idx, entry in enumerate(state_entries): + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind in (None, 'boundary', 'gate'): + continue + rid = entry.get('run_id') + if not (isinstance(rid, str) and rid.strip()): + continue + parsed = _parse_run_index(rid) + if parsed is None: + continue + # Apply same R0+train typo rewrite as _infer_run_id so the hint stays + # consistent with what _infer_run_id would have returned for that + # entry on its own. + canonical = 'R1' if (kind == 'train' and parsed == 0) else rid.strip() + explicit_by_step.setdefault(step, []).append((idx, canonical)) + + hints: list[str | None] = [None] * len(state_entries) + for idx, entry in enumerate(state_entries): + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + candidates = explicit_by_step.get(step) + if not candidates: + continue + best = min(candidates, key=lambda ix_rid: abs(ix_rid[0] - idx)) + hints[idx] = best[1] + return hints + + +def _section_type_for(step: str, run_index: int) -> str | None: + """Which section a card belongs to within its round. + + Train steps under R0 are filtered out by `_infer_run_id` (R0 has no train + phase). If one slips through here, we drop it rather than aliasing to + baseline — silently mixing train cards into a baseline section is what + caused hypothesis to render under R0·Baseline. + """ + kind = _STEP_KIND.get(step) + if kind == 'analysis': + return 'baseline' if run_index == 0 else 'analysis' + if kind == 'train': + return 'train' if run_index >= 1 else None + return None + + +def _derive_section_status(card_statuses: list[str]) -> str: + if not card_statuses: + return 'pending' + if all(s == 'complete' for s in card_statuses): + return 'complete' + if 'failed' in card_statuses and not any(s in ('running', 'complete') for s in card_statuses): + return 'failed' + if 'running' in card_statuses or 'complete' in card_statuses: + return 'running' + return 'pending' + + +def _build_pipeline_items( + state_entries: list[dict[str, Any]], + iteration_log_count: int, +) -> list[dict[str, Any]]: + """Assemble the items[] payload from raw state entries. + + Returns an ordered list of {type:'round', ...} and {type:'gate', ...} items + in display order. Each item shows agent-written progress only — sections + not yet touched do not appear. + """ + # 1. Walk entries: keep latest per step (per round-namespaced key). + # We namespace by run_id so the same step name on different rounds + # doesn't overwrite each other. + latest_step_by_key: dict[str, dict[str, Any]] = {} + latest_gate_by_key: dict[str, dict[str, Any]] = {} + section_first_ts: dict[tuple[str, str], str] = {} # (run_id, section_type) → earliest ts + last_non_gate_run_id: str | None = None + gate_position_index: dict[str, int] = {} # gate_key → position in entries (for ordering) + same_step_hints = _build_same_step_hints(state_entries) + + for idx, entry in enumerate(state_entries): + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind == 'boundary': + continue + if kind == 'gate': + run_id = entry.get('run_id') + if not (isinstance(run_id, str) and run_id.strip()): + run_id = last_non_gate_run_id + if not run_id: + continue + run_id = run_id.strip() + gate_key = f'{run_id}:{step}' + existing = latest_gate_by_key.get(gate_key) + if existing is None: + latest_gate_by_key[gate_key] = { + **entry, + '_run_id': run_id, + '_first_ts': entry.get('ts', ''), + } + gate_position_index[gate_key] = idx + else: + existing.update({k: v for k, v in entry.items() if k != 'ts'}) + # keep first ts for ordering + continue + + # analysis or train + run_id = _infer_run_id( + entry, iteration_log_count, same_step_hint=same_step_hints[idx] + ) + if not run_id: + continue + last_non_gate_run_id = run_id + run_index = _parse_run_index(run_id) + if run_index is None: + continue + section_type = _section_type_for(step, run_index) + if section_type is None: + continue + card_key = f'{run_id}:{step}' + ts = entry.get('ts', '') + if card_key not in latest_step_by_key: + latest_step_by_key[card_key] = { + **entry, + '_run_id': run_id, + '_section_type': section_type, + '_first_ts': ts, + } + else: + existing = latest_step_by_key[card_key] + for k, v in entry.items(): + existing[k] = v + section_key = (run_id, section_type) + if section_key not in section_first_ts: + section_first_ts[section_key] = ts + + # 2. Group cards by section. + sections: dict[tuple[str, str], dict[str, Any]] = {} + for card_key, entry in latest_step_by_key.items(): + run_id = entry['_run_id'] + section_type = entry['_section_type'] + section_key = (run_id, section_type) + if section_key not in sections: + sections[section_key] = { + 'run_id': run_id, + 'section_type': section_type, + 'cards': [], + 'first_ts': section_first_ts.get(section_key, ''), + } + step = entry['step'] + meta = _CARD_META.get((step, section_type), { + 'icon': 'clock', + 'title': step, + 'subtitle': '', + }) + order = _CARD_ORDER.get((step, section_type), 999) + card: dict[str, Any] = { + 'key': card_key, + 'step': step, + 'icon': meta['icon'], + 'title': meta['title'], + 'subtitle': meta['subtitle'], + 'status': entry.get('status') or 'pending', + '_order': order, + } + if 'progress' in entry: + try: + value = float(entry['progress']) + if value > 1: + value /= 100 + card['progress'] = max(0.0, min(1.0, value)) + except (TypeError, ValueError): + pass + sections[section_key]['cards'].append(card) + + # 3. Sort cards within each section. + for sec in sections.values(): + sec['cards'].sort(key=lambda c: c.get('_order', 999)) + for c in sec['cards']: + c.pop('_order', None) + + # 4. Build round items in display order. + # Order: by run_index ascending, then within same round: train before analysis. + section_order = {'baseline': 0, 'train': 0, 'analysis': 1} + section_items: list[dict[str, Any]] = [] + for (run_id, section_type), sec in sections.items(): + run_index = _parse_run_index(run_id) + if run_index is None: + continue + statuses = [c['status'] for c in sec['cards']] + round_status = _derive_section_status(statuses) + label = f'{run_id} · {_SECTION_LABELS[section_type]}' + section_items.append({ + 'type': 'round', + 'run_id': run_id, + 'run_index': run_index, + 'section_type': section_type, + 'label': label, + 'description': _SECTION_DESCRIPTIONS[section_type], + 'status': round_status, + 'cards': sec['cards'], + '_order_key': (run_index, section_order.get(section_type, 9), sec.get('first_ts', '')), + }) + + # 5. Build gate items. + gate_items: list[dict[str, Any]] = [] + for gate_key, entry in latest_gate_by_key.items(): + run_id = entry['_run_id'] + run_index = _parse_run_index(run_id) + if run_index is None: + continue + step = entry['step'] + meta_g = _GATE_TITLES.get(step, {'title': step, 'description': ''}) + # Place gate after the latest section of this run_id (so use a high + # secondary order). Use position index for tie-break across same round. + raw_status = entry.get('status') or 'pending' + # Gate running → 'waiting': the agent has surfaced the checkpoint and + # ended its turn. There's no work in flight on the agent side; we're + # waiting on the user. The frontend renders 'waiting' with a distinct + # amber pill ("等待人工确认") instead of the spinning blue "IN PROGRESS" + # used for actual running steps. + gate_status = 'waiting' if raw_status == 'running' else raw_status + gate_items.append({ + 'type': 'gate', + 'key': gate_key, + 'run_id': run_id, + 'run_index': run_index, + 'gate_kind': step, + 'title': meta_g['title'], + 'description': meta_g['description'], + 'status': gate_status, + 'reason': entry.get('reason'), + '_order_key': (run_index, 99, gate_position_index.get(gate_key, 0)), + }) + + # 6. Filter: drop empty/pending sections (no cards or all pending). Running + # and complete sections both render — RoundSection's status pill conveys + # the difference, so the transition complete→running on the same numbered + # box is in-place rather than a separate chip strip popping in/out. + section_items = [ + s for s in section_items if s['status'] in ('running', 'complete', 'failed') + ] + + # 7. Merge and sort all items by (run_index, section_order, ts/position). + all_items = section_items + gate_items + all_items.sort(key=lambda x: x['_order_key']) + + # 8. Attach 1-based numeric index for display ("01", "02", ...) after + # filtering so numbering is dense (no gaps from hidden running sections). + for i, item in enumerate(all_items, start=1): + item['index'] = i + item.pop('_order_key', None) + + return all_items + + +def _compute_in_flight( + state_entries: list[dict[str, Any]], + iteration_log_count: int, +) -> dict[str, Any] | None: + """Render the in-progress phase as an inline chip row. + + The wrapping section only appears once the whole phase is COMPLETE. + Until then we still want the user to see what's been done in the current + phase: every step that has reached `complete` plus the one currently + `running`. Pending steps stay hidden — they appear when their turn + comes and replace nothing. + + Returns a single object describing the in-progress phase + its + progressively-filling cards, in canonical step order. None when no + phase is currently running (everything settled, or the latest signal + is a gate/boundary). + """ + same_step_hints = _build_same_step_hints(state_entries) + target_run_id: str | None = None + target_section: str | None = None + seen_keys: set[str] = set() + for idx in range(len(state_entries) - 1, -1, -1): + entry = state_entries[idx] + if entry.get('kpi') or entry.get('log'): + continue + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind in ('boundary', 'gate'): + continue + run_id = _infer_run_id( + entry, iteration_log_count, same_step_hint=same_step_hints[idx] + ) + if not run_id: + continue + key = f'{run_id}:{step}' + if key in seen_keys: + continue + seen_keys.add(key) + if entry.get('status') != 'running': + continue + run_index = _parse_run_index(run_id) + if run_index is None: + continue + section_type = _section_type_for(step, run_index) + if section_type is None: + continue + target_run_id = run_id + target_section = section_type + break + + if not target_run_id or not target_section: + return None + + latest_per_step: dict[str, dict[str, Any]] = {} + for idx, entry in enumerate(state_entries): + if entry.get('kpi') or entry.get('log'): + continue + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind in ('boundary', 'gate'): + continue + run_id = _infer_run_id( + entry, iteration_log_count, same_step_hint=same_step_hints[idx] + ) + if run_id != target_run_id: + continue + run_index = _parse_run_index(run_id) + if run_index is None: + continue + section_type = _section_type_for(step, run_index) + if section_type != target_section: + continue + latest_per_step[step] = entry + + cards: list[dict[str, Any]] = [] + for step, entry in latest_per_step.items(): + status = entry.get('status') + if status not in ('complete', 'running'): + continue + meta = _CARD_META.get((step, target_section), { + 'icon': 'clock', + 'title': step, + 'subtitle': '', + }) + progress: float | None = None + if 'progress' in entry: + try: + value = float(entry['progress']) + if value > 1: + value /= 100 + progress = max(0.0, min(1.0, value)) + except (TypeError, ValueError): + pass + cards.append({ + 'key': f'{target_run_id}:{step}', + 'step': step, + 'icon': meta['icon'], + 'title': meta['title'], + 'subtitle': meta['subtitle'], + 'status': status, + 'progress': progress, + }) + + if not cards: + return None + + cards.sort(key=lambda c: _CARD_ORDER.get((c['step'], target_section), 999)) + + return { + 'run_id': target_run_id, + 'section_type': target_section, + 'section_label': f'{target_run_id} · {_SECTION_LABELS[target_section]}', + 'cards': cards, + } + + def _apply_program_state( payload: dict[str, Any], state_entries: list[dict[str, Any]], + *, + iteration_log_count: int = 0, + session_failure_status: str | None = None, + run_active: bool | None = None, + session_id: str | None = None, ) -> None: - """Mutate payload in place. State entries override card status/progress and - append log lines. KPI entries (legacy `{kpi:..,value:..}`) are ignored — + """Mutate payload in place. Builds items[] from state entries and appends + log lines. KPI entries (legacy `{kpi:..,value:..}`) are ignored — KPIs are computed by the backend in _compute_kpis.""" - if not state_entries: - return - latest_by_step: dict[str, dict[str, Any]] = {} + # Append logs first (they are independent of items[]). appended_logs: list[dict[str, Any]] = [] for entry in state_entries: if entry.get('kpi'): - # Backend now owns KPI rendering — explicitly ignore agent-written - # kpi entries to avoid drift between displayed value and authoritative - # source (config.yaml / iteration_log / metric_diff). continue if entry.get('log'): log_obj = entry.get('log') @@ -4903,76 +6305,144 @@ def _apply_program_state( appended_logs.append(log_obj) elif isinstance(log_obj, str): appended_logs.append({'text': log_obj, 'ts': entry.get('ts', '')}) - continue - step = entry.get('step') - if isinstance(step, str) and step: - latest_by_step[step] = entry if appended_logs: existing = payload.setdefault('logs', []) existing.extend(appended_logs) - if not latest_by_step: - # logs may still have been applied; status derivation runs below. - latest_by_step = {} + # Build items (already filtered to complete sections + visible gates). + items = _build_pipeline_items(state_entries, iteration_log_count) + payload['items'] = items + payload['in_flight'] = _compute_in_flight(state_entries, iteration_log_count) - for step, entry in latest_by_step.items(): - applied = False - for phase in payload.get('phases', []): - for card in phase.get('cards', []): - if _step_matches_card(step, card): - status = entry.get('status') - if isinstance(status, str) and status: - card['status'] = status - if 'progress' in entry: - try: - value = float(entry['progress']) - if value > 1: - value /= 100 - card['progress'] = max(0.0, min(1.0, value)) - except (TypeError, ValueError): - pass - applied = True - break - if applied: - break + # Derive overall status from RAW entries — items[] is filtered to complete + # sections only, so deriving from items would falsely report 'complete' + # whenever a running section is hidden. + latest_per_step: dict[str, str] = {} + for entry in state_entries: + if entry.get('kpi') or entry.get('log'): + continue + step = entry.get('step') + status = entry.get('status') + if isinstance(step, str) and step and isinstance(status, str) and status: + # boundary steps don't count toward overall progress + if _STEP_KIND.get(step) == 'boundary': + continue + latest_per_step[step] = status - # Re-derive phase tones and overall state. - # 规则：卡片有任何 "已开始过"（running 或 complete）的活动 → 阶段/整体 - # 都视为 running（迭代进行中）。只有全部 pending 才是 idle，全部 complete - # 才是真的完成。 - for phase in payload.get('phases', []): - statuses = [c.get('status') for c in phase.get('cards', [])] - if not statuses: - phase['tone'] = 'pending' - elif all(s == 'complete' for s in statuses): - phase['tone'] = 'complete' - elif 'running' in statuses or 'complete' in statuses: - phase['tone'] = 'running' - elif 'failed' in statuses: - phase['tone'] = 'pending' - else: - phase['tone'] = 'pending' - - all_cards = [ - c for phase in payload.get('phases', []) for c in phase.get('cards', []) - ] - if not all_cards: - return - statuses = [c.get('status') for c in all_cards] - if all(s == 'complete' for s in statuses): + non_gate_running = any( + status == 'running' and _STEP_KIND.get(step) != 'gate' + for step, status in latest_per_step.items() + ) + gate_running = any( + status == 'running' and _STEP_KIND.get(step) == 'gate' + for step, status in latest_per_step.items() + ) + statuses = list(latest_per_step.values()) + if not statuses: + payload['status']['state'] = 'pending' + elif non_gate_running: + payload['status']['state'] = 'running' + elif gate_running: + # Only HiTL gates are 'running' — agent has handed control back to the + # user. Surface as 'waiting' so the header pill says "等待人工" instead + # of the spinning "Running" badge. + payload['status']['state'] = 'waiting' + elif all(s == 'complete' for s in statuses): payload['status']['state'] = 'complete' - elif 'running' in statuses or 'complete' in statuses: - # 有任意进展（在跑或完成过任一步）→ 整体迭代是运行中 + elif 'complete' in statuses: payload['status']['state'] = 'running' elif 'failed' in statuses: payload['status']['state'] = 'failed' else: payload['status']['state'] = 'pending' + + # Orphan detection: agent wrote `step:running` then turn ended cleanly + # without a closing `complete`/`failed` entry — run_state_store says the + # run is no longer active, but the pipeline still claims `running`. Fold + # this into session_failure_status='failed' so the same red-failed visual + # applies (per user UX choice). Note: only triggers when there's at least + # one explicit running step — purely-pending state stays pending. + # + # Exception: if every running step has an active watcher polling for it, + # it's NOT an orphan — agent legitimately ended its turn and is waiting + # for a remote file to land via the watcher. The watcher will write the + # closing entry when it triggers (file_exists / timeout / cancel). + # + # Gates (human-check / human-review) are also never orphans: agent writes + # `step:running` to surface the gate card, ends the turn, and waits for + # the user to reply. There's no watcher and no remote file — the closing + # `complete` entry comes from the next user-driven turn. + running_steps = { + step for step, status in latest_per_step.items() + if status == 'running' and _STEP_KIND.get(step) != 'gate' + } + all_watched = bool(running_steps) and all( + _watcher_manager.is_watching(session_id, step) + for step in running_steps + if session_id + ) + if ( + run_active is False + and session_failure_status is None + and payload['status']['state'] == 'running' + and running_steps + and not all_watched + ): + session_failure_status = 'failed' + + # Session-level failure (agent process died / cancelled) overrides the + # state derived from program-state.jsonl. The state file can't write its + # own tombstone when the writer is the dead process — we cross-reference + # session.json to surface the failure on the monitor panel. + if session_failure_status in ('failed', 'cancelled', 'interrupted'): + # Cancelled keeps a softer 'cancelled' label so the UI can distinguish + # user-stop from crash; everything else maps to 'failed' (red). + target_state = 'cancelled' if session_failure_status == 'cancelled' else 'failed' + if payload['status']['state'] != 'complete': + payload['status']['state'] = target_state + in_flight = payload.get('in_flight') + if isinstance(in_flight, dict): + cards = in_flight.get('cards') + if isinstance(cards, list): + for card in cards: + if isinstance(card, dict) and card.get('status') == 'running': + card['status'] = target_state + # Same treatment for any running cards inside items[] — running + # round sections also get flipped, and the section status follows. + for item in payload.get('items', []): + if not isinstance(item, dict): + continue + if item.get('type') == 'round': + section_cards = item.get('cards', []) + flipped = False + if isinstance(section_cards, list): + for card in section_cards: + if isinstance(card, dict) and card.get('status') == 'running': + card['status'] = target_state + flipped = True + if flipped or item.get('status') == 'running': + item['status'] = target_state + elif item.get('type') == 'gate' and item.get('status') == 'running': + item['status'] = target_state def _autoresearch_root() -> Path: return Path(os.environ.get('AUTORESEARCH_ROOT', '/mnt/wangsenhao/autoresearch-zk')) +def _chat_root_for_runtime( + runtime: 'JupyterRuntimeSession | None', +) -> str | None: + """Resolve the per-chat autoresearch root as a REMOTE path string. + + Returns the NFS path under /mnt/wangsenhao/autoresearch-zk-users/// + so jupyter pod and SFT training pod both read/write the same directory. + Returns None when no runtime is bound — caller decides whether to fall + back to the legacy global path.""" + if runtime is None: + return None + return runtime.chat_workspace_root + + def _run_history_dir() -> Path: return Path( os.environ.get( @@ -5038,15 +6508,125 @@ def _scan_run_history_max() -> int | None: return best if best >= 0 else None -def _read_iteration_log_count() -> int | None: - path = _autoresearch_root() / 'results' / 'iteration_log.jsonl' - try: - if not path.is_file(): +def _read_iteration_log_count( + session_id: str | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, +) -> int | None: + """Count non-empty lines in iteration_log.jsonl. + + Chat-root lives under the remote workspace_cwd, so reads MUST go through + the bound jupyter runtime — backend host typically can't read this path + locally. Falls back to the legacy global path on the local fs only when + no runtime is bound (session never connected, or transient lookup miss). + """ + runtime = ( + _jupyter_runtime_for_session(agent_state, account_id, session_id) + if session_id and agent_state is not None + else None + ) + remote_root = _chat_root_for_runtime(runtime) + if runtime is not None and remote_root is not None: + remote_path = f'{remote_root}/results/iteration_log.jsonl' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& grep -c "[^[:space:]]" {shlex.quote(remote_path)} ' + f'|| echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=10.0, + max_output_chars=2000, + ) + except Exception: # noqa: BLE001 return None - with path.open('r', encoding='utf-8') as fp: - return sum(1 for line in fp if line.strip()) - except (OSError, UnicodeDecodeError): + stdout = (result.stdout or '').strip() if result is not None else '' + if stdout and stdout != '__MISSING__': + try: + return int(stdout.splitlines()[-1]) + except (ValueError, IndexError): + return None return None + legacy = _autoresearch_root() / 'results' / 'iteration_log.jsonl' + try: + if legacy.is_file(): + with legacy.open('r', encoding='utf-8') as fp: + return sum(1 for line in fp if line.strip()) + except (OSError, UnicodeDecodeError): + pass + return None + + +def _read_iteration_log_h1_label( + runtime: 'JupyterRuntimeSession | None', + run_dic: int | None, + *, + max_chars: int = 32, +) -> str | None: + """读取 iteration_log.jsonl，提取与本轮 dist-analysis 候选集对应的 H1 假设短标签。 + + 优先匹配 runDic 一致的 entry，否则取最新一条；从 ``hypothesis`` / + ``next_hypothesis`` 字段切第一句并截断。读不到或无 iteration_log 返回 None + （调用方据此决定是否在标题里加括号）。 + """ + if runtime is None: + return None + chat_root = _chat_root_for_runtime(runtime) + if not chat_root: + return None + remote_path = f'{chat_root}/results/iteration_log.jsonl' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& cat {shlex.quote(remote_path)} ' + f'|| echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, timeout_seconds=10.0, max_output_chars=200_000 + ) + except Exception: # noqa: BLE001 + return None + stdout = (result.stdout or '').strip() if result is not None else '' + if not stdout or stdout == '__MISSING__': + return None + entries: list[dict[str, Any]] = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except (ValueError, TypeError): + continue + if isinstance(obj, dict): + entries.append(obj) + if not entries: + return None + chosen: dict[str, Any] | None = None + if run_dic is not None: + for entry in reversed(entries): + rd = entry.get('runDic') or entry.get('run_dic') + if isinstance(rd, str) and rd.isdigit(): + rd = int(rd) + if rd == run_dic: + chosen = entry + break + if chosen is None: + chosen = entries[-1] + for field in ('hypothesis', 'next_hypothesis'): + val = chosen.get(field) + if isinstance(val, str) and val.strip(): + label = val.strip() + for sep in ('；', ';', '。', '\n'): + if sep in label: + label = label.split(sep, 1)[0].strip() + break + if len(label) > max_chars: + label = label[: max_chars - 1] + '…' + return label or None + return None def _extract_trigger_from_state(state_entries: list[dict[str, Any]]) -> str | None: @@ -5225,19 +6805,79 @@ def _refresh_target_set_cache_blocking( _llm_extract_target_set(model_config, text) -def _read_workflow_metric_diff(run_id: int) -> dict[str, Any] | None: - path = ( - _run_history_dir() - / f'workflow{run_id}' - / 'metric_diff' - / 'lark_template.json' - ) - try: - if not path.is_file(): - return None - return json.loads(path.read_text(encoding='utf-8')) - except (OSError, json.JSONDecodeError): +_METRIC_DIFF_CACHE: dict[int, dict[str, Any]] = {} +_METRIC_DIFF_NEG_RETRY_AT: dict[int, float] = {} +_METRIC_DIFF_NEG_TTL_SECONDS = 30.0 + + +def _read_workflow_metric_diff( + run_id: int, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, +) -> dict[str, Any] | None: + """Read workflow/metric_diff/lark_template.json. + + metric_diff lives on /mnt/xiaoai-zk-model-train-tj5/... which is mounted on + the remote jupyter workspace but typically not on the backend host. So we + try local first, then fall back to `cat` over the session's jupyter runtime + if available. lark_template.json is written once by cml workflow and never + changes, so successful reads are cached forever; misses honor a 30s + negative cache to avoid hammering the remote during the cml-running phase. + """ + cached = _METRIC_DIFF_CACHE.get(run_id) + if cached is not None: + return cached + now = time.monotonic() + next_retry = _METRIC_DIFF_NEG_RETRY_AT.get(run_id) + if next_retry is not None and now < next_retry: return None + relative = f'workflow{run_id}/metric_diff/lark_template.json' + local_path = _run_history_dir() / relative + try: + if local_path.is_file(): + data = json.loads(local_path.read_text(encoding='utf-8')) + if isinstance(data, dict): + _METRIC_DIFF_CACHE[run_id] = data + _METRIC_DIFF_NEG_RETRY_AT.pop(run_id, None) + return data + except (OSError, json.JSONDecodeError): + pass + if agent_state is not None and session_id is not None: + runtime = _jupyter_runtime_for_session(agent_state, account_id, session_id) + if runtime is not None: + remote_path = f'{_run_history_dir()}/{relative}' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& cat {shlex.quote(remote_path)} || echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=10.0, + max_output_chars=200_000, + ) + except Exception: # noqa: BLE001 + _METRIC_DIFF_NEG_RETRY_AT[run_id] = now + _METRIC_DIFF_NEG_TTL_SECONDS + return None + stdout = (result.stdout or '').strip() if result is not None else '' + if ( + result is not None + and result.exit_code == 0 + and stdout + and stdout != '__MISSING__' + ): + try: + data = json.loads(stdout) + if isinstance(data, dict): + _METRIC_DIFF_CACHE[run_id] = data + _METRIC_DIFF_NEG_RETRY_AT.pop(run_id, None) + return data + except json.JSONDecodeError: + pass + _METRIC_DIFF_NEG_RETRY_AT[run_id] = now + _METRIC_DIFF_NEG_TTL_SECONDS + return None def _format_metric_value(metrics: dict[str, Any] | None, key: str) -> str: @@ -5253,6 +6893,84 @@ def _format_metric_value(metrics: dict[str, Any] | None, key: str) -> str: return str(value) +# lark_template.json 的指标都嵌在 eval_output_info 文本里，行格式： +# 🟢 specific_test || icl_test: 86.11% (1873/2175) -> 86.11% (1873/2175) (+0.00%) +# 我们要 "->" 后面那个值（after-migration pass rate）。 +_EVAL_LINE_RE = re.compile( + r'^\s*[🟢🔴]\s*(?P.+?):\s*[\d.]+%\s*\([^)]+\)\s*->\s*(?P[\d.]+)%' +) + + +def _extract_eval_metric(eval_output_info: str | None, target_path: str) -> float | None: + """Find the line whose 'path' (the bit between emoji and ":") equals + target_path, return the post-value as a 0-100 float, or None. + + Match is exact-equality first, then falls back to path-tail equality — + eval lines often carry full relative paths like + `specific_test || data/specific_test_set/icl_test/.csv` + while target_path is just `specific_test || .csv`.""" + if not isinstance(eval_output_info, str) or not eval_output_info: + return None + target = target_path.strip() + # Strip the optional ` || ` so we can suffix-compare just the path tail. + target_tail = target.split('||', 1)[-1].strip() if '||' in target else target + for line in eval_output_info.splitlines(): + m = _EVAL_LINE_RE.match(line) + if not m: + continue + path = m.group('path').strip() + path_tail = path.split('||', 1)[-1].strip() if '||' in path else path + matched = path == target or path_tail == target_tail or path_tail.endswith( + f'/{target_tail}' + ) + if matched: + try: + return float(m.group('post')) + except ValueError: + return None + return None + + +def _enrich_metrics_from_eval_output( + metrics: dict[str, Any] | None, target_set: str | None +) -> None: + """Backfill structured pass-rate keys parsed out of eval_output_info text. + Mutates metrics in place. No-op if metrics is None or already has the keys.""" + if not metrics: + return + eval_text = metrics.get('eval_output_info') + if not isinstance(eval_text, str): + return + # _format_metric_value 把 0..1 当作 fraction 格式化为 XX.XX%，所以这里 /100 入库。 + # specific_test 大盘 + if metrics.get('specific_test_pass_rate') is None: + v = _extract_eval_metric(eval_text, 'specific_test') + if v is not None: + metrics['specific_test_pass_rate'] = v / 100.0 + # 大盘 overrall || 车载（注意 lark template 里就是 "overrall" 拼写，不是 "overall"） + if metrics.get('overall_car_pass_rate') is None: + v = _extract_eval_metric(eval_text, 'overrall || 车载') + if v is not None: + metrics['overall_car_pass_rate'] = v / 100.0 + # target set: specific_test || + # Trigger 里带 .csv 后缀；lark template 偶尔写不带后缀的。两种都试。 + if ( + metrics.get('target_set_pass_rate') is None + and target_set + and target_set != '—' + ): + candidates = [target_set] + if target_set.endswith('.csv'): + candidates.append(target_set[:-4]) + else: + candidates.append(f'{target_set}.csv') + for name in candidates: + v = _extract_eval_metric(eval_text, f'specific_test || {name}') + if v is not None: + metrics['target_set_pass_rate'] = v / 100.0 + break + + def _compute_step0_start_seconds( state_entries: list[dict[str, Any]], ) -> float | None: @@ -5292,20 +7010,49 @@ def _compute_kpis( session_id: str | None, state_entries: list[dict[str, Any]], model_config: ModelConfig | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, ) -> list[dict[str, Any]]: - """Compute the 7 top KPIs from authoritative sources. Falls back to '—'.""" - run_max = _scan_run_history_max() + """Compute the 7 top KPIs from authoritative sources. Falls back to '—'. + + RUN ID prefers the runDic this session has already written into + program-state.jsonl — that's the only source guaranteed to belong to THIS + chat. Falls back to a global scan of RUN_HISTORY_DIR for sessions that + haven't logged a runDic yet (e.g. baseline pre-cml). Without this priority, + a concurrently-running chat that allocated a larger runDic on the shared + /mnt/xiaoai-zk-model-train-tj5/workflow5/ would shadow this chat's value. + Metric reads use the same jupyter runtime the watcher uses, so /mnt/* paths + reachable only on the remote workspace still work. + """ + run_max = _resolve_run_dic_from_state(state_entries) + if run_max is None: + run_max = _scan_run_history_max() run_id_value = str(run_max) if run_max is not None else '—' version = _read_workflow_version() or '—' trigger = _extract_trigger_from_state(state_entries) or _extract_trigger_from_session( sessions_dir, session_id ) target_set = _parse_target_set(trigger, model_config=model_config) or '—' - metrics = _read_workflow_metric_diff(run_max) if run_max is not None else None + metrics = ( + _read_workflow_metric_diff( + run_max, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + if run_max is not None + else None + ) + _enrich_metrics_from_eval_output(metrics, target_set) target_metric = _format_metric_value(metrics, 'target_set_pass_rate') overall_metric = _format_metric_value(metrics, 'overall_car_pass_rate') specific_metric = _format_metric_value(metrics, 'specific_test_pass_rate') - iter_count = _read_iteration_log_count() + iter_count = _read_iteration_log_count( + session_id, + agent_state=agent_state, + account_id=account_id, + ) if iter_count is None: iteration = 'R0-baseline' elif iter_count == 0: @@ -5330,53 +7077,73 @@ def _compute_kpis( def _resolve_run_dic_from_state( state_entries: list[dict[str, Any]], ) -> int | None: - """Extract runDic from any place agent might leave it in program-state.jsonl: - 1. Top-level `runDic` field on step entries - 2. Any string field containing `workflow` or `runDic=N` (covers - report_path, log.text, watch.path, etc.) - 3. Latest occurrence wins (reverse iteration).""" + """Extract runDic from program-state.jsonl. + + Priority: + 1. Explicit ``runDic`` field on any entry (latest wins). + 2. Fallback: scan string fields recursively for ``workflow`` / + ``runDic=N`` and pick the **max** match. Path strings often contain + both the global mount root (``/mnt/.../workflow5/...``) and the actual + run number (``workflow17793/...``) — max picks the run number, not the + fixed mount-dir digit. + """ workflow_re = re.compile(r'workflow(\d+)') rundic_re = re.compile(r'runDic\s*[=:取]?\s*(\d+)') def _scan_strings(obj: Any) -> int | None: if isinstance(obj, str): - m = workflow_re.search(obj) - if m: - return int(m.group(1)) - m = rundic_re.search(obj) - if m: - return int(m.group(1)) - elif isinstance(obj, dict): + candidates = [int(x) for x in workflow_re.findall(obj)] + candidates += [int(x) for x in rundic_re.findall(obj)] + return max(candidates) if candidates else None + if isinstance(obj, dict): + best: int | None = None for v in obj.values(): hit = _scan_strings(v) - if hit is not None: - return hit - elif isinstance(obj, list): + if hit is not None and (best is None or hit > best): + best = hit + return best + if isinstance(obj, list): + best = None for v in obj: hit = _scan_strings(v) - if hit is not None: - return hit + if hit is not None and (best is None or hit > best): + best = hit + return best return None + # 1. Explicit field — latest wins. for entry in reversed(state_entries): - # 1. 显式字段 runDic run_dic = entry.get('runDic') or entry.get('run_dic') or entry.get('rundic') if isinstance(run_dic, int): return run_dic if isinstance(run_dic, str) and run_dic.isdigit(): return int(run_dic) - # 2. 扫所有字符串字段（递归） + + # 2. Fallback string scan — take max across all entries. + best: int | None = None + for entry in state_entries: hit = _scan_strings(entry) - if hit is not None: - return hit - return None + if hit is not None and (best is None or hit > best): + best = hit + return best -def _step_artifact_paths(step: str, run_dic: int | None) -> list[str]: - """Map step key → ordered list of remote artifact paths to try.""" - auto_root = os.environ.get( - 'AUTORESEARCH_ROOT', '/mnt/wangsenhao/autoresearch-zk' - ) +def _step_artifact_paths( + step: str, + run_dic: int | None, + runtime: 'JupyterRuntimeSession | None' = None, +) -> list[str]: + """Map step key → ordered list of remote artifact paths to try. + + Per-chat artifacts (results/, ai-planning/, output/) live under the chat + workspace root on shared NFS at + `/mnt/wangsenhao/autoresearch-zk-users///`. Reads still + go through the bound jupyter runtime since the path is only mounted there. + metric_diff (cml step) stays on the global RUN_HISTORY_DIR mount that is + shared across chats. When no runtime is bound, falls back to the legacy + global AUTORESEARCH_ROOT so KPI fetches don't crash on unbound sessions. + """ + chat_root = _chat_root_for_runtime(runtime) or str(_autoresearch_root()) metric_root = os.environ.get( 'RUN_HISTORY_DIR', '/mnt/xiaoai-zk-model-train-tj5/workflow5' ) @@ -5391,32 +7158,129 @@ def _step_artifact_paths(step: str, run_dic: int | None) -> list[str]: ) elif step == 'gold-drift': if run_dic is not None: - paths.append(f'{auto_root}/results/gold_drift/drift_{run_dic}.json') - elif step in {'dist-analysis', 'report'}: + paths.append(f'{chat_root}/results/gold_drift/drift_{run_dic}.json') + elif step == 'dist-analysis': if run_dic is not None: - paths.append(f'{auto_root}/results/workflow{run_dic}.md') + paths.append(f'{chat_root}/results/workflow{run_dic}.md') + paths.append( + f'{chat_root}/output/relabel_candidates_{run_dic}.csv' + ) + # 后向兼容：旧 agent 把 csv 写到 jupyter pod 本地 $(pwd)/output/ + if runtime is not None: + paths.append( + f'{runtime.binding.workspace_cwd}/output/relabel_candidates_{run_dic}.csv' + ) + elif step == 'report': + if run_dic is not None: + paths.append(f'{chat_root}/results/workflow{run_dic}.md') elif step == 'hypothesis': - paths.append(f'{auto_root}/results/iteration_log.jsonl') + paths.append(f'{chat_root}/results/iteration_log.jsonl') elif step == 'augment': if run_dic is not None: paths.append( - f'{auto_root}/results/augment_raw/augment_{run_dic}_raw.jsonl' + f'{chat_root}/results/data_clean_{run_dic}/modified_samples.jsonl' ) paths.append( - f'{auto_root}/ai-planning/data/train_set/zk_intent/augment_{run_dic}.jsonl' + f'{chat_root}/results/augment_raw/augment_{run_dic}_raw.jsonl' + ) + paths.append( + f'{chat_root}/ai-planning/data/train_set/zk_intent/augment_{run_dic}.jsonl' + ) + paths.append( + f'{chat_root}/results/data_clean_{run_dic}/label_master_review.jsonl' ) elif step == 'verify': - paths.append(f'{auto_root}/results/iteration_log.jsonl') + paths.append(f'{chat_root}/results/iteration_log.jsonl') elif step == 'sft': - paths.append(f'{auto_root}/results/iteration_log.jsonl') + paths.append(f'{chat_root}/results/iteration_log.jsonl') elif step == 'log': - paths.append(f'{auto_root}/results/iteration_log.jsonl') - paths.append(f'{auto_root}/results/error_registry.jsonl') + paths.append(f'{chat_root}/results/iteration_log.jsonl') + paths.append(f'{chat_root}/results/error_registry.jsonl') elif step == 'next-round': - paths.append(f'{auto_root}/results/iteration_log.jsonl') + paths.append(f'{chat_root}/results/iteration_log.jsonl') return paths +def _csv_to_table( + text: str, max_rows: int | None +) -> tuple[list[str], list[list[str]], int]: + """Parse CSV text into (columns, rows, total). Strips BOM, handles + quoted multi-line cells. ``max_rows=None`` means no row cap.""" + if text.startswith(''): + text = text[1:] + import io as _io + reader = csv.reader(_io.StringIO(text)) + columns: list[str] = [] + rows: list[list[str]] = [] + total = 0 + for i, record in enumerate(reader): + if i == 0: + columns = [c.strip() for c in record] + continue + # 跳过整行全空的视觉分隔行（agent 写人审 CSV 时常插空行） + if not any(cell.strip() for cell in record): + continue + total += 1 + if max_rows is None or len(rows) < max_rows: + row = [_truncate_cell(c) for c in record] + if len(row) < len(columns): + row += [''] * (len(columns) - len(row)) + elif len(row) > len(columns): + row = row[: len(columns)] + rows.append(row) + return columns, rows, total + + +def _jsonl_to_table( + text: str, max_rows: int | None +) -> tuple[list[str], list[list[str]], int]: + """Parse jsonl into (columns, rows, total). Column order is the order + keys appear in the first record, with new keys appended. + ``max_rows=None`` means no row cap.""" + columns: list[str] = [] + seen: set[str] = set() + sample_rows: list[dict[str, Any]] = [] + total = 0 + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + try: + obj = json.loads(stripped) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + total += 1 + for k in obj.keys(): + if k not in seen: + seen.add(k) + columns.append(k) + if max_rows is None or len(sample_rows) < max_rows: + sample_rows.append(obj) + rows: list[list[str]] = [] + for obj in sample_rows: + row: list[str] = [] + for col in columns: + val = obj.get(col) + if val is None: + row.append('') + elif isinstance(val, str): + row.append(_truncate_cell(val)) + elif isinstance(val, (int, float, bool)): + row.append(str(val)) + else: + row.append(_truncate_cell(json.dumps(val, ensure_ascii=False))) + rows.append(row) + return columns, rows, total + + +def _truncate_cell(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[:limit] + f'…（共 {len(text)} 字，已截断）' + + def _detect_artifact_format(path: str) -> str: lower = path.lower() if lower.endswith('.md'): @@ -5469,6 +7333,9 @@ def _hardcoded_autoresearch_pipeline( sessions_dir: Path | None = None, state_entries: list[dict[str, Any]] | None = None, model_config: ModelConfig | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, ) -> dict[str, Any]: """Canonical autoresearch model-iteration pipeline structure. @@ -5484,6 +7351,8 @@ def _hardcoded_autoresearch_pipeline( session_id, state_entries or [], model_config=model_config, + agent_state=agent_state, + account_id=account_id, ) else: # Defensive default if caller didn't pass sessions_dir. @@ -5502,39 +7371,13 @@ def _hardcoded_autoresearch_pipeline( 'available': True, 'status': {'mode': 'Model Iteration', 'state': 'pending'}, 'kpis': kpis, - 'phases': [ - { - 'key': 'eval', - 'label': '评测与诊断', - 'tone': 'pending', - 'cards': [ - {'key': 'cml', 'icon': 'bar-chart', 'title': 'CML 评测', 'subtitle': 'workflow metric_diff', 'status': 'pending'}, - {'key': 'gold-drift', 'icon': 'microscope', 'title': 'Gold Drift', 'subtitle': 'RAG 检查漂移', 'status': 'pending'}, - {'key': 'dist-analysis', 'icon': 'trending-up', 'title': '分层结果分析', 'subtitle': '需求集合 / 大盘车载 / specific', 'status': 'pending'}, - {'key': 'report', 'icon': 'file-text', 'title': '问题分析 & 报告', 'subtitle': 'workflow.md', 'status': 'pending'}, - ], - }, - { - 'key': 'intervene', - 'label': '假设与干预', - 'tone': 'pending', - 'cards': [ - {'key': 'hypothesis', 'icon': 'dna', 'title': '形成假设', 'subtitle': 'iteration_log hypothesis', 'status': 'pending'}, - {'key': 'augment', 'icon': 'flask', 'title': '数据增强', 'subtitle': 'augment_.jsonl', 'status': 'pending'}, - {'key': 'verify', 'icon': 'shield', 'title': '修改返回验证', 'subtitle': 'sanity + 人审', 'status': 'pending'}, - ], - }, - { - 'key': 'train', - 'label': '训练与回归', - 'tone': 'pending', - 'cards': [ - {'key': 'sft', 'icon': 'graduation-cap', 'title': 'SFT 训练', 'subtitle': 'submit_sft_via_cml.sh', 'status': 'pending'}, - {'key': 'log', 'icon': 'clipboard-list', 'title': '记录迭代日志', 'subtitle': 'iteration_log.jsonl', 'status': 'pending'}, - {'key': 'next-round', 'icon': 'refresh-cw', 'title': '下一轮评测', 'subtitle': '回 Step 0', 'status': 'pending'}, - ], - }, - ], + # items[] is heterogeneous: {type:'round',...} for R0·Baseline / + # R{n}·Train / R{n}·Analysis, and {type:'gate',...} for HiTL gates. + # Built dynamically from program-state.jsonl by _build_pipeline_items — + # sections only appear when the agent has touched at least one step + # in them. + 'items': [], + 'in_flight': None, 'logs': [], } diff --git a/frontend/app/app/api/claw/auth/email-login/route.ts b/frontend/app/app/api/claw/auth/email-login/route.ts new file mode 100644 index 0000000..3bf5bf6 --- /dev/null +++ b/frontend/app/app/api/claw/auth/email-login/route.ts @@ -0,0 +1,20 @@ +import { loginByEmail } from "@/lib/claw-auth"; + +export async function POST(req: Request) { + try { + const payload = (await req.json()) as { + email?: string; + password?: string; + }; + const account = await loginByEmail( + payload.email ?? "", + payload.password ?? "", + ); + return Response.json({ account }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "登录失败" }, + { status: 400 }, + ); + } +} diff --git a/frontend/app/app/api/claw/auth/email-register/route.ts b/frontend/app/app/api/claw/auth/email-register/route.ts new file mode 100644 index 0000000..c708dec --- /dev/null +++ b/frontend/app/app/api/claw/auth/email-register/route.ts @@ -0,0 +1,20 @@ +import { registerByEmail } from "@/lib/claw-auth"; + +export async function POST(req: Request) { + try { + const payload = (await req.json()) as { + email?: string; + password?: string; + }; + const account = await registerByEmail( + payload.email ?? "", + payload.password ?? "", + ); + return Response.json({ account }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "注册失败" }, + { status: 400 }, + ); + } +} diff --git a/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/bind/route.ts b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/bind/route.ts new file mode 100644 index 0000000..2720a0c --- /dev/null +++ b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/bind/route.ts @@ -0,0 +1,49 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +type BindBody = { + session_id?: string; +}; + +export async function POST( + req: Request, + { params }: { params: Promise<{ workspaceId: string }> }, +) { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ error: "请先登录账号" }, { status: 401 }); + + const { workspaceId } = await params; + const payload = (await req.json()) as BindBody; + try { + const target = `${CLAW_API_URL}/api/jupyter/workspaces/${encodeURIComponent(workspaceId)}/bind`; + const response = await fetch(target, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + session_id: payload.session_id, + account_id: account.id, + }), + cache: "no-store", + }); + const body = await response.text(); + return new Response(body, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + }, + }); + } catch (err) { + return Response.json( + { + error: + err instanceof Error + ? `Unable to reach Claw backend: ${err.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/route.ts b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/route.ts new file mode 100644 index 0000000..51fd872 --- /dev/null +++ b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/route.ts @@ -0,0 +1,42 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ workspaceId: string }> }, +) { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ error: "请先登录账号" }, { status: 401 }); + + const { workspaceId } = await params; + try { + const target = new URL( + `${CLAW_API_URL}/api/jupyter/workspaces/${encodeURIComponent(workspaceId)}`, + ); + target.searchParams.set("account_id", account.id); + const response = await fetch(target, { + method: "DELETE", + cache: "no-store", + }); + const body = await response.text(); + return new Response(body, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + }, + }); + } catch (err) { + return Response.json( + { + error: + err instanceof Error + ? `Unable to reach Claw backend: ${err.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/api/claw/jupyter/workspaces/route.ts b/frontend/app/app/api/claw/jupyter/workspaces/route.ts new file mode 100644 index 0000000..1a1b248 --- /dev/null +++ b/frontend/app/app/api/claw/jupyter/workspaces/route.ts @@ -0,0 +1,33 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export async function GET() { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ error: "请先登录账号" }, { status: 401 }); + + try { + const target = new URL(`${CLAW_API_URL}/api/jupyter/workspaces`); + target.searchParams.set("account_id", account.id); + const response = await fetch(target, { cache: "no-store" }); + const body = await response.text(); + return new Response(body, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + }, + }); + } catch (err) { + return Response.json( + { + error: + err instanceof Error + ? `Unable to reach Claw backend: ${err.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/assistant.tsx b/frontend/app/app/assistant.tsx index 903159c..0f42dc3 100644 --- a/frontend/app/app/assistant.tsx +++ b/frontend/app/app/assistant.tsx @@ -182,6 +182,8 @@ const APPLY_INTENT_PATTERNS: RegExp[] = [ /(?:进行|开始|启动|开启|执行|发起|做)\s*(?:模型)?\s*训练/i, // 训练 + 目标集合 /训练[\s\S]{0,60}?(?:目标集合|需求集合|specific[\s_-]?test|集合)/i, + // 「(模型)训练，目标是 X」「训练，基模 X」「训练 ... csv」之类自然表达 + /(?:^|[\s，,。.!?])(?:模型)?训练[\s，,][\s\S]{0,80}?(?:目标|基模|specific[\s_-]?test|\.csv)/i, // 跟着 program.md 的旧表达保留 /(?:应用|加载|载入|启用|装载|使用|跑|按照?|根据|用)\s*\S*program\.?md/i, /program\.?md[\s\S]{0,120}?(?:训练|应用|启用|跑起来|跑一下)/i, @@ -204,6 +206,42 @@ function AssistantWorkspace() { const showPipeline = skill.enabled && applied; const messages = useAuiState((s) => s.thread.messages); const lastProcessedUserMsgId = useRef(null); + const pipelineContainerRef = useRef(null); + const lastPipelineYRef = useRef(null); + const [exitButtonVisible, setExitButtonVisible] = useState(false); + + const handlePipelineMouseMove = useCallback( + (event: React.MouseEvent) => { + const node = pipelineContainerRef.current; + if (!node) return; + const rect = node.getBoundingClientRect(); + const relativeY = event.clientY - rect.top; + const inTopThird = relativeY >= 0 && relativeY <= rect.height / 3; + const lastY = lastPipelineYRef.current; + const movingUp = lastY !== null && event.clientY < lastY; + const movingDown = lastY !== null && event.clientY > lastY; + lastPipelineYRef.current = event.clientY; + setExitButtonVisible((prev) => { + if (!inTopThird) return false; + if (movingUp) return true; + if (movingDown) return false; + return prev; + }); + }, + [], + ); + + const handlePipelineMouseLeave = useCallback(() => { + lastPipelineYRef.current = null; + setExitButtonVisible(false); + }, []); + + useEffect(() => { + if (!showPipeline) { + lastPipelineYRef.current = null; + setExitButtonVisible(false); + } + }, [showPipeline]); useEffect(() => { const update = () => setActiveSessionId(readActiveSessionId()); @@ -235,31 +273,52 @@ function AssistantWorkspace() { // 自动应用：skill 启用 + 当前 session 后端的 pipeline 已经有进展（任何 // 卡 running 或 complete）→ applied=true。这样关 tab 重开 / 刷新 / 切回 // 已在跑的会话都能自动恢复 monitor 面板，不依赖触发词。 + // 30 秒轮询一次，覆盖「页面打开时 pipeline 还没起、之后才被外部脚本启动」 + // 这类情形——一次性 fetch 容易错过。 useEffect(() => { if (!skill.enabled) return; if (applied) return; if (!activeSessionId) return; let cancelled = false; const url = `/api/claw/training/pipeline?session_id=${encodeURIComponent(activeSessionId)}`; - fetch(url, { cache: "no-store" }) - .then((res) => (res.ok ? res.json() : null)) - .then((payload) => { + const tick = async () => { + try { + const res = await fetch(url, { cache: "no-store" }); + if (!res.ok) return; + const payload = await res.json(); if (cancelled || !payload) return; - const phases = Array.isArray(payload.phases) ? payload.phases : []; - const hasProgress = phases.some( - (ph: { cards?: Array<{ status?: string }> }) => - Array.isArray(ph.cards) && - ph.cards.some( - (c) => c.status === "running" || c.status === "complete", - ), + const items = Array.isArray(payload.items) ? payload.items : []; + const hasItemProgress = items.some( + (it: { + type?: string; + status?: string; + cards?: Array<{ status?: string }>; + }) => { + if (it.type === "gate") { + return it.status === "running" || it.status === "complete"; + } + return ( + Array.isArray(it.cards) && + it.cards.some( + (c) => + c.status === "running" || c.status === "complete", + ) + ); + }, ); - if (hasProgress) setApplied(true); - }) - .catch(() => { + // items[] 走 phase-complete-only 过滤；R0 只有一张卡在跑时 + // items 是空的，进展信号在 in_flight 字段里——也算进展。 + const hasInFlight = Boolean(payload.in_flight); + if (hasItemProgress || hasInFlight) setApplied(true); + } catch { /* ignore */ - }); + } + }; + tick(); + const id = window.setInterval(tick, 30000); return () => { cancelled = true; + window.clearInterval(id); }; }, [skill.enabled, applied, activeSessionId, setApplied]); @@ -293,15 +352,24 @@ function AssistantWorkspace() { return (
{showPipeline ? ( -
+
@@ -309,9 +377,7 @@ function AssistantWorkspace() {
diff --git a/frontend/app/app/claw-account-gate.tsx b/frontend/app/app/claw-account-gate.tsx index 502c68d..5d19396 100644 --- a/frontend/app/app/claw-account-gate.tsx +++ b/frontend/app/app/claw-account-gate.tsx @@ -36,36 +36,59 @@ export function useClawAccount() { return { account, isLoading, refresh, setAccount }; } +const XIAOMI_EMAIL_RE = /^[\w.-]+@xiaomi\.com$/i; + export function ClawAccountGate({ onAccount, }: { onAccount: (account: ClawAccount) => void; }) { const [mode, setMode] = useState<"login" | "register">("login"); - const [username, setUsername] = useState(""); + const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); const [error, setError] = useState(""); const [isSubmitting, setIsSubmitting] = useState(false); async function submit() { setError(""); + const trimmed = email.trim(); + if (!XIAOMI_EMAIL_RE.test(trimmed)) { + setError("请使用小米邮箱（@xiaomi.com）登录"); + return; + } + if (password.length < 6) { + setError("密码至少 6 位"); + return; + } setIsSubmitting(true); try { - const response = await fetch(`/api/claw/auth/${mode}`, { + const endpoint = + mode === "register" + ? "/api/claw/auth/email-register" + : "/api/claw/auth/email-login"; + const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ username, password }), + body: JSON.stringify({ email: trimmed, password }), }); const payload = (await response.json()) as { account?: ClawAccount; error?: string; }; if (!response.ok || !payload.account) { - throw new Error(payload.error ?? "账号操作失败"); + throw new Error( + payload.error ?? (mode === "register" ? "注册失败" : "登录失败"), + ); } onAccount(payload.account); } catch (err) { - setError(err instanceof Error ? err.message : "账号操作失败"); + setError( + err instanceof Error + ? err.message + : mode === "register" + ? "注册失败" + : "登录失败", + ); } finally { setIsSubmitting(false); } @@ -77,22 +100,25 @@ export function ClawAccountGate({

ZK Data Agent

- 中控数据开发平台会按账号隔离会话、上传文件和生成产物。 + {mode === "register" + ? "创建账号：使用小米邮箱注册，前缀作为 autoresearch 工作根目录。" + : "使用小米邮箱登录。会话/上传/产物按邮箱前缀和 chat 二级隔离。"}

setUsername(event.target.value)} + type="email" + placeholder="your_name@xiaomi.com" + value={email} + onChange={(event) => setEmail(event.target.value)} onKeyDown={(event) => { if (event.key === "Enter") submit(); }} /> setPassword(event.target.value)} onKeyDown={(event) => { @@ -101,7 +127,7 @@ export function ClawAccountGate({ /> {error ?

{error}

: null} + +
+ ); + })} +
+ {error ? ( +
{error}
+ ) : null} +
+ ); +}; + const ClawSessionList: FC = () => { const { replaySession } = useClawSessionReplay(); const [sessions, setSessions] = useState([]); diff --git a/frontend/app/components/assistant-ui/thread.tsx b/frontend/app/components/assistant-ui/thread.tsx index b5bbe3e..fd4c2c7 100644 --- a/frontend/app/components/assistant-ui/thread.tsx +++ b/frontend/app/components/assistant-ui/thread.tsx @@ -273,7 +273,25 @@ function useRefreshCurrentRun() { const runStatus = await fetchLatestRunStatus(sessionId); if (cancelled) return; if (runStatus?.status === "queued" || runStatus?.status === "running") { - activeRunRef.current = runStatus.run_id ?? "active"; + const runKey = runStatus.run_id ?? "active"; + const previousRunKey = activeRunRef.current; + activeRunRef.current = runKey; + // Watcher auto-resume: backend started a new run via _maybe_auto_resume + // without a local SSE stream driving the UI. We need to replay once so + // the active-run placeholder (pending prompt + 思考中 bubble) appears. + // useRefreshReplayedRun will take over polling once replayedRun.active + // becomes true. + if (!runtimeRunning && previousRunKey !== runKey) { + const response = await fetch(`/api/claw/sessions/${sessionId}`, { + cache: "no-store", + }); + if (!response.ok || cancelled) return; + const payload = await response.json(); + replaySession( + sessionId, + toReplayRepository(payload, sessionId, runStatus), + ); + } return; } if (!runtimeRunning && !activeRunRef.current) return; @@ -632,6 +650,7 @@ function WorkspaceSwitchDialog({ if (needsFirstMessageSession) { writePendingWorkspaceSessionId(sessionIdForRequest); } + window.dispatchEvent(new Event("claw-workspaces-changed")); setStatus(payload as JupyterWorkspaceStatus); setProgress({ percent: 100, label: "切换完成" }); setMessage("工作区切换成功，后续工具会在远端 Jupyter 工作区执行。"); @@ -1186,6 +1205,7 @@ function ModelPickerButton({ }; }) { const [query, setQuery] = useState(""); + const [open, setOpen] = useState(false); const normalizedQuery = query.trim().toLowerCase(); const filteredModels = useMemo(() => { if (!normalizedQuery) return status.models; @@ -1199,7 +1219,7 @@ function ModelPickerButton({ }, [normalizedQuery, status.models]); return ( - +
) : null}
- +
+ + + + +
{loading ? ( @@ -106,14 +145,16 @@ export function StepDetailSheet({

无法获取详情

) : detail.format === "empty" ? ( + ) : detail.format === "tables" ? ( + ) : detail.format === "markdown" ? ( - + ) : detail.format === "json" ? ( - + ) : detail.format === "jsonl" ? ( - + ) : ( - + )}
{detail?.source_path && !loading ? ( @@ -939,6 +980,153 @@ function MarkdownView({ text }: { text: string }) { ); } +function TablesView({ sections }: { sections: TableSection[] }) { + if (sections.length === 0) { + return

空内容。

; + } + return ( +
+ {sections.map((section, idx) => ( + + ))} +
+ ); +} + +function TableSectionCard({ + section, + defaultOpen, +}: { + section: TableSection; + defaultOpen: boolean; +}) { + const isMissing = section.kind === "missing" || section.kind === "error"; + const isMarkdown = section.kind === "markdown"; + const truncated = + section.shown_rows > 0 && section.shown_rows < section.total_rows; + const noteParts: string[] = []; + if (section.total_rows > 0) { + noteParts.push(`共 ${section.total_rows} 行`); + if (truncated) { + noteParts.push(`仅显示前 ${section.shown_rows} 行`); + } + } + if (section.kind === "csv") noteParts.push("CSV"); + else if (section.kind === "jsonl") noteParts.push("JSONL"); + else if (section.kind === "text") noteParts.push("TEXT"); + else if (section.kind === "markdown") noteParts.push("MARKDOWN"); + + return ( +
+ + + {section.title} + + {noteParts.length > 0 ? ( + + {noteParts.join(" · ")} + + ) : null} + + ▾ + + +
+
+ + {section.source_path} +
+ {isMissing ? ( +

+ {section.error ?? "产物文件不存在或读取失败"} +

+ ) : isMarkdown ? ( + + ) : section.rows.length === 0 ? ( +

空表。

+ ) : ( + + )} +
+
+ ); +} + +function DataTable({ + columns, + rows, +}: { + columns: string[]; + rows: string[][]; +}) { + return ( +
+ + + + + {columns.map((c) => ( + + ))} + + + + {rows.map((row, rIdx) => ( + + + {columns.map((c, cIdx) => { + const cell = row[cIdx] ?? ""; + return ( + + ); + })} + + ))} + +
+ # + + {c} +
+ {rIdx + 1} + + {cell === "" ? ( + — + ) : ( + + )} +
+
+ ); +} + +function TableCell({ text }: { text: string }) { + if (text.includes("\n")) { + return ( + 
+				{text}
+
+ ); + } + return {text}; +} + function RawView({ text }: { text: string }) { return ( 
diff --git a/frontend/app/components/training/training-pipeline-panel.tsx b/frontend/app/components/training/training-pipeline-panel.tsx
index 71c919c..682689d 100644
--- a/frontend/app/components/training/training-pipeline-panel.tsx
+++ b/frontend/app/components/training/training-pipeline-panel.tsx
@@ -1,34 +1,43 @@
 "use client";
 
 import {
+	AlertTriangleIcon,
 	BarChart3Icon,
 	CheckIcon,
-	ChevronDownIcon,
+	ChevronRightIcon,
 	ClipboardListIcon,
 	ClockIcon,
 	DnaIcon,
 	ExternalLinkIcon,
 	FileTextIcon,
 	FlaskConicalIcon,
-	GitBranchIcon,
 	GraduationCapIcon,
 	HourglassIcon,
 	Loader2Icon,
-	MicroscopeIcon,
 	type LucideIcon,
+	MicroscopeIcon,
 	RefreshCwIcon,
 	ShieldCheckIcon,
 	TrendingUpIcon,
+	UserCheckIcon,
 } from "lucide-react";
 import { useEffect, useRef, useState } from "react";
 import { StepDetailSheet } from "@/components/training/step-detail-sheet";
 import { cn } from "@/lib/utils";
 
-type CardStatus = "pending" | "running" | "complete" | "failed" | "cancelled";
-type PhaseTone = "complete" | "running" | "pending";
+type CardStatus =
+	| "pending"
+	| "running"
+	| "waiting"
+	| "complete"
+	| "failed"
+	| "cancelled";
+type SectionType = "baseline" | "train" | "analysis";
+type GateKind = "human-check" | "human-review";
 
 type PipelineCard = {
 	key: string;
+	step: string;
 	icon?: string;
 	title: string;
 	subtitle: string;
@@ -36,10 +45,37 @@ type PipelineCard = {
 	progress?: number;
 };
 
-type PipelinePhase = {
-	key: string;
+type RoundItem = {
+	type: "round";
+	index: number;
+	run_id: string;
+	run_index: number;
+	section_type: SectionType;
 	label: string;
-	tone: PhaseTone;
+	description: string;
+	status: CardStatus;
+	cards: PipelineCard[];
+};
+
+type GateItem = {
+	type: "gate";
+	index: number;
+	key: string;
+	run_id: string;
+	run_index: number;
+	gate_kind: GateKind;
+	title: string;
+	description: string;
+	status: CardStatus;
+	reason?: string | null;
+};
+
+type PipelineItem = RoundItem | GateItem;
+
+type InFlight = {
+	run_id: string;
+	section_type: SectionType;
+	section_label: string;
 	cards: PipelineCard[];
 };
 
@@ -58,19 +94,21 @@ type LogLine = {
 type PipelinePayload = {
 	session_id: string;
 	generated_at_ms: number;
+	available?: boolean;
 	status: { mode: string; state: CardStatus };
 	kpis: Kpi[];
-	phases: PipelinePhase[];
+	items: PipelineItem[];
+	in_flight?: InFlight | null;
 	logs: LogLine[];
 };
 
 const ICON_MAP: Record = {
+	"alert-triangle": AlertTriangleIcon,
 	"bar-chart": BarChart3Icon,
 	microscope: MicroscopeIcon,
 	"trending-up": TrendingUpIcon,
 	"file-text": FileTextIcon,
 	dna: DnaIcon,
-	"git-branch": GitBranchIcon,
 	flask: FlaskConicalIcon,
 	shield: ShieldCheckIcon,
 	"graduation-cap": GraduationCapIcon,
@@ -78,6 +116,19 @@ const ICON_MAP: Record = {
 	"refresh-cw": RefreshCwIcon,
 	clock: ClockIcon,
 	hourglass: HourglassIcon,
+	"user-check": UserCheckIcon,
+};
+
+const SECTION_PHASE_TONE: Record = {
+	baseline: "from-violet-500/15 to-sky-500/10 border-violet-400/40",
+	train: "from-emerald-500/15 to-amber-500/10 border-emerald-400/40",
+	analysis: "from-sky-500/15 to-violet-500/10 border-sky-400/40",
+};
+
+const SECTION_BADGE_TONE: Record = {
+	baseline: "bg-violet-500/10 text-violet-700 dark:text-violet-300",
+	train: "bg-emerald-500/10 text-emerald-700 dark:text-emerald-300",
+	analysis: "bg-sky-500/10 text-sky-700 dark:text-sky-300",
 };
 
 export function TrainingPipelinePanel({
@@ -102,7 +153,6 @@ export function TrainingPipelinePanel({
 		es.onmessage = (ev) => {
 			try {
 				const payload = JSON.parse(ev.data);
-				// program.md missing → backend pushes {available: false}; clear panel.
 				if (payload && payload.available === false) {
 					setData(null);
 				} else {
@@ -114,7 +164,6 @@ export function TrainingPipelinePanel({
 			}
 		};
 		es.onerror = () => {
-			// EventSource auto-reconnects; just stop showing the loading state.
 			setLoading(false);
 		};
 		return () => {
@@ -122,33 +171,83 @@ export function TrainingPipelinePanel({
 		};
 	}, [sessionId]);
 
+	const stripNamespace = (key: string): string => {
+		// "R1:cml" → "cml" — backend uses round-namespaced keys, but the
+		// step-detail endpoint is keyed only by step name.
+		const idx = key.indexOf(":");
+		return idx >= 0 ? key.slice(idx + 1) : key;
+	};
+
 	return (
 		

 			

 				
 				
-				

-					{!data ? (
-						

-							暂无 pipeline 数据
-						

-					) : (
-						data.phases.map((phase) => (
-							
-									setOpenCard({
-										key: card.key,
-										title: card.title,
-										status: card.status,
-									})
-								}
-							/>
-						))
-					)}
+				

+					{(() => {
+						if (!data) {
+							return (
+								

+									暂无 pipeline 数据
+								

+							);
+						}
+						const items = data.items ?? [];
+						if (items.length === 0) {
+							const isRunning = data.status?.state === "running";
+							return (
+								

+									{isRunning
+										? "首个阶段进行中，卡片即将出现……"
+										: "等待 agent 进入第一个步骤……"}
+								

+							);
+						}
+						return (
+							<>
+								{items.map((item, idx) => {
+									const showConnector = idx < items.length - 1;
+									if (item.type === "round") {
+										return (
+											

+												
+														setOpenCard({
+															key: stripNamespace(card.key),
+															title: card.title,
+															status: card.status,
+														})
+													}
+												/>
+												{showConnector ?  : null}
+											

+										);
+									}
+									return (
+										

+											
+													setOpenCard({
+														key: stripNamespace(item.key),
+														title: item.title,
+														status: item.status,
+													})
+												}
+											/>
+											{showConnector ?  : null}
+										

+									);
+								})}
+							
+						);
+					})()}
 				

 			

+			
 			
 			
 			

@@ -187,27 +290,33 @@ function HeaderRow({
 			

 			

 				
 					
-					{isRunning ? (
-						
+					{isRunning && !isErrored ? (
+						
 					) : null}
 				
 				{statusLabel}
@@ -245,80 +354,204 @@ function KpiRow({ kpis }: { kpis: Kpi[] }) {
 	);
 }
 
-function PhaseSection({
-	phase,
+function ItemConnector() {
+	return (
+		

+			

+		

+	);
+}
+
+function StatusPill({ status }: { status: CardStatus }) {
+	const map: Record<
+		CardStatus,
+		{ label: string; cls: string; icon: LucideIcon | null }
+	> = {
+		complete: {
+			label: "COMPLETED",
+			cls: "bg-emerald-500/15 text-emerald-700 dark:text-emerald-300 border-emerald-500/30",
+			icon: CheckIcon,
+		},
+		running: {
+			label: "IN PROGRESS",
+			cls: "bg-sky-500/15 text-sky-700 dark:text-sky-300 border-sky-500/30",
+			icon: Loader2Icon,
+		},
+		waiting: {
+			label: "WAITING FOR YOU",
+			cls: "bg-amber-500/15 text-amber-700 dark:text-amber-300 border-amber-500/40",
+			icon: HourglassIcon,
+		},
+		pending: {
+			label: "PENDING",
+			cls: "bg-muted text-muted-foreground border-border",
+			icon: null,
+		},
+		failed: {
+			label: "FAILED",
+			cls: "bg-rose-500/15 text-rose-700 dark:text-rose-300 border-rose-500/30",
+			icon: AlertTriangleIcon,
+		},
+		cancelled: {
+			label: "CANCELLED",
+			cls: "bg-muted text-muted-foreground border-border",
+			icon: null,
+		},
+	};
+	const { label, cls, icon: Icon } = map[status];
+	return (
+		
+			{Icon ? (
+				
+			) : null}
+			{label}
+		
+	);
+}
+
+function RoundSection({
+	item,
 	onCardClick,
 }: {
-	phase: PipelinePhase;
+	item: RoundItem;
 	onCardClick?: (card: PipelineCard) => void;
 }) {
-	const dotCls =
-		phase.tone === "complete"
-			? "bg-emerald-500"
-			: phase.tone === "running"
-				? "bg-sky-500"
-				: "bg-violet-500";
-	const arrowComplete = phase.tone === "complete";
+	const numLabel = item.index.toString().padStart(2, "0");
+	const phaseTone = SECTION_PHASE_TONE[item.section_type];
+	const badgeTone = SECTION_BADGE_TONE[item.section_type];
 	return (
-		

-			

-				

-					
-					
-						{phase.label}
+		

+			

+				

+					
+						{numLabel}
 					
-				

-				
-			

-			

-				{phase.cards.map((card, idx) => (
-					

-						 onCardClick(card) : undefined}
-						/>
-						{idx < phase.cards.length - 1 ? (
-							
-						) : null}
+					

+						

+							
+								{item.label}
+							
+						

+						

+							{item.description}
+						

 					

-				))}
+				

+				
 			

+			{item.cards.length > 0 ? (
+				

+					{item.cards.map((card, idx) => (
+						

+							 onCardClick(card) : undefined}
+							/>
+							{idx < item.cards.length - 1 ?  : null}
+						

+					))}
+				

+			) : null}
 		

 	);
 }
 
-function PhaseArrow({ complete }: { complete: boolean }) {
+function CardArrow() {
 	return (
-		

-			
-				
-				
-			
+		

+			
 		

 	);
 }
 
+function GateCard({
+	item,
+	onClick,
+}: {
+	item: GateItem;
+	onClick?: () => void;
+}) {
+	const numLabel = item.index.toString().padStart(2, "0");
+	const isWaiting = item.status === "waiting";
+	const isComplete = item.status === "complete";
+	const Tag = onClick ? "button" : "div";
+	const hint =
+		item.gate_kind === "human-review"
+			? "在右侧对话里给出 OK / 调整后的指令，agent 才会继续。"
+			: "在右侧对话里回 OK / 修改建议，确认后进入下一轮训练。";
+	return (
+		
+			

+				
+					{numLabel}
+				
+				

+					
+				

+				

+					

+						
+							{item.title}
+						
+						
+							{item.run_id}
+						
+					

+					{item.reason ? (
+						

+							{item.reason}
+						

+					) : (
+						

+							{item.description}
+						

+					)}
+					{isWaiting ? (
+						

+							{hint}
+						

+					) : null}
+				

+			

+			
+		
+	);
+}
+
 function PipelineCardView({
 	card,
 	onClick,
@@ -357,7 +590,7 @@ function PipelineCardView({
 						
 					

 				) : (
-					
+					
 				)}
 			

 			

@@ -390,7 +623,7 @@ function PipelineCardView({
 	);
 }
 
-function StatusBadge({ status }: { status: CardStatus }) {
+function MiniStatus({ status }: { status: CardStatus }) {
 	const map: Record = {
 		complete: {
 			label: "DONE",
@@ -400,6 +633,10 @@ function StatusBadge({ status }: { status: CardStatus }) {
 			label: "RUNNING",
 			cls: "bg-sky-500/20 text-sky-700 dark:text-sky-200",
 		},
+		waiting: {
+			label: "WAITING",
+			cls: "bg-amber-500/20 text-amber-700 dark:text-amber-200",
+		},
 		pending: {
 			label: "PENDING",
 			cls: "bg-muted text-muted-foreground",
@@ -429,6 +666,61 @@ function StatusBadge({ status }: { status: CardStatus }) {
 	);
 }
 
+function LegendBar() {
+	const items = [
+		{
+			icon: BarChart3Icon,
+			label: "Measure",
+			sub: "Quantify performance",
+			color:
+				"bg-sky-500/10 text-sky-700 dark:text-sky-300 border-sky-500/30",
+		},
+		{
+			icon: DnaIcon,
+			label: "Improve",
+			sub: "Data, training, quality",
+			color:
+				"bg-emerald-500/10 text-emerald-700 dark:text-emerald-300 border-emerald-500/30",
+		},
+		{
+			icon: MicroscopeIcon,
+			label: "Understand",
+			sub: "Analyze & diagnose",
+			color:
+				"bg-violet-500/10 text-violet-700 dark:text-violet-300 border-violet-500/30",
+		},
+		{
+			icon: RefreshCwIcon,
+			label: "Iterate",
+			sub: "Repeat & grow",
+			color:
+				"bg-amber-500/10 text-amber-700 dark:text-amber-300 border-amber-500/30",
+		},
+	];
+	return (
+		

+			{items.map(({ icon: Icon, label, sub, color }) => (
+				

+					
+						
+					
+					

+						

+							{label}
+						

+						
{sub}

+					

+				

+			))}
+		

+	);
+}
+
 function LogStream({ logs }: { logs: LogLine[] }) {
 	const ref = useRef(null);
 	useEffect(() => {
@@ -471,6 +763,7 @@ function LogStream({ logs }: { logs: LogLine[] }) {
 function statusText(state: CardStatus) {
 	const map: Record = {
 		running: "Running",
+		waiting: "等待人工确认",
 		complete: "Complete",
 		failed: "Failed",
 		pending: "Idle",
diff --git a/frontend/app/components/ui/sheet.tsx b/frontend/app/components/ui/sheet.tsx
index 6507bb7..7353028 100644
--- a/frontend/app/components/ui/sheet.tsx
+++ b/frontend/app/components/ui/sheet.tsx
@@ -48,9 +48,11 @@ function SheetContent({
 	className,
 	children,
 	side = "right",
+	hideClose = false,
 	...props
 }: React.ComponentProps & {
 	side?: "top" | "right" | "bottom" | "left";
+	hideClose?: boolean;
 }) {
 	return (
 		
@@ -72,10 +74,12 @@ function SheetContent({
 				{...props}
 			>
 				{children}
-				
-					
-					Close
-				
+				{hideClose ? null : (
+					
+						
+						Close
+					
+				)}
 			
 		
 	);
diff --git a/frontend/app/lib/claw-auth.ts b/frontend/app/lib/claw-auth.ts
index e5346d7..b864d64 100644
--- a/frontend/app/lib/claw-auth.ts
+++ b/frontend/app/lib/claw-auth.ts
@@ -69,6 +69,58 @@ export async function loginAccount(username: string, password: string) {
 	return createSession(account);
 }
 
+const XIAOMI_EMAIL_RE = /^([\w.-]+)@xiaomi\.com$/i;
+
+export function accountIdFromEmail(email: string): string {
+	const match = XIAOMI_EMAIL_RE.exec((email ?? "").trim());
+	if (!match) {
+		throw new Error("请使用小米邮箱（@xiaomi.com）登录");
+	}
+	const prefix = match[1].toLowerCase();
+	if (!/^[a-z0-9._-]{2,40}$/.test(prefix)) {
+		throw new Error(
+			"邮箱前缀只能包含小写字母、数字、点、下划线或中划线，长度 2-40",
+		);
+	}
+	return prefix;
+}
+
+export async function registerByEmail(email: string, password: string) {
+	const accountId = accountIdFromEmail(email);
+	validatePassword(password);
+	const usersFile = await readUsers();
+	if (usersFile.users.some((user) => user.id === accountId)) {
+		throw new Error("该邮箱已注册，请直接登录");
+	}
+	const account: UserRecord = {
+		id: accountId,
+		username: accountId,
+		passwordHash: hashPassword(password),
+		createdAt: new Date().toISOString(),
+	};
+	usersFile.users.push(account);
+	await writeUsers(usersFile);
+	await createAccountDirectories(account.id);
+	return createSession(account);
+}
+
+export async function loginByEmail(email: string, password: string) {
+	const accountId = accountIdFromEmail(email);
+	const usersFile = await readUsers();
+	const account = usersFile.users.find((user) => user.id === accountId);
+	if (!account) {
+		throw new Error("账号不存在，请先注册");
+	}
+	if (!account.passwordHash) {
+		throw new Error("该账号尚未设置密码，请联系管理员重置");
+	}
+	if (!verifyPassword(password, account.passwordHash)) {
+		throw new Error("邮箱或密码错误");
+	}
+	await createAccountDirectories(account.id);
+	return createSession(account);
+}
+
 export async function logoutAccount() {
 	const cookieStore = await cookies();
 	const token = cookieStore.get(COOKIE_NAME)?.value;
diff --git a/skills/model-iteration/SKILL.md b/skills/model-iteration/SKILL.md
index 8f218ed..da3f256 100644
--- a/skills/model-iteration/SKILL.md
+++ b/skills/model-iteration/SKILL.md
@@ -20,20 +20,22 @@ when_to_use: |
 |---------|---------|
 | "要不要我做训练集近邻检索？" | 这是 §2.3 分析必做项，**直接做**，做完把结果落 report |
 | "下一步可以继续吗？" | 永远不问。看 program.md 流程图自己判断 |
-| "我先把控制权交回，等你来问跑完了吗" | watcher 接手 UI 同步，你还是要在 bash 内 sleep+poll 等 cml 完，看到 complete **直接进 Step 1** |
+| "我先把控制权交回，等你来问跑完了吗" | watcher 接手 UI 同步，长任务用 `bash(run_in_background=true, wait_for_completion=true)` 提交后本轮主动结束；后端会在产物落盘后自动起新一轮把结果送回，**直接进 Step 1** |
 | "这一步是关键决策点，需要你拍板" | program.md 没写就不是。**自主决策 + 落 iteration_log** |
 | "我把现状停在这里，把决策摘要给你 review" | 不准。摘要可以写，但不准停 |
+| "Step 4 augment 完成 ✅，等回调后续做 SFT" | augment=complete 那一刻就是 SFT 启动那一刻——**同一轮 bash** 紧接着跑 §4.5 label-master 复核 + 提交 SFT，不许写简报、不许等回调（详见 program.md "Step 4 → Step 5 硬连接"）|
+| "R1 评测发现 regression，先把诊断给你看，等你拍板再决定要不要回滚" | dist-analysis 发现 regression **也算 dist-analysis 完成**——**同轮 bash** 紧接着写 `results/workflow.md`（包含完整 delta 表 + new_fail/new_fix 子集 + 病灶定位 + 回滚建议）+ append `iteration_log.jsonl` R{n} entry。文件落完了再用 chat reply 给人提回滚选项。**不许把诊断只写在 chat 里、不落盘**——前端「分层结果分析」卡片读的是 `results/workflow.md`，你不写卡片永远停在上轮。 |
 
 **合法暂停只有**：
 
 1. program.md `## Human-in-the-Loop 时机` 章节的 7 类迭代级信号
 2. `### 训练集调整专项 H-i-T-L 信号` 的 T1-T10
 3. **§4.0.1 Step B 量级判定的 HiTL 分支**：
-   - 候选量 51-200 条 → **必须**抽样 30-50 条到飞书 sheet 让人审 1/0，按比例外推（**这就是一次合法暂停**，不是擅自停）
-   - 候选量 >200 条 → 命中触发条件 #3，强制 HiTL 介入让人定更精细 pattern 收窄
+   - 候选量 51-200 条 → **必须**全量导出到飞书 sheet 让人逐条审 1/0（**这就是一次合法暂停**，不是擅自停；**不再抽样外推**）
+   - 候选量 >200 条 → 命中触发条件 #3，强制 HiTL 介入让人定更精细 pattern 收窄（仍然全量交付）
 4. 用户主动发消息打断
 
-这之外**所有**"我觉得这事大、我先停"的本能都要压下来。命中合法暂停时，**不要只是说"等你确认"**——按 §4.0.1 把抽样表/精细 pattern 的具体输出 dump 到飞书 / scratchpad 给人具体可审的东西，再停。
+这之外**所有**"我觉得这事大、我先停"的本能都要压下来。命中合法暂停时，**不要只是说"等你确认"**——按 §4.0.1 把**全量候选清单**（不是抽样）/精细 pattern 的具体输出 dump 到飞书 / scratchpad 给人具体可审的东西，再停。
 
 ## 📋 每个 Step 的强制准入条件（少一项不准进下一步）
 
@@ -44,13 +46,13 @@ when_to_use: |
 - [ ] runDic 已分配（扫 `run_history_dir` 取 max+1）
 - [ ] CML workflow 已提交（拿到 Execution ID）
 - [ ] watcher 声明已写入 program-state.jsonl
-- [ ] 在 bash sleep+poll 等到 cml=complete
+- [ ] cml workflow + 等 metric_diff 落盘已用 `bash(run_in_background=true, wait_for_completion=true)` 提交（**不要**在 bash 内 sleep+poll）
 
 ### Step 1 准入（`dist-analysis`）
 - [ ] 读 `metric_diff/lark_template.json` 提分层指标（specific / 大盘车载 / 目标专项）
-- [ ] 读 `metric_diff/specific_comparison.csv` 抽 baseline 错误分布
+- [ ] 读 `metric_diff/specific_comparison.csv` 提取 baseline 错误分布
 - [ ] **§2.3 失败 case 与训练数据的关联**：每条错例在 `train_set/zk_intent/*.jsonl` 做近邻检索（前 3 近邻），输出"有近邻 / 无近邻"分类
-- [ ] **§2.3 Reward 对齐检查**：抽 10 条失败 case 用 `zk_reward_fn` 验 reward 方向
+- [ ] **§2.3 Reward 对齐检查**：全量失败 case 用 `zk_reward_fn` 验 reward 方向（不抽样）
 - [ ] §0.1 Gold drift 检查（如未做）
 
 ### Step 2 准入（`report`）
@@ -58,18 +60,33 @@ when_to_use: |
 - [ ] 写入 `results/workflow.md`
 - [ ] error_registry 追加本轮错误
 
-### Step 3 准入（`hypothesis`）
+### Step 3 准入（`hypothesis`） — analysis 收尾，不是 train 开头
 - [ ] 写本轮假设到 iteration_log.jsonl 的 hypothesis 字段
 - [ ] 假设必须有依据（指向 §2.3 / §2.4 的具体发现）
+- [ ] **`step:"hypothesis"` entry 写在当前 round（R{n}）名下**——不要写成 R{n+1}。后端把 hypothesis 归类为 analysis 类，是 R{n}·Baseline / R{n}·Analysis 的最后一张卡，不是 R{n+1}·Train 的开头。这样 gate（Human Check / Review）会插在 hypothesis 卡之后、R{n+1}·Train 之前，**用户看到假设内容再拍板是否进 train**。
+
+### Step 1 → Step 2 → Step 3 边界（**NEVER STOP 硬连接**，R0 / R1+ regression 都适用）
+- [ ] dist-analysis 算完 delta（new_fail / new_fix / persistent / 子集分布）那一刻起，**同一轮 bash 不许结束**：紧接着写 `results/workflow.md`（包含完整指标表 + 病灶定位 + 假设 + 回滚/继续建议）→ append `iteration_log.jsonl` R{n} entry（`results` 字段填本轮 metric，`hypothesis` 字段填下一动作）
+- [ ] **regression 场景同样适用**：哪怕 R1 出现导航bvt 纯劣化、可聊可控大幅 -25 这种"必须回滚"信号，**先把分析落到 workflow.md 和 iteration_log，再用 chat reply 给人回滚选项**。文件先落、聊天再发——顺序不能反。
+- [ ] **不许"诊断只写聊天回复 / 等用户拍板再补盘"**：前端「分层结果分析」卡片读的是 `chat_root/results/workflow.md`；你不写盘卡片永远显示上一轮，看不到 R1 结论。
+- [ ] **runDic 推进**：每轮新评测结果落盘后，append `iteration_log.jsonl` 一条新 entry（runDic 写新轮的值，比如 R0 是 17793，R1 评测的 metric_diff 在 workflow17794，这条 entry 的 runDic 就写 17794），UI 卡片才会切到新一行。
+- [ ] **凡是要让用户拍板的检查点（R0 baseline 之后、R{n} regression 之后、>200 候选要定 pattern、Gold drift 复核 等），必须先 append `step:"human-check"` 或 `step:"human-review"` running entry，再用 chat reply 提问**。**禁止只在 chat 里问而不写 gate entry**——UI 看不到拦截 = 视为没做这步，用户面板上看不到任何卡。详见下方 "Human-in-the-Loop 信号 → 写 gate 节点"。
 
 ### Step 4 准入（`augment`，需要修改训练集才进）
 - [ ] **§4.0 原始训练数据清洗（增强前必做）**——按 4 种动作走完逐 pattern 判定
 - [ ] §4.0.1 Step A：候选定位输出 csv，**不直接改**
-- [ ] §4.0.1 Step B：量级判定走对应支线（≤50 自动 / 51-200 抽样审 / >200 触发 #3）
+- [ ] §4.0.1 Step B：量级判定走对应支线（≤50 自动 / 51-200 全量人审 / >200 触发 #3，全量交付）
 - [ ] §4.0.1 Step C：备份 `.bak` 文件
 - [ ] §4.1 输入准备 / §4.2 GPT 调用 / §4.3 sanity check / §4.4 写入
+- [ ] **§4.5 label-master 标签复核（落盘后必做，强制）**：对 H1 `modified_samples.jsonl` + H2 `augment_.jsonl` 跑两层（`validate_label_output.py` 格式 + Skill 调用 `label-master` 语义），verdict 写 `results/data_clean_/label_master_review.jsonl`，不通过比例 ≤ 5% 才能进 Step 5
+
+### Step 4 → Step 5 边界（**NEVER STOP 硬连接**，反复踩坑）
+- [ ] 写完 `augment=complete` 那一刻，**同一轮 bash 不许结束**：紧接着跑 §4.5 label-master 复核 → 写 `sft=running` → 调 `submit_sft_via_cml.sh` → 挂 watcher
+- [ ] **不许写"Step 4 完成"进度简报后 turn 结束**，不许"等回调后续做 SFT"
+- [ ] H2 后台任务回调（`[system] 后台任务 ... exit_code=0`）**不是 turn 结束信号**——它只是 augment 子流程的一个中间节拍，agent 必须在同轮里继续走完 §4.5 → Step 5
 
 ### Step 5 准入（`sft`）
+- [ ] §4.5 label-master 复核报告 `label_master_review.jsonl` 存在且不通过 ≤ 5%
 - [ ] 旧 sft_output 已 `mv sft_output sft_output_r{prev}` 备份
 - [ ] 训练参数从 config.yaml 读，model_path = basemodel（**不从上轮 ckpt 续训**）
 
@@ -101,6 +118,18 @@ when_to_use: |
 > echo '{"step":"cml","status":"complete","ts":"'$(date -Iseconds)'"}' >> "$SESSION_OUTPUT/program-state.jsonl"
 > ```
 
+### R1.5 — 推荐每条 state entry 带 run_id（让 UI 准确归位 round）
+
+新版 UI 按 R0 / R1 / R2 切分 sections。**强烈建议**每行加 `run_id`：
+
+```bash
+echo '{"step":"cml","status":"running","run_id":"R0","ts":"'$(date -Iseconds)'"}' >> $S
+echo '{"step":"hypothesis","status":"complete","run_id":"R0","ts":"..."}' >> $S    # hypothesis 跟当前轮（R0），不是 R1
+echo '{"step":"augment","status":"running","run_id":"R1","ts":"..."}' >> $S        # augment 才是 R1·Train 起点
+```
+
+run_id 缺省时后端会按 `iteration_log.jsonl` 行数自动推断（analysis 类 step 含 hypothesis → R{count}，train 类 step augment/verify/sft → R{count+1}），但显式写更准——尤其是并发跨轮、补写历史 entry、或要在 R0 强制注入 baseline 时。
+
 ### R2 — 写入即生效，最后一行覆盖前面
 
 后端按 `step` 字段取**最后一条**状态。如果你之前写过 failed、现在恢复了，**必须 append 一条新的 `running`**，否则 UI 永远停在 failed。
@@ -122,18 +151,24 @@ UI 卡底部进度条会跟着动；不写就一直显示初始进度。
 ### State 文件位置
 
 ```
-<当前会话目录>/output/program-state.jsonl
+<远端 workspace>/output/program-state.jsonl
 ```
 
-会话目录在系统提示的 `[当前会话目录]` 里给了绝对路径。**直接用那个值**，不要拼。**绝对不要**写到 `/mnt/wangsenhao/...` 或项目根目录——多会话互相覆盖。
+bash 命令是在 **远端 workspace** 里跑的（一般 `/root/zk_agent_workspaces/LOCALID_xxx`），系统提示里的「当前会话目录」是 **后端宿主机的本地路径**，**不能直接拿来当 SESSION_OUTPUT**——拿了等于在远端凭空建一条同名死路径，后端 sync 永远读不到，前端卡片就一直不动。
 
-每次写之前先确保目录存在：
+正确做法：用远端 cwd 派生（`pwd` 就是当前远端 workspace），目录确保存在：
 
 ```bash
-SESSION_OUTPUT="<从系统提示拷过来>/output"
+SESSION_OUTPUT="$(pwd)/output"
 mkdir -p "$SESSION_OUTPUT"
 ```
 
+或者显式写远端绝对路径 `SESSION_OUTPUT="/root/zk_agent_workspaces/LOCALID_<本会话 id>/output"`。**绝对不要**：
+- 拷系统提示里的 `/home/mi/zk-data-agent-wsh/.port_sessions/.../output`（那是后端本地，远端没这条）
+- 写到 `/mnt/wangsenhao/...` 或项目根目录（多会话互相覆盖）
+
+后端 `_sync_remote_program_state` 只读远端 `{workspace_cwd}/output/program-state.jsonl` —— 路径写错 = 前端死锁。
+
 ### Step key 表（必须用这套 key，否则匹配不到卡片）
 
 | step key | 对应卡片 |
@@ -147,7 +182,9 @@ mkdir -p "$SESSION_OUTPUT"
 | `verify` | 修改返回验证 |
 | `sft` | SFT 训练 |
 | `log` | 记录迭代日志 |
-| `next-round` | 下一轮评测 |
+| `next-round` | 下一轮评测（边界标记，**不生成卡片**，仅推进 UI 的 run_id 推断） |
+| `human-check` | Human Check（HiTL 网关；命中 HiTL 时 agent 显式写，UI 在对应 round section 后插入 gate 卡） |
+| `human-review` | Human Review（同上，多用于"建议人审 1/0"或"建议回滚"等需要人拍板的检查点） |
 
 ### KPI（你不要写，后端读权威源）
 
@@ -159,7 +196,7 @@ mkdir -p "$SESSION_OUTPUT"
 | VERSION | `skills/model-iteration/assets/config.yaml` 的 `cml_eval.version` |
 | 目标集合 | 从 session 最新 user 消息里解析 `"开始, "` 中的 name |
 | 大盘车载 / SPECIFIC TEST | `RUN_HISTORY_DIR/workflow/metric_diff/lark_template.json` |
-| ITERATION | `/results/iteration_log.jsonl` 行数 |
+| ITERATION | `$AUTORESEARCH_CHAT_ROOT/results/iteration_log.jsonl` 行数（每 chat 独立） |
 | 耗时 | program-state.jsonl 第一条 `{step:"cml",status:"running"}` 的 `ts` 到现在 |
 
 **你不要再写 `{"kpi":...}` 行**——写了也会被后端忽略，不显示。如果想让用户看到某个 KPI 的当前值，把它落到对应的权威源（比如改 config.yaml、写 iteration_log），后端会自动读到。
@@ -172,7 +209,8 @@ CML 评测 / SFT 训练这类几分钟到几十分钟的长任务，声明 watch
 
 - **不许**给用户回"控制权交回 / 我先停下 / 等你来问"这种话
 - **不许**主动结束本轮（除非命中 program.md 里写明的 H-i-T-L 阻断信号）
-- 提交完任务、声明完 watcher 后，**继续在 bash 里轮询 state file** 等 cml=complete，看到就**直接进入 Step 1**（读 metric_diff、做分层分析、写报告等），一气呵成跑完整轮迭代
+- 长任务（cml 评测、SFT 训练）**必须**用 `bash(run_in_background=true, wait_for_completion=true)` 提交：把"提交远端任务 + 等产物落盘"打包成一个 bg 命令；本轮 agent 主动结束（**不是**交回控制权——是把等待这件事 detach 给后端），产物落盘后后端自动起新一轮，agent 收到 stdout 直接进 Step 1
+- **不许**在 bash 里写 `while true; do sleep 60; done` 这种轮询循环——bash 命令有 30s 硬超时，会被切成几十个 step，把本轮 50 step 上限耗尽留不出做分析的预算
 
 watcher 解决的是 **UI 与 agent 异步**：UI 不用等 agent 写状态，watcher 替 agent 把 complete 落盘。它不是替你"放假"。
 
@@ -202,31 +240,41 @@ echo '{"watch":{"step":"cml","kind":"file_exists","path":"/mnt/xiaoai-zk-model-t
 - step 已经有 complete/failed 行后，watcher 声明会被忽略
 - 后端 scanner 5 秒扫一次所有会话的 state file
 
-**典型用法**（CML workflow，**NEVER STOP**）：
+**典型用法**（CML workflow，**用 bg 模式 + watcher**）：
+
+长任务必须用 `bash(run_in_background=true, wait_for_completion=true)` 提交。bg 命令在远端 detach 跑、立刻返回 task_id；产物落盘后后端自动起新一轮把 stdout 作为 system 消息送回 agent，**不要**在 bash 里 sleep+poll。watcher 仍然要声明——它驱动 UI 卡片状态，跟 bg 任务是两条独立的通道。
 
 ```bash
-# 1. 提交 cml 任务（异步，立刻返回 runDic）
-RUNDIC=$(cml workflow run --workflow_id "$WORKFLOW_ID" ...)
-
-# 2. 写 running + watcher 声明
+# 1. 写 running + watcher 声明（UI 卡片靠它）
 echo '{"step":"cml","status":"running","ts":"'$(date -Iseconds)'"}' >> $S
+
+# 2. 用 bg 模式：把"提交 cml + 等 metric_diff 落盘"一次封装
+#    wait_for_completion=true（默认）：进程结束后后端自动起新一轮把 stdout 送回
+bash(run_in_background=true, command='''
+  set -e
+  RUNDIC=$(cml workflow run --workflow_id "$WORKFLOW_ID" ...)
+  echo "RUNDIC=$RUNDIC"
+  TARGET=/mnt/xiaoai-zk-model-train-tj5/workflow5/workflow$RUNDIC/metric_diff/lark_template.json
+  while [ ! -f "$TARGET" ]; do sleep 60; done
+  echo "EVAL_DONE runDic=$RUNDIC target=$TARGET"
+''')
+
+# 3. 紧接着声明 watcher（UI 卡片切到 complete 靠它，runDic 在 bg 任务 stdout 里）
+#    watcher path 用稍后从 bg stdout 拿到的 runDic 拼；如果先不知道 runDic，
+#    可以等 bg 任务返回 RUNDIC 后这一轮里再补声明 watcher。
 echo '{"watch":{"step":"cml","kind":"file_exists","path":"'$ROOT/workflow$RUNDIC/metric_diff/lark_template.json'","interval":30,"timeout":7200,"run_id":"R'$N'"}}' >> $S
 
-# 3. 跟用户简报一句状态，**不要说"交回控制权"**，紧接着开始等
-echo "已提交评测 runDic=$RUNDIC，开始轮询 state file 等 watcher 写 complete..."
+# 4. 跟用户简报一句"已提交，等产物落盘后会自动续"，agent 这一轮主动结束
+#    （不是交回控制权——是把等待这件事 detach 给后端）
 
-# 4. 在 bash 里轮询 state file（不阻塞你下一步思考）
-while true; do
-  status=$(grep -E '"step":"cml"' $S | tail -1 | python3 -c 'import json,sys; print(json.loads(sys.stdin.read()).get("status",""))')
-  if [ "$status" = "complete" ]; then break; fi
-  if [ "$status" = "failed" ]; then echo "cml failed"; exit 1; fi
-  sleep 60
-done
-
-# 5. cml 完成 → 立刻进入 Step 1，读 metric_diff、做分析、写下一步 running...
-# 整轮迭代不交回控制权，直到达标 / 命中 H-i-T-L 阻断 / 用户主动打断
+# 5. 产物落盘后，后端起新一轮，agent 收到 stdout（含 RUNDIC + EVAL_DONE）
+#    → 立刻进入 Step 1，读 metric_diff、做分层分析、写报告……整轮 50 step 全用在分析上
 ```
 
+**为什么不能在 bash 里 sleep+poll**：bash 工具有 30s 硬超时，`sleep 60` 会被切成 `sleep 25 + sleep 5` 两个 step；评测跑 25 分钟 = ~50 step 全耗在等待上，留不出做分析的预算。bg 模式只占 1 个 step。
+
+**bg 任务排查**：agent 中途想看进度，可以 `bash_status(task_id)` 拿当前 stdout/exit_code 快照；想终止远端进程要 `bash_kill(task_id)`（前端按 stop 不会杀 bg 任务，那是 detach 的本意）。
+
 ### Live log（可选）
 
 要把日志推到 UI 底部那块黑色 Live Log 区，追加：
@@ -249,10 +297,42 @@ echo '{"log":{"ts":"11:55:10","iter":"R3","text":"Step 3 开始: 问题分析 &
 | 触发信号 | `"开始，<需求集合名>"` |
 | 评测 workflow | `f-20260408161444-wu3pz`（版本见 `config.yaml`） |
 | 训练基模 | `/mnt/wangsenhao/verl_zk/Qwen3-4B-Instruct-2507` |
-| 工作目录 | `/mnt/wangsenhao/autoresearch-zk` |
+| 工作目录 | `$AUTORESEARCH_CHAT_ROOT`（每 chat 独立，由后端注入；位于共享 NFS：`/mnt/wangsenhao/autoresearch-zk-users///`，jupyter pod 与训练 pod 都能读写） |
 | 历史记录目录 | `/mnt/xiaoai-zk-model-train-tj5/workflow5/` |
 | 需求集合位置 | `https://git.n.xiaomi.com/ai-service/ai-planning/-/tree/autoresearch-v1` 下 `ai-planning/data/specific_test_set/` |
 
+## 工作空间隔离（按用户 + 每 chat 二级隔离，落在 NFS）
+
+每个 chat session 一份独立工作区 `$AUTORESEARCH_CHAT_ROOT`（后端在 jupyter 启动时自动注入这个 env，agent 每次 bash 都能拿到）。该目录位于共享 NFS：
+
+```
+/mnt/wangsenhao/autoresearch-zk-users///
+```
+
+`` 是用户登录时的小米邮箱前缀（`xxx@xiaomi.com` 取 `xxx`），用作账号根目录；同一用户跨 chat 共享根目录但 chat 之间完全隔离。
+
+**为什么放 NFS 而不是 jupyter pod 私有路径**：SFT 训练在独立的 pytorch pod 里跑，那个 pod 不挂载 jupyter pod 的 workspace；放共享 NFS 是双方都能 `cd` 进去的唯一选择。
+
+**所有写入路径都必须以 `$AUTORESEARCH_CHAT_ROOT` 开头**，不要写 hard-coded `/mnt/wangsenhao/autoresearch-zk/...` 全局路径，也不要写 jupyter pod 本地路径（`/root/zk_agent_workspaces/...`），训练 pod 看不到。
+
+| 资产 | 位置 | 隔离方式 |
+|---|---|---|
+| `prepare_and_train_sft.py` | `$AUTORESEARCH_CHAT_ROOT/scripts/` | 后端 bind 时从 skill bundle 推过来，每次刷新最新版 |
+| `ai-planning/`（含 corpus + augment） | `$AUTORESEARCH_CHAT_ROOT/ai-planning/` | **首次访问前你自己 git clone**（见 program.md 「ai-planning bootstrap」） |
+| `zk_trainer/` | `$AUTORESEARCH_CHAT_ROOT/zk_trainer/` | **Step 5 首次 SFT 前你自己 git clone**（见 program.md §5.0） |
+| `results/iteration_log.jsonl` | `$AUTORESEARCH_CHAT_ROOT/results/` | 每 chat 独立累积；后端 KPI 也读这里 |
+| `results/error_registry.jsonl` | 同上 | 每 chat 独立 |
+| `results/workflow.md / data_clean_/ / augment_raw/` | 同上 | 每 chat 独立 |
+| `sft_output/` | `$AUTORESEARCH_CHAT_ROOT/sft_output/` | 每 chat 独立，互不覆盖；训练 pod 走 NFS 直接写 |
+| `augment_.jsonl` | `$AUTORESEARCH_CHAT_ROOT/ai-planning/data/train_set/zk_intent/` | 写在 chat 自己的 ai-planning clone 内，下一次 SFT prepare 只合并本 chat 的增量 |
+
+**保持全局共享的资产**（不要按 chat 拆）：
+- runDic 计数：`/mnt/xiaoai-zk-model-train-tj5/workflow5/`（max+1 是平台级唯一标识）
+- 训练基模：`/mnt/wangsenhao/verl_zk/Qwen3-4B-Instruct-2507`（只读模型权重）
+- metric_diff 输出：`/mnt/xiaoai-zk-model-train-tj5/workflow5/workflow/metric_diff/`（cml workflow 写）
+
+**每个 step 落产物时务必用 `$AUTORESEARCH_CHAT_ROOT` 作前缀**（例如 `cd $AUTORESEARCH_CHAT_ROOT && python scripts/prepare_and_train_sft.py ...`，或 `echo ... >> $AUTORESEARCH_CHAT_ROOT/results/iteration_log.jsonl`）。
+
 ## 核心原则
 
 1. **评测优先**：`"开始"` 信号的第一个动作永远是评测当前模型，绝不直接训练
@@ -343,6 +423,58 @@ assets/
 - 跨子集净退步（#4）
 - 连续 3 轮目标子集净提升 ≤ 0.5pp（#5）
 
+### 写 gate 节点（让 UI 显示 Human Check / Review 卡）
+
+凡是要让用户拍板的检查点，**必须** append 一条 gate entry 到 `program-state.jsonl`，**再** 用 chat reply 提问。UI 会在当前 round section 之后插入「Human Check」或「Human Review」横向卡片（IN PROGRESS 状态）。
+
+> ⛔ **禁止**：只在 chat 里问用户、不写 gate entry。UI 看不到拦截 = 视为这一步没做——用户在面板里看不到任何卡，会以为流程卡死或还在自动推进。**这是反复踩坑点**：以前 agent 经常在 R0 baseline 跑完后聊天里问"要不要进 R1 train"，但 program-state 一直在写 augment running，UI 里完全看不到 gate。
+
+#### 什么时候要写
+
+需要用户拍板就写——不管 R0 还是 R{n≥1}，触发条件都按 §HiTL 信号判：
+
+| 时机 | gate 类型 | run_id |
+|---|---|---|
+| R0 baseline 分析完成、要让用户确认是否进 R1 train（命中 HiTL 信号 / 候选量大 / 目标子集偏低 / 第一次跑想让用户校方向 等） | `human-check` | `R0` |
+| R{n≥1} analysis 完成、命中 §HiTL 信号（gold drift / regression / 跨子集退步 / 连续 3 轮无提升 等） | `human-review` | `R{n}` |
+| §4.0.1 Step B：候选量 > 200 条命中触发条件 #3 | `human-check` | 当前 round |
+| 其他 HiTL 信号（要批改旧标签 > 50 条、Gold drift ≥ 10 条 等） | `human-check`（开始前）/ `human-review`（结果后） | 当前 round |
+
+判断标准就一条：**只要你下一步打算 chat-ask 用户拍板，就先写 gate entry 再问**。
+
+#### 写法
+
+```bash
+# R0 baseline 之后想让用户拍板是否进 train
+echo '{"step":"human-check","status":"running","run_id":"R0","reason":"R0 baseline 分析完成，目标子集 X.X% 偏低，确认是否按当前 H1 候选进 R1 train","ts":"'$(date -Iseconds)'"}' >> $S
+
+# R1 评测发现 regression，需要用户决定是否回滚
+echo '{"step":"human-review","status":"running","run_id":"R1","reason":"R1 vs R0：导航bvt -3.2pp, 可聊可控 -25pp，建议回滚","ts":"'$(date -Iseconds)'"}' >> $S
+
+# 候选量超 200 条命中触发条件 #3
+echo '{"step":"human-check","status":"running","run_id":"R0","reason":"候选量 412 条命中触发条件 #3","ts":"'$(date -Iseconds)'"}' >> $S
+```
+
+**顺序不能反**：先 echo gate entry → 再 chat reply 给用户。否则用户先看到聊天问话、UI 里却没卡，会困惑"流程是不是卡死了"。
+
+#### 字段说明
+
+| 字段 | 必填 | 说明 |
+|---|---|---|
+| `step` | ✅ | `human-check` 或 `human-review` |
+| `status` | ✅ | 卡进入时写 `running`；用户回复后写 `complete` |
+| `run_id` | ✅ | 当前所在轮次（决定 gate 插在哪个 section 后） |
+| `reason` | ⭕ | 触发原因——直接展示给用户看，写具体一些（"Gold drift 12 条，触发 #1" 比 "需要人审" 强得多） |
+| `ts` | ✅ | ISO 时间戳 |
+
+**用户拍板回复后**：append `status:"complete"` 同 step + run_id 一行，gate 卡变 COMPLETED；然后才继续往下走（继续/回滚/调参）。
+
+**两种 gate 的区分（约定）**：
+- `human-check`：偏"开始前的检查"——baseline → train 网关、>200 条候选定 pattern、Gold drift 复核
+- `human-review`：偏"结果出来要复核"——R{n} 评测出来发现 regression 要不要回滚、目标集合连续 3 轮无提升要不要换思路
+
+实际差别在 UI 上不大（都是横向 dashed 卡），区分主要为了让用户从标题就大致知道是开始前还是结果后的检查点。
+
 ## CML 配置（快速查阅）
 
 ```yaml
@@ -357,13 +489,17 @@ xiaomi_cloudml:
 
 ## 结果文件
 
-| 文件 | 用途 |
-|---|---|
-| `results/iteration_log.jsonl` | 每轮假设/干预/判定完整记录 |
-| `results/error_registry.jsonl` | 跨轮错误追踪（case_hash → 出错轮次） |
-| `results/workflow.md` | 每轮回归分析报告 |
-| `results/augment_raw/augment__raw.jsonl` | GPT-5.4 原始生成产物 |
-| `ai-planning/data/train_set/zk_intent/augment_.jsonl` | 清洗后的增量训练数据 |
-| `results/data_clean_/` | 旧数据清洗存档（必须在覆盖原文件前写入） |
-| `results/gold_drift/drift_.json` | Gold drift 检测结果 |
-| `results/label_rules.md` | 已确立的标签规则集（R1~RN） |
+| 文件 | 用途 | 前端卡片 |
+|---|---|---|
+| `results/iteration_log.jsonl` | 每轮假设/干预/判定完整记录 | hypothesis / log |
+| `results/error_registry.jsonl` | 跨轮错误追踪（case_hash → 出错轮次） | log |
+| `results/workflow.md` | 每轮回归分析报告 | dist-analysis / report |
+| `output/relabel_candidates_.csv` | **阶段一**：预计修改训练集候选清单（complex 翻转候选等） | **dist-analysis（分层结果分析）** |
+| `results/data_clean_/modified_samples.jsonl` | **阶段二**：确认修改最终落盘（用户审核 1/0 后实际生效的改动） | **augment（数据增强）** |
+| `results/data_clean_/deleted_samples.jsonl` | 旧数据清洗存档（删除项） | — |
+| `results/augment_raw/augment__raw.jsonl` | GPT-5.4 仿写原始产物 | augment |
+| `ai-planning/data/train_set/zk_intent/augment_.jsonl` | 本轮合入训练集的增量（H1 重标 + H2 仿写） | augment |
+| `results/gold_drift/drift_.json` | Gold drift 检测结果 | gold-drift |
+| `results/label_rules.md` | 已确立的标签规则集（R1~RN） | — |
+
+**两阶段训练集修改的展示约定**：候选阶段（量级判定 + 人审 1/0 之前）写在 `output/relabel_candidates_.csv`，前端「分层结果分析」卡片读取展示供人审；确认阶段（人审后落盘）写在 `results/data_clean_/modified_samples.jsonl`，前端「数据增强」卡片读取展示最终改动。详见 [references/program.md §4.0.1 Step A](references/program.md)。
diff --git a/skills/model-iteration/references/program.md b/skills/model-iteration/references/program.md
index 8a34790..0e089dd 100644
--- a/skills/model-iteration/references/program.md
+++ b/skills/model-iteration/references/program.md
@@ -51,6 +51,23 @@ ssh -T git@git.n.xiaomi.com
 #   打开 https://git.n.xiaomi.com/-/profile/keys 粘贴公钥
 ```
 
+### Chat 工作区 bootstrap（首次启动时）
+
+每个 chat session 一份独立工作区 `$AUTORESEARCH_CHAT_ROOT`（后端 jupyter 启动时自动注入这个 env）。该路径位于共享 NFS：`/mnt/wangsenhao/autoresearch-zk-users///`，`` 是用户登录的小米邮箱前缀（账号根目录），chat 二级隔离。**jupyter pod 与 SFT 训练 pod 共用这条 NFS**，所以训练 yaml 里 `cd $AUTORESEARCH_CHAT_ROOT` 不会再像旧版（jupyter pod 私有 `/root/zk_agent_workspaces/...`）那样在训练 pod 报 No such file。
+
+`scripts/`、`results/`、`sft_output/` 由后端 mkdir + 推送 `prepare_and_train_sft.py`；**ai-planning corpus 需要 agent 自己 git clone**（之前依赖全局共享，已废弃）：
+
+```bash
+# 检测是否已 clone，没有就拉
+[ -d "$AUTORESEARCH_CHAT_ROOT/ai-planning" ] || git clone -b autoresearch-v1 \
+  git@git.n.xiaomi.com:ai-service/ai-planning.git \
+  "$AUTORESEARCH_CHAT_ROOT/ai-planning"
+```
+
+之后所有训练数据读写都走 `$AUTORESEARCH_CHAT_ROOT/ai-planning/...`（包括 augment_.jsonl 写入、近邻检索、训练集修改）。`prepare_and_train_sft.py` 用 `Path(__file__)/../ai-planning` 解析数据目录，因为脚本被推到了 `$AUTORESEARCH_CHAT_ROOT/scripts/` 旁边，自然落在 chat 副本上。
+
+zk_trainer 的 clone 在首次 SFT 前进行（见 §5.0），目标也是 `$AUTORESEARCH_CHAT_ROOT/zk_trainer`。
+
 ### cml 环境检查（每次评测前必做）
 
 ```bash
@@ -231,10 +248,11 @@ for k in prev:
 当前瓶颈是需求集合，单独做一节：
 
 1. **子集级表格**：每个需求子集 pass rate、距离 95% 的 gap、对比上轮、对比基线。
-2. **失败 case 与训练数据的关联**：对每个失败 case 在 `ai-planning/data/train_set/zk_intent/` 下所有 `.jsonl`（包括历史增强 `augment_*.jsonl` 和原始种子训练文件，排除 `*_valid.jsonl`）里做近似检索（前 3 近邻），判断：
-   - **数据缺失**：训练集中没有类似样本 → 补数据
-   - **数据充足但仍错**：训练集中有类似样本 → 数据质量 or 学不会，不是加数据的问题
-3. **Reward 对齐检查**（抽样 10 条失败 case）：用 `zk_reward_fn` 对"正确 label"和"实际输出"分别打分，验证 reward 方向是否和准确率一致。如果 reward 给错误输出的分更高 → reward 函数本身就有问题。
+2. **失败 case 与训练数据的关联**：对每个失败 case 在 `ai-planning/data/train_set/zk_intent/` 下所有 `.jsonl`（包括历史增强 `augment_*.jsonl` 和原始种子训练文件，排除 `*_valid.jsonl`）里做近似检索（前 3 近邻），三档判定：
+   - **无近邻**（最高相似度 < 阈值）→ 分布外，补数据
+   - **有近邻 + 近邻 label 与本 case gold 全部一致** → 训练数据正确，SFT 学不动 → 加 epoch / 改 reward
+   - **有近邻 + 任一近邻 label 与 gold 矛盾** ⚠️ → **训练集打错标**，必须**逐条列出该 mislabeled 训练样本**：`{train_file:line, hash, 当前 label, 应改 label, 与失败 case 相似度, 违反的标签规则}`，进入下一轮的数据修订清单
+3. **Reward 对齐检查**（全量失败 case，不抽样）：用 `zk_reward_fn` 对"正确 label"和"实际输出"分别打分，验证 reward 方向是否和准确率一致。如果 reward 给错误输出的分更高 → reward 函数本身就有问题。
 #### 2.4 根因归类
 
 把发现映射到一类根因，**每个 pattern 必须归一类**（可并列，但要主次分明）：
@@ -243,8 +261,9 @@ for k in prev:
 |---|---|---|
 | 分流错误占比高 (>30%) | reward 对 complex 误判惩罚不足 | 改 `zk_reward_fn` 加分流项 |
 | 语义错误集中在少数 tag | 训练数据该类分布不足 | 定向补数据 |
-| 需求集合失败 case 在训练集有近邻 | 数据质量 / 标签噪声 / SFT 学不动 | 审核数据 + 加大 SFT epoch |
-| 需求集合失败 case 在训练集无近邻 | 分布外 | 补数据（最直接） |
+| 需求集合失败 case 训练集近邻 **label 一致** | SFT 学不动 / reward 信号弱 | 加大 SFT epoch / 改 reward |
+| 需求集合失败 case 训练集近邻 **label 矛盾**（mislabeled） ⚠️ | 训练集错标 | 按 §2.3 错标清单逐条修标，进 Step 4 数据修订 |
+| 需求集合失败 case 在训练集**无近邻** | 分布外 | 补数据（最直接） |
 | new/regressed case 多 | 上轮干预有副作用 | 回退 or 缩小干预范围 |
 | 训练 prompt ≠ eval prompt | 格式不匹配（silent bug） | 对齐模板，常常能"白捡"几个点 |
 | 错误 query 含状态描述句但训练集标 Agent | 标签规则违反（R3） | 改训练集对应样本为 CT，按 4.3.1 规则检查 |
@@ -313,7 +332,14 @@ conflict_rate = len(conflicts) / len(query_to_golds)
 | 子集 | 准确率 | 总数 | 错误数 | 距 95% gap |
 
 ### 失败 case 与训练数据关联
-- <子集>: N 条失败中 X 条在训练集有近邻，Y 条无近邻 → 主要缺口: <数据缺失/质量>
+- <子集>: N 条失败中 X 条**有近邻且 label 一致**、Y 条**无近邻**、Z 条**近邻 mislabeled** ⚠️ → 主要缺口: <分布外补数据 / SFT 学不动 / 训练集错标>
+
+#### 训练集错标清单（逐条，对应 Z 条 mislabeled）
+| 失败 case query | 训练样本 file:line | hash | 当前 label | 应改 label | 相似度 | 违反规则 |
+|---|---|---|---|---|---|---|
+|  | augment_.jsonl:42 | 
 | Agent | CT | 0.93 | R1（单 POI+多形容词） |
+
+> 没有 mislabeled 时这张表省略；有则进入下一轮 Step 4 的修订清单。
 
 ## 设备维度（只分析车载）
 | 设备 | 准确率 | 总数 | 错误数 |
@@ -332,7 +358,7 @@ query: ...
 label: ...
 模型输出: ...
 归类: 分流/语义
-训练集近邻: 有/无
+训练集近邻: 有 label 一致 / 无 / ⚠️ mislabeled（hash=xxx, 相似度 0.xx, 当前=Agent / 应改=CT）
 
 ## 初始错误分布总结
 | 根因类 | 错误数 | 占比 | 代表 pattern |
@@ -375,8 +401,15 @@ label: ...
 > 准确率来自 lark_template.json（含 complex 门禁检查），B/G 数来自 CSV 的纯模型 GSB（不含 complex 门禁），两者口径不同。
 
 ### 失败 case 与训练数据关联
-- <子集>: N 条失败中 X 条在训练集有近邻，Y 条无近邻 → 主要缺口: <数据缺失/质量>
-- Reward 对齐抽样: N/10 条 reward 方向与准确率一致
+- <子集>: N 条失败中 X 条**有近邻且 label 一致**、Y 条**无近邻**、Z 条**近邻 mislabeled** ⚠️ → 主要缺口: <分布外补数据 / SFT 学不动 / 训练集错标>
+- Reward 对齐（全量）: N/N 条 reward 方向与准确率一致
+
+#### 训练集错标清单（逐条，对应 Z 条 mislabeled）
+| 失败 case query | 训练样本 file:line | hash | 当前 label | 应改 label | 相似度 | 违反规则 |
+|---|---|---|---|---|---|---|
+|  | augment_.jsonl:42 | 
 | Agent | CT | 0.93 | R1（单 POI+多形容词） |
+
+> 没有 mislabeled 时这张表省略；有则 Z 条样本自动进入下一轮 Step 4 的修订清单。
 
 ## 设备维度（只分析车载）
 | 设备 | 基线准确率 | 新模型准确率 | 变化 | 旧对新错 |
@@ -421,7 +454,7 @@ label: xxx
 旧:    xxx      # origin_predict_base
 新:    xxx      # origin_predict_dev（保留 complex= 前缀）
 归类:  分流/语义/两者  | 跨轮: persistent/new/regressed
-训练集近邻: 有/无（hash=xxx, 相似度 0.xx）
+训练集近邻: 有 label 一致（hash=xxx, 相似度 0.xx）/ 无 / ⚠️ mislabeled（hash=xxx, 相似度 0.xx, 当前=Agent / 应改=CT）
 ```
 
 > **重要**：对话历史从 input 字段的 `[对话历史]` 段提取，不能省略。很多错误（如短句闲聊被误判为 Agent）只有在多轮上下文中才能理解根因。
@@ -589,19 +622,33 @@ for f in glob.glob('ai-planning/data/train_set/zk_intent/*.jsonl'):
                 candidates.append((f, ln, q, d['output']))
 
 # 3. 输出 csv 让人审核（不改文件）
-import csv
-with open(f'/tmp/candidates_r{RUNDIC}.csv', 'w', encoding='utf-8-sig') as fp:
+#    必须写到 $AUTORESEARCH_CHAT_ROOT/output/ —— 这是 NFS 路径，前端「分层结果分析」
+#    卡片从 chat_root/output/ 读这条；写到 jupyter pod 本地 cwd 会导致前端 not found。
+import csv, os
+out_csv = os.path.join(os.environ['AUTORESEARCH_CHAT_ROOT'],
+                      'output', f'relabel_candidates_{RUNDIC}.csv')
+os.makedirs(os.path.dirname(out_csv), exist_ok=True)
+with open(out_csv, 'w', encoding='utf-8-sig') as fp:
     w = csv.writer(fp)
     w.writerow(['file','line','query','old_label','是否改(1/0)','建议新label'])
     for c in candidates: w.writerow(list(c)+['',''])
 ```
 
+**两阶段 UI 展示约定（强制）**：
+
+| 阶段 | 产物 | 落盘路径 | 前端卡片 |
+|---|---|---|---|
+| 阶段一：预计修改候选 | `relabel_candidates_.csv` | `$AUTORESEARCH_CHAT_ROOT/output/`（NFS） | **分层结果分析（dist-analysis）** |
+| 阶段二：确认修改最终 | `modified_samples.jsonl` | `$AUTORESEARCH_CHAT_ROOT/results/data_clean_/` | **数据增强（augment）** |
+
+候选阶段（246 条等量级）的 CSV 必须写到 `output/relabel_candidates_.csv` 让分析卡片可读；确认修改阶段（用户审核后落盘的 135 条）写入 `results/data_clean_/modified_samples.jsonl` 让数据增强卡片可读。**不要混落**——把候选 CSV 写到 `/tmp/` 或 `data_clean_/` 都会让 UI 看不到。
+
 **Step B：量级判定（决定是否走人审）**
 
 | 候选量 | 处理 |
 |---|---|
 | ≤ 50 条 | 程序化 sanity check + 自动改 |
-| 51 ~ 200 条 | **必须**抽样 30~50 条到飞书 sheet 让人审 1/0，按抽样比例外推到全量 |
+| 51 ~ 200 条 | **必须**全量导出到飞书 sheet 让人逐条审 1/0（不抽样） |
 | > 200 条 | **强制 H-i-T-L 介入**（触发条件 #3），让人定更精细 pattern 收窄 |
 
 **Step C：备份（强制，覆盖前必做）**
@@ -642,7 +689,7 @@ for f, edits in edits_by_file.items():
 
 改完不能直接训：
 1. **数据 sanity**：`prepare_and_train_sft.py prepare`，对比合并后总数（旧总数 - 删除数 = 新总数）
-2. **抽样人审**：随机抽 5 条，看 query/label/instruction 符合预期
+2. **全量人审**：所有改动条目逐条过一遍，看 query/label/instruction 符合预期
 3. **格式校验**：用 4.1.1 字段约束扫一遍改后 output
 
 **Step F：删除 vs 修改的选择**
@@ -828,7 +875,7 @@ def gen_augment(badcase: dict, n: int = 8) -> list[dict]:
 - **ground_truth 格式校验**：`Agent(tag=...)` / `ComplexTask(...)` / `QA()` / `Chat()` 是否和测试集一致？GPT-5.4 偶尔会把 tag 改写或加空格，必须用 `zk_reward_fn` 里的 parser 过一遍，parse 失败的扔掉。
 - **context 完整性**：`context` 必须是 dict 且含 `location` / `rag` 两键（可为空字符串但不能缺）；`prev_session` 必须是 list。结构不符的丢弃。
 - **泄漏检测**：生成 `current_query` 与所有测试集（需求集合 + 大盘 + specific）做**精确匹配 + 近似匹配**（MinHash or embedding cos > 0.9）。命中则整条丢弃。
-- **Label 自洽**：生成的 `(current_query, prev_session, context) → ground_truth` 必须和原 badcase 的 ground_truth 语义一致。抽 10% 跑一次 GPT-5.4 自检（"下面这条 current_query 的正确 ground_truth 是什么？"），和声明 ground_truth 不一致的丢弃。
+- **Label 自洽**：生成的 `(current_query, prev_session, context) → ground_truth` 必须和原 badcase 的 ground_truth 语义一致。**全量**跑一次 GPT-5.4 自检（"下面这条 current_query 的正确 ground_truth 是什么？"），和声明 ground_truth 不一致的丢弃。
 - **Prompt 模板一致性**：并入训练前，把 `(prev_session, context, current_query)` 渲染成最终 SFT prompt，和 eval prompt 逐字段比对（`complex=true/false` 前缀、system prompt、function schema），不一致就对齐模板——常常能白捡几个点。
 - **去重**：与 `ai-planning/data/train_set/zk_intent/` 下所有已存在的 `*_train.jsonl`（历史增强 + 原始种子训练文件）去重（`current_query` 精确 + 近似）。
 
@@ -1014,10 +1061,11 @@ def induce_rule(cluster_cases, all_test_rows, existing_rules):
 ### R{N} 自动归纳（{runDic} 轮）
 **规则**：含 pattern `{regex}` → {gold}
 **置信度**：c1={c1:.2f}（错例 {support_err}/{support_err_total} 一致）, c2={c2:.2f}（全测试集 {full_majority}/{support_total} 一致）
-**锚定 case**（自动从测试集抽 3 条）：
+**锚定 case**（列出该 pattern 在测试集命中的全部 case，不抽样）：
 - {case1}
 - {case2}
 - {case3}
+- ...（共 {N} 条）
 **自动检查正则**：`{regex}`
 ```
 
@@ -1053,16 +1101,95 @@ ai-planning/data/train_set/zk_intent/augment_.jsonl
 
 JSONL 行里的 `sub_cate` 字段仅用于内部路由/去重，归档时丢弃不写入 CSV。因为 4.2 的输出 schema 已经和 CSV 列名对齐，归档就是把每个 `augment_.jsonl` 行序列化成 CSV 单元格（`prev_session` / `context` 两列用 `json.dumps` 回写成字符串），没有字段重命名。
 
+#### 4.5 label-master 标签复核（落盘后、SFT 前，**强制**）
+
+任何写入训练目录的修改/新增样本都必须经 [label-master skill](../../label-master/SKILL.md) 复核**两层**：格式 + 语义。**没过这关不许进 Step 5。**
+
+**复核范围**（两份必须全过）：
+
+| 文件 | 来源 | 复核什么 |
+|---|---|---|
+| `$AUTORESEARCH_CHAT_ROOT/results/data_clean_/modified_samples.jsonl` | H1 改标（complex 翻转 / tag 改写等） | 改后的 `output` 字段 |
+| `$AUTORESEARCH_CHAT_ROOT/ai-planning/data/train_set/zk_intent/augment_.jsonl` | H2 仿写（新增训练样本） | `output` 字段 |
+
+**层 1：格式校验（确定性，自动跑）**
+
+```bash
+cd /home/mi/zk-data-agent-wsh   # 或 wherever skills 包在的项目根
+# H1 改标
+python skills/label-master/scripts/validate_label_output.py \
+  --file "$AUTORESEARCH_CHAT_ROOT/results/data_clean_/modified_samples.jsonl" \
+  --field output_after
+# H2 仿写
+python skills/label-master/scripts/validate_label_output.py \
+  --file "$AUTORESEARCH_CHAT_ROOT/ai-planning/data/train_set/zk_intent/augment_.jsonl" \
+  --field output
+```
+
+任一行 fail（tag 不存在 / Agent 包装错 / function 引用错）→ **必须修，不许直接跳过**。修完原地重跑直到全过。
+
+**层 2：语义复核（Skill 调用 label-master Agent）**
+
+格式过了不代表标对了。每条修改/新增样本要用 label-master 做边界判断：
+
+1. 抽出 `(query, output_after)` 或 `(query, output)` 对，按 `sub_cate` 分组（每组 ≤30 条作为一批）
+2. 每批用 `Skill(skill="label-master", args=...)` 调用，args 里给：
+   - 全部 `(query, 当前 label)` 对
+   - 让 label-master 按 §决策流程 / §候选召回索引 / §高频混淆边界 判定每条
+   - 输出格式：每条 → `{verdict: 通过 | 不通过, 推荐标签, 排除理由, 易混淆边界}`
+3. 收集 verdict，写入 `$AUTORESEARCH_CHAT_ROOT/results/data_clean_/label_master_review.jsonl`，每行一条 verdict
+
+**verdict 处置规则（自动）**：
+
+| 比例 | 处置 |
+|---|---|
+| 不通过 ≤ 5% | 自动丢弃这些样本（H2 → 从 augment_.jsonl 删行；H1 → 回滚改动到 .bak）+ 写明丢弃数到 iteration_log |
+| 不通过 5%~20% | **触发 H-i-T-L #6**（gold 可疑），把不通过样本导出飞书 sheet 让人审 |
+| 不通过 > 20% | **强制暂停**，Step 4 假设本身有问题，回 Step 3 重新形成假设 |
+
+**写完复核报告才允许进 Step 5。**verdict 文件不存在 / 没跑层 2 → Step 5 拒绝启动（5.0 的准入检查会查 `label_master_review.jsonl` 存在）。
+
+🚨 **注意**：label-master 是"知识 + Agent 推理"型 Skill，不是黑盒分类器。批量调用时严禁要求它返回 1/0 标量；必须输出 `verdict + 推荐标签 + 排除理由` 三件套，便于人类回查。
+
 ### 5. SFT 训练
 
+🚨 **Step 4 → Step 5 硬连接（NEVER STOP，反惯性反复强调）**
+
+写完 `{"step":"augment","status":"complete",...}` 那一行**之后**，**同一轮 bash 不许结束、不许写"Step 4 完成"简报、不许等回调**。必须**紧接着**：
+
+1. `echo '{"step":"sft","status":"running",...}' >> $SESSION_OUTPUT/program-state.jsonl`
+2. 跑层 1 + 层 2 label-master 复核（§4.5）
+3. 如果复核全过：在 SAME bash turn 里调 `bash scripts/submit_sft_via_cml.sh  ` 并立刻挂 watcher（§5.2 的 cml custom_train + workflow chain）
+
+**反模式**（被反复踩坑，永远禁止）：
+
+- ❌ 写完 `augment=complete` → 写一份漂亮的"Step 4 完成"进度简报 → turn 结束 → 等下一次触发
+- ❌ "augment 已完成，下一步等回调后续做 SFT" — 不存在等回调，augment 完成的那一刻就是 SFT 启动的那一刻
+- ❌ 把 H2 后台任务的 stdout 当成"用户消息"处理完就 turn 结束 — bg 任务回调不是 turn 结束信号，augment=complete 也不是
+
+**模型本能会想停（"任务段落感"），SKILL NEVER STOP 压不住这个本能**——所以这里写成硬命令清单，照清单第一条到第三条机械执行，做完才允许 turn 结束。
+
+
 使用 `prepare_and_train_sft.py` 完成数据组装和训练。
 
 #### 5.0 环境准备
 
+**准入检查（少一项不许进）**：
+
 ```bash
-# 克隆 zk_trainer（仅首次）
-cd /mnt/wangsenhao/autoresearch-zk
-git clone git@git.n.xiaomi.com:wangsenhao/zk_trainer.git
+# 1. label-master 复核报告必须存在且全通过
+REVIEW="$AUTORESEARCH_CHAT_ROOT/results/data_clean_${RUNDIC}/label_master_review.jsonl"
+[ -f "$REVIEW" ] || { echo "label-master 复核未完成，回 §4.5"; exit 1; }
+NOT_PASS=$(grep -c '"verdict":"不通过"' "$REVIEW" || echo 0)
+TOTAL=$(wc -l < "$REVIEW")
+if [ "$TOTAL" -gt 0 ] && [ $((NOT_PASS * 100 / TOTAL)) -gt 5 ]; then
+  echo "不通过比例 $NOT_PASS/$TOTAL > 5%，按 §4.5 处置规则走 H-i-T-L 或回 Step 3"
+  exit 1
+fi
+
+# 2. zk_trainer 仓库已 clone
+cd "$AUTORESEARCH_CHAT_ROOT"
+[ -d zk_trainer ] || git clone git@git.n.xiaomi.com:wangsenhao/zk_trainer.git
 ```
 
 **训练配置**：
@@ -1098,8 +1225,8 @@ git clone git@git.n.xiaomi.com:wangsenhao/zk_trainer.git
 - `prepare_and_train_sft.py train` 命令行参数 `--model_path` 必须检查为 basemodel 路径
 
 **反模式**：
-- ❌ `--model_path /mnt/wangsenhao/autoresearch-zk/sft_output`（继续训练）
-- ❌ `--model_path /mnt/wangsenhao/autoresearch-zk/sft_output_r17776`（从历史轮 ckpt）
+- ❌ `--model_path $AUTORESEARCH_CHAT_ROOT/sft_output`（继续训练）
+- ❌ `--model_path $AUTORESEARCH_CHAT_ROOT/sft_output_r17776`（从历史轮 ckpt）
 - ❌ 任何形式的 "增量 SFT"（除非显式声明是为了验证"从 X ckpt 继续是否更好"的对照实验，且单次性，log 要明确标记）
 
 #### 5.0.1 训练产物备份（R23-R28 经验沉淀）
@@ -1127,13 +1254,15 @@ R29 起 SFT 走 cml custom_train submit（见 5.2），watcher 直接复用 `sub
 
 ```bash
 # 预清理：必须在挂 watcher 之前执行
-rm -f /mnt/wangsenhao/autoresearch-zk/sft_output/_SUCCESS
+rm -f "$AUTORESEARCH_CHAT_ROOT/sft_output/_SUCCESS"
 
 # Watcher 1：训练阶段 —— cml state 为主判据，_SUCCESS 辅助
+# 注意：heredoc 用 'EOF' 防止外层 shell 展开，$AUTORESEARCH_CHAT_ROOT
+# 在 watcher 子进程里运行时再展开（远端 bash 会自动注入这个 env）。
 cat > /tmp/wait_train_.sh <<'EOF'
 #!/bin/bash
 JOB_ID=
-SUCCESS=/mnt/wangsenhao/autoresearch-zk/sft_output/_SUCCESS
+SUCCESS="$AUTORESEARCH_CHAT_ROOT/sft_output/_SUCCESS"
 export PATH=$HOME/.cloudml-cli/bin:$PATH
 while true; do
   sleep 60
@@ -1174,7 +1303,7 @@ EOF
 # /tmp/auto_eval_after_train.sh —— 等本地训练 PID 结束 → 自动起 cml workflow
 TRAIN_PID=$1
 RUN_DIC=$2
-MODEL_NEW=/mnt/wangsenhao/autoresearch-zk/sft_output
+MODEL_NEW=$AUTORESEARCH_CHAT_ROOT/sft_output
 MODEL_OLD=<基线模型>
 
 while kill -0 $TRAIN_PID 2>/dev/null; do sleep 60; done
@@ -1217,7 +1346,7 @@ R23-R28 期间用本地 `nohup python3 prepare_and_train_sft.py train ...` 启
 ##### 一键提交脚本
 
 ```bash
-cd /mnt/wangsenhao/autoresearch-zk
+cd "$AUTORESEARCH_CHAT_ROOT"
 ./scripts/submit_sft_via_cml.sh  [PREV_RUNDIC]
 
 # 例：R29 训练（上一轮 R28）
@@ -1333,7 +1462,7 @@ tail -f /tmp/r_logs/cml_watcher.log
 | **跨子集净退步：目标子集 +X / 其他子集合计 -Y, 净 < 0.3pp** | **回退或缩小干预范围**（R27 经验：单看目标子集涨容易自欺）|
 | **连续 3 轮目标子集净提升 ≤ 0.5pp** | **进入瓶颈期**：输出 SFT 天花板报告，触发 H-i-T-L #6 |
 | **跨子集 gold 矛盾率 > 5%** | **结构性天花板**：触发 H-i-T-L #2，停 SFT 转 RL 或返工标注 |
-| **要批改旧标签 > 50 条** | **触发 H-i-T-L #3**，抽样人审，按比例外推 |
+| **要批改旧标签 > 50 条** | **触发 H-i-T-L #3**，全量人审 |
 | **Gold drift ≥ 10 条** | **触发 H-i-T-L #1**，暂停迭代确认是否同步训练集 |
 | 训练前未备份 sft_output | **强制 mv sft_output sft_output_r{prev}**，不允许覆盖 |
 
@@ -1347,7 +1476,7 @@ tail -f /tmp/r_logs/cml_watcher.log
 |---|---|---|---|
 | 1 | **Gold drift 检测到 ≥10 条** | 暂停迭代，让人确认是否同步更新训练集 | R23 100 条翻转 |
 | 2 | **跨子集同形 query gold 矛盾率 > 5%** | 输出"结构性天花板"报告，让人选：硬推目标子集 / 接受 / 转 RL | 复杂导航 vs 可聊可控 矛盾 |
-| 3 | **要批改旧标签 > 50 条** | 暂停，导出抽样 ≥30 条到飞书 sheet 让人逐条确认 | R28 改 230 条引发副作用 |
+| 3 | **要批改旧标签 > 50 条** | 暂停，导出全量到飞书 sheet 让人逐条确认 | R28 改 230 条引发副作用 |
 | 4 | **跨子集净退步**（目标 +X / 其他合计 -Y, 净 < 0.3pp） | 暂停，让人决策回退 / 接受 / 换策略 | R27 可聊可控 -1.34pp |
 | 5 | **连续 3 轮目标子集净提升 ≤ 0.5pp** | 输出 SFT 天花板报告，让人选继续 SFT / 转 RL / 接受 / 换基模 | R25-R28 复杂导航 +0.7~3pp 递减 |
 | 6 | **测试集错例里 gold 可疑（自相矛盾的同 pattern）** | 列出可疑 gold 让人确认 / 反馈标注团队 | R28「找+模糊属性」双向标注 |
@@ -1359,8 +1488,8 @@ tail -f /tmp/r_logs/cml_watcher.log
 
 | # | 触发条件 | 应对动作 | 经验来源 |
 |---|---|---|---|
-| T1 | **单轮批改旧标签 > 50 条** | 抽样 30 条到飞书 sheet 逐条审；按抽样的"改/不改"比例外推全量 | R28 改 230 条引发跨子集副作用 |
-| T2 | **单轮新仿写 > 100 条 OR 单类 (同 sub_cate / 同 pattern) > 50 条** | 抽样 20 条让人审风险，量级大要拆批 | R23 一次 670 条仿写过载，规则错全反 |
+| T1 | **单轮批改旧标签 > 50 条** | 全量导出到飞书 sheet 逐条审，逐条标 1/0 | R28 改 230 条引发跨子集副作用 |
+| T2 | **单轮新仿写 > 100 条 OR 单类 (同 sub_cate / 同 pattern) > 50 条** | 全量让人审风险，量级大要拆批 | R23 一次 670 条仿写过载，规则错全反 |
 | T3 | **删除训练样本 > 30 条** | 列删除清单 + 删除原因，人确认后才执行（删比改更不可逆） | R26 删 23 P0 通用短词产生副作用 |
 | T4 | **修改 `all_train.jsonl` 主集 > 20 条** | 核心训练集动一行都贵；列改动给人确认 | 主集影响所有 sub_cate，副作用范围最大 |
 | T5 | **跨子集冲突 query：同结构 query 在 ≥3 条训练样本里 gold 不一致** | 让人定边界规则（如「沿途搜 X」是 Agent 还是 CT），不许两边都加 | R26 augment_17752 同 pattern 矛盾仿写 |
@@ -1396,7 +1525,7 @@ tail -f /tmp/r_logs/cml_watcher.log
 2. **现状量化**：目标子集 +X / 其他子集 -Y / 大盘 ±Z
 3. **2-4 个选项**（用 AskUserQuestion 工具）：每个选项含预期收益 + 风险
 4. **推荐选项**：基于经验给出推荐（标 "(推荐)"）
-5. **本地产物**：抽样 / 错例 csv / 飞书 sheet 链接
+5. **本地产物**：全量错例 csv / 飞书 sheet 链接
 
 ### 介入后的恢复
 
diff --git a/skills/model-iteration/scripts/prepare_and_train_sft.py b/skills/model-iteration/scripts/prepare_and_train_sft.py
index f06380f..1dcb978 100644
--- a/skills/model-iteration/scripts/prepare_and_train_sft.py
+++ b/skills/model-iteration/scripts/prepare_and_train_sft.py
@@ -2,16 +2,16 @@
 """
 从 ai-planning 组装 SFT 训练/验证集，并启动 zk_trainer SFT 训练。
 
-用法:
+用法（先 cd "$AUTORESEARCH_CHAT_ROOT"，所有产物落 chat 工作区）:
   # 仅组装数据
-  python prepare_and_train_sft.py prepare --output_dir /mnt/wangsenhao/autoresearch-zk/sft_data
+  python scripts/prepare_and_train_sft.py prepare --output_dir "$AUTORESEARCH_CHAT_ROOT/sft_data"
 
   # 启动训练（需先 prepare）
-  python prepare_and_train_sft.py train \
-    --data_dir /mnt/wangsenhao/autoresearch-zk/sft_data \
+  python scripts/prepare_and_train_sft.py train \
+    --data_dir "$AUTORESEARCH_CHAT_ROOT/sft_data" \
     --model_path /mnt/wangsenhao/verl_zk/Qwen3-4B-Instruct-2507 \
     --model_type qwen3 \
-    --train_output /mnt/wangsenhao/autoresearch-zk/sft_output
+    --train_output "$AUTORESEARCH_CHAT_ROOT/sft_output"
 """
 
 import argparse
diff --git a/src/agent_prompting.py b/src/agent_prompting.py
index 1f2264f..e946922 100644
--- a/src/agent_prompting.py
+++ b/src/agent_prompting.py
@@ -249,6 +249,9 @@ def get_using_your_tools_section(enabled_tool_names: set[str]) -> str:
         items.append(
             '需要可靠保存文件时，可以在 python_exec.code 中使用 pathlib.Path(...).write_text(...)、json.dump/json.dumps、csv.writer 或流式逐行写入；这比把大段内容塞进 write_file 参数更稳定。'
         )
+        items.append(
+            'python_exec 默认 timeout=30s。读 CSV/JSONL 大文件、pandas 处理、批量校验、训练评测脚本等可能 >30s 的任务，调用时显式传 timeout_seconds（常规 120-300、训练/长跑 600+），不要靠默认值。被 timeout(1) kill 后会以 timed_out=true 报错并提示同样的内容，看到就重跑并调大。'
+        )
         if 'python_package' in enabled_tool_names:
             items.append(
                 '当 python_exec 因缺少 pandas、pyarrow、openpyxl 等 Python 包失败时，使用 python_package 在当前用户独立 venv 中安装缺失包，然后重试；不要安装到系统 Python 或项目 .venv。'
diff --git a/src/agent_tool_core.py b/src/agent_tool_core.py
index 0ec8e0f..baabebc 100644
--- a/src/agent_tool_core.py
+++ b/src/agent_tool_core.py
@@ -10,6 +10,7 @@ from .agent_types import AgentPermissions, AgentRuntimeConfig, ToolExecutionResu
 if TYPE_CHECKING:
     from .account_runtime import AccountRuntime
     from .ask_user_runtime import AskUserRuntime
+    from .bash_bg_store import BgTaskSpec, BgTaskStatus
     from .config_runtime import ConfigRuntime
     from .lsp_runtime import LSPRuntime
     from .mcp_runtime import MCPRuntime
@@ -58,6 +59,12 @@ class ToolExecutionContext:
     team_runtime: 'TeamRuntime | None' = None
     workflow_runtime: 'WorkflowRuntime | None' = None
     worktree_runtime: 'WorktreeRuntime | None' = None
+    bg_register: Callable[['BgTaskSpec'], None] | None = None
+    bg_status_query: Callable[[str], 'BgTaskStatus | None'] | None = None
+    bg_kill_request: Callable[[str], bool] | None = None
+    account_id: str | None = None
+    session_id: str | None = None
+    run_id: str | None = None
 
 
 ToolHandler = Callable[
diff --git a/src/agent_tool_specs/execution.py b/src/agent_tool_specs/execution.py
index 2a72979..d1eb41b 100644
--- a/src/agent_tool_specs/execution.py
+++ b/src/agent_tool_specs/execution.py
@@ -44,7 +44,7 @@ def build_execution_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentTool
                     'timeout_seconds': {
                         'type': 'number',
                         'minimum': 1,
-                        'description': '可选超时时间，默认使用当前会话命令超时。',
+                        'description': '可选超时时间，默认 30 秒。读大 CSV/JSONL、pandas 分析、批量校验、训练评测脚本等可能 >30s 的任务必须显式传值（常规 120-300、训练/长跑 600+）。',
                     },
                     'max_output_chars': {
                         'type': 'integer',
@@ -98,16 +98,84 @@ def build_execution_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentTool
                 '不要用 bash 执行 python、python3、pip 或 .venv/bin/python；结构化文件分析、'
                 'JSON/JSONL 处理、批量校验、数据抽样和快速计算必须优先使用 python_exec。'
                 'Python 包安装必须优先使用 python_package。'
+                '需要等待远端长任务（>30s 的训练、文件落盘、外部 API 回调）时，'
+                '设置 run_in_background=true 让命令在后台 detach 跑、立刻返回 task_id；'
+                '默认 wait_for_completion=true 时进程结束后会自动起新一轮把结果送回。'
+                '后台任务输出落在 /.bg_tasks//output。'
             ),
             parameters={
                 'type': 'object',
                 'properties': {
                     'command': {'type': 'string'},
+                    'run_in_background': {
+                        'type': 'boolean',
+                        'description': (
+                            '后台 detach 跑。立刻返回 task_id，不等命令结束。'
+                            '适合 >30s 的远端任务、需要等异步事件的场景。'
+                        ),
+                    },
+                    'wait_for_completion': {
+                        'type': 'boolean',
+                        'description': (
+                            '仅在 run_in_background=true 时生效。'
+                            'true（默认）：进程结束后由后端自动起新一轮把结果作为 system 消息送回；'
+                            'false：不自动续，agent 需自行调 bash_status 取结果。'
+                        ),
+                    },
                 },
                 'required': ['command'],
             },
             handler=resolve_handler(handlers, 'bash', 'execution'),
         ),
+        AgentTool(
+            name='bash_status',
+            description=(
+                '查询通过 bash(run_in_background=true) 启动的后台任务状态。'
+                'block=true 会阻塞到任务终止或 timeout_seconds 到期再返回；'
+                'block=false（默认）立刻返回当前快照。'
+                '返回 status (running/completed/failed/cancelled)、exit_code 和最近输出预览。'
+            ),
+            parameters={
+                'type': 'object',
+                'properties': {
+                    'task_id': {
+                        'type': 'string',
+                        'description': 'bash run_in_background 启动时返回的 task_id。',
+                    },
+                    'block': {
+                        'type': 'boolean',
+                        'description': '是否等到任务终止再返回；默认 false。',
+                    },
+                    'timeout_seconds': {
+                        'type': 'number',
+                        'minimum': 0,
+                        'maximum': 600,
+                        'description': 'block=true 时最长等待秒数，默认 30，上限 600。',
+                    },
+                },
+                'required': ['task_id'],
+            },
+            handler=resolve_handler(handlers, 'bash_status', 'execution'),
+        ),
+        AgentTool(
+            name='bash_kill',
+            description=(
+                '主动终止通过 bash(run_in_background=true) 启动的后台任务。'
+                '注意：用户在前端按 stop 取消当前 run 不会自动杀后台进程（detach 的本意），'
+                '需要明确调用本工具才能终止远端进程。'
+            ),
+            parameters={
+                'type': 'object',
+                'properties': {
+                    'task_id': {
+                        'type': 'string',
+                        'description': 'bash run_in_background 启动时返回的 task_id。',
+                    },
+                },
+                'required': ['task_id'],
+            },
+            handler=resolve_handler(handlers, 'bash_kill', 'execution'),
+        ),
         AgentTool(
             name='sleep',
             description='Pause execution briefly for bounded local wait flows.',
diff --git a/src/agent_tools.py b/src/agent_tools.py
index 550548e..4ea911e 100644
--- a/src/agent_tools.py
+++ b/src/agent_tools.py
@@ -7,7 +7,9 @@ import io
 import json
 import os
 import re
+import secrets
 import selectors
+import shlex
 import shutil
 import signal
 import subprocess
@@ -32,6 +34,7 @@ from .agent_tool_specs.data_agent import build_data_agent_tools
 from .agent_tool_specs.execution import build_execution_tools
 from .agent_tool_specs.files import build_file_tools
 from .agent_types import DEFAULT_MAX_TURNS, ToolExecutionResult
+from .bash_bg_store import BgTaskSpec, BgTaskStatus
 from .data_agent_records import (
     DataRecordError,
     confirm_generation_goal,
@@ -117,6 +120,8 @@ def default_tool_registry() -> dict[str, AgentTool]:
             'python_exec': _run_python_exec,
             'python_package': _run_python_package,
             'bash': _run_bash,
+            'bash_status': _run_bash_status,
+            'bash_kill': _run_bash_kill,
             'sleep': _sleep,
         }),
         AgentTool(
@@ -1734,46 +1739,29 @@ def _ensure_shell_allowed(command: str, context: ToolExecutionContext) -> None:
             'Shell command appears to write platform code. '
             'Platform code directories are read-only in data-agent sessions.'
         )
-    if context.permissions.allow_destructive_shell_commands:
-        return
-    destructive_patterns = [
-        r'(^|[;&|])\s*rm\s',
-        r'(^|[;&|])\s*mv\s',
-        r'(^|[;&|])\s*dd\s',
-        r'(^|[;&|])\s*shutdown\s',
-        r'(^|[;&|])\s*reboot\s',
-        r'(^|[;&|])\s*mkfs',
-        r'(^|[;&|])\s*chmod\s+-r\s+777',
-        r'(^|[;&|])\s*chown\s+-r',
-        r'(^|[;&|])\s*git\s+reset\s+--hard',
-        r'(^|[;&|])\s*git\s+clean\s+-fd',
-        r'(^|[;&|])\s*:\s*>\s*',
-    ]
-    lowered = command.lower()
-    if any(re.search(pattern, lowered) for pattern in destructive_patterns):
-        raise ToolPermissionError(
-            'Potentially destructive shell command blocked. Re-run with --unsafe to allow it.'
-        )
+    return
 
 
 def _looks_like_platform_code_shell_write(
     command: str,
     context: ToolExecutionContext,
 ) -> bool:
+    if context.jupyter_runtime is not None:
+        return False
     if not _is_platform_app_root(context.root):
         return False
     lowered = command.lower()
     write_marker = re.search(
-        r'>|(^|[\s;&|])(tee|touch|python|python3|sed\s+-i|perl\s+-pi)\b',
+        r'(?(?!&)|(^|[\s;&|])(tee|touch|python|python3|sed\s+-i|perl\s+-pi)\b',
         lowered,
     )
     if write_marker is None:
         return False
+    root_prefix = context.root.as_posix().lower()
     platform_fragments = [
-        f'{name}/' for name in _PLATFORM_READONLY_DIRS
+        f'{root_prefix}/{name}/' for name in _PLATFORM_READONLY_DIRS
     ] + [
-        f'{context.root.as_posix().lower()}/{name}/'
-        for name in _PLATFORM_READONLY_DIRS
+        f'./{name}/' for name in _PLATFORM_READONLY_DIRS
     ]
     return any(fragment in lowered for fragment in platform_fragments)
 
@@ -2322,6 +2310,26 @@ def _run_python_exec(arguments: dict[str, Any], context: ToolExecutionContext) -
         ]
         if result.timed_out:
             payload.insert(0, f'timed_out=true\ntimeout_seconds={timeout_seconds:g}')
+            # 被 timeout(1) 砍掉时 ok=True 会让 agent 误以为脚本跑完了；翻成 ok=False，
+            # 错误消息里直接给出建议（调大 timeout、拆分操作、流式读 CSV 等）。
+            raise ToolExecutionError(
+                _truncate_output(
+                    '\n'.join(
+                        [
+                            *payload,
+                            (
+                                f'[hint] python_exec 在 {timeout_seconds:g}s 被 timeout(1) kill，没有跑完。\n'
+                                '首选做法：把 timeout_seconds 调大（数据分析类任务起步给 300，处理大 CSV/JSONL 给 600+）。\n'
+                                '不要把脚本拆成多个更小的 python_exec 调用——每次调用都要重新 import、重读数据、'
+                                '重跑一遍带完整对话上下文的远端 round-trip，拆得越碎总耗时越长。'
+                                '把多步分析（解析、聚合、统计、写出）合并到一个脚本里才是对的。\n'
+                                '只有当数据本身远超内存或单次跑要 1h+ 时，才考虑 pandas chunksize / 流式过滤先把数据降量。'
+                            ),
+                        ]
+                    ).strip(),
+                    max_output_chars,
+                )
+            )
         return (
             _truncate_output('\n'.join(payload).strip(), max_output_chars),
             {
@@ -2393,21 +2401,17 @@ def _run_python_exec(arguments: dict[str, Any], context: ToolExecutionContext) -
             stdout.rstrip(),
             '[stderr]',
             stderr.rstrip(),
+            (
+                f'[hint] python_exec 在 {timeout_seconds:g}s 被 kill，没有跑完。\n'
+                '首选做法：把 timeout_seconds 调大（数据分析类任务起步给 300，处理大 CSV/JSONL 给 600+）。\n'
+                '不要把脚本拆成多个更小的 python_exec 调用——每次调用都要重新 import、重读数据、'
+                '重跑一遍带完整对话上下文的远端 round-trip，拆得越碎总耗时越长。'
+                '把多步分析（解析、聚合、统计、写出）合并到一个脚本里才是对的。\n'
+                '只有当数据本身远超内存或单次跑要 1h+ 时，才考虑 pandas chunksize / 流式过滤先把数据降量。'
+            ),
         ]
-        return (
-            _truncate_output('\n'.join(payload).strip(), max_output_chars),
-            {
-                'action': 'python_exec',
-                'mode': mode,
-                'interpreter': interpreter,
-                'scratchpad_directory': str(context.scratchpad_directory) if context.scratchpad_directory else '',
-                'python_env_dir': str(context.python_env_dir) if context.python_env_dir else '',
-                'timed_out': True,
-                'timeout_seconds': timeout_seconds,
-                'stdout_preview': _snapshot_text(stdout),
-                'stderr_preview': _snapshot_text(stderr),
-                'output_preview': _snapshot_text('\n'.join(payload).strip()),
-            },
+        raise ToolExecutionError(
+            _truncate_output('\n'.join(payload).strip(), max_output_chars)
         )
     except ToolExecutionError:
         raise
@@ -2690,7 +2694,15 @@ def _resolve_python_interpreter(context: ToolExecutionContext) -> str:
 
 
 def _run_bash(arguments: dict[str, Any], context: ToolExecutionContext) -> str:
-    command = _require_string(arguments, 'command')
+    if arguments.get('run_in_background'):
+        return _run_bash_background(arguments, context)
+    raw_command = arguments.get('command')
+    if not isinstance(raw_command, str) or not raw_command.strip():
+        return (
+            '(no-op: empty bash command — pass a non-empty `command` string '
+            'to actually execute something)'
+        )
+    command = raw_command
     _ensure_shell_allowed(command, context)
     if context.jupyter_runtime is not None:
         result = context.jupyter_runtime.run_command(
@@ -2752,6 +2764,276 @@ def _run_bash(arguments: dict[str, Any], context: ToolExecutionContext) -> str:
     )
 
 
+def _new_bg_task_id() -> str:
+    return f'bg_{secrets.token_hex(4)}'
+
+
+def _bg_extract_pid(stdout: str) -> int | None:
+    match = re.search(r'BG_TASK_PID=(\d+)', stdout)
+    if match is None:
+        return None
+    try:
+        return int(match.group(1))
+    except ValueError:
+        return None
+
+
+def _bg_remote_launcher(
+    *,
+    task_dir: str,
+    output_path: str,
+    pid_path: str,
+    exit_code_path: str,
+    user_command: str,
+) -> str:
+    inner = (
+        f'( {user_command} ) > {shlex.quote(output_path)} 2>&1'
+        f' ; echo $? > {shlex.quote(exit_code_path)}'
+    )
+    return (
+        f'mkdir -p {shlex.quote(task_dir)}\n'
+        f'nohup bash -c {shlex.quote(inner)} /dev/null 2>&1 &\n'
+        'PID=$!\n'
+        f'echo $PID > {shlex.quote(pid_path)}\n'
+        'echo "BG_TASK_PID=$PID"\n'
+    )
+
+
+def _run_bash_background(
+    arguments: dict[str, Any],
+    context: ToolExecutionContext,
+) -> tuple[str, dict[str, Any]]:
+    raw_command = arguments.get('command')
+    if not isinstance(raw_command, str) or not raw_command.strip():
+        return (
+            '(no-op: empty bash command — pass a non-empty `command` string '
+            'to actually execute something)',
+            {'action': 'bash_background_noop'},
+        )
+    command = raw_command
+    _ensure_shell_allowed(command, context)
+    wait_for_completion = bool(arguments.get('wait_for_completion', True))
+    task_id = _new_bg_task_id()
+
+    if context.jupyter_runtime is not None:
+        workspace = context.jupyter_runtime.binding.workspace_cwd
+        task_dir = f'{workspace}/.bg_tasks/{task_id}'
+        output_path = f'{task_dir}/output'
+        pid_path = f'{task_dir}/pid'
+        exit_code_path = f'{task_dir}/exit_code'
+        launcher = _bg_remote_launcher(
+            task_dir=task_dir,
+            output_path=output_path,
+            pid_path=pid_path,
+            exit_code_path=exit_code_path,
+            user_command=command,
+        )
+        result = context.jupyter_runtime.run_command(
+            launcher,
+            timeout_seconds=min(context.command_timeout_seconds, 15.0),
+            max_output_chars=4096,
+            cancel_event=context.cancel_event,
+        )
+        if result.exit_code != 0:
+            raise ToolExecutionError(
+                f'后台 bash 启动失败 exit_code={result.exit_code}\n'
+                f'stderr={result.stderr.strip()}'
+            )
+        pid = _bg_extract_pid(result.stdout)
+        if pid is None:
+            raise ToolExecutionError(
+                f'后台 bash 启动后未拿到 pid\nstdout={result.stdout.strip()}'
+            )
+        spec = BgTaskSpec(
+            task_id=task_id,
+            account_key=context.account_id or '',
+            session_id=context.session_id or '',
+            run_id=context.run_id or '',
+            pid=pid,
+            task_dir=task_dir,
+            output_path=output_path,
+            pid_path=pid_path,
+            exit_code_path=exit_code_path,
+            command=command,
+            started_at=time.time(),
+            wait_for_completion=wait_for_completion,
+        )
+        if context.bg_register is not None:
+            context.bg_register(spec)
+        content = (
+            f'已在远端后台启动\n'
+            f'task_id={task_id}\n'
+            f'pid={pid}\n'
+            f'task_dir={task_dir}\n'
+            f'output={output_path}\n'
+            f'wait_for_completion={wait_for_completion}\n'
+            '完成后将自动起新一轮把结果送回；'
+            '中途可用 bash_status(task_id) 查看，bash_kill(task_id) 终止。'
+        )
+        return (
+            content,
+            {
+                'action': 'remote_bash_bg',
+                'task_id': task_id,
+                'pid': pid,
+                'task_dir': task_dir,
+                'output_path': output_path,
+                'command': command,
+                'wait_for_completion': wait_for_completion,
+            },
+        )
+
+    # Local fallback: spawn detached subprocess with file redirects.
+    base_dir = (context.scratchpad_directory or context.root) / '.bg_tasks' / task_id
+    base_dir.mkdir(parents=True, exist_ok=True)
+    output_path = base_dir / 'output'
+    pid_path = base_dir / 'pid'
+    exit_code_path = base_dir / 'exit_code'
+    inner = (
+        f'( {command} ) > {shlex.quote(str(output_path))} 2>&1'
+        f' ; echo $? > {shlex.quote(str(exit_code_path))}'
+    )
+    process = subprocess.Popen(
+        ['bash', '-c', inner],
+        cwd=_execution_cwd(context),
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        env=_build_subprocess_env(context),
+        start_new_session=True,
+    )
+    pid_path.write_text(f'{process.pid}\n', encoding='utf-8')
+    spec = BgTaskSpec(
+        task_id=task_id,
+        account_key=context.account_id or '',
+        session_id=context.session_id or '',
+        run_id=context.run_id or '',
+        pid=process.pid,
+        task_dir=str(base_dir),
+        output_path=str(output_path),
+        pid_path=str(pid_path),
+        exit_code_path=str(exit_code_path),
+        command=command,
+        started_at=time.time(),
+        wait_for_completion=wait_for_completion,
+    )
+    if context.bg_register is not None:
+        context.bg_register(spec)
+    content = (
+        f'已在本地后台启动\n'
+        f'task_id={task_id}\n'
+        f'pid={process.pid}\n'
+        f'output={output_path}\n'
+    )
+    return (
+        content,
+        {
+            'action': 'bash_bg',
+            'task_id': task_id,
+            'pid': process.pid,
+            'task_dir': str(base_dir),
+            'output_path': str(output_path),
+            'command': command,
+            'wait_for_completion': wait_for_completion,
+        },
+    )
+
+
+def _format_bash_status(status: BgTaskStatus, max_output_chars: int) -> str:
+    finished = (
+        time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(status.finished_at))
+        if status.finished_at is not None
+        else '-'
+    )
+    started = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(status.started_at))
+    body = [
+        f'task_id={status.task_id}',
+        f'status={status.status}',
+        f'pid={status.pid}',
+        f'started_at={started}',
+        f'finished_at={finished}',
+        f'exit_code={status.exit_code if status.exit_code is not None else "-"}',
+        f'auto_resumed={status.auto_resumed}',
+        '[output_preview]',
+        status.output_preview or '(empty)',
+    ]
+    return _truncate_output('\n'.join(body), max_output_chars)
+
+
+def _run_bash_status(
+    arguments: dict[str, Any],
+    context: ToolExecutionContext,
+) -> tuple[str, dict[str, Any]]:
+    task_id = _require_string(arguments, 'task_id')
+    if context.bg_status_query is None:
+        raise ToolExecutionError('bg_status_query 未注入；当前运行环境不支持 bash 后台任务')
+    block = bool(arguments.get('block', False))
+    timeout_seconds = float(arguments.get('timeout_seconds', 30))
+    timeout_seconds = max(0.0, min(timeout_seconds, 600.0))
+    deadline = time.monotonic() + timeout_seconds if block else None
+
+    while True:
+        status = context.bg_status_query(task_id)
+        if status is None:
+            raise ToolExecutionError(f'未找到 task_id={task_id}')
+        if not block or status.status != 'running':
+            return (
+                _format_bash_status(status, context.max_output_chars),
+                {
+                    'action': 'bash_status',
+                    'task_id': task_id,
+                    'status': status.status,
+                    'exit_code': status.exit_code,
+                    'auto_resumed': status.auto_resumed,
+                },
+            )
+        if deadline is None or time.monotonic() >= deadline:
+            return (
+                _format_bash_status(status, context.max_output_chars),
+                {
+                    'action': 'bash_status',
+                    'task_id': task_id,
+                    'status': status.status,
+                    'timed_out_block': True,
+                },
+            )
+        if context.cancel_event is not None and context.cancel_event.is_set():
+            return (
+                _format_bash_status(status, context.max_output_chars),
+                {
+                    'action': 'bash_status',
+                    'task_id': task_id,
+                    'status': status.status,
+                    'cancelled': True,
+                },
+            )
+        time.sleep(min(2.0, max(0.5, deadline - time.monotonic())))
+
+
+def _run_bash_kill(
+    arguments: dict[str, Any],
+    context: ToolExecutionContext,
+) -> tuple[str, dict[str, Any]]:
+    task_id = _require_string(arguments, 'task_id')
+    if context.bg_kill_request is None:
+        raise ToolExecutionError('bg_kill_request 未注入；当前运行环境不支持 bash 后台任务')
+    killed = context.bg_kill_request(task_id)
+    content = (
+        f'已终止 task_id={task_id}'
+        if killed
+        else f'未能终止 task_id={task_id}（可能已结束或不存在）'
+    )
+    return (
+        content,
+        {
+            'action': 'bash_kill',
+            'task_id': task_id,
+            'killed': killed,
+        },
+    )
+
+
 def _web_fetch(arguments: dict[str, Any], context: ToolExecutionContext) -> str:
     raw_url = _require_string(arguments, 'url')
     max_chars = _coerce_int(arguments, 'max_chars', context.max_output_chars)
diff --git a/src/agent_types.py b/src/agent_types.py
index 227a64f..4e5d50a 100644
--- a/src/agent_types.py
+++ b/src/agent_types.py
@@ -157,7 +157,7 @@ DEFAULT_MAX_TURNS = 50
 class AgentRuntimeConfig:
     cwd: Path
     max_turns: int = DEFAULT_MAX_TURNS
-    command_timeout_seconds: float = 30.0
+    command_timeout_seconds: float = 300.0
     max_output_chars: int = 50000  # ≈ 20k token (Chinese/code mix ≈ 2.5 chars/token)
     stream_model_responses: bool = False
     auto_snip_threshold_tokens: int | None = None
diff --git a/src/bash_bg_store.py b/src/bash_bg_store.py
new file mode 100644
index 0000000..ef055d7
--- /dev/null
+++ b/src/bash_bg_store.py
@@ -0,0 +1,211 @@
+"""Persist background bash task records.
+
+Sibling to run_state_store.py: a SQLite table tracking bg shell tasks the
+agent spawns via bash(run_in_background=true). The store survives backend
+restarts so polling can be resumed; the actual remote process is owned by
+nohup on the Jupyter terminal and is independent of this store.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from threading import RLock
+from typing import Any
+
+
+ACTIVE_BG_STATUSES = {'running'}
+TERMINAL_BG_STATUSES = {'completed', 'failed', 'cancelled'}
+
+
+@dataclass(frozen=True)
+class BgTaskSpec:
+    """Payload an agent-side handler hands to the backend on registration."""
+
+    task_id: str
+    account_key: str
+    session_id: str
+    run_id: str
+    pid: int
+    task_dir: str
+    output_path: str
+    pid_path: str
+    exit_code_path: str
+    command: str
+    started_at: float
+    wait_for_completion: bool
+
+
+@dataclass(frozen=True)
+class BgTaskStatus:
+    """Snapshot the agent reads back via bash_status."""
+
+    task_id: str
+    status: str
+    pid: int
+    started_at: float
+    finished_at: float | None
+    exit_code: int | None
+    output_path: str
+    output_preview: str
+    auto_resumed: bool
+
+
+class BashBgStore:
+    def __init__(self, db_path: Path) -> None:
+        self.db_path = db_path
+        self._lock = RLock()
+
+    def record_start(self, spec: BgTaskSpec) -> None:
+        now = time.time()
+        with self._connect() as conn:
+            conn.execute(
+                """
+                insert into bash_bg_tasks (
+                    task_id, account_key, session_id, run_id, pid,
+                    task_dir, output_path, pid_path, exit_code_path,
+                    command, status, started_at, updated_at,
+                    wait_for_completion, auto_resumed
+                )
+                values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?, ?, 0)
+                on conflict(task_id) do nothing
+                """,
+                (
+                    spec.task_id,
+                    spec.account_key,
+                    spec.session_id,
+                    spec.run_id,
+                    spec.pid,
+                    spec.task_dir,
+                    spec.output_path,
+                    spec.pid_path,
+                    spec.exit_code_path,
+                    spec.command,
+                    spec.started_at,
+                    now,
+                    1 if spec.wait_for_completion else 0,
+                ),
+            )
+
+    def mark_completed(
+        self,
+        task_id: str,
+        *,
+        exit_code: int,
+        finished_at: float | None = None,
+    ) -> None:
+        finished = finished_at if finished_at is not None else time.time()
+        status = 'completed' if exit_code == 0 else 'failed'
+        with self._connect() as conn:
+            conn.execute(
+                """
+                update bash_bg_tasks
+                set status = ?, exit_code = ?, finished_at = ?, updated_at = ?
+                where task_id = ? and status = 'running'
+                """,
+                (status, exit_code, finished, finished, task_id),
+            )
+
+    def mark_killed(self, task_id: str) -> None:
+        now = time.time()
+        with self._connect() as conn:
+            conn.execute(
+                """
+                update bash_bg_tasks
+                set status = 'cancelled', finished_at = ?, updated_at = ?
+                where task_id = ? and status = 'running'
+                """,
+                (now, now, task_id),
+            )
+
+    def mark_auto_resumed(self, task_id: str) -> bool:
+        """Set auto_resumed=1 atomically; return True if we won the race."""
+        with self._connect() as conn:
+            cursor = conn.execute(
+                """
+                update bash_bg_tasks
+                set auto_resumed = 1, updated_at = ?
+                where task_id = ? and auto_resumed = 0
+                """,
+                (time.time(), task_id),
+            )
+            return cursor.rowcount > 0
+
+    def list_active(self) -> list[dict[str, Any]]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                "select * from bash_bg_tasks where status = 'running'"
+            ).fetchall()
+            return [dict(row) for row in rows]
+
+    def list_for_session(
+        self,
+        account_key: str,
+        session_id: str,
+    ) -> list[dict[str, Any]]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                """
+                select * from bash_bg_tasks
+                where account_key = ? and session_id = ?
+                order by started_at desc
+                """,
+                (account_key, session_id),
+            ).fetchall()
+            return [dict(row) for row in rows]
+
+    def get(self, task_id: str) -> dict[str, Any] | None:
+        with self._connect() as conn:
+            row = conn.execute(
+                "select * from bash_bg_tasks where task_id = ?",
+                (task_id,),
+            ).fetchone()
+            return dict(row) if row is not None else None
+
+    def _connect(self) -> sqlite3.Connection:
+        with self._lock:
+            self.db_path.parent.mkdir(parents=True, exist_ok=True)
+            conn = sqlite3.connect(self.db_path, timeout=10)
+            conn.row_factory = sqlite3.Row
+            self._init_schema(conn)
+            return conn
+
+    def _init_schema(self, conn: sqlite3.Connection) -> None:
+        conn.execute('pragma journal_mode=wal')
+        conn.execute(
+            """
+            create table if not exists bash_bg_tasks (
+                task_id text primary key,
+                account_key text not null,
+                session_id text not null,
+                run_id text not null default '',
+                pid integer not null,
+                task_dir text not null,
+                output_path text not null,
+                pid_path text not null,
+                exit_code_path text not null,
+                command text not null,
+                status text not null,
+                started_at real not null,
+                finished_at real,
+                exit_code integer,
+                updated_at real not null,
+                wait_for_completion integer not null default 1,
+                auto_resumed integer not null default 0
+            )
+            """
+        )
+        conn.execute(
+            """
+            create index if not exists idx_bash_bg_session
+            on bash_bg_tasks(account_key, session_id, started_at)
+            """
+        )
+        conn.execute(
+            """
+            create index if not exists idx_bash_bg_active
+            on bash_bg_tasks(status)
+            """
+        )
diff --git a/src/bash_security.py b/src/bash_security.py
index 7517324..a2b71a2 100644
--- a/src/bash_security.py
+++ b/src/bash_security.py
@@ -1251,11 +1251,4 @@ def check_shell_security(
         if result.is_misparsing:
             return (False, f'Security check: {result.message}')
 
-    # Check destructive patterns if not allowed
-    if not allow_destructive:
-        warning = get_destructive_command_warning(command)
-        if warning:
-            return (False, f'Potentially destructive command blocked: {warning}. '
-                          'Re-run with --unsafe to allow it.')
-
     return (True, '')
diff --git a/src/jupyter_runtime.py b/src/jupyter_runtime.py
index 9eda94c..31c0d67 100644
--- a/src/jupyter_runtime.py
+++ b/src/jupyter_runtime.py
@@ -233,12 +233,41 @@ class JupyterRuntimeSession:
             'persisted_at': time.time(),
         }
 
+    @property
+    def chat_workspace_root(self) -> str:
+        """Per-user-per-chat root on shared NFS so the SFT training pod
+        (which doesn't mount the jupyter pod's private workspace) can read
+        and write the same artifacts. Layout:
+        /mnt/wangsenhao/autoresearch-zk-users///"""
+        account_part = sanitize_remote_path_part(self.binding.account_id)
+        session_part = sanitize_remote_path_part(self.binding.session_id)
+        return (
+            f'/mnt/wangsenhao/autoresearch-zk-users/'
+            f'{account_part}/{session_part}'
+        )
+
     def bootstrap_workspace(
         self,
         *,
         timeout_seconds: float = 20.0,
         project_root: Path | None = None,
     ) -> None:
+        chat_root = self.chat_workspace_root
+        nfs_users_root = '/mnt/wangsenhao/autoresearch-zk-users'
+        probe = self.run_command(
+            (
+                f'mkdir -p {shlex.quote(nfs_users_root)} && '
+                f'touch {shlex.quote(nfs_users_root)}/.write_probe && '
+                f'rm -f {shlex.quote(nfs_users_root)}/.write_probe'
+            ),
+            timeout_seconds=timeout_seconds,
+            max_output_chars=2000,
+        )
+        if probe.exit_code != 0:
+            raise JupyterRuntimeError(
+                '远端 jupyter pod 未挂载 /mnt/wangsenhao 或不可写：'
+                + (probe.stdout.strip() or probe.stderr.strip() or 'unknown error')
+            )
         result = self.run_command(
             (
                 'mkdir -p '
@@ -247,7 +276,10 @@ class JupyterRuntimeSession:
                 f'{shlex.quote(self.binding.workspace_cwd)}/scratchpad '
                 f'{shlex.quote(self.binding.workspace_root)}/.runtime/uploads '
                 f'{shlex.quote(self.binding.workspace_root)}/runtime_uploads '
-                f'{shlex.quote(self.account_runtime_root)}/python'
+                f'{shlex.quote(self.account_runtime_root)}/python '
+                f'{shlex.quote(chat_root)}/results '
+                f'{shlex.quote(chat_root)}/sft_output '
+                f'{shlex.quote(chat_root)}/scripts'
             ),
             timeout_seconds=timeout_seconds,
             max_output_chars=4000,
@@ -259,6 +291,27 @@ class JupyterRuntimeSession:
         self.ensure_python_environment(timeout_seconds=max(timeout_seconds, 300.0))
         if project_root is not None:
             self.sync_runtime_bundle(project_root, timeout_seconds=max(timeout_seconds, 180.0))
+            self._sync_chat_workspace_scripts(timeout_seconds=timeout_seconds)
+
+    def _sync_chat_workspace_scripts(self, *, timeout_seconds: float = 20.0) -> None:
+        if not self.skills_root:
+            return
+        chat_root = self.chat_workspace_root
+        src = f'{self.skills_root}/model-iteration/scripts/prepare_and_train_sft.py'
+        dst = f'{chat_root}/scripts/prepare_and_train_sft.py'
+        result = self.run_command(
+            (
+                f'if [ -f {shlex.quote(src)} ]; then '
+                f'cp {shlex.quote(src)} {shlex.quote(dst)}; '
+                f'fi'
+            ),
+            timeout_seconds=timeout_seconds,
+            max_output_chars=2000,
+        )
+        if result.exit_code != 0:
+            raise JupyterRuntimeError(
+                result.stdout.strip() or 'Unable to sync chat workspace SFT script.'
+            )
 
     def to_dict(self) -> dict[str, Any]:
         payload = self.binding.to_dict()
@@ -835,6 +888,7 @@ print(f"{'appended' if MODE == 'a' else 'wrote'} {target} ({len(content)} chars)
             'ZK_AGENT_OUTPUT': f'{self.binding.workspace_cwd}/output',
             'ZK_AGENT_SCRATCHPAD': f'{self.binding.workspace_cwd}/scratchpad',
             'ZK_AGENT_PYTHON_ENV': self.python_env_path,
+            'AUTORESEARCH_CHAT_ROOT': self.chat_workspace_root,
         }
         if self.platform_root:
             exports['ZK_AGENT_PLATFORM_ROOT'] = self.platform_root
diff --git a/src/openai_compat.py b/src/openai_compat.py
index 030fc23..b59dfa7 100644
--- a/src/openai_compat.py
+++ b/src/openai_compat.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import sys
 from typing import Any, Iterator
 from urllib import error, request
 
@@ -31,7 +32,7 @@ ANTHROPIC_MESSAGES_MODEL_PREFIXES = (
 )
 
 ANTHROPIC_VERSION = '2023-06-01'
-ANTHROPIC_MAX_TOKENS = 4096
+ANTHROPIC_MAX_TOKENS = 16384
 DEFAULT_MODEL_IDLE_TIMEOUT_SECONDS = 60.0
 
 
@@ -218,6 +219,57 @@ def _uses_anthropic_messages_api(model: str, base_url: str) -> bool:
     )
 
 
+_UNKNOWN_EVENT_LOG_LIMIT = 8
+_unknown_event_seen: dict[str, int] = {}
+
+
+def _log_unknown_anthropic_event(kind: str, payload: dict[str, Any]) -> None:
+    """Emit a one-line stderr warning the first few times we see an unknown event.
+
+    The streaming handler used to silently drop any block/delta type it didn't
+    recognise. When the upstream proxy started emitting frames we didn't decode,
+    the model's output budget was burned with no observable effect. This log
+    surfaces the situation without flooding logs on every turn.
+    """
+    seen = _unknown_event_seen.get(kind, 0)
+    if seen >= _UNKNOWN_EVENT_LOG_LIMIT:
+        return
+    _unknown_event_seen[kind] = seen + 1
+    try:
+        snippet = json.dumps(payload, ensure_ascii=False)[:400]
+    except Exception:  # pragma: no cover - defensive
+        snippet = repr(payload)[:400]
+    print(
+        f'[openai_compat] unknown anthropic stream event {kind!r}: {snippet}',
+        file=sys.stderr,
+        flush=True,
+    )
+
+
+def _mark_last_message_cache_breakpoint(messages: list[dict[str, Any]]) -> None:
+    """Attach cache_control to the last content block of the last message.
+
+    Anthropic prompt cache requires the breakpoint at the *end* of the cacheable
+    prefix. The next turn appends a new message → the previous last becomes the
+    second-to-last with its breakpoint still in the prefix, so everything up to
+    that point is reused on the next call.
+    """
+    if not messages:
+        return
+    last = messages[-1]
+    content = last.get('content')
+    if not isinstance(content, list) or not content:
+        return
+    last_block = content[-1]
+    if not isinstance(last_block, dict):
+        return
+    new_block = dict(last_block)
+    new_block['cache_control'] = {'type': 'ephemeral'}
+    new_content = list(content)
+    new_content[-1] = new_block
+    last['content'] = new_content
+
+
 def _anthropic_base_url(base_url: str) -> str:
     base = base_url.rstrip('/')
     if base.lower().endswith('/anthropic'):
@@ -486,25 +538,46 @@ class OpenAICompatClient:
                 if blocks:
                     self._append_anthropic_message(anthropic_messages, 'user', blocks)
 
+        system_text = '\n\n'.join(system_parts) if system_parts else ''
+        if output_schema is not None:
+            schema_hint = (
+                f'请严格输出符合 JSON Schema `{output_schema.name}` 的 JSON，'
+                '不要输出额外解释。'
+            )
+            system_text = f'{system_text}\n\n{schema_hint}'.strip() if system_text else schema_hint
+
+        cache_enabled = (
+            os.environ.get('CLAW_DISABLE_PROMPT_CACHE', '').strip().lower()
+            not in {'1', 'true', 'yes', 'on'}
+        )
+
         payload: dict[str, Any] = {
             'model': self.config.model,
             'messages': anthropic_messages,
             'max_tokens': ANTHROPIC_MAX_TOKENS,
             'stream': stream,
         }
-        if system_parts:
-            payload['system'] = '\n\n'.join(system_parts)
+        if system_text:
+            if cache_enabled:
+                payload['system'] = [
+                    {
+                        'type': 'text',
+                        'text': system_text,
+                        'cache_control': {'type': 'ephemeral'},
+                    }
+                ]
+            else:
+                payload['system'] = system_text
         converted_tools = self._anthropic_tools(tools)
         if converted_tools:
+            if cache_enabled:
+                converted_tools = list(converted_tools)
+                last_tool = dict(converted_tools[-1])
+                last_tool['cache_control'] = {'type': 'ephemeral'}
+                converted_tools[-1] = last_tool
             payload['tools'] = converted_tools
-        if output_schema is not None:
-            schema_hint = (
-                f'请严格输出符合 JSON Schema `{output_schema.name}` 的 JSON，'
-                '不要输出额外解释。'
-            )
-            payload['system'] = (
-                f'{payload.get("system", "")}\n\n{schema_hint}'.strip()
-            )
+        if cache_enabled and anthropic_messages:
+            _mark_last_message_cache_breakpoint(anthropic_messages)
         return payload
 
     def _anthropic_headers(self) -> dict[str, str]:
@@ -604,7 +677,12 @@ class OpenAICompatClient:
                         content_block = event_payload.get('content_block')
                         if not isinstance(block_index, int) or not isinstance(content_block, dict):
                             continue
-                        if content_block.get('type') != 'tool_use':
+                        block_type = content_block.get('type')
+                        if block_type != 'tool_use':
+                            if block_type not in ('text', 'thinking', 'redacted_thinking'):
+                                _log_unknown_anthropic_event(
+                                    'content_block_start', event_payload
+                                )
                             continue
                         tool_index = next_tool_index
                         next_tool_index += 1
@@ -636,7 +714,8 @@ class OpenAICompatClient:
                         delta = event_payload.get('delta')
                         if not isinstance(delta, dict):
                             continue
-                        if delta.get('type') == 'text_delta':
+                        delta_type = delta.get('type')
+                        if delta_type == 'text_delta':
                             text = delta.get('text')
                             if isinstance(text, str) and text:
                                 yield StreamEvent(
@@ -645,7 +724,7 @@ class OpenAICompatClient:
                                     raw_event=event_payload,
                                 )
                             continue
-                        if delta.get('type') == 'input_json_delta':
+                        if delta_type == 'input_json_delta':
                             block_index = event_payload.get('index')
                             partial_json = delta.get('partial_json')
                             if (
@@ -661,6 +740,17 @@ class OpenAICompatClient:
                                     raw_event=event_payload,
                                 )
                             continue
+                        if delta_type in ('thinking_delta', 'signature_delta'):
+                            # Extended-thinking blocks: we don't surface them as
+                            # content (the runtime has nowhere to put them), but
+                            # they previously vanished silently and burned the
+                            # whole 4k output budget before tool_use could start.
+                            # Logging once is enough to confirm they're flowing.
+                            continue
+                        _log_unknown_anthropic_event(
+                            f'content_block_delta:{delta_type}', event_payload
+                        )
+                        continue
                     if event_type == 'message_delta':
                         delta = event_payload.get('delta')
                         if isinstance(delta, dict) and isinstance(delta.get('stop_reason'), str):
diff --git a/src/session_store.py b/src/session_store.py
index 0ad9cf2..52f557d 100644
--- a/src/session_store.py
+++ b/src/session_store.py
@@ -223,7 +223,7 @@ def deserialize_runtime_config(payload: JSONDict) -> AgentRuntimeConfig:
     return AgentRuntimeConfig(
         cwd=Path(str(payload['cwd'])).resolve(),
         max_turns=int(payload.get('max_turns', DEFAULT_MAX_TURNS)),
-        command_timeout_seconds=float(payload.get('command_timeout_seconds', 30.0)),
+        command_timeout_seconds=float(payload.get('command_timeout_seconds', 300.0)),
         max_output_chars=int(payload.get('max_output_chars', 50000)),
         stream_model_responses=bool(payload.get('stream_model_responses', False)),
         auto_snip_threshold_tokens=_optional_int(payload.get('auto_snip_threshold_tokens')),
diff --git a/tests/test_bash_bg.py b/tests/test_bash_bg.py
new file mode 100644
index 0000000..86ea6b2
--- /dev/null
+++ b/tests/test_bash_bg.py
@@ -0,0 +1,324 @@
+"""Tests for the bash run_in_background path (local fallback only).
+
+Covers:
+- BashBgStore: record_start / mark_completed / mark_auto_resumed race semantics.
+- _run_bash_background local fallback: spawns a detached process with file
+  redirects, registers it via the bg_register callback, output/pid/exit_code
+  files materialize.
+- _run_bash_status / _run_bash_kill: route through bg_status_query / bg_kill_request
+  callbacks and format their result.
+
+Remote (jupyter_runtime) path is intentionally not exercised here — that
+requires a live wsh account and is covered by the e2e checklist in the plan.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+import pytest
+
+from src.agent_tool_core import ToolExecutionContext
+from src.agent_tools import (
+    _run_bash_background,
+    _run_bash_kill,
+    _run_bash_status,
+)
+from src.agent_types import AgentPermissions
+from src.bash_bg_store import BashBgStore, BgTaskSpec, BgTaskStatus
+
+
+def _make_context(
+    tmp_path: Path,
+    *,
+    bg_register=None,
+    bg_status_query=None,
+    bg_kill_request=None,
+) -> ToolExecutionContext:
+    return ToolExecutionContext(
+        root=tmp_path,
+        command_timeout_seconds=10.0,
+        max_output_chars=4096,
+        permissions=AgentPermissions(allow_shell_commands=True),
+        scratchpad_directory=tmp_path,
+        bg_register=bg_register,
+        bg_status_query=bg_status_query,
+        bg_kill_request=bg_kill_request,
+        account_id='acct-test',
+        session_id='sess-test',
+        run_id='run-test',
+    )
+
+
+# ----- BashBgStore --------------------------------------------------------
+
+
+def test_store_record_and_complete(tmp_path: Path) -> None:
+    store = BashBgStore(tmp_path / 'bg.db')
+    spec = BgTaskSpec(
+        task_id='bg_abc',
+        account_key='acct',
+        session_id='sess',
+        run_id='run',
+        pid=12345,
+        task_dir=str(tmp_path / 'bg_abc'),
+        output_path=str(tmp_path / 'bg_abc' / 'output'),
+        pid_path=str(tmp_path / 'bg_abc' / 'pid'),
+        exit_code_path=str(tmp_path / 'bg_abc' / 'exit_code'),
+        command='echo hi',
+        started_at=time.time(),
+        wait_for_completion=True,
+    )
+    store.record_start(spec)
+    row = store.get('bg_abc')
+    assert row is not None
+    assert row['status'] == 'running'
+    assert row['exit_code'] is None
+
+    store.mark_completed('bg_abc', exit_code=0)
+    row = store.get('bg_abc')
+    assert row['status'] == 'completed'
+    assert row['exit_code'] == 0
+    assert row['finished_at'] is not None
+
+
+def test_store_mark_auto_resumed_is_race_safe(tmp_path: Path) -> None:
+    store = BashBgStore(tmp_path / 'bg.db')
+    spec = BgTaskSpec(
+        task_id='bg_race',
+        account_key='acct',
+        session_id='sess',
+        run_id='run',
+        pid=1,
+        task_dir='/tmp/x',
+        output_path='/tmp/x/output',
+        pid_path='/tmp/x/pid',
+        exit_code_path='/tmp/x/exit_code',
+        command='true',
+        started_at=time.time(),
+        wait_for_completion=True,
+    )
+    store.record_start(spec)
+    store.mark_completed('bg_race', exit_code=0)
+
+    # First caller wins; subsequent callers always see False.
+    assert store.mark_auto_resumed('bg_race') is True
+    assert store.mark_auto_resumed('bg_race') is False
+    assert store.mark_auto_resumed('bg_race') is False
+
+
+def test_store_mark_killed_only_running(tmp_path: Path) -> None:
+    store = BashBgStore(tmp_path / 'bg.db')
+    spec = BgTaskSpec(
+        task_id='bg_kill',
+        account_key='acct',
+        session_id='sess',
+        run_id='run',
+        pid=2,
+        task_dir='/tmp/y',
+        output_path='/tmp/y/output',
+        pid_path='/tmp/y/pid',
+        exit_code_path='/tmp/y/exit_code',
+        command='sleep 100',
+        started_at=time.time(),
+        wait_for_completion=False,
+    )
+    store.record_start(spec)
+    store.mark_killed('bg_kill')
+    row = store.get('bg_kill')
+    assert row['status'] == 'cancelled'
+
+    # Re-killing a non-running task is a no-op (status stays cancelled).
+    store.mark_killed('bg_kill')
+    row = store.get('bg_kill')
+    assert row['status'] == 'cancelled'
+
+
+# ----- _run_bash_background local fallback --------------------------------
+
+
+def test_run_bash_background_local_spawns_detached(tmp_path: Path) -> None:
+    captured: list[BgTaskSpec] = []
+
+    ctx = _make_context(tmp_path, bg_register=captured.append)
+    content, metadata = _run_bash_background(
+        {
+            'command': 'echo hello-from-bg',
+            'wait_for_completion': True,
+        },
+        ctx,
+    )
+
+    assert metadata['action'] == 'bash_bg'
+    task_id = metadata['task_id']
+    assert task_id.startswith('bg_')
+    assert metadata['wait_for_completion'] is True
+    assert 'pid=' in content
+
+    # bg_register received exactly one spec, with the agent's session/account.
+    assert len(captured) == 1
+    spec = captured[0]
+    assert spec.task_id == task_id
+    assert spec.account_key == 'acct-test'
+    assert spec.session_id == 'sess-test'
+    assert spec.run_id == 'run-test'
+    assert spec.command == 'echo hello-from-bg'
+    assert spec.wait_for_completion is True
+
+    # Wait for the detached subprocess to finish — bounded by exit_code file.
+    deadline = time.monotonic() + 5.0
+    exit_code_path = Path(spec.exit_code_path)
+    while time.monotonic() < deadline and not exit_code_path.exists():
+        time.sleep(0.05)
+    assert exit_code_path.exists(), 'exit_code file did not appear in 5s'
+    assert exit_code_path.read_text(encoding='utf-8').strip() == '0'
+
+    output_text = Path(spec.output_path).read_text(encoding='utf-8')
+    assert 'hello-from-bg' in output_text
+
+
+def test_run_bash_background_failed_command_records_nonzero_exit(
+    tmp_path: Path,
+) -> None:
+    captured: list[BgTaskSpec] = []
+    ctx = _make_context(tmp_path, bg_register=captured.append)
+
+    _content, metadata = _run_bash_background(
+        {'command': 'exit 7'},
+        ctx,
+    )
+    spec = captured[0]
+    assert metadata['action'] == 'bash_bg'
+
+    deadline = time.monotonic() + 5.0
+    exit_code_path = Path(spec.exit_code_path)
+    while time.monotonic() < deadline and not exit_code_path.exists():
+        time.sleep(0.05)
+    assert exit_code_path.exists()
+    assert exit_code_path.read_text(encoding='utf-8').strip() == '7'
+
+
+def test_run_bash_background_requires_shell_permission(tmp_path: Path) -> None:
+    from src.agent_tool_core import ToolPermissionError
+
+    ctx = ToolExecutionContext(
+        root=tmp_path,
+        command_timeout_seconds=10.0,
+        max_output_chars=4096,
+        permissions=AgentPermissions(allow_shell_commands=False),
+        scratchpad_directory=tmp_path,
+    )
+    with pytest.raises(ToolPermissionError):
+        _run_bash_background({'command': 'echo nope'}, ctx)
+
+
+# ----- bash_status / bash_kill via callbacks ------------------------------
+
+
+def test_bash_status_routes_through_callback(tmp_path: Path) -> None:
+    fake_status = BgTaskStatus(
+        task_id='bg_xyz',
+        status='completed',
+        pid=999,
+        started_at=time.time() - 5.0,
+        finished_at=time.time(),
+        exit_code=0,
+        output_path='/tmp/x/output',
+        output_preview='all good\n',
+        auto_resumed=False,
+    )
+
+    queries: list[str] = []
+
+    def query(task_id: str) -> BgTaskStatus | None:
+        queries.append(task_id)
+        return fake_status
+
+    ctx = _make_context(tmp_path, bg_status_query=query)
+    content, metadata = _run_bash_status({'task_id': 'bg_xyz'}, ctx)
+
+    assert queries == ['bg_xyz']
+    assert metadata['action'] == 'bash_status'
+    assert metadata['status'] == 'completed'
+    assert metadata['exit_code'] == 0
+    assert 'all good' in content
+    assert 'task_id=bg_xyz' in content
+
+
+def test_bash_status_unknown_task_raises(tmp_path: Path) -> None:
+    from src.agent_tool_core import ToolExecutionError
+
+    def query(_task_id: str):
+        return None
+
+    ctx = _make_context(tmp_path, bg_status_query=query)
+    with pytest.raises(ToolExecutionError):
+        _run_bash_status({'task_id': 'bg_missing'}, ctx)
+
+
+def test_bash_status_without_callback_raises(tmp_path: Path) -> None:
+    from src.agent_tool_core import ToolExecutionError
+
+    ctx = _make_context(tmp_path)
+    with pytest.raises(ToolExecutionError):
+        _run_bash_status({'task_id': 'bg_anything'}, ctx)
+
+
+def test_bash_kill_routes_through_callback(tmp_path: Path) -> None:
+    killed: list[str] = []
+
+    def kill(task_id: str) -> bool:
+        killed.append(task_id)
+        return True
+
+    ctx = _make_context(tmp_path, bg_kill_request=kill)
+    content, metadata = _run_bash_kill({'task_id': 'bg_kk'}, ctx)
+    assert killed == ['bg_kk']
+    assert metadata['killed'] is True
+    assert 'bg_kk' in content
+
+
+def test_bash_kill_failure_reports(tmp_path: Path) -> None:
+    ctx = _make_context(tmp_path, bg_kill_request=lambda _t: False)
+    content, metadata = _run_bash_kill({'task_id': 'bg_dead'}, ctx)
+    assert metadata['killed'] is False
+    assert '未能终止' in content
+
+
+# ----- end-to-end: spawn + register + sync via real BashBgStore -----------
+
+
+def test_local_bg_lifecycle_with_real_store(tmp_path: Path) -> None:
+    """Plug a real BashBgStore into bg_register and check the row appears."""
+    store = BashBgStore(tmp_path / 'bg.db')
+
+    def register(spec: BgTaskSpec) -> None:
+        store.record_start(spec)
+
+    ctx = _make_context(tmp_path, bg_register=register)
+    _content, metadata = _run_bash_background(
+        {'command': 'echo lifecycle && exit 0'}, ctx,
+    )
+    task_id = metadata['task_id']
+    row = store.get(task_id)
+    assert row is not None
+    assert row['status'] == 'running'
+
+    # Wait for child to write exit_code (more reliable than kill -0, since
+    # the unreaped child becomes a zombie and kill -0 still reports it alive).
+    deadline = time.monotonic() + 5.0
+    exit_code_path = Path(row['exit_code_path'])
+    while time.monotonic() < deadline and not exit_code_path.exists():
+        time.sleep(0.05)
+    if not exit_code_path.exists():
+        pytest.fail('background process did not exit within 5 seconds')
+
+    # Simulate manager finalize step.
+    exit_code = int(
+        exit_code_path.read_text(encoding='utf-8').strip()
+    )
+    store.mark_completed(task_id, exit_code=exit_code)
+    row = store.get(task_id)
+    assert row['status'] == 'completed'
+    assert row['exit_code'] == 0