diff --git a/.gitignore b/.gitignore index 7ed6b58..399604e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ archive/ scratchpad/ tasks/ router_session_parquet/ +.logs/ # Environment files .env diff --git a/.logs/server-20260521-094247.log b/.logs/server-20260521-094247.log new file mode 100644 index 0000000..45d73f4 --- /dev/null +++ b/.logs/server-20260521-094247.log @@ -0,0 +1,14 @@ +INFO: Started server process [2637646] +INFO: Waiting for application startup. +INFO: Application startup complete. +ERROR: [Errno 98] error while attempting to bind on address ('127.0.0.1', 8975): address already in use +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +Claw Code GUI listening on http://127.0.0.1:8975 + cwd : /home/mi/zk-data-agent-wsh + model : xiaomi/mimo-v2-flash + base-url : http://model.mify.ai.srv/v1 + timeout : 3600s + sessions : /home/mi/zk-data-agent-wsh/.port_sessions/agent + shell : on + write : on diff --git a/backend/api/server.py b/backend/api/server.py index 6fdd383..d83520b 100644 --- a/backend/api/server.py +++ b/backend/api/server.py @@ -7,8 +7,10 @@ chat, slash commands, and saved sessions, and serves the static SPA. from __future__ import annotations import asyncio +from contextlib import asynccontextmanager import csv import hashlib +import shlex import json import os import pwd @@ -23,12 +25,12 @@ import venv from dataclasses import dataclass, field, replace from pathlib import Path from threading import Lock, RLock -from typing import Any +from typing import Any, Callable from urllib import error, request from urllib.parse import quote, unquote, urlparse from uuid import uuid4 -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field @@ -56,6 +58,13 @@ from src.personal_memory import ( USER_MEMORY_FILENAME, PersonalMemoryManager, ) +from src.bash_bg_store import ( + ACTIVE_BG_STATUSES, + TERMINAL_BG_STATUSES, + BashBgStore, + BgTaskSpec, + BgTaskStatus, +) from src.run_state_store import ACTIVE_RUN_STATUSES, RunStateStore from src.session_store import ( DEFAULT_AGENT_SESSION_DIR, @@ -88,6 +97,7 @@ FEISHU_SPREADSHEET_MAX_CELL_CHARS = 5000 FEISHU_SPREADSHEET_WRITE_BATCH_ROWS = 500 FEISHU_ONLINE_DOCS_FILENAME = 'online-docs.json' JUPYTER_WORKSPACE_FILENAME = 'jupyter_workspace.json' +JUPYTER_WORKSPACES_REGISTRY_FILENAME = 'workspaces.json' JUPYTER_STREAM_CHUNK_BYTES = 1024 * 1024 FEISHU_REMOTE_DOC_MAX_BYTES = 50 * 1024 * 1024 FEISHU_REMOTE_SHEET_MAX_BYTES = 100 * 1024 * 1024 @@ -545,6 +555,8 @@ class AgentState: self._account_configs: dict[str, AgentInstanceConfig] = {} self.run_manager = RunManager() self.run_state_store = RunStateStore(self.session_directory.parent / 'run_state.db') + self.bash_bg_store = BashBgStore(self.session_directory.parent / 'bash_bg.db') + self.event_loop: 'asyncio.AbstractEventLoop | None' = None self.jupyter_runtime_manager = JupyterRuntimeManager() self._default_config = AgentInstanceConfig( cwd=cwd.resolve(), @@ -853,6 +865,7 @@ class AgentState: python_env_dir=paths['python_env'], runtime_user=runtime_user, enabled_skill_names=self.enabled_skill_names(account_id), + auto_compact_threshold_tokens=180_000, ) model_config = ModelConfig( model=config.model, @@ -1027,6 +1040,11 @@ class JupyterWorkspaceBindRequest(BaseModel): account_id: str | None = None +class SavedJupyterWorkspaceBindRequest(BaseModel): + session_id: str = Field(min_length=1) + account_id: str | None = None + + class SkillPreferenceUpdate(BaseModel): skill: str | None = None enabled: bool @@ -1038,8 +1056,9 @@ class SkillSyncRequest(BaseModel): account_id: str | None = None -class SessionTitleUpdate(BaseModel): - title: str = Field(min_length=1, max_length=80) +class SessionUpdate(BaseModel): + title: str | None = Field(default=None, max_length=80) + is_training: bool | None = None class FeishuAccountRequest(BaseModel): @@ -1077,7 +1096,21 @@ def _append_runtime_context(current: str | None, addition: str) -> str: # --------------------------------------------------------------------------- def create_app(state: AgentState) -> FastAPI: - app = FastAPI(title='Claw Code GUI', version='1.0') + @asynccontextmanager + async def lifespan(app: FastAPI): # noqa: ARG001 + state.event_loop = asyncio.get_running_loop() + _bash_bg_manager.set_chat_runner(_run_chat_payload) + await _bash_bg_manager.recover_from_store(state) + scanner_task = asyncio.create_task(_watcher_scanner_loop(state)) + try: + yield + finally: + scanner_task.cancel() + _watcher_manager.cancel_all() + _bash_bg_manager.cancel_all() + state.event_loop = None + + app = FastAPI(title='Claw Code GUI', version='1.0', lifespan=lifespan) # ------------- static + index ------------------------------------------ app.mount( @@ -1217,8 +1250,83 @@ def create_app(state: AgentState) -> FastAPI: state.account_paths(payload.account_id)['sessions'], runtime, ) + _register_jupyter_workspace( + state, + payload.account_id, + base_url=payload.base_url, + workspace_root=payload.workspace_root, + password=payload.password, + ) return runtime.to_dict() + @app.get('/api/jupyter/workspaces') + async def list_jupyter_workspaces( + account_id: str | None = None, + ) -> list[dict[str, Any]]: + entries = _load_jupyter_workspaces_registry(state, account_id) + return [_public_jupyter_workspace_entry(e) for e in entries] + + @app.post('/api/jupyter/workspaces/{workspace_id}/bind') + async def bind_saved_jupyter_workspace( + workspace_id: str, + payload: SavedJupyterWorkspaceBindRequest, + ) -> dict[str, Any]: + safe_session_id = _safe_session_id(payload.session_id) + if not safe_session_id: + raise HTTPException(status_code=400, detail='session_id is required') + entries = _load_jupyter_workspaces_registry(state, payload.account_id) + match = next((e for e in entries if e.get('id') == workspace_id), None) + if match is None: + raise HTTPException(status_code=404, detail='workspace not found') + password = match.get('password') + base_url = match.get('base_url') + workspace_root = match.get('workspace_root') or DEFAULT_JUPYTER_WORKSPACE_ROOT + if not (isinstance(password, str) and isinstance(base_url, str)): + raise HTTPException(status_code=400, detail='workspace entry is incomplete') + account_key = state._account_key(payload.account_id) + try: + runtime = await asyncio.to_thread( + state.jupyter_runtime_manager.bind_session, + account_id=account_key, + session_id=safe_session_id, + base_url=base_url, + password=password, + workspace_root=workspace_root, + project_root=state.config_for(payload.account_id).cwd, + ) + except JupyterRuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + status_code=502, + detail=f'Unable to connect Jupyter runtime: {exc}', + ) from exc + _save_jupyter_runtime_binding( + state.account_paths(payload.account_id)['sessions'], + runtime, + ) + _register_jupyter_workspace( + state, + payload.account_id, + base_url=base_url, + workspace_root=workspace_root, + password=password, + label=match.get('label') if isinstance(match.get('label'), str) else None, + ) + return runtime.to_dict() + + @app.delete('/api/jupyter/workspaces/{workspace_id}') + async def delete_saved_jupyter_workspace( + workspace_id: str, + account_id: str | None = None, + ) -> dict[str, Any]: + entries = _load_jupyter_workspaces_registry(state, account_id) + kept = [e for e in entries if e.get('id') != workspace_id] + if len(kept) == len(entries): + raise HTTPException(status_code=404, detail='workspace not found') + _save_jupyter_workspaces_registry(state, account_id, kept) + return {'ok': True} + @app.get('/api/slash-commands') async def list_slash_commands() -> list[dict[str, Any]]: commands: list[dict[str, Any]] = [] @@ -1568,6 +1676,7 @@ def create_app(state: AgentState) -> FastAPI: 'modified_at': _session_mtime(path), 'model': model_config.get('model'), 'usage': usage, + 'is_training': bool(data.get('is_training', False)), } ) return results @@ -1595,17 +1704,446 @@ def create_app(state: AgentState) -> FastAPI: @app.patch('/api/sessions/{session_id}') async def update_session( session_id: str, - payload: SessionTitleUpdate, + payload: SessionUpdate, account_id: str | None = None, ) -> dict[str, Any]: directory = state.account_paths(account_id)['sessions'] safe_id = _safe_session_id(session_id) - title = _clean_manual_session_title(payload.title) - if safe_id is None or title is None: - raise HTTPException(status_code=400, detail='Invalid session title') - if not _update_session_title(directory, safe_id, title): - raise HTTPException(status_code=404, detail='Session not found') - return {'session_id': safe_id, 'title': title} + if safe_id is None: + raise HTTPException(status_code=400, detail='Invalid session id') + if payload.title is None and payload.is_training is None: + raise HTTPException(status_code=400, detail='Nothing to update') + + result: dict[str, Any] = {'session_id': safe_id} + + if payload.title is not None: + title = _clean_manual_session_title(payload.title) + if title is None: + raise HTTPException(status_code=400, detail='Invalid session title') + if not _update_session_title(directory, safe_id, title): + raise HTTPException(status_code=404, detail='Session not found') + result['title'] = title + + if payload.is_training is not None: + value = bool(payload.is_training) + if not _update_session_training(directory, safe_id, value): + raise HTTPException(status_code=404, detail='Session not found') + result['is_training'] = value + + return result + + @app.get('/api/training/step-detail') + async def get_training_step_detail( + session_id: str, + step: str, + run_id: str | None = None, + account_id: str | None = None, + ) -> Response: + sessions_dir = state.account_paths(account_id)['sessions'] + state_path = _session_state_path(sessions_dir, session_id) + state_entries, _ = _read_program_state(state_path) + # Per-round: if frontend passed run_id, use that round's actual runDic + # so historical step views (e.g., R1's augment) read the right + # workflow directory. Fall back to "max in state" only when run_id + # is absent or the round has no resolvable runDic — preserves prior + # behaviour for unbundled callers. + run_dic: int | None = None + if isinstance(run_id, str) and run_id.strip(): + rid = run_id.strip() + per_round = _resolve_run_dic_per_round(state_entries) + # augment 处理的是「上一轮评测的错例」，产物按 R{n-1} 的 runDic 命名 + # （data_clean_/、augment_.jsonl ...），不是本轮 eval workflow + if step == 'augment': + m = re.match(r'^[Rr](\d+)$', rid) + if m and int(m.group(1)) >= 1: + run_dic = per_round.get(f'R{int(m.group(1)) - 1}') + if run_dic is None: + run_dic = per_round.get(rid) + if run_dic is None: + run_dic = _resolve_run_dic_from_state(state_entries) + runtime = _jupyter_runtime_for_session(state, account_id, session_id) + artifact_paths = _step_artifact_paths(step, run_dic, runtime) + + if not artifact_paths: + payload = { + 'step': step, + 'run_dic': run_dic, + 'format': 'empty', + 'content': '', + 'source_path': None, + 'fetched_at_ms': int(time.time() * 1000), + 'error': '该 step 没有定义产物路径', + } + return Response( + content=json.dumps(payload, ensure_ascii=False), + media_type='application/json', + headers={'Cache-Control': 'no-store'}, + ) + + # 多分区表格视图：augment（H1 修改 / H2 仿写）与 dist-analysis（候选 + 报告）共用 + # path_indices 指向 _step_artifact_paths 返回的路径列表；多个候选按顺序尝试， + # 用第一个能读到的（让 dist-analysis 的 NFS 主路径 + jupyter pod 本地兜底 + # 路径合并成同一个 section，避免出现重复卡片）。 + multi_section_specs: dict[str, list[dict[str, Any]]] = { + 'augment': [ + {'title': 'H1 确认修改（complex 翻转 → 原文件 in-place 改标）', 'max_rows': None, 'path_indices': [0]}, + {'title': 'H2 仿写候选 raw（GPT 原始输出，未过 sanity）', 'max_rows': None, 'path_indices': [1]}, + {'title': 'H2 仿写最终（本轮新增训练样本，合入 augment.jsonl）', 'max_rows': None, 'path_indices': [2]}, + {'title': 'label-master 标签复核（H1+H2 全量 verdict）', 'max_rows': None, 'path_indices': [3]}, + ], + 'dist-analysis': [ + {'title': '问题分析报告（workflow.md）', 'max_rows': None, 'path_indices': [0]}, + {'title': 'H1 预计修改候选', 'max_rows': None, 'path_indices': [1, 2]}, + ], + } + if step == 'dist-analysis': + # 从 iteration_log.jsonl 读 H1 假设短标签，注入到候选集 section 标题 + h1_label = await asyncio.to_thread( + _read_iteration_log_h1_label, runtime, run_dic + ) + if h1_label: + for spec in multi_section_specs['dist-analysis']: + if spec['title'] == 'H1 预计修改候选': + spec['title'] = f'H1 预计修改候选（{h1_label}）' + break + if step in multi_section_specs: + section_specs = multi_section_specs[step] + table_sections: list[dict[str, Any]] = [] + tried: list[str] = [] + errors: list[str] = [] + for spec in section_specs: + title = spec['title'] + max_rows = spec.get('max_rows') + candidate_paths = [ + artifact_paths[i] + for i in spec.get('path_indices', []) + if 0 <= i < len(artifact_paths) + ] + if not candidate_paths: + continue + # 多候选按顺序尝试，用第一个能读到的；都读不到则用首个候选作为缺失占位 + content: str | None = None + path_str: str = candidate_paths[0] + last_err: str | None = None + for cand in candidate_paths: + tried.append(cand) + content_try, err = await asyncio.to_thread( + _read_artifact, runtime, cand + ) + if content_try is not None: + content = content_try + path_str = cand + break + last_err = err + if content is None: + if last_err: + errors.append(f'{title}: {last_err}') + table_sections.append( + { + 'title': title, + 'source_path': candidate_paths[0], + 'kind': 'missing', + 'columns': [], + 'rows': [], + 'total_rows': 0, + 'shown_rows': 0, + 'error': last_err or '产物文件不存在', + } + ) + continue + fmt = _detect_artifact_format(path_str) + try: + if fmt == 'markdown': + # markdown 直接以原文渲染，不走表格 + table_sections.append( + { + 'title': title, + 'source_path': path_str, + 'kind': 'markdown', + 'columns': [], + 'rows': [], + 'total_rows': 0, + 'shown_rows': 0, + 'body': content, + } + ) + continue + if fmt == 'csv': + cols, table_rows, total = _csv_to_table(content, max_rows) + kind = 'csv' + elif fmt in {'jsonl', 'json'}: + cols, table_rows, total = _jsonl_to_table( + content, max_rows + ) + kind = 'jsonl' + else: + lines = content.splitlines() + total = len(lines) + capped = lines if max_rows is None else lines[:max_rows] + cols = ['line'] + table_rows = [[_truncate_cell(line)] for line in capped] + kind = 'text' + except Exception as exc: # noqa: BLE001 + errors.append(f'{title}: parse failed: {exc}') + table_sections.append( + { + 'title': title, + 'source_path': path_str, + 'kind': 'error', + 'columns': [], + 'rows': [], + 'total_rows': 0, + 'shown_rows': 0, + 'error': f'解析失败：{exc}', + } + ) + continue + table_sections.append( + { + 'title': title, + 'source_path': path_str, + 'kind': kind, + 'columns': cols, + 'rows': table_rows, + 'total_rows': total, + 'shown_rows': len(table_rows), + } + ) + if step == 'dist-analysis': + # 软警告：dist-analysis 标 complete 时若 H1 候选 CSV 缺失，把 section + # 的兜底 "产物文件不存在" 换成更明确的提示。某些轮次（hparam 类假设、 + # 无标签问题）确实可能没有候选，所以不 fail 这一步、不阻塞 human-check。 + target_run = (run_id or '').strip() + last_status: str | None = None + for entry in state_entries: + if not isinstance(entry, dict) or entry.get('step') != 'dist-analysis': + continue + eid = (entry.get('run_id') or '').strip() + if target_run and eid and eid != target_run: + continue + if 'status' in entry: + last_status = entry.get('status') + if last_status == 'complete': + rd_label = str(run_dic) if run_dic is not None else '' + for sec in table_sections: + if ( + str(sec.get('title', '')).startswith('H1 预计修改候选') + and sec.get('kind') == 'missing' + ): + sec['error'] = ( + f'未检测到 output/relabel_candidates_{rd_label}.csv —— ' + f'若本轮无 H1 候选可忽略；如有候选请按 SKILL.md 「阶段一」' + f'落盘 file,line,query,old_label,是否改(1/0),建议新label。' + ) + break + payload = { + 'step': step, + 'run_dic': run_dic, + 'format': 'tables', + 'sections': table_sections, + 'source_path': artifact_paths[0], + 'tried_paths': tried, + 'fetched_at_ms': int(time.time() * 1000), + } + if errors: + payload['error'] = '; '.join(errors) + return Response( + content=json.dumps(payload, ensure_ascii=False), + media_type='application/json', + headers={'Cache-Control': 'no-store'}, + ) + + last_error: str | None = None + for path_str in artifact_paths: + content, err = await asyncio.to_thread( + _read_artifact, runtime, path_str + ) + if content is not None: + fmt = _detect_artifact_format(path_str) + if fmt == 'jsonl': + # 只取前 50 行，避免超大 + content = '\n'.join(content.splitlines()[:50]) + payload = { + 'step': step, + 'run_dic': run_dic, + 'format': fmt, + 'content': content, + 'source_path': path_str, + 'fetched_at_ms': int(time.time() * 1000), + } + return Response( + content=json.dumps(payload, ensure_ascii=False), + media_type='application/json', + headers={'Cache-Control': 'no-store'}, + ) + if err: + last_error = err + + payload = { + 'step': step, + 'run_dic': run_dic, + 'format': 'empty', + 'content': '', + 'source_path': artifact_paths[0] if artifact_paths else None, + 'fetched_at_ms': int(time.time() * 1000), + 'error': last_error or '产物文件不存在或读取失败', + 'tried_paths': artifact_paths, + } + return Response( + content=json.dumps(payload, ensure_ascii=False), + media_type='application/json', + headers={'Cache-Control': 'no-store'}, + ) + + @app.get('/api/training/pipeline') + async def get_training_pipeline( + session_id: str | None = None, + account_id: str | None = None, + ) -> Response: + sessions_dir = state.account_paths(account_id)['sessions'] + state_path = _session_state_path(sessions_dir, session_id) + state_entries, _ = _read_program_state(state_path) + iter_count = await asyncio.to_thread( + _read_iteration_log_count, + session_id, + agent_state=state, + account_id=account_id, + ) + metrics_history = await asyncio.to_thread( + _read_iteration_log_metrics_history, + session_id, + agent_state=state, + account_id=account_id, + ) + payload = await asyncio.to_thread( + _hardcoded_autoresearch_pipeline, + session_id, + sessions_dir=sessions_dir, + state_entries=state_entries, + model_config=state.model_config_for(account_id), + agent_state=state, + account_id=account_id, + iteration_log_count=iter_count, + metrics_history=metrics_history, + ) + failure_status = await asyncio.to_thread( + _read_session_failure_status, + sessions_dir, + session_id, + ) + run_active = _is_run_active(state, account_id, session_id) + _apply_program_state( + payload, + state_entries, + iteration_log_count=iter_count or 0, + session_failure_status=failure_status, + run_active=run_active, + session_id=session_id, + ) + return Response( + content=json.dumps(payload, ensure_ascii=False), + media_type='application/json', + headers={'Cache-Control': 'no-store'}, + ) + + @app.get('/api/training/pipeline/stream') + async def stream_training_pipeline( + request: Request, + session_id: str | None = None, + account_id: str | None = None, + ) -> StreamingResponse: + sessions_dir = state.account_paths(account_id)['sessions'] + state_path = _session_state_path(sessions_dir, session_id) + + async def event_gen(): + last_signature: str | None = None + last_heartbeat = time.monotonic() + refresher = asyncio.create_task( + _iter_count_refresh_loop( + session_id, + agent_state=state, + account_id=account_id, + ) + ) + metrics_refresher = asyncio.create_task( + _metrics_history_refresh_loop( + session_id, + agent_state=state, + account_id=account_id, + ) + ) + try: + while True: + if await request.is_disconnected(): + break + state_entries, state_mtime = _read_program_state(state_path) + iter_count = await asyncio.to_thread( + _read_iteration_log_count, + session_id, + agent_state=state, + account_id=account_id, + ) + metrics_history = await asyncio.to_thread( + _read_iteration_log_metrics_history, + session_id, + agent_state=state, + account_id=account_id, + ) + payload = await asyncio.to_thread( + _hardcoded_autoresearch_pipeline, + session_id, + sessions_dir=sessions_dir, + state_entries=state_entries, + agent_state=state, + account_id=account_id, + iteration_log_count=iter_count, + metrics_history=metrics_history, + ) + failure_status = await asyncio.to_thread( + _read_session_failure_status, + sessions_dir, + session_id, + ) + run_active = _is_run_active(state, account_id, session_id) + _apply_program_state( + payload, + state_entries, + iteration_log_count=iter_count or 0, + session_failure_status=failure_status, + run_active=run_active, + session_id=session_id, + ) + payload_str = json.dumps(payload, ensure_ascii=False) + signature = f'{state_mtime}|{hash(payload_str)}' + if signature != last_signature: + last_signature = signature + yield f'data: {payload_str}\n\n' + now = time.monotonic() + if now - last_heartbeat > 15: + yield ': heartbeat\n\n' + last_heartbeat = now + await asyncio.sleep(0.5) + finally: + refresher.cancel() + try: + await refresher + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + metrics_refresher.cancel() + try: + await metrics_refresher + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + + return StreamingResponse( + event_gen(), + media_type='text/event-stream', + headers={ + 'Cache-Control': 'no-store', + 'X-Accel-Buffering': 'no', + 'Connection': 'keep-alive', + }, + ) @app.get('/api/context-budget') async def get_context_budget( @@ -1859,11 +2397,44 @@ def create_app(state: AgentState) -> FastAPI: if stored_event is not None: state.run_state_store.record_event(run_record.run_id, stored_event) _emit_runtime_event(event_sink, started_event) + event_loop = state.event_loop + + def _bg_register(spec: BgTaskSpec) -> None: + if event_loop is None: + raise RuntimeError( + 'event loop unavailable; backend not started via lifespan' + ) + asyncio.run_coroutine_threadsafe( + _bash_bg_manager.register(state, spec), + event_loop, + ) + + def _bg_status_query(task_id: str) -> 'BgTaskStatus | None': + return _bash_bg_manager.status_query(state, task_id) + + def _bg_kill_request(task_id: str) -> bool: + if event_loop is None: + return False + future = asyncio.run_coroutine_threadsafe( + _bash_bg_manager.cancel(state, task_id), + event_loop, + ) + try: + return bool(future.result(timeout=15)) + except Exception: # noqa: BLE001 + return False + agent.tool_context = replace( agent.tool_context, cancel_event=run_record.cancel_event, process_registry=run_record.process_registry, jupyter_runtime=jupyter_runtime, + bg_register=_bg_register, + bg_status_query=_bg_status_query, + bg_kill_request=_bg_kill_request, + account_id=account_key, + session_id=requested_session_id, + run_id=run_record.run_id, ) started_at = time.perf_counter() try: @@ -1933,6 +2504,12 @@ def create_app(state: AgentState) -> FastAPI: cancel_event=None, process_registry=None, jupyter_runtime=None, + bg_register=None, + bg_status_query=None, + bg_kill_request=None, + account_id=None, + session_id=None, + run_id=None, ) elapsed_ms = max(0, int((time.perf_counter() - started_at) * 1000)) payload = _serialize_run_result(result) @@ -2140,6 +2717,7 @@ def _serialize_stored_session(stored: StoredAgentSession) -> dict[str, Any]: 'usage': stored.usage, 'total_cost_usd': stored.total_cost_usd, 'model': stored.model_config.get('model'), + 'is_training': stored.is_training, } @@ -2151,6 +2729,53 @@ def _stored_session_has_incomplete_tail(stored: StoredAgentSession) -> bool: return budget_state.get('status') in {'queued', 'running'} +def _is_run_active(state: 'AgentState', account_id: str | None, session_id: str | None) -> bool | None: + """True iff `run_state_store` says the latest run for this session is in + `queued`/`running`. None when the lookup is impossible (no session id, or + no row yet). The pipeline endpoint uses this to detect orphan `running` + step entries — agent wrote `step:running`, finished its turn cleanly, and + never wrote the closing entry; from the user's POV the run is idle but + the panel was still spinning. + """ + if not session_id: + return None + try: + account_key = state._account_key(account_id) + snapshot = state.run_state_store.snapshot_latest(account_key, session_id) + except Exception: + return None + if snapshot is None: + return None + return snapshot.get('status') in ACTIVE_RUN_STATUSES + + +def _read_session_failure_status( + directory: Path, + session_id: str | None, +) -> str | None: + """Returns 'failed' / 'cancelled' / 'interrupted' iff the stored session + was explicitly tombstoned by `_mark_session_interrupted`. Returns None for + healthy sessions AND for live runs (budget_state.status='running'/'queued'). + + The pipeline panel uses this to paint a failed run red. We deliberately do + NOT use `_stored_session_has_incomplete_tail` here — that helper is for + detecting incomplete tails when LOADING a session for resume, where + budget_state='running' implies a dead writer. Polled live, 'running' just + means the agent is still chugging. + """ + if not session_id: + return None + try: + stored = load_agent_session(session_id, directory=directory) + except (FileNotFoundError, OSError, json.JSONDecodeError): + return None + bs = stored.budget_state if isinstance(stored.budget_state, dict) else {} + status = bs.get('status') + if status in ('failed', 'cancelled', 'interrupted'): + return str(status) + return None + + def _mark_session_interrupted( directory: Path, session_id: str, @@ -2389,7 +3014,7 @@ def _normalize_run_event(event: dict[str, object]) -> dict[str, Any] | None: 'exceeds_soft_limit', } normalized = { - key: _json_safe_limited(value) + key: _json_safe_limited(value, parent_key=key) for key, value in event.items() if key in allowed and value is not None } @@ -2397,20 +3022,25 @@ def _normalize_run_event(event: dict[str, object]) -> dict[str, Any] | None: return normalized -def _json_safe_limited(value: Any, *, depth: int = 0) -> Any: +_VERBOSE_STRING_KEYS = {'code', 'command', 'content', 'delta', 'script', 'stdout', 'output'} +_VERBOSE_STRING_LIMIT = 20000 + + +def _json_safe_limited(value: Any, *, depth: int = 0, parent_key: str | None = None) -> Any: if depth > 4: return str(value)[:500] if isinstance(value, str): - return value[:2000] + limit = _VERBOSE_STRING_LIMIT if parent_key in _VERBOSE_STRING_KEYS else 2000 + return value[:limit] if isinstance(value, (int, float, bool)) or value is None: return value if isinstance(value, dict): return { - str(key)[:200]: _json_safe_limited(item, depth=depth + 1) + str(key)[:200]: _json_safe_limited(item, depth=depth + 1, parent_key=str(key)) for key, item in list(value.items())[:80] } if isinstance(value, (list, tuple)): - return [_json_safe_limited(item, depth=depth + 1) for item in value[:80]] + return [_json_safe_limited(item, depth=depth + 1, parent_key=parent_key) for item in value[:80]] try: json.dumps(value) return value @@ -2928,6 +3558,107 @@ def _save_jupyter_runtime_binding( return +# ── Per-account remote-Jupyter workspace registry ───────────────────────────── +# Saved at accounts//workspaces.json. Each entry stores enough to re-bind a +# new chat without re-prompting (base_url + workspace_root + password). Cookies +# expire and aren't reusable across sessions, so for one-click re-bind we store +# the password — same plaintext-on-disk security posture as the per-session +# jupyter_workspace.json that already keeps login cookies. + +def _jupyter_workspaces_registry_path(state: 'AgentState', account_id: str | None) -> Path: + return state.account_paths(account_id)['base'] / JUPYTER_WORKSPACES_REGISTRY_FILENAME + + +def _load_jupyter_workspaces_registry( + state: 'AgentState', account_id: str | None +) -> list[dict[str, Any]]: + path = _jupyter_workspaces_registry_path(state, account_id) + try: + payload = json.loads(path.read_text(encoding='utf-8')) + except (OSError, json.JSONDecodeError): + return [] + if not isinstance(payload, list): + return [] + out: list[dict[str, Any]] = [] + for item in payload: + if isinstance(item, dict) and isinstance(item.get('id'), str): + out.append(item) + return out + + +def _save_jupyter_workspaces_registry( + state: 'AgentState', account_id: str | None, entries: list[dict[str, Any]] +) -> None: + path = _jupyter_workspaces_registry_path(state, account_id) + try: + path.parent.mkdir(parents=True, exist_ok=True) + _write_session_metadata(path, entries) + except OSError: + return + + +def _jupyter_workspace_registry_id(base_url: str, workspace_root: str) -> str: + """Deterministic ID = sha1(base_url|workspace_root). Same (host, root) tuple + de-duplicates rather than piling up entries on every successful bind.""" + digest = hashlib.sha1( + f'{base_url}|{workspace_root}'.encode('utf-8'), + ).hexdigest() + return digest[:16] + + +def _public_jupyter_workspace_entry(entry: dict[str, Any]) -> dict[str, Any]: + """Strip password before returning to the frontend.""" + return { + 'id': entry.get('id'), + 'label': entry.get('label'), + 'base_url': entry.get('base_url'), + 'workspace_root': entry.get('workspace_root'), + 'created_at': entry.get('created_at'), + 'last_used_at': entry.get('last_used_at'), + } + + +def _register_jupyter_workspace( + state: 'AgentState', + account_id: str | None, + *, + base_url: str, + workspace_root: str, + password: str, + label: str | None = None, +) -> dict[str, Any]: + entries = _load_jupyter_workspaces_registry(state, account_id) + workspace_id = _jupyter_workspace_registry_id(base_url, workspace_root) + now = time.time() + found: dict[str, Any] | None = None + rest: list[dict[str, Any]] = [] + for entry in entries: + if entry.get('id') == workspace_id: + found = entry + else: + rest.append(entry) + if found is None: + found = { + 'id': workspace_id, + 'label': label or base_url, + 'base_url': base_url, + 'workspace_root': workspace_root, + 'created_at': now, + } + else: + if label: + found['label'] = label + elif not found.get('label'): + found['label'] = base_url + found['base_url'] = base_url + found['workspace_root'] = workspace_root + found['password'] = password + found['last_used_at'] = now + rest.insert(0, found) # most-recently-used first + _save_jupyter_workspaces_registry(state, account_id, rest) + return found + + def _jupyter_runtime_for_session( state: AgentState, account_id: str | None, @@ -4246,6 +4977,2995 @@ def _update_session_title(directory: Path, session_id: str, title: str) -> bool: return True +def _update_session_training(directory: Path, session_id: str, is_training: bool) -> bool: + path = _session_json_path(directory, session_id) + if not path.exists(): + return False + try: + data = json.loads(path.read_text(encoding='utf-8')) + except (OSError, json.JSONDecodeError): + return False + data['is_training'] = is_training + _write_session_metadata(path, data) + return True + + +# --------------------------------------------------------------------------- +# Pipeline state watcher: agent declares 「watch」 entries in program-state.jsonl; +# a backend scanner picks them up and polls the target file/dir on its own, +# writing complete/failed entries when the watcher resolves. This decouples UI +# state from agent's discipline (or lack thereof). +# --------------------------------------------------------------------------- + + +def _now_iso_local() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).astimezone().isoformat(timespec='seconds') + + +def _watcher_append(path: Path, entry: dict[str, Any]) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('a', encoding='utf-8') as fp: + fp.write(json.dumps(entry, ensure_ascii=False) + '\n') + except OSError: + return + + +class _WatcherManager: + def __init__(self) -> None: + self._tasks: dict[tuple[str, str], asyncio.Task[None]] = {} + + def is_watching(self, session_id: str, step: str) -> bool: + return (session_id, step) in self._tasks + + def register_file_exists( + self, + *, + session_id: str, + state_path: Path, + step: str, + target: Path, + interval: float, + timeout: float, + run_id: str, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + ) -> None: + key = (session_id, step) + if key in self._tasks: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + task = loop.create_task( + self._poll_file_exists( + key=key, + state_path=state_path, + step=step, + target=target, + interval=max(2.0, interval), + timeout=max(60.0, timeout), + run_id=run_id, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + ) + self._tasks[key] = task + + async def _poll_file_exists( + self, + *, + key: tuple[str, str], + state_path: Path, + step: str, + target: Path, + interval: float, + timeout: float, + run_id: str, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, + ) -> None: + try: + mode = 'remote' if ( + agent_state is not None + and account_id is not None + and session_id is not None + and _jupyter_runtime_for_session(agent_state, account_id, session_id) + is not None + ) else 'local' + await self._write_log( + state_path, + run_id, + f'watcher 启动 step={step} target={target.name} 周期={int(interval)}s 超时={int(timeout)}s mode={mode}', + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + start = time.monotonic() + while True: + exists = await self._check_target_exists( + target, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + if exists: + await self._write_step( + state_path, + step=step, + status='complete', + run_id=run_id, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + await self._write_log( + state_path, + run_id, + f'✅ watcher 检测到 {target.name} 落盘 → step {step} complete', + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + # R{n>=1} 链路：SFT 训练完后 submit_sft_via_cml.sh 内置 nohup + # watcher 自动起 R{n} 评测，agent 不在回路中，没人写 + # cml=running，UI 会从 sft=complete 直接跳到 cml=complete， + # 中间 ~20min 评测期看起来像"流程卡死"。检测到同 session + # 已注册 cml watch（说明 agent 提交 SFT 时预注册了下游评测 + # 的 watch）就顺手补写 cml=running 同 run_id。 + if ( + step == 'sft' + and session_id is not None + and self.is_watching(session_id, 'cml') + ): + await self._write_step( + state_path, + step='cml', + status='running', + run_id=run_id, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + await self._write_log( + state_path, + run_id, + '↪ sft 完成自动衔接 → step cml running ' + '(submit_sft_via_cml.sh 内置 watcher 已触发 R 评测)', + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + return + if time.monotonic() - start > timeout: + await self._write_step( + state_path, + step=step, + status='failed', + run_id=run_id, + error=f'watcher 超时 {int(timeout)}s, target 未出现: {target}', + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + await self._write_log( + state_path, + run_id, + f'⚠️ watcher 超时 step={step} 未在 {int(timeout)}s 内看到 {target.name}', + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + return + await asyncio.sleep(interval) + except asyncio.CancelledError: + return + except Exception as exc: # noqa: BLE001 + await self._write_log( + state_path, + run_id, + f'⚠️ watcher 异常 step={step}: {exc}', + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + finally: + self._tasks.pop(key, None) + + async def _write_step( + self, + state_path: Path, + *, + step: str, + status: str, + run_id: str | None = None, + error: str | None = None, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, + ) -> None: + entry: dict[str, Any] = { + 'step': step, + 'status': status, + 'ts': _now_iso_local(), + } + if run_id: + entry['run_id'] = run_id + if error: + entry['error'] = error + await self._append_state_entry( + state_path, + entry, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + + async def _write_log( + self, + state_path: Path, + run_id: str, + text: str, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, + ) -> None: + entry = { + 'log': { + 'ts': time.strftime('%H:%M:%S'), + 'iter': run_id, + 'text': text, + } + } + await self._append_state_entry( + state_path, + entry, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + + def cancel_all(self) -> None: + for task in self._tasks.values(): + task.cancel() + self._tasks.clear() + + async def _append_state_entry( + self, + state_path: Path, + entry: dict[str, Any], + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, + ) -> None: + """Append a state entry. Prefers remote (so sync doesn't overwrite our + writes); falls back to local when no Jupyter binding exists.""" + line = json.dumps(entry, ensure_ascii=False) + '\n' + if ( + agent_state is not None + and account_id is not None + and session_id is not None + ): + runtime = _jupyter_runtime_for_session( + agent_state, account_id, session_id + ) + if runtime is not None: + remote_dir = f'{runtime.binding.workspace_cwd}/output' + remote_path = f'{remote_dir}/program-state.jsonl' + cmd = ( + f'mkdir -p {shlex.quote(remote_dir)} && ' + f'printf %s {shlex.quote(line)} >> {shlex.quote(remote_path)}' + ) + try: + await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=10.0, + max_output_chars=64, + ) + return + except Exception: # noqa: BLE001 + pass + # Local fallback + await asyncio.to_thread(_watcher_append, state_path, entry) + + async def _check_target_exists( + self, + target: Path, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, + ) -> bool: + """File-existence check that prefers remote Jupyter when bound (so + targets on /mnt/* mounts visible to the remote workspace are reachable).""" + if ( + agent_state is not None + and account_id is not None + and session_id is not None + ): + runtime = _jupyter_runtime_for_session( + agent_state, account_id, session_id + ) + if runtime is not None: + cmd = ( + f'test -f {shlex.quote(str(target))} ' + f'&& echo OK || echo MISSING' + ) + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=10.0, + max_output_chars=64, + ) + except Exception: # noqa: BLE001 + return False + return bool(result and result.stdout.strip().endswith('OK')) + # Fallback: local file system + try: + return target.is_file() + except OSError: + return False + + @staticmethod + def _append_step(state_path: Path, **fields: Any) -> None: + entry: dict[str, Any] = {**fields, 'ts': _now_iso_local()} + _watcher_append(state_path, entry) + + @staticmethod + def _append_log(state_path: Path, run_id: str, text: str) -> None: + entry = { + 'log': { + 'ts': time.strftime('%H:%M:%S'), + 'iter': run_id, + 'text': text, + } + } + _watcher_append(state_path, entry) + + +_watcher_manager = _WatcherManager() + + +class _BashBackgroundManager: + """Polls remote/local bg bash tasks; finalizes them and (optionally) fires + an auto-resume turn when the agent has paused with the previous run already + completed. + + Lifecycle mirrors `_WatcherManager`: one asyncio.Task per task_id, written + to/read from `BashBgStore` so polling can resume after a server restart. + """ + + POLL_INTERVAL_SECONDS = 5.0 + MAX_TASK_LIFETIME_SECONDS = 6 * 3600.0 + REMOTE_PROBE_TIMEOUT_SECONDS = 10.0 + OUTPUT_PREVIEW_BYTES = 4096 + + def __init__(self) -> None: + self._tasks: dict[str, asyncio.Task[None]] = {} + self._chat_runner: 'Callable[[ChatRequest], dict[str, Any]] | None' = None + + def set_chat_runner( + self, + runner: 'Callable[[ChatRequest], dict[str, Any]]', + ) -> None: + self._chat_runner = runner + + async def register(self, state: 'AgentState', spec: BgTaskSpec) -> None: + state.bash_bg_store.record_start(spec) + if spec.task_id in self._tasks: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._tasks[spec.task_id] = loop.create_task( + self._poll_alive(state, spec) + ) + + async def recover_from_store(self, state: 'AgentState') -> None: + for row in state.bash_bg_store.list_active(): + spec = self._spec_from_row(row) + await self.register(state, spec) + + async def cancel(self, state: 'AgentState', task_id: str) -> bool: + row = state.bash_bg_store.get(task_id) + if row is None or row['status'] != 'running': + return False + await self._kill_remote_or_local(state, row) + state.bash_bg_store.mark_killed(task_id) + existing = self._tasks.pop(task_id, None) + if existing is not None: + existing.cancel() + return True + + def status_query( + self, state: 'AgentState', task_id: str + ) -> 'BgTaskStatus | None': + row = state.bash_bg_store.get(task_id) + if row is None: + return None + preview = self._read_output_preview_sync(state, row) + return BgTaskStatus( + task_id=row['task_id'], + status=row['status'], + pid=row['pid'], + started_at=row['started_at'], + finished_at=row['finished_at'], + exit_code=row['exit_code'], + output_path=row['output_path'], + output_preview=preview, + auto_resumed=bool(row['auto_resumed']), + ) + + def cancel_all(self) -> None: + for task in list(self._tasks.values()): + task.cancel() + self._tasks.clear() + + async def _poll_alive(self, state: 'AgentState', spec: BgTaskSpec) -> None: + try: + start = time.monotonic() + while True: + alive = await self._check_pid_alive(state, spec) + if not alive: + await self._finalize(state, spec) + return + if time.monotonic() - start > self.MAX_TASK_LIFETIME_SECONDS: + state.bash_bg_store.mark_completed( + spec.task_id, exit_code=124, + ) + return + await asyncio.sleep(self.POLL_INTERVAL_SECONDS) + except asyncio.CancelledError: + return + except Exception as exc: # noqa: BLE001 + print( + f'[bg-bash] poll error task={spec.task_id}: {exc}', flush=True, + ) + finally: + self._tasks.pop(spec.task_id, None) + + async def _check_pid_alive( + self, state: 'AgentState', spec: BgTaskSpec + ) -> bool: + runtime = _jupyter_runtime_for_session( + state, spec.account_key or None, spec.session_id, + ) + if runtime is not None: + cmd = ( + f'kill -0 {spec.pid} 2>/dev/null && echo alive || echo dead' + ) + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=64, + cancel_event=None, + ) + except Exception: # noqa: BLE001 + # Treat probe failure as "still running" — don't finalize on + # transient runtime errors. The 6h lifetime cap is the backstop. + return True + return 'alive' in result.stdout + # Local fallback. Note: the spawned child is detached via + # start_new_session=True but the FastAPI process never reaps it, so + # after exit it becomes a zombie and kill(pid, 0) keeps reporting it + # alive. The MAX_TASK_LIFETIME_SECONDS cap is the eventual backstop; + # production runs use jupyter_runtime where the child belongs to a + # different process tree. + try: + os.kill(spec.pid, 0) + return True + except (ProcessLookupError, PermissionError): + return False + except OSError: + return True + + async def _finalize(self, state: 'AgentState', spec: BgTaskSpec) -> None: + exit_code = await self._read_exit_code(state, spec) + state.bash_bg_store.mark_completed(spec.task_id, exit_code=exit_code) + if not spec.wait_for_completion: + return + await self._maybe_auto_resume(state, spec, exit_code) + + async def _read_exit_code( + self, state: 'AgentState', spec: BgTaskSpec + ) -> int: + runtime = _jupyter_runtime_for_session( + state, spec.account_key or None, spec.session_id, + ) + if runtime is not None: + cmd = f'cat {shlex.quote(spec.exit_code_path)} 2>/dev/null || echo' + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=64, + cancel_event=None, + ) + text = result.stdout.strip() + return int(text) if text else -1 + except (ValueError, Exception): # noqa: BLE001 + return -1 + try: + text = Path(spec.exit_code_path).read_text( + encoding='utf-8', + ).strip() + return int(text) if text else -1 + except (OSError, ValueError): + return -1 + + async def _read_output_preview_async( + self, state: 'AgentState', spec_row: 'dict[str, Any] | BgTaskSpec' + ) -> str: + if isinstance(spec_row, BgTaskSpec): + account = spec_row.account_key + session = spec_row.session_id + output_path = spec_row.output_path + else: + account = spec_row['account_key'] + session = spec_row['session_id'] + output_path = spec_row['output_path'] + runtime = _jupyter_runtime_for_session(state, account or None, session) + if runtime is not None: + cmd = ( + f'tail -c {self.OUTPUT_PREVIEW_BYTES} ' + f'{shlex.quote(output_path)} 2>/dev/null || true' + ) + try: + result = await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=self.OUTPUT_PREVIEW_BYTES + 256, + cancel_event=None, + ) + return result.stdout.strip() + except Exception: # noqa: BLE001 + return '(读取远端输出失败)' + try: + text = Path(output_path).read_text( + encoding='utf-8', errors='replace', + ) + return text[-self.OUTPUT_PREVIEW_BYTES:].strip() + except OSError: + return '(无法读取输出文件)' + + def _read_output_preview_sync( + self, state: 'AgentState', row: dict[str, Any] + ) -> str: + runtime = _jupyter_runtime_for_session( + state, row['account_key'] or None, row['session_id'], + ) + if runtime is not None: + cmd = ( + f'tail -c {self.OUTPUT_PREVIEW_BYTES} ' + f'{shlex.quote(row["output_path"])} 2>/dev/null || true' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=self.OUTPUT_PREVIEW_BYTES + 256, + cancel_event=None, + ) + return result.stdout.strip() + except Exception: # noqa: BLE001 + return '(读取远端输出失败)' + try: + text = Path(row['output_path']).read_text( + encoding='utf-8', errors='replace', + ) + return text[-self.OUTPUT_PREVIEW_BYTES:].strip() + except OSError: + return '(无法读取输出文件)' + + async def _kill_remote_or_local( + self, state: 'AgentState', row: dict[str, Any] + ) -> None: + runtime = _jupyter_runtime_for_session( + state, row['account_key'] or None, row['session_id'], + ) + pid = int(row['pid']) + if runtime is not None: + cmd = ( + f'kill -- -$(ps -o pgid= -p {pid} 2>/dev/null ' + f'| tr -d " ") 2>/dev/null; ' + f'kill {pid} 2>/dev/null; true' + ) + try: + await asyncio.to_thread( + runtime.run_command, + cmd, + timeout_seconds=self.REMOTE_PROBE_TIMEOUT_SECONDS, + max_output_chars=64, + cancel_event=None, + ) + except Exception: # noqa: BLE001 + pass + return + try: + os.killpg(os.getpgid(pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + try: + os.kill(pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + pass + + async def _maybe_auto_resume( + self, state: 'AgentState', spec: BgTaskSpec, exit_code: int, + ) -> None: + runner = self._chat_runner + if runner is None: + return + account_key = spec.account_key or state._account_key(None) + session_id = spec.session_id + if not session_id: + return + snapshot = state.run_state_store.snapshot_latest( + account_key, session_id, + ) + if not snapshot or snapshot.get('status') != 'completed': + return + run_lock = state.run_lock_for(account_key, session_id) + if not run_lock.acquire(blocking=False): + return + run_lock.release() + if not state.bash_bg_store.mark_auto_resumed(spec.task_id): + return + output_preview = await self._read_output_preview_async(state, spec) + prompt = ( + f'[system] 后台任务 {spec.task_id} 已结束，' + f'exit_code={exit_code}\n' + f'命令：{spec.command}\n' + f'输出预览（最后 {self.OUTPUT_PREVIEW_BYTES} 字节）：\n' + f'{output_preview}' + ) + request = ChatRequest( + prompt=prompt, + account_id=spec.account_key or None, + session_id=session_id, + resume_session_id=session_id, + ) + loop = asyncio.get_running_loop() + loop.run_in_executor(None, _safe_run_chat_payload, runner, request) + + @staticmethod + def _spec_from_row(row: dict[str, Any]) -> BgTaskSpec: + return BgTaskSpec( + task_id=row['task_id'], + account_key=row['account_key'] or '', + session_id=row['session_id'] or '', + run_id=row['run_id'] or '', + pid=int(row['pid']), + task_dir=row['task_dir'], + output_path=row['output_path'], + pid_path=row['pid_path'], + exit_code_path=row['exit_code_path'], + command=row['command'], + started_at=float(row['started_at']), + wait_for_completion=bool(row['wait_for_completion']), + ) + + +_bash_bg_manager = _BashBackgroundManager() + + +def _safe_run_chat_payload( + runner: 'Callable[[ChatRequest], dict[str, Any]]', + request: ChatRequest, +) -> None: + """Fire-and-forget wrapper around the chat payload runner for auto-resume. + + Runs in the default thread pool. Any exception is logged but swallowed — + the bg task is already finalized in SQLite, so the user can still + bash_status / 继续 manually if the auto-resume fails. + """ + try: + runner(request) + except Exception as exc: # noqa: BLE001 + print( + f'[bg-bash] auto-resume failed session={request.session_id}: {exc}', + flush=True, + ) + + +def _sync_remote_program_state( + state: AgentState, + account_id: str, + session_id: str, + local_state_path: Path, +) -> None: + """If a Jupyter binding exists, cat the remote program-state.jsonl into local. + Backend reads only the local file; this keeps both in sync so agent writes + on remote (where its bash runs) are visible to UI/watcher logic.""" + runtime = _jupyter_runtime_for_session(state, account_id, session_id) + if runtime is None: + return + remote_state_path = ( + f'{runtime.binding.workspace_cwd}/output/program-state.jsonl' + ) + try: + result = runtime.run_command( + f'test -f {shlex.quote(remote_state_path)} && ' + f'cat {shlex.quote(remote_state_path)} || true', + timeout_seconds=10.0, + max_output_chars=200_000, + ) + except Exception: # noqa: BLE001 + return + if not result or result.exit_code != 0: + return + new_content = result.stdout or '' + if not new_content.strip(): + return + # Skip rewrite if local already matches remote (avoid touching mtime, which + # SSE uses to detect changes). + try: + if local_state_path.is_file() and ( + local_state_path.read_text(encoding='utf-8') == new_content + ): + return + except OSError: + pass + try: + local_state_path.parent.mkdir(parents=True, exist_ok=True) + tmp = local_state_path.with_suffix('.jsonl.sync-tmp') + tmp.write_text(new_content, encoding='utf-8') + tmp.replace(local_state_path) + except OSError: + return + + +def _scan_for_watchers(state: AgentState) -> None: + """Synchronous scan body — used as fallback when no event loop is available. + Prefer _scan_for_watchers_async in normal operation so blocking calls + (remote sync, LLM) run in worker threads instead of the loop.""" + accounts_root = state.session_directory.parent / 'accounts' + if not accounts_root.is_dir(): + return + for account_dir in accounts_root.iterdir(): + if not account_dir.is_dir(): + continue + account_id = account_dir.name + sessions_dir = account_dir / 'sessions' + if not sessions_dir.is_dir(): + continue + for session_dir in sessions_dir.iterdir(): + session_id = session_dir.name + state_path = session_dir / 'output' / 'program-state.jsonl' + try: + _sync_remote_program_state( + state, account_id, session_id, state_path + ) + except Exception as exc: # noqa: BLE001 + print( + f'[scanner] remote sync failed for {account_id}/{session_id}: {exc}', + flush=True, + ) + if not state_path.is_file(): + continue + entries, _ = _read_program_state(state_path) + last_status_per_step: dict[str, str] = {} + for entry in entries: + step = entry.get('step') + status = entry.get('status') + if isinstance(step, str) and step and isinstance(status, str): + last_status_per_step[step] = status + # See note in _scan_for_watchers_async: only the latest watch + # entry per step is current. + latest_watch_per_step: dict[str, dict[str, Any]] = {} + latest_running_run_id: dict[str, str] = {} + for entry in entries: + watch = entry.get('watch') + if isinstance(watch, dict): + w_step = watch.get('step') + if isinstance(w_step, str) and w_step: + latest_watch_per_step[w_step] = watch + continue + e_step = entry.get('step') + e_status = entry.get('status') + e_run = entry.get('run_id') + if ( + isinstance(e_step, str) and e_step + and e_status == 'running' + and isinstance(e_run, str) and e_run + ): + latest_running_run_id[e_step] = e_run + for step, watch in latest_watch_per_step.items(): + if last_status_per_step.get(step) in { + 'complete', + 'failed', + 'cancelled', + }: + continue + if _watcher_manager.is_watching(session_id, step): + continue + kind = watch.get('kind', 'file_exists') + if kind != 'file_exists': + continue + path_str = watch.get('path') + if not isinstance(path_str, str) or not path_str: + continue + try: + interval = float(watch.get('interval', 30) or 30) + timeout = float(watch.get('timeout', 3600) or 3600) + except (TypeError, ValueError): + interval = 30.0 + timeout = 3600.0 + run_id = str(watch.get('run_id') or '?') + running_run_id = latest_running_run_id.get(step) + if running_run_id and run_id != running_run_id: + continue + _watcher_manager.register_file_exists( + session_id=session_id, + state_path=state_path, + step=step, + target=Path(path_str), + interval=interval, + timeout=timeout, + run_id=run_id, + agent_state=state, + account_id=account_id, + ) + + +async def _scan_for_watchers_async(state: AgentState) -> None: + """Async-friendly scan: runs blocking I/O (remote sync, LLM target_set + extraction) in worker threads so the event loop stays responsive for SSE.""" + accounts_root = state.session_directory.parent / 'accounts' + if not accounts_root.is_dir(): + return + try: + account_dirs = list(accounts_root.iterdir()) + except OSError: + return + for account_dir in account_dirs: + if not account_dir.is_dir(): + continue + account_id = account_dir.name + sessions_dir = account_dir / 'sessions' + if not sessions_dir.is_dir(): + continue + try: + session_dirs = list(sessions_dir.iterdir()) + except OSError: + continue + for session_dir in session_dirs: + session_id = session_dir.name + state_path = session_dir / 'output' / 'program-state.jsonl' + # Remote sync runs HTTP/WS — blocking. Off the loop. + try: + await asyncio.to_thread( + _sync_remote_program_state, + state, + account_id, + session_id, + state_path, + ) + except Exception as exc: # noqa: BLE001 + print( + f'[scanner] remote sync failed for {account_id}/{session_id}: {exc}', + flush=True, + ) + if not state_path.is_file(): + continue + try: + entries, _ = await asyncio.to_thread( + _read_program_state, state_path + ) + except Exception: # noqa: BLE001 + continue + # Refresh LLM-derived target_set cache (also off the loop). + trigger_text = await asyncio.to_thread( + _extract_trigger_from_session, sessions_dir, session_id + ) + if trigger_text: + try: + await asyncio.to_thread( + _refresh_target_set_cache_blocking, + state.model_config_for(account_id), + trigger_text, + ) + except Exception as exc: # noqa: BLE001 + print( + f'[scanner] target_set LLM refresh failed: {exc}', + flush=True, + ) + # Watch registration is fast (just spawns asyncio tasks); stay on loop. + last_status_per_step: dict[str, str] = {} + for entry in entries: + step = entry.get('step') + status = entry.get('status') + if isinstance(step, str) and step and isinstance(status, str): + last_status_per_step[step] = status + # Only the LAST watch entry per step represents the current round's + # intent. Earlier watch entries from previous rounds must NOT be + # re-registered when a new round resets the step to running, or + # the resulting watcher will fire with a stale run_id and the UI + # will never see a complete tagged with the current round. + latest_watch_per_step: dict[str, dict[str, Any]] = {} + latest_running_run_id: dict[str, str] = {} + for entry in entries: + watch = entry.get('watch') + if isinstance(watch, dict): + w_step = watch.get('step') + if isinstance(w_step, str) and w_step: + latest_watch_per_step[w_step] = watch + continue + e_step = entry.get('step') + e_status = entry.get('status') + e_run = entry.get('run_id') + if ( + isinstance(e_step, str) and e_step + and e_status == 'running' + and isinstance(e_run, str) and e_run + ): + latest_running_run_id[e_step] = e_run + for step, watch in latest_watch_per_step.items(): + if last_status_per_step.get(step) in { + 'complete', + 'failed', + 'cancelled', + }: + continue + if _watcher_manager.is_watching(session_id, step): + continue + kind = watch.get('kind', 'file_exists') + if kind != 'file_exists': + continue + path_str = watch.get('path') + if not isinstance(path_str, str) or not path_str: + continue + try: + interval_s = float(watch.get('interval', 30) or 30) + timeout_s = float(watch.get('timeout', 3600) or 3600) + except (TypeError, ValueError): + interval_s = 30.0 + timeout_s = 3600.0 + run_id = str(watch.get('run_id') or '?') + # If a newer round has written `step=, status=running` + # but the matching watch entry hasn't landed yet, skip and + # wait for the next scan rather than spawning a watcher tied + # to the previous round's run_id. + running_run_id = latest_running_run_id.get(step) + if running_run_id and run_id != running_run_id: + continue + _watcher_manager.register_file_exists( + session_id=session_id, + state_path=state_path, + step=step, + target=Path(path_str), + interval=interval_s, + timeout=timeout_s, + run_id=run_id, + agent_state=state, + account_id=account_id, + ) + + +async def _watcher_scanner_loop( + state: AgentState, interval: float = 5.0 +) -> None: + """Periodically scan all session program-state.jsonl files for new watch + declarations and spawn watchers for them. Blocking calls run in threads.""" + while True: + try: + await _scan_for_watchers_async(state) + except asyncio.CancelledError: + return + except Exception as exc: # noqa: BLE001 + print(f'[watcher-scanner] scan error: {exc}', flush=True) + try: + await asyncio.sleep(interval) + except asyncio.CancelledError: + return + + +def _session_state_path(sessions_dir: Path, session_id: str | None) -> Path | None: + """Resolve //output/program-state.jsonl. Returns None if unsafe id.""" + safe_id = _safe_session_id(session_id) + if safe_id is None: + return None + return sessions_dir / safe_id / 'output' / 'program-state.jsonl' + + +def _read_program_state(path: Path | None) -> tuple[list[dict[str, Any]], float | None]: + """Returns (entries, mtime_seconds). Empty if path is None / missing / unreadable.""" + if path is None: + return [], None + try: + if not path.is_file(): + return [], None + st = path.stat() + text = path.read_text(encoding='utf-8', errors='replace') + except (FileNotFoundError, OSError, PermissionError): + return [], None + entries: list[dict[str, Any]] = [] + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith('#'): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + entries.append(obj) + return entries, st.st_mtime + + +def _normalize_step_token(value: str) -> str: + return re.sub(r'[\s_\-/.]+', '', value.strip().lower()) + + +def _step_matches_card(step: str, card: dict[str, Any]) -> bool: + s = _normalize_step_token(step) + if not s: + return False + key = _normalize_step_token(str(card.get('key') or '')) + if s == key: + return True + title = _normalize_step_token(str(card.get('title') or '')) + if title and (s in title or title.startswith(s) or s in key): + return True + return False + + +# Round-section pipeline data model. +# +# Old: 3 fixed phases (eval / intervene / train) with cards dynamically injected. +# New: per-round numbered sections (R0·Baseline → R1·Train → R1·Analysis → ...), +# with optional HiTL gates inserted between rounds when agent writes +# step:"human-check" or step:"human-review". +# +# Each step belongs to a "section type" (baseline / train / analysis): +# - R0 has only one section: R0·Baseline (cml/gold-drift/dist-analysis/report/hypothesis/log) +# - R{n>=1} has two sections: R{n}·Train (augment/verify/sft) +# R{n}·Analysis (cml/gold-drift/dist-analysis/report/hypothesis/log) +# +# 'hypothesis' is classified as 'analysis' (not 'train'): forming the next-round +# hypothesis is the conclusion of the current round's analysis cycle, not the +# start of the next round's training. Putting it in analysis makes the gate +# (Human Check / Review) — which sorts after all sections of run_id=R{n} but +# before R{n+1}·Train — block training kickoff cleanly. +# +# 'next-round' is a boundary marker — does not produce a card. +# 'human-check' / 'human-review' produce gate items, not cards. + +# step → kind: 'analysis' | 'train' | 'gate' | 'boundary' +_STEP_KIND: dict[str, str] = { + 'cml': 'analysis', + 'gold-drift': 'analysis', + 'dist-analysis': 'analysis', + 'hypothesis': 'analysis', + 'log': 'analysis', + 'augment': 'train', + 'verify': 'train', + 'sft': 'train', + 'human-check': 'gate', + 'human-review': 'gate', + 'next-round': 'boundary', +} + +_GATE_TITLES: dict[str, dict[str, str]] = { + 'human-check': { + 'title': '人工确认', + 'description': '已生成本轮假设与候选，等待你点头进入下一轮训练', + }, + 'human-review': { + 'title': '人工复核', + 'description': '触发了 §4.0.1 / Gold-drift 规则，需要你逐条审一下再继续', + }, +} + +# (step, section_type) → display metadata. section_type is 'baseline' for R0 +# steps, 'train'/'analysis' for R{n>=1}. +_CARD_META: dict[tuple[str, str], dict[str, Any]] = { + # R0·Baseline + ('cml', 'baseline'): {'icon': 'bar-chart', 'title': 'Baseline 评测', 'subtitle': 'CML workflow metric_diff'}, + ('gold-drift', 'baseline'): {'icon': 'alert-triangle', 'title': 'Gold Drift 检查', 'subtitle': 'drift_.json'}, + ('dist-analysis', 'baseline'): {'icon': 'trending-up', 'title': '分层结果分析 & 报告', 'subtitle': '根因归类 + workflow.md'}, + ('hypothesis', 'baseline'): {'icon': 'dna', 'title': '形成假设', 'subtitle': 'iteration_log hypothesis 字段'}, + ('log', 'baseline'): {'icon': 'clipboard-list', 'title': '记录迭代日志', 'subtitle': 'iteration_log.jsonl'}, + # R{n}·Train + ('augment', 'train'): {'icon': 'flask', 'title': '数据增强', 'subtitle': 'augment_.jsonl'}, + ('verify', 'train'): {'icon': 'shield', 'title': '修改返回验证', 'subtitle': 'sanity + 人审'}, + ('sft', 'train'): {'icon': 'graduation-cap','title': 'SFT 训练', 'subtitle': 'submit_sft_via_cml.sh'}, + # R{n}·Analysis + ('cml', 'analysis'): {'icon': 'bar-chart', 'title': 'CML 评测', 'subtitle': 'workflow metric_diff'}, + ('gold-drift', 'analysis'): {'icon': 'alert-triangle', 'title': 'Gold Drift 检查', 'subtitle': 'drift_.json'}, + ('dist-analysis', 'analysis'): {'icon': 'trending-up', 'title': '分层结果分析 & 报告', 'subtitle': '根因归类 + workflow.md'}, + ('hypothesis', 'analysis'): {'icon': 'dna', 'title': '形成假设', 'subtitle': 'iteration_log hypothesis 字段'}, + ('log', 'analysis'): {'icon': 'clipboard-list', 'title': '记录迭代日志', 'subtitle': 'iteration_log.jsonl'}, +} + +# Order within a section. +_CARD_ORDER: dict[tuple[str, str], int] = { + # baseline + ('cml', 'baseline'): 0, + ('gold-drift', 'baseline'): 1, + ('dist-analysis', 'baseline'): 2, + ('hypothesis', 'baseline'): 3, + ('log', 'baseline'): 4, + # train + ('augment', 'train'): 0, + ('verify', 'train'): 1, + ('sft', 'train'): 2, + # analysis + ('cml', 'analysis'): 0, + ('gold-drift', 'analysis'): 1, + ('dist-analysis', 'analysis'): 2, + ('hypothesis', 'analysis'): 3, + ('log', 'analysis'): 4, +} + +_SECTION_LABELS: dict[str, str] = { + 'baseline': 'Baseline', + 'train': 'Train', + 'analysis': 'Analysis', +} + +_SECTION_DESCRIPTIONS: dict[str, str] = { + 'baseline': 'Establish baseline performance and metrics', + 'train': 'Form hypothesis & retrain on augmented data', + 'analysis': 'Evaluate new checkpoint & diagnose regression', +} + + +def _parse_run_index(run_id: str | None) -> int | None: + """'R0' → 0, 'R12' → 12, anything else → None.""" + if not isinstance(run_id, str): + return None + m = re.match(r'^[Rr](\d+)$', run_id.strip()) + if not m: + return None + try: + return int(m.group(1)) + except ValueError: + return None + + +def _infer_run_id( + entry: dict[str, Any], + iteration_log_count: int, + *, + same_step_hint: str | None = None, +) -> str | None: + """Resolve which round this state entry belongs to. + + 1. explicit entry.run_id wins — UNLESS it's semantically invalid (a train + step tagged R0; R0 is baseline-only by design). In that case we treat + the explicit value as a typo and fall through to inference. + 2. else: same_step_hint (run_id from nearest same-step entry) — handles + stateless writes like the watcher's complete/failed entries that lack + run_id but should obviously inherit the round of the agent-tagged + cml/running line they accompany. + 3. else: derive from kind + iteration_log line count + - analysis steps: evaluating R{count}'s output → R{count} + - train steps: preparing R{count+1} → R{count+1} + - gates / boundary: cannot infer alone — caller back-fills from neighbor + """ + step = entry.get('step') + kind = _STEP_KIND.get(step) if isinstance(step, str) else None + + explicit = entry.get('run_id') + if isinstance(explicit, str) and explicit.strip(): + explicit_idx = _parse_run_index(explicit) + if explicit_idx is not None: + # R0+train is a common agent typo: hypothesis/augment/verify/sft are + # "preparing R1", but agents sometimes tag them with the round they + # see in the report. We rewrite to R1 specifically (NOT current-time + # inference) so retroactive iteration_log changes don't shift the + # cards into R{n>=2}. + if kind == 'train' and explicit_idx == 0: + return 'R1' + return explicit.strip() + + if not isinstance(step, str): + return None + if same_step_hint: + return same_step_hint + if kind == 'analysis': + return f'R{iteration_log_count}' + if kind == 'train': + return f'R{max(iteration_log_count, 0) + 1}' + return None + + +def _build_same_step_hints( + state_entries: list[dict[str, Any]], +) -> list[str | None]: + """For each entry, find the nearest same-step entry (by index distance) + that carries an explicit, semantically-valid run_id, and return its + run_id. Used as a fallback for stateless writes (e.g. watcher's complete + line lacks run_id but the agent's cml/running line right above has it). + + Returns a parallel list aligned to state_entries; None for entries with + no usable neighbor. + """ + explicit_by_step: dict[str, list[tuple[int, str]]] = {} + for idx, entry in enumerate(state_entries): + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind in (None, 'boundary', 'gate'): + continue + rid = entry.get('run_id') + if not (isinstance(rid, str) and rid.strip()): + continue + parsed = _parse_run_index(rid) + if parsed is None: + continue + # Apply same R0+train typo rewrite as _infer_run_id so the hint stays + # consistent with what _infer_run_id would have returned for that + # entry on its own. + canonical = 'R1' if (kind == 'train' and parsed == 0) else rid.strip() + explicit_by_step.setdefault(step, []).append((idx, canonical)) + + hints: list[str | None] = [None] * len(state_entries) + for idx, entry in enumerate(state_entries): + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + candidates = explicit_by_step.get(step) + if not candidates: + continue + best = min(candidates, key=lambda ix_rid: abs(ix_rid[0] - idx)) + hints[idx] = best[1] + return hints + + +def _section_type_for(step: str, run_index: int) -> str | None: + """Which section a card belongs to within its round. + + Train steps under R0 are filtered out by `_infer_run_id` (R0 has no train + phase). If one slips through here, we drop it rather than aliasing to + baseline — silently mixing train cards into a baseline section is what + caused hypothesis to render under R0·Baseline. + """ + kind = _STEP_KIND.get(step) + if kind == 'analysis': + return 'baseline' if run_index == 0 else 'analysis' + if kind == 'train': + return 'train' if run_index >= 1 else None + return None + + +def _derive_section_status(card_statuses: list[str]) -> str: + if not card_statuses: + return 'pending' + if all(s == 'complete' for s in card_statuses): + return 'complete' + if 'failed' in card_statuses and not any(s in ('running', 'complete') for s in card_statuses): + return 'failed' + if 'running' in card_statuses or 'complete' in card_statuses: + return 'running' + return 'pending' + + +def _build_pipeline_items( + state_entries: list[dict[str, Any]], + iteration_log_count: int, +) -> list[dict[str, Any]]: + """Assemble the items[] payload from raw state entries. + + Returns an ordered list of {type:'round', ...} and {type:'gate', ...} items + in display order. Each item shows agent-written progress only — sections + not yet touched do not appear. + """ + # 1. Walk entries: keep latest per step (per round-namespaced key). + # We namespace by run_id so the same step name on different rounds + # doesn't overwrite each other. + latest_step_by_key: dict[str, dict[str, Any]] = {} + latest_gate_by_key: dict[str, dict[str, Any]] = {} + section_first_ts: dict[tuple[str, str], str] = {} # (run_id, section_type) → earliest ts + last_non_gate_run_id: str | None = None + gate_position_index: dict[str, int] = {} # gate_key → position in entries (for ordering) + same_step_hints = _build_same_step_hints(state_entries) + + for idx, entry in enumerate(state_entries): + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind == 'boundary': + continue + if kind == 'gate': + run_id = entry.get('run_id') + if not (isinstance(run_id, str) and run_id.strip()): + run_id = last_non_gate_run_id + if not run_id: + continue + run_id = run_id.strip() + gate_key = f'{run_id}:{step}' + existing = latest_gate_by_key.get(gate_key) + if existing is None: + latest_gate_by_key[gate_key] = { + **entry, + '_run_id': run_id, + '_first_ts': entry.get('ts', ''), + } + gate_position_index[gate_key] = idx + else: + existing.update({k: v for k, v in entry.items() if k != 'ts'}) + # keep first ts for ordering + continue + + # analysis or train + run_id = _infer_run_id( + entry, iteration_log_count, same_step_hint=same_step_hints[idx] + ) + if not run_id: + continue + last_non_gate_run_id = run_id + run_index = _parse_run_index(run_id) + if run_index is None: + continue + section_type = _section_type_for(step, run_index) + if section_type is None: + continue + card_key = f'{run_id}:{step}' + ts = entry.get('ts', '') + if card_key not in latest_step_by_key: + latest_step_by_key[card_key] = { + **entry, + '_run_id': run_id, + '_section_type': section_type, + '_first_ts': ts, + } + else: + existing = latest_step_by_key[card_key] + for k, v in entry.items(): + existing[k] = v + section_key = (run_id, section_type) + if section_key not in section_first_ts: + section_first_ts[section_key] = ts + + # 2. Group cards by section. + sections: dict[tuple[str, str], dict[str, Any]] = {} + for card_key, entry in latest_step_by_key.items(): + run_id = entry['_run_id'] + section_type = entry['_section_type'] + section_key = (run_id, section_type) + if section_key not in sections: + sections[section_key] = { + 'run_id': run_id, + 'section_type': section_type, + 'cards': [], + 'first_ts': section_first_ts.get(section_key, ''), + } + step = entry['step'] + meta = _CARD_META.get((step, section_type), { + 'icon': 'clock', + 'title': step, + 'subtitle': '', + }) + order = _CARD_ORDER.get((step, section_type), 999) + card: dict[str, Any] = { + 'key': card_key, + 'step': step, + 'icon': meta['icon'], + 'title': meta['title'], + 'subtitle': meta['subtitle'], + 'status': entry.get('status') or 'pending', + '_order': order, + } + if 'progress' in entry: + try: + value = float(entry['progress']) + if value > 1: + value /= 100 + card['progress'] = max(0.0, min(1.0, value)) + except (TypeError, ValueError): + pass + sections[section_key]['cards'].append(card) + + # 3. Sort cards within each section. + for sec in sections.values(): + sec['cards'].sort(key=lambda c: c.get('_order', 999)) + for c in sec['cards']: + c.pop('_order', None) + + # 4. Build round items in display order. + # Order: by run_index ascending, then within same round: train before analysis. + section_order = {'baseline': 0, 'train': 0, 'analysis': 1} + section_items: list[dict[str, Any]] = [] + for (run_id, section_type), sec in sections.items(): + run_index = _parse_run_index(run_id) + if run_index is None: + continue + statuses = [c['status'] for c in sec['cards']] + round_status = _derive_section_status(statuses) + label = f'{run_id} · {_SECTION_LABELS[section_type]}' + section_items.append({ + 'type': 'round', + 'run_id': run_id, + 'run_index': run_index, + 'section_type': section_type, + 'label': label, + 'description': _SECTION_DESCRIPTIONS[section_type], + 'status': round_status, + 'cards': sec['cards'], + '_order_key': (run_index, section_order.get(section_type, 9), sec.get('first_ts', '')), + }) + + # 5. Build gate items. + gate_items: list[dict[str, Any]] = [] + for gate_key, entry in latest_gate_by_key.items(): + run_id = entry['_run_id'] + run_index = _parse_run_index(run_id) + if run_index is None: + continue + step = entry['step'] + meta_g = _GATE_TITLES.get(step, {'title': step, 'description': ''}) + # Place gate after the latest section of this run_id (so use a high + # secondary order). Use position index for tie-break across same round. + raw_status = entry.get('status') or 'pending' + # Gate running → 'waiting': the agent has surfaced the checkpoint and + # ended its turn. There's no work in flight on the agent side; we're + # waiting on the user. The frontend renders 'waiting' with a distinct + # amber pill ("等待人工确认") instead of the spinning blue "IN PROGRESS" + # used for actual running steps. + gate_status = 'waiting' if raw_status == 'running' else raw_status + # Structured fields for the gate body — agent should prefer these so + # the UI can render labeled rows (现状 / 提议 / 请选). `reason` is kept + # as a free-text fallback for entries that haven't been migrated. + gate_items.append({ + 'type': 'gate', + 'key': gate_key, + 'run_id': run_id, + 'run_index': run_index, + 'gate_kind': step, + 'title': meta_g['title'], + 'description': meta_g['description'], + 'status': gate_status, + 'reason': entry.get('reason'), + 'summary': entry.get('summary'), + 'proposal': entry.get('proposal'), + 'ask': entry.get('ask'), + '_order_key': (run_index, 99, gate_position_index.get(gate_key, 0)), + }) + + # 6. Filter: drop empty/pending sections (no cards or all pending). Running + # and complete sections both render — RoundSection's status pill conveys + # the difference, so the transition complete→running on the same numbered + # box is in-place rather than a separate chip strip popping in/out. + section_items = [ + s for s in section_items if s['status'] in ('running', 'complete', 'failed') + ] + + # 7. Merge and sort all items by (run_index, section_order, ts/position). + all_items = section_items + gate_items + all_items.sort(key=lambda x: x['_order_key']) + + # 8. Attach 1-based numeric index for display ("01", "02", ...) after + # filtering so numbering is dense (no gaps from hidden running sections). + for i, item in enumerate(all_items, start=1): + item['index'] = i + item.pop('_order_key', None) + + return all_items + + +def _compute_in_flight( + state_entries: list[dict[str, Any]], + iteration_log_count: int, +) -> dict[str, Any] | None: + """Render the in-progress phase as an inline chip row. + + The wrapping section only appears once the whole phase is COMPLETE. + Until then we still want the user to see what's been done in the current + phase: every step that has reached `complete` plus the one currently + `running`. Pending steps stay hidden — they appear when their turn + comes and replace nothing. + + Returns a single object describing the in-progress phase + its + progressively-filling cards, in canonical step order. None when no + phase is currently running (everything settled, or the latest signal + is a gate/boundary). + """ + same_step_hints = _build_same_step_hints(state_entries) + target_run_id: str | None = None + target_section: str | None = None + seen_keys: set[str] = set() + for idx in range(len(state_entries) - 1, -1, -1): + entry = state_entries[idx] + if entry.get('kpi') or entry.get('log'): + continue + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind in ('boundary', 'gate'): + continue + run_id = _infer_run_id( + entry, iteration_log_count, same_step_hint=same_step_hints[idx] + ) + if not run_id: + continue + key = f'{run_id}:{step}' + if key in seen_keys: + continue + seen_keys.add(key) + if entry.get('status') != 'running': + continue + run_index = _parse_run_index(run_id) + if run_index is None: + continue + section_type = _section_type_for(step, run_index) + if section_type is None: + continue + target_run_id = run_id + target_section = section_type + break + + if not target_run_id or not target_section: + return None + + latest_per_step: dict[str, dict[str, Any]] = {} + for idx, entry in enumerate(state_entries): + if entry.get('kpi') or entry.get('log'): + continue + step = entry.get('step') + if not isinstance(step, str) or not step: + continue + kind = _STEP_KIND.get(step) + if kind in ('boundary', 'gate'): + continue + run_id = _infer_run_id( + entry, iteration_log_count, same_step_hint=same_step_hints[idx] + ) + if run_id != target_run_id: + continue + run_index = _parse_run_index(run_id) + if run_index is None: + continue + section_type = _section_type_for(step, run_index) + if section_type != target_section: + continue + latest_per_step[step] = entry + + cards: list[dict[str, Any]] = [] + for step, entry in latest_per_step.items(): + status = entry.get('status') + if status not in ('complete', 'running'): + continue + meta = _CARD_META.get((step, target_section), { + 'icon': 'clock', + 'title': step, + 'subtitle': '', + }) + progress: float | None = None + if 'progress' in entry: + try: + value = float(entry['progress']) + if value > 1: + value /= 100 + progress = max(0.0, min(1.0, value)) + except (TypeError, ValueError): + pass + cards.append({ + 'key': f'{target_run_id}:{step}', + 'step': step, + 'icon': meta['icon'], + 'title': meta['title'], + 'subtitle': meta['subtitle'], + 'status': status, + 'progress': progress, + }) + + if not cards: + return None + + cards.sort(key=lambda c: _CARD_ORDER.get((c['step'], target_section), 999)) + + return { + 'run_id': target_run_id, + 'section_type': target_section, + 'section_label': f'{target_run_id} · {_SECTION_LABELS[target_section]}', + 'cards': cards, + } + + +def _apply_program_state( + payload: dict[str, Any], + state_entries: list[dict[str, Any]], + *, + iteration_log_count: int = 0, + session_failure_status: str | None = None, + run_active: bool | None = None, + session_id: str | None = None, +) -> None: + """Mutate payload in place. Builds items[] from state entries and appends + log lines. KPI entries (legacy `{kpi:..,value:..}`) are ignored — + KPIs are computed by the backend in _compute_kpis.""" + # Append logs first (they are independent of items[]). + appended_logs: list[dict[str, Any]] = [] + for entry in state_entries: + if entry.get('kpi'): + continue + if entry.get('log'): + log_obj = entry.get('log') + if isinstance(log_obj, dict): + appended_logs.append(log_obj) + elif isinstance(log_obj, str): + appended_logs.append({'text': log_obj, 'ts': entry.get('ts', '')}) + + if appended_logs: + existing = payload.setdefault('logs', []) + existing.extend(appended_logs) + + # Build items (already filtered to complete sections + visible gates). + items = _build_pipeline_items(state_entries, iteration_log_count) + payload['items'] = items + payload['in_flight'] = _compute_in_flight(state_entries, iteration_log_count) + + # Derive overall status from RAW entries — items[] is filtered to complete + # sections only, so deriving from items would falsely report 'complete' + # whenever a running section is hidden. + latest_per_step: dict[str, str] = {} + for entry in state_entries: + if entry.get('kpi') or entry.get('log'): + continue + step = entry.get('step') + status = entry.get('status') + if isinstance(step, str) and step and isinstance(status, str) and status: + # boundary steps don't count toward overall progress + if _STEP_KIND.get(step) == 'boundary': + continue + latest_per_step[step] = status + + non_gate_running = any( + status == 'running' and _STEP_KIND.get(step) != 'gate' + for step, status in latest_per_step.items() + ) + gate_running = any( + status == 'running' and _STEP_KIND.get(step) == 'gate' + for step, status in latest_per_step.items() + ) + statuses = list(latest_per_step.values()) + if not statuses: + payload['status']['state'] = 'pending' + elif non_gate_running: + payload['status']['state'] = 'running' + elif gate_running: + # Only HiTL gates are 'running' — agent has handed control back to the + # user. Surface as 'waiting' so the header pill says "等待人工" instead + # of the spinning "Running" badge. + payload['status']['state'] = 'waiting' + elif all(s == 'complete' for s in statuses): + payload['status']['state'] = 'complete' + elif 'complete' in statuses: + payload['status']['state'] = 'running' + elif 'failed' in statuses: + payload['status']['state'] = 'failed' + else: + payload['status']['state'] = 'pending' + + # Orphan detection: agent wrote `step:running` then turn ended cleanly + # without a closing `complete`/`failed` entry — run_state_store says the + # run is no longer active, but the pipeline still claims `running`. Fold + # this into session_failure_status='failed' so the same red-failed visual + # applies (per user UX choice). Note: only triggers when there's at least + # one explicit running step — purely-pending state stays pending. + # + # Exception: if every running step has an active watcher polling for it, + # it's NOT an orphan — agent legitimately ended its turn and is waiting + # for a remote file to land via the watcher. The watcher will write the + # closing entry when it triggers (file_exists / timeout / cancel). + # + # Gates (human-check / human-review) are also never orphans: agent writes + # `step:running` to surface the gate card, ends the turn, and waits for + # the user to reply. There's no watcher and no remote file — the closing + # `complete` entry comes from the next user-driven turn. + running_steps = { + step for step, status in latest_per_step.items() + if status == 'running' and _STEP_KIND.get(step) != 'gate' + } + all_watched = bool(running_steps) and all( + _watcher_manager.is_watching(session_id, step) + for step in running_steps + if session_id + ) + if ( + run_active is False + and session_failure_status is None + and payload['status']['state'] == 'running' + and running_steps + and not all_watched + ): + session_failure_status = 'failed' + + # Session-level failure (agent process died / cancelled) overrides the + # state derived from program-state.jsonl. The state file can't write its + # own tombstone when the writer is the dead process — we cross-reference + # session.json to surface the failure on the monitor panel. + if session_failure_status in ('failed', 'cancelled', 'interrupted'): + # Cancelled keeps a softer 'cancelled' label so the UI can distinguish + # user-stop from crash; everything else maps to 'failed' (red). + target_state = 'cancelled' if session_failure_status == 'cancelled' else 'failed' + if payload['status']['state'] != 'complete': + payload['status']['state'] = target_state + in_flight = payload.get('in_flight') + if isinstance(in_flight, dict): + cards = in_flight.get('cards') + if isinstance(cards, list): + for card in cards: + if isinstance(card, dict) and card.get('status') == 'running': + card['status'] = target_state + # Same treatment for any running cards inside items[] — running + # round sections also get flipped, and the section status follows. + for item in payload.get('items', []): + if not isinstance(item, dict): + continue + if item.get('type') == 'round': + section_cards = item.get('cards', []) + flipped = False + if isinstance(section_cards, list): + for card in section_cards: + if isinstance(card, dict) and card.get('status') == 'running': + card['status'] = target_state + flipped = True + if flipped or item.get('status') == 'running': + item['status'] = target_state + elif item.get('type') == 'gate' and item.get('status') == 'running': + item['status'] = target_state +def _autoresearch_root() -> Path: + return Path(os.environ.get('AUTORESEARCH_ROOT', '/mnt/wangsenhao/autoresearch-zk')) + + +def _chat_root_for_runtime( + runtime: 'JupyterRuntimeSession | None', +) -> str | None: + """Resolve the per-chat autoresearch root as a REMOTE path string. + + Returns the NFS path under /mnt/wangsenhao/autoresearch-zk-users/// + so jupyter pod and SFT training pod both read/write the same directory. + Returns None when no runtime is bound — caller decides whether to fall + back to the legacy global path.""" + if runtime is None: + return None + return runtime.chat_workspace_root + + +def _run_history_dir() -> Path: + return Path( + os.environ.get( + 'RUN_HISTORY_DIR', + '/mnt/xiaoai-zk-model-train-tj5/workflow5', + ) + ) + + +def _skill_config_yaml_path() -> Path: + # backend/api/server.py → ../../skills/model-iteration/assets/config.yaml + return ( + Path(__file__).resolve().parent.parent.parent + / 'skills' + / 'model-iteration' + / 'assets' + / 'config.yaml' + ) + + +def _read_workflow_version() -> str | None: + path = _skill_config_yaml_path() + try: + text = path.read_text(encoding='utf-8') + except (FileNotFoundError, OSError): + return None + # Tiny parser to avoid pulling pyyaml just for two lines. + in_cml = False + for raw in text.splitlines(): + stripped = raw.rstrip() + if not stripped or stripped.lstrip().startswith('#'): + continue + if not stripped.startswith(' '): + in_cml = stripped.split(':', 1)[0].strip() == 'cml_eval' + continue + if in_cml: + inner = stripped.strip() + if inner.startswith('version:'): + value = inner.split(':', 1)[1].strip().strip('"\'') + return value or None + return None + + +def _scan_run_history_max() -> int | None: + root = _run_history_dir() + try: + if not root.is_dir(): + return None + except OSError: + return None + best = -1 + for entry in root.iterdir(): + if not entry.is_dir(): + continue + m = re.match(r'^workflow(\d+)$', entry.name) + if m: + try: + value = int(m.group(1)) + except ValueError: + continue + if value > best: + best = value + return best if best >= 0 else None + + +_ITER_COUNT_CACHE: dict[str, int | None] = {} + + +def _iter_count_cache_key(session_id: str | None, account_id: str | None) -> str: + return f'{account_id or "_"}|{session_id or "_"}' + + +def _read_iteration_log_count( + session_id: str | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, +) -> int | None: + """Count non-empty lines in iteration_log.jsonl. + + Chat-root lives under the remote workspace_cwd, so reads MUST go through + the bound jupyter runtime — the round-trip is multi-second. To keep the + SSE pipeline loop responsive, this function returns the last cached value + when one exists; only a cold cache pays the synchronous remote cost. The + cache is refreshed in the background by `_iter_count_refresh_loop`, which + the SSE handler spawns alongside its event loop. + """ + cache_key = _iter_count_cache_key(session_id, account_id) + if cache_key in _ITER_COUNT_CACHE: + return _ITER_COUNT_CACHE[cache_key] + value = _read_iteration_log_count_uncached( + session_id, agent_state=agent_state, account_id=account_id, + ) + _ITER_COUNT_CACHE[cache_key] = value + return value + + +async def _iter_count_refresh_loop( + session_id: str | None, + *, + agent_state: 'AgentState | None', + account_id: str | None, + interval_seconds: float = 3.0, +) -> None: + """Background task that re-reads iteration_log_count every `interval_seconds` + and updates the in-memory cache. Cancelled when the caller (SSE handler) + exits.""" + cache_key = _iter_count_cache_key(session_id, account_id) + while True: + try: + value = await asyncio.to_thread( + _read_iteration_log_count_uncached, + session_id, + agent_state=agent_state, + account_id=account_id, + ) + _ITER_COUNT_CACHE[cache_key] = value + except Exception: # noqa: BLE001 + pass + await asyncio.sleep(interval_seconds) + + +def _read_iteration_log_count_uncached( + session_id: str | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, +) -> int | None: + runtime = ( + _jupyter_runtime_for_session(agent_state, account_id, session_id) + if session_id and agent_state is not None + else None + ) + remote_root = _chat_root_for_runtime(runtime) + if runtime is not None and remote_root is not None: + remote_path = f'{remote_root}/results/iteration_log.jsonl' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& grep -c "[^[:space:]]" {shlex.quote(remote_path)} ' + f'|| echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=10.0, + max_output_chars=2000, + ) + except Exception: # noqa: BLE001 + return None + stdout = (result.stdout or '').strip() if result is not None else '' + if stdout and stdout != '__MISSING__': + try: + return int(stdout.splitlines()[-1]) + except (ValueError, IndexError): + return None + return None + legacy = _autoresearch_root() / 'results' / 'iteration_log.jsonl' + try: + if legacy.is_file(): + with legacy.open('r', encoding='utf-8') as fp: + return sum(1 for line in fp if line.strip()) + except (OSError, UnicodeDecodeError): + pass + return None + + +_METRICS_HISTORY_CACHE: dict[str, dict[str, Any] | None] = {} + + +def _read_iteration_log_metrics_history( + session_id: str | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, +) -> dict[str, Any] | None: + """Cached read of full iteration_log.jsonl as metrics time-series.""" + cache_key = _iter_count_cache_key(session_id, account_id) + if cache_key in _METRICS_HISTORY_CACHE: + return _METRICS_HISTORY_CACHE[cache_key] + value = _read_iteration_log_metrics_history_uncached( + session_id, agent_state=agent_state, account_id=account_id, + ) + _METRICS_HISTORY_CACHE[cache_key] = value + return value + + +async def _metrics_history_refresh_loop( + session_id: str | None, + *, + agent_state: 'AgentState | None', + account_id: str | None, + interval_seconds: float = 3.0, +) -> None: + cache_key = _iter_count_cache_key(session_id, account_id) + while True: + try: + value = await asyncio.to_thread( + _read_iteration_log_metrics_history_uncached, + session_id, + agent_state=agent_state, + account_id=account_id, + ) + _METRICS_HISTORY_CACHE[cache_key] = value + except Exception: # noqa: BLE001 + pass + await asyncio.sleep(interval_seconds) + + +def _read_iteration_log_metrics_history_uncached( + session_id: str | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, +) -> dict[str, Any] | None: + """Parse iteration_log.jsonl into {'metrics': [keys], 'rounds': [...]}. + + Each entry's `results` dict contributes one round point; metric key union + is taken across all rounds (sorted by first-appearance for stable column + order). Missing values produce nulls so frontend line charts break the + series naturally.""" + runtime = ( + _jupyter_runtime_for_session(agent_state, account_id, session_id) + if session_id and agent_state is not None + else None + ) + text: str | None = None + if runtime is not None: + remote_root = _chat_root_for_runtime(runtime) + if remote_root is not None: + remote_path = f'{remote_root}/results/iteration_log.jsonl' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& cat {shlex.quote(remote_path)} ' + f'|| echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, timeout_seconds=10.0, max_output_chars=400_000, + ) + except Exception: # noqa: BLE001 + return None + stdout = (result.stdout or '').strip() if result is not None else '' + if stdout and stdout != '__MISSING__': + text = stdout + if text is None: + legacy = _autoresearch_root() / 'results' / 'iteration_log.jsonl' + try: + if legacy.is_file(): + text = legacy.read_text(encoding='utf-8', errors='replace') + except OSError: + return None + if not text: + return None + + metrics_order: list[str] = [] + metrics_seen: set[str] = set() + rounds: list[dict[str, Any]] = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except (ValueError, TypeError): + continue + if not isinstance(obj, dict): + continue + results = obj.get('results') + if not isinstance(results, dict): + continue + values: dict[str, float] = {} + for k, v in results.items(): + if not isinstance(k, str): + continue + try: + fv = float(v) + except (TypeError, ValueError): + continue + values[k] = fv + if k not in metrics_seen: + metrics_seen.add(k) + metrics_order.append(k) + if not values: + continue + iteration = obj.get('iteration') + if isinstance(iteration, str) and iteration.lstrip('-').isdigit(): + iteration = int(iteration) + if not isinstance(iteration, int): + iteration = len(rounds) + run_dic = obj.get('runDic') or obj.get('run_dic') + if isinstance(run_dic, str) and run_dic.isdigit(): + run_dic = int(run_dic) + if not isinstance(run_dic, int): + run_dic = 0 + timestamp = obj.get('timestamp') + if not isinstance(timestamp, str): + timestamp = '' + rounds.append({ + 'iteration': iteration, + 'runDic': run_dic, + 'label': f'R{iteration}', + 'timestamp': timestamp, + 'values': values, + }) + + if not rounds: + return None + return {'metrics': metrics_order, 'rounds': rounds} + + +def _read_iteration_log_h1_label( + runtime: 'JupyterRuntimeSession | None', + run_dic: int | None, + *, + max_chars: int = 32, +) -> str | None: + """读取 iteration_log.jsonl，提取与本轮 dist-analysis 候选集对应的 H1 假设短标签。 + + 优先匹配 runDic 一致的 entry，否则取最新一条；从 ``hypothesis`` / + ``next_hypothesis`` 字段切第一句并截断。读不到或无 iteration_log 返回 None + （调用方据此决定是否在标题里加括号）。 + """ + if runtime is None: + return None + chat_root = _chat_root_for_runtime(runtime) + if not chat_root: + return None + remote_path = f'{chat_root}/results/iteration_log.jsonl' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& cat {shlex.quote(remote_path)} ' + f'|| echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, timeout_seconds=10.0, max_output_chars=200_000 + ) + except Exception: # noqa: BLE001 + return None + stdout = (result.stdout or '').strip() if result is not None else '' + if not stdout or stdout == '__MISSING__': + return None + entries: list[dict[str, Any]] = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except (ValueError, TypeError): + continue + if isinstance(obj, dict): + entries.append(obj) + if not entries: + return None + chosen: dict[str, Any] | None = None + if run_dic is not None: + for entry in reversed(entries): + rd = entry.get('runDic') or entry.get('run_dic') + if isinstance(rd, str) and rd.isdigit(): + rd = int(rd) + if rd == run_dic: + chosen = entry + break + if chosen is None: + chosen = entries[-1] + for field in ('hypothesis', 'next_hypothesis'): + val = chosen.get(field) + if isinstance(val, str) and val.strip(): + label = val.strip() + for sep in ('；', ';', '。', '\n'): + if sep in label: + label = label.split(sep, 1)[0].strip() + break + if len(label) > max_chars: + label = label[: max_chars - 1] + '…' + return label or None + return None + + +def _extract_trigger_from_state(state_entries: list[dict[str, Any]]) -> str | None: + """Backwards compat: if agent wrote {kpi:'TRIGGER',value:...}, use that. + Backend now owns KPI rendering, but TRIGGER is a passthrough of the user's + own phrasing — fine to read from state.""" + last: str | None = None + for entry in state_entries: + if entry.get('kpi') == 'TRIGGER': + value = entry.get('value') + if isinstance(value, str) and value.strip(): + last = value + return last + + +def _extract_trigger_from_session( + sessions_dir: Path, session_id: str | None +) -> str | None: + """Return the user's first non-empty message that looks like a training + trigger. Used as raw input for _parse_target_set.""" + if not session_id: + return None + safe_id = _safe_session_id(session_id) + if safe_id is None: + return None + path = sessions_dir / safe_id / 'session.json' + try: + data = json.loads(path.read_text(encoding='utf-8')) + except (OSError, json.JSONDecodeError): + return None + messages = data.get('messages') + if not isinstance(messages, list): + return None + for msg in messages: + if not isinstance(msg, dict): + continue + if msg.get('role') != 'user': + continue + content = msg.get('content') + text = content if isinstance(content, str) else '' + if not text: + continue + # Strip blocks etc. + cleaned = re.sub(r'<[^>]+>', ' ', text).strip() + if not cleaned: + continue + if _looks_like_training_trigger(cleaned): + return cleaned + return None + + +def _looks_like_training_trigger(text: str) -> bool: + keywords = ( + '训练', + 'training', + '评测', + '迭代', + '开始', + '需求集合', + '目标集合', + '目标是', + '针对', + 'cml', + '复杂导航', + '.csv', + ) + lower = text.lower() + return any(k.lower() in lower for k in keywords) + + +_TARGET_SET_LLM_CACHE: dict[str, str | None] = {} + + +def _llm_extract_target_set( + model_config: ModelConfig | None, text: str +) -> str | None: + """Use the configured LLM to extract a target set name from a user message. + Cached by message hash so the same input doesn't repeatedly call the model.""" + if not text or not model_config: + return None + text = text.strip() + if len(text) > 4000: + text = text[:4000] + cache_key = hashlib.sha256(text.encode('utf-8')).hexdigest() + if cache_key in _TARGET_SET_LLM_CACHE: + return _TARGET_SET_LLM_CACHE[cache_key] + + # Tighten timeout for this short auxiliary call so SSE doesn't stall. + cfg = replace(model_config, timeout_seconds=min(20.0, model_config.timeout_seconds)) + try: + from src.openai_compat import OpenAICompatClient, OpenAICompatError + except ImportError: + _TARGET_SET_LLM_CACHE[cache_key] = None + return None + + client = OpenAICompatClient(cfg) + system_prompt = ( + '你是一个文本抽取器。任务：从用户的训练触发消息里抽出"目标需求集合"的名称。\n' + '- 名称可能是 CSV 文件名、目录名、或一个标识词\n' + '- 只返回名称字符串本身，不要任何前后缀、引号、说明、标点\n' + '- 没有明确目标时返回大写 NONE' + ) + few_shot = [ + {'role': 'user', 'content': '开始, icl_test'}, + {'role': 'assistant', 'content': 'icl_test'}, + {'role': 'user', 'content': '开始，复杂导航过召专项0511.csv'}, + {'role': 'assistant', 'content': '复杂导航过召专项0511.csv'}, + { + 'role': 'user', + 'content': '我要进行模型训练，目标是icl_test中的复杂导航过召专项0511.csv，基模在/mnt/zhangzhaowen/icl/v2/test/', + }, + {'role': 'assistant', 'content': '复杂导航过召专项0511.csv'}, + {'role': 'user', 'content': '针对 dapan_test 跑一轮 SFT'}, + {'role': 'assistant', 'content': 'dapan_test'}, + {'role': 'user', 'content': '你好'}, + {'role': 'assistant', 'content': 'NONE'}, + ] + messages: list[dict[str, Any]] = [{'role': 'system', 'content': system_prompt}] + messages.extend(few_shot) + messages.append({'role': 'user', 'content': text}) + + try: + result = client.complete(messages, tools=[]) + except OpenAICompatError: + _TARGET_SET_LLM_CACHE[cache_key] = None + return None + except Exception: # noqa: BLE001 — keep KPI render robust to upstream LLM hiccups + _TARGET_SET_LLM_CACHE[cache_key] = None + return None + + answer = (getattr(result, 'content', '') or '').strip().strip('"\'`，,。()（） ') + if not answer or answer.upper() == 'NONE' or len(answer) > 200: + _TARGET_SET_LLM_CACHE[cache_key] = None + return None + # Single-line: take only the first line in case model added explanation + answer = answer.split('\n', 1)[0].strip() + if not answer or answer.upper() == 'NONE': + _TARGET_SET_LLM_CACHE[cache_key] = None + return None + _TARGET_SET_LLM_CACHE[cache_key] = answer + return answer + + +def _parse_target_set( + trigger: str | None, model_config: ModelConfig | None = None +) -> str | None: + """Sync, non-blocking. Reads cache only — never makes the LLM call inline. + Cache is filled by the watcher scanner background loop via + _refresh_target_set_cache_blocking (which is run in a thread).""" + if not trigger: + return None + text = trigger.strip() + if not text: + return None + if len(text) > 4000: + text = text[:4000] + cache_key = hashlib.sha256(text.encode('utf-8')).hexdigest() + return _TARGET_SET_LLM_CACHE.get(cache_key) + + +def _refresh_target_set_cache_blocking( + model_config: ModelConfig | None, trigger: str +) -> None: + """Synchronous wrapper that calls the LLM and populates the cache. + Designed to be invoked via asyncio.to_thread from a background task.""" + if not model_config or not trigger: + return + text = trigger.strip() + if not text: + return + if len(text) > 4000: + text = text[:4000] + cache_key = hashlib.sha256(text.encode('utf-8')).hexdigest() + if cache_key in _TARGET_SET_LLM_CACHE: + return + _llm_extract_target_set(model_config, text) + + +_METRIC_DIFF_CACHE: dict[int, dict[str, Any]] = {} +_METRIC_DIFF_NEG_RETRY_AT: dict[int, float] = {} +_METRIC_DIFF_NEG_TTL_SECONDS = 30.0 + + +def _read_workflow_metric_diff( + run_id: int, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + session_id: str | None = None, +) -> dict[str, Any] | None: + """Read workflow/metric_diff/lark_template.json. + + metric_diff lives on /mnt/xiaoai-zk-model-train-tj5/... which is mounted on + the remote jupyter workspace but typically not on the backend host. So we + try local first, then fall back to `cat` over the session's jupyter runtime + if available. lark_template.json is written once by cml workflow and never + changes, so successful reads are cached forever; misses honor a 30s + negative cache to avoid hammering the remote during the cml-running phase. + """ + cached = _METRIC_DIFF_CACHE.get(run_id) + if cached is not None: + return cached + now = time.monotonic() + next_retry = _METRIC_DIFF_NEG_RETRY_AT.get(run_id) + if next_retry is not None and now < next_retry: + return None + relative = f'workflow{run_id}/metric_diff/lark_template.json' + local_path = _run_history_dir() / relative + try: + if local_path.is_file(): + data = json.loads(local_path.read_text(encoding='utf-8')) + if isinstance(data, dict): + _METRIC_DIFF_CACHE[run_id] = data + _METRIC_DIFF_NEG_RETRY_AT.pop(run_id, None) + return data + except (OSError, json.JSONDecodeError): + pass + if agent_state is not None and session_id is not None: + runtime = _jupyter_runtime_for_session(agent_state, account_id, session_id) + if runtime is not None: + remote_path = f'{_run_history_dir()}/{relative}' + cmd = ( + f'test -f {shlex.quote(remote_path)} ' + f'&& cat {shlex.quote(remote_path)} || echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=10.0, + max_output_chars=200_000, + ) + except Exception: # noqa: BLE001 + _METRIC_DIFF_NEG_RETRY_AT[run_id] = now + _METRIC_DIFF_NEG_TTL_SECONDS + return None + stdout = (result.stdout or '').strip() if result is not None else '' + if ( + result is not None + and result.exit_code == 0 + and stdout + and stdout != '__MISSING__' + ): + try: + data = json.loads(stdout) + if isinstance(data, dict): + _METRIC_DIFF_CACHE[run_id] = data + _METRIC_DIFF_NEG_RETRY_AT.pop(run_id, None) + return data + except json.JSONDecodeError: + pass + _METRIC_DIFF_NEG_RETRY_AT[run_id] = now + _METRIC_DIFF_NEG_TTL_SECONDS + return None + + +def _format_metric_value(metrics: dict[str, Any] | None, key: str) -> str: + if not metrics: + return '—' + value = metrics.get(key) + if value is None: + return '—' + if isinstance(value, (int, float)): + if 0 <= value <= 1: + return f'{value * 100:.2f}%' + return f'{value:.2f}' + return str(value) + + +# lark_template.json 的指标都嵌在 eval_output_info 文本里，行格式： +# 🟢 specific_test || icl_test: 86.11% (1873/2175) -> 86.11% (1873/2175) (+0.00%) +# 我们要 "->" 后面那个值（after-migration pass rate）。 +_EVAL_LINE_RE = re.compile( + r'^\s*[🟢🔴]\s*(?P.+?):\s*[\d.]+%\s*\([^)]+\)\s*->\s*(?P[\d.]+)%' +) + + +def _extract_eval_metric(eval_output_info: str | None, target_path: str) -> float | None: + """Find the line whose 'path' (the bit between emoji and ":") equals + target_path, return the post-value as a 0-100 float, or None. + + Match is exact-equality first, then falls back to path-tail equality — + eval lines often carry full relative paths like + `specific_test || data/specific_test_set/icl_test/.csv` + while target_path is just `specific_test || .csv`.""" + if not isinstance(eval_output_info, str) or not eval_output_info: + return None + target = target_path.strip() + # Strip the optional ` || ` so we can suffix-compare just the path tail. + target_tail = target.split('||', 1)[-1].strip() if '||' in target else target + for line in eval_output_info.splitlines(): + m = _EVAL_LINE_RE.match(line) + if not m: + continue + path = m.group('path').strip() + path_tail = path.split('||', 1)[-1].strip() if '||' in path else path + matched = path == target or path_tail == target_tail or path_tail.endswith( + f'/{target_tail}' + ) + if matched: + try: + return float(m.group('post')) + except ValueError: + return None + return None + + +def _enrich_metrics_from_eval_output( + metrics: dict[str, Any] | None, target_set: str | None +) -> None: + """Backfill structured pass-rate keys parsed out of eval_output_info text. + Mutates metrics in place. No-op if metrics is None or already has the keys.""" + if not metrics: + return + eval_text = metrics.get('eval_output_info') + if not isinstance(eval_text, str): + return + # _format_metric_value 把 0..1 当作 fraction 格式化为 XX.XX%，所以这里 /100 入库。 + # specific_test 大盘 + if metrics.get('specific_test_pass_rate') is None: + v = _extract_eval_metric(eval_text, 'specific_test') + if v is not None: + metrics['specific_test_pass_rate'] = v / 100.0 + # 大盘 overrall || 车载（注意 lark template 里就是 "overrall" 拼写，不是 "overall"） + if metrics.get('overall_car_pass_rate') is None: + v = _extract_eval_metric(eval_text, 'overrall || 车载') + if v is not None: + metrics['overall_car_pass_rate'] = v / 100.0 + # target set: specific_test || + # Trigger 里带 .csv 后缀；lark template 偶尔写不带后缀的。两种都试。 + if ( + metrics.get('target_set_pass_rate') is None + and target_set + and target_set != '—' + ): + candidates = [target_set] + if target_set.endswith('.csv'): + candidates.append(target_set[:-4]) + else: + candidates.append(f'{target_set}.csv') + for name in candidates: + v = _extract_eval_metric(eval_text, f'specific_test || {name}') + if v is not None: + metrics['target_set_pass_rate'] = v / 100.0 + break + + +def _compute_step0_start_seconds( + state_entries: list[dict[str, Any]], +) -> float | None: + """Earliest cml running ts seen in state file.""" + earliest: float | None = None + for entry in state_entries: + if entry.get('step') != 'cml': + continue + if entry.get('status') != 'running': + continue + ts = entry.get('ts') + if not isinstance(ts, str): + continue + try: + from datetime import datetime + dt = datetime.fromisoformat(ts.replace('Z', '+00:00')) + seconds = dt.timestamp() + except (ValueError, OSError): + continue + if earliest is None or seconds < earliest: + earliest = seconds + return earliest + + +def _format_elapsed(start_seconds: float | None) -> str: + if start_seconds is None: + return '00:00:00' + delta = max(0.0, time.time() - start_seconds) + total = int(delta) + h, rem = divmod(total, 3600) + m, s = divmod(rem, 60) + return f'{h:02d}:{m:02d}:{s:02d}' + + +def _compute_kpis( + sessions_dir: Path, + session_id: str | None, + state_entries: list[dict[str, Any]], + model_config: ModelConfig | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + iteration_log_count: int | None = None, + metrics_history: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Compute the 7 top KPIs from authoritative sources. Falls back to '—'. + + RUN ID prefers the runDic this session has already written into + program-state.jsonl — that's the only source guaranteed to belong to THIS + chat. Falls back to a global scan of RUN_HISTORY_DIR for sessions that + haven't logged a runDic yet (e.g. baseline pre-cml). Without this priority, + a concurrently-running chat that allocated a larger runDic on the shared + /mnt/xiaoai-zk-model-train-tj5/workflow5/ would shadow this chat's value. + Metric reads use the same jupyter runtime the watcher uses, so /mnt/* paths + reachable only on the remote workspace still work. + """ + run_max = _resolve_run_dic_from_state(state_entries) + if run_max is None: + run_max = _scan_run_history_max() + run_id_value = str(run_max) if run_max is not None else '—' + version = _read_workflow_version() or '—' + trigger = _extract_trigger_from_state(state_entries) or _extract_trigger_from_session( + sessions_dir, session_id + ) + target_set = _parse_target_set(trigger, model_config=model_config) or '—' + metrics = ( + _read_workflow_metric_diff( + run_max, + agent_state=agent_state, + account_id=account_id, + session_id=session_id, + ) + if run_max is not None + else None + ) + _enrich_metrics_from_eval_output(metrics, target_set) + target_metric = _format_metric_value(metrics, 'target_set_pass_rate') + overall_metric = _format_metric_value(metrics, 'overall_car_pass_rate') + specific_metric = _format_metric_value(metrics, 'specific_test_pass_rate') + # Round number = MAX numeric `iteration` field across iteration_log.jsonl + # entries, NOT the line count. iteration_log.jsonl carries multiple rows + # per round (separate hypothesis-only / results / final-summary entries), + # so line count overcounts. Falls back to line-count for backwards + # compatibility when metrics_history isn't available. + max_iteration: int | None = None + if metrics_history and isinstance(metrics_history.get('rounds'), list): + for r in metrics_history['rounds']: + it = r.get('iteration') if isinstance(r, dict) else None + if isinstance(it, int): + if max_iteration is None or it > max_iteration: + max_iteration = it + if max_iteration is not None: + iteration = 'R0-baseline' if max_iteration == 0 else f'R{max_iteration}' + else: + if iteration_log_count is not None: + iter_count = iteration_log_count + else: + iter_count = _read_iteration_log_count( + session_id, + agent_state=agent_state, + account_id=account_id, + ) + if iter_count is None or iter_count == 0: + iteration = 'R0-baseline' + else: + iteration = f'R{iter_count}' + elapsed = _format_elapsed(_compute_step0_start_seconds(state_entries)) + target_value = ( + f'{target_set} · {target_metric}' if target_metric != '—' else target_set + ) + return [ + {'label': 'RUN ID', 'value': run_id_value}, + {'label': 'VERSION', 'value': version}, + {'label': '目标集合', 'value': target_value}, + {'label': '大盘车载', 'value': overall_metric}, + {'label': 'SPECIFIC TEST', 'value': specific_metric}, + {'label': 'ITERATION', 'value': iteration}, + {'label': '耗时', 'value': elapsed, 'icon': 'clock'}, + ] + + +def _resolve_run_dic_from_state( + state_entries: list[dict[str, Any]], +) -> int | None: + """Extract runDic from program-state.jsonl. + + Priority: + 1. Explicit ``runDic`` field on any entry (latest wins). + 2. Fallback: scan string fields recursively for ``workflow`` / + ``runDic=N`` and pick the **max** match. Path strings often contain + both the global mount root (``/mnt/.../workflow5/...``) and the actual + run number (``workflow17793/...``) — max picks the run number, not the + fixed mount-dir digit. + """ + workflow_re = re.compile(r'workflow(\d+)') + rundic_re = re.compile(r'runDic\s*[=:取]?\s*(\d+)') + + def _scan_strings(obj: Any) -> int | None: + if isinstance(obj, str): + candidates = [int(x) for x in workflow_re.findall(obj)] + candidates += [int(x) for x in rundic_re.findall(obj)] + return max(candidates) if candidates else None + if isinstance(obj, dict): + best: int | None = None + for v in obj.values(): + hit = _scan_strings(v) + if hit is not None and (best is None or hit > best): + best = hit + return best + if isinstance(obj, list): + best = None + for v in obj: + hit = _scan_strings(v) + if hit is not None and (best is None or hit > best): + best = hit + return best + return None + + # 1. Explicit field — latest wins. + for entry in reversed(state_entries): + run_dic = entry.get('runDic') or entry.get('run_dic') or entry.get('rundic') + if isinstance(run_dic, int): + return run_dic + if isinstance(run_dic, str) and run_dic.isdigit(): + return int(run_dic) + + # 2. Fallback string scan — take max across all entries. + best: int | None = None + for entry in state_entries: + hit = _scan_strings(entry) + if hit is not None and (best is None or hit > best): + best = hit + return best + + +def _resolve_run_dic_per_round( + state_entries: list[dict[str, Any]], +) -> dict[str, int]: + """Map each run_id (e.g., 'R2') to the runDic that round actually used. + + Same priority as ``_resolve_run_dic_from_state`` but bucketed per run_id + so per-round step-detail views read the right artifact directory instead + of always using the latest workflow id seen anywhere in state.""" + workflow_re = re.compile(r'workflow(\d+)') + rundic_re = re.compile(r'runDic\s*[=:取]?\s*(\d+)') + + def _scan(obj: Any) -> int | None: + if isinstance(obj, str): + cands = [int(x) for x in workflow_re.findall(obj)] + cands += [int(x) for x in rundic_re.findall(obj)] + return max(cands) if cands else None + if isinstance(obj, dict): + best: int | None = None + for v in obj.values(): + hit = _scan(v) + if hit is not None and (best is None or hit > best): + best = hit + return best + if isinstance(obj, list): + best = None + for v in obj: + hit = _scan(v) + if hit is not None and (best is None or hit > best): + best = hit + return best + return None + + def _entry_run_id(entry: dict[str, Any]) -> str | None: + # 顶层 run_id 优先；否则从 watch.run_id / log.iter 兜底，让 watcher/log + # 这种 agent 不写顶层 run_id 但嵌套里带轮次信号的条目也能贡献 runDic。 + rid = entry.get('run_id') + if isinstance(rid, str) and rid.strip(): + return rid.strip() + watch = entry.get('watch') + if isinstance(watch, dict): + wrid = watch.get('run_id') + if isinstance(wrid, str) and wrid.strip(): + return wrid.strip() + log = entry.get('log') + if isinstance(log, dict): + lit = log.get('iter') + if isinstance(lit, str) and lit.strip(): + return lit.strip() + return None + + explicit: dict[str, int] = {} + fallback: dict[str, int] = {} + for entry in state_entries: + run_id = _entry_run_id(entry) + if not run_id: + continue + rd = entry.get('runDic') or entry.get('run_dic') or entry.get('rundic') + if isinstance(rd, str) and rd.isdigit(): + rd = int(rd) + if isinstance(rd, int): + explicit[run_id] = rd # latest wins + continue + hit = _scan(entry) + if hit is not None: + cur = fallback.get(run_id) + if cur is None or hit > cur: + fallback[run_id] = hit + + out = dict(fallback) + out.update(explicit) + return out + + +def _step_artifact_paths( + step: str, + run_dic: int | None, + runtime: 'JupyterRuntimeSession | None' = None, +) -> list[str]: + """Map step key → ordered list of remote artifact paths to try. + + Per-chat artifacts (results/, ai-planning/, output/) live under the chat + workspace root on shared NFS at + `/mnt/wangsenhao/autoresearch-zk-users///`. Reads still + go through the bound jupyter runtime since the path is only mounted there. + metric_diff (cml step) stays on the global RUN_HISTORY_DIR mount that is + shared across chats. When no runtime is bound, falls back to the legacy + global AUTORESEARCH_ROOT so KPI fetches don't crash on unbound sessions. + """ + chat_root = _chat_root_for_runtime(runtime) or str(_autoresearch_root()) + metric_root = os.environ.get( + 'RUN_HISTORY_DIR', '/mnt/xiaoai-zk-model-train-tj5/workflow5' + ) + paths: list[str] = [] + if step == 'cml': + if run_dic is not None: + paths.append( + f'{metric_root}/workflow{run_dic}/metric_diff/lark_template.json' + ) + paths.append( + f'{metric_root}/workflow{run_dic}/metric_diff/specific_comparison.csv' + ) + elif step == 'gold-drift': + if run_dic is not None: + paths.append(f'{chat_root}/results/gold_drift/drift_{run_dic}.json') + elif step == 'dist-analysis': + if run_dic is not None: + paths.append(f'{chat_root}/results/workflow{run_dic}.md') + paths.append( + f'{chat_root}/output/relabel_candidates_{run_dic}.csv' + ) + # 后向兼容：旧 agent 把 csv 写到 jupyter pod 本地 $(pwd)/output/ + if runtime is not None: + paths.append( + f'{runtime.binding.workspace_cwd}/output/relabel_candidates_{run_dic}.csv' + ) + elif step == 'hypothesis': + paths.append(f'{chat_root}/results/iteration_log.jsonl') + elif step == 'augment': + if run_dic is not None: + paths.append( + f'{chat_root}/results/data_clean_{run_dic}/modified_samples.jsonl' + ) + paths.append( + f'{chat_root}/results/augment_raw/augment_{run_dic}_raw.jsonl' + ) + paths.append( + f'{chat_root}/ai-planning/data/train_set/zk_intent/augment_{run_dic}.jsonl' + ) + paths.append( + f'{chat_root}/results/data_clean_{run_dic}/label_master_review.jsonl' + ) + elif step == 'verify': + paths.append(f'{chat_root}/results/iteration_log.jsonl') + elif step == 'sft': + paths.append(f'{chat_root}/results/iteration_log.jsonl') + elif step == 'log': + paths.append(f'{chat_root}/results/iteration_log.jsonl') + paths.append(f'{chat_root}/results/error_registry.jsonl') + elif step == 'next-round': + paths.append(f'{chat_root}/results/iteration_log.jsonl') + return paths + + +def _csv_to_table( + text: str, max_rows: int | None +) -> tuple[list[str], list[list[str]], int]: + """Parse CSV text into (columns, rows, total). Strips BOM, handles + quoted multi-line cells. ``max_rows=None`` means no row cap.""" + if text.startswith(''): + text = text[1:] + import io as _io + reader = csv.reader(_io.StringIO(text)) + columns: list[str] = [] + rows: list[list[str]] = [] + total = 0 + for i, record in enumerate(reader): + if i == 0: + columns = [c.strip() for c in record] + continue + # 跳过整行全空的视觉分隔行（agent 写人审 CSV 时常插空行） + if not any(cell.strip() for cell in record): + continue + total += 1 + if max_rows is None or len(rows) < max_rows: + row = [_truncate_cell(c) for c in record] + if len(row) < len(columns): + row += [''] * (len(columns) - len(row)) + elif len(row) > len(columns): + row = row[: len(columns)] + rows.append(row) + return columns, rows, total + + +def _jsonl_to_table( + text: str, max_rows: int | None +) -> tuple[list[str], list[list[str]], int]: + """Parse jsonl into (columns, rows, total). Column order is the order + keys appear in the first record, with new keys appended. + ``max_rows=None`` means no row cap.""" + columns: list[str] = [] + seen: set[str] = set() + sample_rows: list[dict[str, Any]] = [] + total = 0 + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + try: + obj = json.loads(stripped) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + total += 1 + for k in obj.keys(): + if k not in seen: + seen.add(k) + columns.append(k) + if max_rows is None or len(sample_rows) < max_rows: + sample_rows.append(obj) + rows: list[list[str]] = [] + for obj in sample_rows: + row: list[str] = [] + for col in columns: + val = obj.get(col) + if val is None: + row.append('') + elif isinstance(val, str): + row.append(_truncate_cell(val)) + elif isinstance(val, (int, float, bool)): + row.append(str(val)) + else: + row.append(_truncate_cell(json.dumps(val, ensure_ascii=False))) + rows.append(row) + return columns, rows, total + + +def _truncate_cell(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[:limit] + f'…（共 {len(text)} 字，已截断）' + + +def _detect_artifact_format(path: str) -> str: + lower = path.lower() + if lower.endswith('.md'): + return 'markdown' + if lower.endswith('.jsonl'): + return 'jsonl' + if lower.endswith('.json'): + return 'json' + if lower.endswith('.csv'): + return 'csv' + return 'text' + + +def _read_artifact( + runtime: 'JupyterRuntimeSession | None', remote_path: str +) -> tuple[str | None, str | None]: + """Read a remote artifact via Jupyter `cat`. Falls back to local Path read + if no remote runtime (dev / unbound sessions). Returns (content, error).""" + if runtime is not None: + cmd = ( + f'test -f {shlex.quote(remote_path)} && ' + f'cat {shlex.quote(remote_path)} || echo __MISSING__' + ) + try: + result = runtime.run_command( + cmd, + timeout_seconds=15.0, + max_output_chars=400_000, + ) + except Exception as exc: # noqa: BLE001 + return None, f'remote read failed: {exc}' + if not result: + return None, 'no result' + out = result.stdout or '' + # Trim our own marker if file missing + if out.strip().endswith('__MISSING__'): + return None, f'remote file not found: {remote_path}' + return out, None + try: + p = Path(remote_path) + if not p.is_file(): + return None, f'file not found: {remote_path}' + return p.read_text(encoding='utf-8', errors='replace'), None + except OSError as exc: + return None, f'local read failed: {exc}' + + +def _hardcoded_autoresearch_pipeline( + session_id: str | None, + sessions_dir: Path | None = None, + state_entries: list[dict[str, Any]] | None = None, + model_config: ModelConfig | None = None, + *, + agent_state: 'AgentState | None' = None, + account_id: str | None = None, + iteration_log_count: int | None = None, + metrics_history: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Canonical autoresearch model-iteration pipeline structure. + + Card / phase / log dynamics still come from program-state.jsonl, but the + 7 top KPIs are now backend-owned (read from skills/config.yaml, + iteration_log, run history dir, CML metric_diff, session state) — agent + cannot override them via state file. + """ + now_ms = int(time.time() * 1000) + if sessions_dir is not None: + kpis = _compute_kpis( + sessions_dir, + session_id, + state_entries or [], + model_config=model_config, + agent_state=agent_state, + account_id=account_id, + iteration_log_count=iteration_log_count, + metrics_history=metrics_history, + ) + else: + # Defensive default if caller didn't pass sessions_dir. + kpis = [ + {'label': 'RUN ID', 'value': '—'}, + {'label': 'VERSION', 'value': '—'}, + {'label': '目标集合', 'value': '—'}, + {'label': '大盘车载', 'value': '—'}, + {'label': 'SPECIFIC TEST', 'value': '—'}, + {'label': 'ITERATION', 'value': 'R0-baseline'}, + {'label': '耗时', 'value': '00:00:00', 'icon': 'clock'}, + ] + return { + 'session_id': session_id or '', + 'generated_at_ms': now_ms, + 'available': True, + 'status': {'mode': 'Model Iteration', 'state': 'pending'}, + 'kpis': kpis, + # items[] is heterogeneous: {type:'round',...} for R0·Baseline / + # R{n}·Train / R{n}·Analysis, and {type:'gate',...} for HiTL gates. + # Built dynamically from program-state.jsonl by _build_pipeline_items — + # sections only appear when the agent has touched at least one step + # in them. + 'items': [], + 'in_flight': None, + 'logs': [], + 'metrics_history': metrics_history, + } + + def _session_mtime(path: Path) -> float: try: return path.stat().st_mtime diff --git a/frontend/app/app/api/claw/auth/email-login/route.ts b/frontend/app/app/api/claw/auth/email-login/route.ts new file mode 100644 index 0000000..3bf5bf6 --- /dev/null +++ b/frontend/app/app/api/claw/auth/email-login/route.ts @@ -0,0 +1,20 @@ +import { loginByEmail } from "@/lib/claw-auth"; + +export async function POST(req: Request) { + try { + const payload = (await req.json()) as { + email?: string; + password?: string; + }; + const account = await loginByEmail( + payload.email ?? "", + payload.password ?? "", + ); + return Response.json({ account }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "登录失败" }, + { status: 400 }, + ); + } +} diff --git a/frontend/app/app/api/claw/auth/email-register/route.ts b/frontend/app/app/api/claw/auth/email-register/route.ts new file mode 100644 index 0000000..c708dec --- /dev/null +++ b/frontend/app/app/api/claw/auth/email-register/route.ts @@ -0,0 +1,20 @@ +import { registerByEmail } from "@/lib/claw-auth"; + +export async function POST(req: Request) { + try { + const payload = (await req.json()) as { + email?: string; + password?: string; + }; + const account = await registerByEmail( + payload.email ?? "", + payload.password ?? "", + ); + return Response.json({ account }); + } catch (err) { + return Response.json( + { error: err instanceof Error ? err.message : "注册失败" }, + { status: 400 }, + ); + } +} diff --git a/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/bind/route.ts b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/bind/route.ts new file mode 100644 index 0000000..2720a0c --- /dev/null +++ b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/bind/route.ts @@ -0,0 +1,49 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +type BindBody = { + session_id?: string; +}; + +export async function POST( + req: Request, + { params }: { params: Promise<{ workspaceId: string }> }, +) { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ error: "请先登录账号" }, { status: 401 }); + + const { workspaceId } = await params; + const payload = (await req.json()) as BindBody; + try { + const target = `${CLAW_API_URL}/api/jupyter/workspaces/${encodeURIComponent(workspaceId)}/bind`; + const response = await fetch(target, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + session_id: payload.session_id, + account_id: account.id, + }), + cache: "no-store", + }); + const body = await response.text(); + return new Response(body, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + }, + }); + } catch (err) { + return Response.json( + { + error: + err instanceof Error + ? `Unable to reach Claw backend: ${err.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/route.ts b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/route.ts new file mode 100644 index 0000000..51fd872 --- /dev/null +++ b/frontend/app/app/api/claw/jupyter/workspaces/[workspaceId]/route.ts @@ -0,0 +1,42 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ workspaceId: string }> }, +) { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ error: "请先登录账号" }, { status: 401 }); + + const { workspaceId } = await params; + try { + const target = new URL( + `${CLAW_API_URL}/api/jupyter/workspaces/${encodeURIComponent(workspaceId)}`, + ); + target.searchParams.set("account_id", account.id); + const response = await fetch(target, { + method: "DELETE", + cache: "no-store", + }); + const body = await response.text(); + return new Response(body, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + }, + }); + } catch (err) { + return Response.json( + { + error: + err instanceof Error + ? `Unable to reach Claw backend: ${err.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/api/claw/jupyter/workspaces/route.ts b/frontend/app/app/api/claw/jupyter/workspaces/route.ts new file mode 100644 index 0000000..1a1b248 --- /dev/null +++ b/frontend/app/app/api/claw/jupyter/workspaces/route.ts @@ -0,0 +1,33 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export async function GET() { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ error: "请先登录账号" }, { status: 401 }); + + try { + const target = new URL(`${CLAW_API_URL}/api/jupyter/workspaces`); + target.searchParams.set("account_id", account.id); + const response = await fetch(target, { cache: "no-store" }); + const body = await response.text(); + return new Response(body, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + }, + }); + } catch (err) { + return Response.json( + { + error: + err instanceof Error + ? `Unable to reach Claw backend: ${err.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/api/claw/training/pipeline/route.ts b/frontend/app/app/api/claw/training/pipeline/route.ts new file mode 100644 index 0000000..3bd10e8 --- /dev/null +++ b/frontend/app/app/api/claw/training/pipeline/route.ts @@ -0,0 +1,26 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export async function GET(request: Request) { + const account = await getCurrentAccount(); + if (!account) return Response.json({ error: "unauthorized" }, { status: 401 }); + + const incoming = new URL(request.url); + const sessionId = incoming.searchParams.get("session_id") ?? ""; + + const url = new URL(`${CLAW_API_URL}/api/training/pipeline`); + url.searchParams.set("account_id", account.id); + if (sessionId) url.searchParams.set("session_id", sessionId); + + const response = await fetch(url, { cache: "no-store" }); + const payload = await response.text(); + return new Response(payload, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + "cache-control": "no-store", + }, + }); +} diff --git a/frontend/app/app/api/claw/training/pipeline/stream/route.ts b/frontend/app/app/api/claw/training/pipeline/stream/route.ts new file mode 100644 index 0000000..5952f3c --- /dev/null +++ b/frontend/app/app/api/claw/training/pipeline/stream/route.ts @@ -0,0 +1,38 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export const dynamic = "force-dynamic"; + +export async function GET(request: Request) { + const account = await getCurrentAccount(); + if (!account) { + return new Response("unauthorized", { status: 401 }); + } + + const incoming = new URL(request.url); + const sessionId = incoming.searchParams.get("session_id") ?? ""; + + const url = new URL(`${CLAW_API_URL}/api/training/pipeline/stream`); + url.searchParams.set("account_id", account.id); + if (sessionId) url.searchParams.set("session_id", sessionId); + + const upstream = await fetch(url, { + cache: "no-store", + signal: request.signal, + }); + + if (!upstream.body) { + return new Response("upstream has no body", { status: 502 }); + } + + return new Response(upstream.body, { + status: upstream.status, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-store", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); +} diff --git a/frontend/app/app/api/claw/training/step-detail/route.ts b/frontend/app/app/api/claw/training/step-detail/route.ts new file mode 100644 index 0000000..1184b30 --- /dev/null +++ b/frontend/app/app/api/claw/training/step-detail/route.ts @@ -0,0 +1,36 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +export async function GET(request: Request) { + const account = await getCurrentAccount(); + if (!account) return Response.json({ error: "unauthorized" }, { status: 401 }); + + const incoming = new URL(request.url); + const sessionId = incoming.searchParams.get("session_id") ?? ""; + const step = incoming.searchParams.get("step") ?? ""; + const runId = incoming.searchParams.get("run_id"); + if (!sessionId || !step) { + return Response.json( + { error: "session_id and step are required" }, + { status: 400 }, + ); + } + + const url = new URL(`${CLAW_API_URL}/api/training/step-detail`); + url.searchParams.set("account_id", account.id); + url.searchParams.set("session_id", sessionId); + url.searchParams.set("step", step); + if (runId) url.searchParams.set("run_id", runId); + + const response = await fetch(url, { cache: "no-store" }); + const payload = await response.text(); + return new Response(payload, { + status: response.status, + headers: { + "content-type": + response.headers.get("content-type") ?? "application/json", + "cache-control": "no-store", + }, + }); +} diff --git a/frontend/app/app/assistant.tsx b/frontend/app/app/assistant.tsx index 20a0f31..e71e5f4 100644 --- a/frontend/app/app/assistant.tsx +++ b/frontend/app/app/assistant.tsx @@ -1,16 +1,22 @@ "use client"; import type { ExportedMessageRepository } from "@assistant-ui/core"; -import { AssistantRuntimeProvider } from "@assistant-ui/react"; +import { + AssistantRuntimeProvider, + useAui, + useAuiState, +} from "@assistant-ui/react"; import { AssistantChatTransport, useChatRuntime, } from "@assistant-ui/react-ai-sdk"; import type { UIMessage } from "ai"; -import { useCallback, useEffect, useMemo, useRef } from "react"; +import { PanelRightOpenIcon, XIcon } from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ActivityPanel, ActivityProvider, + useActivityPanel, } from "@/components/assistant-ui/activity-panel"; import { Thread } from "@/components/assistant-ui/thread"; import { @@ -18,18 +24,27 @@ import { toReplayRepository, } from "@/components/assistant-ui/thread-list"; import { ThreadListSidebar } from "@/components/assistant-ui/threadlist-sidebar"; +import { TrainingPipelinePanel } from "@/components/training/training-pipeline-panel"; +import { Button } from "@/components/ui/button"; import { SidebarInset, SidebarProvider, SidebarTrigger, } from "@/components/ui/sidebar"; import { + ACTIVE_SESSION_CHANGED_EVENT, + consumeFreshLocalId, readActiveSessionId, readPendingWorkspaceSessionId, writeActiveSessionId, } from "@/lib/claw-active-session"; import { ClawSessionReplayProvider } from "@/lib/claw-session-replay"; import { pushSessionUrl } from "@/lib/claw-session-url"; +import { cn } from "@/lib/utils"; +import { + useAppliedTraining, + useModelIterationEnabled, +} from "@/lib/use-training-mode"; import { ClawAccountGate, useClawAccount } from "./claw-account-gate"; type AssistantProps = { @@ -81,6 +96,7 @@ export const Assistant = ({ initialSessionId }: AssistantProps = {}) => { }); const replaySession = useCallback( (sessionId: string, repository: ExportedMessageRepository) => { + console.log("[replay-session] called", { sessionId }); writeActiveSessionId(sessionId); pushSessionUrl(sessionId); // 后端已经落盘/结束后，用回放结果接管当前会话。 @@ -153,12 +169,7 @@ export const Assistant = ({ initialSessionId }: AssistantProps = {}) => { onLogout={() => setAccount(null)} /> -
-
- -
- -
+ @@ -168,6 +179,363 @@ export const Assistant = ({ initialSessionId }: AssistantProps = {}) => { ); }; +// 共享 composer 草稿缓存里 newTask（无 sessionId）使用的 key。 +const NEWTASK_KEY = "__newtask__"; + +// AssistantWorkspace cache effect 通知 ImeComposerInput 直接更新 textarea +// 内容（绕过 store→local 间接路径，避免 switchToNewThread 时序问题）。 +export const COMPOSER_RESTORE_EVENT = "claw-composer-restore-draft"; + +// __LOCALID_xxx 是侧栏在 newTask 状态下点工作区时临时分配的 placeholder +// session id（让 jupyter bind 有 id 可传）。每个 LOCALID 拥有独立的 draft +// key，避免多个 LOCALID 会话在 cache 里互相覆盖。newTask → 首次生成 LOCALID +// 这一瞬的 textarea 闪空，由下面 save/restore effect 里的一次性继承处理。 +function getDraftKey(activeSessionId: string | null): string { + if (!activeSessionId) return NEWTASK_KEY; + return activeSessionId; +} + +// 用户在对话里表达「我要开始/进行模型训练（这一轮迭代）」时命中的关键短语。 +// 要求动词或专属触发词必须存在；避免 "什么是模型训练" 之类无意义匹配。 +const APPLY_INTENT_PATTERNS: RegExp[] = [ + // 触发短语：开始，xxx / 开始, xxx + /(?:^|\s|[，,。.!?])开始\s*[，,]\s*\S+/, + // 单纯训练动作：进行/开始/启动/开启/执行/发起/做 + (模型) + 训练 + /(?:进行|开始|启动|开启|执行|发起|做)\s*(?:模型)?\s*训练/i, + // 训练 + 目标集合 + /训练[\s\S]{0,60}?(?:目标集合|需求集合|specific[\s_-]?test|集合)/i, + // 「(模型)训练，目标是 X」「训练，基模 X」「训练 ... csv」之类自然表达 + /(?:^|[\s，,。.!?])(?:模型)?训练[\s，,][\s\S]{0,80}?(?:目标|基模|specific[\s_-]?test|\.csv)/i, + // 跟着 program.md 的旧表达保留 + /(?:应用|加载|载入|启用|装载|使用|跑|按照?|根据|用)\s*\S*program\.?md/i, + /program\.?md[\s\S]{0,120}?(?:训练|应用|启用|跑起来|跑一下)/i, + /apply\s+program(?:\.md)?/i, + /load\s+program(?:\.md)?/i, + /(?:start|begin|run)\s+(?:the\s+)?training/i, +]; + +function detectApplyIntent(text: string): boolean { + return APPLY_INTENT_PATTERNS.some((re) => re.test(text)); +} + +function AssistantWorkspace() { + const aui = useAui(); + const [activeSessionId, setActiveSessionId] = useState(() => + readActiveSessionId(), + ); + const skill = useModelIterationEnabled(); + const { applied, setApplied } = useAppliedTraining(activeSessionId); + const { close: closeActivityPanel } = useActivityPanel(); + const showPipeline = skill.enabled && applied; + const messages = useAuiState((s) => s.thread.messages); + const lastProcessedUserMsgId = useRef(null); + const pipelineContainerRef = useRef(null); + const lastPipelineYRef = useRef(null); + const [exitButtonVisible, setExitButtonVisible] = useState(false); + // Composer 是 per-thread 的，但本应用所有 backend session 都共用同一个 + // assistant-ui thread（从不调 switchToThread），因此 composer.text 跨 session + // 共享。在 activeSessionId 切换时手动 save+restore，给每个 session 一份 + // 独立的 draft。null sessionId（newTask）用 NEWTASK_KEY 落盘。 + const composerCacheRef = useRef>(new Map()); + const prevSessionKeyRef = useRef(getDraftKey(activeSessionId)); + + const handlePipelineMouseMove = useCallback( + (event: React.MouseEvent) => { + const node = pipelineContainerRef.current; + if (!node) return; + const rect = node.getBoundingClientRect(); + const relativeY = event.clientY - rect.top; + const inTopThird = relativeY >= 0 && relativeY <= rect.height / 3; + const lastY = lastPipelineYRef.current; + const movingUp = lastY !== null && event.clientY < lastY; + const movingDown = lastY !== null && event.clientY > lastY; + lastPipelineYRef.current = event.clientY; + setExitButtonVisible((prev) => { + if (!inTopThird) return false; + if (movingUp) return true; + if (movingDown) return false; + return prev; + }); + }, + [], + ); + + const handlePipelineMouseLeave = useCallback(() => { + lastPipelineYRef.current = null; + setExitButtonVisible(false); + }, []); + + useEffect(() => { + if (!showPipeline) { + lastPipelineYRef.current = null; + setExitButtonVisible(false); + } + }, [showPipeline]); + + useEffect(() => { + const update = () => { + const next = readActiveSessionId(); + console.log("[active-session] focus-update", { next }); + setActiveSessionId(next); + }; + const handleChanged = (event: Event) => { + const detail = (event as CustomEvent<{ sessionId?: string | null }>) + .detail; + const next = detail?.sessionId ?? readActiveSessionId(); + console.log("[active-session] event-changed", { + detailSessionId: detail?.sessionId, + next, + }); + setActiveSessionId(next); + }; + window.addEventListener(ACTIVE_SESSION_CHANGED_EVENT, handleChanged); + window.addEventListener("focus", update); + update(); + return () => { + window.removeEventListener(ACTIVE_SESSION_CHANGED_EVENT, handleChanged); + window.removeEventListener("focus", update); + }; + }, []); + + // 切 session 时保存当前 composer 草稿到旧 key、恢复新 key 的草稿。 + useEffect(() => { + const cache = composerCacheRef.current; + const nextKey = getDraftKey(activeSessionId); + const prevKey = prevSessionKeyRef.current; + console.log("[draft-cache] effect-fire", { + prevKey, + nextKey, + activeSessionId, + willSkip: prevKey === nextKey, + }); + if (prevKey === nextKey) return; + // 不能信任 aui store 的 composer.text——assistant-ui 内部 reducer 在 + // 切换 / isEditing 翻转时会让 store 与 textarea 脱钩（典型现象：textarea + // 显示着 "hello" 但 store 已经是 ""），后果是这里把空串存进 cache、 + // 用户的草稿被静默吞掉。直接读 DOM 拿 textarea 当前真实值。 + // EditComposer 用的是 aui-edit-composer-input，不会撞 selector。 + const composerEl = document.querySelector( + "textarea.aui-composer-input", + ); + const composerState = aui.composer().getState(); + const savingText = + composerEl?.value ?? composerState.text ?? ""; + cache.set(prevKey, savingText); + // newTask 里点工作区会**新生成**一个 LOCALID 并切过去，用户视角里 + // 这是同一份草稿的延续——把 newTask 的文本继承到 LOCALID 名下， + // 避免 textarea 闪空。consumeFreshLocalId 只对前端刚生成的 LOCALID + // 命中，点击侧栏老 LOCALID 不命中，确保历史会话之间互不串。 + const inheritFromNewTask = + prevKey === NEWTASK_KEY && consumeFreshLocalId(activeSessionId); + if (inheritFromNewTask) { + cache.set(nextKey, savingText); + } + const restored = cache.get(nextKey) ?? ""; + console.log("[draft-cache] do-switch", { + prevKey, + nextKey, + savingText, + savingTextSource: composerEl ? "dom" : "store", + storeText: composerState.text, + restored, + inheritFromNewTask, + }); + aui.composer().setText(restored); + window.dispatchEvent( + new CustomEvent(COMPOSER_RESTORE_EVENT, { + detail: { text: restored }, + }), + ); + console.log("[draft-cache] dispatched", { restored }); + prevSessionKeyRef.current = nextKey; + }, [activeSessionId, aui]); + + useEffect(() => { + closeActivityPanel(); + }, [showPipeline, closeActivityPanel]); + + // Skill 关掉时撤销 applied，避免重新启用就立刻弹开。 + useEffect(() => { + if (!skill.enabled && applied) { + setApplied(false); + } + }, [skill.enabled, applied, setApplied]); + + // 自动应用：skill 启用 + 当前 session 后端的 pipeline 已经有进展（任何 + // 卡 running 或 complete）→ applied=true。这样关 tab 重开 / 刷新 / 切回 + // 已在跑的会话都能自动恢复 monitor 面板，不依赖触发词。 + // 30 秒轮询一次，覆盖「页面打开时 pipeline 还没起、之后才被外部脚本启动」 + // 这类情形——一次性 fetch 容易错过。 + useEffect(() => { + if (!skill.enabled) return; + if (applied) return; + if (!activeSessionId) return; + let cancelled = false; + const pipelineUrl = `/api/claw/training/pipeline?session_id=${encodeURIComponent(activeSessionId)}`; + const runUrl = `/api/claw/runs/latest?session_id=${encodeURIComponent(activeSessionId)}`; + const tick = async () => { + try { + const res = await fetch(pipelineUrl, { cache: "no-store" }); + if (res.ok) { + const payload = await res.json(); + if (cancelled) return; + if (payload) { + const items = Array.isArray(payload.items) ? payload.items : []; + const hasItemProgress = items.some( + (it: { + type?: string; + status?: string; + cards?: Array<{ status?: string }>; + }) => { + if (it.type === "gate") { + return it.status === "running" || it.status === "complete"; + } + return ( + Array.isArray(it.cards) && + it.cards.some( + (c) => + c.status === "running" || c.status === "complete", + ) + ); + }, + ); + // items[] 走 phase-complete-only 过滤；R0 只有一张卡在跑时 + // items 是空的，进展信号在 in_flight 字段里——也算进展。 + const hasInFlight = Boolean(payload.in_flight); + if (hasItemProgress || hasInFlight) { + setApplied(true); + return; + } + } + } + } catch { + /* ignore, fall through to runs/latest fallback */ + } + // Pipeline 还没写出 items/in_flight（agent 处于 prep 阶段，比如在 watcher + // 里 sleep 等远端文件）时，pipeline 端点会返回 state=pending、items=[]， + // 但后端 run 其实已经在跑。直接看 run 状态兜底，免得用户起了训练但 + // monitor 面板等到第一个 phase 才弹。 + try { + const res = await fetch(runUrl, { cache: "no-store" }); + if (!res.ok) return; + const payload = await res.json(); + if (cancelled || !payload) return; + if (payload.status === "running" || payload.status === "queued") { + setApplied(true); + } + } catch { + /* ignore */ + } + }; + tick(); + const id = window.setInterval(tick, 30000); + return () => { + cancelled = true; + window.clearInterval(id); + }; + }, [skill.enabled, applied, activeSessionId, setApplied]); + + // 监听用户消息：skill 启用 + 检测到训练触发短语 → applied = true。 + useEffect(() => { + if (!skill.enabled) return; + if (applied) return; + if (!Array.isArray(messages) || messages.length === 0) return; + // 找最新一条 user 消息 + let latestUser: (typeof messages)[number] | null = null; + for (let i = messages.length - 1; i >= 0; i -= 1) { + const m = messages[i]; + if (m.role === "user") { + latestUser = m; + break; + } + } + if (!latestUser) return; + if (lastProcessedUserMsgId.current === latestUser.id) return; + lastProcessedUserMsgId.current = latestUser.id; + const text = collectUserMessageText(latestUser); + if (text && detectApplyIntent(text)) { + setApplied(true); + } + }, [messages, skill.enabled, applied, setApplied]); + + // 注意：用同一棵树 + 条件渲染，保证 和 在 + // pipeline 切换前后都保持挂载，否则它们会 remount， + // useJupyterWorkspaceStatus 的内部 state 会被清空，UI 上 Jupyter 工作区 + // indicator 会闪一下变成"切换工作区"。 + return ( +
+ {showPipeline ? ( +
+ + +
+ ) : null} +
+
+ +
+
+ + {showPipeline ? : null} +
+ ); +} + +function collectUserMessageText(message: { content?: unknown }): string { + const content = message.content; + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + const parts: string[] = []; + for (const part of content) { + if (typeof part === "string") { + parts.push(part); + } else if (part && typeof part === "object") { + const text = (part as { text?: unknown }).text; + if (typeof text === "string") parts.push(text); + } + } + return parts.join("\n"); +} + +function ActivityDrawerTrigger() { + const { open, openActivity } = useActivityPanel(); + if (open) return null; + return ( + + ); +} + function getLastSessionId(messages: UIMessage[]) { for (const message of [...messages].reverse()) { if (message.role !== "assistant") continue; diff --git a/frontend/app/app/claw-account-gate.tsx b/frontend/app/app/claw-account-gate.tsx index 502c68d..5d19396 100644 --- a/frontend/app/app/claw-account-gate.tsx +++ b/frontend/app/app/claw-account-gate.tsx @@ -36,36 +36,59 @@ export function useClawAccount() { return { account, isLoading, refresh, setAccount }; } +const XIAOMI_EMAIL_RE = /^[\w.-]+@xiaomi\.com$/i; + export function ClawAccountGate({ onAccount, }: { onAccount: (account: ClawAccount) => void; }) { const [mode, setMode] = useState<"login" | "register">("login"); - const [username, setUsername] = useState(""); + const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); const [error, setError] = useState(""); const [isSubmitting, setIsSubmitting] = useState(false); async function submit() { setError(""); + const trimmed = email.trim(); + if (!XIAOMI_EMAIL_RE.test(trimmed)) { + setError("请使用小米邮箱（@xiaomi.com）登录"); + return; + } + if (password.length < 6) { + setError("密码至少 6 位"); + return; + } setIsSubmitting(true); try { - const response = await fetch(`/api/claw/auth/${mode}`, { + const endpoint = + mode === "register" + ? "/api/claw/auth/email-register" + : "/api/claw/auth/email-login"; + const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ username, password }), + body: JSON.stringify({ email: trimmed, password }), }); const payload = (await response.json()) as { account?: ClawAccount; error?: string; }; if (!response.ok || !payload.account) { - throw new Error(payload.error ?? "账号操作失败"); + throw new Error( + payload.error ?? (mode === "register" ? "注册失败" : "登录失败"), + ); } onAccount(payload.account); } catch (err) { - setError(err instanceof Error ? err.message : "账号操作失败"); + setError( + err instanceof Error + ? err.message + : mode === "register" + ? "注册失败" + : "登录失败", + ); } finally { setIsSubmitting(false); } @@ -77,22 +100,25 @@ export function ClawAccountGate({

ZK Data Agent

- 中控数据开发平台会按账号隔离会话、上传文件和生成产物。 + {mode === "register" + ? "创建账号：使用小米邮箱注册，前缀作为 autoresearch 工作根目录。" + : "使用小米邮箱登录。会话/上传/产物按邮箱前缀和 chat 二级隔离。"}

setUsername(event.target.value)} + type="email" + placeholder="your_name@xiaomi.com" + value={email} + onChange={(event) => setEmail(event.target.value)} onKeyDown={(event) => { if (event.key === "Enter") submit(); }} /> setPassword(event.target.value)} onKeyDown={(event) => { @@ -101,7 +127,7 @@ export function ClawAccountGate({ /> {error ?

{error}

: null} + {!asDrawer ? ( + + ) : null}
{visibleItems.length === 0 ? ( @@ -274,12 +270,59 @@ export function ActivityPanel() { open={selectedId === item.id} onOpenChange={(nextOpen) => { if (nextOpen) openItem(item.id); + else if (selectedId === item.id) openActivity(); }} /> ))}
)}
+ + ); + + if (asDrawer) { + return ( + { + if (!next) close(); + }} + > + + + {isFilesMode ? "聊天中的文件" : "活动"} + + {isFilesMode ? ( + + ) : ( + activityBody + )} + + + ); + } + + if (isFilesMode) { + return ( + + ); + } + + return ( + ); } @@ -308,9 +351,11 @@ type SessionFilesPayload = { function SessionFilesPanel({ sessionId, onClose, + embedded = false, }: { sessionId: string | null; onClose: () => void; + embedded?: boolean; }) { const [payload, setPayload] = useState(null); const [status, setStatus] = useState("正在读取聊天中的文件..."); @@ -377,8 +422,8 @@ function SessionFilesPanel({ const outputFiles = payload?.output ?? []; const total = inputFiles.length + outputFiles.length; - return ( -