feat: add skill evaluation workbench

2026-07-23 19:38:25 +08:00
parent 9067914ecc
commit 5be9434ba3
8 changed files with 5232 additions and 1 deletions
@@ -47,6 +47,7 @@ from src.agent_types import (
 )
 from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills
 from src.data_agent_inputs import DataAgentInputError, load_input_sources
+from src.evaluation_runtime import EvaluationError, EvaluationRuntime
 from src.jupyter_runtime import (
    DEFAULT_JUPYTER_WORKSPACE_ROOT,
    JupyterRuntimeError,
@@ -586,6 +587,12 @@ class AgentState:
            self.model_config_for,
        )
        self.memory_manager.start()
+        self.evaluation_runtime = EvaluationRuntime(
+            root=self.session_directory.parent / 'evaluations',
+            cwd_for_account=lambda account_id: self.config_for(account_id).cwd,
+            model_config_for=self.model_config_for,
+            account_paths_for=self.account_paths,
+        )

    @property
    def cwd(self) -> Path:
@@ -1125,6 +1132,56 @@ class SkillSyncRequest(BaseModel):
    account_id: str | None = None


+class EvaluationDatasetCreateRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    name: str = ''
+    filename: str = Field(min_length=1)
+    content_base64: str | None = None
+    rows: list[dict[str, Any]] | None = None
+
+
+class EvaluationDatasetMappingRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    mapping: dict[str, Any]
+
+
+class EvaluationAnalyzeRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    query: str = Field(min_length=1, max_length=20_000)
+    skill_name: str = 'label-master'
+    model: str | None = None
+    history: list[dict[str, Any]] = Field(default_factory=list)
+    context: dict[str, Any] = Field(default_factory=dict)
+    domain: str = ''
+    request_id: str = ''
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class EvaluationExperimentCreateRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    dataset_id: str = Field(min_length=1)
+    name: str = ''
+    skill_name: str = 'label-master'
+    model: str | None = None
+    concurrency: int = Field(default=2, ge=1, le=8)
+
+
+class EvaluationExperimentActionRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+
+
+class EvaluationRetryRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    case_ids: list[str] | None = None
+    disagreements_only: bool = False
+
+
+class EvaluationReviewRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    review_status: str = Field(max_length=80)
+    review_note: str = Field(default='', max_length=4000)
+
+
 class SessionUpdate(BaseModel):
    title: str | None = Field(default=None, max_length=80)
    is_training: bool | None = None
@@ -1178,6 +1235,7 @@ def create_app(state: AgentState) -> FastAPI:
            scanner_task.cancel()
            _watcher_manager.cancel_all()
            _bash_bg_manager.cancel_all()
+            state.evaluation_runtime.shutdown()
            state.event_loop = None

    app = FastAPI(title='Claw Code GUI', version='1.0', lifespan=lifespan)
@@ -1468,6 +1526,222 @@ def create_app(state: AgentState) -> FastAPI:
        )
        return result

+    # ------------- Skill evaluations ---------------------------------------
+    @app.get('/api/evaluations/metadata')
+    async def evaluation_metadata(account_id: str) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.metadata(_safe_account_id(account_id))
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/analyze')
+    async def analyze_evaluation_case(
+        payload: EvaluationAnalyzeRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.analyze_case,
+                account_id=_safe_account_id(payload.account_id),
+                query=payload.query,
+                skill_name=payload.skill_name,
+                model=payload.model,
+                history=payload.history,
+                context=payload.context,
+                domain=payload.domain,
+                request_id=payload.request_id,
+                metadata=payload.metadata,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/datasets')
+    async def list_evaluation_datasets(account_id: str) -> list[dict[str, Any]]:
+        return state.evaluation_runtime.list_datasets(_safe_account_id(account_id))
+
+    @app.post('/api/evaluations/datasets')
+    async def create_evaluation_dataset(
+        payload: EvaluationDatasetCreateRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.create_dataset,
+                account_id=_safe_account_id(payload.account_id),
+                name=payload.name,
+                filename=payload.filename,
+                content_base64=payload.content_base64,
+                rows=payload.rows,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/datasets/{dataset_id}')
+    async def get_evaluation_dataset(
+        dataset_id: str,
+        account_id: str,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.get_dataset(
+                dataset_id,
+                _safe_account_id(account_id),
+                include_rows=False,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+    @app.patch('/api/evaluations/datasets/{dataset_id}/mapping')
+    async def update_evaluation_dataset_mapping(
+        dataset_id: str,
+        payload: EvaluationDatasetMappingRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.update_dataset_mapping(
+                dataset_id,
+                _safe_account_id(payload.account_id),
+                payload.mapping,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/experiments')
+    async def list_evaluation_experiments(
+        account_id: str,
+    ) -> list[dict[str, Any]]:
+        return state.evaluation_runtime.list_experiments(
+            _safe_account_id(account_id)
+        )
+
+    @app.post('/api/evaluations/experiments')
+    async def create_evaluation_experiment(
+        payload: EvaluationExperimentCreateRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.create_experiment,
+                account_id=_safe_account_id(payload.account_id),
+                dataset_id=payload.dataset_id,
+                name=payload.name,
+                skill_name=payload.skill_name,
+                model=payload.model,
+                concurrency=payload.concurrency,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/experiments/{experiment_id}')
+    async def get_evaluation_experiment(
+        experiment_id: str,
+        account_id: str,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.get_experiment(
+                experiment_id,
+                _safe_account_id(account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/start')
+    async def start_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.start(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/pause')
+    async def pause_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.pause(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/resume')
+    async def resume_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.resume(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/cancel')
+    async def cancel_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.cancel(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/retry')
+    async def retry_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationRetryRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.retry(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+                case_ids=payload.case_ids,
+                disagreements_only=payload.disagreements_only,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.patch('/api/evaluations/cases/{case_id}/review')
+    async def review_evaluation_case(
+        case_id: str,
+        payload: EvaluationReviewRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.update_review(
+                case_id,
+                _safe_account_id(payload.account_id),
+                review_status=payload.review_status,
+                review_note=payload.review_note,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/experiments/{experiment_id}/export')
+    async def export_evaluation_experiment(
+        experiment_id: str,
+        account_id: str,
+    ) -> Response:
+        try:
+            filename, content = state.evaluation_runtime.export_csv(
+                experiment_id,
+                _safe_account_id(account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+        return Response(
+            content,
+            media_type='text/csv; charset=utf-8',
+            headers={
+                'Content-Disposition': f"attachment; filename*=UTF-8''{quote(filename)}"
+            },
+        )
+
    # ------------- memory ----------------------------------------------------
    @app.get('/api/memory/user')
    async def get_user_memory(account_id: str) -> dict[str, Any]: