feat: add skill evaluation workbench

backend/api/server.py

@@ -47,6 +47,7 @@ from src.agent_types import (
 )
 from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills
 from src.data_agent_inputs import DataAgentInputError, load_input_sources
+from src.evaluation_runtime import EvaluationError, EvaluationRuntime
 from src.jupyter_runtime import (
     DEFAULT_JUPYTER_WORKSPACE_ROOT,
     JupyterRuntimeError,
@@ -586,6 +587,12 @@ class AgentState:
             self.model_config_for,
         )
         self.memory_manager.start()
+        self.evaluation_runtime = EvaluationRuntime(
+            root=self.session_directory.parent / 'evaluations',
+            cwd_for_account=lambda account_id: self.config_for(account_id).cwd,
+            model_config_for=self.model_config_for,
+            account_paths_for=self.account_paths,
+        )
 
     @property
     def cwd(self) -> Path:
@@ -1125,6 +1132,56 @@ class SkillSyncRequest(BaseModel):
     account_id: str | None = None
 
 
+class EvaluationDatasetCreateRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    name: str = ''
+    filename: str = Field(min_length=1)
+    content_base64: str | None = None
+    rows: list[dict[str, Any]] | None = None
+
+
+class EvaluationDatasetMappingRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    mapping: dict[str, Any]
+
+
+class EvaluationAnalyzeRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    query: str = Field(min_length=1, max_length=20_000)
+    skill_name: str = 'label-master'
+    model: str | None = None
+    history: list[dict[str, Any]] = Field(default_factory=list)
+    context: dict[str, Any] = Field(default_factory=dict)
+    domain: str = ''
+    request_id: str = ''
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class EvaluationExperimentCreateRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    dataset_id: str = Field(min_length=1)
+    name: str = ''
+    skill_name: str = 'label-master'
+    model: str | None = None
+    concurrency: int = Field(default=2, ge=1, le=8)
+
+
+class EvaluationExperimentActionRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+
+
+class EvaluationRetryRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    case_ids: list[str] | None = None
+    disagreements_only: bool = False
+
+
+class EvaluationReviewRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    review_status: str = Field(max_length=80)
+    review_note: str = Field(default='', max_length=4000)
+
+
 class SessionUpdate(BaseModel):
     title: str | None = Field(default=None, max_length=80)
     is_training: bool | None = None
@@ -1178,6 +1235,7 @@ def create_app(state: AgentState) -> FastAPI:
             scanner_task.cancel()
             _watcher_manager.cancel_all()
             _bash_bg_manager.cancel_all()
+            state.evaluation_runtime.shutdown()
             state.event_loop = None
 
     app = FastAPI(title='Claw Code GUI', version='1.0', lifespan=lifespan)
@@ -1468,6 +1526,222 @@ def create_app(state: AgentState) -> FastAPI:
         )
         return result
 
+    # ------------- Skill evaluations ---------------------------------------
+    @app.get('/api/evaluations/metadata')
+    async def evaluation_metadata(account_id: str) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.metadata(_safe_account_id(account_id))
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/analyze')
+    async def analyze_evaluation_case(
+        payload: EvaluationAnalyzeRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.analyze_case,
+                account_id=_safe_account_id(payload.account_id),
+                query=payload.query,
+                skill_name=payload.skill_name,
+                model=payload.model,
+                history=payload.history,
+                context=payload.context,
+                domain=payload.domain,
+                request_id=payload.request_id,
+                metadata=payload.metadata,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/datasets')
+    async def list_evaluation_datasets(account_id: str) -> list[dict[str, Any]]:
+        return state.evaluation_runtime.list_datasets(_safe_account_id(account_id))
+
+    @app.post('/api/evaluations/datasets')
+    async def create_evaluation_dataset(
+        payload: EvaluationDatasetCreateRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.create_dataset,
+                account_id=_safe_account_id(payload.account_id),
+                name=payload.name,
+                filename=payload.filename,
+                content_base64=payload.content_base64,
+                rows=payload.rows,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/datasets/{dataset_id}')
+    async def get_evaluation_dataset(
+        dataset_id: str,
+        account_id: str,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.get_dataset(
+                dataset_id,
+                _safe_account_id(account_id),
+                include_rows=False,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+    @app.patch('/api/evaluations/datasets/{dataset_id}/mapping')
+    async def update_evaluation_dataset_mapping(
+        dataset_id: str,
+        payload: EvaluationDatasetMappingRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.update_dataset_mapping(
+                dataset_id,
+                _safe_account_id(payload.account_id),
+                payload.mapping,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/experiments')
+    async def list_evaluation_experiments(
+        account_id: str,
+    ) -> list[dict[str, Any]]:
+        return state.evaluation_runtime.list_experiments(
+            _safe_account_id(account_id)
+        )
+
+    @app.post('/api/evaluations/experiments')
+    async def create_evaluation_experiment(
+        payload: EvaluationExperimentCreateRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.create_experiment,
+                account_id=_safe_account_id(payload.account_id),
+                dataset_id=payload.dataset_id,
+                name=payload.name,
+                skill_name=payload.skill_name,
+                model=payload.model,
+                concurrency=payload.concurrency,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/experiments/{experiment_id}')
+    async def get_evaluation_experiment(
+        experiment_id: str,
+        account_id: str,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.get_experiment(
+                experiment_id,
+                _safe_account_id(account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/start')
+    async def start_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.start(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/pause')
+    async def pause_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.pause(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/resume')
+    async def resume_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.resume(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/cancel')
+    async def cancel_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationExperimentActionRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.cancel(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.post('/api/evaluations/experiments/{experiment_id}/retry')
+    async def retry_evaluation_experiment(
+        experiment_id: str,
+        payload: EvaluationRetryRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.retry(
+                experiment_id,
+                _safe_account_id(payload.account_id),
+                case_ids=payload.case_ids,
+                disagreements_only=payload.disagreements_only,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.patch('/api/evaluations/cases/{case_id}/review')
+    async def review_evaluation_case(
+        case_id: str,
+        payload: EvaluationReviewRequest,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.update_review(
+                case_id,
+                _safe_account_id(payload.account_id),
+                review_status=payload.review_status,
+                review_note=payload.review_note,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @app.get('/api/evaluations/experiments/{experiment_id}/export')
+    async def export_evaluation_experiment(
+        experiment_id: str,
+        account_id: str,
+    ) -> Response:
+        try:
+            filename, content = state.evaluation_runtime.export_csv(
+                experiment_id,
+                _safe_account_id(account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+        return Response(
+            content,
+            media_type='text/csv; charset=utf-8',
+            headers={
+                'Content-Disposition': f"attachment; filename*=UTF-8''{quote(filename)}"
+            },
+        )
+
     # ------------- memory ----------------------------------------------------
     @app.get('/api/memory/user')
     async def get_user_memory(account_id: str) -> dict[str, Any]:

frontend/app/app/api/claw/evaluations/[...path]/route.ts

@@ -0,0 +1,78 @@
+import { getCurrentAccount } from "@/lib/claw-auth";
+
+const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765";
+
+type RouteContext = {
+	params: Promise<{ path: string[] }>;
+};
+
+export async function GET(request: Request, context: RouteContext) {
+	return proxyEvaluationRequest("GET", request, context);
+}
+
+export async function POST(request: Request, context: RouteContext) {
+	return proxyEvaluationRequest("POST", request, context);
+}
+
+export async function PATCH(request: Request, context: RouteContext) {
+	return proxyEvaluationRequest("PATCH", request, context);
+}
+
+async function proxyEvaluationRequest(
+	method: "GET" | "POST" | "PATCH",
+	request: Request,
+	context: RouteContext,
+) {
+	const account = await getCurrentAccount();
+	if (!account)
+		return Response.json({ detail: "unauthorized" }, { status: 401 });
+
+	const { path } = await context.params;
+	const incomingUrl = new URL(request.url);
+	const targetUrl = new URL(
+		`${CLAW_API_URL}/api/evaluations/${path.map(encodeURIComponent).join("/")}`,
+	);
+	for (const [key, value] of incomingUrl.searchParams.entries()) {
+		targetUrl.searchParams.append(key, value);
+	}
+
+	const headers: Record<string, string> = {};
+	let body: string | undefined;
+	if (method === "GET") {
+		targetUrl.searchParams.set("account_id", account.id);
+	} else {
+		const payload = (await request.json()) as Record<string, unknown>;
+		headers["content-type"] = "application/json";
+		body = JSON.stringify({ ...payload, account_id: account.id });
+	}
+
+	try {
+		const response = await fetch(targetUrl, {
+			method,
+			headers,
+			body,
+			cache: "no-store",
+		});
+		const responseHeaders = new Headers();
+		responseHeaders.set(
+			"content-type",
+			response.headers.get("content-type") ?? "application/json",
+		);
+		const disposition = response.headers.get("content-disposition");
+		if (disposition) responseHeaders.set("content-disposition", disposition);
+		return new Response(await response.arrayBuffer(), {
+			status: response.status,
+			headers: responseHeaders,
+		});
+	} catch (error) {
+		return Response.json(
+			{
+				detail:
+					error instanceof Error
+						? `Unable to reach Claw backend: ${error.message}`
+						: "Unable to reach Claw backend",
+			},
+			{ status: 502 },
+		);
+	}
+}

frontend/app/components/assistant-ui/thread-list.tsx

@@ -6,6 +6,7 @@ import {
 import { ThreadListPrimitive } from "@assistant-ui/react";
 import type { UIMessage } from "ai";
 import {
+	FlaskConicalIcon,
 	Loader2Icon,
 	MoreHorizontalIcon,
 	PencilIcon,
@@ -148,6 +149,7 @@ export const ThreadList: FC = () => {
 	return (
 		<ThreadListPrimitive.Root className="aui-root aui-thread-list-root flex min-h-0 flex-1 flex-col gap-1">
 			<ThreadListNew />
+			<ThreadListEvaluation />
 			<JupyterWorkspaceList />
 			<ClawSessionList />
 		</ThreadListPrimitive.Root>
@@ -174,6 +176,21 @@ const ThreadListNew: FC = () => {
 	);
 };
 
+const ThreadListEvaluation: FC = () => {
+	return (
+		<Button
+			variant="ghost"
+			className="h-9 w-full justify-start gap-2 rounded-lg px-3 text-sm"
+			asChild
+		>
+			<a href="/evaluations">
+				<FlaskConicalIcon className="size-4" />
+				Skill 评测
+			</a>
+		</Button>
+	);
+};
+
 type JupyterWorkspaceEntry = {
 	id: string;
 	label?: string;
@@ -1220,7 +1237,7 @@ function resolvePendingPrompt(
 			: "";
 	if (!pendingPrompt) return null;
 	for (const message of messages) {
-		if (!message || message.role !== "user") continue;
+		if (message?.role !== "user") continue;
 		const content = cleanStoredContent(message.content ?? "").trim();
 		if (!content || content.trimStart().startsWith("<system-reminder>"))
 			continue;

frontend/app/components/assistant-ui/threadlist-sidebar.tsx

@@ -5,6 +5,7 @@ import {
 	BrainIcon,
 	ChevronDownIcon,
 	Clock3Icon,
+	FlaskConicalIcon,
 	LogOutIcon,
 	PanelLeftOpenIcon,
 	SaveIcon,
@@ -215,6 +216,12 @@ function CollapsedSidebarRail({
 				>
 					<SquarePenIcon className="size-4" />
 				</CollapsedIconButton>
+				<CollapsedIconButton
+					label="Skill 评测"
+					onClick={() => window.location.assign("/evaluations")}
+				>
+					<FlaskConicalIcon className="size-4" />
+				</CollapsedIconButton>
 				<CollapsedSessionSearch />
 				<CollapsedRecentSessions />
 			</div>

tests/test_evaluation_api.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import base64
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from fastapi.testclient import TestClient
+
+from backend.api.server import AgentState, create_app
+
+
+def _build_state(root: Path) -> AgentState:
+    skill_dir = root / 'skills' / 'label-master'
+    manifest_dir = skill_dir / 'knowledge' / '索引'
+    manifest_dir.mkdir(parents=True)
+    (skill_dir / 'SKILL.md').write_text(
+        (
+            '---\n'
+            'name: label-master\n'
+            'description: Test label master.\n'
+            'allowed_tools: read_file\n'
+            '---\n'
+            'Use the label catalog.\n'
+        ),
+        encoding='utf-8',
+    )
+    (manifest_dir / 'label_manifest.json').write_text(
+        json.dumps(
+            {
+                'counts': {'labels': 1},
+                'labels': [
+                    {
+                        'name': '地图导航',
+                        'domain': '地图',
+                        'path': 'knowledge/标签/地图导航.md',
+                    }
+                ],
+                'agent_tags': ['地图导航'],
+                'functions': [],
+                'intents': [],
+            },
+            ensure_ascii=False,
+        ),
+        encoding='utf-8',
+    )
+    return AgentState(
+        cwd=root,
+        model='test-model',
+        base_url='http://127.0.0.1:8000/v1',
+        api_key='local-token',
+        timeout_seconds=30,
+        allow_shell=False,
+        allow_write=False,
+        session_directory=root / '.port_sessions' / 'agent',
+    )
+
+
+class EvaluationApiTests(unittest.TestCase):
+    def test_single_analysis_endpoint(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            state = _build_state(root)
+            with (
+                TestClient(create_app(state)) as client,
+                patch.object(
+                    state.evaluation_runtime,
+                    'analyze_case',
+                    return_value={
+                        'query': '导航去公司',
+                        'prediction': '地图导航',
+                        'status': 'completed',
+                    },
+                ) as analyze,
+            ):
+                response = client.post(
+                    '/api/evaluations/analyze',
+                    json={
+                        'account_id': 'alice',
+                        'query': '导航去公司',
+                        'skill_name': 'label-master',
+                        'model': 'test-model',
+                    },
+                )
+                self.assertEqual(response.status_code, 200)
+                self.assertEqual(response.json()['prediction'], '地图导航')
+                analyze.assert_called_once()
+
+    def test_dataset_experiment_and_export_flow(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            state = _build_state(root)
+            with TestClient(create_app(state)) as client:
+                metadata = client.get(
+                    '/api/evaluations/metadata',
+                    params={'account_id': 'alice'},
+                )
+                self.assertEqual(metadata.status_code, 200)
+                self.assertEqual(metadata.json()['default_skill'], 'label-master')
+
+                encoded = base64.b64encode(
+                    'query,label\n导航去公司,地图导航\n'.encode()
+                ).decode()
+                dataset_response = client.post(
+                    '/api/evaluations/datasets',
+                    json={
+                        'account_id': 'alice',
+                        'name': 'routing',
+                        'filename': 'routing.csv',
+                        'content_base64': encoded,
+                    },
+                )
+                self.assertEqual(dataset_response.status_code, 200)
+                dataset = dataset_response.json()
+                self.assertEqual(dataset['row_count'], 1)
+
+                experiment_response = client.post(
+                    '/api/evaluations/experiments',
+                    json={
+                        'account_id': 'alice',
+                        'dataset_id': dataset['id'],
+                        'name': 'routing test',
+                        'skill_name': 'label-master',
+                        'concurrency': 1,
+                    },
+                )
+                self.assertEqual(experiment_response.status_code, 200)
+                experiment = experiment_response.json()
+                self.assertEqual(experiment['status'], 'draft')
+                self.assertEqual(experiment['cases'][0]['query'], '导航去公司')
+
+                export = client.get(
+                    f"/api/evaluations/experiments/{experiment['id']}/export",
+                    params={'account_id': 'alice'},
+                )
+                self.assertEqual(export.status_code, 200)
+                self.assertIn('text/csv', export.headers['content-type'])
+                self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_evaluation_runtime.py

@@ -0,0 +1,502 @@
+from __future__ import annotations
+
+import base64
+import json
+import tempfile
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from src.agent_types import AgentRunResult, ModelConfig, UsageStats
+from src.bundled_skills import BundledSkill
+from src.evaluation_runtime import (
+    EvaluationRuntime,
+    build_evaluation_prompt,
+    calculate_metrics,
+    extract_accessed_refs,
+    labels_equivalent,
+    normalize_label_master_result,
+    normalize_evaluation_output,
+    parse_dataset_bytes,
+    prediction_is_allowed,
+)
+
+
+def _write_test_skill(root: Path) -> None:
+    skill_dir = root / 'skills' / 'label-master'
+    manifest_dir = skill_dir / 'knowledge' / '索引'
+    manifest_dir.mkdir(parents=True)
+    (skill_dir / 'SKILL.md').write_text(
+        (
+            '---\n'
+            'name: label-master\n'
+            'description: Test label skill.\n'
+            'allowed_tools: read_file, grep_search\n'
+            '---\n'
+            'Read the manifest and classify the query.\n'
+        ),
+        encoding='utf-8',
+    )
+    (manifest_dir / 'label_manifest.json').write_text(
+        json.dumps(
+            {
+                'counts': {'labels': 2, 'functions': 1},
+                'labels': [
+                    {
+                        'name': '地图导航',
+                        'domain': '地图',
+                        'path': 'knowledge/标签/地图导航.md',
+                    },
+                    {
+                        'name': 'QA',
+                        'domain': '问答',
+                        'path': 'knowledge/标签/QA.md',
+                    },
+                ],
+                'agent_tags': ['地图导航'],
+                'functions': [{'name': 'QA'}],
+                'intents': [],
+            },
+            ensure_ascii=False,
+        ),
+        encoding='utf-8',
+    )
+
+
+def _write_fast_slow_skill(root: Path) -> None:
+    skill_dir = root / 'skills' / 'label-fast-slow-routing'
+    skill_dir.mkdir(parents=True)
+    (skill_dir / 'SKILL.md').write_text(
+        (
+            '---\n'
+            'name: label-fast-slow-routing\n'
+            'description: Test fast slow routing skill.\n'
+            '---\n'
+            'Classify the query as 快、慢 or 模糊.\n'
+        ),
+        encoding='utf-8',
+    )
+
+
+class EvaluationRuntimeTests(unittest.TestCase):
+    def test_parse_csv_and_jsonl(self) -> None:
+        csv_rows, csv_format = parse_dataset_bytes(
+            'sample.csv',
+            'query,label\n打开地图,地图导航\n'.encode(),
+        )
+        self.assertEqual(csv_format, 'csv')
+        self.assertEqual(csv_rows[0]['query'], '打开地图')
+
+        jsonl_rows, jsonl_format = parse_dataset_bytes(
+            'sample.jsonl',
+            b'{"query":"hello","label":"QA"}\n',
+        )
+        self.assertEqual(jsonl_format, 'jsonl')
+        self.assertEqual(jsonl_rows[0]['label'], 'QA')
+
+    def test_dataset_mapping_snapshot_and_experiment(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            _write_test_skill(root)
+            runtime = EvaluationRuntime(
+                root=root / 'evaluations',
+                cwd_for_account=lambda _account: root,
+                model_config_for=lambda _account: ModelConfig(model='test-model'),
+                account_paths_for=lambda _account: {'python_env': root / '.venv'},
+                max_workers=1,
+            )
+            try:
+                metadata = runtime.metadata('alice')
+                self.assertEqual(metadata['default_skill'], 'label-master')
+                self.assertEqual(
+                    [item['name'] for item in metadata['label_catalog']['labels']],
+                    ['地图导航', 'QA'],
+                )
+
+                content = base64.b64encode(
+                    'rid,query,人工标签\n1,导航去公司,Agent(tag="地图导航")\n'.encode()
+                ).decode()
+                dataset = runtime.create_dataset(
+                    account_id='alice',
+                    name='test',
+                    filename='test.csv',
+                    content_base64=content,
+                )
+                self.assertEqual(dataset['mapping']['fields']['query'], 'query')
+                self.assertEqual(
+                    dataset['mapping']['fields']['gold_label'],
+                    '人工标签',
+                )
+                self.assertEqual(dataset['mapping']['fields']['request_id'], 'rid')
+
+                experiment = runtime.create_experiment(
+                    account_id='alice',
+                    dataset_id=dataset['id'],
+                    name='first',
+                )
+                self.assertEqual(experiment['total_cases'], 1)
+                self.assertEqual(experiment['skill_name'], 'label-master')
+                self.assertEqual(len(experiment['skill_version']), 12)
+                self.assertEqual(experiment['cases'][0]['request_id'], '1')
+            finally:
+                runtime.shutdown()
+
+    def test_fast_slow_skill_is_the_evaluation_default(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            _write_test_skill(root)
+            _write_fast_slow_skill(root)
+            runtime = EvaluationRuntime(
+                root=root / 'evaluations',
+                cwd_for_account=lambda _account: root,
+                model_config_for=lambda _account: ModelConfig(model='test-model'),
+                account_paths_for=lambda _account: {'python_env': root / '.venv'},
+                max_workers=1,
+            )
+            try:
+                metadata = runtime.metadata('alice')
+                self.assertEqual(
+                    metadata['default_skill'],
+                    'label-fast-slow-routing',
+                )
+                defaults = [
+                    item['name']
+                    for item in metadata['skills']
+                    if item['default']
+                ]
+                self.assertEqual(defaults, ['label-fast-slow-routing'])
+            finally:
+                runtime.shutdown()
+
+    def test_result_normalization_and_label_equivalence(self) -> None:
+        result = normalize_evaluation_output(
+            json.dumps(
+                {
+                    'prediction': '地图导航',
+                    'function_output': 'Agent(tag="地图导航")',
+                    'complex': False,
+                    'reason': '导航执行',
+                    'candidates': ['地图导航', '地图问答'],
+                    'evidence_refs': ['knowledge/标签/地图导航.md'],
+                    'confidence': 1.4,
+                    'uncertainty': '',
+                },
+                ensure_ascii=False,
+            )
+        )
+        self.assertEqual(result['prediction'], '地图导航')
+        self.assertEqual(result['confidence'], 1.0)
+        self.assertTrue(labels_equivalent('Agent(tag="地图导航")', '地图导航'))
+        self.assertTrue(
+            labels_equivalent(
+                'complex=false\nAgent(tag="地图导航")',
+                '地图导航',
+            )
+        )
+        snapshot_metadata = json.dumps(
+            {
+                'label_catalog': {
+                    'labels': ['地图导航'],
+                    'agent_tags': ['地图导航'],
+                    'functions': ['QA'],
+                    'intents': [],
+                }
+            },
+            ensure_ascii=False,
+        )
+        self.assertTrue(
+            prediction_is_allowed(
+                'label-master',
+                'Agent(tag="地图导航")',
+                snapshot_metadata,
+            )
+        )
+        self.assertFalse(
+            prediction_is_allowed(
+                'label-master',
+                '已完成第一步知识读取',
+                snapshot_metadata,
+            )
+        )
+        self.assertTrue(
+            prediction_is_allowed(
+                'label-fast-slow-routing',
+                '模糊',
+                '{}',
+            )
+        )
+        self.assertFalse(
+            prediction_is_allowed(
+                'label-fast-slow-routing',
+                'fast',
+                '{}',
+            )
+        )
+        normalized_label = normalize_label_master_result(
+            {
+                **result,
+                'prediction': 'Agent(tag="地图导航")',
+                'function_output': '一段解释文字',
+            },
+            snapshot_metadata,
+        )
+        self.assertEqual(normalized_label['prediction'], '地图导航')
+        self.assertEqual(
+            normalized_label['function_output'],
+            'Agent(tag="地图导航")',
+        )
+
+    def test_fast_slow_prompt_uses_skill_contract_and_relative_paths(self) -> None:
+        skill = BundledSkill(
+            name='label-fast-slow-routing',
+            description='Test fast slow routing skill.',
+            source='directory',
+            get_prompt=lambda _agent, _args: 'Read references/policy.md.',
+        )
+        prompt, runtime_context = build_evaluation_prompt(
+            skill=skill,
+            snapshot_root=Path('/tmp/evaluation-snapshot'),
+            canonical_case={'query': '打开空调', 'gold_label': '快'},
+        )
+        self.assertIn('填写“快”“慢”或“模糊”', prompt)
+        self.assertNotIn('填写 fast 或 slow', prompt)
+        self.assertIn('`references/...`', runtime_context)
+        self.assertNotIn(
+            '不要添加 `skills/label-fast-slow-routing/` 前缀',
+            prompt,
+        )
+
+    def test_single_analysis_uses_snapshot_without_dataset_records(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            _write_test_skill(root)
+            runtime = EvaluationRuntime(
+                root=root / 'evaluations',
+                cwd_for_account=lambda _account: root,
+                model_config_for=lambda _account: ModelConfig(model='test-model'),
+                account_paths_for=lambda _account: {'python_env': root / '.venv'},
+                max_workers=1,
+            )
+            fake_result = AgentRunResult(
+                final_output=json.dumps(
+                    {
+                        'prediction': '地图导航',
+                        'function_output': 'Agent(tag="地图导航")',
+                        'complex': False,
+                        'reason': '导航执行',
+                        'candidates': ['地图导航'],
+                        'evidence_refs': [
+                            'knowledge/索引/label_manifest.json'
+                        ],
+                        'confidence': 0.98,
+                        'uncertainty': '',
+                    },
+                    ensure_ascii=False,
+                ),
+                turns=1,
+                tool_calls=1,
+                transcript=(
+                    {
+                        'role': 'assistant',
+                        'tool_calls': [
+                            {
+                                'function': {
+                                    'name': 'read_file',
+                                    'arguments': json.dumps(
+                                        {
+                                            'path': (
+                                                'knowledge/索引/'
+                                                'label_manifest.json'
+                                            )
+                                        },
+                                        ensure_ascii=False,
+                                    ),
+                                }
+                            }
+                        ],
+                    },
+                ),
+                usage=UsageStats(input_tokens=10, output_tokens=5),
+            )
+            try:
+                with patch(
+                    'src.evaluation_runtime.LocalCodingAgent.run',
+                    return_value=fake_result,
+                ):
+                    result = runtime.analyze_case(
+                        account_id='alice',
+                        query='导航去公司',
+                    )
+                self.assertEqual(result['prediction'], '地图导航')
+                self.assertEqual(result['skill_name'], 'label-master')
+                self.assertEqual(result['model'], 'test-model')
+                self.assertEqual(runtime.list_datasets('alice'), [])
+                self.assertEqual(runtime.list_experiments('alice'), [])
+            finally:
+                runtime.shutdown()
+
+    def test_background_runner_completes_cases(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            _write_test_skill(root)
+            runtime = EvaluationRuntime(
+                root=root / 'evaluations',
+                cwd_for_account=lambda _account: root,
+                model_config_for=lambda _account: ModelConfig(model='test-model'),
+                account_paths_for=lambda _account: {'python_env': root / '.venv'},
+                max_workers=1,
+            )
+            try:
+                dataset = runtime.create_dataset(
+                    account_id='alice',
+                    name='test',
+                    filename='test.csv',
+                    rows=[
+                        {'query': '导航去公司', 'label': '地图导航'},
+                        {'query': '导航回家', 'label': '地图导航'},
+                    ],
+                )
+                experiment = runtime.create_experiment(
+                    account_id='alice',
+                    dataset_id=dataset['id'],
+                    name='runner',
+                    concurrency=1,
+                )
+                fake_result = AgentRunResult(
+                    final_output=json.dumps(
+                        {
+                            'prediction': '地图导航',
+                            'function_output': 'Agent(tag="地图导航")',
+                            'complex': False,
+                            'reason': '导航执行',
+                            'candidates': ['地图导航'],
+                            'evidence_refs': ['knowledge/标签/地图导航.md'],
+                            'confidence': 0.98,
+                            'uncertainty': '',
+                        },
+                        ensure_ascii=False,
+                    ),
+                    turns=1,
+                    tool_calls=1,
+                    transcript=(
+                        {
+                            'role': 'assistant',
+                            'tool_calls': [
+                                {
+                                    'function': {
+                                        'name': 'read_file',
+                                        'arguments': json.dumps(
+                                            {
+                                                'path': (
+                                                    'knowledge/索引/'
+                                                    'label_manifest.json'
+                                                )
+                                            },
+                                            ensure_ascii=False,
+                                        ),
+                                    }
+                                }
+                            ],
+                        },
+                    ),
+                    usage=UsageStats(input_tokens=10, output_tokens=5),
+                )
+                with patch(
+                    'src.evaluation_runtime.LocalCodingAgent.run',
+                    return_value=fake_result,
+                ):
+                    runtime.start(experiment['id'], 'alice')
+                    deadline = time.time() + 3
+                    current = runtime.get_experiment(experiment['id'], 'alice')
+                    while (
+                        current['status'] not in {'completed', 'completed_with_errors'}
+                        and time.time() < deadline
+                    ):
+                        time.sleep(0.02)
+                        current = runtime.get_experiment(
+                            experiment['id'],
+                            'alice',
+                        )
+                self.assertEqual(current['status'], 'completed')
+                self.assertEqual(current['completed_cases'], 2)
+                self.assertEqual(current['metrics']['accuracy'], 1.0)
+            finally:
+                runtime.shutdown()
+
+    def test_accessed_refs_only_include_snapshot_paths(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            snapshot_root = Path(directory)
+            evidence = snapshot_root / 'skills' / 'label-master' / 'knowledge.md'
+            evidence.parent.mkdir(parents=True)
+            evidence.write_text('evidence', encoding='utf-8')
+            transcript = (
+                {
+                    'role': 'assistant',
+                    'tool_calls': [
+                        {
+                            'function': {
+                                'name': 'read_file',
+                                'arguments': json.dumps(
+                                    {
+                                        'path': (
+                                            'skills/label-master/knowledge.md'
+                                        )
+                                    }
+                                ),
+                            }
+                        },
+                        {
+                            'function': {
+                                'name': 'read_file',
+                                'arguments': json.dumps({'path': '/etc/hosts'}),
+                            }
+                        },
+                    ],
+                },
+            )
+            self.assertEqual(
+                extract_accessed_refs(
+                    transcript,
+                    snapshot_root=snapshot_root,
+                ),
+                [
+                    {
+                        'tool': 'read_file',
+                        'path': 'skills/label-master/knowledge.md',
+                    }
+                ],
+            )
+
+    def test_metrics_count_disagreements(self) -> None:
+        metrics = calculate_metrics(
+            [
+                {
+                    'status': 'completed',
+                    'gold_label': 'A',
+                    'prediction': 'A',
+                    'correct': True,
+                    'confidence': 0.9,
+                    'evidence_refs': ['a.md'],
+                    'domain': 'one',
+                },
+                {
+                    'status': 'completed',
+                    'gold_label': 'A',
+                    'prediction': 'B',
+                    'correct': False,
+                    'confidence': 0.5,
+                    'evidence_refs': [],
+                    'domain': 'one',
+                },
+            ]
+        )
+        self.assertEqual(metrics['labeled'], 2)
+        self.assertEqual(metrics['disagreements'], 1)
+        self.assertEqual(metrics['accuracy'], 0.5)
+        self.assertEqual(metrics['low_confidence'], 1)
+        self.assertEqual(metrics['no_evidence'], 1)
+
+
+if __name__ == '__main__':
+    unittest.main()