feat: persist single evaluation history

2026-07-23 19:46:15 +08:00
parent 5be9434ba3
commit 6116d10e8f
5 changed files with 398 additions and 13 deletions
@@ -1554,6 +1554,27 @@ def create_app(state: AgentState) -> FastAPI:
        except EvaluationError as exc:
            raise HTTPException(status_code=400, detail=str(exc)) from exc

+    @app.get('/api/evaluations/analyses')
+    async def list_evaluation_analyses(
+        account_id: str,
+    ) -> list[dict[str, Any]]:
+        return state.evaluation_runtime.list_single_analyses(
+            _safe_account_id(account_id)
+        )
+
+    @app.get('/api/evaluations/analyses/{analysis_id}')
+    async def get_evaluation_analysis(
+        analysis_id: str,
+        account_id: str,
+    ) -> dict[str, Any]:
+        try:
+            return state.evaluation_runtime.get_single_analysis(
+                analysis_id,
+                _safe_account_id(account_id),
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
    @app.get('/api/evaluations/datasets')
    async def list_evaluation_datasets(account_id: str) -> list[dict[str, Any]]:
        return state.evaluation_runtime.list_datasets(_safe_account_id(account_id))
@@ -11,6 +11,7 @@ import {
 	LoaderCircleIcon,
 	PauseIcon,
 	PlayIcon,
+	PlusIcon,
 	RefreshCwIcon,
 	RotateCcwIcon,
 	SearchIcon,
@@ -122,6 +123,8 @@ type EvaluationCase = {
 	skill_name?: string;
 	skill_version?: string;
 	model?: string;
+	created_at?: number;
+	updated_at?: number;
 };

 type Experiment = {
@@ -213,6 +216,8 @@ function EvaluationWorkspace() {
 	const [selectedDatasetId, setSelectedDatasetId] = useState("");
 	const [selectedExperimentId, setSelectedExperimentId] = useState("");
 	const [experiment, setExperiment] = useState<Experiment | null>(null);
+	const [singleAnalyses, setSingleAnalyses] = useState<EvaluationCase[]>([]);
+	const [selectedSingleAnalysisId, setSelectedSingleAnalysisId] = useState("");
 	const [mapping, setMapping] = useState<DatasetMapping>({
 		fields: {},
 		label_map: {},
@@ -233,6 +238,7 @@ function EvaluationWorkspace() {
 	const uploadRef = useRef<HTMLInputElement>(null);
 	const selectedDatasetIdRef = useRef("");
 	const selectedExperimentIdRef = useRef("");
+	const selectedSingleAnalysisIdRef = useRef("");

 	useEffect(() => {
 		selectedDatasetIdRef.current = selectedDatasetId;
@@ -242,6 +248,10 @@ function EvaluationWorkspace() {
 		selectedExperimentIdRef.current = selectedExperimentId;
 	}, [selectedExperimentId]);

+	useEffect(() => {
+		selectedSingleAnalysisIdRef.current = selectedSingleAnalysisId;
+	}, [selectedSingleAnalysisId]);
+
 	const selectedDataset = useMemo(
 		() => datasets.find((item) => item.id === selectedDatasetId) ?? null,
 		[datasets, selectedDatasetId],
@@ -264,12 +274,14 @@ function EvaluationWorkspace() {
 		try {
 			const [
 				metadataResponse,
+				analysisResponse,
 				datasetResponse,
 				experimentResponse,
 				modelResponse,
 				stateResponse,
 			] = await Promise.all([
 				fetch("/api/claw/evaluations/metadata", { cache: "no-store" }),
+				fetch("/api/claw/evaluations/analyses", { cache: "no-store" }),
 				fetch("/api/claw/evaluations/datasets", { cache: "no-store" }),
 				fetch("/api/claw/evaluations/experiments", { cache: "no-store" }),
 				fetch("/api/claw/models", { cache: "no-store" }),
@@ -277,6 +289,8 @@ function EvaluationWorkspace() {
 			]);
 			const metadataPayload =
 				await readJson<EvaluationMetadata>(metadataResponse);
+			const analysisPayload =
+				await readJson<EvaluationCase[]>(analysisResponse);
 			const datasetPayload = await readJson<Dataset[]>(datasetResponse);
 			const experimentPayload =
 				await readJson<Experiment[]>(experimentResponse);
@@ -287,11 +301,21 @@ function EvaluationWorkspace() {
 				? ((await stateResponse.json()) as ClawState)
 				: {};
 			setMetadata(metadataPayload);
+			setSingleAnalyses(analysisPayload);
 			setDatasets(datasetPayload);
 			setExperiments(experimentPayload);
 			setSkillName(
 				(current) => current || metadataPayload.default_skill || "label-master",
 			);
+			const nextSingleAnalysis =
+				analysisPayload.find(
+					(item) => item.id === selectedSingleAnalysisIdRef.current,
+				) ?? analysisPayload[0];
+			if (nextSingleAnalysis) {
+				setSelectedSingleAnalysisId(nextSingleAnalysis.id);
+				setSingleResult((current) => current ?? nextSingleAnalysis);
+				setSingleQuery((current) => current || nextSingleAnalysis.query);
+			}
 			const nextModels = modelPayload.models ?? [];
 			setModels(nextModels);
 			setModel((current) => {
@@ -454,7 +478,13 @@ function EvaluationWorkspace() {
 					model: model || undefined,
 				}),
 			});
-			setSingleResult(await readJson<EvaluationCase>(response));
+			const payload = await readJson<EvaluationCase>(response);
+			setSingleResult(payload);
+			setSelectedSingleAnalysisId(payload.id);
+			setSingleAnalyses((current) => [
+				payload,
+				...current.filter((item) => item.id !== payload.id),
+			]);
 		} catch (reason) {
 			setError(messageFrom(reason));
 		} finally {
@@ -462,6 +492,35 @@ function EvaluationWorkspace() {
 		}
 	}

+	async function selectSingleAnalysis(analysisId: string) {
+		setSelectedSingleAnalysisId(analysisId);
+		setBusy("single-history");
+		setError("");
+		try {
+			const response = await fetch(
+				`/api/claw/evaluations/analyses/${encodeURIComponent(analysisId)}`,
+				{ cache: "no-store" },
+			);
+			const payload = await readJson<EvaluationCase>(response);
+			setSingleResult(payload);
+			setSingleQuery(payload.query);
+			setSingleAnalyses((current) =>
+				current.map((item) => (item.id === payload.id ? payload : item)),
+			);
+		} catch (reason) {
+			setError(messageFrom(reason));
+		} finally {
+			setBusy("");
+		}
+	}
+
+	function newSingleAnalysis() {
+		setSelectedSingleAnalysisId("");
+		setSingleResult(null);
+		setSingleQuery("");
+		setError("");
+	}
+
 	async function act(action: "start" | "pause" | "resume" | "cancel") {
 		if (!experiment) return;
 		setBusy(action);
@@ -617,7 +676,12 @@ function EvaluationWorkspace() {
 								))}
 							</select>
 						</label>
-						{workspaceMode === "batch" ? (
+						{workspaceMode === "single" ? (
+							<Button size="sm" onClick={newSingleAnalysis}>
+								<PlusIcon />
+								新建单条分析
+							</Button>
+						) : (
 							<Button
 								size="sm"
 								onClick={() => {
@@ -627,7 +691,7 @@ function EvaluationWorkspace() {
 							>
 								新建批量评测
 							</Button>
-						) : null}
+						)}
 					</div>
 				</div>
 			</div>
@@ -640,13 +704,60 @@ function EvaluationWorkspace() {
 			) : null}

 			{workspaceMode === "single" ? (
-				<SingleQueryWorkspace
-					query={singleQuery}
-					setQuery={setSingleQuery}
-					result={singleResult}
-					onAnalyze={analyzeSingleQuery}
-					busy={busy}
-				/>
+				<div className="grid min-h-[calc(100dvh-6.5rem)] lg:grid-cols-[260px_minmax(0,1fr)]">
+					<aside className="border-r bg-muted/15 p-3">
+						<div className="mb-2 flex items-center justify-between px-2">
+							<span className="font-medium text-xs">单条分析记录</span>
+							<span className="text-muted-foreground text-xs">
+								{singleAnalyses.length}
+							</span>
+						</div>
+						<div className="space-y-1">
+							{singleAnalyses.map((item) => (
+								<button
+									type="button"
+									key={item.id}
+									className={`w-full rounded-md px-2.5 py-2 text-left transition-colors ${
+										selectedSingleAnalysisId === item.id
+											? "bg-accent text-accent-foreground"
+											: "hover:bg-accent/60"
+									}`}
+									onClick={() => void selectSingleAnalysis(item.id)}
+								>
+									<div className="flex items-center gap-2">
+										<StatusDot status={item.status} />
+										<span className="min-w-0 flex-1 truncate font-medium text-sm">
+											{item.query}
+										</span>
+										<ChevronRightIcon className="size-3.5 shrink-0 text-muted-foreground" />
+									</div>
+									<div className="mt-1 flex items-center justify-between gap-2 pl-5 text-muted-foreground text-xs">
+										<span className="truncate">
+											{item.prediction || item.skill_name || "分析失败"}
+										</span>
+										<span className="shrink-0">
+											{formatTimestamp(item.created_at)}
+										</span>
+									</div>
+								</button>
+							))}
+							{!singleAnalyses.length ? (
+								<div className="px-2 py-8 text-center text-muted-foreground text-xs">
+									暂无单条分析
+								</div>
+							) : null}
+						</div>
+					</aside>
+					<section className="min-w-0">
+						<SingleQueryWorkspace
+							query={singleQuery}
+							setQuery={setSingleQuery}
+							result={singleResult}
+							onAnalyze={analyzeSingleQuery}
+							busy={busy}
+						/>
+					</section>
+				</div>
 			) : (
 				<div className="grid min-h-[calc(100dvh-6.5rem)] lg:grid-cols-[260px_minmax(0,1fr)]">
 					<aside className="border-r bg-muted/15 p-3">
@@ -1752,6 +1863,23 @@ function formatDuration(value: number | null) {
 	return `${(value / 1000).toFixed(value < 10_000 ? 1 : 0)}s`;
 }

+function formatTimestamp(value?: number) {
+	if (!value) return "";
+	const date = new Date(value * 1000);
+	const now = new Date();
+	if (date.toDateString() === now.toDateString()) {
+		return date.toLocaleTimeString("zh-CN", {
+			hour: "2-digit",
+			minute: "2-digit",
+			hour12: false,
+		});
+	}
+	return date.toLocaleDateString("zh-CN", {
+		month: "2-digit",
+		day: "2-digit",
+	});
+}
+
 function buildMappedPreview(
 	rows: Array<Record<string, unknown>>,
 	mapping: DatasetMapping,
@@ -205,10 +205,27 @@ class EvaluationStore:
                    FOREIGN KEY(experiment_id) REFERENCES evaluation_experiments(id)
                );

+                CREATE TABLE IF NOT EXISTS evaluation_single_analyses (
+                    id TEXT PRIMARY KEY,
+                    account_id TEXT NOT NULL,
+                    query TEXT NOT NULL,
+                    skill_name TEXT NOT NULL,
+                    snapshot_id TEXT NOT NULL,
+                    model TEXT NOT NULL,
+                    status TEXT NOT NULL,
+                    result_json TEXT NOT NULL DEFAULT '{}',
+                    error TEXT,
+                    created_at REAL NOT NULL,
+                    updated_at REAL NOT NULL,
+                    FOREIGN KEY(snapshot_id) REFERENCES skill_snapshots(id)
+                );
+
                CREATE INDEX IF NOT EXISTS idx_eval_datasets_account
                    ON evaluation_datasets(account_id, created_at DESC);
                CREATE INDEX IF NOT EXISTS idx_eval_experiments_account
                    ON evaluation_experiments(account_id, created_at DESC);
+                CREATE INDEX IF NOT EXISTS idx_eval_single_analyses_account
+                    ON evaluation_single_analyses(account_id, created_at DESC);
                CREATE INDEX IF NOT EXISTS idx_eval_cases_experiment
                    ON evaluation_cases(experiment_id, case_index);
                CREATE INDEX IF NOT EXISTS idx_eval_cases_status
@@ -231,6 +248,16 @@ class EvaluationStore:
                """,
                (time.time(),),
            )
+            connection.execute(
+                """
+                UPDATE evaluation_single_analyses
+                SET status = 'failed',
+                    error = COALESCE(error, '服务重启前分析未完成'),
+                    updated_at = ?
+                WHERE status = 'running'
+                """,
+                (time.time(),),
+            )

    def execute(
        self,
@@ -478,6 +505,7 @@ class EvaluationRuntime:
        model_name = model or self.model_config_for(account_id).model
        analysis_id = f'single_{uuid4().hex[:16]}'
        case_id = f'case_{uuid4().hex[:16]}'
+        created_at = time.time()
        canonical = {
            'case_id': case_id,
            'query': normalized_query,
@@ -488,6 +516,24 @@ class EvaluationRuntime:
            'request_id': request_id.strip(),
            'metadata': metadata or {},
        }
+        self.store.execute(
+            """
+            INSERT INTO evaluation_single_analyses (
+                id, account_id, query, skill_name, snapshot_id, model,
+                status, result_json, error, created_at, updated_at
+            ) VALUES (?, ?, ?, ?, ?, ?, 'running', '{}', NULL, ?, ?)
+            """,
+            (
+                analysis_id,
+                account_id,
+                normalized_query,
+                skill_name,
+                snapshot['id'],
+                model_name,
+                created_at,
+                created_at,
+            ),
+        )
        execution = self._execute_case_analysis(
            account_id=account_id,
            run_id=analysis_id,
@@ -503,9 +549,24 @@ class EvaluationRuntime:
            cancel_event=threading.Event(),
        )
        if execution.status != 'completed' or execution.normalized is None:
-            raise EvaluationError(execution.error or '单条分析失败')
+            error_message = execution.error or '单条分析失败'
+            self.store.execute(
+                """
+                UPDATE evaluation_single_analyses
+                SET status = ?, error = ?, updated_at = ?
+                WHERE id = ? AND account_id = ?
+                """,
+                (
+                    execution.status,
+                    error_message,
+                    time.time(),
+                    analysis_id,
+                    account_id,
+                ),
+            )
+            raise EvaluationError(error_message)
        result = execution.normalized
-        return {
+        payload = {
            'id': analysis_id,
            'case_index': 0,
            'case_id': case_id,
@@ -538,9 +599,64 @@ class EvaluationRuntime:
            'skill_name': skill_name,
            'skill_version': snapshot['content_hash'][:12],
            'model': model_name,
-            'created_at': time.time(),
+            'created_at': created_at,
            'updated_at': time.time(),
        }
+        self.store.execute(
+            """
+            UPDATE evaluation_single_analyses
+            SET status = 'completed', result_json = ?, error = NULL,
+                updated_at = ?
+            WHERE id = ? AND account_id = ?
+            """,
+            (
+                json.dumps(payload, ensure_ascii=False),
+                payload['updated_at'],
+                analysis_id,
+                account_id,
+            ),
+        )
+        return payload
+
+    def list_single_analyses(
+        self,
+        account_id: str,
+        *,
+        limit: int = 100,
+    ) -> list[dict[str, Any]]:
+        rows = self.store.fetchall(
+            """
+            SELECT a.*, s.content_hash AS snapshot_hash
+            FROM evaluation_single_analyses AS a
+            JOIN skill_snapshots AS s ON s.id = a.snapshot_id
+            WHERE a.account_id = ?
+            ORDER BY a.created_at DESC
+            LIMIT ?
+            """,
+            (account_id, max(1, min(limit, 200))),
+        )
+        return [
+            self._single_analysis_payload(row, include_detail=False)
+            for row in rows
+        ]
+
+    def get_single_analysis(
+        self,
+        analysis_id: str,
+        account_id: str,
+    ) -> dict[str, Any]:
+        row = self.store.fetchone(
+            """
+            SELECT a.*, s.content_hash AS snapshot_hash
+            FROM evaluation_single_analyses AS a
+            JOIN skill_snapshots AS s ON s.id = a.snapshot_id
+            WHERE a.id = ? AND a.account_id = ?
+            """,
+            (analysis_id, account_id),
+        )
+        if row is None:
+            raise EvaluationError('单条分析记录不存在')
+        return self._single_analysis_payload(row, include_detail=True)

    # ------------------------------------------------------------------
    # Experiments
@@ -1391,6 +1507,78 @@ class EvaluationRuntime:
    # Serialization helpers
    # ------------------------------------------------------------------

+    def _single_analysis_payload(
+        self,
+        row: dict[str, Any],
+        *,
+        include_detail: bool,
+    ) -> dict[str, Any]:
+        try:
+            stored_result = json.loads(row.get('result_json') or '{}')
+        except json.JSONDecodeError:
+            stored_result = {}
+        if not isinstance(stored_result, dict):
+            stored_result = {}
+        payload: dict[str, Any] = {
+            'id': row['id'],
+            'case_index': 0,
+            'case_id': '',
+            'query': row['query'],
+            'gold_label': '',
+            'prediction': '',
+            'function_output': '',
+            'complex': None,
+            'correct': None,
+            'reason': '',
+            'candidates': [],
+            'confidence': None,
+            'uncertainty': '',
+            'evidence_refs': [],
+            'accessed_refs': [],
+            'raw_output': '',
+            'transcript_path': '',
+            'latency_ms': None,
+            'tool_calls': 0,
+            'turns': 0,
+            'input_tokens': 0,
+            'output_tokens': 0,
+            'domain': '',
+            'request_id': '',
+            'status': row['status'],
+            'error': row.get('error'),
+            'review_status': '',
+            'review_note': '',
+            'canonical': {},
+            'skill_name': row['skill_name'],
+            'skill_version': str(row.get('snapshot_hash') or '')[:12],
+            'model': row['model'],
+            'created_at': row['created_at'],
+            'updated_at': row['updated_at'],
+        }
+        payload.update(stored_result)
+        payload.update(
+            {
+                'id': row['id'],
+                'query': row['query'],
+                'skill_name': row['skill_name'],
+                'skill_version': str(row.get('snapshot_hash') or '')[:12],
+                'model': row['model'],
+                'status': row['status'],
+                'error': row.get('error'),
+                'created_at': row['created_at'],
+                'updated_at': row['updated_at'],
+            }
+        )
+        if not include_detail:
+            for key in (
+                'raw_output',
+                'transcript_path',
+                'canonical',
+                'accessed_refs',
+            ):
+                payload.pop(key, None)
+        return payload
+
    def _dataset_payload(
        self,
        row: dict[str, Any],
@@ -88,6 +88,45 @@ class EvaluationApiTests(unittest.TestCase):
                self.assertEqual(response.json()['prediction'], '地图导航')
                analyze.assert_called_once()

+    def test_single_analysis_history_endpoints(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            state = _build_state(root)
+            record = {
+                'id': 'single_1',
+                'query': '导航去公司',
+                'prediction': '地图导航',
+                'status': 'completed',
+            }
+            with (
+                TestClient(create_app(state)) as client,
+                patch.object(
+                    state.evaluation_runtime,
+                    'list_single_analyses',
+                    return_value=[record],
+                ) as list_history,
+                patch.object(
+                    state.evaluation_runtime,
+                    'get_single_analysis',
+                    return_value=record,
+                ) as get_history,
+            ):
+                response = client.get(
+                    '/api/evaluations/analyses',
+                    params={'account_id': 'alice'},
+                )
+                self.assertEqual(response.status_code, 200)
+                self.assertEqual(response.json(), [record])
+                list_history.assert_called_once_with('alice')
+
+                detail = client.get(
+                    '/api/evaluations/analyses/single_1',
+                    params={'account_id': 'alice'},
+                )
+                self.assertEqual(detail.status_code, 200)
+                self.assertEqual(detail.json(), record)
+                get_history.assert_called_once_with('single_1', 'alice')
+
    def test_dataset_experiment_and_export_flow(self) -> None:
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
@@ -333,6 +333,15 @@ class EvaluationRuntimeTests(unittest.TestCase):
                self.assertEqual(result['model'], 'test-model')
                self.assertEqual(runtime.list_datasets('alice'), [])
                self.assertEqual(runtime.list_experiments('alice'), [])
+                history = runtime.list_single_analyses('alice')
+                self.assertEqual(len(history), 1)
+                self.assertEqual(history[0]['id'], result['id'])
+                self.assertEqual(history[0]['query'], '导航去公司')
+                self.assertEqual(history[0]['prediction'], '地图导航')
+                self.assertNotIn('raw_output', history[0])
+                detail = runtime.get_single_analysis(result['id'], 'alice')
+                self.assertEqual(detail['prediction'], '地图导航')
+                self.assertEqual(detail['canonical']['query'], '导航去公司')
            finally:
                runtime.shutdown()