diff --git a/backend/api/server.py b/backend/api/server.py index c299293..4cf1f3b 100644 --- a/backend/api/server.py +++ b/backend/api/server.py @@ -47,6 +47,7 @@ from src.agent_types import ( ) from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills from src.data_agent_inputs import DataAgentInputError, load_input_sources +from src.evaluation_runtime import EvaluationError, EvaluationRuntime from src.jupyter_runtime import ( DEFAULT_JUPYTER_WORKSPACE_ROOT, JupyterRuntimeError, @@ -586,6 +587,12 @@ class AgentState: self.model_config_for, ) self.memory_manager.start() + self.evaluation_runtime = EvaluationRuntime( + root=self.session_directory.parent / 'evaluations', + cwd_for_account=lambda account_id: self.config_for(account_id).cwd, + model_config_for=self.model_config_for, + account_paths_for=self.account_paths, + ) @property def cwd(self) -> Path: @@ -1125,6 +1132,56 @@ class SkillSyncRequest(BaseModel): account_id: str | None = None +class EvaluationDatasetCreateRequest(BaseModel): + account_id: str = Field(min_length=1) + name: str = '' + filename: str = Field(min_length=1) + content_base64: str | None = None + rows: list[dict[str, Any]] | None = None + + +class EvaluationDatasetMappingRequest(BaseModel): + account_id: str = Field(min_length=1) + mapping: dict[str, Any] + + +class EvaluationAnalyzeRequest(BaseModel): + account_id: str = Field(min_length=1) + query: str = Field(min_length=1, max_length=20_000) + skill_name: str = 'label-master' + model: str | None = None + history: list[dict[str, Any]] = Field(default_factory=list) + context: dict[str, Any] = Field(default_factory=dict) + domain: str = '' + request_id: str = '' + metadata: dict[str, Any] = Field(default_factory=dict) + + +class EvaluationExperimentCreateRequest(BaseModel): + account_id: str = Field(min_length=1) + dataset_id: str = Field(min_length=1) + name: str = '' + skill_name: str = 'label-master' + model: str | None = None + concurrency: int = Field(default=2, ge=1, le=8) + + +class EvaluationExperimentActionRequest(BaseModel): + account_id: str = Field(min_length=1) + + +class EvaluationRetryRequest(BaseModel): + account_id: str = Field(min_length=1) + case_ids: list[str] | None = None + disagreements_only: bool = False + + +class EvaluationReviewRequest(BaseModel): + account_id: str = Field(min_length=1) + review_status: str = Field(max_length=80) + review_note: str = Field(default='', max_length=4000) + + class SessionUpdate(BaseModel): title: str | None = Field(default=None, max_length=80) is_training: bool | None = None @@ -1178,6 +1235,7 @@ def create_app(state: AgentState) -> FastAPI: scanner_task.cancel() _watcher_manager.cancel_all() _bash_bg_manager.cancel_all() + state.evaluation_runtime.shutdown() state.event_loop = None app = FastAPI(title='Claw Code GUI', version='1.0', lifespan=lifespan) @@ -1468,6 +1526,222 @@ def create_app(state: AgentState) -> FastAPI: ) return result + # ------------- Skill evaluations --------------------------------------- + @app.get('/api/evaluations/metadata') + async def evaluation_metadata(account_id: str) -> dict[str, Any]: + try: + return state.evaluation_runtime.metadata(_safe_account_id(account_id)) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.post('/api/evaluations/analyze') + async def analyze_evaluation_case( + payload: EvaluationAnalyzeRequest, + ) -> dict[str, Any]: + try: + return await asyncio.to_thread( + state.evaluation_runtime.analyze_case, + account_id=_safe_account_id(payload.account_id), + query=payload.query, + skill_name=payload.skill_name, + model=payload.model, + history=payload.history, + context=payload.context, + domain=payload.domain, + request_id=payload.request_id, + metadata=payload.metadata, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.get('/api/evaluations/datasets') + async def list_evaluation_datasets(account_id: str) -> list[dict[str, Any]]: + return state.evaluation_runtime.list_datasets(_safe_account_id(account_id)) + + @app.post('/api/evaluations/datasets') + async def create_evaluation_dataset( + payload: EvaluationDatasetCreateRequest, + ) -> dict[str, Any]: + try: + return await asyncio.to_thread( + state.evaluation_runtime.create_dataset, + account_id=_safe_account_id(payload.account_id), + name=payload.name, + filename=payload.filename, + content_base64=payload.content_base64, + rows=payload.rows, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.get('/api/evaluations/datasets/{dataset_id}') + async def get_evaluation_dataset( + dataset_id: str, + account_id: str, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.get_dataset( + dataset_id, + _safe_account_id(account_id), + include_rows=False, + ) + except EvaluationError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + @app.patch('/api/evaluations/datasets/{dataset_id}/mapping') + async def update_evaluation_dataset_mapping( + dataset_id: str, + payload: EvaluationDatasetMappingRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.update_dataset_mapping( + dataset_id, + _safe_account_id(payload.account_id), + payload.mapping, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.get('/api/evaluations/experiments') + async def list_evaluation_experiments( + account_id: str, + ) -> list[dict[str, Any]]: + return state.evaluation_runtime.list_experiments( + _safe_account_id(account_id) + ) + + @app.post('/api/evaluations/experiments') + async def create_evaluation_experiment( + payload: EvaluationExperimentCreateRequest, + ) -> dict[str, Any]: + try: + return await asyncio.to_thread( + state.evaluation_runtime.create_experiment, + account_id=_safe_account_id(payload.account_id), + dataset_id=payload.dataset_id, + name=payload.name, + skill_name=payload.skill_name, + model=payload.model, + concurrency=payload.concurrency, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.get('/api/evaluations/experiments/{experiment_id}') + async def get_evaluation_experiment( + experiment_id: str, + account_id: str, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.get_experiment( + experiment_id, + _safe_account_id(account_id), + ) + except EvaluationError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + @app.post('/api/evaluations/experiments/{experiment_id}/start') + async def start_evaluation_experiment( + experiment_id: str, + payload: EvaluationExperimentActionRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.start( + experiment_id, + _safe_account_id(payload.account_id), + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.post('/api/evaluations/experiments/{experiment_id}/pause') + async def pause_evaluation_experiment( + experiment_id: str, + payload: EvaluationExperimentActionRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.pause( + experiment_id, + _safe_account_id(payload.account_id), + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.post('/api/evaluations/experiments/{experiment_id}/resume') + async def resume_evaluation_experiment( + experiment_id: str, + payload: EvaluationExperimentActionRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.resume( + experiment_id, + _safe_account_id(payload.account_id), + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.post('/api/evaluations/experiments/{experiment_id}/cancel') + async def cancel_evaluation_experiment( + experiment_id: str, + payload: EvaluationExperimentActionRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.cancel( + experiment_id, + _safe_account_id(payload.account_id), + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.post('/api/evaluations/experiments/{experiment_id}/retry') + async def retry_evaluation_experiment( + experiment_id: str, + payload: EvaluationRetryRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.retry( + experiment_id, + _safe_account_id(payload.account_id), + case_ids=payload.case_ids, + disagreements_only=payload.disagreements_only, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.patch('/api/evaluations/cases/{case_id}/review') + async def review_evaluation_case( + case_id: str, + payload: EvaluationReviewRequest, + ) -> dict[str, Any]: + try: + return state.evaluation_runtime.update_review( + case_id, + _safe_account_id(payload.account_id), + review_status=payload.review_status, + review_note=payload.review_note, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @app.get('/api/evaluations/experiments/{experiment_id}/export') + async def export_evaluation_experiment( + experiment_id: str, + account_id: str, + ) -> Response: + try: + filename, content = state.evaluation_runtime.export_csv( + experiment_id, + _safe_account_id(account_id), + ) + except EvaluationError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + return Response( + content, + media_type='text/csv; charset=utf-8', + headers={ + 'Content-Disposition': f"attachment; filename*=UTF-8''{quote(filename)}" + }, + ) + # ------------- memory ---------------------------------------------------- @app.get('/api/memory/user') async def get_user_memory(account_id: str) -> dict[str, Any]: diff --git a/frontend/app/app/api/claw/evaluations/[...path]/route.ts b/frontend/app/app/api/claw/evaluations/[...path]/route.ts new file mode 100644 index 0000000..7d51291 --- /dev/null +++ b/frontend/app/app/api/claw/evaluations/[...path]/route.ts @@ -0,0 +1,78 @@ +import { getCurrentAccount } from "@/lib/claw-auth"; + +const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765"; + +type RouteContext = { + params: Promise<{ path: string[] }>; +}; + +export async function GET(request: Request, context: RouteContext) { + return proxyEvaluationRequest("GET", request, context); +} + +export async function POST(request: Request, context: RouteContext) { + return proxyEvaluationRequest("POST", request, context); +} + +export async function PATCH(request: Request, context: RouteContext) { + return proxyEvaluationRequest("PATCH", request, context); +} + +async function proxyEvaluationRequest( + method: "GET" | "POST" | "PATCH", + request: Request, + context: RouteContext, +) { + const account = await getCurrentAccount(); + if (!account) + return Response.json({ detail: "unauthorized" }, { status: 401 }); + + const { path } = await context.params; + const incomingUrl = new URL(request.url); + const targetUrl = new URL( + `${CLAW_API_URL}/api/evaluations/${path.map(encodeURIComponent).join("/")}`, + ); + for (const [key, value] of incomingUrl.searchParams.entries()) { + targetUrl.searchParams.append(key, value); + } + + const headers: Record = {}; + let body: string | undefined; + if (method === "GET") { + targetUrl.searchParams.set("account_id", account.id); + } else { + const payload = (await request.json()) as Record; + headers["content-type"] = "application/json"; + body = JSON.stringify({ ...payload, account_id: account.id }); + } + + try { + const response = await fetch(targetUrl, { + method, + headers, + body, + cache: "no-store", + }); + const responseHeaders = new Headers(); + responseHeaders.set( + "content-type", + response.headers.get("content-type") ?? "application/json", + ); + const disposition = response.headers.get("content-disposition"); + if (disposition) responseHeaders.set("content-disposition", disposition); + return new Response(await response.arrayBuffer(), { + status: response.status, + headers: responseHeaders, + }); + } catch (error) { + return Response.json( + { + detail: + error instanceof Error + ? `Unable to reach Claw backend: ${error.message}` + : "Unable to reach Claw backend", + }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/app/evaluations/page.tsx b/frontend/app/app/evaluations/page.tsx new file mode 100644 index 0000000..a100b11 --- /dev/null +++ b/frontend/app/app/evaluations/page.tsx @@ -0,0 +1,1832 @@ +"use client"; + +import { + ArrowLeftIcon, + CheckCircle2Icon, + ChevronRightIcon, + CircleAlertIcon, + DownloadIcon, + FileSpreadsheetIcon, + FlaskConicalIcon, + LoaderCircleIcon, + PauseIcon, + PlayIcon, + RefreshCwIcon, + RotateCcwIcon, + SearchIcon, + SquareIcon, + UploadIcon, + XCircleIcon, +} from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { ClawAccountGate, useClawAccount } from "@/app/claw-account-gate"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet"; + +type SkillMetadata = { + name: string; + description: string; + default?: boolean; +}; + +type EvaluationMetadata = { + skills: SkillMetadata[]; + default_skill: string; + max_concurrency: number; + label_catalog: { + labels: Array<{ name: string; domain: string; path: string }>; + agent_tags: string[]; + functions: string[]; + intents: string[]; + counts?: Record; + }; +}; + +type ModelOption = { + id: string; + name?: string; +}; + +type ClawState = { + model?: string; +}; + +type DatasetMapping = { + fields: Record; + label_map: Record; +}; + +type Dataset = { + id: string; + name: string; + filename: string; + format: string; + columns: string[]; + row_count: number; + mapping: DatasetMapping; + preview: Array>; + mapped_preview: CanonicalCase[]; + validation: { + total_rows: number; + missing_query: number; + duplicate_ids: number; + has_gold_label: number; + }; +}; + +type CanonicalCase = { + case_id: string; + query: string; + gold_label: string; + domain: string; + request_id: string; +}; + +type EvaluationCase = { + id: string; + case_index: number; + case_id: string; + query: string; + gold_label: string; + prediction: string; + function_output: string; + complex: boolean | null; + correct: boolean | null; + reason: string; + candidates: string[]; + confidence: number | null; + uncertainty: string; + evidence_refs: string[]; + accessed_refs: Array<{ tool: string; path: string }>; + raw_output: string; + transcript_path: string; + latency_ms: number | null; + tool_calls: number; + turns: number; + input_tokens: number; + output_tokens: number; + domain: string; + request_id: string; + status: string; + error?: string | null; + review_status: string; + review_note: string; + canonical: Record; + skill_name?: string; + skill_version?: string; + model?: string; +}; + +type Experiment = { + id: string; + name: string; + dataset_name?: string; + dataset_filename?: string; + skill_name: string; + skill_version: string; + git_commit: string; + git_status: string; + model: string; + concurrency: number; + status: string; + total_cases: number; + completed_cases: number; + failed_cases: number; + cancelled_cases: number; + processed_cases: number; + progress: number; + created_at: number; + started_at?: number | null; + finished_at?: number | null; + error?: string | null; + cases?: EvaluationCase[]; + metrics?: { + completed: number; + labeled: number; + correct: number; + disagreements: number; + accuracy: number | null; + low_confidence: number; + no_evidence: number; + prediction_distribution: Record; + domain_metrics: Array<{ + domain: string; + total: number; + correct: number; + accuracy: number | null; + }>; + }; +}; + +const FIELD_OPTIONS = [ + ["query", "当前 query", true], + ["gold_label", "人工标签", false], + ["case_id", "Case ID", false], + ["history", "对话历史", false], + ["context", "上下文", false], + ["domain", "垂域", false], + ["request_id", "Request ID", false], + ["metadata", "扩展信息", false], +] as const; + +const ACTIVE_STATUSES = new Set(["queued", "running", "pausing"]); +const REVIEW_OPTIONS = [ + ["", "未复核"], + ["gold_correct", "人工原标签正确"], + ["prediction_correct", "模型判断正确"], + ["ambiguous", "分类标准有歧义"], + ["capability_missing", "能力信息缺失"], + ["uncertain", "暂不确定"], +]; + +const DEFAULT_EVALUATION_MODEL = "ppio/pa/gpt-5.5"; + +export default function EvaluationsPage() { + const { account, isLoading, setAccount } = useClawAccount(); + + if (isLoading) { + return ( +
+ +
+ ); + } + if (!account) return ; + return ; +} + +function EvaluationWorkspace() { + const [workspaceMode, setWorkspaceMode] = useState<"single" | "batch">( + "single", + ); + const [metadata, setMetadata] = useState(null); + const [models, setModels] = useState([]); + const [datasets, setDatasets] = useState([]); + const [experiments, setExperiments] = useState([]); + const [selectedDatasetId, setSelectedDatasetId] = useState(""); + const [selectedExperimentId, setSelectedExperimentId] = useState(""); + const [experiment, setExperiment] = useState(null); + const [mapping, setMapping] = useState({ + fields: {}, + label_map: {}, + }); + const [skillName, setSkillName] = useState(""); + const [model, setModel] = useState(""); + const [concurrency, setConcurrency] = useState(2); + const [experimentName, setExperimentName] = useState(""); + const [singleQuery, setSingleQuery] = useState(""); + const [singleResult, setSingleResult] = useState(null); + const [tab, setTab] = useState<"overview" | "disagreements" | "all">( + "overview", + ); + const [selectedCase, setSelectedCase] = useState(null); + const [queryFilter, setQueryFilter] = useState(""); + const [busy, setBusy] = useState(""); + const [error, setError] = useState(""); + const uploadRef = useRef(null); + const selectedDatasetIdRef = useRef(""); + const selectedExperimentIdRef = useRef(""); + + useEffect(() => { + selectedDatasetIdRef.current = selectedDatasetId; + }, [selectedDatasetId]); + + useEffect(() => { + selectedExperimentIdRef.current = selectedExperimentId; + }, [selectedExperimentId]); + + const selectedDataset = useMemo( + () => datasets.find((item) => item.id === selectedDatasetId) ?? null, + [datasets, selectedDatasetId], + ); + + const refreshExperiment = useCallback(async (experimentId: string) => { + const response = await fetch( + `/api/claw/evaluations/experiments/${encodeURIComponent(experimentId)}`, + { cache: "no-store" }, + ); + const payload = await readJson(response); + setExperiment(payload); + setExperiments((current) => + current.map((item) => (item.id === payload.id ? payload : item)), + ); + }, []); + + const loadWorkspace = useCallback(async () => { + setError(""); + try { + const [ + metadataResponse, + datasetResponse, + experimentResponse, + modelResponse, + stateResponse, + ] = await Promise.all([ + fetch("/api/claw/evaluations/metadata", { cache: "no-store" }), + fetch("/api/claw/evaluations/datasets", { cache: "no-store" }), + fetch("/api/claw/evaluations/experiments", { cache: "no-store" }), + fetch("/api/claw/models", { cache: "no-store" }), + fetch("/api/claw/state", { cache: "no-store" }), + ]); + const metadataPayload = + await readJson(metadataResponse); + const datasetPayload = await readJson(datasetResponse); + const experimentPayload = + await readJson(experimentResponse); + const modelPayload = await readJson<{ models?: ModelOption[] }>( + modelResponse, + ); + const statePayload = stateResponse.ok + ? ((await stateResponse.json()) as ClawState) + : {}; + setMetadata(metadataPayload); + setDatasets(datasetPayload); + setExperiments(experimentPayload); + setSkillName( + (current) => current || metadataPayload.default_skill || "label-master", + ); + const nextModels = modelPayload.models ?? []; + setModels(nextModels); + setModel((current) => { + if (current) return current; + const defaultEvaluationModel = nextModels.find( + (item) => item.id === DEFAULT_EVALUATION_MODEL, + ); + const accountModel = nextModels.find( + (item) => item.id === statePayload.model, + ); + return ( + defaultEvaluationModel?.id || + accountModel?.id || + nextModels[0]?.id || + "" + ); + }); + const nextDataset = + datasetPayload.find( + (item) => item.id === selectedDatasetIdRef.current, + ) ?? datasetPayload[0]; + if (nextDataset && nextDataset.id !== selectedDatasetIdRef.current) { + setSelectedDatasetId(nextDataset.id); + setMapping(nextDataset.mapping); + } + const nextExperiment = + experimentPayload.find( + (item) => item.id === selectedExperimentIdRef.current, + ) ?? experimentPayload[0]; + if (nextExperiment) { + setSelectedExperimentId(nextExperiment.id); + await refreshExperiment(nextExperiment.id); + } + } catch (reason) { + setError(messageFrom(reason)); + } + }, [refreshExperiment]); + + useEffect(() => { + void loadWorkspace(); + }, [loadWorkspace]); + + useEffect(() => { + if (!experiment || !ACTIVE_STATUSES.has(experiment.status)) return; + const timer = window.setInterval(() => { + void refreshExperiment(experiment.id).catch((reason) => + setError(messageFrom(reason)), + ); + }, 1500); + return () => window.clearInterval(timer); + }, [experiment, refreshExperiment]); + + async function uploadDataset(file: File) { + setBusy("upload"); + setError(""); + try { + const content = await fileToBase64(file); + const response = await fetch("/api/claw/evaluations/datasets", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + name: file.name.replace(/\.[^.]+$/, ""), + filename: file.name, + content_base64: content, + }), + }); + const payload = await readJson(response); + setDatasets((current) => [payload, ...current]); + setSelectedDatasetId(payload.id); + setMapping(payload.mapping); + setExperimentName( + `${metadata?.default_skill || "label-master"} · ${payload.name}`, + ); + setExperiment(null); + setSelectedExperimentId(""); + } catch (reason) { + setError(messageFrom(reason)); + } finally { + setBusy(""); + if (uploadRef.current) uploadRef.current.value = ""; + } + } + + async function saveMapping() { + if (!selectedDataset) return null; + const response = await fetch( + `/api/claw/evaluations/datasets/${encodeURIComponent(selectedDataset.id)}/mapping`, + { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ mapping }), + }, + ); + const payload = await readJson(response); + setDatasets((current) => + current.map((item) => (item.id === payload.id ? payload : item)), + ); + return payload; + } + + async function createAndStart() { + if (!selectedDataset) return; + if (!mapping.fields.query) { + setError("请先映射当前 query 字段"); + return; + } + setBusy("create"); + setError(""); + try { + await saveMapping(); + const response = await fetch("/api/claw/evaluations/experiments", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + dataset_id: selectedDataset.id, + name: + experimentName.trim() || `${skillName} · ${selectedDataset.name}`, + skill_name: skillName, + model: model || undefined, + concurrency, + }), + }); + const created = await readJson(response); + setExperiments((current) => [created, ...current]); + setSelectedExperimentId(created.id); + setExperiment(created); + setTab("overview"); + const startResponse = await fetch( + `/api/claw/evaluations/experiments/${encodeURIComponent(created.id)}/start`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}", + }, + ); + setExperiment(await readJson(startResponse)); + } catch (reason) { + setError(messageFrom(reason)); + } finally { + setBusy(""); + } + } + + async function analyzeSingleQuery() { + const query = singleQuery.trim(); + if (!query) { + setError("请输入需要分析的 query"); + return; + } + setBusy("single"); + setError(""); + setSingleResult(null); + try { + const response = await fetch("/api/claw/evaluations/analyze", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + query, + skill_name: skillName, + model: model || undefined, + }), + }); + setSingleResult(await readJson(response)); + } catch (reason) { + setError(messageFrom(reason)); + } finally { + setBusy(""); + } + } + + async function act(action: "start" | "pause" | "resume" | "cancel") { + if (!experiment) return; + setBusy(action); + setError(""); + try { + const response = await fetch( + `/api/claw/evaluations/experiments/${encodeURIComponent(experiment.id)}/${action}`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{}", + }, + ); + setExperiment(await readJson(response)); + } catch (reason) { + setError(messageFrom(reason)); + } finally { + setBusy(""); + } + } + + async function retry(caseIds?: string[]) { + if (!experiment) return; + setBusy("retry"); + setError(""); + try { + const response = await fetch( + `/api/claw/evaluations/experiments/${encodeURIComponent(experiment.id)}/retry`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ case_ids: caseIds }), + }, + ); + setExperiment(await readJson(response)); + } catch (reason) { + setError(messageFrom(reason)); + } finally { + setBusy(""); + } + } + + const visibleCases = useMemo(() => { + const cases = experiment?.cases ?? []; + const keyword = queryFilter.trim().toLocaleLowerCase("zh-CN"); + return cases.filter((item) => { + if (tab === "disagreements") { + const shouldShow = + item.correct === false || + (item.confidence !== null && item.confidence < 0.6) || + (item.status === "completed" && !item.evidence_refs.length); + if (!shouldShow) return false; + } + if (!keyword) return true; + return `${item.query}\n${item.gold_label}\n${item.prediction}\n${item.domain}` + .toLocaleLowerCase("zh-CN") + .includes(keyword); + }); + }, [experiment?.cases, queryFilter, tab]); + + const labelCounts = metadata?.label_catalog.counts ?? {}; + + return ( +
+
+
+ +
+ +
+
+

Skill 评测

+

+ {metadata + ? `${labelCounts.labels ?? metadata.label_catalog.labels.length} 个标签 · ${labelCounts.functions ?? metadata.label_catalog.functions.length} 个 Function` + : "正在读取 Skill"} +

+
+
+ +
+
+
+ +
+
+
+ + +
+
+ + + {workspaceMode === "batch" ? ( + + ) : null} +
+
+
+ + {error ? ( +
+ + {error} +
+ ) : null} + + {workspaceMode === "single" ? ( + + ) : ( +
+ + +
+ {experiment ? ( + + ) : ( + { + setSelectedDatasetId(value); + const next = datasets.find((item) => item.id === value); + if (next) { + setMapping(next.mapping); + setExperimentName(`${skillName} · ${next.name}`); + } + }} + mapping={mapping} + setMapping={setMapping} + concurrency={concurrency} + setConcurrency={setConcurrency} + experimentName={experimentName} + setExperimentName={setExperimentName} + uploadRef={uploadRef} + onUpload={uploadDataset} + onCreate={createAndStart} + busy={busy} + /> + )} +
+
+ )} + + setSelectedCase(null)} + onUpdated={(updated) => { + setSelectedCase(updated); + setExperiment((current) => + current + ? { + ...current, + cases: current.cases?.map((item) => + item.id === updated.id ? updated : item, + ), + } + : current, + ); + }} + /> +
+ ); +} + +function SingleQueryWorkspace(props: { + query: string; + setQuery: (value: string) => void; + result: EvaluationCase | null; + onAnalyze: () => void; + busy: string; +}) { + const { query, setQuery, result, onAnalyze, busy } = props; + return ( +
+
+

单条分析

+

+ 输入 query，查看分类结果、判断理由和 Skill 依据。 +

+
+ +
+