diff --git a/backend/api/server.py b/backend/api/server.py index f04a19a..ac85cee 100644 --- a/backend/api/server.py +++ b/backend/api/server.py @@ -47,7 +47,12 @@ from src.agent_types import ( ) from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills from src.data_agent_inputs import DataAgentInputError, load_input_sources -from src.evaluation_runtime import EvaluationError, EvaluationRuntime +from src.evaluation_runtime import ( + DEFAULT_CONCURRENCY, + MAX_CONCURRENCY, + EvaluationError, + EvaluationRuntime, +) from src.jupyter_runtime import ( DEFAULT_JUPYTER_WORKSPACE_ROOT, JupyterRuntimeError, @@ -1145,6 +1150,14 @@ class EvaluationDatasetMappingRequest(BaseModel): mapping: dict[str, Any] +class EvaluationLabelMappingSuggestRequest(BaseModel): + account_id: str = Field(min_length=1) + mapping: dict[str, Any] + skill_name: str = 'label-master' + snapshot_id: str | None = None + model: str | None = None + + class EvaluationAnalyzeRequest(BaseModel): account_id: str = Field(min_length=1) query: str = Field(min_length=1, max_length=20_000) @@ -1165,7 +1178,11 @@ class EvaluationExperimentCreateRequest(BaseModel): skill_name: str = 'label-master' snapshot_id: str | None = None model: str | None = None - concurrency: int = Field(default=2, ge=1, le=8) + concurrency: int = Field( + default=DEFAULT_CONCURRENCY, + ge=1, + le=MAX_CONCURRENCY, + ) class EvaluationSkillVersionSaveRequest(BaseModel): @@ -1702,6 +1719,24 @@ def create_app(state: AgentState) -> FastAPI: except EvaluationError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + @app.post('/api/evaluations/datasets/{dataset_id}/label-mapping/suggest') + async def suggest_evaluation_label_mapping( + dataset_id: str, + payload: EvaluationLabelMappingSuggestRequest, + ) -> dict[str, Any]: + try: + return await asyncio.to_thread( + state.evaluation_runtime.suggest_label_mapping, + dataset_id, + _safe_account_id(payload.account_id), + mapping=payload.mapping, + skill_name=payload.skill_name, + snapshot_id=payload.snapshot_id, + model=payload.model, + ) + except EvaluationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + @app.get('/api/evaluations/experiments') async def list_evaluation_experiments( account_id: str, diff --git a/frontend/app/app/evaluations/page.tsx b/frontend/app/app/evaluations/page.tsx index a6c160f..748f1c6 100644 --- a/frontend/app/app/evaluations/page.tsx +++ b/frontend/app/app/evaluations/page.tsx @@ -20,6 +20,7 @@ import { RotateCcwIcon, SaveIcon, SearchIcon, + SparklesIcon, SquareIcon, UploadIcon, XCircleIcon, @@ -126,6 +127,16 @@ type Dataset = { }; }; +type LabelMappingSuggestion = { + label_map: Record; + unmapped: string[]; + reason: string; + raw_labels: string[]; + skill_name: string; + skill_version: string; + model: string; +}; + type CanonicalCase = { case_id: string; query: string; @@ -270,7 +281,7 @@ function EvaluationWorkspace() { const [skillVersions, setSkillVersions] = useState([]); const [snapshotId, setSnapshotId] = useState(""); const [model, setModel] = useState(""); - const [concurrency, setConcurrency] = useState(2); + const [concurrency, setConcurrency] = useState(8); const [experimentName, setExperimentName] = useState(""); const [singleQuery, setSingleQuery] = useState(""); const [singleResult, setSingleResult] = useState(null); @@ -281,6 +292,7 @@ function EvaluationWorkspace() { const [queryFilter, setQueryFilter] = useState(""); const [busy, setBusy] = useState(""); const [error, setError] = useState(""); + const [labelMappingNote, setLabelMappingNote] = useState(""); const [skillEditorOpen, setSkillEditorOpen] = useState(false); const [skillEditorBusy, setSkillEditorBusy] = useState(""); const [skillEditorError, setSkillEditorError] = useState(""); @@ -386,6 +398,9 @@ function EvaluationWorkspace() { ? ((await stateResponse.json()) as ClawState) : {}; setMetadata(metadataPayload); + setConcurrency((current) => + Math.max(1, Math.min(current, metadataPayload.max_concurrency)), + ); setSingleAnalyses(analysisPayload); setDatasets(datasetPayload); setExperiments(experimentPayload); @@ -425,6 +440,7 @@ function EvaluationWorkspace() { if (nextDataset && nextDataset.id !== selectedDatasetIdRef.current) { setSelectedDatasetId(nextDataset.id); setMapping(nextDataset.mapping); + setLabelMappingNote(""); } const nextExperiment = experimentPayload.find( @@ -605,6 +621,7 @@ function EvaluationWorkspace() { setDatasets((current) => [payload, ...current]); setSelectedDatasetId(payload.id); setMapping(payload.mapping); + setLabelMappingNote(""); setExperimentName( `${metadata?.default_skill || "label-master"} · ${payload.name}`, ); @@ -635,6 +652,46 @@ function EvaluationWorkspace() { return payload; } + async function suggestLabelMapping() { + if (!selectedDataset) return; + if (!mapping.fields.gold_label) { + setError("请先映射人工标签字段"); + return; + } + setBusy("label-map"); + setError(""); + setLabelMappingNote(""); + try { + const response = await fetch( + `/api/claw/evaluations/datasets/${encodeURIComponent(selectedDataset.id)}/label-mapping/suggest`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + mapping, + skill_name: skillName, + snapshot_id: snapshotId || undefined, + model: model || undefined, + }), + }, + ); + const payload = await readJson(response); + setMapping((current) => ({ + ...current, + label_map: payload.label_map, + })); + const notes = [ + payload.reason, + payload.unmapped.length ? `未映射:${payload.unmapped.join("、")}` : "", + ].filter(Boolean); + setLabelMappingNote(notes.join(" ")); + } catch (reason) { + setError(messageFrom(reason)); + } finally { + setBusy(""); + } + } + async function createAndStart() { if (!selectedDataset) return; if (!mapping.fields.query) { @@ -1084,6 +1141,7 @@ function EvaluationWorkspace() { const next = datasets.find((item) => item.id === value); if (next) { setMapping(next.mapping); + setLabelMappingNote(""); setExperimentName(`${skillName} · ${next.name}`); } }} @@ -1096,6 +1154,8 @@ function EvaluationWorkspace() { uploadRef={uploadRef} onUpload={uploadDataset} onCreate={createAndStart} + onSuggestLabelMapping={suggestLabelMapping} + labelMappingNote={labelMappingNote} busy={busy} /> )} @@ -1476,6 +1536,8 @@ function ExperimentSetup(props: { uploadRef: React.RefObject; onUpload: (file: File) => void; onCreate: () => void; + onSuggestLabelMapping: () => void; + labelMappingNote: string; busy: string; }) { const { @@ -1493,6 +1555,8 @@ function ExperimentSetup(props: { uploadRef, onUpload, onCreate, + onSuggestLabelMapping, + labelMappingNote, busy, } = props; const mappedPreview = useMemo( @@ -1566,14 +1630,33 @@ function ExperimentSetup(props: {
-
+

2. 字段映射

- {selectedDataset ? ( - - {selectedDataset.validation.has_gold_label}/ - {selectedDataset.row_count} 条含人工标签 - - ) : null} +
+ {selectedDataset ? ( + + {selectedDataset.validation.has_gold_label}/ + {selectedDataset.row_count} 条含人工标签 + + ) : null} + +
{FIELD_OPTIONS.map(([key, label, required]) => ( @@ -1609,6 +1692,55 @@ function ExperimentSetup(props: { ))}
+ {Object.keys(mapping.label_map).length ? ( +
+
标签映射
+
+ {Object.entries(mapping.label_map).map(([source, target]) => ( +
+ + {source} + + +