feat: add evaluation label mapping
This commit is contained in:
wuyang6
@@ -20,6 +20,7 @@ import {
|
||||
RotateCcwIcon,
|
||||
SaveIcon,
|
||||
SearchIcon,
|
||||
SparklesIcon,
|
||||
SquareIcon,
|
||||
UploadIcon,
|
||||
XCircleIcon,
|
||||
@@ -126,6 +127,16 @@ type Dataset = {
|
||||
};
|
||||
};
|
||||
|
||||
type LabelMappingSuggestion = {
|
||||
label_map: Record<string, string>;
|
||||
unmapped: string[];
|
||||
reason: string;
|
||||
raw_labels: string[];
|
||||
skill_name: string;
|
||||
skill_version: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
type CanonicalCase = {
|
||||
case_id: string;
|
||||
query: string;
|
||||
@@ -270,7 +281,7 @@ function EvaluationWorkspace() {
|
||||
const [skillVersions, setSkillVersions] = useState<SkillVersion[]>([]);
|
||||
const [snapshotId, setSnapshotId] = useState("");
|
||||
const [model, setModel] = useState("");
|
||||
const [concurrency, setConcurrency] = useState(2);
|
||||
const [concurrency, setConcurrency] = useState(8);
|
||||
const [experimentName, setExperimentName] = useState("");
|
||||
const [singleQuery, setSingleQuery] = useState("");
|
||||
const [singleResult, setSingleResult] = useState<EvaluationCase | null>(null);
|
||||
@@ -281,6 +292,7 @@ function EvaluationWorkspace() {
|
||||
const [queryFilter, setQueryFilter] = useState("");
|
||||
const [busy, setBusy] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
const [labelMappingNote, setLabelMappingNote] = useState("");
|
||||
const [skillEditorOpen, setSkillEditorOpen] = useState(false);
|
||||
const [skillEditorBusy, setSkillEditorBusy] = useState("");
|
||||
const [skillEditorError, setSkillEditorError] = useState("");
|
||||
@@ -386,6 +398,9 @@ function EvaluationWorkspace() {
|
||||
? ((await stateResponse.json()) as ClawState)
|
||||
: {};
|
||||
setMetadata(metadataPayload);
|
||||
setConcurrency((current) =>
|
||||
Math.max(1, Math.min(current, metadataPayload.max_concurrency)),
|
||||
);
|
||||
setSingleAnalyses(analysisPayload);
|
||||
setDatasets(datasetPayload);
|
||||
setExperiments(experimentPayload);
|
||||
@@ -425,6 +440,7 @@ function EvaluationWorkspace() {
|
||||
if (nextDataset && nextDataset.id !== selectedDatasetIdRef.current) {
|
||||
setSelectedDatasetId(nextDataset.id);
|
||||
setMapping(nextDataset.mapping);
|
||||
setLabelMappingNote("");
|
||||
}
|
||||
const nextExperiment =
|
||||
experimentPayload.find(
|
||||
@@ -605,6 +621,7 @@ function EvaluationWorkspace() {
|
||||
setDatasets((current) => [payload, ...current]);
|
||||
setSelectedDatasetId(payload.id);
|
||||
setMapping(payload.mapping);
|
||||
setLabelMappingNote("");
|
||||
setExperimentName(
|
||||
`${metadata?.default_skill || "label-master"} · ${payload.name}`,
|
||||
);
|
||||
@@ -635,6 +652,46 @@ function EvaluationWorkspace() {
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function suggestLabelMapping() {
|
||||
if (!selectedDataset) return;
|
||||
if (!mapping.fields.gold_label) {
|
||||
setError("请先映射人工标签字段");
|
||||
return;
|
||||
}
|
||||
setBusy("label-map");
|
||||
setError("");
|
||||
setLabelMappingNote("");
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/claw/evaluations/datasets/${encodeURIComponent(selectedDataset.id)}/label-mapping/suggest`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
mapping,
|
||||
skill_name: skillName,
|
||||
snapshot_id: snapshotId || undefined,
|
||||
model: model || undefined,
|
||||
}),
|
||||
},
|
||||
);
|
||||
const payload = await readJson<LabelMappingSuggestion>(response);
|
||||
setMapping((current) => ({
|
||||
...current,
|
||||
label_map: payload.label_map,
|
||||
}));
|
||||
const notes = [
|
||||
payload.reason,
|
||||
payload.unmapped.length ? `未映射：${payload.unmapped.join("、")}` : "",
|
||||
].filter(Boolean);
|
||||
setLabelMappingNote(notes.join(" "));
|
||||
} catch (reason) {
|
||||
setError(messageFrom(reason));
|
||||
} finally {
|
||||
setBusy("");
|
||||
}
|
||||
}
|
||||
|
||||
async function createAndStart() {
|
||||
if (!selectedDataset) return;
|
||||
if (!mapping.fields.query) {
|
||||
@@ -1084,6 +1141,7 @@ function EvaluationWorkspace() {
|
||||
const next = datasets.find((item) => item.id === value);
|
||||
if (next) {
|
||||
setMapping(next.mapping);
|
||||
setLabelMappingNote("");
|
||||
setExperimentName(`${skillName} · ${next.name}`);
|
||||
}
|
||||
}}
|
||||
@@ -1096,6 +1154,8 @@ function EvaluationWorkspace() {
|
||||
uploadRef={uploadRef}
|
||||
onUpload={uploadDataset}
|
||||
onCreate={createAndStart}
|
||||
onSuggestLabelMapping={suggestLabelMapping}
|
||||
labelMappingNote={labelMappingNote}
|
||||
busy={busy}
|
||||
/>
|
||||
)}
|
||||
@@ -1476,6 +1536,8 @@ function ExperimentSetup(props: {
|
||||
uploadRef: React.RefObject<HTMLInputElement | null>;
|
||||
onUpload: (file: File) => void;
|
||||
onCreate: () => void;
|
||||
onSuggestLabelMapping: () => void;
|
||||
labelMappingNote: string;
|
||||
busy: string;
|
||||
}) {
|
||||
const {
|
||||
@@ -1493,6 +1555,8 @@ function ExperimentSetup(props: {
|
||||
uploadRef,
|
||||
onUpload,
|
||||
onCreate,
|
||||
onSuggestLabelMapping,
|
||||
labelMappingNote,
|
||||
busy,
|
||||
} = props;
|
||||
const mappedPreview = useMemo(
|
||||
@@ -1566,14 +1630,33 @@ function ExperimentSetup(props: {
|
||||
</section>
|
||||
|
||||
<section className="mt-4 border-y py-5 sm:rounded-md sm:border sm:px-5">
|
||||
<div className="mb-4 flex items-center justify-between">
|
||||
<div className="mb-4 flex items-center justify-between gap-3">
|
||||
<h3 className="font-medium text-sm">2. 字段映射</h3>
|
||||
{selectedDataset ? (
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{selectedDataset.validation.has_gold_label}/
|
||||
{selectedDataset.row_count} 条含人工标签
|
||||
</span>
|
||||
) : null}
|
||||
<div className="flex items-center gap-3">
|
||||
{selectedDataset ? (
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{selectedDataset.validation.has_gold_label}/
|
||||
{selectedDataset.row_count} 条含人工标签
|
||||
</span>
|
||||
) : null}
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={
|
||||
!selectedDataset ||
|
||||
!mapping.fields.gold_label ||
|
||||
busy === "label-map"
|
||||
}
|
||||
onClick={onSuggestLabelMapping}
|
||||
>
|
||||
{busy === "label-map" ? (
|
||||
<LoaderCircleIcon className="animate-spin" />
|
||||
) : (
|
||||
<SparklesIcon />
|
||||
)}
|
||||
生成标签映射
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid gap-x-5 gap-y-3 sm:grid-cols-2">
|
||||
{FIELD_OPTIONS.map(([key, label, required]) => (
|
||||
@@ -1609,6 +1692,55 @@ function ExperimentSetup(props: {
|
||||
</label>
|
||||
))}
|
||||
</div>
|
||||
{Object.keys(mapping.label_map).length ? (
|
||||
<div className="mt-5 border-t pt-4">
|
||||
<div className="mb-2 font-medium text-xs">标签映射</div>
|
||||
<div className="grid gap-2">
|
||||
{Object.entries(mapping.label_map).map(([source, target]) => (
|
||||
<div
|
||||
key={source}
|
||||
className="grid grid-cols-[minmax(0,1fr)_20px_minmax(0,1fr)_28px] items-center gap-2"
|
||||
>
|
||||
<code className="truncate rounded bg-muted px-2 py-1.5 text-xs">
|
||||
{source}
|
||||
</code>
|
||||
<span className="text-center text-muted-foreground">→</span>
|
||||
<textarea
|
||||
value={target}
|
||||
onChange={(event) =>
|
||||
setMapping({
|
||||
...mapping,
|
||||
label_map: {
|
||||
...mapping.label_map,
|
||||
[source]: event.target.value,
|
||||
},
|
||||
})
|
||||
}
|
||||
rows={target.includes("\n") ? 2 : 1}
|
||||
className="min-h-8 resize-y rounded-md border bg-transparent px-3 py-1.5 font-mono text-xs outline-none focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
title="删除映射"
|
||||
onClick={() => {
|
||||
const nextLabelMap = { ...mapping.label_map };
|
||||
delete nextLabelMap[source];
|
||||
setMapping({ ...mapping, label_map: nextLabelMap });
|
||||
}}
|
||||
className="grid size-7 place-items-center rounded text-muted-foreground hover:bg-muted hover:text-foreground"
|
||||
>
|
||||
<XCircleIcon className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{labelMappingNote ? (
|
||||
<p className="mt-2 text-muted-foreground text-xs">
|
||||
{labelMappingNote}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
{mappedPreview.length ? (
|
||||
<div className="mt-5 overflow-hidden rounded-md border">
|
||||
<div className="grid grid-cols-[72px_minmax(0,1fr)_140px] gap-3 border-b bg-muted/40 px-3 py-2 font-medium text-muted-foreground text-xs">
|
||||
@@ -1647,7 +1779,7 @@ function ExperimentSetup(props: {
|
||||
<input
|
||||
type="range"
|
||||
min={1}
|
||||
max={metadata?.max_concurrency ?? 2}
|
||||
max={metadata?.max_concurrency ?? 8}
|
||||
value={concurrency}
|
||||
onChange={(event) => setConcurrency(Number(event.target.value))}
|
||||
className="min-w-0 flex-1 accent-foreground"
|
||||
@@ -1664,9 +1796,7 @@ function ExperimentSetup(props: {
|
||||
<Button
|
||||
size="lg"
|
||||
onClick={onCreate}
|
||||
disabled={
|
||||
!selectedDataset || !mapping.fields.query || busy === "create"
|
||||
}
|
||||
disabled={!selectedDataset || !mapping.fields.query || Boolean(busy)}
|
||||
>
|
||||
{busy === "create" ? (
|
||||
<LoaderCircleIcon className="animate-spin" />
|
||||
|
||||
Reference in New Issue
Block a user