feat: add evaluation label mapping

2026-07-24 17:34:32 +08:00
parent e2bf012f60
commit 174d1d3d01
4 changed files with 653 additions and 31 deletions
@@ -47,7 +47,12 @@ from src.agent_types import (
 )
 from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills
 from src.data_agent_inputs import DataAgentInputError, load_input_sources
-from src.evaluation_runtime import EvaluationError, EvaluationRuntime
+from src.evaluation_runtime import (
+    DEFAULT_CONCURRENCY,
+    MAX_CONCURRENCY,
+    EvaluationError,
+    EvaluationRuntime,
+)
 from src.jupyter_runtime import (
    DEFAULT_JUPYTER_WORKSPACE_ROOT,
    JupyterRuntimeError,
@@ -1145,6 +1150,14 @@ class EvaluationDatasetMappingRequest(BaseModel):
    mapping: dict[str, Any]


+class EvaluationLabelMappingSuggestRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    mapping: dict[str, Any]
+    skill_name: str = 'label-master'
+    snapshot_id: str | None = None
+    model: str | None = None
+
+
 class EvaluationAnalyzeRequest(BaseModel):
    account_id: str = Field(min_length=1)
    query: str = Field(min_length=1, max_length=20_000)
@@ -1165,7 +1178,11 @@ class EvaluationExperimentCreateRequest(BaseModel):
    skill_name: str = 'label-master'
    snapshot_id: str | None = None
    model: str | None = None
-    concurrency: int = Field(default=2, ge=1, le=8)
+    concurrency: int = Field(
+        default=DEFAULT_CONCURRENCY,
+        ge=1,
+        le=MAX_CONCURRENCY,
+    )


 class EvaluationSkillVersionSaveRequest(BaseModel):
@@ -1702,6 +1719,24 @@ def create_app(state: AgentState) -> FastAPI:
        except EvaluationError as exc:
            raise HTTPException(status_code=400, detail=str(exc)) from exc

+    @app.post('/api/evaluations/datasets/{dataset_id}/label-mapping/suggest')
+    async def suggest_evaluation_label_mapping(
+        dataset_id: str,
+        payload: EvaluationLabelMappingSuggestRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.suggest_label_mapping,
+                dataset_id,
+                _safe_account_id(payload.account_id),
+                mapping=payload.mapping,
+                skill_name=payload.skill_name,
+                snapshot_id=payload.snapshot_id,
+                model=payload.model,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
    @app.get('/api/evaluations/experiments')
    async def list_evaluation_experiments(
        account_id: str,
@@ -20,6 +20,7 @@ import {
 	RotateCcwIcon,
 	SaveIcon,
 	SearchIcon,
+	SparklesIcon,
 	SquareIcon,
 	UploadIcon,
 	XCircleIcon,
@@ -126,6 +127,16 @@ type Dataset = {
 	};
 };

+type LabelMappingSuggestion = {
+	label_map: Record<string, string>;
+	unmapped: string[];
+	reason: string;
+	raw_labels: string[];
+	skill_name: string;
+	skill_version: string;
+	model: string;
+};
+
 type CanonicalCase = {
 	case_id: string;
 	query: string;
@@ -270,7 +281,7 @@ function EvaluationWorkspace() {
 	const [skillVersions, setSkillVersions] = useState<SkillVersion[]>([]);
 	const [snapshotId, setSnapshotId] = useState("");
 	const [model, setModel] = useState("");
-	const [concurrency, setConcurrency] = useState(2);
+	const [concurrency, setConcurrency] = useState(8);
 	const [experimentName, setExperimentName] = useState("");
 	const [singleQuery, setSingleQuery] = useState("");
 	const [singleResult, setSingleResult] = useState<EvaluationCase | null>(null);
@@ -281,6 +292,7 @@ function EvaluationWorkspace() {
 	const [queryFilter, setQueryFilter] = useState("");
 	const [busy, setBusy] = useState("");
 	const [error, setError] = useState("");
+	const [labelMappingNote, setLabelMappingNote] = useState("");
 	const [skillEditorOpen, setSkillEditorOpen] = useState(false);
 	const [skillEditorBusy, setSkillEditorBusy] = useState("");
 	const [skillEditorError, setSkillEditorError] = useState("");
@@ -386,6 +398,9 @@ function EvaluationWorkspace() {
 				? ((await stateResponse.json()) as ClawState)
 				: {};
 			setMetadata(metadataPayload);
+			setConcurrency((current) =>
+				Math.max(1, Math.min(current, metadataPayload.max_concurrency)),
+			);
 			setSingleAnalyses(analysisPayload);
 			setDatasets(datasetPayload);
 			setExperiments(experimentPayload);
@@ -425,6 +440,7 @@ function EvaluationWorkspace() {
 			if (nextDataset && nextDataset.id !== selectedDatasetIdRef.current) {
 				setSelectedDatasetId(nextDataset.id);
 				setMapping(nextDataset.mapping);
+				setLabelMappingNote("");
 			}
 			const nextExperiment =
 				experimentPayload.find(
@@ -605,6 +621,7 @@ function EvaluationWorkspace() {
 			setDatasets((current) => [payload, ...current]);
 			setSelectedDatasetId(payload.id);
 			setMapping(payload.mapping);
+			setLabelMappingNote("");
 			setExperimentName(
 				`${metadata?.default_skill || "label-master"} · ${payload.name}`,
 			);
@@ -635,6 +652,46 @@ function EvaluationWorkspace() {
 		return payload;
 	}

+	async function suggestLabelMapping() {
+		if (!selectedDataset) return;
+		if (!mapping.fields.gold_label) {
+			setError("请先映射人工标签字段");
+			return;
+		}
+		setBusy("label-map");
+		setError("");
+		setLabelMappingNote("");
+		try {
+			const response = await fetch(
+				`/api/claw/evaluations/datasets/${encodeURIComponent(selectedDataset.id)}/label-mapping/suggest`,
+				{
+					method: "POST",
+					headers: { "content-type": "application/json" },
+					body: JSON.stringify({
+						mapping,
+						skill_name: skillName,
+						snapshot_id: snapshotId || undefined,
+						model: model || undefined,
+					}),
+				},
+			);
+			const payload = await readJson<LabelMappingSuggestion>(response);
+			setMapping((current) => ({
+				...current,
+				label_map: payload.label_map,
+			}));
+			const notes = [
+				payload.reason,
+				payload.unmapped.length ? `未映射：${payload.unmapped.join("、")}` : "",
+			].filter(Boolean);
+			setLabelMappingNote(notes.join(" "));
+		} catch (reason) {
+			setError(messageFrom(reason));
+		} finally {
+			setBusy("");
+		}
+	}
+
 	async function createAndStart() {
 		if (!selectedDataset) return;
 		if (!mapping.fields.query) {
@@ -1084,6 +1141,7 @@ function EvaluationWorkspace() {
 									const next = datasets.find((item) => item.id === value);
 									if (next) {
 										setMapping(next.mapping);
+										setLabelMappingNote("");
 										setExperimentName(`${skillName} · ${next.name}`);
 									}
 								}}
@@ -1096,6 +1154,8 @@ function EvaluationWorkspace() {
 								uploadRef={uploadRef}
 								onUpload={uploadDataset}
 								onCreate={createAndStart}
+								onSuggestLabelMapping={suggestLabelMapping}
+								labelMappingNote={labelMappingNote}
 								busy={busy}
 							/>
 						)}
@@ -1476,6 +1536,8 @@ function ExperimentSetup(props: {
 	uploadRef: React.RefObject<HTMLInputElement | null>;
 	onUpload: (file: File) => void;
 	onCreate: () => void;
+	onSuggestLabelMapping: () => void;
+	labelMappingNote: string;
 	busy: string;
 }) {
 	const {
@@ -1493,6 +1555,8 @@ function ExperimentSetup(props: {
 		uploadRef,
 		onUpload,
 		onCreate,
+		onSuggestLabelMapping,
+		labelMappingNote,
 		busy,
 	} = props;
 	const mappedPreview = useMemo(
@@ -1566,14 +1630,33 @@ function ExperimentSetup(props: {
 			</section>

 			<section className="mt-4 border-y py-5 sm:rounded-md sm:border sm:px-5">
-				<div className="mb-4 flex items-center justify-between">
+				<div className="mb-4 flex items-center justify-between gap-3">
 					<h3 className="font-medium text-sm">2. 字段映射</h3>
-					{selectedDataset ? (
-						<span className="text-muted-foreground text-xs">
-							{selectedDataset.validation.has_gold_label}/
-							{selectedDataset.row_count} 条含人工标签
-						</span>
-					) : null}
+					<div className="flex items-center gap-3">
+						{selectedDataset ? (
+							<span className="text-muted-foreground text-xs">
+								{selectedDataset.validation.has_gold_label}/
+								{selectedDataset.row_count} 条含人工标签
+							</span>
+						) : null}
+						<Button
+							variant="outline"
+							size="sm"
+							disabled={
+								!selectedDataset ||
+								!mapping.fields.gold_label ||
+								busy === "label-map"
+							}
+							onClick={onSuggestLabelMapping}
+						>
+							{busy === "label-map" ? (
+								<LoaderCircleIcon className="animate-spin" />
+							) : (
+								<SparklesIcon />
+							)}
+							生成标签映射
+						</Button>
+					</div>
 				</div>
 				<div className="grid gap-x-5 gap-y-3 sm:grid-cols-2">
 					{FIELD_OPTIONS.map(([key, label, required]) => (
@@ -1609,6 +1692,55 @@ function ExperimentSetup(props: {
 						</label>
 					))}
 				</div>
+				{Object.keys(mapping.label_map).length ? (
+					<div className="mt-5 border-t pt-4">
+						<div className="mb-2 font-medium text-xs">标签映射</div>
+						<div className="grid gap-2">
+							{Object.entries(mapping.label_map).map(([source, target]) => (
+								<div
+									key={source}
+									className="grid grid-cols-[minmax(0,1fr)_20px_minmax(0,1fr)_28px] items-center gap-2"
+								>
+									<code className="truncate rounded bg-muted px-2 py-1.5 text-xs">
+										{source}
+									</code>
+									<span className="text-center text-muted-foreground">→</span>
+									<textarea
+										value={target}
+										onChange={(event) =>
+											setMapping({
+												...mapping,
+												label_map: {
+													...mapping.label_map,
+													[source]: event.target.value,
+												},
+											})
+										}
+										rows={target.includes("\n") ? 2 : 1}
+										className="min-h-8 resize-y rounded-md border bg-transparent px-3 py-1.5 font-mono text-xs outline-none focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50"
+									/>
+									<button
+										type="button"
+										title="删除映射"
+										onClick={() => {
+											const nextLabelMap = { ...mapping.label_map };
+											delete nextLabelMap[source];
+											setMapping({ ...mapping, label_map: nextLabelMap });
+										}}
+										className="grid size-7 place-items-center rounded text-muted-foreground hover:bg-muted hover:text-foreground"
+									>
+										<XCircleIcon className="size-4" />
+									</button>
+								</div>
+							))}
+						</div>
+						{labelMappingNote ? (
+							<p className="mt-2 text-muted-foreground text-xs">
+								{labelMappingNote}
+							</p>
+						) : null}
+					</div>
+				) : null}
 				{mappedPreview.length ? (
 					<div className="mt-5 overflow-hidden rounded-md border">
 						<div className="grid grid-cols-[72px_minmax(0,1fr)_140px] gap-3 border-b bg-muted/40 px-3 py-2 font-medium text-muted-foreground text-xs">
@@ -1647,7 +1779,7 @@ function ExperimentSetup(props: {
 							<input
 								type="range"
 								min={1}
-								max={metadata?.max_concurrency ?? 2}
+								max={metadata?.max_concurrency ?? 8}
 								value={concurrency}
 								onChange={(event) => setConcurrency(Number(event.target.value))}
 								className="min-w-0 flex-1 accent-foreground"
@@ -1664,9 +1796,7 @@ function ExperimentSetup(props: {
 				<Button
 					size="lg"
 					onClick={onCreate}
-					disabled={
-						!selectedDataset || !mapping.fields.query || busy === "create"
-					}
+					disabled={!selectedDataset || !mapping.fields.query || Boolean(busy)}
 				>
 					{busy === "create" ? (
 						<LoaderCircleIcon className="animate-spin" />
@@ -32,9 +32,10 @@ from src.bundled_skills import BundledSkill, load_directory_skills
 TERMINAL_CASE_STATUSES = {'completed', 'failed', 'cancelled'}
 ACTIVE_EXPERIMENT_STATUSES = {'queued', 'running', 'pausing'}
 SUPPORTED_DATASET_SUFFIXES = {'.csv', '.tsv', '.json', '.jsonl', '.xlsx'}
-DEFAULT_CONCURRENCY = 2
-MAX_CONCURRENCY = 8
+DEFAULT_CONCURRENCY = 8
+MAX_CONCURRENCY = 32
 MAX_DATASET_ROWS = 20_000
+MAX_LABEL_MAPPING_VALUES = 200
 VISIBLE_SKILL_FILE_SUFFIXES = {'.json', '.md', '.py', '.txt', '.yaml', '.yml'}
 EDITABLE_SKILL_FILE_SUFFIXES = {'.json', '.md'}
 MAX_SKILL_FILE_BYTES = 512 * 1024
@@ -75,6 +76,27 @@ EVALUATION_OUTPUT_SCHEMA = OutputSchemaConfig(
    },
 )

+LABEL_MAPPING_OUTPUT_SCHEMA = OutputSchemaConfig(
+    name='evaluation_label_mapping',
+    strict=False,
+    schema={
+        'type': 'object',
+        'properties': {
+            'label_map': {
+                'type': 'object',
+                'additionalProperties': {'type': 'string'},
+            },
+            'unmapped': {
+                'type': 'array',
+                'items': {'type': 'string'},
+            },
+            'reason': {'type': 'string'},
+        },
+        'required': ['label_map', 'unmapped', 'reason'],
+        'additionalProperties': False,
+    },
+)
+

@dataclass
 class ExperimentControl:
@@ -489,6 +511,78 @@ class EvaluationRuntime:
        payload['validation'] = validation
        return payload

+    def suggest_label_mapping(
+        self,
+        dataset_id: str,
+        account_id: str,
+        *,
+        mapping: dict[str, Any] | None,
+        skill_name: str,
+        snapshot_id: str | None = None,
+        model: str | None = None,
+    ) -> dict[str, Any]:
+        dataset = self.get_dataset(dataset_id, account_id, include_rows=True)
+        normalized_mapping = normalize_mapping(mapping or dataset['mapping'])
+        gold_field = normalized_mapping['fields']['gold_label']
+        if not gold_field:
+            raise EvaluationError('请先映射人工标签字段')
+        raw_labels: list[str] = []
+        seen: set[str] = set()
+        for row in dataset['rows']:
+            value = stringify_value(resolve_field(row, gold_field)).strip()
+            if not value or value in seen:
+                continue
+            seen.add(value)
+            raw_labels.append(value)
+            if len(raw_labels) >= MAX_LABEL_MAPPING_VALUES:
+                break
+        if not raw_labels:
+            raise EvaluationError('人工标签字段中没有可用于映射的值')
+
+        snapshot = self.resolve_skill_snapshot(
+            account_id,
+            skill_name,
+            snapshot_id=snapshot_id,
+        )
+        snapshot_root = Path(snapshot['path'])
+        skill = self._load_snapshot_skill(snapshot_root, skill_name)
+        model_name = model or self.model_config_for(account_id).model
+        run_id = f'label_map_{uuid4().hex[:16]}'
+        case_row_id = f'dataset_{sanitize_filename(dataset_id)}'
+        cancel_event = threading.Event()
+        agent = self._build_evaluation_agent(
+            account_id=account_id,
+            experiment_id=run_id,
+            case_row_id=case_row_id,
+            model=model_name,
+            snapshot_root=snapshot_root,
+            skill=skill,
+            cancel_event=cancel_event,
+            output_schema=LABEL_MAPPING_OUTPUT_SCHEMA,
+        )
+        prompt, runtime_context = build_label_mapping_prompt(
+            skill=skill,
+            snapshot_root=snapshot_root,
+            raw_labels=raw_labels,
+        )
+        run_result = agent.run(
+            prompt,
+            session_id=f'{run_id}_{case_row_id}',
+            runtime_context=runtime_context,
+        )
+        result = normalize_label_mapping_output(
+            run_result.final_output,
+            raw_labels=raw_labels,
+            skill_name=skill_name,
+        )
+        return {
+            **result,
+            'raw_labels': raw_labels,
+            'skill_name': skill_name,
+            'skill_version': snapshot['content_hash'][:12],
+            'model': model_name,
+        }
+
    def analyze_case(
        self,
        *,
@@ -909,7 +1003,30 @@ class EvaluationRuntime:
            params.extend(case_ids)
        elif disagreements_only:
            where.append("status = 'completed'")
-            where.append("gold_label <> '' AND prediction <> gold_label")
+            completed_rows = self.store.fetchall(
+                """
+                SELECT id, gold_label, prediction, complex_json
+                FROM evaluation_cases
+                WHERE experiment_id = ? AND status = 'completed'
+                  AND gold_label <> ''
+                """,
+                (experiment_id,),
+            )
+            disagreement_ids = [
+                row['id']
+                for row in completed_rows
+                if not labels_equivalent(
+                    row.get('gold_label') or '',
+                    row.get('prediction') or '',
+                    parse_complex_json(row.get('complex_json')),
+                )
+            ]
+            if disagreement_ids:
+                placeholders = ','.join('?' for _ in disagreement_ids)
+                where.append(f'id IN ({placeholders})')
+                params.extend(disagreement_ids)
+            else:
+                where.append('1 = 0')
        else:
            where.append("status IN ('failed', 'cancelled')")
        self.store.execute(
@@ -1734,6 +1851,7 @@ class EvaluationRuntime:
        snapshot_root: Path,
        skill: BundledSkill,
        cancel_event: threading.Event,
+        output_schema: OutputSchemaConfig = EVALUATION_OUTPUT_SCHEMA,
    ) -> LocalCodingAgent:
        account_paths = self.account_paths_for(account_id)
        snapshot_skill_dir = snapshot_root / 'skills' / skill.name
@@ -1769,7 +1887,7 @@ class EvaluationRuntime:
            ),
            additional_working_directories=(snapshot_skill_dir,),
            disable_claude_md_discovery=True,
-            output_schema=EVALUATION_OUTPUT_SCHEMA,
+            output_schema=output_schema,
            session_directory=session_dir,
            scratchpad_root=scratchpad_root,
            python_env_dir=account_paths.get('python_env'),
@@ -2032,8 +2150,9 @@ class EvaluationRuntime:
    def _case_payload(self, row: dict[str, Any]) -> dict[str, Any]:
        gold_label = row.get('gold_label') or ''
        prediction = row.get('prediction') or ''
+        complex_value = parse_complex_json(row.get('complex_json'))
        correct = (
-            labels_equivalent(gold_label, prediction)
+            labels_equivalent(gold_label, prediction, complex_value)
            if gold_label and prediction and row.get('status') == 'completed'
            else None
        )
@@ -2045,7 +2164,7 @@ class EvaluationRuntime:
            'gold_label': gold_label,
            'prediction': prediction,
            'function_output': row.get('function_output') or '',
-            'complex': json.loads(row.get('complex_json') or 'null'),
+            'complex': complex_value,
            'correct': correct,
            'reason': row.get('reason') or '',
            'candidates': json.loads(row.get('candidates_json') or '[]'),
@@ -2575,22 +2694,96 @@ def build_evaluation_prompt(
 不要把人工标签当作输入依据；下面数据不会提供 gold_label。

 <case>
-{json.dumps({key: value for key, value in canonical_case.items() if key != 'gold_label'}, ensure_ascii=False, indent=2)}
+{json.dumps(evaluation_case_for_prompt(canonical_case), ensure_ascii=False, indent=2)}
 </case>
 """.strip()
    return prompt, runtime_context


+def build_label_mapping_prompt(
+    *,
+    skill: BundledSkill,
+    snapshot_root: Path,
+    raw_labels: list[str],
+) -> tuple[str, str]:
+    snapshot_skill_dir = snapshot_root / 'skills' / skill.name
+    skill_instructions = skill.get_prompt(
+        None,  # type: ignore[arg-type]
+        '离线评测标签映射模式：只比较标签表达，不分析具体 case。',
+    )
+    if skill.name == 'label-fast-slow-routing':
+        target_contract = """
+本 Skill 的主标签规范值为：`快`、`慢`、`模糊`。
+复杂度是独立维度，规范写法为 `complex=true` 或 `complex=false`。
+同一原始值同时包含复杂度与主标签时，用两行保留，例如：
+
+complex=true
+慢
+
+`慢系统`与`慢`等价，`快系统`与`快`等价；`Ture`按明显拼写错误
+`true`处理。不得把只有复杂度的信息猜成快慢标签。
+""".strip()
+    else:
+        target_contract = (
+            '规范值必须遵循该 Skill 的实际输出协议。'
+            '只合并表达形式不同但语义完全相同的标签，不要创造新类别。'
+        )
+    prompt = f"""
+你正在为离线评测生成一次性标签映射。
+
+指定 Skill：{skill.name}
+不可变快照目录：{snapshot_skill_dir}
+
+{skill_instructions}
+
+{target_contract}
+
+要求：
+1. 第一步必须调用 `read_file`、`grep_search` 或 `glob_search`，读取快照中
+   至少一份直接定义输出标签或判断协议的文件。
+2. 只比较下面的原始人工标签，不读取 query，不进行逐条分类。
+3. `label_map` 的 key 必须原样使用输入值，value 使用规范标签。
+4. 可以返回恒等映射，便于用户检查。
+5. 无法可靠对应的值放入 `unmapped`，不要猜测。
+6. 最终只返回符合输出 schema 的 JSON。
+
+<raw_labels>
+{json.dumps(raw_labels, ensure_ascii=False, indent=2)}
+</raw_labels>
+""".strip()
+    runtime_context = f"""
+离线标签映射运行时。
+当前工作目录：{snapshot_skill_dir}
+只能读取当前 Skill 快照；禁止修改文件。
+""".strip()
+    return prompt, runtime_context
+
+
+PROMPT_CASE_FIELDS = (
+    'case_id',
+    'query',
+    'history',
+    'context',
+    'domain',
+    'request_id',
+    'metadata',
+)
+
+
+def evaluation_case_for_prompt(canonical_case: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: canonical_case[key]
+        for key in PROMPT_CASE_FIELDS
+        if key in canonical_case
+    }
+
+
 def build_evidence_retry_prompt(
    skill_name: str,
    reason: str,
    canonical_case: dict[str, Any],
 ) -> str:
-    case_payload = {
-        key: value
-        for key, value in canonical_case.items()
-        if key != 'gold_label'
-    }
+    case_payload = evaluation_case_for_prompt(canonical_case)
    return f"""
 上一轮结果不能进入评测指标：{reason}。

@@ -2732,6 +2925,101 @@ def normalize_evaluation_output(raw_output: str) -> dict[str, Any]:
    }


+def normalize_label_mapping_output(
+    raw_output: str,
+    *,
+    raw_labels: list[str],
+    skill_name: str,
+) -> dict[str, Any]:
+    payload = parse_json_object_output(
+        raw_output,
+        error_message='模型返回的标签映射不是合法 JSON',
+    )
+    raw_mapping = payload.get('label_map')
+    if not isinstance(raw_mapping, dict):
+        raise EvaluationError('模型标签映射缺少 label_map')
+    by_stripped = {value.strip(): value for value in raw_labels}
+    label_map = deterministic_label_mapping(raw_labels, skill_name=skill_name)
+    for raw_key, raw_value in raw_mapping.items():
+        source = by_stripped.get(str(raw_key).strip())
+        if source is None:
+            continue
+        target = canonicalize_mapping_target(
+            str(raw_value),
+            skill_name=skill_name,
+        )
+        if target:
+            label_map[source] = target
+    unmapped = [
+        value
+        for value in raw_labels
+        if value not in label_map
+    ]
+    return {
+        'label_map': label_map,
+        'unmapped': unmapped,
+        'reason': str(payload.get('reason') or '').strip(),
+    }
+
+
+def parse_json_object_output(
+    raw_output: str,
+    *,
+    error_message: str,
+) -> dict[str, Any]:
+    text = raw_output.strip()
+    if text.startswith('```'):
+        text = re.sub(r'^```(?:json)?\s*', '', text)
+        text = re.sub(r'\s*```$', '', text)
+    try:
+        payload = json.loads(text)
+    except json.JSONDecodeError:
+        match = re.search(r'\{.*\}', text, re.DOTALL)
+        if not match:
+            raise EvaluationError(error_message)
+        try:
+            payload = json.loads(match.group(0))
+        except json.JSONDecodeError as exc:
+            raise EvaluationError(error_message) from exc
+    if not isinstance(payload, dict):
+        raise EvaluationError(error_message)
+    return payload
+
+
+def deterministic_label_mapping(
+    raw_labels: list[str],
+    *,
+    skill_name: str,
+) -> dict[str, str]:
+    if skill_name != 'label-fast-slow-routing':
+        return {}
+    result: dict[str, str] = {}
+    for raw_label in raw_labels:
+        target = canonicalize_mapping_target(
+            raw_label,
+            skill_name=skill_name,
+        )
+        if target:
+            result[raw_label] = target
+    return result
+
+
+def canonicalize_mapping_target(value: str, *, skill_name: str) -> str:
+    text = str(value or '').strip()
+    if not text:
+        return ''
+    if skill_name != 'label-fast-slow-routing':
+        return text
+    label = normalize_label_value(text)
+    complex_value = extract_complex_value(text)
+    parts: list[str] = []
+    if complex_value is not None:
+        parts.append(f"complex={'true' if complex_value else 'false'}")
+    if label in {'快', '慢', '模糊'}:
+        parts.append(label)
+    return '\n'.join(parts)
+
+
 def clean_string_list(value: Any) -> list[str]:
    if isinstance(value, str):
        return [value] if value.strip() else []
@@ -2883,8 +3171,31 @@ def calculate_metrics(cases: list[dict[str, Any]]) -> dict[str, Any]:
    }


-def labels_equivalent(gold: str, prediction: str) -> bool:
-    return normalize_label_value(gold) == normalize_label_value(prediction)
+def labels_equivalent(
+    gold: str,
+    prediction: str,
+    prediction_complex: bool | None = None,
+) -> bool:
+    gold_label = normalize_label_value(gold)
+    prediction_label = normalize_label_value(prediction)
+    gold_complex = extract_complex_value(gold)
+    embedded_prediction_complex = extract_complex_value(prediction)
+    effective_prediction_complex = (
+        prediction_complex
+        if prediction_complex is not None
+        else embedded_prediction_complex
+    )
+
+    if gold_label and gold_label != prediction_label:
+        return False
+    if not gold_label and gold_complex is None:
+        return prediction_label == ''
+    if gold_complex is not None:
+        if effective_prediction_complex is None:
+            return bool(gold_label)
+        if gold_complex != effective_prediction_complex:
+            return False
+    return True


 def normalize_label_value(value: str) -> str:
@@ -2896,8 +3207,53 @@ def normalize_label_value(value: str) -> str:
    )
    if agent_match:
        value = agent_match.group(1)
-    value = re.sub(r'^complex\s*=\s*(?:true|false)\s*', '', value, flags=re.I)
-    return re.sub(r'\s+', '', value).lower()
+    value = re.sub(
+        r'complex\s*[:=]\s*(?:true|ture|false)',
+        '',
+        value,
+        flags=re.I,
+    )
+    value = re.sub(
+        r'^(?:prediction|label|route|标签|路由)\s*[:=：]\s*',
+        '',
+        value,
+        flags=re.I,
+    )
+    normalized = re.sub(r'[\s,，;；|]+', '', value).lower()
+    aliases = {
+        '快系统': '快',
+        '快速系统': '快',
+        'fast': '快',
+        'fastsystem': '快',
+        '慢系统': '慢',
+        '慢速系统': '慢',
+        'slow': '慢',
+        'slowsystem': '慢',
+        '模糊系统': '模糊',
+        'ambiguous': '模糊',
+    }
+    return aliases.get(normalized, normalized)
+
+
+def extract_complex_value(value: str) -> bool | None:
+    match = re.search(
+        r'complex\s*[:=]\s*(true|ture|false)',
+        str(value or ''),
+        flags=re.I,
+    )
+    if not match:
+        return None
+    return match.group(1).lower() in {'true', 'ture'}
+
+
+def parse_complex_json(value: Any) -> bool | None:
+    if isinstance(value, bool):
+        return value
+    try:
+        parsed = json.loads(str(value or 'null'))
+    except (TypeError, ValueError, json.JSONDecodeError):
+        return None
+    return parsed if isinstance(parsed, bool) else None


 def hash_directory(directory: Path) -> str:
@@ -16,6 +16,7 @@ from src.evaluation_runtime import (
    calculate_metrics,
    extract_accessed_refs,
    labels_equivalent,
+    normalize_label_mapping_output,
    normalize_label_master_result,
    normalize_evaluation_output,
    parse_dataset_bytes,
@@ -288,6 +289,22 @@ class EvaluationRuntimeTests(unittest.TestCase):
                '地图导航',
            )
        )
+        self.assertTrue(labels_equivalent('慢系统', '慢'))
+        self.assertTrue(
+            labels_equivalent(
+                'complex=Ture\n慢系统',
+                '慢',
+                True,
+            )
+        )
+        self.assertTrue(labels_equivalent('complex=Ture', '慢', True))
+        self.assertFalse(
+            labels_equivalent(
+                'complex=Ture\n慢系统',
+                '慢',
+                False,
+            )
+        )
        snapshot_metadata = json.dumps(
            {
                'label_catalog': {
@@ -340,6 +357,24 @@ class EvaluationRuntimeTests(unittest.TestCase):
            normalized_label['function_output'],
            'Agent(tag="地图导航")',
        )
+        mapping = normalize_label_mapping_output(
+            json.dumps(
+                {
+                    'label_map': {
+                        '慢系统': '慢',
+                        'complex=Ture': 'complex=true',
+                    },
+                    'unmapped': [],
+                    'reason': '统一快慢和复杂度格式',
+                },
+                ensure_ascii=False,
+            ),
+            raw_labels=['慢系统', 'complex=Ture', '未知'],
+            skill_name='label-fast-slow-routing',
+        )
+        self.assertEqual(mapping['label_map']['慢系统'], '慢')
+        self.assertEqual(mapping['label_map']['complex=Ture'], 'complex=true')
+        self.assertEqual(mapping['unmapped'], ['未知'])

    def test_fast_slow_prompt_uses_skill_contract_and_relative_paths(self) -> None:
        skill = BundledSkill(
@@ -351,7 +386,15 @@ class EvaluationRuntimeTests(unittest.TestCase):
        prompt, runtime_context = build_evaluation_prompt(
            skill=skill,
            snapshot_root=Path('/tmp/evaluation-snapshot'),
-            canonical_case={'query': '打开空调', 'gold_label': '快'},
+            canonical_case={
+                'query': '打开空调',
+                'gold_label': '快',
+                'source_row': {
+                    'query': '打开空调',
+                    '人工标签': '快',
+                    'secret': 'should-not-leak',
+                },
+            },
        )
        self.assertIn('填写“快”“慢”或“模糊”', prompt)
        self.assertNotIn('填写 fast 或 slow', prompt)
@@ -360,6 +403,64 @@ class EvaluationRuntimeTests(unittest.TestCase):
            '不要添加 `skills/label-fast-slow-routing/` 前缀',
            prompt,
        )
+        self.assertNotIn('should-not-leak', prompt)
+        self.assertNotIn('"人工标签"', prompt)
+
+    def test_label_mapping_suggestion_uses_one_agent_run(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            _write_fast_slow_skill(root)
+            runtime = EvaluationRuntime(
+                root=root / 'evaluations',
+                cwd_for_account=lambda _account: root,
+                model_config_for=lambda _account: ModelConfig(model='test-model'),
+                account_paths_for=lambda _account: {'python_env': root / '.venv'},
+                max_workers=1,
+            )
+            try:
+                dataset = runtime.create_dataset(
+                    account_id='alice',
+                    name='fast-slow',
+                    filename='fast-slow.csv',
+                    rows=[
+                        {'query': '打开空调', 'label': '快系统'},
+                        {'query': '帮我规划路线', 'label': '慢系统'},
+                    ],
+                )
+                fake_result = AgentRunResult(
+                    final_output=json.dumps(
+                        {
+                            'label_map': {
+                                '快系统': '快',
+                                '慢系统': '慢',
+                            },
+                            'unmapped': [],
+                            'reason': '统一路由标签',
+                        },
+                        ensure_ascii=False,
+                    ),
+                    turns=1,
+                    tool_calls=1,
+                    transcript=(),
+                    usage=UsageStats(input_tokens=10, output_tokens=5),
+                )
+                with patch(
+                    'src.evaluation_runtime.LocalCodingAgent.run',
+                    return_value=fake_result,
+                ) as run:
+                    suggestion = runtime.suggest_label_mapping(
+                        dataset['id'],
+                        'alice',
+                        mapping=dataset['mapping'],
+                        skill_name='label-fast-slow-routing',
+                    )
+                self.assertEqual(run.call_count, 1)
+                self.assertEqual(
+                    suggestion['label_map'],
+                    {'快系统': '快', '慢系统': '慢'},
+                )
+            finally:
+                runtime.shutdown()

    def test_single_analysis_uses_snapshot_without_dataset_records(self) -> None:
        with tempfile.TemporaryDirectory() as directory: