feat: add evaluation label mapping

backend/api/server.py

@@ -47,7 +47,12 @@ from src.agent_types import (
 )
 from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills
 from src.data_agent_inputs import DataAgentInputError, load_input_sources
-from src.evaluation_runtime import EvaluationError, EvaluationRuntime
+from src.evaluation_runtime import (
+    DEFAULT_CONCURRENCY,
+    MAX_CONCURRENCY,
+    EvaluationError,
+    EvaluationRuntime,
+)
 from src.jupyter_runtime import (
     DEFAULT_JUPYTER_WORKSPACE_ROOT,
     JupyterRuntimeError,
@@ -1145,6 +1150,14 @@ class EvaluationDatasetMappingRequest(BaseModel):
     mapping: dict[str, Any]
 
 
+class EvaluationLabelMappingSuggestRequest(BaseModel):
+    account_id: str = Field(min_length=1)
+    mapping: dict[str, Any]
+    skill_name: str = 'label-master'
+    snapshot_id: str | None = None
+    model: str | None = None
+
+
 class EvaluationAnalyzeRequest(BaseModel):
     account_id: str = Field(min_length=1)
     query: str = Field(min_length=1, max_length=20_000)
@@ -1165,7 +1178,11 @@ class EvaluationExperimentCreateRequest(BaseModel):
     skill_name: str = 'label-master'
     snapshot_id: str | None = None
     model: str | None = None
-    concurrency: int = Field(default=2, ge=1, le=8)
+    concurrency: int = Field(
+        default=DEFAULT_CONCURRENCY,
+        ge=1,
+        le=MAX_CONCURRENCY,
+    )
 
 
 class EvaluationSkillVersionSaveRequest(BaseModel):
@@ -1702,6 +1719,24 @@ def create_app(state: AgentState) -> FastAPI:
         except EvaluationError as exc:
             raise HTTPException(status_code=400, detail=str(exc)) from exc
 
+    @app.post('/api/evaluations/datasets/{dataset_id}/label-mapping/suggest')
+    async def suggest_evaluation_label_mapping(
+        dataset_id: str,
+        payload: EvaluationLabelMappingSuggestRequest,
+    ) -> dict[str, Any]:
+        try:
+            return await asyncio.to_thread(
+                state.evaluation_runtime.suggest_label_mapping,
+                dataset_id,
+                _safe_account_id(payload.account_id),
+                mapping=payload.mapping,
+                skill_name=payload.skill_name,
+                snapshot_id=payload.snapshot_id,
+                model=payload.model,
+            )
+        except EvaluationError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
     @app.get('/api/evaluations/experiments')
     async def list_evaluation_experiments(
         account_id: str,