feat: add skill evaluation workbench
This commit is contained in:
wuyang6
@@ -47,6 +47,7 @@ from src.agent_types import (
|
||||
)
|
||||
from src.bundled_skills import ALWAYS_ENABLED_HIDDEN_SKILL_NAMES, get_bundled_skills
|
||||
from src.data_agent_inputs import DataAgentInputError, load_input_sources
|
||||
from src.evaluation_runtime import EvaluationError, EvaluationRuntime
|
||||
from src.jupyter_runtime import (
|
||||
DEFAULT_JUPYTER_WORKSPACE_ROOT,
|
||||
JupyterRuntimeError,
|
||||
@@ -586,6 +587,12 @@ class AgentState:
|
||||
self.model_config_for,
|
||||
)
|
||||
self.memory_manager.start()
|
||||
self.evaluation_runtime = EvaluationRuntime(
|
||||
root=self.session_directory.parent / 'evaluations',
|
||||
cwd_for_account=lambda account_id: self.config_for(account_id).cwd,
|
||||
model_config_for=self.model_config_for,
|
||||
account_paths_for=self.account_paths,
|
||||
)
|
||||
|
||||
@property
|
||||
def cwd(self) -> Path:
|
||||
@@ -1125,6 +1132,56 @@ class SkillSyncRequest(BaseModel):
|
||||
account_id: str | None = None
|
||||
|
||||
|
||||
class EvaluationDatasetCreateRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
name: str = ''
|
||||
filename: str = Field(min_length=1)
|
||||
content_base64: str | None = None
|
||||
rows: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
class EvaluationDatasetMappingRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
mapping: dict[str, Any]
|
||||
|
||||
|
||||
class EvaluationAnalyzeRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
query: str = Field(min_length=1, max_length=20_000)
|
||||
skill_name: str = 'label-master'
|
||||
model: str | None = None
|
||||
history: list[dict[str, Any]] = Field(default_factory=list)
|
||||
context: dict[str, Any] = Field(default_factory=dict)
|
||||
domain: str = ''
|
||||
request_id: str = ''
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class EvaluationExperimentCreateRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
dataset_id: str = Field(min_length=1)
|
||||
name: str = ''
|
||||
skill_name: str = 'label-master'
|
||||
model: str | None = None
|
||||
concurrency: int = Field(default=2, ge=1, le=8)
|
||||
|
||||
|
||||
class EvaluationExperimentActionRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class EvaluationRetryRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
case_ids: list[str] | None = None
|
||||
disagreements_only: bool = False
|
||||
|
||||
|
||||
class EvaluationReviewRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
review_status: str = Field(max_length=80)
|
||||
review_note: str = Field(default='', max_length=4000)
|
||||
|
||||
|
||||
class SessionUpdate(BaseModel):
|
||||
title: str | None = Field(default=None, max_length=80)
|
||||
is_training: bool | None = None
|
||||
@@ -1178,6 +1235,7 @@ def create_app(state: AgentState) -> FastAPI:
|
||||
scanner_task.cancel()
|
||||
_watcher_manager.cancel_all()
|
||||
_bash_bg_manager.cancel_all()
|
||||
state.evaluation_runtime.shutdown()
|
||||
state.event_loop = None
|
||||
|
||||
app = FastAPI(title='Claw Code GUI', version='1.0', lifespan=lifespan)
|
||||
@@ -1468,6 +1526,222 @@ def create_app(state: AgentState) -> FastAPI:
|
||||
)
|
||||
return result
|
||||
|
||||
# ------------- Skill evaluations ---------------------------------------
|
||||
@app.get('/api/evaluations/metadata')
|
||||
async def evaluation_metadata(account_id: str) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.metadata(_safe_account_id(account_id))
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/analyze')
|
||||
async def analyze_evaluation_case(
|
||||
payload: EvaluationAnalyzeRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
state.evaluation_runtime.analyze_case,
|
||||
account_id=_safe_account_id(payload.account_id),
|
||||
query=payload.query,
|
||||
skill_name=payload.skill_name,
|
||||
model=payload.model,
|
||||
history=payload.history,
|
||||
context=payload.context,
|
||||
domain=payload.domain,
|
||||
request_id=payload.request_id,
|
||||
metadata=payload.metadata,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get('/api/evaluations/datasets')
|
||||
async def list_evaluation_datasets(account_id: str) -> list[dict[str, Any]]:
|
||||
return state.evaluation_runtime.list_datasets(_safe_account_id(account_id))
|
||||
|
||||
@app.post('/api/evaluations/datasets')
|
||||
async def create_evaluation_dataset(
|
||||
payload: EvaluationDatasetCreateRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
state.evaluation_runtime.create_dataset,
|
||||
account_id=_safe_account_id(payload.account_id),
|
||||
name=payload.name,
|
||||
filename=payload.filename,
|
||||
content_base64=payload.content_base64,
|
||||
rows=payload.rows,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get('/api/evaluations/datasets/{dataset_id}')
|
||||
async def get_evaluation_dataset(
|
||||
dataset_id: str,
|
||||
account_id: str,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.get_dataset(
|
||||
dataset_id,
|
||||
_safe_account_id(account_id),
|
||||
include_rows=False,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@app.patch('/api/evaluations/datasets/{dataset_id}/mapping')
|
||||
async def update_evaluation_dataset_mapping(
|
||||
dataset_id: str,
|
||||
payload: EvaluationDatasetMappingRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.update_dataset_mapping(
|
||||
dataset_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
payload.mapping,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get('/api/evaluations/experiments')
|
||||
async def list_evaluation_experiments(
|
||||
account_id: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
return state.evaluation_runtime.list_experiments(
|
||||
_safe_account_id(account_id)
|
||||
)
|
||||
|
||||
@app.post('/api/evaluations/experiments')
|
||||
async def create_evaluation_experiment(
|
||||
payload: EvaluationExperimentCreateRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
state.evaluation_runtime.create_experiment,
|
||||
account_id=_safe_account_id(payload.account_id),
|
||||
dataset_id=payload.dataset_id,
|
||||
name=payload.name,
|
||||
skill_name=payload.skill_name,
|
||||
model=payload.model,
|
||||
concurrency=payload.concurrency,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get('/api/evaluations/experiments/{experiment_id}')
|
||||
async def get_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
account_id: str,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.get_experiment(
|
||||
experiment_id,
|
||||
_safe_account_id(account_id),
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/experiments/{experiment_id}/start')
|
||||
async def start_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
payload: EvaluationExperimentActionRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.start(
|
||||
experiment_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/experiments/{experiment_id}/pause')
|
||||
async def pause_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
payload: EvaluationExperimentActionRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.pause(
|
||||
experiment_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/experiments/{experiment_id}/resume')
|
||||
async def resume_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
payload: EvaluationExperimentActionRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.resume(
|
||||
experiment_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/experiments/{experiment_id}/cancel')
|
||||
async def cancel_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
payload: EvaluationExperimentActionRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.cancel(
|
||||
experiment_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/experiments/{experiment_id}/retry')
|
||||
async def retry_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
payload: EvaluationRetryRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.retry(
|
||||
experiment_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
case_ids=payload.case_ids,
|
||||
disagreements_only=payload.disagreements_only,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.patch('/api/evaluations/cases/{case_id}/review')
|
||||
async def review_evaluation_case(
|
||||
case_id: str,
|
||||
payload: EvaluationReviewRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.update_review(
|
||||
case_id,
|
||||
_safe_account_id(payload.account_id),
|
||||
review_status=payload.review_status,
|
||||
review_note=payload.review_note,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get('/api/evaluations/experiments/{experiment_id}/export')
|
||||
async def export_evaluation_experiment(
|
||||
experiment_id: str,
|
||||
account_id: str,
|
||||
) -> Response:
|
||||
try:
|
||||
filename, content = state.evaluation_runtime.export_csv(
|
||||
experiment_id,
|
||||
_safe_account_id(account_id),
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
return Response(
|
||||
content,
|
||||
media_type='text/csv; charset=utf-8',
|
||||
headers={
|
||||
'Content-Disposition': f"attachment; filename*=UTF-8''{quote(filename)}"
|
||||
},
|
||||
)
|
||||
|
||||
# ------------- memory ----------------------------------------------------
|
||||
@app.get('/api/memory/user')
|
||||
async def get_user_memory(account_id: str) -> dict[str, Any]:
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { getCurrentAccount } from "@/lib/claw-auth";
|
||||
|
||||
const CLAW_API_URL = process.env.CLAW_API_URL ?? "http://127.0.0.1:8765";
|
||||
|
||||
type RouteContext = {
|
||||
params: Promise<{ path: string[] }>;
|
||||
};
|
||||
|
||||
export async function GET(request: Request, context: RouteContext) {
|
||||
return proxyEvaluationRequest("GET", request, context);
|
||||
}
|
||||
|
||||
export async function POST(request: Request, context: RouteContext) {
|
||||
return proxyEvaluationRequest("POST", request, context);
|
||||
}
|
||||
|
||||
export async function PATCH(request: Request, context: RouteContext) {
|
||||
return proxyEvaluationRequest("PATCH", request, context);
|
||||
}
|
||||
|
||||
async function proxyEvaluationRequest(
|
||||
method: "GET" | "POST" | "PATCH",
|
||||
request: Request,
|
||||
context: RouteContext,
|
||||
) {
|
||||
const account = await getCurrentAccount();
|
||||
if (!account)
|
||||
return Response.json({ detail: "unauthorized" }, { status: 401 });
|
||||
|
||||
const { path } = await context.params;
|
||||
const incomingUrl = new URL(request.url);
|
||||
const targetUrl = new URL(
|
||||
`${CLAW_API_URL}/api/evaluations/${path.map(encodeURIComponent).join("/")}`,
|
||||
);
|
||||
for (const [key, value] of incomingUrl.searchParams.entries()) {
|
||||
targetUrl.searchParams.append(key, value);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
let body: string | undefined;
|
||||
if (method === "GET") {
|
||||
targetUrl.searchParams.set("account_id", account.id);
|
||||
} else {
|
||||
const payload = (await request.json()) as Record<string, unknown>;
|
||||
headers["content-type"] = "application/json";
|
||||
body = JSON.stringify({ ...payload, account_id: account.id });
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(targetUrl, {
|
||||
method,
|
||||
headers,
|
||||
body,
|
||||
cache: "no-store",
|
||||
});
|
||||
const responseHeaders = new Headers();
|
||||
responseHeaders.set(
|
||||
"content-type",
|
||||
response.headers.get("content-type") ?? "application/json",
|
||||
);
|
||||
const disposition = response.headers.get("content-disposition");
|
||||
if (disposition) responseHeaders.set("content-disposition", disposition);
|
||||
return new Response(await response.arrayBuffer(), {
|
||||
status: response.status,
|
||||
headers: responseHeaders,
|
||||
});
|
||||
} catch (error) {
|
||||
return Response.json(
|
||||
{
|
||||
detail:
|
||||
error instanceof Error
|
||||
? `Unable to reach Claw backend: ${error.message}`
|
||||
: "Unable to reach Claw backend",
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,7 @@ import {
|
||||
import { ThreadListPrimitive } from "@assistant-ui/react";
|
||||
import type { UIMessage } from "ai";
|
||||
import {
|
||||
FlaskConicalIcon,
|
||||
Loader2Icon,
|
||||
MoreHorizontalIcon,
|
||||
PencilIcon,
|
||||
@@ -148,6 +149,7 @@ export const ThreadList: FC = () => {
|
||||
return (
|
||||
<ThreadListPrimitive.Root className="aui-root aui-thread-list-root flex min-h-0 flex-1 flex-col gap-1">
|
||||
<ThreadListNew />
|
||||
<ThreadListEvaluation />
|
||||
<JupyterWorkspaceList />
|
||||
<ClawSessionList />
|
||||
</ThreadListPrimitive.Root>
|
||||
@@ -174,6 +176,21 @@ const ThreadListNew: FC = () => {
|
||||
);
|
||||
};
|
||||
|
||||
const ThreadListEvaluation: FC = () => {
|
||||
return (
|
||||
<Button
|
||||
variant="ghost"
|
||||
className="h-9 w-full justify-start gap-2 rounded-lg px-3 text-sm"
|
||||
asChild
|
||||
>
|
||||
<a href="/evaluations">
|
||||
<FlaskConicalIcon className="size-4" />
|
||||
Skill 评测
|
||||
</a>
|
||||
</Button>
|
||||
);
|
||||
};
|
||||
|
||||
type JupyterWorkspaceEntry = {
|
||||
id: string;
|
||||
label?: string;
|
||||
@@ -1220,7 +1237,7 @@ function resolvePendingPrompt(
|
||||
: "";
|
||||
if (!pendingPrompt) return null;
|
||||
for (const message of messages) {
|
||||
if (!message || message.role !== "user") continue;
|
||||
if (message?.role !== "user") continue;
|
||||
const content = cleanStoredContent(message.content ?? "").trim();
|
||||
if (!content || content.trimStart().startsWith("<system-reminder>"))
|
||||
continue;
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
BrainIcon,
|
||||
ChevronDownIcon,
|
||||
Clock3Icon,
|
||||
FlaskConicalIcon,
|
||||
LogOutIcon,
|
||||
PanelLeftOpenIcon,
|
||||
SaveIcon,
|
||||
@@ -215,6 +216,12 @@ function CollapsedSidebarRail({
|
||||
>
|
||||
<SquarePenIcon className="size-4" />
|
||||
</CollapsedIconButton>
|
||||
<CollapsedIconButton
|
||||
label="Skill 评测"
|
||||
onClick={() => window.location.assign("/evaluations")}
|
||||
>
|
||||
<FlaskConicalIcon className="size-4" />
|
||||
</CollapsedIconButton>
|
||||
<CollapsedSessionSearch />
|
||||
<CollapsedRecentSessions />
|
||||
</div>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from backend.api.server import AgentState, create_app
|
||||
|
||||
|
||||
def _build_state(root: Path) -> AgentState:
|
||||
skill_dir = root / 'skills' / 'label-master'
|
||||
manifest_dir = skill_dir / 'knowledge' / '索引'
|
||||
manifest_dir.mkdir(parents=True)
|
||||
(skill_dir / 'SKILL.md').write_text(
|
||||
(
|
||||
'---\n'
|
||||
'name: label-master\n'
|
||||
'description: Test label master.\n'
|
||||
'allowed_tools: read_file\n'
|
||||
'---\n'
|
||||
'Use the label catalog.\n'
|
||||
),
|
||||
encoding='utf-8',
|
||||
)
|
||||
(manifest_dir / 'label_manifest.json').write_text(
|
||||
json.dumps(
|
||||
{
|
||||
'counts': {'labels': 1},
|
||||
'labels': [
|
||||
{
|
||||
'name': '地图导航',
|
||||
'domain': '地图',
|
||||
'path': 'knowledge/标签/地图导航.md',
|
||||
}
|
||||
],
|
||||
'agent_tags': ['地图导航'],
|
||||
'functions': [],
|
||||
'intents': [],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding='utf-8',
|
||||
)
|
||||
return AgentState(
|
||||
cwd=root,
|
||||
model='test-model',
|
||||
base_url='http://127.0.0.1:8000/v1',
|
||||
api_key='local-token',
|
||||
timeout_seconds=30,
|
||||
allow_shell=False,
|
||||
allow_write=False,
|
||||
session_directory=root / '.port_sessions' / 'agent',
|
||||
)
|
||||
|
||||
|
||||
class EvaluationApiTests(unittest.TestCase):
|
||||
def test_single_analysis_endpoint(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
state = _build_state(root)
|
||||
with (
|
||||
TestClient(create_app(state)) as client,
|
||||
patch.object(
|
||||
state.evaluation_runtime,
|
||||
'analyze_case',
|
||||
return_value={
|
||||
'query': '导航去公司',
|
||||
'prediction': '地图导航',
|
||||
'status': 'completed',
|
||||
},
|
||||
) as analyze,
|
||||
):
|
||||
response = client.post(
|
||||
'/api/evaluations/analyze',
|
||||
json={
|
||||
'account_id': 'alice',
|
||||
'query': '导航去公司',
|
||||
'skill_name': 'label-master',
|
||||
'model': 'test-model',
|
||||
},
|
||||
)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.json()['prediction'], '地图导航')
|
||||
analyze.assert_called_once()
|
||||
|
||||
def test_dataset_experiment_and_export_flow(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
state = _build_state(root)
|
||||
with TestClient(create_app(state)) as client:
|
||||
metadata = client.get(
|
||||
'/api/evaluations/metadata',
|
||||
params={'account_id': 'alice'},
|
||||
)
|
||||
self.assertEqual(metadata.status_code, 200)
|
||||
self.assertEqual(metadata.json()['default_skill'], 'label-master')
|
||||
|
||||
encoded = base64.b64encode(
|
||||
'query,label\n导航去公司,地图导航\n'.encode()
|
||||
).decode()
|
||||
dataset_response = client.post(
|
||||
'/api/evaluations/datasets',
|
||||
json={
|
||||
'account_id': 'alice',
|
||||
'name': 'routing',
|
||||
'filename': 'routing.csv',
|
||||
'content_base64': encoded,
|
||||
},
|
||||
)
|
||||
self.assertEqual(dataset_response.status_code, 200)
|
||||
dataset = dataset_response.json()
|
||||
self.assertEqual(dataset['row_count'], 1)
|
||||
|
||||
experiment_response = client.post(
|
||||
'/api/evaluations/experiments',
|
||||
json={
|
||||
'account_id': 'alice',
|
||||
'dataset_id': dataset['id'],
|
||||
'name': 'routing test',
|
||||
'skill_name': 'label-master',
|
||||
'concurrency': 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(experiment_response.status_code, 200)
|
||||
experiment = experiment_response.json()
|
||||
self.assertEqual(experiment['status'], 'draft')
|
||||
self.assertEqual(experiment['cases'][0]['query'], '导航去公司')
|
||||
|
||||
export = client.get(
|
||||
f"/api/evaluations/experiments/{experiment['id']}/export",
|
||||
params={'account_id': 'alice'},
|
||||
)
|
||||
self.assertEqual(export.status_code, 200)
|
||||
self.assertIn('text/csv', export.headers['content-type'])
|
||||
self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,502 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from src.agent_types import AgentRunResult, ModelConfig, UsageStats
|
||||
from src.bundled_skills import BundledSkill
|
||||
from src.evaluation_runtime import (
|
||||
EvaluationRuntime,
|
||||
build_evaluation_prompt,
|
||||
calculate_metrics,
|
||||
extract_accessed_refs,
|
||||
labels_equivalent,
|
||||
normalize_label_master_result,
|
||||
normalize_evaluation_output,
|
||||
parse_dataset_bytes,
|
||||
prediction_is_allowed,
|
||||
)
|
||||
|
||||
|
||||
def _write_test_skill(root: Path) -> None:
|
||||
skill_dir = root / 'skills' / 'label-master'
|
||||
manifest_dir = skill_dir / 'knowledge' / '索引'
|
||||
manifest_dir.mkdir(parents=True)
|
||||
(skill_dir / 'SKILL.md').write_text(
|
||||
(
|
||||
'---\n'
|
||||
'name: label-master\n'
|
||||
'description: Test label skill.\n'
|
||||
'allowed_tools: read_file, grep_search\n'
|
||||
'---\n'
|
||||
'Read the manifest and classify the query.\n'
|
||||
),
|
||||
encoding='utf-8',
|
||||
)
|
||||
(manifest_dir / 'label_manifest.json').write_text(
|
||||
json.dumps(
|
||||
{
|
||||
'counts': {'labels': 2, 'functions': 1},
|
||||
'labels': [
|
||||
{
|
||||
'name': '地图导航',
|
||||
'domain': '地图',
|
||||
'path': 'knowledge/标签/地图导航.md',
|
||||
},
|
||||
{
|
||||
'name': 'QA',
|
||||
'domain': '问答',
|
||||
'path': 'knowledge/标签/QA.md',
|
||||
},
|
||||
],
|
||||
'agent_tags': ['地图导航'],
|
||||
'functions': [{'name': 'QA'}],
|
||||
'intents': [],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
|
||||
def _write_fast_slow_skill(root: Path) -> None:
|
||||
skill_dir = root / 'skills' / 'label-fast-slow-routing'
|
||||
skill_dir.mkdir(parents=True)
|
||||
(skill_dir / 'SKILL.md').write_text(
|
||||
(
|
||||
'---\n'
|
||||
'name: label-fast-slow-routing\n'
|
||||
'description: Test fast slow routing skill.\n'
|
||||
'---\n'
|
||||
'Classify the query as 快、慢 or 模糊.\n'
|
||||
),
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
|
||||
class EvaluationRuntimeTests(unittest.TestCase):
|
||||
def test_parse_csv_and_jsonl(self) -> None:
|
||||
csv_rows, csv_format = parse_dataset_bytes(
|
||||
'sample.csv',
|
||||
'query,label\n打开地图,地图导航\n'.encode(),
|
||||
)
|
||||
self.assertEqual(csv_format, 'csv')
|
||||
self.assertEqual(csv_rows[0]['query'], '打开地图')
|
||||
|
||||
jsonl_rows, jsonl_format = parse_dataset_bytes(
|
||||
'sample.jsonl',
|
||||
b'{"query":"hello","label":"QA"}\n',
|
||||
)
|
||||
self.assertEqual(jsonl_format, 'jsonl')
|
||||
self.assertEqual(jsonl_rows[0]['label'], 'QA')
|
||||
|
||||
def test_dataset_mapping_snapshot_and_experiment(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
_write_test_skill(root)
|
||||
runtime = EvaluationRuntime(
|
||||
root=root / 'evaluations',
|
||||
cwd_for_account=lambda _account: root,
|
||||
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||
max_workers=1,
|
||||
)
|
||||
try:
|
||||
metadata = runtime.metadata('alice')
|
||||
self.assertEqual(metadata['default_skill'], 'label-master')
|
||||
self.assertEqual(
|
||||
[item['name'] for item in metadata['label_catalog']['labels']],
|
||||
['地图导航', 'QA'],
|
||||
)
|
||||
|
||||
content = base64.b64encode(
|
||||
'rid,query,人工标签\n1,导航去公司,Agent(tag="地图导航")\n'.encode()
|
||||
).decode()
|
||||
dataset = runtime.create_dataset(
|
||||
account_id='alice',
|
||||
name='test',
|
||||
filename='test.csv',
|
||||
content_base64=content,
|
||||
)
|
||||
self.assertEqual(dataset['mapping']['fields']['query'], 'query')
|
||||
self.assertEqual(
|
||||
dataset['mapping']['fields']['gold_label'],
|
||||
'人工标签',
|
||||
)
|
||||
self.assertEqual(dataset['mapping']['fields']['request_id'], 'rid')
|
||||
|
||||
experiment = runtime.create_experiment(
|
||||
account_id='alice',
|
||||
dataset_id=dataset['id'],
|
||||
name='first',
|
||||
)
|
||||
self.assertEqual(experiment['total_cases'], 1)
|
||||
self.assertEqual(experiment['skill_name'], 'label-master')
|
||||
self.assertEqual(len(experiment['skill_version']), 12)
|
||||
self.assertEqual(experiment['cases'][0]['request_id'], '1')
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_fast_slow_skill_is_the_evaluation_default(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
_write_test_skill(root)
|
||||
_write_fast_slow_skill(root)
|
||||
runtime = EvaluationRuntime(
|
||||
root=root / 'evaluations',
|
||||
cwd_for_account=lambda _account: root,
|
||||
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||
max_workers=1,
|
||||
)
|
||||
try:
|
||||
metadata = runtime.metadata('alice')
|
||||
self.assertEqual(
|
||||
metadata['default_skill'],
|
||||
'label-fast-slow-routing',
|
||||
)
|
||||
defaults = [
|
||||
item['name']
|
||||
for item in metadata['skills']
|
||||
if item['default']
|
||||
]
|
||||
self.assertEqual(defaults, ['label-fast-slow-routing'])
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_result_normalization_and_label_equivalence(self) -> None:
|
||||
result = normalize_evaluation_output(
|
||||
json.dumps(
|
||||
{
|
||||
'prediction': '地图导航',
|
||||
'function_output': 'Agent(tag="地图导航")',
|
||||
'complex': False,
|
||||
'reason': '导航执行',
|
||||
'candidates': ['地图导航', '地图问答'],
|
||||
'evidence_refs': ['knowledge/标签/地图导航.md'],
|
||||
'confidence': 1.4,
|
||||
'uncertainty': '',
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
self.assertEqual(result['prediction'], '地图导航')
|
||||
self.assertEqual(result['confidence'], 1.0)
|
||||
self.assertTrue(labels_equivalent('Agent(tag="地图导航")', '地图导航'))
|
||||
self.assertTrue(
|
||||
labels_equivalent(
|
||||
'complex=false\nAgent(tag="地图导航")',
|
||||
'地图导航',
|
||||
)
|
||||
)
|
||||
snapshot_metadata = json.dumps(
|
||||
{
|
||||
'label_catalog': {
|
||||
'labels': ['地图导航'],
|
||||
'agent_tags': ['地图导航'],
|
||||
'functions': ['QA'],
|
||||
'intents': [],
|
||||
}
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
self.assertTrue(
|
||||
prediction_is_allowed(
|
||||
'label-master',
|
||||
'Agent(tag="地图导航")',
|
||||
snapshot_metadata,
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
prediction_is_allowed(
|
||||
'label-master',
|
||||
'已完成第一步知识读取',
|
||||
snapshot_metadata,
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
prediction_is_allowed(
|
||||
'label-fast-slow-routing',
|
||||
'模糊',
|
||||
'{}',
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
prediction_is_allowed(
|
||||
'label-fast-slow-routing',
|
||||
'fast',
|
||||
'{}',
|
||||
)
|
||||
)
|
||||
normalized_label = normalize_label_master_result(
|
||||
{
|
||||
**result,
|
||||
'prediction': 'Agent(tag="地图导航")',
|
||||
'function_output': '一段解释文字',
|
||||
},
|
||||
snapshot_metadata,
|
||||
)
|
||||
self.assertEqual(normalized_label['prediction'], '地图导航')
|
||||
self.assertEqual(
|
||||
normalized_label['function_output'],
|
||||
'Agent(tag="地图导航")',
|
||||
)
|
||||
|
||||
def test_fast_slow_prompt_uses_skill_contract_and_relative_paths(self) -> None:
|
||||
skill = BundledSkill(
|
||||
name='label-fast-slow-routing',
|
||||
description='Test fast slow routing skill.',
|
||||
source='directory',
|
||||
get_prompt=lambda _agent, _args: 'Read references/policy.md.',
|
||||
)
|
||||
prompt, runtime_context = build_evaluation_prompt(
|
||||
skill=skill,
|
||||
snapshot_root=Path('/tmp/evaluation-snapshot'),
|
||||
canonical_case={'query': '打开空调', 'gold_label': '快'},
|
||||
)
|
||||
self.assertIn('填写“快”“慢”或“模糊”', prompt)
|
||||
self.assertNotIn('填写 fast 或 slow', prompt)
|
||||
self.assertIn('`references/...`', runtime_context)
|
||||
self.assertNotIn(
|
||||
'不要添加 `skills/label-fast-slow-routing/` 前缀',
|
||||
prompt,
|
||||
)
|
||||
|
||||
def test_single_analysis_uses_snapshot_without_dataset_records(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
_write_test_skill(root)
|
||||
runtime = EvaluationRuntime(
|
||||
root=root / 'evaluations',
|
||||
cwd_for_account=lambda _account: root,
|
||||
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||
max_workers=1,
|
||||
)
|
||||
fake_result = AgentRunResult(
|
||||
final_output=json.dumps(
|
||||
{
|
||||
'prediction': '地图导航',
|
||||
'function_output': 'Agent(tag="地图导航")',
|
||||
'complex': False,
|
||||
'reason': '导航执行',
|
||||
'candidates': ['地图导航'],
|
||||
'evidence_refs': [
|
||||
'knowledge/索引/label_manifest.json'
|
||||
],
|
||||
'confidence': 0.98,
|
||||
'uncertainty': '',
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
turns=1,
|
||||
tool_calls=1,
|
||||
transcript=(
|
||||
{
|
||||
'role': 'assistant',
|
||||
'tool_calls': [
|
||||
{
|
||||
'function': {
|
||||
'name': 'read_file',
|
||||
'arguments': json.dumps(
|
||||
{
|
||||
'path': (
|
||||
'knowledge/索引/'
|
||||
'label_manifest.json'
|
||||
)
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
),
|
||||
usage=UsageStats(input_tokens=10, output_tokens=5),
|
||||
)
|
||||
try:
|
||||
with patch(
|
||||
'src.evaluation_runtime.LocalCodingAgent.run',
|
||||
return_value=fake_result,
|
||||
):
|
||||
result = runtime.analyze_case(
|
||||
account_id='alice',
|
||||
query='导航去公司',
|
||||
)
|
||||
self.assertEqual(result['prediction'], '地图导航')
|
||||
self.assertEqual(result['skill_name'], 'label-master')
|
||||
self.assertEqual(result['model'], 'test-model')
|
||||
self.assertEqual(runtime.list_datasets('alice'), [])
|
||||
self.assertEqual(runtime.list_experiments('alice'), [])
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_background_runner_completes_cases(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
_write_test_skill(root)
|
||||
runtime = EvaluationRuntime(
|
||||
root=root / 'evaluations',
|
||||
cwd_for_account=lambda _account: root,
|
||||
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||
max_workers=1,
|
||||
)
|
||||
try:
|
||||
dataset = runtime.create_dataset(
|
||||
account_id='alice',
|
||||
name='test',
|
||||
filename='test.csv',
|
||||
rows=[
|
||||
{'query': '导航去公司', 'label': '地图导航'},
|
||||
{'query': '导航回家', 'label': '地图导航'},
|
||||
],
|
||||
)
|
||||
experiment = runtime.create_experiment(
|
||||
account_id='alice',
|
||||
dataset_id=dataset['id'],
|
||||
name='runner',
|
||||
concurrency=1,
|
||||
)
|
||||
fake_result = AgentRunResult(
|
||||
final_output=json.dumps(
|
||||
{
|
||||
'prediction': '地图导航',
|
||||
'function_output': 'Agent(tag="地图导航")',
|
||||
'complex': False,
|
||||
'reason': '导航执行',
|
||||
'candidates': ['地图导航'],
|
||||
'evidence_refs': ['knowledge/标签/地图导航.md'],
|
||||
'confidence': 0.98,
|
||||
'uncertainty': '',
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
turns=1,
|
||||
tool_calls=1,
|
||||
transcript=(
|
||||
{
|
||||
'role': 'assistant',
|
||||
'tool_calls': [
|
||||
{
|
||||
'function': {
|
||||
'name': 'read_file',
|
||||
'arguments': json.dumps(
|
||||
{
|
||||
'path': (
|
||||
'knowledge/索引/'
|
||||
'label_manifest.json'
|
||||
)
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
),
|
||||
usage=UsageStats(input_tokens=10, output_tokens=5),
|
||||
)
|
||||
with patch(
|
||||
'src.evaluation_runtime.LocalCodingAgent.run',
|
||||
return_value=fake_result,
|
||||
):
|
||||
runtime.start(experiment['id'], 'alice')
|
||||
deadline = time.time() + 3
|
||||
current = runtime.get_experiment(experiment['id'], 'alice')
|
||||
while (
|
||||
current['status'] not in {'completed', 'completed_with_errors'}
|
||||
and time.time() < deadline
|
||||
):
|
||||
time.sleep(0.02)
|
||||
current = runtime.get_experiment(
|
||||
experiment['id'],
|
||||
'alice',
|
||||
)
|
||||
self.assertEqual(current['status'], 'completed')
|
||||
self.assertEqual(current['completed_cases'], 2)
|
||||
self.assertEqual(current['metrics']['accuracy'], 1.0)
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_accessed_refs_only_include_snapshot_paths(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
snapshot_root = Path(directory)
|
||||
evidence = snapshot_root / 'skills' / 'label-master' / 'knowledge.md'
|
||||
evidence.parent.mkdir(parents=True)
|
||||
evidence.write_text('evidence', encoding='utf-8')
|
||||
transcript = (
|
||||
{
|
||||
'role': 'assistant',
|
||||
'tool_calls': [
|
||||
{
|
||||
'function': {
|
||||
'name': 'read_file',
|
||||
'arguments': json.dumps(
|
||||
{
|
||||
'path': (
|
||||
'skills/label-master/knowledge.md'
|
||||
)
|
||||
}
|
||||
),
|
||||
}
|
||||
},
|
||||
{
|
||||
'function': {
|
||||
'name': 'read_file',
|
||||
'arguments': json.dumps({'path': '/etc/hosts'}),
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
self.assertEqual(
|
||||
extract_accessed_refs(
|
||||
transcript,
|
||||
snapshot_root=snapshot_root,
|
||||
),
|
||||
[
|
||||
{
|
||||
'tool': 'read_file',
|
||||
'path': 'skills/label-master/knowledge.md',
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_metrics_count_disagreements(self) -> None:
|
||||
metrics = calculate_metrics(
|
||||
[
|
||||
{
|
||||
'status': 'completed',
|
||||
'gold_label': 'A',
|
||||
'prediction': 'A',
|
||||
'correct': True,
|
||||
'confidence': 0.9,
|
||||
'evidence_refs': ['a.md'],
|
||||
'domain': 'one',
|
||||
},
|
||||
{
|
||||
'status': 'completed',
|
||||
'gold_label': 'A',
|
||||
'prediction': 'B',
|
||||
'correct': False,
|
||||
'confidence': 0.5,
|
||||
'evidence_refs': [],
|
||||
'domain': 'one',
|
||||
},
|
||||
]
|
||||
)
|
||||
self.assertEqual(metrics['labeled'], 2)
|
||||
self.assertEqual(metrics['disagreements'], 1)
|
||||
self.assertEqual(metrics['accuracy'], 0.5)
|
||||
self.assertEqual(metrics['low_confidence'], 1)
|
||||
self.assertEqual(metrics['no_evidence'], 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user