feat: add editable skill evaluation versions

This commit is contained in:
wuyang6
2026-07-24 10:11:51 +08:00
parent ecc02c06b7
commit 6630e82069
5 changed files with 1070 additions and 3 deletions
+80
View File
@@ -1149,6 +1149,7 @@ class EvaluationAnalyzeRequest(BaseModel):
account_id: str = Field(min_length=1) account_id: str = Field(min_length=1)
query: str = Field(min_length=1, max_length=20_000) query: str = Field(min_length=1, max_length=20_000)
skill_name: str = 'label-master' skill_name: str = 'label-master'
snapshot_id: str | None = None
model: str | None = None model: str | None = None
history: list[dict[str, Any]] = Field(default_factory=list) history: list[dict[str, Any]] = Field(default_factory=list)
context: dict[str, Any] = Field(default_factory=dict) context: dict[str, Any] = Field(default_factory=dict)
@@ -1162,10 +1163,19 @@ class EvaluationExperimentCreateRequest(BaseModel):
dataset_id: str = Field(min_length=1) dataset_id: str = Field(min_length=1)
name: str = '' name: str = ''
skill_name: str = 'label-master' skill_name: str = 'label-master'
snapshot_id: str | None = None
model: str | None = None model: str | None = None
concurrency: int = Field(default=2, ge=1, le=8) concurrency: int = Field(default=2, ge=1, le=8)
class EvaluationSkillVersionSaveRequest(BaseModel):
account_id: str = Field(min_length=1)
base_snapshot_id: str = Field(min_length=1)
version_name: str = Field(min_length=1, max_length=80)
note: str = Field(default='', max_length=1000)
files: dict[str, str]
class EvaluationExperimentActionRequest(BaseModel): class EvaluationExperimentActionRequest(BaseModel):
account_id: str = Field(min_length=1) account_id: str = Field(min_length=1)
@@ -1534,6 +1544,74 @@ def create_app(state: AgentState) -> FastAPI:
except EvaluationError as exc: except EvaluationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.get('/api/evaluations/skills/{skill_name}/versions')
async def list_evaluation_skill_versions(
skill_name: str,
account_id: str,
) -> dict[str, Any]:
try:
return await asyncio.to_thread(
state.evaluation_runtime.list_skill_versions,
_safe_account_id(account_id),
skill_name,
)
except EvaluationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.get(
'/api/evaluations/skills/{skill_name}/versions/{snapshot_id}'
)
async def get_evaluation_skill_version(
skill_name: str,
snapshot_id: str,
account_id: str,
) -> dict[str, Any]:
try:
return state.evaluation_runtime.get_skill_version_manifest(
_safe_account_id(account_id),
skill_name,
snapshot_id,
)
except EvaluationError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get(
'/api/evaluations/skills/{skill_name}/versions/{snapshot_id}/file'
)
async def get_evaluation_skill_version_file(
skill_name: str,
snapshot_id: str,
account_id: str,
path: str,
) -> dict[str, Any]:
try:
return state.evaluation_runtime.read_skill_version_file(
_safe_account_id(account_id),
skill_name,
snapshot_id,
path,
)
except EvaluationError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post('/api/evaluations/skills/{skill_name}/versions')
async def save_evaluation_skill_version(
skill_name: str,
payload: EvaluationSkillVersionSaveRequest,
) -> dict[str, Any]:
try:
return await asyncio.to_thread(
state.evaluation_runtime.save_skill_version,
_safe_account_id(payload.account_id),
skill_name,
base_snapshot_id=payload.base_snapshot_id,
version_name=payload.version_name,
note=payload.note,
files=payload.files,
)
except EvaluationError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.post('/api/evaluations/analyze') @app.post('/api/evaluations/analyze')
async def analyze_evaluation_case( async def analyze_evaluation_case(
payload: EvaluationAnalyzeRequest, payload: EvaluationAnalyzeRequest,
@@ -1544,6 +1622,7 @@ def create_app(state: AgentState) -> FastAPI:
account_id=_safe_account_id(payload.account_id), account_id=_safe_account_id(payload.account_id),
query=payload.query, query=payload.query,
skill_name=payload.skill_name, skill_name=payload.skill_name,
snapshot_id=payload.snapshot_id,
model=payload.model, model=payload.model,
history=payload.history, history=payload.history,
context=payload.context, context=payload.context,
@@ -1642,6 +1721,7 @@ def create_app(state: AgentState) -> FastAPI:
dataset_id=payload.dataset_id, dataset_id=payload.dataset_id,
name=payload.name, name=payload.name,
skill_name=payload.skill_name, skill_name=payload.skill_name,
snapshot_id=payload.snapshot_id,
model=payload.model, model=payload.model,
concurrency=payload.concurrency, concurrency=payload.concurrency,
) )
+461
View File
@@ -6,14 +6,19 @@ import {
ChevronRightIcon, ChevronRightIcon,
CircleAlertIcon, CircleAlertIcon,
DownloadIcon, DownloadIcon,
FileCode2Icon,
FilePenLineIcon,
FileSpreadsheetIcon, FileSpreadsheetIcon,
FileTextIcon,
FlaskConicalIcon, FlaskConicalIcon,
GitBranchIcon,
LoaderCircleIcon, LoaderCircleIcon,
PauseIcon, PauseIcon,
PlayIcon, PlayIcon,
PlusIcon, PlusIcon,
RefreshCwIcon, RefreshCwIcon,
RotateCcwIcon, RotateCcwIcon,
SaveIcon,
SearchIcon, SearchIcon,
SquareIcon, SquareIcon,
UploadIcon, UploadIcon,
@@ -50,6 +55,45 @@ type EvaluationMetadata = {
}; };
}; };
type SkillVersion = {
id: string;
skill_name: string;
version_name: string;
version_note: string;
source_type: "git" | "local";
parent_snapshot_id: string;
content_hash: string;
git_commit: string;
git_status: string;
file_count: number;
created_at: number;
is_current: boolean;
};
type SkillVersionFile = {
path: string;
size: number;
editable: boolean;
previewable: boolean;
};
type SkillVersionManifest = SkillVersion & {
files: SkillVersionFile[];
};
type SkillVersionsResponse = {
skill_name: string;
current_snapshot_id: string;
versions: SkillVersion[];
};
type SkillFileContent = {
path: string;
content: string;
size: number;
editable: boolean;
};
type ModelOption = { type ModelOption = {
id: string; id: string;
name?: string; name?: string;
@@ -223,6 +267,8 @@ function EvaluationWorkspace() {
label_map: {}, label_map: {},
}); });
const [skillName, setSkillName] = useState(""); const [skillName, setSkillName] = useState("");
const [skillVersions, setSkillVersions] = useState<SkillVersion[]>([]);
const [snapshotId, setSnapshotId] = useState("");
const [model, setModel] = useState(""); const [model, setModel] = useState("");
const [concurrency, setConcurrency] = useState(2); const [concurrency, setConcurrency] = useState(2);
const [experimentName, setExperimentName] = useState(""); const [experimentName, setExperimentName] = useState("");
@@ -235,6 +281,18 @@ function EvaluationWorkspace() {
const [queryFilter, setQueryFilter] = useState(""); const [queryFilter, setQueryFilter] = useState("");
const [busy, setBusy] = useState(""); const [busy, setBusy] = useState("");
const [error, setError] = useState(""); const [error, setError] = useState("");
const [skillEditorOpen, setSkillEditorOpen] = useState(false);
const [skillEditorBusy, setSkillEditorBusy] = useState("");
const [skillEditorError, setSkillEditorError] = useState("");
const [skillManifest, setSkillManifest] =
useState<SkillVersionManifest | null>(null);
const [selectedSkillFile, setSelectedSkillFile] = useState("");
const [skillFileContents, setSkillFileContents] = useState<
Record<string, SkillFileContent>
>({});
const [dirtySkillFiles, setDirtySkillFiles] = useState<string[]>([]);
const [skillVersionName, setSkillVersionName] = useState("");
const [skillVersionNote, setSkillVersionNote] = useState("");
const uploadRef = useRef<HTMLInputElement>(null); const uploadRef = useRef<HTMLInputElement>(null);
const selectedDatasetIdRef = useRef(""); const selectedDatasetIdRef = useRef("");
const selectedExperimentIdRef = useRef(""); const selectedExperimentIdRef = useRef("");
@@ -256,6 +314,33 @@ function EvaluationWorkspace() {
() => datasets.find((item) => item.id === selectedDatasetId) ?? null, () => datasets.find((item) => item.id === selectedDatasetId) ?? null,
[datasets, selectedDatasetId], [datasets, selectedDatasetId],
); );
const selectedSkillVersion = useMemo(
() => skillVersions.find((item) => item.id === snapshotId) ?? null,
[skillVersions, snapshotId],
);
const loadSkillVersions = useCallback(
async (nextSkillName: string, preferredSnapshotId = "") => {
if (!nextSkillName) return null;
const response = await fetch(
`/api/claw/evaluations/skills/${encodeURIComponent(nextSkillName)}/versions`,
{ cache: "no-store" },
);
const payload = await readJson<SkillVersionsResponse>(response);
setSkillVersions(payload.versions);
const preferred = payload.versions.find(
(item) => item.id === preferredSnapshotId,
);
setSnapshotId(
preferred?.id ||
payload.current_snapshot_id ||
payload.versions[0]?.id ||
"",
);
return payload;
},
[],
);
const refreshExperiment = useCallback(async (experimentId: string) => { const refreshExperiment = useCallback(async (experimentId: string) => {
const response = await fetch( const response = await fetch(
@@ -358,6 +443,15 @@ function EvaluationWorkspace() {
void loadWorkspace(); void loadWorkspace();
}, [loadWorkspace]); }, [loadWorkspace]);
useEffect(() => {
if (!skillName) return;
setSkillVersions([]);
setSnapshotId("");
void loadSkillVersions(skillName).catch((reason) =>
setError(messageFrom(reason)),
);
}, [loadSkillVersions, skillName]);
useEffect(() => { useEffect(() => {
if (!experiment || !ACTIVE_STATUSES.has(experiment.status)) return; if (!experiment || !ACTIVE_STATUSES.has(experiment.status)) return;
const timer = window.setInterval(() => { const timer = window.setInterval(() => {
@@ -368,6 +462,131 @@ function EvaluationWorkspace() {
return () => window.clearInterval(timer); return () => window.clearInterval(timer);
}, [experiment, refreshExperiment]); }, [experiment, refreshExperiment]);
async function fetchSkillFile(
versionId: string,
filePath: string,
force = false,
) {
setSelectedSkillFile(filePath);
const cached = skillFileContents[filePath];
if (cached && !force) return cached;
setSkillEditorBusy("file");
setSkillEditorError("");
try {
const response = await fetch(
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions/${encodeURIComponent(versionId)}/file?path=${encodeURIComponent(filePath)}`,
{ cache: "no-store" },
);
const payload = await readJson<SkillFileContent>(response);
setSkillFileContents((current) => ({
...current,
[filePath]: payload,
}));
return payload;
} catch (reason) {
setSkillEditorError(messageFrom(reason));
return null;
} finally {
setSkillEditorBusy("");
}
}
async function openSkillEditor() {
if (!skillName) return;
setSkillEditorOpen(true);
setSkillEditorBusy("manifest");
setSkillEditorError("");
setSkillManifest(null);
setSelectedSkillFile("");
setSkillFileContents({});
setDirtySkillFiles([]);
setSkillVersionName("");
setSkillVersionNote("");
try {
let versionId = snapshotId;
if (!versionId) {
const versions = await loadSkillVersions(skillName);
versionId = versions?.current_snapshot_id ?? "";
}
if (!versionId) throw new Error("当前 Skill 没有可用版本");
const response = await fetch(
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions/${encodeURIComponent(versionId)}`,
{ cache: "no-store" },
);
const manifest = await readJson<SkillVersionManifest>(response);
setSkillManifest(manifest);
const firstFile =
manifest.files.find((item) => item.path === "SKILL.md") ??
manifest.files.find((item) => item.previewable);
if (firstFile) {
await fetchSkillFile(manifest.id, firstFile.path, true);
}
} catch (reason) {
setSkillEditorError(messageFrom(reason));
} finally {
setSkillEditorBusy("");
}
}
function updateSkillFile(content: string) {
if (!selectedSkillFile) return;
setSkillFileContents((current) => {
const existing = current[selectedSkillFile];
if (!existing) return current;
return {
...current,
[selectedSkillFile]: { ...existing, content },
};
});
setDirtySkillFiles((current) =>
current.includes(selectedSkillFile)
? current
: [...current, selectedSkillFile],
);
}
async function saveSkillVersion() {
if (!skillManifest) return;
if (!skillVersionName.trim()) {
setSkillEditorError("请填写新版本名称");
return;
}
if (!dirtySkillFiles.length) {
setSkillEditorError("当前没有文件修改");
return;
}
setSkillEditorBusy("save");
setSkillEditorError("");
try {
const files = Object.fromEntries(
dirtySkillFiles.map((path) => [
path,
skillFileContents[path]?.content ?? "",
]),
);
const response = await fetch(
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions`,
{
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
base_snapshot_id: skillManifest.id,
version_name: skillVersionName.trim(),
note: skillVersionNote.trim(),
files,
}),
},
);
const saved = await readJson<SkillVersion>(response);
await loadSkillVersions(skillName, saved.id);
setSkillEditorOpen(false);
} catch (reason) {
setSkillEditorError(messageFrom(reason));
} finally {
setSkillEditorBusy("");
}
}
async function uploadDataset(file: File) { async function uploadDataset(file: File) {
setBusy("upload"); setBusy("upload");
setError(""); setError("");
@@ -434,6 +653,7 @@ function EvaluationWorkspace() {
name: name:
experimentName.trim() || `${skillName} · ${selectedDataset.name}`, experimentName.trim() || `${skillName} · ${selectedDataset.name}`,
skill_name: skillName, skill_name: skillName,
snapshot_id: snapshotId || undefined,
model: model || undefined, model: model || undefined,
concurrency, concurrency,
}), }),
@@ -475,6 +695,7 @@ function EvaluationWorkspace() {
body: JSON.stringify({ body: JSON.stringify({
query, query,
skill_name: skillName, skill_name: skillName,
snapshot_id: snapshotId || undefined,
model: model || undefined, model: model || undefined,
}), }),
}); });
@@ -661,6 +882,34 @@ function EvaluationWorkspace() {
))} ))}
</select> </select>
</label> </label>
<label className="flex h-8 items-center gap-2 rounded-md border bg-background px-2">
<span className="text-muted-foreground text-xs"></span>
<select
aria-label="全局 Skill 版本"
value={snapshotId}
onChange={(event) => setSnapshotId(event.target.value)}
disabled={!skillVersions.length}
className="h-7 min-w-36 max-w-56 bg-transparent font-medium text-xs outline-none disabled:opacity-60"
>
{skillVersions.map((version) => (
<option key={version.id} value={version.id}>
{version.version_name}
{version.source_type === "local" ? " · 本地" : ""}
{` · ${version.content_hash.slice(0, 8)}`}
</option>
))}
</select>
</label>
<Button
variant="outline"
size="sm"
onClick={() => void openSkillEditor()}
disabled={!snapshotId}
title="查看并编辑当前 Skill 版本"
>
<FilePenLineIcon />
</Button>
<label className="flex h-8 min-w-0 items-center gap-2 rounded-md border bg-background px-2"> <label className="flex h-8 min-w-0 items-center gap-2 rounded-md border bg-background px-2">
<span className="text-muted-foreground text-xs"></span> <span className="text-muted-foreground text-xs"></span>
<select <select
@@ -871,10 +1120,222 @@ function EvaluationWorkspace() {
); );
}} }}
/> />
<SkillEditorSheet
open={skillEditorOpen}
onOpenChange={setSkillEditorOpen}
skillName={skillName}
version={selectedSkillVersion}
manifest={skillManifest}
selectedFile={selectedSkillFile}
fileContent={
selectedSkillFile
? (skillFileContents[selectedSkillFile] ?? null)
: null
}
dirtyFiles={dirtySkillFiles}
busy={skillEditorBusy}
error={skillEditorError}
versionName={skillVersionName}
setVersionName={setSkillVersionName}
versionNote={skillVersionNote}
setVersionNote={setSkillVersionNote}
onSelectFile={(filePath) => {
if (skillManifest) {
void fetchSkillFile(skillManifest.id, filePath);
}
}}
onChangeContent={updateSkillFile}
onSave={() => void saveSkillVersion()}
/>
</main> </main>
); );
} }
function SkillEditorSheet(props: {
open: boolean;
onOpenChange: (open: boolean) => void;
skillName: string;
version: SkillVersion | null;
manifest: SkillVersionManifest | null;
selectedFile: string;
fileContent: SkillFileContent | null;
dirtyFiles: string[];
busy: string;
error: string;
versionName: string;
setVersionName: (value: string) => void;
versionNote: string;
setVersionNote: (value: string) => void;
onSelectFile: (path: string) => void;
onChangeContent: (content: string) => void;
onSave: () => void;
}) {
const {
open,
onOpenChange,
skillName,
version,
manifest,
selectedFile,
fileContent,
dirtyFiles,
busy,
error,
versionName,
setVersionName,
versionNote,
setVersionNote,
onSelectFile,
onChangeContent,
onSave,
} = props;
return (
<Sheet open={open} onOpenChange={onOpenChange}>
<SheetContent className="w-[min(96vw,1100px)] gap-0 sm:max-w-[1100px]">
<SheetHeader className="border-b px-5 py-4">
<SheetTitle className="flex items-center gap-2">
<FilePenLineIcon className="size-4" />
Skill
</SheetTitle>
<SheetDescription className="flex flex-wrap items-center gap-x-3 gap-y-1">
<span>{skillName}</span>
<span className="inline-flex items-center gap-1">
<GitBranchIcon className="size-3.5" />
{version?.version_name || "正在读取版本"}
</span>
{version?.content_hash ? (
<code>{version.content_hash.slice(0, 12)}</code>
) : null}
</SheetDescription>
</SheetHeader>
{error ? (
<div className="mx-5 mt-3 flex items-start gap-2 border border-destructive/30 bg-destructive/5 px-3 py-2 text-destructive text-sm">
<CircleAlertIcon className="mt-0.5 size-4 shrink-0" />
<span>{error}</span>
</div>
) : null}
<div className="grid min-h-0 flex-1 grid-cols-[230px_minmax(0,1fr)]">
<aside className="min-h-0 overflow-y-auto border-r bg-muted/15 p-3">
<div className="mb-2 flex items-center justify-between px-2 text-xs">
<span className="font-medium">Skill </span>
<span className="text-muted-foreground">
{manifest?.files.length ?? 0}
</span>
</div>
<div className="space-y-0.5">
{manifest?.files.map((file) => {
const dirty = dirtyFiles.includes(file.path);
return (
<button
type="button"
key={file.path}
onClick={() => onSelectFile(file.path)}
disabled={!file.previewable}
className={`flex w-full items-center gap-2 rounded-md px-2 py-1.5 text-left text-xs transition-colors ${
selectedFile === file.path
? "bg-accent text-accent-foreground"
: "hover:bg-accent/60"
} disabled:cursor-not-allowed disabled:opacity-50`}
>
{file.path.endsWith(".md") ? (
<FileTextIcon className="size-3.5 shrink-0" />
) : (
<FileCode2Icon className="size-3.5 shrink-0" />
)}
<span className="min-w-0 flex-1 truncate">{file.path}</span>
{dirty ? (
<span
className="size-1.5 shrink-0 rounded-full bg-sky-500"
title="已修改"
/>
) : null}
</button>
);
})}
</div>
</aside>
<section className="flex min-h-0 min-w-0 flex-col">
<div className="flex h-10 shrink-0 items-center gap-2 border-b px-4">
<span className="min-w-0 flex-1 truncate font-medium text-sm">
{selectedFile || "选择一个文件"}
</span>
{fileContent ? (
<span className="text-muted-foreground text-xs">
{fileContent.editable ? "可编辑" : "只读"}
</span>
) : null}
{busy === "file" ? (
<LoaderCircleIcon className="size-4 animate-spin text-muted-foreground" />
) : null}
</div>
<div className="min-h-0 flex-1 p-4">
{fileContent ? (
<textarea
value={fileContent.content}
onChange={(event) => onChangeContent(event.target.value)}
readOnly={!fileContent.editable}
spellCheck={false}
className="h-full min-h-72 w-full resize-none border bg-muted/10 p-4 font-mono text-[13px] leading-6 outline-none focus:ring-2 focus:ring-ring/40 read-only:cursor-default read-only:opacity-80"
/>
) : (
<div className="grid h-full min-h-72 place-items-center text-muted-foreground text-sm">
{busy === "manifest" ? (
<LoaderCircleIcon className="size-5 animate-spin" />
) : (
"选择左侧文件查看内容"
)}
</div>
)}
</div>
<div className="shrink-0 border-t bg-muted/10 px-4 py-3">
<div className="grid gap-3 md:grid-cols-[minmax(0,220px)_minmax(0,1fr)_auto] md:items-end">
<Field label="新版本名称">
<Input
value={versionName}
onChange={(event) => setVersionName(event.target.value)}
placeholder="例如:导航边界调整 v1"
maxLength={80}
/>
</Field>
<Field label="版本备注">
<Input
value={versionNote}
onChange={(event) => setVersionNote(event.target.value)}
placeholder="记录本次调整内容"
maxLength={1000}
/>
</Field>
<Button
onClick={onSave}
disabled={busy === "save" || !dirtyFiles.length}
>
{busy === "save" ? (
<LoaderCircleIcon className="animate-spin" />
) : (
<SaveIcon />
)}
</Button>
</div>
<p className="mt-2 text-muted-foreground text-xs">
Markdown JSON
{dirtyFiles.length
? ` 当前修改 ${dirtyFiles.length} 个文件。`
: ""}
</p>
</div>
</section>
</div>
</SheetContent>
</Sheet>
);
}
function SingleQueryWorkspace(props: { function SingleQueryWorkspace(props: {
query: string; query: string;
setQuery: (value: string) => void; setQuery: (value: string) => void;
+379 -3
View File
@@ -14,7 +14,7 @@ import threading
import time import time
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
from dataclasses import dataclass, field, replace from dataclasses import dataclass, field, replace
from pathlib import Path from pathlib import Path, PurePosixPath
from typing import Any, Callable, Iterable from typing import Any, Callable, Iterable
from uuid import uuid4 from uuid import uuid4
@@ -35,6 +35,10 @@ SUPPORTED_DATASET_SUFFIXES = {'.csv', '.tsv', '.json', '.jsonl', '.xlsx'}
DEFAULT_CONCURRENCY = 2 DEFAULT_CONCURRENCY = 2
MAX_CONCURRENCY = 8 MAX_CONCURRENCY = 8
MAX_DATASET_ROWS = 20_000 MAX_DATASET_ROWS = 20_000
VISIBLE_SKILL_FILE_SUFFIXES = {'.json', '.md', '.py', '.txt', '.yaml', '.yml'}
EDITABLE_SKILL_FILE_SUFFIXES = {'.json', '.md'}
MAX_SKILL_FILE_BYTES = 512 * 1024
MAX_SKILL_VERSION_EDIT_BYTES = 2 * 1024 * 1024
EVALUATION_OUTPUT_SCHEMA = OutputSchemaConfig( EVALUATION_OUTPUT_SCHEMA = OutputSchemaConfig(
name='skill_evaluation_result', name='skill_evaluation_result',
@@ -491,6 +495,7 @@ class EvaluationRuntime:
account_id: str, account_id: str,
query: str, query: str,
skill_name: str = 'label-master', skill_name: str = 'label-master',
snapshot_id: str | None = None,
model: str | None = None, model: str | None = None,
history: list[dict[str, Any]] | None = None, history: list[dict[str, Any]] | None = None,
context: dict[str, Any] | None = None, context: dict[str, Any] | None = None,
@@ -501,7 +506,11 @@ class EvaluationRuntime:
normalized_query = query.strip() normalized_query = query.strip()
if not normalized_query: if not normalized_query:
raise EvaluationError('Query 不能为空') raise EvaluationError('Query 不能为空')
snapshot = self.create_skill_snapshot(account_id, skill_name) snapshot = self.resolve_skill_snapshot(
account_id,
skill_name,
snapshot_id=snapshot_id,
)
model_name = model or self.model_config_for(account_id).model model_name = model or self.model_config_for(account_id).model
analysis_id = f'single_{uuid4().hex[:16]}' analysis_id = f'single_{uuid4().hex[:16]}'
case_id = f'case_{uuid4().hex[:16]}' case_id = f'case_{uuid4().hex[:16]}'
@@ -669,6 +678,7 @@ class EvaluationRuntime:
dataset_id: str, dataset_id: str,
name: str, name: str,
skill_name: str = 'label-master', skill_name: str = 'label-master',
snapshot_id: str | None = None,
model: str | None = None, model: str | None = None,
concurrency: int = DEFAULT_CONCURRENCY, concurrency: int = DEFAULT_CONCURRENCY,
) -> dict[str, Any]: ) -> dict[str, Any]:
@@ -678,7 +688,11 @@ class EvaluationRuntime:
canonical_rows = [row for row in canonical_rows if row['query']] canonical_rows = [row for row in canonical_rows if row['query']]
if not canonical_rows: if not canonical_rows:
raise EvaluationError('当前字段映射没有生成有效 case') raise EvaluationError('当前字段映射没有生成有效 case')
snapshot = self.create_skill_snapshot(account_id, skill_name) snapshot = self.resolve_skill_snapshot(
account_id,
skill_name,
snapshot_id=snapshot_id,
)
model_config = self.model_config_for(account_id) model_config = self.model_config_for(account_id)
experiment_id = f'eval_{uuid4().hex[:16]}' experiment_id = f'eval_{uuid4().hex[:16]}'
now = time.time() now = time.time()
@@ -1004,6 +1018,274 @@ class EvaluationRuntime:
# Snapshots and execution # Snapshots and execution
# ------------------------------------------------------------------ # ------------------------------------------------------------------
def resolve_skill_snapshot(
self,
account_id: str,
skill_name: str,
*,
snapshot_id: str | None = None,
) -> dict[str, Any]:
if not snapshot_id:
return self.create_skill_snapshot(account_id, skill_name)
row = self.store.fetchone(
"""
SELECT *
FROM skill_snapshots
WHERE id = ? AND account_id = ? AND skill_name = ?
""",
(snapshot_id, account_id, skill_name),
)
if row is None or not Path(row['path']).is_dir():
raise EvaluationError('Skill 版本不存在或已失效')
return self._snapshot_payload(row)
def list_skill_versions(
self,
account_id: str,
skill_name: str,
) -> dict[str, Any]:
current = self.create_skill_snapshot(account_id, skill_name)
rows = self.store.fetchall(
"""
SELECT *
FROM skill_snapshots
WHERE account_id = ? AND skill_name = ?
ORDER BY created_at DESC
""",
(account_id, skill_name),
)
rows.sort(
key=lambda row: (
row['id'] != current['id'],
-float(row['created_at']),
)
)
return {
'skill_name': skill_name,
'current_snapshot_id': current['id'],
'versions': [
self._public_skill_version(
row,
is_current=row['id'] == current['id'],
)
for row in rows
if Path(row['path']).is_dir()
],
}
def get_skill_version_manifest(
self,
account_id: str,
skill_name: str,
snapshot_id: str,
) -> dict[str, Any]:
snapshot = self.resolve_skill_snapshot(
account_id,
skill_name,
snapshot_id=snapshot_id,
)
snapshot_skill_dir = self._snapshot_skill_dir(snapshot, skill_name)
files: list[dict[str, Any]] = []
for path in sorted(
item for item in snapshot_skill_dir.rglob('*') if item.is_file()
):
relative = path.relative_to(snapshot_skill_dir).as_posix()
if any(part.startswith('.') for part in PurePosixPath(relative).parts):
continue
suffix = path.suffix.lower()
if suffix not in VISIBLE_SKILL_FILE_SUFFIXES:
continue
size = path.stat().st_size
files.append(
{
'path': relative,
'size': size,
'editable': (
suffix in EDITABLE_SKILL_FILE_SUFFIXES
and size <= MAX_SKILL_FILE_BYTES
),
'previewable': size <= MAX_SKILL_FILE_BYTES,
}
)
row = self._skill_snapshot_row(account_id, skill_name, snapshot_id)
return {
**self._public_skill_version(row),
'files': files,
}
def read_skill_version_file(
self,
account_id: str,
skill_name: str,
snapshot_id: str,
file_path: str,
) -> dict[str, Any]:
snapshot = self.resolve_skill_snapshot(
account_id,
skill_name,
snapshot_id=snapshot_id,
)
snapshot_skill_dir = self._snapshot_skill_dir(snapshot, skill_name)
relative, target = self._resolve_skill_file(
snapshot_skill_dir,
file_path,
)
if target.suffix.lower() not in VISIBLE_SKILL_FILE_SUFFIXES:
raise EvaluationError('该文件类型不支持预览')
size = target.stat().st_size
if size > MAX_SKILL_FILE_BYTES:
raise EvaluationError('文件过大,暂不支持在线预览')
try:
content = target.read_text(encoding='utf-8')
except UnicodeDecodeError as exc:
raise EvaluationError('文件不是 UTF-8 文本') from exc
return {
'path': relative,
'content': content,
'size': size,
'editable': target.suffix.lower() in EDITABLE_SKILL_FILE_SUFFIXES,
}
def save_skill_version(
self,
account_id: str,
skill_name: str,
*,
base_snapshot_id: str,
version_name: str,
note: str = '',
files: dict[str, str],
) -> dict[str, Any]:
normalized_name = version_name.strip()
if not normalized_name:
raise EvaluationError('请填写版本名称')
if len(normalized_name) > 80:
raise EvaluationError('版本名称不能超过 80 个字符')
normalized_note = note.strip()
if len(normalized_note) > 1000:
raise EvaluationError('版本备注不能超过 1000 个字符')
if not files:
raise EvaluationError('没有需要保存的文件修改')
base = self.resolve_skill_snapshot(
account_id,
skill_name,
snapshot_id=base_snapshot_id,
)
base_skill_dir = self._snapshot_skill_dir(base, skill_name)
total_bytes = sum(
len(str(content).encode('utf-8')) for content in files.values()
)
if total_bytes > MAX_SKILL_VERSION_EDIT_BYTES:
raise EvaluationError('本次修改内容过大,请拆分后保存')
draft_root: Path | None = (
self.root / '.skill_version_drafts' / uuid4().hex
)
draft_skill_dir = draft_root / 'skills' / skill_name
draft_skill_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(base_skill_dir, draft_skill_dir)
try:
for file_path, raw_content in files.items():
relative, target = self._resolve_skill_file(
draft_skill_dir,
file_path,
)
if target.suffix.lower() not in EDITABLE_SKILL_FILE_SUFFIXES:
raise EvaluationError(f'该文件只读:{relative}')
content = str(raw_content)
if len(content.encode('utf-8')) > MAX_SKILL_FILE_BYTES:
raise EvaluationError(f'文件过大:{relative}')
if target.suffix.lower() == '.json':
try:
json.loads(content)
except json.JSONDecodeError as exc:
raise EvaluationError(
f'JSON 格式错误:{relative},第 {exc.lineno}'
) from exc
target.write_text(content, encoding='utf-8')
loaded = self._load_snapshot_skill(draft_root, skill_name)
if loaded.name != skill_name:
raise EvaluationError('SKILL.md 中的 name 不允许修改')
content_hash = hash_directory(draft_skill_dir)
existing = self.store.fetchone(
"""
SELECT *
FROM skill_snapshots
WHERE account_id = ? AND skill_name = ? AND content_hash = ?
""",
(account_id, skill_name, content_hash),
)
if existing is not None and Path(existing['path']).is_dir():
return self._public_skill_version(existing)
snapshot_id = f'snapshot_{uuid4().hex[:16]}'
snapshot_root = (
self.root
/ 'skill_snapshots'
/ sanitize_filename(account_id)
/ sanitize_filename(skill_name)
/ content_hash[:16]
)
with self._lock:
if snapshot_root.exists():
shutil.rmtree(snapshot_root)
snapshot_root.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(draft_root), str(snapshot_root))
draft_root = None
snapshot_skill_dir = snapshot_root / 'skills' / skill_name
metadata = self._skill_snapshot_metadata(
snapshot_skill_dir,
skill_name,
)
metadata.update(
{
'source_type': 'local',
'version_name': normalized_name,
'version_note': normalized_note,
'parent_snapshot_id': base_snapshot_id,
}
)
try:
self.store.execute(
"""
INSERT INTO skill_snapshots (
id, account_id, skill_name, content_hash, git_commit,
git_status, path, metadata_json, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
snapshot_id,
account_id,
skill_name,
content_hash,
base.get('git_commit', ''),
'local',
str(snapshot_root),
json.dumps(metadata, ensure_ascii=False),
time.time(),
),
)
except sqlite3.IntegrityError:
existing = self.store.fetchone(
"""
SELECT *
FROM skill_snapshots
WHERE account_id = ? AND skill_name = ?
AND content_hash = ?
""",
(account_id, skill_name, content_hash),
)
if existing is None:
raise
return self._public_skill_version(existing)
row = self._skill_snapshot_row(account_id, skill_name, snapshot_id)
return self._public_skill_version(row)
finally:
if draft_root is not None and draft_root.exists():
shutil.rmtree(draft_root, ignore_errors=True)
def create_skill_snapshot( def create_skill_snapshot(
self, self,
account_id: str, account_id: str,
@@ -1043,6 +1325,14 @@ class EvaluationRuntime:
ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '.DS_Store'), ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '.DS_Store'),
) )
metadata = self._skill_snapshot_metadata(snapshot_skill_dir, skill_name) metadata = self._skill_snapshot_metadata(snapshot_skill_dir, skill_name)
metadata.update(
{
'source_type': 'git',
'version_name': '',
'version_note': '',
'parent_snapshot_id': '',
}
)
now = time.time() now = time.time()
try: try:
self.store.execute( self.store.execute(
@@ -1619,6 +1909,92 @@ class EvaluationRuntime:
'created_at': row['created_at'], 'created_at': row['created_at'],
} }
def _public_skill_version(
self,
row: dict[str, Any],
*,
is_current: bool = False,
) -> dict[str, Any]:
metadata = json.loads(row.get('metadata_json') or '{}')
source_type = str(metadata.get('source_type') or 'git')
version_name = str(metadata.get('version_name') or '').strip()
if not version_name:
if is_current:
version_name = '当前 Git'
elif source_type == 'local':
version_name = f"本地版本 {str(row['content_hash'])[:8]}"
else:
version_name = f"Git 快照 {str(row['content_hash'])[:8]}"
return {
'id': row['id'],
'skill_name': row['skill_name'],
'version_name': version_name,
'version_note': str(metadata.get('version_note') or ''),
'source_type': source_type,
'parent_snapshot_id': str(
metadata.get('parent_snapshot_id') or ''
),
'content_hash': row['content_hash'],
'git_commit': row.get('git_commit', ''),
'git_status': row.get('git_status', 'unknown'),
'file_count': int(metadata.get('file_count') or 0),
'created_at': row['created_at'],
'is_current': is_current,
}
def _skill_snapshot_row(
self,
account_id: str,
skill_name: str,
snapshot_id: str,
) -> dict[str, Any]:
row = self.store.fetchone(
"""
SELECT *
FROM skill_snapshots
WHERE id = ? AND account_id = ? AND skill_name = ?
""",
(snapshot_id, account_id, skill_name),
)
if row is None:
raise EvaluationError('Skill 版本不存在')
return row
def _snapshot_skill_dir(
self,
snapshot: dict[str, Any],
skill_name: str,
) -> Path:
skill_dir = Path(snapshot['path']) / 'skills' / skill_name
if not skill_dir.is_dir() or not (skill_dir / 'SKILL.md').is_file():
raise EvaluationError('Skill 版本文件已失效')
return skill_dir
def _resolve_skill_file(
self,
skill_dir: Path,
file_path: str,
) -> tuple[str, Path]:
candidate = PurePosixPath(str(file_path).strip())
if (
not candidate.parts
or candidate.is_absolute()
or '..' in candidate.parts
or any(part.startswith('.') for part in candidate.parts)
):
raise EvaluationError('文件路径无效')
relative = candidate.as_posix()
target = skill_dir.joinpath(*candidate.parts)
if target.is_symlink() or not target.is_file():
raise EvaluationError(f'Skill 文件不存在:{relative}')
resolved_root = skill_dir.resolve()
resolved_target = target.resolve()
try:
resolved_target.relative_to(resolved_root)
except ValueError as exc:
raise EvaluationError('文件路径越界') from exc
return relative, resolved_target
def _experiment_summary(self, row: dict[str, Any]) -> dict[str, Any]: def _experiment_summary(self, row: dict[str, Any]) -> dict[str, Any]:
total = int(row.get('total_cases') or 0) total = int(row.get('total_cases') or 0)
completed = int(row.get('completed_cases') or 0) completed = int(row.get('completed_cases') or 0)
+56
View File
@@ -178,6 +178,62 @@ class EvaluationApiTests(unittest.TestCase):
self.assertIn('text/csv', export.headers['content-type']) self.assertIn('text/csv', export.headers['content-type'])
self.assertIn('导航去公司', export.content.decode('utf-8-sig')) self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
def test_skill_version_endpoints(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
state = _build_state(root)
with TestClient(create_app(state)) as client:
versions_response = client.get(
'/api/evaluations/skills/label-master/versions',
params={'account_id': 'alice'},
)
self.assertEqual(versions_response.status_code, 200)
versions = versions_response.json()
current_id = versions['current_snapshot_id']
manifest = client.get(
(
'/api/evaluations/skills/label-master/versions/'
f'{current_id}'
),
params={'account_id': 'alice'},
)
self.assertEqual(manifest.status_code, 200)
self.assertTrue(
any(
item['path'] == 'SKILL.md'
for item in manifest.json()['files']
)
)
file_response = client.get(
(
'/api/evaluations/skills/label-master/versions/'
f'{current_id}/file'
),
params={'account_id': 'alice', 'path': 'SKILL.md'},
)
self.assertEqual(file_response.status_code, 200)
content = file_response.json()['content'].replace(
'Use the label catalog.',
'Use the label catalog carefully.',
)
saved_response = client.post(
'/api/evaluations/skills/label-master/versions',
json={
'account_id': 'alice',
'base_snapshot_id': current_id,
'version_name': '测试版本',
'note': 'API 测试',
'files': {'SKILL.md': content},
},
)
self.assertEqual(saved_response.status_code, 200)
self.assertEqual(
saved_response.json()['version_name'],
'测试版本',
)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()
+94
View File
@@ -169,6 +169,100 @@ class EvaluationRuntimeTests(unittest.TestCase):
finally: finally:
runtime.shutdown() runtime.shutdown()
def test_skill_versions_are_immutable_and_account_scoped(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
_write_test_skill(root)
runtime = EvaluationRuntime(
root=root / 'evaluations',
cwd_for_account=lambda _account: root,
model_config_for=lambda _account: ModelConfig(model='test-model'),
account_paths_for=lambda _account: {'python_env': root / '.venv'},
max_workers=1,
)
try:
versions = runtime.list_skill_versions('alice', 'label-master')
self.assertEqual(len(versions['versions']), 1)
current_id = versions['current_snapshot_id']
self.assertTrue(versions['versions'][0]['is_current'])
manifest = runtime.get_skill_version_manifest(
'alice',
'label-master',
current_id,
)
skill_file = next(
item
for item in manifest['files']
if item['path'] == 'SKILL.md'
)
self.assertTrue(skill_file['editable'])
current_file = runtime.read_skill_version_file(
'alice',
'label-master',
current_id,
'SKILL.md',
)
updated_content = current_file['content'].replace(
'Read the manifest',
'Read the manifest carefully',
)
saved = runtime.save_skill_version(
'alice',
'label-master',
base_snapshot_id=current_id,
version_name='边界调整 v1',
note='测试版本',
files={'SKILL.md': updated_content},
)
self.assertEqual(saved['version_name'], '边界调整 v1')
self.assertEqual(saved['source_type'], 'local')
self.assertNotEqual(saved['id'], current_id)
original = runtime.read_skill_version_file(
'alice',
'label-master',
current_id,
'SKILL.md',
)
forked = runtime.read_skill_version_file(
'alice',
'label-master',
saved['id'],
'SKILL.md',
)
self.assertNotIn('carefully', original['content'])
self.assertIn('carefully', forked['content'])
dataset = runtime.create_dataset(
account_id='alice',
name='versioned',
filename='versioned.csv',
rows=[{'query': '导航去公司'}],
)
experiment = runtime.create_experiment(
account_id='alice',
dataset_id=dataset['id'],
name='local version',
snapshot_id=saved['id'],
)
self.assertEqual(experiment['snapshot_id'], saved['id'])
self.assertEqual(
experiment['skill_version'],
saved['content_hash'][:12],
)
with self.assertRaisesRegex(
ValueError,
'Skill 版本不存在',
):
runtime.read_skill_version_file(
'bob',
'label-master',
saved['id'],
'SKILL.md',
)
finally:
runtime.shutdown()
def test_result_normalization_and_label_equivalence(self) -> None: def test_result_normalization_and_label_equivalence(self) -> None:
result = normalize_evaluation_output( result = normalize_evaluation_output(
json.dumps( json.dumps(