feat: add editable skill evaluation versions
This commit is contained in:
wuyang6
@@ -1149,6 +1149,7 @@ class EvaluationAnalyzeRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
query: str = Field(min_length=1, max_length=20_000)
|
||||
skill_name: str = 'label-master'
|
||||
snapshot_id: str | None = None
|
||||
model: str | None = None
|
||||
history: list[dict[str, Any]] = Field(default_factory=list)
|
||||
context: dict[str, Any] = Field(default_factory=dict)
|
||||
@@ -1162,10 +1163,19 @@ class EvaluationExperimentCreateRequest(BaseModel):
|
||||
dataset_id: str = Field(min_length=1)
|
||||
name: str = ''
|
||||
skill_name: str = 'label-master'
|
||||
snapshot_id: str | None = None
|
||||
model: str | None = None
|
||||
concurrency: int = Field(default=2, ge=1, le=8)
|
||||
|
||||
|
||||
class EvaluationSkillVersionSaveRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
base_snapshot_id: str = Field(min_length=1)
|
||||
version_name: str = Field(min_length=1, max_length=80)
|
||||
note: str = Field(default='', max_length=1000)
|
||||
files: dict[str, str]
|
||||
|
||||
|
||||
class EvaluationExperimentActionRequest(BaseModel):
|
||||
account_id: str = Field(min_length=1)
|
||||
|
||||
@@ -1534,6 +1544,74 @@ def create_app(state: AgentState) -> FastAPI:
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get('/api/evaluations/skills/{skill_name}/versions')
|
||||
async def list_evaluation_skill_versions(
|
||||
skill_name: str,
|
||||
account_id: str,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
state.evaluation_runtime.list_skill_versions,
|
||||
_safe_account_id(account_id),
|
||||
skill_name,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get(
|
||||
'/api/evaluations/skills/{skill_name}/versions/{snapshot_id}'
|
||||
)
|
||||
async def get_evaluation_skill_version(
|
||||
skill_name: str,
|
||||
snapshot_id: str,
|
||||
account_id: str,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.get_skill_version_manifest(
|
||||
_safe_account_id(account_id),
|
||||
skill_name,
|
||||
snapshot_id,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@app.get(
|
||||
'/api/evaluations/skills/{skill_name}/versions/{snapshot_id}/file'
|
||||
)
|
||||
async def get_evaluation_skill_version_file(
|
||||
skill_name: str,
|
||||
snapshot_id: str,
|
||||
account_id: str,
|
||||
path: str,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return state.evaluation_runtime.read_skill_version_file(
|
||||
_safe_account_id(account_id),
|
||||
skill_name,
|
||||
snapshot_id,
|
||||
path,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/skills/{skill_name}/versions')
|
||||
async def save_evaluation_skill_version(
|
||||
skill_name: str,
|
||||
payload: EvaluationSkillVersionSaveRequest,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
state.evaluation_runtime.save_skill_version,
|
||||
_safe_account_id(payload.account_id),
|
||||
skill_name,
|
||||
base_snapshot_id=payload.base_snapshot_id,
|
||||
version_name=payload.version_name,
|
||||
note=payload.note,
|
||||
files=payload.files,
|
||||
)
|
||||
except EvaluationError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post('/api/evaluations/analyze')
|
||||
async def analyze_evaluation_case(
|
||||
payload: EvaluationAnalyzeRequest,
|
||||
@@ -1544,6 +1622,7 @@ def create_app(state: AgentState) -> FastAPI:
|
||||
account_id=_safe_account_id(payload.account_id),
|
||||
query=payload.query,
|
||||
skill_name=payload.skill_name,
|
||||
snapshot_id=payload.snapshot_id,
|
||||
model=payload.model,
|
||||
history=payload.history,
|
||||
context=payload.context,
|
||||
@@ -1642,6 +1721,7 @@ def create_app(state: AgentState) -> FastAPI:
|
||||
dataset_id=payload.dataset_id,
|
||||
name=payload.name,
|
||||
skill_name=payload.skill_name,
|
||||
snapshot_id=payload.snapshot_id,
|
||||
model=payload.model,
|
||||
concurrency=payload.concurrency,
|
||||
)
|
||||
|
||||
@@ -6,14 +6,19 @@ import {
|
||||
ChevronRightIcon,
|
||||
CircleAlertIcon,
|
||||
DownloadIcon,
|
||||
FileCode2Icon,
|
||||
FilePenLineIcon,
|
||||
FileSpreadsheetIcon,
|
||||
FileTextIcon,
|
||||
FlaskConicalIcon,
|
||||
GitBranchIcon,
|
||||
LoaderCircleIcon,
|
||||
PauseIcon,
|
||||
PlayIcon,
|
||||
PlusIcon,
|
||||
RefreshCwIcon,
|
||||
RotateCcwIcon,
|
||||
SaveIcon,
|
||||
SearchIcon,
|
||||
SquareIcon,
|
||||
UploadIcon,
|
||||
@@ -50,6 +55,45 @@ type EvaluationMetadata = {
|
||||
};
|
||||
};
|
||||
|
||||
type SkillVersion = {
|
||||
id: string;
|
||||
skill_name: string;
|
||||
version_name: string;
|
||||
version_note: string;
|
||||
source_type: "git" | "local";
|
||||
parent_snapshot_id: string;
|
||||
content_hash: string;
|
||||
git_commit: string;
|
||||
git_status: string;
|
||||
file_count: number;
|
||||
created_at: number;
|
||||
is_current: boolean;
|
||||
};
|
||||
|
||||
type SkillVersionFile = {
|
||||
path: string;
|
||||
size: number;
|
||||
editable: boolean;
|
||||
previewable: boolean;
|
||||
};
|
||||
|
||||
type SkillVersionManifest = SkillVersion & {
|
||||
files: SkillVersionFile[];
|
||||
};
|
||||
|
||||
type SkillVersionsResponse = {
|
||||
skill_name: string;
|
||||
current_snapshot_id: string;
|
||||
versions: SkillVersion[];
|
||||
};
|
||||
|
||||
type SkillFileContent = {
|
||||
path: string;
|
||||
content: string;
|
||||
size: number;
|
||||
editable: boolean;
|
||||
};
|
||||
|
||||
type ModelOption = {
|
||||
id: string;
|
||||
name?: string;
|
||||
@@ -223,6 +267,8 @@ function EvaluationWorkspace() {
|
||||
label_map: {},
|
||||
});
|
||||
const [skillName, setSkillName] = useState("");
|
||||
const [skillVersions, setSkillVersions] = useState<SkillVersion[]>([]);
|
||||
const [snapshotId, setSnapshotId] = useState("");
|
||||
const [model, setModel] = useState("");
|
||||
const [concurrency, setConcurrency] = useState(2);
|
||||
const [experimentName, setExperimentName] = useState("");
|
||||
@@ -235,6 +281,18 @@ function EvaluationWorkspace() {
|
||||
const [queryFilter, setQueryFilter] = useState("");
|
||||
const [busy, setBusy] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
const [skillEditorOpen, setSkillEditorOpen] = useState(false);
|
||||
const [skillEditorBusy, setSkillEditorBusy] = useState("");
|
||||
const [skillEditorError, setSkillEditorError] = useState("");
|
||||
const [skillManifest, setSkillManifest] =
|
||||
useState<SkillVersionManifest | null>(null);
|
||||
const [selectedSkillFile, setSelectedSkillFile] = useState("");
|
||||
const [skillFileContents, setSkillFileContents] = useState<
|
||||
Record<string, SkillFileContent>
|
||||
>({});
|
||||
const [dirtySkillFiles, setDirtySkillFiles] = useState<string[]>([]);
|
||||
const [skillVersionName, setSkillVersionName] = useState("");
|
||||
const [skillVersionNote, setSkillVersionNote] = useState("");
|
||||
const uploadRef = useRef<HTMLInputElement>(null);
|
||||
const selectedDatasetIdRef = useRef("");
|
||||
const selectedExperimentIdRef = useRef("");
|
||||
@@ -256,6 +314,33 @@ function EvaluationWorkspace() {
|
||||
() => datasets.find((item) => item.id === selectedDatasetId) ?? null,
|
||||
[datasets, selectedDatasetId],
|
||||
);
|
||||
const selectedSkillVersion = useMemo(
|
||||
() => skillVersions.find((item) => item.id === snapshotId) ?? null,
|
||||
[skillVersions, snapshotId],
|
||||
);
|
||||
|
||||
const loadSkillVersions = useCallback(
|
||||
async (nextSkillName: string, preferredSnapshotId = "") => {
|
||||
if (!nextSkillName) return null;
|
||||
const response = await fetch(
|
||||
`/api/claw/evaluations/skills/${encodeURIComponent(nextSkillName)}/versions`,
|
||||
{ cache: "no-store" },
|
||||
);
|
||||
const payload = await readJson<SkillVersionsResponse>(response);
|
||||
setSkillVersions(payload.versions);
|
||||
const preferred = payload.versions.find(
|
||||
(item) => item.id === preferredSnapshotId,
|
||||
);
|
||||
setSnapshotId(
|
||||
preferred?.id ||
|
||||
payload.current_snapshot_id ||
|
||||
payload.versions[0]?.id ||
|
||||
"",
|
||||
);
|
||||
return payload;
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const refreshExperiment = useCallback(async (experimentId: string) => {
|
||||
const response = await fetch(
|
||||
@@ -358,6 +443,15 @@ function EvaluationWorkspace() {
|
||||
void loadWorkspace();
|
||||
}, [loadWorkspace]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!skillName) return;
|
||||
setSkillVersions([]);
|
||||
setSnapshotId("");
|
||||
void loadSkillVersions(skillName).catch((reason) =>
|
||||
setError(messageFrom(reason)),
|
||||
);
|
||||
}, [loadSkillVersions, skillName]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!experiment || !ACTIVE_STATUSES.has(experiment.status)) return;
|
||||
const timer = window.setInterval(() => {
|
||||
@@ -368,6 +462,131 @@ function EvaluationWorkspace() {
|
||||
return () => window.clearInterval(timer);
|
||||
}, [experiment, refreshExperiment]);
|
||||
|
||||
async function fetchSkillFile(
|
||||
versionId: string,
|
||||
filePath: string,
|
||||
force = false,
|
||||
) {
|
||||
setSelectedSkillFile(filePath);
|
||||
const cached = skillFileContents[filePath];
|
||||
if (cached && !force) return cached;
|
||||
setSkillEditorBusy("file");
|
||||
setSkillEditorError("");
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions/${encodeURIComponent(versionId)}/file?path=${encodeURIComponent(filePath)}`,
|
||||
{ cache: "no-store" },
|
||||
);
|
||||
const payload = await readJson<SkillFileContent>(response);
|
||||
setSkillFileContents((current) => ({
|
||||
...current,
|
||||
[filePath]: payload,
|
||||
}));
|
||||
return payload;
|
||||
} catch (reason) {
|
||||
setSkillEditorError(messageFrom(reason));
|
||||
return null;
|
||||
} finally {
|
||||
setSkillEditorBusy("");
|
||||
}
|
||||
}
|
||||
|
||||
async function openSkillEditor() {
|
||||
if (!skillName) return;
|
||||
setSkillEditorOpen(true);
|
||||
setSkillEditorBusy("manifest");
|
||||
setSkillEditorError("");
|
||||
setSkillManifest(null);
|
||||
setSelectedSkillFile("");
|
||||
setSkillFileContents({});
|
||||
setDirtySkillFiles([]);
|
||||
setSkillVersionName("");
|
||||
setSkillVersionNote("");
|
||||
try {
|
||||
let versionId = snapshotId;
|
||||
if (!versionId) {
|
||||
const versions = await loadSkillVersions(skillName);
|
||||
versionId = versions?.current_snapshot_id ?? "";
|
||||
}
|
||||
if (!versionId) throw new Error("当前 Skill 没有可用版本");
|
||||
const response = await fetch(
|
||||
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions/${encodeURIComponent(versionId)}`,
|
||||
{ cache: "no-store" },
|
||||
);
|
||||
const manifest = await readJson<SkillVersionManifest>(response);
|
||||
setSkillManifest(manifest);
|
||||
const firstFile =
|
||||
manifest.files.find((item) => item.path === "SKILL.md") ??
|
||||
manifest.files.find((item) => item.previewable);
|
||||
if (firstFile) {
|
||||
await fetchSkillFile(manifest.id, firstFile.path, true);
|
||||
}
|
||||
} catch (reason) {
|
||||
setSkillEditorError(messageFrom(reason));
|
||||
} finally {
|
||||
setSkillEditorBusy("");
|
||||
}
|
||||
}
|
||||
|
||||
function updateSkillFile(content: string) {
|
||||
if (!selectedSkillFile) return;
|
||||
setSkillFileContents((current) => {
|
||||
const existing = current[selectedSkillFile];
|
||||
if (!existing) return current;
|
||||
return {
|
||||
...current,
|
||||
[selectedSkillFile]: { ...existing, content },
|
||||
};
|
||||
});
|
||||
setDirtySkillFiles((current) =>
|
||||
current.includes(selectedSkillFile)
|
||||
? current
|
||||
: [...current, selectedSkillFile],
|
||||
);
|
||||
}
|
||||
|
||||
async function saveSkillVersion() {
|
||||
if (!skillManifest) return;
|
||||
if (!skillVersionName.trim()) {
|
||||
setSkillEditorError("请填写新版本名称");
|
||||
return;
|
||||
}
|
||||
if (!dirtySkillFiles.length) {
|
||||
setSkillEditorError("当前没有文件修改");
|
||||
return;
|
||||
}
|
||||
setSkillEditorBusy("save");
|
||||
setSkillEditorError("");
|
||||
try {
|
||||
const files = Object.fromEntries(
|
||||
dirtySkillFiles.map((path) => [
|
||||
path,
|
||||
skillFileContents[path]?.content ?? "",
|
||||
]),
|
||||
);
|
||||
const response = await fetch(
|
||||
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
base_snapshot_id: skillManifest.id,
|
||||
version_name: skillVersionName.trim(),
|
||||
note: skillVersionNote.trim(),
|
||||
files,
|
||||
}),
|
||||
},
|
||||
);
|
||||
const saved = await readJson<SkillVersion>(response);
|
||||
await loadSkillVersions(skillName, saved.id);
|
||||
setSkillEditorOpen(false);
|
||||
} catch (reason) {
|
||||
setSkillEditorError(messageFrom(reason));
|
||||
} finally {
|
||||
setSkillEditorBusy("");
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadDataset(file: File) {
|
||||
setBusy("upload");
|
||||
setError("");
|
||||
@@ -434,6 +653,7 @@ function EvaluationWorkspace() {
|
||||
name:
|
||||
experimentName.trim() || `${skillName} · ${selectedDataset.name}`,
|
||||
skill_name: skillName,
|
||||
snapshot_id: snapshotId || undefined,
|
||||
model: model || undefined,
|
||||
concurrency,
|
||||
}),
|
||||
@@ -475,6 +695,7 @@ function EvaluationWorkspace() {
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
skill_name: skillName,
|
||||
snapshot_id: snapshotId || undefined,
|
||||
model: model || undefined,
|
||||
}),
|
||||
});
|
||||
@@ -661,6 +882,34 @@ function EvaluationWorkspace() {
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label className="flex h-8 items-center gap-2 rounded-md border bg-background px-2">
|
||||
<span className="text-muted-foreground text-xs">版本</span>
|
||||
<select
|
||||
aria-label="全局 Skill 版本"
|
||||
value={snapshotId}
|
||||
onChange={(event) => setSnapshotId(event.target.value)}
|
||||
disabled={!skillVersions.length}
|
||||
className="h-7 min-w-36 max-w-56 bg-transparent font-medium text-xs outline-none disabled:opacity-60"
|
||||
>
|
||||
{skillVersions.map((version) => (
|
||||
<option key={version.id} value={version.id}>
|
||||
{version.version_name}
|
||||
{version.source_type === "local" ? " · 本地" : ""}
|
||||
{` · ${version.content_hash.slice(0, 8)}`}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => void openSkillEditor()}
|
||||
disabled={!snapshotId}
|
||||
title="查看并编辑当前 Skill 版本"
|
||||
>
|
||||
<FilePenLineIcon />
|
||||
编辑
|
||||
</Button>
|
||||
<label className="flex h-8 min-w-0 items-center gap-2 rounded-md border bg-background px-2">
|
||||
<span className="text-muted-foreground text-xs">模型</span>
|
||||
<select
|
||||
@@ -871,10 +1120,222 @@ function EvaluationWorkspace() {
|
||||
);
|
||||
}}
|
||||
/>
|
||||
<SkillEditorSheet
|
||||
open={skillEditorOpen}
|
||||
onOpenChange={setSkillEditorOpen}
|
||||
skillName={skillName}
|
||||
version={selectedSkillVersion}
|
||||
manifest={skillManifest}
|
||||
selectedFile={selectedSkillFile}
|
||||
fileContent={
|
||||
selectedSkillFile
|
||||
? (skillFileContents[selectedSkillFile] ?? null)
|
||||
: null
|
||||
}
|
||||
dirtyFiles={dirtySkillFiles}
|
||||
busy={skillEditorBusy}
|
||||
error={skillEditorError}
|
||||
versionName={skillVersionName}
|
||||
setVersionName={setSkillVersionName}
|
||||
versionNote={skillVersionNote}
|
||||
setVersionNote={setSkillVersionNote}
|
||||
onSelectFile={(filePath) => {
|
||||
if (skillManifest) {
|
||||
void fetchSkillFile(skillManifest.id, filePath);
|
||||
}
|
||||
}}
|
||||
onChangeContent={updateSkillFile}
|
||||
onSave={() => void saveSkillVersion()}
|
||||
/>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
function SkillEditorSheet(props: {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
skillName: string;
|
||||
version: SkillVersion | null;
|
||||
manifest: SkillVersionManifest | null;
|
||||
selectedFile: string;
|
||||
fileContent: SkillFileContent | null;
|
||||
dirtyFiles: string[];
|
||||
busy: string;
|
||||
error: string;
|
||||
versionName: string;
|
||||
setVersionName: (value: string) => void;
|
||||
versionNote: string;
|
||||
setVersionNote: (value: string) => void;
|
||||
onSelectFile: (path: string) => void;
|
||||
onChangeContent: (content: string) => void;
|
||||
onSave: () => void;
|
||||
}) {
|
||||
const {
|
||||
open,
|
||||
onOpenChange,
|
||||
skillName,
|
||||
version,
|
||||
manifest,
|
||||
selectedFile,
|
||||
fileContent,
|
||||
dirtyFiles,
|
||||
busy,
|
||||
error,
|
||||
versionName,
|
||||
setVersionName,
|
||||
versionNote,
|
||||
setVersionNote,
|
||||
onSelectFile,
|
||||
onChangeContent,
|
||||
onSave,
|
||||
} = props;
|
||||
|
||||
return (
|
||||
<Sheet open={open} onOpenChange={onOpenChange}>
|
||||
<SheetContent className="w-[min(96vw,1100px)] gap-0 sm:max-w-[1100px]">
|
||||
<SheetHeader className="border-b px-5 py-4">
|
||||
<SheetTitle className="flex items-center gap-2">
|
||||
<FilePenLineIcon className="size-4" />
|
||||
Skill 版本编辑
|
||||
</SheetTitle>
|
||||
<SheetDescription className="flex flex-wrap items-center gap-x-3 gap-y-1">
|
||||
<span>{skillName}</span>
|
||||
<span className="inline-flex items-center gap-1">
|
||||
<GitBranchIcon className="size-3.5" />
|
||||
{version?.version_name || "正在读取版本"}
|
||||
</span>
|
||||
{version?.content_hash ? (
|
||||
<code>{version.content_hash.slice(0, 12)}</code>
|
||||
) : null}
|
||||
</SheetDescription>
|
||||
</SheetHeader>
|
||||
|
||||
{error ? (
|
||||
<div className="mx-5 mt-3 flex items-start gap-2 border border-destructive/30 bg-destructive/5 px-3 py-2 text-destructive text-sm">
|
||||
<CircleAlertIcon className="mt-0.5 size-4 shrink-0" />
|
||||
<span>{error}</span>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="grid min-h-0 flex-1 grid-cols-[230px_minmax(0,1fr)]">
|
||||
<aside className="min-h-0 overflow-y-auto border-r bg-muted/15 p-3">
|
||||
<div className="mb-2 flex items-center justify-between px-2 text-xs">
|
||||
<span className="font-medium">Skill 文件</span>
|
||||
<span className="text-muted-foreground">
|
||||
{manifest?.files.length ?? 0}
|
||||
</span>
|
||||
</div>
|
||||
<div className="space-y-0.5">
|
||||
{manifest?.files.map((file) => {
|
||||
const dirty = dirtyFiles.includes(file.path);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
key={file.path}
|
||||
onClick={() => onSelectFile(file.path)}
|
||||
disabled={!file.previewable}
|
||||
className={`flex w-full items-center gap-2 rounded-md px-2 py-1.5 text-left text-xs transition-colors ${
|
||||
selectedFile === file.path
|
||||
? "bg-accent text-accent-foreground"
|
||||
: "hover:bg-accent/60"
|
||||
} disabled:cursor-not-allowed disabled:opacity-50`}
|
||||
>
|
||||
{file.path.endsWith(".md") ? (
|
||||
<FileTextIcon className="size-3.5 shrink-0" />
|
||||
) : (
|
||||
<FileCode2Icon className="size-3.5 shrink-0" />
|
||||
)}
|
||||
<span className="min-w-0 flex-1 truncate">{file.path}</span>
|
||||
{dirty ? (
|
||||
<span
|
||||
className="size-1.5 shrink-0 rounded-full bg-sky-500"
|
||||
title="已修改"
|
||||
/>
|
||||
) : null}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<section className="flex min-h-0 min-w-0 flex-col">
|
||||
<div className="flex h-10 shrink-0 items-center gap-2 border-b px-4">
|
||||
<span className="min-w-0 flex-1 truncate font-medium text-sm">
|
||||
{selectedFile || "选择一个文件"}
|
||||
</span>
|
||||
{fileContent ? (
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{fileContent.editable ? "可编辑" : "只读"}
|
||||
</span>
|
||||
) : null}
|
||||
{busy === "file" ? (
|
||||
<LoaderCircleIcon className="size-4 animate-spin text-muted-foreground" />
|
||||
) : null}
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 p-4">
|
||||
{fileContent ? (
|
||||
<textarea
|
||||
value={fileContent.content}
|
||||
onChange={(event) => onChangeContent(event.target.value)}
|
||||
readOnly={!fileContent.editable}
|
||||
spellCheck={false}
|
||||
className="h-full min-h-72 w-full resize-none border bg-muted/10 p-4 font-mono text-[13px] leading-6 outline-none focus:ring-2 focus:ring-ring/40 read-only:cursor-default read-only:opacity-80"
|
||||
/>
|
||||
) : (
|
||||
<div className="grid h-full min-h-72 place-items-center text-muted-foreground text-sm">
|
||||
{busy === "manifest" ? (
|
||||
<LoaderCircleIcon className="size-5 animate-spin" />
|
||||
) : (
|
||||
"选择左侧文件查看内容"
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="shrink-0 border-t bg-muted/10 px-4 py-3">
|
||||
<div className="grid gap-3 md:grid-cols-[minmax(0,220px)_minmax(0,1fr)_auto] md:items-end">
|
||||
<Field label="新版本名称">
|
||||
<Input
|
||||
value={versionName}
|
||||
onChange={(event) => setVersionName(event.target.value)}
|
||||
placeholder="例如：导航边界调整 v1"
|
||||
maxLength={80}
|
||||
/>
|
||||
</Field>
|
||||
<Field label="版本备注">
|
||||
<Input
|
||||
value={versionNote}
|
||||
onChange={(event) => setVersionNote(event.target.value)}
|
||||
placeholder="记录本次调整内容"
|
||||
maxLength={1000}
|
||||
/>
|
||||
</Field>
|
||||
<Button
|
||||
onClick={onSave}
|
||||
disabled={busy === "save" || !dirtyFiles.length}
|
||||
>
|
||||
{busy === "save" ? (
|
||||
<LoaderCircleIcon className="animate-spin" />
|
||||
) : (
|
||||
<SaveIcon />
|
||||
)}
|
||||
保存为新版本
|
||||
</Button>
|
||||
</div>
|
||||
<p className="mt-2 text-muted-foreground text-xs">
|
||||
已保存版本保持只读；Markdown 和 JSON 可编辑，脚本暂只读。
|
||||
{dirtyFiles.length
|
||||
? ` 当前修改 ${dirtyFiles.length} 个文件。`
|
||||
: ""}
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</SheetContent>
|
||||
</Sheet>
|
||||
);
|
||||
}
|
||||
|
||||
function SingleQueryWorkspace(props: {
|
||||
query: string;
|
||||
setQuery: (value: string) => void;
|
||||
|
||||
+379
-3
@@ -14,7 +14,7 @@ import threading
|
||||
import time
|
||||
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
|
||||
from dataclasses import dataclass, field, replace
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Callable, Iterable
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -35,6 +35,10 @@ SUPPORTED_DATASET_SUFFIXES = {'.csv', '.tsv', '.json', '.jsonl', '.xlsx'}
|
||||
DEFAULT_CONCURRENCY = 2
|
||||
MAX_CONCURRENCY = 8
|
||||
MAX_DATASET_ROWS = 20_000
|
||||
VISIBLE_SKILL_FILE_SUFFIXES = {'.json', '.md', '.py', '.txt', '.yaml', '.yml'}
|
||||
EDITABLE_SKILL_FILE_SUFFIXES = {'.json', '.md'}
|
||||
MAX_SKILL_FILE_BYTES = 512 * 1024
|
||||
MAX_SKILL_VERSION_EDIT_BYTES = 2 * 1024 * 1024
|
||||
|
||||
EVALUATION_OUTPUT_SCHEMA = OutputSchemaConfig(
|
||||
name='skill_evaluation_result',
|
||||
@@ -491,6 +495,7 @@ class EvaluationRuntime:
|
||||
account_id: str,
|
||||
query: str,
|
||||
skill_name: str = 'label-master',
|
||||
snapshot_id: str | None = None,
|
||||
model: str | None = None,
|
||||
history: list[dict[str, Any]] | None = None,
|
||||
context: dict[str, Any] | None = None,
|
||||
@@ -501,7 +506,11 @@ class EvaluationRuntime:
|
||||
normalized_query = query.strip()
|
||||
if not normalized_query:
|
||||
raise EvaluationError('Query 不能为空')
|
||||
snapshot = self.create_skill_snapshot(account_id, skill_name)
|
||||
snapshot = self.resolve_skill_snapshot(
|
||||
account_id,
|
||||
skill_name,
|
||||
snapshot_id=snapshot_id,
|
||||
)
|
||||
model_name = model or self.model_config_for(account_id).model
|
||||
analysis_id = f'single_{uuid4().hex[:16]}'
|
||||
case_id = f'case_{uuid4().hex[:16]}'
|
||||
@@ -669,6 +678,7 @@ class EvaluationRuntime:
|
||||
dataset_id: str,
|
||||
name: str,
|
||||
skill_name: str = 'label-master',
|
||||
snapshot_id: str | None = None,
|
||||
model: str | None = None,
|
||||
concurrency: int = DEFAULT_CONCURRENCY,
|
||||
) -> dict[str, Any]:
|
||||
@@ -678,7 +688,11 @@ class EvaluationRuntime:
|
||||
canonical_rows = [row for row in canonical_rows if row['query']]
|
||||
if not canonical_rows:
|
||||
raise EvaluationError('当前字段映射没有生成有效 case')
|
||||
snapshot = self.create_skill_snapshot(account_id, skill_name)
|
||||
snapshot = self.resolve_skill_snapshot(
|
||||
account_id,
|
||||
skill_name,
|
||||
snapshot_id=snapshot_id,
|
||||
)
|
||||
model_config = self.model_config_for(account_id)
|
||||
experiment_id = f'eval_{uuid4().hex[:16]}'
|
||||
now = time.time()
|
||||
@@ -1004,6 +1018,274 @@ class EvaluationRuntime:
|
||||
# Snapshots and execution
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def resolve_skill_snapshot(
|
||||
self,
|
||||
account_id: str,
|
||||
skill_name: str,
|
||||
*,
|
||||
snapshot_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if not snapshot_id:
|
||||
return self.create_skill_snapshot(account_id, skill_name)
|
||||
row = self.store.fetchone(
|
||||
"""
|
||||
SELECT *
|
||||
FROM skill_snapshots
|
||||
WHERE id = ? AND account_id = ? AND skill_name = ?
|
||||
""",
|
||||
(snapshot_id, account_id, skill_name),
|
||||
)
|
||||
if row is None or not Path(row['path']).is_dir():
|
||||
raise EvaluationError('Skill 版本不存在或已失效')
|
||||
return self._snapshot_payload(row)
|
||||
|
||||
def list_skill_versions(
|
||||
self,
|
||||
account_id: str,
|
||||
skill_name: str,
|
||||
) -> dict[str, Any]:
|
||||
current = self.create_skill_snapshot(account_id, skill_name)
|
||||
rows = self.store.fetchall(
|
||||
"""
|
||||
SELECT *
|
||||
FROM skill_snapshots
|
||||
WHERE account_id = ? AND skill_name = ?
|
||||
ORDER BY created_at DESC
|
||||
""",
|
||||
(account_id, skill_name),
|
||||
)
|
||||
rows.sort(
|
||||
key=lambda row: (
|
||||
row['id'] != current['id'],
|
||||
-float(row['created_at']),
|
||||
)
|
||||
)
|
||||
return {
|
||||
'skill_name': skill_name,
|
||||
'current_snapshot_id': current['id'],
|
||||
'versions': [
|
||||
self._public_skill_version(
|
||||
row,
|
||||
is_current=row['id'] == current['id'],
|
||||
)
|
||||
for row in rows
|
||||
if Path(row['path']).is_dir()
|
||||
],
|
||||
}
|
||||
|
||||
def get_skill_version_manifest(
|
||||
self,
|
||||
account_id: str,
|
||||
skill_name: str,
|
||||
snapshot_id: str,
|
||||
) -> dict[str, Any]:
|
||||
snapshot = self.resolve_skill_snapshot(
|
||||
account_id,
|
||||
skill_name,
|
||||
snapshot_id=snapshot_id,
|
||||
)
|
||||
snapshot_skill_dir = self._snapshot_skill_dir(snapshot, skill_name)
|
||||
files: list[dict[str, Any]] = []
|
||||
for path in sorted(
|
||||
item for item in snapshot_skill_dir.rglob('*') if item.is_file()
|
||||
):
|
||||
relative = path.relative_to(snapshot_skill_dir).as_posix()
|
||||
if any(part.startswith('.') for part in PurePosixPath(relative).parts):
|
||||
continue
|
||||
suffix = path.suffix.lower()
|
||||
if suffix not in VISIBLE_SKILL_FILE_SUFFIXES:
|
||||
continue
|
||||
size = path.stat().st_size
|
||||
files.append(
|
||||
{
|
||||
'path': relative,
|
||||
'size': size,
|
||||
'editable': (
|
||||
suffix in EDITABLE_SKILL_FILE_SUFFIXES
|
||||
and size <= MAX_SKILL_FILE_BYTES
|
||||
),
|
||||
'previewable': size <= MAX_SKILL_FILE_BYTES,
|
||||
}
|
||||
)
|
||||
row = self._skill_snapshot_row(account_id, skill_name, snapshot_id)
|
||||
return {
|
||||
**self._public_skill_version(row),
|
||||
'files': files,
|
||||
}
|
||||
|
||||
def read_skill_version_file(
|
||||
self,
|
||||
account_id: str,
|
||||
skill_name: str,
|
||||
snapshot_id: str,
|
||||
file_path: str,
|
||||
) -> dict[str, Any]:
|
||||
snapshot = self.resolve_skill_snapshot(
|
||||
account_id,
|
||||
skill_name,
|
||||
snapshot_id=snapshot_id,
|
||||
)
|
||||
snapshot_skill_dir = self._snapshot_skill_dir(snapshot, skill_name)
|
||||
relative, target = self._resolve_skill_file(
|
||||
snapshot_skill_dir,
|
||||
file_path,
|
||||
)
|
||||
if target.suffix.lower() not in VISIBLE_SKILL_FILE_SUFFIXES:
|
||||
raise EvaluationError('该文件类型不支持预览')
|
||||
size = target.stat().st_size
|
||||
if size > MAX_SKILL_FILE_BYTES:
|
||||
raise EvaluationError('文件过大，暂不支持在线预览')
|
||||
try:
|
||||
content = target.read_text(encoding='utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
raise EvaluationError('文件不是 UTF-8 文本') from exc
|
||||
return {
|
||||
'path': relative,
|
||||
'content': content,
|
||||
'size': size,
|
||||
'editable': target.suffix.lower() in EDITABLE_SKILL_FILE_SUFFIXES,
|
||||
}
|
||||
|
||||
def save_skill_version(
|
||||
self,
|
||||
account_id: str,
|
||||
skill_name: str,
|
||||
*,
|
||||
base_snapshot_id: str,
|
||||
version_name: str,
|
||||
note: str = '',
|
||||
files: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
normalized_name = version_name.strip()
|
||||
if not normalized_name:
|
||||
raise EvaluationError('请填写版本名称')
|
||||
if len(normalized_name) > 80:
|
||||
raise EvaluationError('版本名称不能超过 80 个字符')
|
||||
normalized_note = note.strip()
|
||||
if len(normalized_note) > 1000:
|
||||
raise EvaluationError('版本备注不能超过 1000 个字符')
|
||||
if not files:
|
||||
raise EvaluationError('没有需要保存的文件修改')
|
||||
|
||||
base = self.resolve_skill_snapshot(
|
||||
account_id,
|
||||
skill_name,
|
||||
snapshot_id=base_snapshot_id,
|
||||
)
|
||||
base_skill_dir = self._snapshot_skill_dir(base, skill_name)
|
||||
total_bytes = sum(
|
||||
len(str(content).encode('utf-8')) for content in files.values()
|
||||
)
|
||||
if total_bytes > MAX_SKILL_VERSION_EDIT_BYTES:
|
||||
raise EvaluationError('本次修改内容过大，请拆分后保存')
|
||||
|
||||
draft_root: Path | None = (
|
||||
self.root / '.skill_version_drafts' / uuid4().hex
|
||||
)
|
||||
draft_skill_dir = draft_root / 'skills' / skill_name
|
||||
draft_skill_dir.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(base_skill_dir, draft_skill_dir)
|
||||
try:
|
||||
for file_path, raw_content in files.items():
|
||||
relative, target = self._resolve_skill_file(
|
||||
draft_skill_dir,
|
||||
file_path,
|
||||
)
|
||||
if target.suffix.lower() not in EDITABLE_SKILL_FILE_SUFFIXES:
|
||||
raise EvaluationError(f'该文件只读：{relative}')
|
||||
content = str(raw_content)
|
||||
if len(content.encode('utf-8')) > MAX_SKILL_FILE_BYTES:
|
||||
raise EvaluationError(f'文件过大：{relative}')
|
||||
if target.suffix.lower() == '.json':
|
||||
try:
|
||||
json.loads(content)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise EvaluationError(
|
||||
f'JSON 格式错误：{relative}，第 {exc.lineno} 行'
|
||||
) from exc
|
||||
target.write_text(content, encoding='utf-8')
|
||||
|
||||
loaded = self._load_snapshot_skill(draft_root, skill_name)
|
||||
if loaded.name != skill_name:
|
||||
raise EvaluationError('SKILL.md 中的 name 不允许修改')
|
||||
content_hash = hash_directory(draft_skill_dir)
|
||||
existing = self.store.fetchone(
|
||||
"""
|
||||
SELECT *
|
||||
FROM skill_snapshots
|
||||
WHERE account_id = ? AND skill_name = ? AND content_hash = ?
|
||||
""",
|
||||
(account_id, skill_name, content_hash),
|
||||
)
|
||||
if existing is not None and Path(existing['path']).is_dir():
|
||||
return self._public_skill_version(existing)
|
||||
|
||||
snapshot_id = f'snapshot_{uuid4().hex[:16]}'
|
||||
snapshot_root = (
|
||||
self.root
|
||||
/ 'skill_snapshots'
|
||||
/ sanitize_filename(account_id)
|
||||
/ sanitize_filename(skill_name)
|
||||
/ content_hash[:16]
|
||||
)
|
||||
with self._lock:
|
||||
if snapshot_root.exists():
|
||||
shutil.rmtree(snapshot_root)
|
||||
snapshot_root.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(draft_root), str(snapshot_root))
|
||||
draft_root = None
|
||||
snapshot_skill_dir = snapshot_root / 'skills' / skill_name
|
||||
metadata = self._skill_snapshot_metadata(
|
||||
snapshot_skill_dir,
|
||||
skill_name,
|
||||
)
|
||||
metadata.update(
|
||||
{
|
||||
'source_type': 'local',
|
||||
'version_name': normalized_name,
|
||||
'version_note': normalized_note,
|
||||
'parent_snapshot_id': base_snapshot_id,
|
||||
}
|
||||
)
|
||||
try:
|
||||
self.store.execute(
|
||||
"""
|
||||
INSERT INTO skill_snapshots (
|
||||
id, account_id, skill_name, content_hash, git_commit,
|
||||
git_status, path, metadata_json, created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
snapshot_id,
|
||||
account_id,
|
||||
skill_name,
|
||||
content_hash,
|
||||
base.get('git_commit', ''),
|
||||
'local',
|
||||
str(snapshot_root),
|
||||
json.dumps(metadata, ensure_ascii=False),
|
||||
time.time(),
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError:
|
||||
existing = self.store.fetchone(
|
||||
"""
|
||||
SELECT *
|
||||
FROM skill_snapshots
|
||||
WHERE account_id = ? AND skill_name = ?
|
||||
AND content_hash = ?
|
||||
""",
|
||||
(account_id, skill_name, content_hash),
|
||||
)
|
||||
if existing is None:
|
||||
raise
|
||||
return self._public_skill_version(existing)
|
||||
row = self._skill_snapshot_row(account_id, skill_name, snapshot_id)
|
||||
return self._public_skill_version(row)
|
||||
finally:
|
||||
if draft_root is not None and draft_root.exists():
|
||||
shutil.rmtree(draft_root, ignore_errors=True)
|
||||
|
||||
def create_skill_snapshot(
|
||||
self,
|
||||
account_id: str,
|
||||
@@ -1043,6 +1325,14 @@ class EvaluationRuntime:
|
||||
ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '.DS_Store'),
|
||||
)
|
||||
metadata = self._skill_snapshot_metadata(snapshot_skill_dir, skill_name)
|
||||
metadata.update(
|
||||
{
|
||||
'source_type': 'git',
|
||||
'version_name': '',
|
||||
'version_note': '',
|
||||
'parent_snapshot_id': '',
|
||||
}
|
||||
)
|
||||
now = time.time()
|
||||
try:
|
||||
self.store.execute(
|
||||
@@ -1619,6 +1909,92 @@ class EvaluationRuntime:
|
||||
'created_at': row['created_at'],
|
||||
}
|
||||
|
||||
def _public_skill_version(
|
||||
self,
|
||||
row: dict[str, Any],
|
||||
*,
|
||||
is_current: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
metadata = json.loads(row.get('metadata_json') or '{}')
|
||||
source_type = str(metadata.get('source_type') or 'git')
|
||||
version_name = str(metadata.get('version_name') or '').strip()
|
||||
if not version_name:
|
||||
if is_current:
|
||||
version_name = '当前 Git'
|
||||
elif source_type == 'local':
|
||||
version_name = f"本地版本 {str(row['content_hash'])[:8]}"
|
||||
else:
|
||||
version_name = f"Git 快照 {str(row['content_hash'])[:8]}"
|
||||
return {
|
||||
'id': row['id'],
|
||||
'skill_name': row['skill_name'],
|
||||
'version_name': version_name,
|
||||
'version_note': str(metadata.get('version_note') or ''),
|
||||
'source_type': source_type,
|
||||
'parent_snapshot_id': str(
|
||||
metadata.get('parent_snapshot_id') or ''
|
||||
),
|
||||
'content_hash': row['content_hash'],
|
||||
'git_commit': row.get('git_commit', ''),
|
||||
'git_status': row.get('git_status', 'unknown'),
|
||||
'file_count': int(metadata.get('file_count') or 0),
|
||||
'created_at': row['created_at'],
|
||||
'is_current': is_current,
|
||||
}
|
||||
|
||||
def _skill_snapshot_row(
|
||||
self,
|
||||
account_id: str,
|
||||
skill_name: str,
|
||||
snapshot_id: str,
|
||||
) -> dict[str, Any]:
|
||||
row = self.store.fetchone(
|
||||
"""
|
||||
SELECT *
|
||||
FROM skill_snapshots
|
||||
WHERE id = ? AND account_id = ? AND skill_name = ?
|
||||
""",
|
||||
(snapshot_id, account_id, skill_name),
|
||||
)
|
||||
if row is None:
|
||||
raise EvaluationError('Skill 版本不存在')
|
||||
return row
|
||||
|
||||
def _snapshot_skill_dir(
|
||||
self,
|
||||
snapshot: dict[str, Any],
|
||||
skill_name: str,
|
||||
) -> Path:
|
||||
skill_dir = Path(snapshot['path']) / 'skills' / skill_name
|
||||
if not skill_dir.is_dir() or not (skill_dir / 'SKILL.md').is_file():
|
||||
raise EvaluationError('Skill 版本文件已失效')
|
||||
return skill_dir
|
||||
|
||||
def _resolve_skill_file(
|
||||
self,
|
||||
skill_dir: Path,
|
||||
file_path: str,
|
||||
) -> tuple[str, Path]:
|
||||
candidate = PurePosixPath(str(file_path).strip())
|
||||
if (
|
||||
not candidate.parts
|
||||
or candidate.is_absolute()
|
||||
or '..' in candidate.parts
|
||||
or any(part.startswith('.') for part in candidate.parts)
|
||||
):
|
||||
raise EvaluationError('文件路径无效')
|
||||
relative = candidate.as_posix()
|
||||
target = skill_dir.joinpath(*candidate.parts)
|
||||
if target.is_symlink() or not target.is_file():
|
||||
raise EvaluationError(f'Skill 文件不存在：{relative}')
|
||||
resolved_root = skill_dir.resolve()
|
||||
resolved_target = target.resolve()
|
||||
try:
|
||||
resolved_target.relative_to(resolved_root)
|
||||
except ValueError as exc:
|
||||
raise EvaluationError('文件路径越界') from exc
|
||||
return relative, resolved_target
|
||||
|
||||
def _experiment_summary(self, row: dict[str, Any]) -> dict[str, Any]:
|
||||
total = int(row.get('total_cases') or 0)
|
||||
completed = int(row.get('completed_cases') or 0)
|
||||
|
||||
@@ -178,6 +178,62 @@ class EvaluationApiTests(unittest.TestCase):
|
||||
self.assertIn('text/csv', export.headers['content-type'])
|
||||
self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
|
||||
|
||||
def test_skill_version_endpoints(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
state = _build_state(root)
|
||||
with TestClient(create_app(state)) as client:
|
||||
versions_response = client.get(
|
||||
'/api/evaluations/skills/label-master/versions',
|
||||
params={'account_id': 'alice'},
|
||||
)
|
||||
self.assertEqual(versions_response.status_code, 200)
|
||||
versions = versions_response.json()
|
||||
current_id = versions['current_snapshot_id']
|
||||
|
||||
manifest = client.get(
|
||||
(
|
||||
'/api/evaluations/skills/label-master/versions/'
|
||||
f'{current_id}'
|
||||
),
|
||||
params={'account_id': 'alice'},
|
||||
)
|
||||
self.assertEqual(manifest.status_code, 200)
|
||||
self.assertTrue(
|
||||
any(
|
||||
item['path'] == 'SKILL.md'
|
||||
for item in manifest.json()['files']
|
||||
)
|
||||
)
|
||||
|
||||
file_response = client.get(
|
||||
(
|
||||
'/api/evaluations/skills/label-master/versions/'
|
||||
f'{current_id}/file'
|
||||
),
|
||||
params={'account_id': 'alice', 'path': 'SKILL.md'},
|
||||
)
|
||||
self.assertEqual(file_response.status_code, 200)
|
||||
content = file_response.json()['content'].replace(
|
||||
'Use the label catalog.',
|
||||
'Use the label catalog carefully.',
|
||||
)
|
||||
saved_response = client.post(
|
||||
'/api/evaluations/skills/label-master/versions',
|
||||
json={
|
||||
'account_id': 'alice',
|
||||
'base_snapshot_id': current_id,
|
||||
'version_name': '测试版本',
|
||||
'note': 'API 测试',
|
||||
'files': {'SKILL.md': content},
|
||||
},
|
||||
)
|
||||
self.assertEqual(saved_response.status_code, 200)
|
||||
self.assertEqual(
|
||||
saved_response.json()['version_name'],
|
||||
'测试版本',
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -169,6 +169,100 @@ class EvaluationRuntimeTests(unittest.TestCase):
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_skill_versions_are_immutable_and_account_scoped(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
_write_test_skill(root)
|
||||
runtime = EvaluationRuntime(
|
||||
root=root / 'evaluations',
|
||||
cwd_for_account=lambda _account: root,
|
||||
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||
max_workers=1,
|
||||
)
|
||||
try:
|
||||
versions = runtime.list_skill_versions('alice', 'label-master')
|
||||
self.assertEqual(len(versions['versions']), 1)
|
||||
current_id = versions['current_snapshot_id']
|
||||
self.assertTrue(versions['versions'][0]['is_current'])
|
||||
|
||||
manifest = runtime.get_skill_version_manifest(
|
||||
'alice',
|
||||
'label-master',
|
||||
current_id,
|
||||
)
|
||||
skill_file = next(
|
||||
item
|
||||
for item in manifest['files']
|
||||
if item['path'] == 'SKILL.md'
|
||||
)
|
||||
self.assertTrue(skill_file['editable'])
|
||||
current_file = runtime.read_skill_version_file(
|
||||
'alice',
|
||||
'label-master',
|
||||
current_id,
|
||||
'SKILL.md',
|
||||
)
|
||||
updated_content = current_file['content'].replace(
|
||||
'Read the manifest',
|
||||
'Read the manifest carefully',
|
||||
)
|
||||
saved = runtime.save_skill_version(
|
||||
'alice',
|
||||
'label-master',
|
||||
base_snapshot_id=current_id,
|
||||
version_name='边界调整 v1',
|
||||
note='测试版本',
|
||||
files={'SKILL.md': updated_content},
|
||||
)
|
||||
self.assertEqual(saved['version_name'], '边界调整 v1')
|
||||
self.assertEqual(saved['source_type'], 'local')
|
||||
self.assertNotEqual(saved['id'], current_id)
|
||||
|
||||
original = runtime.read_skill_version_file(
|
||||
'alice',
|
||||
'label-master',
|
||||
current_id,
|
||||
'SKILL.md',
|
||||
)
|
||||
forked = runtime.read_skill_version_file(
|
||||
'alice',
|
||||
'label-master',
|
||||
saved['id'],
|
||||
'SKILL.md',
|
||||
)
|
||||
self.assertNotIn('carefully', original['content'])
|
||||
self.assertIn('carefully', forked['content'])
|
||||
dataset = runtime.create_dataset(
|
||||
account_id='alice',
|
||||
name='versioned',
|
||||
filename='versioned.csv',
|
||||
rows=[{'query': '导航去公司'}],
|
||||
)
|
||||
experiment = runtime.create_experiment(
|
||||
account_id='alice',
|
||||
dataset_id=dataset['id'],
|
||||
name='local version',
|
||||
snapshot_id=saved['id'],
|
||||
)
|
||||
self.assertEqual(experiment['snapshot_id'], saved['id'])
|
||||
self.assertEqual(
|
||||
experiment['skill_version'],
|
||||
saved['content_hash'][:12],
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
'Skill 版本不存在',
|
||||
):
|
||||
runtime.read_skill_version_file(
|
||||
'bob',
|
||||
'label-master',
|
||||
saved['id'],
|
||||
'SKILL.md',
|
||||
)
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_result_normalization_and_label_equivalence(self) -> None:
|
||||
result = normalize_evaluation_output(
|
||||
json.dumps(
|
||||
|
||||
Reference in New Issue
Block a user