feat: add editable skill evaluation versions
This commit is contained in:
@@ -1149,6 +1149,7 @@ class EvaluationAnalyzeRequest(BaseModel):
|
|||||||
account_id: str = Field(min_length=1)
|
account_id: str = Field(min_length=1)
|
||||||
query: str = Field(min_length=1, max_length=20_000)
|
query: str = Field(min_length=1, max_length=20_000)
|
||||||
skill_name: str = 'label-master'
|
skill_name: str = 'label-master'
|
||||||
|
snapshot_id: str | None = None
|
||||||
model: str | None = None
|
model: str | None = None
|
||||||
history: list[dict[str, Any]] = Field(default_factory=list)
|
history: list[dict[str, Any]] = Field(default_factory=list)
|
||||||
context: dict[str, Any] = Field(default_factory=dict)
|
context: dict[str, Any] = Field(default_factory=dict)
|
||||||
@@ -1162,10 +1163,19 @@ class EvaluationExperimentCreateRequest(BaseModel):
|
|||||||
dataset_id: str = Field(min_length=1)
|
dataset_id: str = Field(min_length=1)
|
||||||
name: str = ''
|
name: str = ''
|
||||||
skill_name: str = 'label-master'
|
skill_name: str = 'label-master'
|
||||||
|
snapshot_id: str | None = None
|
||||||
model: str | None = None
|
model: str | None = None
|
||||||
concurrency: int = Field(default=2, ge=1, le=8)
|
concurrency: int = Field(default=2, ge=1, le=8)
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluationSkillVersionSaveRequest(BaseModel):
|
||||||
|
account_id: str = Field(min_length=1)
|
||||||
|
base_snapshot_id: str = Field(min_length=1)
|
||||||
|
version_name: str = Field(min_length=1, max_length=80)
|
||||||
|
note: str = Field(default='', max_length=1000)
|
||||||
|
files: dict[str, str]
|
||||||
|
|
||||||
|
|
||||||
class EvaluationExperimentActionRequest(BaseModel):
|
class EvaluationExperimentActionRequest(BaseModel):
|
||||||
account_id: str = Field(min_length=1)
|
account_id: str = Field(min_length=1)
|
||||||
|
|
||||||
@@ -1534,6 +1544,74 @@ def create_app(state: AgentState) -> FastAPI:
|
|||||||
except EvaluationError as exc:
|
except EvaluationError as exc:
|
||||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
@app.get('/api/evaluations/skills/{skill_name}/versions')
|
||||||
|
async def list_evaluation_skill_versions(
|
||||||
|
skill_name: str,
|
||||||
|
account_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return await asyncio.to_thread(
|
||||||
|
state.evaluation_runtime.list_skill_versions,
|
||||||
|
_safe_account_id(account_id),
|
||||||
|
skill_name,
|
||||||
|
)
|
||||||
|
except EvaluationError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
@app.get(
|
||||||
|
'/api/evaluations/skills/{skill_name}/versions/{snapshot_id}'
|
||||||
|
)
|
||||||
|
async def get_evaluation_skill_version(
|
||||||
|
skill_name: str,
|
||||||
|
snapshot_id: str,
|
||||||
|
account_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return state.evaluation_runtime.get_skill_version_manifest(
|
||||||
|
_safe_account_id(account_id),
|
||||||
|
skill_name,
|
||||||
|
snapshot_id,
|
||||||
|
)
|
||||||
|
except EvaluationError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
@app.get(
|
||||||
|
'/api/evaluations/skills/{skill_name}/versions/{snapshot_id}/file'
|
||||||
|
)
|
||||||
|
async def get_evaluation_skill_version_file(
|
||||||
|
skill_name: str,
|
||||||
|
snapshot_id: str,
|
||||||
|
account_id: str,
|
||||||
|
path: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return state.evaluation_runtime.read_skill_version_file(
|
||||||
|
_safe_account_id(account_id),
|
||||||
|
skill_name,
|
||||||
|
snapshot_id,
|
||||||
|
path,
|
||||||
|
)
|
||||||
|
except EvaluationError as exc:
|
||||||
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
@app.post('/api/evaluations/skills/{skill_name}/versions')
|
||||||
|
async def save_evaluation_skill_version(
|
||||||
|
skill_name: str,
|
||||||
|
payload: EvaluationSkillVersionSaveRequest,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
return await asyncio.to_thread(
|
||||||
|
state.evaluation_runtime.save_skill_version,
|
||||||
|
_safe_account_id(payload.account_id),
|
||||||
|
skill_name,
|
||||||
|
base_snapshot_id=payload.base_snapshot_id,
|
||||||
|
version_name=payload.version_name,
|
||||||
|
note=payload.note,
|
||||||
|
files=payload.files,
|
||||||
|
)
|
||||||
|
except EvaluationError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||||
|
|
||||||
@app.post('/api/evaluations/analyze')
|
@app.post('/api/evaluations/analyze')
|
||||||
async def analyze_evaluation_case(
|
async def analyze_evaluation_case(
|
||||||
payload: EvaluationAnalyzeRequest,
|
payload: EvaluationAnalyzeRequest,
|
||||||
@@ -1544,6 +1622,7 @@ def create_app(state: AgentState) -> FastAPI:
|
|||||||
account_id=_safe_account_id(payload.account_id),
|
account_id=_safe_account_id(payload.account_id),
|
||||||
query=payload.query,
|
query=payload.query,
|
||||||
skill_name=payload.skill_name,
|
skill_name=payload.skill_name,
|
||||||
|
snapshot_id=payload.snapshot_id,
|
||||||
model=payload.model,
|
model=payload.model,
|
||||||
history=payload.history,
|
history=payload.history,
|
||||||
context=payload.context,
|
context=payload.context,
|
||||||
@@ -1642,6 +1721,7 @@ def create_app(state: AgentState) -> FastAPI:
|
|||||||
dataset_id=payload.dataset_id,
|
dataset_id=payload.dataset_id,
|
||||||
name=payload.name,
|
name=payload.name,
|
||||||
skill_name=payload.skill_name,
|
skill_name=payload.skill_name,
|
||||||
|
snapshot_id=payload.snapshot_id,
|
||||||
model=payload.model,
|
model=payload.model,
|
||||||
concurrency=payload.concurrency,
|
concurrency=payload.concurrency,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,14 +6,19 @@ import {
|
|||||||
ChevronRightIcon,
|
ChevronRightIcon,
|
||||||
CircleAlertIcon,
|
CircleAlertIcon,
|
||||||
DownloadIcon,
|
DownloadIcon,
|
||||||
|
FileCode2Icon,
|
||||||
|
FilePenLineIcon,
|
||||||
FileSpreadsheetIcon,
|
FileSpreadsheetIcon,
|
||||||
|
FileTextIcon,
|
||||||
FlaskConicalIcon,
|
FlaskConicalIcon,
|
||||||
|
GitBranchIcon,
|
||||||
LoaderCircleIcon,
|
LoaderCircleIcon,
|
||||||
PauseIcon,
|
PauseIcon,
|
||||||
PlayIcon,
|
PlayIcon,
|
||||||
PlusIcon,
|
PlusIcon,
|
||||||
RefreshCwIcon,
|
RefreshCwIcon,
|
||||||
RotateCcwIcon,
|
RotateCcwIcon,
|
||||||
|
SaveIcon,
|
||||||
SearchIcon,
|
SearchIcon,
|
||||||
SquareIcon,
|
SquareIcon,
|
||||||
UploadIcon,
|
UploadIcon,
|
||||||
@@ -50,6 +55,45 @@ type EvaluationMetadata = {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type SkillVersion = {
|
||||||
|
id: string;
|
||||||
|
skill_name: string;
|
||||||
|
version_name: string;
|
||||||
|
version_note: string;
|
||||||
|
source_type: "git" | "local";
|
||||||
|
parent_snapshot_id: string;
|
||||||
|
content_hash: string;
|
||||||
|
git_commit: string;
|
||||||
|
git_status: string;
|
||||||
|
file_count: number;
|
||||||
|
created_at: number;
|
||||||
|
is_current: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type SkillVersionFile = {
|
||||||
|
path: string;
|
||||||
|
size: number;
|
||||||
|
editable: boolean;
|
||||||
|
previewable: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type SkillVersionManifest = SkillVersion & {
|
||||||
|
files: SkillVersionFile[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type SkillVersionsResponse = {
|
||||||
|
skill_name: string;
|
||||||
|
current_snapshot_id: string;
|
||||||
|
versions: SkillVersion[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type SkillFileContent = {
|
||||||
|
path: string;
|
||||||
|
content: string;
|
||||||
|
size: number;
|
||||||
|
editable: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
type ModelOption = {
|
type ModelOption = {
|
||||||
id: string;
|
id: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
@@ -223,6 +267,8 @@ function EvaluationWorkspace() {
|
|||||||
label_map: {},
|
label_map: {},
|
||||||
});
|
});
|
||||||
const [skillName, setSkillName] = useState("");
|
const [skillName, setSkillName] = useState("");
|
||||||
|
const [skillVersions, setSkillVersions] = useState<SkillVersion[]>([]);
|
||||||
|
const [snapshotId, setSnapshotId] = useState("");
|
||||||
const [model, setModel] = useState("");
|
const [model, setModel] = useState("");
|
||||||
const [concurrency, setConcurrency] = useState(2);
|
const [concurrency, setConcurrency] = useState(2);
|
||||||
const [experimentName, setExperimentName] = useState("");
|
const [experimentName, setExperimentName] = useState("");
|
||||||
@@ -235,6 +281,18 @@ function EvaluationWorkspace() {
|
|||||||
const [queryFilter, setQueryFilter] = useState("");
|
const [queryFilter, setQueryFilter] = useState("");
|
||||||
const [busy, setBusy] = useState("");
|
const [busy, setBusy] = useState("");
|
||||||
const [error, setError] = useState("");
|
const [error, setError] = useState("");
|
||||||
|
const [skillEditorOpen, setSkillEditorOpen] = useState(false);
|
||||||
|
const [skillEditorBusy, setSkillEditorBusy] = useState("");
|
||||||
|
const [skillEditorError, setSkillEditorError] = useState("");
|
||||||
|
const [skillManifest, setSkillManifest] =
|
||||||
|
useState<SkillVersionManifest | null>(null);
|
||||||
|
const [selectedSkillFile, setSelectedSkillFile] = useState("");
|
||||||
|
const [skillFileContents, setSkillFileContents] = useState<
|
||||||
|
Record<string, SkillFileContent>
|
||||||
|
>({});
|
||||||
|
const [dirtySkillFiles, setDirtySkillFiles] = useState<string[]>([]);
|
||||||
|
const [skillVersionName, setSkillVersionName] = useState("");
|
||||||
|
const [skillVersionNote, setSkillVersionNote] = useState("");
|
||||||
const uploadRef = useRef<HTMLInputElement>(null);
|
const uploadRef = useRef<HTMLInputElement>(null);
|
||||||
const selectedDatasetIdRef = useRef("");
|
const selectedDatasetIdRef = useRef("");
|
||||||
const selectedExperimentIdRef = useRef("");
|
const selectedExperimentIdRef = useRef("");
|
||||||
@@ -256,6 +314,33 @@ function EvaluationWorkspace() {
|
|||||||
() => datasets.find((item) => item.id === selectedDatasetId) ?? null,
|
() => datasets.find((item) => item.id === selectedDatasetId) ?? null,
|
||||||
[datasets, selectedDatasetId],
|
[datasets, selectedDatasetId],
|
||||||
);
|
);
|
||||||
|
const selectedSkillVersion = useMemo(
|
||||||
|
() => skillVersions.find((item) => item.id === snapshotId) ?? null,
|
||||||
|
[skillVersions, snapshotId],
|
||||||
|
);
|
||||||
|
|
||||||
|
const loadSkillVersions = useCallback(
|
||||||
|
async (nextSkillName: string, preferredSnapshotId = "") => {
|
||||||
|
if (!nextSkillName) return null;
|
||||||
|
const response = await fetch(
|
||||||
|
`/api/claw/evaluations/skills/${encodeURIComponent(nextSkillName)}/versions`,
|
||||||
|
{ cache: "no-store" },
|
||||||
|
);
|
||||||
|
const payload = await readJson<SkillVersionsResponse>(response);
|
||||||
|
setSkillVersions(payload.versions);
|
||||||
|
const preferred = payload.versions.find(
|
||||||
|
(item) => item.id === preferredSnapshotId,
|
||||||
|
);
|
||||||
|
setSnapshotId(
|
||||||
|
preferred?.id ||
|
||||||
|
payload.current_snapshot_id ||
|
||||||
|
payload.versions[0]?.id ||
|
||||||
|
"",
|
||||||
|
);
|
||||||
|
return payload;
|
||||||
|
},
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
const refreshExperiment = useCallback(async (experimentId: string) => {
|
const refreshExperiment = useCallback(async (experimentId: string) => {
|
||||||
const response = await fetch(
|
const response = await fetch(
|
||||||
@@ -358,6 +443,15 @@ function EvaluationWorkspace() {
|
|||||||
void loadWorkspace();
|
void loadWorkspace();
|
||||||
}, [loadWorkspace]);
|
}, [loadWorkspace]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!skillName) return;
|
||||||
|
setSkillVersions([]);
|
||||||
|
setSnapshotId("");
|
||||||
|
void loadSkillVersions(skillName).catch((reason) =>
|
||||||
|
setError(messageFrom(reason)),
|
||||||
|
);
|
||||||
|
}, [loadSkillVersions, skillName]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!experiment || !ACTIVE_STATUSES.has(experiment.status)) return;
|
if (!experiment || !ACTIVE_STATUSES.has(experiment.status)) return;
|
||||||
const timer = window.setInterval(() => {
|
const timer = window.setInterval(() => {
|
||||||
@@ -368,6 +462,131 @@ function EvaluationWorkspace() {
|
|||||||
return () => window.clearInterval(timer);
|
return () => window.clearInterval(timer);
|
||||||
}, [experiment, refreshExperiment]);
|
}, [experiment, refreshExperiment]);
|
||||||
|
|
||||||
|
async function fetchSkillFile(
|
||||||
|
versionId: string,
|
||||||
|
filePath: string,
|
||||||
|
force = false,
|
||||||
|
) {
|
||||||
|
setSelectedSkillFile(filePath);
|
||||||
|
const cached = skillFileContents[filePath];
|
||||||
|
if (cached && !force) return cached;
|
||||||
|
setSkillEditorBusy("file");
|
||||||
|
setSkillEditorError("");
|
||||||
|
try {
|
||||||
|
const response = await fetch(
|
||||||
|
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions/${encodeURIComponent(versionId)}/file?path=${encodeURIComponent(filePath)}`,
|
||||||
|
{ cache: "no-store" },
|
||||||
|
);
|
||||||
|
const payload = await readJson<SkillFileContent>(response);
|
||||||
|
setSkillFileContents((current) => ({
|
||||||
|
...current,
|
||||||
|
[filePath]: payload,
|
||||||
|
}));
|
||||||
|
return payload;
|
||||||
|
} catch (reason) {
|
||||||
|
setSkillEditorError(messageFrom(reason));
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
setSkillEditorBusy("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openSkillEditor() {
|
||||||
|
if (!skillName) return;
|
||||||
|
setSkillEditorOpen(true);
|
||||||
|
setSkillEditorBusy("manifest");
|
||||||
|
setSkillEditorError("");
|
||||||
|
setSkillManifest(null);
|
||||||
|
setSelectedSkillFile("");
|
||||||
|
setSkillFileContents({});
|
||||||
|
setDirtySkillFiles([]);
|
||||||
|
setSkillVersionName("");
|
||||||
|
setSkillVersionNote("");
|
||||||
|
try {
|
||||||
|
let versionId = snapshotId;
|
||||||
|
if (!versionId) {
|
||||||
|
const versions = await loadSkillVersions(skillName);
|
||||||
|
versionId = versions?.current_snapshot_id ?? "";
|
||||||
|
}
|
||||||
|
if (!versionId) throw new Error("当前 Skill 没有可用版本");
|
||||||
|
const response = await fetch(
|
||||||
|
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions/${encodeURIComponent(versionId)}`,
|
||||||
|
{ cache: "no-store" },
|
||||||
|
);
|
||||||
|
const manifest = await readJson<SkillVersionManifest>(response);
|
||||||
|
setSkillManifest(manifest);
|
||||||
|
const firstFile =
|
||||||
|
manifest.files.find((item) => item.path === "SKILL.md") ??
|
||||||
|
manifest.files.find((item) => item.previewable);
|
||||||
|
if (firstFile) {
|
||||||
|
await fetchSkillFile(manifest.id, firstFile.path, true);
|
||||||
|
}
|
||||||
|
} catch (reason) {
|
||||||
|
setSkillEditorError(messageFrom(reason));
|
||||||
|
} finally {
|
||||||
|
setSkillEditorBusy("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateSkillFile(content: string) {
|
||||||
|
if (!selectedSkillFile) return;
|
||||||
|
setSkillFileContents((current) => {
|
||||||
|
const existing = current[selectedSkillFile];
|
||||||
|
if (!existing) return current;
|
||||||
|
return {
|
||||||
|
...current,
|
||||||
|
[selectedSkillFile]: { ...existing, content },
|
||||||
|
};
|
||||||
|
});
|
||||||
|
setDirtySkillFiles((current) =>
|
||||||
|
current.includes(selectedSkillFile)
|
||||||
|
? current
|
||||||
|
: [...current, selectedSkillFile],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveSkillVersion() {
|
||||||
|
if (!skillManifest) return;
|
||||||
|
if (!skillVersionName.trim()) {
|
||||||
|
setSkillEditorError("请填写新版本名称");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!dirtySkillFiles.length) {
|
||||||
|
setSkillEditorError("当前没有文件修改");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setSkillEditorBusy("save");
|
||||||
|
setSkillEditorError("");
|
||||||
|
try {
|
||||||
|
const files = Object.fromEntries(
|
||||||
|
dirtySkillFiles.map((path) => [
|
||||||
|
path,
|
||||||
|
skillFileContents[path]?.content ?? "",
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
const response = await fetch(
|
||||||
|
`/api/claw/evaluations/skills/${encodeURIComponent(skillName)}/versions`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
base_snapshot_id: skillManifest.id,
|
||||||
|
version_name: skillVersionName.trim(),
|
||||||
|
note: skillVersionNote.trim(),
|
||||||
|
files,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const saved = await readJson<SkillVersion>(response);
|
||||||
|
await loadSkillVersions(skillName, saved.id);
|
||||||
|
setSkillEditorOpen(false);
|
||||||
|
} catch (reason) {
|
||||||
|
setSkillEditorError(messageFrom(reason));
|
||||||
|
} finally {
|
||||||
|
setSkillEditorBusy("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function uploadDataset(file: File) {
|
async function uploadDataset(file: File) {
|
||||||
setBusy("upload");
|
setBusy("upload");
|
||||||
setError("");
|
setError("");
|
||||||
@@ -434,6 +653,7 @@ function EvaluationWorkspace() {
|
|||||||
name:
|
name:
|
||||||
experimentName.trim() || `${skillName} · ${selectedDataset.name}`,
|
experimentName.trim() || `${skillName} · ${selectedDataset.name}`,
|
||||||
skill_name: skillName,
|
skill_name: skillName,
|
||||||
|
snapshot_id: snapshotId || undefined,
|
||||||
model: model || undefined,
|
model: model || undefined,
|
||||||
concurrency,
|
concurrency,
|
||||||
}),
|
}),
|
||||||
@@ -475,6 +695,7 @@ function EvaluationWorkspace() {
|
|||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
query,
|
query,
|
||||||
skill_name: skillName,
|
skill_name: skillName,
|
||||||
|
snapshot_id: snapshotId || undefined,
|
||||||
model: model || undefined,
|
model: model || undefined,
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
@@ -661,6 +882,34 @@ function EvaluationWorkspace() {
|
|||||||
))}
|
))}
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>
|
||||||
|
<label className="flex h-8 items-center gap-2 rounded-md border bg-background px-2">
|
||||||
|
<span className="text-muted-foreground text-xs">版本</span>
|
||||||
|
<select
|
||||||
|
aria-label="全局 Skill 版本"
|
||||||
|
value={snapshotId}
|
||||||
|
onChange={(event) => setSnapshotId(event.target.value)}
|
||||||
|
disabled={!skillVersions.length}
|
||||||
|
className="h-7 min-w-36 max-w-56 bg-transparent font-medium text-xs outline-none disabled:opacity-60"
|
||||||
|
>
|
||||||
|
{skillVersions.map((version) => (
|
||||||
|
<option key={version.id} value={version.id}>
|
||||||
|
{version.version_name}
|
||||||
|
{version.source_type === "local" ? " · 本地" : ""}
|
||||||
|
{` · ${version.content_hash.slice(0, 8)}`}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onClick={() => void openSkillEditor()}
|
||||||
|
disabled={!snapshotId}
|
||||||
|
title="查看并编辑当前 Skill 版本"
|
||||||
|
>
|
||||||
|
<FilePenLineIcon />
|
||||||
|
编辑
|
||||||
|
</Button>
|
||||||
<label className="flex h-8 min-w-0 items-center gap-2 rounded-md border bg-background px-2">
|
<label className="flex h-8 min-w-0 items-center gap-2 rounded-md border bg-background px-2">
|
||||||
<span className="text-muted-foreground text-xs">模型</span>
|
<span className="text-muted-foreground text-xs">模型</span>
|
||||||
<select
|
<select
|
||||||
@@ -871,10 +1120,222 @@ function EvaluationWorkspace() {
|
|||||||
);
|
);
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
<SkillEditorSheet
|
||||||
|
open={skillEditorOpen}
|
||||||
|
onOpenChange={setSkillEditorOpen}
|
||||||
|
skillName={skillName}
|
||||||
|
version={selectedSkillVersion}
|
||||||
|
manifest={skillManifest}
|
||||||
|
selectedFile={selectedSkillFile}
|
||||||
|
fileContent={
|
||||||
|
selectedSkillFile
|
||||||
|
? (skillFileContents[selectedSkillFile] ?? null)
|
||||||
|
: null
|
||||||
|
}
|
||||||
|
dirtyFiles={dirtySkillFiles}
|
||||||
|
busy={skillEditorBusy}
|
||||||
|
error={skillEditorError}
|
||||||
|
versionName={skillVersionName}
|
||||||
|
setVersionName={setSkillVersionName}
|
||||||
|
versionNote={skillVersionNote}
|
||||||
|
setVersionNote={setSkillVersionNote}
|
||||||
|
onSelectFile={(filePath) => {
|
||||||
|
if (skillManifest) {
|
||||||
|
void fetchSkillFile(skillManifest.id, filePath);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
onChangeContent={updateSkillFile}
|
||||||
|
onSave={() => void saveSkillVersion()}
|
||||||
|
/>
|
||||||
</main>
|
</main>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function SkillEditorSheet(props: {
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
skillName: string;
|
||||||
|
version: SkillVersion | null;
|
||||||
|
manifest: SkillVersionManifest | null;
|
||||||
|
selectedFile: string;
|
||||||
|
fileContent: SkillFileContent | null;
|
||||||
|
dirtyFiles: string[];
|
||||||
|
busy: string;
|
||||||
|
error: string;
|
||||||
|
versionName: string;
|
||||||
|
setVersionName: (value: string) => void;
|
||||||
|
versionNote: string;
|
||||||
|
setVersionNote: (value: string) => void;
|
||||||
|
onSelectFile: (path: string) => void;
|
||||||
|
onChangeContent: (content: string) => void;
|
||||||
|
onSave: () => void;
|
||||||
|
}) {
|
||||||
|
const {
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
skillName,
|
||||||
|
version,
|
||||||
|
manifest,
|
||||||
|
selectedFile,
|
||||||
|
fileContent,
|
||||||
|
dirtyFiles,
|
||||||
|
busy,
|
||||||
|
error,
|
||||||
|
versionName,
|
||||||
|
setVersionName,
|
||||||
|
versionNote,
|
||||||
|
setVersionNote,
|
||||||
|
onSelectFile,
|
||||||
|
onChangeContent,
|
||||||
|
onSave,
|
||||||
|
} = props;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Sheet open={open} onOpenChange={onOpenChange}>
|
||||||
|
<SheetContent className="w-[min(96vw,1100px)] gap-0 sm:max-w-[1100px]">
|
||||||
|
<SheetHeader className="border-b px-5 py-4">
|
||||||
|
<SheetTitle className="flex items-center gap-2">
|
||||||
|
<FilePenLineIcon className="size-4" />
|
||||||
|
Skill 版本编辑
|
||||||
|
</SheetTitle>
|
||||||
|
<SheetDescription className="flex flex-wrap items-center gap-x-3 gap-y-1">
|
||||||
|
<span>{skillName}</span>
|
||||||
|
<span className="inline-flex items-center gap-1">
|
||||||
|
<GitBranchIcon className="size-3.5" />
|
||||||
|
{version?.version_name || "正在读取版本"}
|
||||||
|
</span>
|
||||||
|
{version?.content_hash ? (
|
||||||
|
<code>{version.content_hash.slice(0, 12)}</code>
|
||||||
|
) : null}
|
||||||
|
</SheetDescription>
|
||||||
|
</SheetHeader>
|
||||||
|
|
||||||
|
{error ? (
|
||||||
|
<div className="mx-5 mt-3 flex items-start gap-2 border border-destructive/30 bg-destructive/5 px-3 py-2 text-destructive text-sm">
|
||||||
|
<CircleAlertIcon className="mt-0.5 size-4 shrink-0" />
|
||||||
|
<span>{error}</span>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
|
<div className="grid min-h-0 flex-1 grid-cols-[230px_minmax(0,1fr)]">
|
||||||
|
<aside className="min-h-0 overflow-y-auto border-r bg-muted/15 p-3">
|
||||||
|
<div className="mb-2 flex items-center justify-between px-2 text-xs">
|
||||||
|
<span className="font-medium">Skill 文件</span>
|
||||||
|
<span className="text-muted-foreground">
|
||||||
|
{manifest?.files.length ?? 0}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="space-y-0.5">
|
||||||
|
{manifest?.files.map((file) => {
|
||||||
|
const dirty = dirtyFiles.includes(file.path);
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
key={file.path}
|
||||||
|
onClick={() => onSelectFile(file.path)}
|
||||||
|
disabled={!file.previewable}
|
||||||
|
className={`flex w-full items-center gap-2 rounded-md px-2 py-1.5 text-left text-xs transition-colors ${
|
||||||
|
selectedFile === file.path
|
||||||
|
? "bg-accent text-accent-foreground"
|
||||||
|
: "hover:bg-accent/60"
|
||||||
|
} disabled:cursor-not-allowed disabled:opacity-50`}
|
||||||
|
>
|
||||||
|
{file.path.endsWith(".md") ? (
|
||||||
|
<FileTextIcon className="size-3.5 shrink-0" />
|
||||||
|
) : (
|
||||||
|
<FileCode2Icon className="size-3.5 shrink-0" />
|
||||||
|
)}
|
||||||
|
<span className="min-w-0 flex-1 truncate">{file.path}</span>
|
||||||
|
{dirty ? (
|
||||||
|
<span
|
||||||
|
className="size-1.5 shrink-0 rounded-full bg-sky-500"
|
||||||
|
title="已修改"
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</aside>
|
||||||
|
|
||||||
|
<section className="flex min-h-0 min-w-0 flex-col">
|
||||||
|
<div className="flex h-10 shrink-0 items-center gap-2 border-b px-4">
|
||||||
|
<span className="min-w-0 flex-1 truncate font-medium text-sm">
|
||||||
|
{selectedFile || "选择一个文件"}
|
||||||
|
</span>
|
||||||
|
{fileContent ? (
|
||||||
|
<span className="text-muted-foreground text-xs">
|
||||||
|
{fileContent.editable ? "可编辑" : "只读"}
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
{busy === "file" ? (
|
||||||
|
<LoaderCircleIcon className="size-4 animate-spin text-muted-foreground" />
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
<div className="min-h-0 flex-1 p-4">
|
||||||
|
{fileContent ? (
|
||||||
|
<textarea
|
||||||
|
value={fileContent.content}
|
||||||
|
onChange={(event) => onChangeContent(event.target.value)}
|
||||||
|
readOnly={!fileContent.editable}
|
||||||
|
spellCheck={false}
|
||||||
|
className="h-full min-h-72 w-full resize-none border bg-muted/10 p-4 font-mono text-[13px] leading-6 outline-none focus:ring-2 focus:ring-ring/40 read-only:cursor-default read-only:opacity-80"
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div className="grid h-full min-h-72 place-items-center text-muted-foreground text-sm">
|
||||||
|
{busy === "manifest" ? (
|
||||||
|
<LoaderCircleIcon className="size-5 animate-spin" />
|
||||||
|
) : (
|
||||||
|
"选择左侧文件查看内容"
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="shrink-0 border-t bg-muted/10 px-4 py-3">
|
||||||
|
<div className="grid gap-3 md:grid-cols-[minmax(0,220px)_minmax(0,1fr)_auto] md:items-end">
|
||||||
|
<Field label="新版本名称">
|
||||||
|
<Input
|
||||||
|
value={versionName}
|
||||||
|
onChange={(event) => setVersionName(event.target.value)}
|
||||||
|
placeholder="例如:导航边界调整 v1"
|
||||||
|
maxLength={80}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
<Field label="版本备注">
|
||||||
|
<Input
|
||||||
|
value={versionNote}
|
||||||
|
onChange={(event) => setVersionNote(event.target.value)}
|
||||||
|
placeholder="记录本次调整内容"
|
||||||
|
maxLength={1000}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
<Button
|
||||||
|
onClick={onSave}
|
||||||
|
disabled={busy === "save" || !dirtyFiles.length}
|
||||||
|
>
|
||||||
|
{busy === "save" ? (
|
||||||
|
<LoaderCircleIcon className="animate-spin" />
|
||||||
|
) : (
|
||||||
|
<SaveIcon />
|
||||||
|
)}
|
||||||
|
保存为新版本
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
<p className="mt-2 text-muted-foreground text-xs">
|
||||||
|
已保存版本保持只读;Markdown 和 JSON 可编辑,脚本暂只读。
|
||||||
|
{dirtyFiles.length
|
||||||
|
? ` 当前修改 ${dirtyFiles.length} 个文件。`
|
||||||
|
: ""}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
</SheetContent>
|
||||||
|
</Sheet>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function SingleQueryWorkspace(props: {
|
function SingleQueryWorkspace(props: {
|
||||||
query: string;
|
query: string;
|
||||||
setQuery: (value: string) => void;
|
setQuery: (value: string) => void;
|
||||||
|
|||||||
+379
-3
@@ -14,7 +14,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
|
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
|
||||||
from dataclasses import dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
from pathlib import Path
|
from pathlib import Path, PurePosixPath
|
||||||
from typing import Any, Callable, Iterable
|
from typing import Any, Callable, Iterable
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -35,6 +35,10 @@ SUPPORTED_DATASET_SUFFIXES = {'.csv', '.tsv', '.json', '.jsonl', '.xlsx'}
|
|||||||
DEFAULT_CONCURRENCY = 2
|
DEFAULT_CONCURRENCY = 2
|
||||||
MAX_CONCURRENCY = 8
|
MAX_CONCURRENCY = 8
|
||||||
MAX_DATASET_ROWS = 20_000
|
MAX_DATASET_ROWS = 20_000
|
||||||
|
VISIBLE_SKILL_FILE_SUFFIXES = {'.json', '.md', '.py', '.txt', '.yaml', '.yml'}
|
||||||
|
EDITABLE_SKILL_FILE_SUFFIXES = {'.json', '.md'}
|
||||||
|
MAX_SKILL_FILE_BYTES = 512 * 1024
|
||||||
|
MAX_SKILL_VERSION_EDIT_BYTES = 2 * 1024 * 1024
|
||||||
|
|
||||||
EVALUATION_OUTPUT_SCHEMA = OutputSchemaConfig(
|
EVALUATION_OUTPUT_SCHEMA = OutputSchemaConfig(
|
||||||
name='skill_evaluation_result',
|
name='skill_evaluation_result',
|
||||||
@@ -491,6 +495,7 @@ class EvaluationRuntime:
|
|||||||
account_id: str,
|
account_id: str,
|
||||||
query: str,
|
query: str,
|
||||||
skill_name: str = 'label-master',
|
skill_name: str = 'label-master',
|
||||||
|
snapshot_id: str | None = None,
|
||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
history: list[dict[str, Any]] | None = None,
|
history: list[dict[str, Any]] | None = None,
|
||||||
context: dict[str, Any] | None = None,
|
context: dict[str, Any] | None = None,
|
||||||
@@ -501,7 +506,11 @@ class EvaluationRuntime:
|
|||||||
normalized_query = query.strip()
|
normalized_query = query.strip()
|
||||||
if not normalized_query:
|
if not normalized_query:
|
||||||
raise EvaluationError('Query 不能为空')
|
raise EvaluationError('Query 不能为空')
|
||||||
snapshot = self.create_skill_snapshot(account_id, skill_name)
|
snapshot = self.resolve_skill_snapshot(
|
||||||
|
account_id,
|
||||||
|
skill_name,
|
||||||
|
snapshot_id=snapshot_id,
|
||||||
|
)
|
||||||
model_name = model or self.model_config_for(account_id).model
|
model_name = model or self.model_config_for(account_id).model
|
||||||
analysis_id = f'single_{uuid4().hex[:16]}'
|
analysis_id = f'single_{uuid4().hex[:16]}'
|
||||||
case_id = f'case_{uuid4().hex[:16]}'
|
case_id = f'case_{uuid4().hex[:16]}'
|
||||||
@@ -669,6 +678,7 @@ class EvaluationRuntime:
|
|||||||
dataset_id: str,
|
dataset_id: str,
|
||||||
name: str,
|
name: str,
|
||||||
skill_name: str = 'label-master',
|
skill_name: str = 'label-master',
|
||||||
|
snapshot_id: str | None = None,
|
||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
concurrency: int = DEFAULT_CONCURRENCY,
|
concurrency: int = DEFAULT_CONCURRENCY,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
@@ -678,7 +688,11 @@ class EvaluationRuntime:
|
|||||||
canonical_rows = [row for row in canonical_rows if row['query']]
|
canonical_rows = [row for row in canonical_rows if row['query']]
|
||||||
if not canonical_rows:
|
if not canonical_rows:
|
||||||
raise EvaluationError('当前字段映射没有生成有效 case')
|
raise EvaluationError('当前字段映射没有生成有效 case')
|
||||||
snapshot = self.create_skill_snapshot(account_id, skill_name)
|
snapshot = self.resolve_skill_snapshot(
|
||||||
|
account_id,
|
||||||
|
skill_name,
|
||||||
|
snapshot_id=snapshot_id,
|
||||||
|
)
|
||||||
model_config = self.model_config_for(account_id)
|
model_config = self.model_config_for(account_id)
|
||||||
experiment_id = f'eval_{uuid4().hex[:16]}'
|
experiment_id = f'eval_{uuid4().hex[:16]}'
|
||||||
now = time.time()
|
now = time.time()
|
||||||
@@ -1004,6 +1018,274 @@ class EvaluationRuntime:
|
|||||||
# Snapshots and execution
|
# Snapshots and execution
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def resolve_skill_snapshot(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
skill_name: str,
|
||||||
|
*,
|
||||||
|
snapshot_id: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not snapshot_id:
|
||||||
|
return self.create_skill_snapshot(account_id, skill_name)
|
||||||
|
row = self.store.fetchone(
|
||||||
|
"""
|
||||||
|
SELECT *
|
||||||
|
FROM skill_snapshots
|
||||||
|
WHERE id = ? AND account_id = ? AND skill_name = ?
|
||||||
|
""",
|
||||||
|
(snapshot_id, account_id, skill_name),
|
||||||
|
)
|
||||||
|
if row is None or not Path(row['path']).is_dir():
|
||||||
|
raise EvaluationError('Skill 版本不存在或已失效')
|
||||||
|
return self._snapshot_payload(row)
|
||||||
|
|
||||||
|
def list_skill_versions(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
skill_name: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
current = self.create_skill_snapshot(account_id, skill_name)
|
||||||
|
rows = self.store.fetchall(
|
||||||
|
"""
|
||||||
|
SELECT *
|
||||||
|
FROM skill_snapshots
|
||||||
|
WHERE account_id = ? AND skill_name = ?
|
||||||
|
ORDER BY created_at DESC
|
||||||
|
""",
|
||||||
|
(account_id, skill_name),
|
||||||
|
)
|
||||||
|
rows.sort(
|
||||||
|
key=lambda row: (
|
||||||
|
row['id'] != current['id'],
|
||||||
|
-float(row['created_at']),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
'skill_name': skill_name,
|
||||||
|
'current_snapshot_id': current['id'],
|
||||||
|
'versions': [
|
||||||
|
self._public_skill_version(
|
||||||
|
row,
|
||||||
|
is_current=row['id'] == current['id'],
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
if Path(row['path']).is_dir()
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_skill_version_manifest(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
skill_name: str,
|
||||||
|
snapshot_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
snapshot = self.resolve_skill_snapshot(
|
||||||
|
account_id,
|
||||||
|
skill_name,
|
||||||
|
snapshot_id=snapshot_id,
|
||||||
|
)
|
||||||
|
snapshot_skill_dir = self._snapshot_skill_dir(snapshot, skill_name)
|
||||||
|
files: list[dict[str, Any]] = []
|
||||||
|
for path in sorted(
|
||||||
|
item for item in snapshot_skill_dir.rglob('*') if item.is_file()
|
||||||
|
):
|
||||||
|
relative = path.relative_to(snapshot_skill_dir).as_posix()
|
||||||
|
if any(part.startswith('.') for part in PurePosixPath(relative).parts):
|
||||||
|
continue
|
||||||
|
suffix = path.suffix.lower()
|
||||||
|
if suffix not in VISIBLE_SKILL_FILE_SUFFIXES:
|
||||||
|
continue
|
||||||
|
size = path.stat().st_size
|
||||||
|
files.append(
|
||||||
|
{
|
||||||
|
'path': relative,
|
||||||
|
'size': size,
|
||||||
|
'editable': (
|
||||||
|
suffix in EDITABLE_SKILL_FILE_SUFFIXES
|
||||||
|
and size <= MAX_SKILL_FILE_BYTES
|
||||||
|
),
|
||||||
|
'previewable': size <= MAX_SKILL_FILE_BYTES,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
row = self._skill_snapshot_row(account_id, skill_name, snapshot_id)
|
||||||
|
return {
|
||||||
|
**self._public_skill_version(row),
|
||||||
|
'files': files,
|
||||||
|
}
|
||||||
|
|
||||||
|
def read_skill_version_file(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
skill_name: str,
|
||||||
|
snapshot_id: str,
|
||||||
|
file_path: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
snapshot = self.resolve_skill_snapshot(
|
||||||
|
account_id,
|
||||||
|
skill_name,
|
||||||
|
snapshot_id=snapshot_id,
|
||||||
|
)
|
||||||
|
snapshot_skill_dir = self._snapshot_skill_dir(snapshot, skill_name)
|
||||||
|
relative, target = self._resolve_skill_file(
|
||||||
|
snapshot_skill_dir,
|
||||||
|
file_path,
|
||||||
|
)
|
||||||
|
if target.suffix.lower() not in VISIBLE_SKILL_FILE_SUFFIXES:
|
||||||
|
raise EvaluationError('该文件类型不支持预览')
|
||||||
|
size = target.stat().st_size
|
||||||
|
if size > MAX_SKILL_FILE_BYTES:
|
||||||
|
raise EvaluationError('文件过大,暂不支持在线预览')
|
||||||
|
try:
|
||||||
|
content = target.read_text(encoding='utf-8')
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise EvaluationError('文件不是 UTF-8 文本') from exc
|
||||||
|
return {
|
||||||
|
'path': relative,
|
||||||
|
'content': content,
|
||||||
|
'size': size,
|
||||||
|
'editable': target.suffix.lower() in EDITABLE_SKILL_FILE_SUFFIXES,
|
||||||
|
}
|
||||||
|
|
||||||
|
def save_skill_version(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
skill_name: str,
|
||||||
|
*,
|
||||||
|
base_snapshot_id: str,
|
||||||
|
version_name: str,
|
||||||
|
note: str = '',
|
||||||
|
files: dict[str, str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
normalized_name = version_name.strip()
|
||||||
|
if not normalized_name:
|
||||||
|
raise EvaluationError('请填写版本名称')
|
||||||
|
if len(normalized_name) > 80:
|
||||||
|
raise EvaluationError('版本名称不能超过 80 个字符')
|
||||||
|
normalized_note = note.strip()
|
||||||
|
if len(normalized_note) > 1000:
|
||||||
|
raise EvaluationError('版本备注不能超过 1000 个字符')
|
||||||
|
if not files:
|
||||||
|
raise EvaluationError('没有需要保存的文件修改')
|
||||||
|
|
||||||
|
base = self.resolve_skill_snapshot(
|
||||||
|
account_id,
|
||||||
|
skill_name,
|
||||||
|
snapshot_id=base_snapshot_id,
|
||||||
|
)
|
||||||
|
base_skill_dir = self._snapshot_skill_dir(base, skill_name)
|
||||||
|
total_bytes = sum(
|
||||||
|
len(str(content).encode('utf-8')) for content in files.values()
|
||||||
|
)
|
||||||
|
if total_bytes > MAX_SKILL_VERSION_EDIT_BYTES:
|
||||||
|
raise EvaluationError('本次修改内容过大,请拆分后保存')
|
||||||
|
|
||||||
|
draft_root: Path | None = (
|
||||||
|
self.root / '.skill_version_drafts' / uuid4().hex
|
||||||
|
)
|
||||||
|
draft_skill_dir = draft_root / 'skills' / skill_name
|
||||||
|
draft_skill_dir.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copytree(base_skill_dir, draft_skill_dir)
|
||||||
|
try:
|
||||||
|
for file_path, raw_content in files.items():
|
||||||
|
relative, target = self._resolve_skill_file(
|
||||||
|
draft_skill_dir,
|
||||||
|
file_path,
|
||||||
|
)
|
||||||
|
if target.suffix.lower() not in EDITABLE_SKILL_FILE_SUFFIXES:
|
||||||
|
raise EvaluationError(f'该文件只读:{relative}')
|
||||||
|
content = str(raw_content)
|
||||||
|
if len(content.encode('utf-8')) > MAX_SKILL_FILE_BYTES:
|
||||||
|
raise EvaluationError(f'文件过大:{relative}')
|
||||||
|
if target.suffix.lower() == '.json':
|
||||||
|
try:
|
||||||
|
json.loads(content)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise EvaluationError(
|
||||||
|
f'JSON 格式错误:{relative},第 {exc.lineno} 行'
|
||||||
|
) from exc
|
||||||
|
target.write_text(content, encoding='utf-8')
|
||||||
|
|
||||||
|
loaded = self._load_snapshot_skill(draft_root, skill_name)
|
||||||
|
if loaded.name != skill_name:
|
||||||
|
raise EvaluationError('SKILL.md 中的 name 不允许修改')
|
||||||
|
content_hash = hash_directory(draft_skill_dir)
|
||||||
|
existing = self.store.fetchone(
|
||||||
|
"""
|
||||||
|
SELECT *
|
||||||
|
FROM skill_snapshots
|
||||||
|
WHERE account_id = ? AND skill_name = ? AND content_hash = ?
|
||||||
|
""",
|
||||||
|
(account_id, skill_name, content_hash),
|
||||||
|
)
|
||||||
|
if existing is not None and Path(existing['path']).is_dir():
|
||||||
|
return self._public_skill_version(existing)
|
||||||
|
|
||||||
|
snapshot_id = f'snapshot_{uuid4().hex[:16]}'
|
||||||
|
snapshot_root = (
|
||||||
|
self.root
|
||||||
|
/ 'skill_snapshots'
|
||||||
|
/ sanitize_filename(account_id)
|
||||||
|
/ sanitize_filename(skill_name)
|
||||||
|
/ content_hash[:16]
|
||||||
|
)
|
||||||
|
with self._lock:
|
||||||
|
if snapshot_root.exists():
|
||||||
|
shutil.rmtree(snapshot_root)
|
||||||
|
snapshot_root.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.move(str(draft_root), str(snapshot_root))
|
||||||
|
draft_root = None
|
||||||
|
snapshot_skill_dir = snapshot_root / 'skills' / skill_name
|
||||||
|
metadata = self._skill_snapshot_metadata(
|
||||||
|
snapshot_skill_dir,
|
||||||
|
skill_name,
|
||||||
|
)
|
||||||
|
metadata.update(
|
||||||
|
{
|
||||||
|
'source_type': 'local',
|
||||||
|
'version_name': normalized_name,
|
||||||
|
'version_note': normalized_note,
|
||||||
|
'parent_snapshot_id': base_snapshot_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
self.store.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO skill_snapshots (
|
||||||
|
id, account_id, skill_name, content_hash, git_commit,
|
||||||
|
git_status, path, metadata_json, created_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
snapshot_id,
|
||||||
|
account_id,
|
||||||
|
skill_name,
|
||||||
|
content_hash,
|
||||||
|
base.get('git_commit', ''),
|
||||||
|
'local',
|
||||||
|
str(snapshot_root),
|
||||||
|
json.dumps(metadata, ensure_ascii=False),
|
||||||
|
time.time(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except sqlite3.IntegrityError:
|
||||||
|
existing = self.store.fetchone(
|
||||||
|
"""
|
||||||
|
SELECT *
|
||||||
|
FROM skill_snapshots
|
||||||
|
WHERE account_id = ? AND skill_name = ?
|
||||||
|
AND content_hash = ?
|
||||||
|
""",
|
||||||
|
(account_id, skill_name, content_hash),
|
||||||
|
)
|
||||||
|
if existing is None:
|
||||||
|
raise
|
||||||
|
return self._public_skill_version(existing)
|
||||||
|
row = self._skill_snapshot_row(account_id, skill_name, snapshot_id)
|
||||||
|
return self._public_skill_version(row)
|
||||||
|
finally:
|
||||||
|
if draft_root is not None and draft_root.exists():
|
||||||
|
shutil.rmtree(draft_root, ignore_errors=True)
|
||||||
|
|
||||||
def create_skill_snapshot(
|
def create_skill_snapshot(
|
||||||
self,
|
self,
|
||||||
account_id: str,
|
account_id: str,
|
||||||
@@ -1043,6 +1325,14 @@ class EvaluationRuntime:
|
|||||||
ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '.DS_Store'),
|
ignore=shutil.ignore_patterns('__pycache__', '*.pyc', '.DS_Store'),
|
||||||
)
|
)
|
||||||
metadata = self._skill_snapshot_metadata(snapshot_skill_dir, skill_name)
|
metadata = self._skill_snapshot_metadata(snapshot_skill_dir, skill_name)
|
||||||
|
metadata.update(
|
||||||
|
{
|
||||||
|
'source_type': 'git',
|
||||||
|
'version_name': '',
|
||||||
|
'version_note': '',
|
||||||
|
'parent_snapshot_id': '',
|
||||||
|
}
|
||||||
|
)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
try:
|
try:
|
||||||
self.store.execute(
|
self.store.execute(
|
||||||
@@ -1619,6 +1909,92 @@ class EvaluationRuntime:
|
|||||||
'created_at': row['created_at'],
|
'created_at': row['created_at'],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _public_skill_version(
|
||||||
|
self,
|
||||||
|
row: dict[str, Any],
|
||||||
|
*,
|
||||||
|
is_current: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
metadata = json.loads(row.get('metadata_json') or '{}')
|
||||||
|
source_type = str(metadata.get('source_type') or 'git')
|
||||||
|
version_name = str(metadata.get('version_name') or '').strip()
|
||||||
|
if not version_name:
|
||||||
|
if is_current:
|
||||||
|
version_name = '当前 Git'
|
||||||
|
elif source_type == 'local':
|
||||||
|
version_name = f"本地版本 {str(row['content_hash'])[:8]}"
|
||||||
|
else:
|
||||||
|
version_name = f"Git 快照 {str(row['content_hash'])[:8]}"
|
||||||
|
return {
|
||||||
|
'id': row['id'],
|
||||||
|
'skill_name': row['skill_name'],
|
||||||
|
'version_name': version_name,
|
||||||
|
'version_note': str(metadata.get('version_note') or ''),
|
||||||
|
'source_type': source_type,
|
||||||
|
'parent_snapshot_id': str(
|
||||||
|
metadata.get('parent_snapshot_id') or ''
|
||||||
|
),
|
||||||
|
'content_hash': row['content_hash'],
|
||||||
|
'git_commit': row.get('git_commit', ''),
|
||||||
|
'git_status': row.get('git_status', 'unknown'),
|
||||||
|
'file_count': int(metadata.get('file_count') or 0),
|
||||||
|
'created_at': row['created_at'],
|
||||||
|
'is_current': is_current,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _skill_snapshot_row(
|
||||||
|
self,
|
||||||
|
account_id: str,
|
||||||
|
skill_name: str,
|
||||||
|
snapshot_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
row = self.store.fetchone(
|
||||||
|
"""
|
||||||
|
SELECT *
|
||||||
|
FROM skill_snapshots
|
||||||
|
WHERE id = ? AND account_id = ? AND skill_name = ?
|
||||||
|
""",
|
||||||
|
(snapshot_id, account_id, skill_name),
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
raise EvaluationError('Skill 版本不存在')
|
||||||
|
return row
|
||||||
|
|
||||||
|
def _snapshot_skill_dir(
|
||||||
|
self,
|
||||||
|
snapshot: dict[str, Any],
|
||||||
|
skill_name: str,
|
||||||
|
) -> Path:
|
||||||
|
skill_dir = Path(snapshot['path']) / 'skills' / skill_name
|
||||||
|
if not skill_dir.is_dir() or not (skill_dir / 'SKILL.md').is_file():
|
||||||
|
raise EvaluationError('Skill 版本文件已失效')
|
||||||
|
return skill_dir
|
||||||
|
|
||||||
|
def _resolve_skill_file(
|
||||||
|
self,
|
||||||
|
skill_dir: Path,
|
||||||
|
file_path: str,
|
||||||
|
) -> tuple[str, Path]:
|
||||||
|
candidate = PurePosixPath(str(file_path).strip())
|
||||||
|
if (
|
||||||
|
not candidate.parts
|
||||||
|
or candidate.is_absolute()
|
||||||
|
or '..' in candidate.parts
|
||||||
|
or any(part.startswith('.') for part in candidate.parts)
|
||||||
|
):
|
||||||
|
raise EvaluationError('文件路径无效')
|
||||||
|
relative = candidate.as_posix()
|
||||||
|
target = skill_dir.joinpath(*candidate.parts)
|
||||||
|
if target.is_symlink() or not target.is_file():
|
||||||
|
raise EvaluationError(f'Skill 文件不存在:{relative}')
|
||||||
|
resolved_root = skill_dir.resolve()
|
||||||
|
resolved_target = target.resolve()
|
||||||
|
try:
|
||||||
|
resolved_target.relative_to(resolved_root)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise EvaluationError('文件路径越界') from exc
|
||||||
|
return relative, resolved_target
|
||||||
|
|
||||||
def _experiment_summary(self, row: dict[str, Any]) -> dict[str, Any]:
|
def _experiment_summary(self, row: dict[str, Any]) -> dict[str, Any]:
|
||||||
total = int(row.get('total_cases') or 0)
|
total = int(row.get('total_cases') or 0)
|
||||||
completed = int(row.get('completed_cases') or 0)
|
completed = int(row.get('completed_cases') or 0)
|
||||||
|
|||||||
@@ -178,6 +178,62 @@ class EvaluationApiTests(unittest.TestCase):
|
|||||||
self.assertIn('text/csv', export.headers['content-type'])
|
self.assertIn('text/csv', export.headers['content-type'])
|
||||||
self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
|
self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
|
||||||
|
|
||||||
|
def test_skill_version_endpoints(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
state = _build_state(root)
|
||||||
|
with TestClient(create_app(state)) as client:
|
||||||
|
versions_response = client.get(
|
||||||
|
'/api/evaluations/skills/label-master/versions',
|
||||||
|
params={'account_id': 'alice'},
|
||||||
|
)
|
||||||
|
self.assertEqual(versions_response.status_code, 200)
|
||||||
|
versions = versions_response.json()
|
||||||
|
current_id = versions['current_snapshot_id']
|
||||||
|
|
||||||
|
manifest = client.get(
|
||||||
|
(
|
||||||
|
'/api/evaluations/skills/label-master/versions/'
|
||||||
|
f'{current_id}'
|
||||||
|
),
|
||||||
|
params={'account_id': 'alice'},
|
||||||
|
)
|
||||||
|
self.assertEqual(manifest.status_code, 200)
|
||||||
|
self.assertTrue(
|
||||||
|
any(
|
||||||
|
item['path'] == 'SKILL.md'
|
||||||
|
for item in manifest.json()['files']
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
file_response = client.get(
|
||||||
|
(
|
||||||
|
'/api/evaluations/skills/label-master/versions/'
|
||||||
|
f'{current_id}/file'
|
||||||
|
),
|
||||||
|
params={'account_id': 'alice', 'path': 'SKILL.md'},
|
||||||
|
)
|
||||||
|
self.assertEqual(file_response.status_code, 200)
|
||||||
|
content = file_response.json()['content'].replace(
|
||||||
|
'Use the label catalog.',
|
||||||
|
'Use the label catalog carefully.',
|
||||||
|
)
|
||||||
|
saved_response = client.post(
|
||||||
|
'/api/evaluations/skills/label-master/versions',
|
||||||
|
json={
|
||||||
|
'account_id': 'alice',
|
||||||
|
'base_snapshot_id': current_id,
|
||||||
|
'version_name': '测试版本',
|
||||||
|
'note': 'API 测试',
|
||||||
|
'files': {'SKILL.md': content},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(saved_response.status_code, 200)
|
||||||
|
self.assertEqual(
|
||||||
|
saved_response.json()['version_name'],
|
||||||
|
'测试版本',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -169,6 +169,100 @@ class EvaluationRuntimeTests(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
runtime.shutdown()
|
runtime.shutdown()
|
||||||
|
|
||||||
|
def test_skill_versions_are_immutable_and_account_scoped(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
_write_test_skill(root)
|
||||||
|
runtime = EvaluationRuntime(
|
||||||
|
root=root / 'evaluations',
|
||||||
|
cwd_for_account=lambda _account: root,
|
||||||
|
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||||
|
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||||
|
max_workers=1,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
versions = runtime.list_skill_versions('alice', 'label-master')
|
||||||
|
self.assertEqual(len(versions['versions']), 1)
|
||||||
|
current_id = versions['current_snapshot_id']
|
||||||
|
self.assertTrue(versions['versions'][0]['is_current'])
|
||||||
|
|
||||||
|
manifest = runtime.get_skill_version_manifest(
|
||||||
|
'alice',
|
||||||
|
'label-master',
|
||||||
|
current_id,
|
||||||
|
)
|
||||||
|
skill_file = next(
|
||||||
|
item
|
||||||
|
for item in manifest['files']
|
||||||
|
if item['path'] == 'SKILL.md'
|
||||||
|
)
|
||||||
|
self.assertTrue(skill_file['editable'])
|
||||||
|
current_file = runtime.read_skill_version_file(
|
||||||
|
'alice',
|
||||||
|
'label-master',
|
||||||
|
current_id,
|
||||||
|
'SKILL.md',
|
||||||
|
)
|
||||||
|
updated_content = current_file['content'].replace(
|
||||||
|
'Read the manifest',
|
||||||
|
'Read the manifest carefully',
|
||||||
|
)
|
||||||
|
saved = runtime.save_skill_version(
|
||||||
|
'alice',
|
||||||
|
'label-master',
|
||||||
|
base_snapshot_id=current_id,
|
||||||
|
version_name='边界调整 v1',
|
||||||
|
note='测试版本',
|
||||||
|
files={'SKILL.md': updated_content},
|
||||||
|
)
|
||||||
|
self.assertEqual(saved['version_name'], '边界调整 v1')
|
||||||
|
self.assertEqual(saved['source_type'], 'local')
|
||||||
|
self.assertNotEqual(saved['id'], current_id)
|
||||||
|
|
||||||
|
original = runtime.read_skill_version_file(
|
||||||
|
'alice',
|
||||||
|
'label-master',
|
||||||
|
current_id,
|
||||||
|
'SKILL.md',
|
||||||
|
)
|
||||||
|
forked = runtime.read_skill_version_file(
|
||||||
|
'alice',
|
||||||
|
'label-master',
|
||||||
|
saved['id'],
|
||||||
|
'SKILL.md',
|
||||||
|
)
|
||||||
|
self.assertNotIn('carefully', original['content'])
|
||||||
|
self.assertIn('carefully', forked['content'])
|
||||||
|
dataset = runtime.create_dataset(
|
||||||
|
account_id='alice',
|
||||||
|
name='versioned',
|
||||||
|
filename='versioned.csv',
|
||||||
|
rows=[{'query': '导航去公司'}],
|
||||||
|
)
|
||||||
|
experiment = runtime.create_experiment(
|
||||||
|
account_id='alice',
|
||||||
|
dataset_id=dataset['id'],
|
||||||
|
name='local version',
|
||||||
|
snapshot_id=saved['id'],
|
||||||
|
)
|
||||||
|
self.assertEqual(experiment['snapshot_id'], saved['id'])
|
||||||
|
self.assertEqual(
|
||||||
|
experiment['skill_version'],
|
||||||
|
saved['content_hash'][:12],
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(
|
||||||
|
ValueError,
|
||||||
|
'Skill 版本不存在',
|
||||||
|
):
|
||||||
|
runtime.read_skill_version_file(
|
||||||
|
'bob',
|
||||||
|
'label-master',
|
||||||
|
saved['id'],
|
||||||
|
'SKILL.md',
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
runtime.shutdown()
|
||||||
|
|
||||||
def test_result_normalization_and_label_equivalence(self) -> None:
|
def test_result_normalization_and_label_equivalence(self) -> None:
|
||||||
result = normalize_evaluation_output(
|
result = normalize_evaluation_output(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
|
|||||||
Reference in New Issue
Block a user