wuyang/agent
2
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refresh papers and define Agent evaluation

Browse Source
This commit is contained in:
wuyang
2026-07-27 16:28:23 +08:00
parent 8e4ff3779b
commit df475e8d90
180 changed files with 31313 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/check_project.py
+31
View File
@@ -21,6 +21,9 @@ REQUIRED_FILES = (
"data/summary.json",
"research/memory/findings.md",
"research/memory/evidence-ledger.md",
"research/evaluation/k1412-agent-evaluation-v1.md",
"data/evaluation/k1412-agent-eval-task-contracts-v1.json",
"tools/evaluation/score_agent_runs.py",
"web/app.py",
"web/agent-knowledge-atlas.service",
"web/manage.sh",
@@ -109,6 +112,32 @@ def check_experiment_state() -> None:
require("status: failed" in pilot, "pilot experiment status is stale")
def check_evaluation_contract() -> int:
suite = load_json("data/evaluation/k1412-agent-eval-task-contracts-v1.json")
require(
suite.get("schema_version") == "agent-eval-task-contracts/v1",
"unknown Agent evaluation task schema",
)
tasks = suite.get("tasks")
require(isinstance(tasks, list) and tasks, "Agent evaluation task suite is empty")
budget_profiles = suite.get("budget_profiles") or {}
require(isinstance(budget_profiles, dict) and budget_profiles, "evaluation budget profiles are empty")
task_ids = [str(task.get("task_id") or "") for task in tasks]
require(all(task_ids), "Agent evaluation task is missing task_id")
require(len(task_ids) == len(set(task_ids)), "Agent evaluation task ids are not unique")
for task in tasks:
require(task.get("title_zh"), f"{task['task_id']} is missing Chinese title")
require(task.get("version"), f"{task['task_id']} is missing version")
require(task.get("runner_status"), f"{task['task_id']} is missing runner_status")
require(
task.get("budget_profile") in budget_profiles,
f"{task['task_id']} references an unknown budget profile",
)
require(task.get("observable_completion"), f"{task['task_id']} has no completion contract")
require(task.get("forbidden_effects") is not None, f"{task['task_id']} has no forbidden effects")
return len(tasks)
def main() -> int:
check_required_files()
collections = check_collection_index()
@@ -116,12 +145,14 @@ def main() -> int:
python_files = check_python_syntax()
check_ollama_contract()
check_experiment_state()
evaluation_tasks = check_evaluation_contract()
print(
json.dumps(
{
"ok": True,
"collections": collections,
"research": research,
"evaluation_tasks": evaluation_tasks,
"python_files_checked": python_files,
},
ensure_ascii=False,