fix: require downloadable Work reports

This commit is contained in:
wuyang
2026-07-26 16:48:28 +08:00
parent 5c9f827357
commit b49b93dcf7
2 changed files with 198 additions and 1 deletions
+127 -1
View File
@@ -9,6 +9,7 @@ from agent_platform.auth import UserIdentity
from agent_platform.models import get_model_spec
from agent_platform.runtime.loop import (
AgentLoop,
deliverable_evidence_failure,
is_substantive_execution,
non_source_execution_error,
normalize_write_file_content,
@@ -68,6 +69,31 @@ def test_python_cannot_execute_documentation_or_data_files() -> None:
assert non_source_execution_error({"command": "python3 -c 'print(1)' "}) is None
def test_script_and_report_require_distinct_verified_artifacts() -> None:
assert (
deliverable_evidence_failure(
source_required=True,
report_required=True,
written_source_paths={"benchmark.py"},
written_report_paths=set(),
reports_written_after_execution=set(),
verified_paths={"benchmark.py"},
)
== "Create a separate report file such as `report.md`."
)
assert (
deliverable_evidence_failure(
source_required=True,
report_required=True,
written_source_paths={"benchmark.py"},
written_report_paths={"report.md"},
reports_written_after_execution={"report.md"},
verified_paths={"benchmark.py", "report.md"},
)
is None
)
def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None:
assert not is_substantive_execution("exec", {"command": "cat report.txt"})
assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"})
@@ -355,6 +381,106 @@ async def test_artifact_completion_is_rejected_until_written_and_verified(settin
await store.close()
async def test_script_and_report_completion_requires_both_files(settings) -> None:
responses = [
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [
tool_call("1", "write_file", {"path": "benchmark.py", "content": "print('1 ms')\n"})
],
},
"finish_reason": "tool_calls",
}
]
},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [tool_call("2", "read_file", {"path": "benchmark.py"})],
},
"finish_reason": "tool_calls",
}
]
},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [tool_call("3", "exec", {"command": "python3 benchmark.py"})],
},
"finish_reason": "tool_calls",
}
]
},
{"choices": [{"message": {"role": "assistant", "content": "完成"}, "finish_reason": "stop"}]},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [
tool_call("4", "write_file", {"path": "report.md", "content": "# Report\n\n1 ms\n"})
],
},
"finish_reason": "tool_calls",
}
]
},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [tool_call("5", "read_file", {"path": "report.md"})],
},
"finish_reason": "tool_calls",
}
]
},
{"choices": [{"message": {"role": "assistant", "content": "已完成两个文件"}, "finish_reason": "stop"}]},
]
registry = RecordingRegistry()
store = RuntimeStore(settings.database_url)
await store.initialize()
events = []
async def callback(event_type, payload):
events.append(event_type)
try:
answer = await AgentLoop(
ScriptedProvider(responses),
registry,
store,
max_tool_output_chars=10_000,
).run(
spec=get_model_spec("work-light"),
messages=[{"role": "user", "content": "写脚本对比算法并给一个报告"}],
identity=UserIdentity("u1", "", "", "user"),
raw_user_jwt="jwt",
chat_id="script-report",
callback=callback,
)
finally:
await store.close()
assert answer == "已完成两个文件"
assert events.count("completion.rejected") == 1
assert [name for name, _ in registry.calls] == [
"write_file",
"read_file",
"exec",
"write_file",
"read_file",
]
class FailingExecutionRegistry(RecordingRegistry):
def __init__(self) -> None:
super().__init__()
@@ -441,7 +567,7 @@ async def test_failed_execution_must_be_repaired_before_completion(settings) ->
try:
answer = await AgentLoop(provider, registry, store, max_tool_output_chars=10_000).run(
spec=get_model_spec("work-light"),
messages=[{"role": "user", "content": "写脚本对比排序算法并生成报告"}],
messages=[{"role": "user", "content": "写脚本对比排序算法"}],
identity=UserIdentity("u1", "", "", "user"),
raw_user_jwt="jwt",
chat_id="c1",