fix: require downloadable Work reports
This commit is contained in:
+127
-1
@@ -9,6 +9,7 @@ from agent_platform.auth import UserIdentity
|
||||
from agent_platform.models import get_model_spec
|
||||
from agent_platform.runtime.loop import (
|
||||
AgentLoop,
|
||||
deliverable_evidence_failure,
|
||||
is_substantive_execution,
|
||||
non_source_execution_error,
|
||||
normalize_write_file_content,
|
||||
@@ -68,6 +69,31 @@ def test_python_cannot_execute_documentation_or_data_files() -> None:
|
||||
assert non_source_execution_error({"command": "python3 -c 'print(1)' "}) is None
|
||||
|
||||
|
||||
def test_script_and_report_require_distinct_verified_artifacts() -> None:
|
||||
assert (
|
||||
deliverable_evidence_failure(
|
||||
source_required=True,
|
||||
report_required=True,
|
||||
written_source_paths={"benchmark.py"},
|
||||
written_report_paths=set(),
|
||||
reports_written_after_execution=set(),
|
||||
verified_paths={"benchmark.py"},
|
||||
)
|
||||
== "Create a separate report file such as `report.md`."
|
||||
)
|
||||
assert (
|
||||
deliverable_evidence_failure(
|
||||
source_required=True,
|
||||
report_required=True,
|
||||
written_source_paths={"benchmark.py"},
|
||||
written_report_paths={"report.md"},
|
||||
reports_written_after_execution={"report.md"},
|
||||
verified_paths={"benchmark.py", "report.md"},
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None:
|
||||
assert not is_substantive_execution("exec", {"command": "cat report.txt"})
|
||||
assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"})
|
||||
@@ -355,6 +381,106 @@ async def test_artifact_completion_is_rejected_until_written_and_verified(settin
|
||||
await store.close()
|
||||
|
||||
|
||||
async def test_script_and_report_completion_requires_both_files(settings) -> None:
|
||||
responses = [
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
tool_call("1", "write_file", {"path": "benchmark.py", "content": "print('1 ms')\n"})
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [tool_call("2", "read_file", {"path": "benchmark.py"})],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [tool_call("3", "exec", {"command": "python3 benchmark.py"})],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
]
|
||||
},
|
||||
{"choices": [{"message": {"role": "assistant", "content": "完成"}, "finish_reason": "stop"}]},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
tool_call("4", "write_file", {"path": "report.md", "content": "# Report\n\n1 ms\n"})
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"tool_calls": [tool_call("5", "read_file", {"path": "report.md"})],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
]
|
||||
},
|
||||
{"choices": [{"message": {"role": "assistant", "content": "已完成两个文件"}, "finish_reason": "stop"}]},
|
||||
]
|
||||
registry = RecordingRegistry()
|
||||
store = RuntimeStore(settings.database_url)
|
||||
await store.initialize()
|
||||
events = []
|
||||
|
||||
async def callback(event_type, payload):
|
||||
events.append(event_type)
|
||||
|
||||
try:
|
||||
answer = await AgentLoop(
|
||||
ScriptedProvider(responses),
|
||||
registry,
|
||||
store,
|
||||
max_tool_output_chars=10_000,
|
||||
).run(
|
||||
spec=get_model_spec("work-light"),
|
||||
messages=[{"role": "user", "content": "写脚本对比算法并给一个报告"}],
|
||||
identity=UserIdentity("u1", "", "", "user"),
|
||||
raw_user_jwt="jwt",
|
||||
chat_id="script-report",
|
||||
callback=callback,
|
||||
)
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
assert answer == "已完成两个文件"
|
||||
assert events.count("completion.rejected") == 1
|
||||
assert [name for name, _ in registry.calls] == [
|
||||
"write_file",
|
||||
"read_file",
|
||||
"exec",
|
||||
"write_file",
|
||||
"read_file",
|
||||
]
|
||||
|
||||
|
||||
class FailingExecutionRegistry(RecordingRegistry):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
@@ -441,7 +567,7 @@ async def test_failed_execution_must_be_repaired_before_completion(settings) ->
|
||||
try:
|
||||
answer = await AgentLoop(provider, registry, store, max_tool_output_chars=10_000).run(
|
||||
spec=get_model_spec("work-light"),
|
||||
messages=[{"role": "user", "content": "写脚本对比排序算法并生成报告"}],
|
||||
messages=[{"role": "user", "content": "写脚本对比排序算法"}],
|
||||
identity=UserIdentity("u1", "", "", "user"),
|
||||
raw_user_jwt="jwt",
|
||||
chat_id="c1",
|
||||
|
||||
Reference in New Issue
Block a user