diff --git a/agent_platform/runtime/loop.py b/agent_platform/runtime/loop.py index f1ce859..17b4dea 100644 --- a/agent_platform/runtime/loop.py +++ b/agent_platform/runtime/loop.py @@ -80,6 +80,8 @@ EXECUTION_REQUEST = re.compile( r"\b(?:run|execute|test|script|program|code|compare|benchmark)\b)", re.IGNORECASE, ) +SOURCE_FILE_REQUEST = re.compile(r"(脚本|程序|代码|\b(?:script|program|code)\b)", re.IGNORECASE) +REPORT_FILE_REQUEST = re.compile(r"(报告|\breport\b)", re.IGNORECASE) MUTATION_TOOLS = {"write_file", "apply_patch"} VERIFICATION_TOOLS = {"read_file", "git_status", "git_diff", "exec", "poll_process"} EXECUTION_TOOLS = {"exec", "start_process", "poll_process"} @@ -122,6 +124,7 @@ PLAIN_TEXT_FILE_SUFFIXES = { ".yaml", ".yml", } +REPORT_FILE_SUFFIXES = {".md", ".rst", ".txt"} READ_ONLY_EXEC_COMMAND = re.compile( r"^(?:cat|cut|diff|find|git\s+(?:diff|show|status)|grep|head|jq|ls|pwd|rg|sed\s+-n|stat|tail|wc)\b", re.IGNORECASE, @@ -177,6 +180,7 @@ def completion_failure( execution_required: bool = False, successful_executions: int = 0, unresolved_execution_failure: str | None = None, + deliverable_failure: str | None = None, ) -> str | None: reasons: list[str] = [] if artifact_required and mutation_count == 0: @@ -199,11 +203,45 @@ def completion_failure( "The task requires a script, code, test, benchmark, or comparison, but no command or process " "completed successfully. Run the relevant verification and use its real output." ) + if deliverable_failure: + reasons.append(deliverable_failure) if not reasons: return None return " Do not claim completion. ".join(reasons) +def workspace_artifact_path(arguments: dict[str, Any]) -> str | None: + raw_path = arguments.get("path") + if not isinstance(raw_path, str) or not raw_path.strip(): + return None + path = PurePosixPath(raw_path.strip()) + try: + return str(path.relative_to("/workspace")) + except ValueError: + return str(path) + + +def deliverable_evidence_failure( + *, + source_required: bool, + report_required: bool, + written_source_paths: set[str], + written_report_paths: set[str], + reports_written_after_execution: set[str], + verified_paths: set[str], +) -> str | None: + reasons: list[str] = [] + if source_required and not written_source_paths: + reasons.append("Create a separate executable source file such as a `.py` file.") + if report_required and not written_report_paths: + reasons.append("Create a separate report file such as `report.md`.") + elif report_required and not reports_written_after_execution: + reasons.append("Write or update the report only after a successful execution, using its measured output.") + elif report_required and not (reports_written_after_execution & verified_paths): + reasons.append("Read the generated report file after writing it to verify the delivered content.") + return " ".join(reasons) or None + + def recovery_message(failure: str) -> str: return ( "Your attempted checkpoint was rejected by the execution policy. Continue the original task now. " @@ -513,10 +551,16 @@ class AgentLoop: artifact_required = depth == 0 and bool(ARTIFACT_REQUEST.search(request_text)) action_required = depth == 0 and bool(ACTION_REQUEST.search(request_text)) execution_required = depth == 0 and bool(EXECUTION_REQUEST.search(request_text)) + source_required = depth == 0 and bool(SOURCE_FILE_REQUEST.search(request_text)) + report_required = depth == 0 and bool(REPORT_FILE_REQUEST.search(request_text)) mutation_count = 0 verified_mutation = -1 successful_tools = 0 successful_executions = 0 + written_source_paths: set[str] = set() + written_report_paths: set[str] = set() + reports_written_after_execution: set[str] = set() + verified_paths: set[str] = set() unresolved_execution_failure: str | None = None failed_execution_revisions: dict[str, int] = {} successful_write_fingerprints: set[str] = set() @@ -554,6 +598,14 @@ class AgentLoop: execution_required, successful_executions, unresolved_execution_failure, + deliverable_evidence_failure( + source_required=source_required, + report_required=report_required, + written_source_paths=written_source_paths, + written_report_paths=written_report_paths, + reports_written_after_execution=reports_written_after_execution, + verified_paths=verified_paths, + ), ) if not failure and action_required and successful_tools == 0: failure = ( @@ -662,6 +714,17 @@ class AgentLoop: successful_tools += 1 if name in MUTATION_TOOLS: mutation_count += 1 + artifact_path = workspace_artifact_path(arguments) if isinstance(arguments, dict) else None + if name == "write_file" and artifact_path: + suffix = PurePosixPath(artifact_path).suffix.lower() + if suffix in CODE_FILE_SUFFIXES: + written_source_paths.add(artifact_path) + if suffix in REPORT_FILE_SUFFIXES: + written_report_paths.add(artifact_path) + if successful_executions or not execution_required: + reports_written_after_execution.add(artifact_path) + elif name == "read_file" and artifact_path: + verified_paths.add(artifact_path) fingerprint = write_call_fingerprint(name, arguments) if isinstance(arguments, dict) else None if fingerprint: successful_write_fingerprints.add(fingerprint) @@ -711,6 +774,14 @@ class AgentLoop: execution_required, successful_executions, unresolved_execution_failure, + deliverable_evidence_failure( + source_required=source_required, + report_required=report_required, + written_source_paths=written_source_paths, + written_report_paths=written_report_paths, + reports_written_after_execution=reports_written_after_execution, + verified_paths=verified_paths, + ), ): return ( "未完成：工具迭代已用尽，但交付物没有形成完整的“写入后验证”证据链。" diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 94135af..b69c39c 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -9,6 +9,7 @@ from agent_platform.auth import UserIdentity from agent_platform.models import get_model_spec from agent_platform.runtime.loop import ( AgentLoop, + deliverable_evidence_failure, is_substantive_execution, non_source_execution_error, normalize_write_file_content, @@ -68,6 +69,31 @@ def test_python_cannot_execute_documentation_or_data_files() -> None: assert non_source_execution_error({"command": "python3 -c 'print(1)' "}) is None +def test_script_and_report_require_distinct_verified_artifacts() -> None: + assert ( + deliverable_evidence_failure( + source_required=True, + report_required=True, + written_source_paths={"benchmark.py"}, + written_report_paths=set(), + reports_written_after_execution=set(), + verified_paths={"benchmark.py"}, + ) + == "Create a separate report file such as `report.md`." + ) + assert ( + deliverable_evidence_failure( + source_required=True, + report_required=True, + written_source_paths={"benchmark.py"}, + written_report_paths={"report.md"}, + reports_written_after_execution={"report.md"}, + verified_paths={"benchmark.py", "report.md"}, + ) + is None + ) + + def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None: assert not is_substantive_execution("exec", {"command": "cat report.txt"}) assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"}) @@ -355,6 +381,106 @@ async def test_artifact_completion_is_rejected_until_written_and_verified(settin await store.close() +async def test_script_and_report_completion_requires_both_files(settings) -> None: + responses = [ + { + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [ + tool_call("1", "write_file", {"path": "benchmark.py", "content": "print('1 ms')\n"}) + ], + }, + "finish_reason": "tool_calls", + } + ] + }, + { + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [tool_call("2", "read_file", {"path": "benchmark.py"})], + }, + "finish_reason": "tool_calls", + } + ] + }, + { + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [tool_call("3", "exec", {"command": "python3 benchmark.py"})], + }, + "finish_reason": "tool_calls", + } + ] + }, + {"choices": [{"message": {"role": "assistant", "content": "完成"}, "finish_reason": "stop"}]}, + { + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [ + tool_call("4", "write_file", {"path": "report.md", "content": "# Report\n\n1 ms\n"}) + ], + }, + "finish_reason": "tool_calls", + } + ] + }, + { + "choices": [ + { + "message": { + "role": "assistant", + "tool_calls": [tool_call("5", "read_file", {"path": "report.md"})], + }, + "finish_reason": "tool_calls", + } + ] + }, + {"choices": [{"message": {"role": "assistant", "content": "已完成两个文件"}, "finish_reason": "stop"}]}, + ] + registry = RecordingRegistry() + store = RuntimeStore(settings.database_url) + await store.initialize() + events = [] + + async def callback(event_type, payload): + events.append(event_type) + + try: + answer = await AgentLoop( + ScriptedProvider(responses), + registry, + store, + max_tool_output_chars=10_000, + ).run( + spec=get_model_spec("work-light"), + messages=[{"role": "user", "content": "写脚本对比算法并给一个报告"}], + identity=UserIdentity("u1", "", "", "user"), + raw_user_jwt="jwt", + chat_id="script-report", + callback=callback, + ) + finally: + await store.close() + + assert answer == "已完成两个文件" + assert events.count("completion.rejected") == 1 + assert [name for name, _ in registry.calls] == [ + "write_file", + "read_file", + "exec", + "write_file", + "read_file", + ] + + class FailingExecutionRegistry(RecordingRegistry): def __init__(self) -> None: super().__init__() @@ -441,7 +567,7 @@ async def test_failed_execution_must_be_repaired_before_completion(settings) -> try: answer = await AgentLoop(provider, registry, store, max_tool_output_chars=10_000).run( spec=get_model_spec("work-light"), - messages=[{"role": "user", "content": "写脚本对比排序算法并生成报告"}], + messages=[{"role": "user", "content": "写脚本对比排序算法"}], identity=UserIdentity("u1", "", "", "user"), raw_user_jwt="jwt", chat_id="c1",