From f64380a0f4f2c04937d2161639b229976c30ca17 Mon Sep 17 00:00:00 2001 From: wuyang <5700876+banisherwy@user.noreply.gitee.com> Date: Sun, 26 Jul 2026 17:00:23 +0800 Subject: [PATCH] fix: ground Work reports in measurements --- agent_platform/runtime/loop.py | 38 +++++++++++++++++++++++++++------- tests/test_agent_loop.py | 14 ++++++++++--- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/agent_platform/runtime/loop.py b/agent_platform/runtime/loop.py index 17b4dea..c908ff8 100644 --- a/agent_platform/runtime/loop.py +++ b/agent_platform/runtime/loop.py @@ -39,6 +39,10 @@ For a code-and-report task, follow this order: write a real source file, read it file with a complete command, write the report using only the measured output, then read the report. The executable source and the report must be separate files: run the `.py`, `.js`, or other real source file, never a Markdown, text, JSON, CSV, YAML, or other documentation/data file. +For comparisons and benchmarks, implement every compared subject, give each one equivalent independent +inputs, validate results against a trusted reference, use a suitable high-resolution timer and meaningful +sizes/repeats, and print concise numeric measurements instead of raw input arrays. Include exact measured +values from successful execution output in the report; never substitute estimated or invented numbers. Write source files with real line breaks rather than literal `\\n` escape sequences. If any command fails, inspect the failure, repair the underlying problem, and successfully rerun the relevant check; never replace failed output with invented numbers, placeholders, or a hand-written success report. @@ -82,6 +86,7 @@ EXECUTION_REQUEST = re.compile( ) SOURCE_FILE_REQUEST = re.compile(r"(脚本|程序|代码|\b(?:script|program|code)\b)", re.IGNORECASE) REPORT_FILE_REQUEST = re.compile(r"(报告|\breport\b)", re.IGNORECASE) +COMPARISON_REQUEST = re.compile(r"(比较|对比|基准|\b(?:compare|comparison|benchmark)\b)", re.IGNORECASE) MUTATION_TOOLS = {"write_file", "apply_patch"} VERIFICATION_TOOLS = {"read_file", "git_status", "git_diff", "exec", "poll_process"} EXECUTION_TOOLS = {"exec", "start_process", "poll_process"} @@ -228,7 +233,8 @@ def deliverable_evidence_failure( written_source_paths: set[str], written_report_paths: set[str], reports_written_after_execution: set[str], - verified_paths: set[str], + report_grounding_required: bool, + grounded_report_paths: set[str], ) -> str | None: reasons: list[str] = [] if source_required and not written_source_paths: @@ -237,11 +243,19 @@ def deliverable_evidence_failure( reasons.append("Create a separate report file such as `report.md`.") elif report_required and not reports_written_after_execution: reasons.append("Write or update the report only after a successful execution, using its measured output.") - elif report_required and not (reports_written_after_execution & verified_paths): - reasons.append("Read the generated report file after writing it to verify the delivered content.") + elif report_grounding_required and not grounded_report_paths: + reasons.append( + "Update the report with at least one exact decimal or scientific-notation measurement from the " + "latest successful execution output; estimated or invented values are not accepted." + ) return " ".join(reasons) or None +def measurement_tokens(value: Any) -> set[str]: + text = str(value or "").lower() + return set(re.findall(r"(? str: return ( "Your attempted checkpoint was rejected by the execution policy. Continue the original task now. " @@ -553,6 +567,7 @@ class AgentLoop: execution_required = depth == 0 and bool(EXECUTION_REQUEST.search(request_text)) source_required = depth == 0 and bool(SOURCE_FILE_REQUEST.search(request_text)) report_required = depth == 0 and bool(REPORT_FILE_REQUEST.search(request_text)) + report_grounding_required = report_required and bool(COMPARISON_REQUEST.search(request_text)) mutation_count = 0 verified_mutation = -1 successful_tools = 0 @@ -560,7 +575,8 @@ class AgentLoop: written_source_paths: set[str] = set() written_report_paths: set[str] = set() reports_written_after_execution: set[str] = set() - verified_paths: set[str] = set() + grounded_report_paths: set[str] = set() + latest_execution_measurements: set[str] = set() unresolved_execution_failure: str | None = None failed_execution_revisions: dict[str, int] = {} successful_write_fingerprints: set[str] = set() @@ -604,7 +620,8 @@ class AgentLoop: written_source_paths=written_source_paths, written_report_paths=written_report_paths, reports_written_after_execution=reports_written_after_execution, - verified_paths=verified_paths, + report_grounding_required=report_grounding_required, + grounded_report_paths=grounded_report_paths, ), ) if not failure and action_required and successful_tools == 0: @@ -723,8 +740,11 @@ class AgentLoop: written_report_paths.add(artifact_path) if successful_executions or not execution_required: reports_written_after_execution.add(artifact_path) - elif name == "read_file" and artifact_path: - verified_paths.add(artifact_path) + report_content = str(arguments.get("content", "")) + if not report_grounding_required or ( + latest_execution_measurements & measurement_tokens(report_content) + ): + grounded_report_paths.add(artifact_path) fingerprint = write_call_fingerprint(name, arguments) if isinstance(arguments, dict) else None if fingerprint: successful_write_fingerprints.add(fingerprint) @@ -733,6 +753,7 @@ class AgentLoop: if is_substantive_execution(name, arguments): successful_executions += 1 unresolved_execution_failure = None + latest_execution_measurements = measurement_tokens(result.get("output")) elif is_substantive_execution(name, arguments) and not result.get("policy_duplicate", False): unresolved_execution_failure = f"{name}: {public_tool_summary(result, 1000)}" execution_failures.append(unresolved_execution_failure) @@ -780,7 +801,8 @@ class AgentLoop: written_source_paths=written_source_paths, written_report_paths=written_report_paths, reports_written_after_execution=reports_written_after_execution, - verified_paths=verified_paths, + report_grounding_required=report_grounding_required, + grounded_report_paths=grounded_report_paths, ), ): return ( diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index b69c39c..3fe8109 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -11,6 +11,7 @@ from agent_platform.runtime.loop import ( AgentLoop, deliverable_evidence_failure, is_substantive_execution, + measurement_tokens, non_source_execution_error, normalize_write_file_content, python_source_error, @@ -77,7 +78,8 @@ def test_script_and_report_require_distinct_verified_artifacts() -> None: written_source_paths={"benchmark.py"}, written_report_paths=set(), reports_written_after_execution=set(), - verified_paths={"benchmark.py"}, + report_grounding_required=True, + grounded_report_paths=set(), ) == "Create a separate report file such as `report.md`." ) @@ -88,12 +90,18 @@ def test_script_and_report_require_distinct_verified_artifacts() -> None: written_source_paths={"benchmark.py"}, written_report_paths={"report.md"}, reports_written_after_execution={"report.md"}, - verified_paths={"benchmark.py", "report.md"}, + report_grounding_required=True, + grounded_report_paths={"report.md"}, ) is None ) +def test_measurement_tokens_only_capture_exact_numeric_measurements() -> None: + assert measurement_tokens("n=1000 elapsed=0.012345s score=4.7e-06") == {"0.012345", "4.7e-06"} + assert measurement_tokens("O(n²), version 3") == set() + + def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None: assert not is_substantive_execution("exec", {"command": "cat report.txt"}) assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"}) @@ -461,7 +469,7 @@ async def test_script_and_report_completion_requires_both_files(settings) -> Non max_tool_output_chars=10_000, ).run( spec=get_model_spec("work-light"), - messages=[{"role": "user", "content": "写脚本对比算法并给一个报告"}], + messages=[{"role": "user", "content": "写一个脚本并给一个报告"}], identity=UserIdentity("u1", "", "", "user"), raw_user_jwt="jwt", chat_id="script-report",