wuyang/zk-data-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: ground Work reports in measurements

Browse Source
This commit is contained in:
wuyang
2026-07-26 17:00:23 +08:00
parent b49b93dcf7
commit f64380a0f4
2 changed files with 41 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent_platform/runtime/loop.py
+30 -8
View File
@@ -39,6 +39,10 @@ For a code-and-report task, follow this order: write a real source file, read it
file with a complete command, write the report using only the measured output, then read the report.
The executable source and the report must be separate files: run the `.py`, `.js`, or other real
source file, never a Markdown, text, JSON, CSV, YAML, or other documentation/data file.
For comparisons and benchmarks, implement every compared subject, give each one equivalent independent
inputs, validate results against a trusted reference, use a suitable high-resolution timer and meaningful
sizes/repeats, and print concise numeric measurements instead of raw input arrays. Include exact measured
values from successful execution output in the report; never substitute estimated or invented numbers.
Write source files with real line breaks rather than literal `\\n` escape sequences. If any command
fails, inspect the failure, repair the underlying problem, and successfully rerun the relevant check;
never replace failed output with invented numbers, placeholders, or a hand-written success report.
@@ -82,6 +86,7 @@ EXECUTION_REQUEST = re.compile(
)
SOURCE_FILE_REQUEST = re.compile(r"(脚本|程序|代码|\b(?:script|program|code)\b)", re.IGNORECASE)
REPORT_FILE_REQUEST = re.compile(r"(报告|\breport\b)", re.IGNORECASE)
COMPARISON_REQUEST = re.compile(r"(比较|对比|基准|\b(?:compare|comparison|benchmark)\b)", re.IGNORECASE)
MUTATION_TOOLS = {"write_file", "apply_patch"}
VERIFICATION_TOOLS = {"read_file", "git_status", "git_diff", "exec", "poll_process"}
EXECUTION_TOOLS = {"exec", "start_process", "poll_process"}
@@ -228,7 +233,8 @@ def deliverable_evidence_failure(
written_source_paths: set[str],
written_report_paths: set[str],
reports_written_after_execution: set[str],
verified_paths: set[str],
report_grounding_required: bool,
grounded_report_paths: set[str],
) -> str | None:
reasons: list[str] = []
if source_required and not written_source_paths:
@@ -237,11 +243,19 @@ def deliverable_evidence_failure(
reasons.append("Create a separate report file such as `report.md`.")
elif report_required and not reports_written_after_execution:
reasons.append("Write or update the report only after a successful execution, using its measured output.")
elif report_required and not (reports_written_after_execution & verified_paths):
reasons.append("Read the generated report file after writing it to verify the delivered content.")
elif report_grounding_required and not grounded_report_paths:
reasons.append(
"Update the report with at least one exact decimal or scientific-notation measurement from the "
"latest successful execution output; estimated or invented values are not accepted."
)
return " ".join(reasons) or None
def measurement_tokens(value: Any) -> set[str]:
text = str(value or "").lower()
return set(re.findall(r"(?<![\d.])(?:\d+\.\d+(?:e[+-]?\d+)?|\d+e[+-]?\d+)(?![\d.])", text))
def recovery_message(failure: str) -> str:
return (
"Your attempted checkpoint was rejected by the execution policy. Continue the original task now. "
@@ -553,6 +567,7 @@ class AgentLoop:
execution_required = depth == 0 and bool(EXECUTION_REQUEST.search(request_text))
source_required = depth == 0 and bool(SOURCE_FILE_REQUEST.search(request_text))
report_required = depth == 0 and bool(REPORT_FILE_REQUEST.search(request_text))
report_grounding_required = report_required and bool(COMPARISON_REQUEST.search(request_text))
mutation_count = 0
verified_mutation = -1
successful_tools = 0
@@ -560,7 +575,8 @@ class AgentLoop:
written_source_paths: set[str] = set()
written_report_paths: set[str] = set()
reports_written_after_execution: set[str] = set()
verified_paths: set[str] = set()
grounded_report_paths: set[str] = set()
latest_execution_measurements: set[str] = set()
unresolved_execution_failure: str | None = None
failed_execution_revisions: dict[str, int] = {}
successful_write_fingerprints: set[str] = set()
@@ -604,7 +620,8 @@ class AgentLoop:
written_source_paths=written_source_paths,
written_report_paths=written_report_paths,
reports_written_after_execution=reports_written_after_execution,
verified_paths=verified_paths,
report_grounding_required=report_grounding_required,
grounded_report_paths=grounded_report_paths,
),
)
if not failure and action_required and successful_tools == 0:
@@ -723,8 +740,11 @@ class AgentLoop:
written_report_paths.add(artifact_path)
if successful_executions or not execution_required:
reports_written_after_execution.add(artifact_path)
elif name == "read_file" and artifact_path:
verified_paths.add(artifact_path)
report_content = str(arguments.get("content", ""))
if not report_grounding_required or (
latest_execution_measurements & measurement_tokens(report_content)
):
grounded_report_paths.add(artifact_path)
fingerprint = write_call_fingerprint(name, arguments) if isinstance(arguments, dict) else None
if fingerprint:
successful_write_fingerprints.add(fingerprint)
@@ -733,6 +753,7 @@ class AgentLoop:
if is_substantive_execution(name, arguments):
successful_executions += 1
unresolved_execution_failure = None
latest_execution_measurements = measurement_tokens(result.get("output"))
elif is_substantive_execution(name, arguments) and not result.get("policy_duplicate", False):
unresolved_execution_failure = f"{name}: {public_tool_summary(result, 1000)}"
execution_failures.append(unresolved_execution_failure)
@@ -780,7 +801,8 @@ class AgentLoop:
written_source_paths=written_source_paths,
written_report_paths=written_report_paths,
reports_written_after_execution=reports_written_after_execution,
verified_paths=verified_paths,
report_grounding_required=report_grounding_required,
grounded_report_paths=grounded_report_paths,
),
):
return (
tests/test_agent_loop.py
+11 -3
View File
@@ -11,6 +11,7 @@ from agent_platform.runtime.loop import (
AgentLoop,
deliverable_evidence_failure,
is_substantive_execution,
measurement_tokens,
non_source_execution_error,
normalize_write_file_content,
python_source_error,
@@ -77,7 +78,8 @@ def test_script_and_report_require_distinct_verified_artifacts() -> None:
written_source_paths={"benchmark.py"},
written_report_paths=set(),
reports_written_after_execution=set(),
verified_paths={"benchmark.py"},
report_grounding_required=True,
grounded_report_paths=set(),
)
== "Create a separate report file such as `report.md`."
)
@@ -88,12 +90,18 @@ def test_script_and_report_require_distinct_verified_artifacts() -> None:
written_source_paths={"benchmark.py"},
written_report_paths={"report.md"},
reports_written_after_execution={"report.md"},
verified_paths={"benchmark.py", "report.md"},
report_grounding_required=True,
grounded_report_paths={"report.md"},
)
is None
)
def test_measurement_tokens_only_capture_exact_numeric_measurements() -> None:
assert measurement_tokens("n=1000 elapsed=0.012345s score=4.7e-06") == {"0.012345", "4.7e-06"}
assert measurement_tokens("O(n²), version 3") == set()
def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None:
assert not is_substantive_execution("exec", {"command": "cat report.txt"})
assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"})
@@ -461,7 +469,7 @@ async def test_script_and_report_completion_requires_both_files(settings) -> Non
max_tool_output_chars=10_000,
).run(
spec=get_model_spec("work-light"),
messages=[{"role": "user", "content": "写脚本对比算法并给一个报告"}],
messages=[{"role": "user", "content": "写一个脚本并给一个报告"}],
identity=UserIdentity("u1", "", "", "user"),
raw_user_jwt="jwt",
chat_id="script-report",