fix: require executable source for Work reports

This commit is contained in:
wuyang
2026-07-26 16:35:58 +08:00
parent 87689931b5
commit 5c9f827357
2 changed files with 30 additions and 0 deletions
+21
View File
@@ -37,6 +37,8 @@ successful tool result from this run. For scripts, reports, code, and other deli
files under /workspace, inspect them after writing, run relevant checks, and cite their exact paths. files under /workspace, inspect them after writing, run relevant checks, and cite their exact paths.
For a code-and-report task, follow this order: write a real source file, read it, run that exact source For a code-and-report task, follow this order: write a real source file, read it, run that exact source
file with a complete command, write the report using only the measured output, then read the report. file with a complete command, write the report using only the measured output, then read the report.
The executable source and the report must be separate files: run the `.py`, `.js`, or other real
source file, never a Markdown, text, JSON, CSV, YAML, or other documentation/data file.
Write source files with real line breaks rather than literal `\\n` escape sequences. If any command Write source files with real line breaks rather than literal `\\n` escape sequences. If any command
fails, inspect the failure, repair the underlying problem, and successfully rerun the relevant check; fails, inspect the failure, repair the underlying problem, and successfully rerun the relevant check;
never replace failed output with invented numbers, placeholders, or a hand-written success report. never replace failed output with invented numbers, placeholders, or a hand-written success report.
@@ -57,6 +59,12 @@ INTERACTIVE_ONLY_COMMAND = re.compile(
r"^(?:python(?:3(?:\.\d+)?)?|ipython|node|bash|sh|zsh|fish|jupyter(?:\s+(?:console|lab|notebook))?)\s*$", r"^(?:python(?:3(?:\.\d+)?)?|ipython|node|bash|sh|zsh|fish|jupyter(?:\s+(?:console|lab|notebook))?)\s*$",
re.IGNORECASE, re.IGNORECASE,
) )
PYTHON_NON_SOURCE_TARGET = re.compile(
r"(?:^|[;&|]\s*)python(?:3(?:\.\d+)?)?\s+"
r"(?!-[cm]\b)(?:-[A-Za-z]+(?:=\S+)?\s+)*"
r"[\"']?[^\"'\s;&|]+\.(?:csv|json|md|rst|txt|ya?ml)[\"']?(?=\s|$|[;&|])",
re.IGNORECASE,
)
ARTIFACT_REQUEST = re.compile( ARTIFACT_REQUEST = re.compile(
r"(写|创建|生成|修改|实现|重构|修复|开发|报告|脚本|文件|代码|网页|项目|" r"(写|创建|生成|修改|实现|重构|修复|开发|报告|脚本|文件|代码|网页|项目|"
r"\b(?:write|create|generate|modify|implement|refactor|fix|build|report|script|file|code)\b)", r"\b(?:write|create|generate|modify|implement|refactor|fix|build|report|script|file|code)\b)",
@@ -353,6 +361,17 @@ def python_source_error(arguments: dict[str, Any]) -> str | None:
return None return None
def non_source_execution_error(arguments: dict[str, Any]) -> str | None:
command = str(arguments.get("command", "")).strip()
if not PYTHON_NON_SOURCE_TARGET.search(command):
return None
return (
"Refused to execute a documentation or data file with Python. Create a separate valid `.py` "
"source file, read it, run that exact source file with `python3 <file>.py`, and only then write "
"or update the report from the measured output."
)
def is_substantive_execution(name: str, arguments: dict[str, Any]) -> bool: def is_substantive_execution(name: str, arguments: dict[str, Any]) -> bool:
if name == "start_process": if name == "start_process":
return True return True
@@ -822,6 +841,8 @@ class AgentLoop:
result = {"ok": False, "error": f"Unknown tool: {name}"} result = {"ok": False, "error": f"Unknown tool: {name}"}
elif read_only and name not in READ_ONLY_TOOL_NAMES: elif read_only and name not in READ_ONLY_TOOL_NAMES:
result = {"ok": False, "error": f"Tool {name} is not available to a read-only delegate"} result = {"ok": False, "error": f"Tool {name} is not available to a read-only delegate"}
elif name in {"exec", "start_process"} and (target_error := non_source_execution_error(arguments)):
result = {"ok": False, "error": target_error}
elif name in {"exec", "start_process"} and INTERACTIVE_ONLY_COMMAND.fullmatch( elif name in {"exec", "start_process"} and INTERACTIVE_ONLY_COMMAND.fullmatch(
str(arguments.get("command", "")).strip() str(arguments.get("command", "")).strip()
): ):
+9
View File
@@ -10,6 +10,7 @@ from agent_platform.models import get_model_spec
from agent_platform.runtime.loop import ( from agent_platform.runtime.loop import (
AgentLoop, AgentLoop,
is_substantive_execution, is_substantive_execution,
non_source_execution_error,
normalize_write_file_content, normalize_write_file_content,
python_source_error, python_source_error,
select_tool_specs, select_tool_specs,
@@ -59,6 +60,14 @@ def test_invalid_complete_python_source_is_detected_before_write() -> None:
assert python_source_error({"path": "benchmark.py", "content": "print('ok')\n"}) is None assert python_source_error({"path": "benchmark.py", "content": "print('ok')\n"}) is None
def test_python_cannot_execute_documentation_or_data_files() -> None:
assert non_source_execution_error({"command": "python3 SortingReport.md"}) is not None
assert non_source_execution_error({"command": "cd /workspace && python report.json"}) is not None
assert non_source_execution_error({"command": "python3 benchmark.py"}) is None
assert non_source_execution_error({"command": "python3 -m pytest"}) is None
assert non_source_execution_error({"command": "python3 -c 'print(1)' "}) is None
def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None: def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None:
assert not is_substantive_execution("exec", {"command": "cat report.txt"}) assert not is_substantive_execution("exec", {"command": "cat report.txt"})
assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"}) assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"})