wuyang/zk-data-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: require downloadable Work reports

Browse Source
This commit is contained in:
wuyang
2026-07-26 16:48:28 +08:00
parent 5c9f827357
commit b49b93dcf7
2 changed files with 198 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent_platform/runtime/loop.py
+71
View File
@@ -80,6 +80,8 @@ EXECUTION_REQUEST = re.compile(
r"\b(?:run|execute|test|script|program|code|compare|benchmark)\b)",
re.IGNORECASE,
)
SOURCE_FILE_REQUEST = re.compile(r"(脚本|程序|代码|\b(?:script|program|code)\b)", re.IGNORECASE)
REPORT_FILE_REQUEST = re.compile(r"(报告|\breport\b)", re.IGNORECASE)
MUTATION_TOOLS = {"write_file", "apply_patch"}
VERIFICATION_TOOLS = {"read_file", "git_status", "git_diff", "exec", "poll_process"}
EXECUTION_TOOLS = {"exec", "start_process", "poll_process"}
@@ -122,6 +124,7 @@ PLAIN_TEXT_FILE_SUFFIXES = {
".yaml",
".yml",
}
REPORT_FILE_SUFFIXES = {".md", ".rst", ".txt"}
READ_ONLY_EXEC_COMMAND = re.compile(
r"^(?:cat|cut|diff|find|git\s+(?:diff|show|status)|grep|head|jq|ls|pwd|rg|sed\s+-n|stat|tail|wc)\b",
re.IGNORECASE,
@@ -177,6 +180,7 @@ def completion_failure(
execution_required: bool = False,
successful_executions: int = 0,
unresolved_execution_failure: str | None = None,
deliverable_failure: str | None = None,
) -> str | None:
reasons: list[str] = []
if artifact_required and mutation_count == 0:
@@ -199,11 +203,45 @@ def completion_failure(
"The task requires a script, code, test, benchmark, or comparison, but no command or process "
"completed successfully. Run the relevant verification and use its real output."
)
if deliverable_failure:
reasons.append(deliverable_failure)
if not reasons:
return None
return " Do not claim completion. ".join(reasons)
def workspace_artifact_path(arguments: dict[str, Any]) -> str | None:
raw_path = arguments.get("path")
if not isinstance(raw_path, str) or not raw_path.strip():
return None
path = PurePosixPath(raw_path.strip())
try:
return str(path.relative_to("/workspace"))
except ValueError:
return str(path)
def deliverable_evidence_failure(
*,
source_required: bool,
report_required: bool,
written_source_paths: set[str],
written_report_paths: set[str],
reports_written_after_execution: set[str],
verified_paths: set[str],
) -> str | None:
reasons: list[str] = []
if source_required and not written_source_paths:
reasons.append("Create a separate executable source file such as a `.py` file.")
if report_required and not written_report_paths:
reasons.append("Create a separate report file such as `report.md`.")
elif report_required and not reports_written_after_execution:
reasons.append("Write or update the report only after a successful execution, using its measured output.")
elif report_required and not (reports_written_after_execution & verified_paths):
reasons.append("Read the generated report file after writing it to verify the delivered content.")
return " ".join(reasons) or None
def recovery_message(failure: str) -> str:
return (
"Your attempted checkpoint was rejected by the execution policy. Continue the original task now. "
@@ -513,10 +551,16 @@ class AgentLoop:
artifact_required = depth == 0 and bool(ARTIFACT_REQUEST.search(request_text))
action_required = depth == 0 and bool(ACTION_REQUEST.search(request_text))
execution_required = depth == 0 and bool(EXECUTION_REQUEST.search(request_text))
source_required = depth == 0 and bool(SOURCE_FILE_REQUEST.search(request_text))
report_required = depth == 0 and bool(REPORT_FILE_REQUEST.search(request_text))
mutation_count = 0
verified_mutation = -1
successful_tools = 0
successful_executions = 0
written_source_paths: set[str] = set()
written_report_paths: set[str] = set()
reports_written_after_execution: set[str] = set()
verified_paths: set[str] = set()
unresolved_execution_failure: str | None = None
failed_execution_revisions: dict[str, int] = {}
successful_write_fingerprints: set[str] = set()
@@ -554,6 +598,14 @@ class AgentLoop:
execution_required,
successful_executions,
unresolved_execution_failure,
deliverable_evidence_failure(
source_required=source_required,
report_required=report_required,
written_source_paths=written_source_paths,
written_report_paths=written_report_paths,
reports_written_after_execution=reports_written_after_execution,
verified_paths=verified_paths,
),
)
if not failure and action_required and successful_tools == 0:
failure = (
@@ -662,6 +714,17 @@ class AgentLoop:
successful_tools += 1
if name in MUTATION_TOOLS:
mutation_count += 1
artifact_path = workspace_artifact_path(arguments) if isinstance(arguments, dict) else None
if name == "write_file" and artifact_path:
suffix = PurePosixPath(artifact_path).suffix.lower()
if suffix in CODE_FILE_SUFFIXES:
written_source_paths.add(artifact_path)
if suffix in REPORT_FILE_SUFFIXES:
written_report_paths.add(artifact_path)
if successful_executions or not execution_required:
reports_written_after_execution.add(artifact_path)
elif name == "read_file" and artifact_path:
verified_paths.add(artifact_path)
fingerprint = write_call_fingerprint(name, arguments) if isinstance(arguments, dict) else None
if fingerprint:
successful_write_fingerprints.add(fingerprint)
@@ -711,6 +774,14 @@ class AgentLoop:
execution_required,
successful_executions,
unresolved_execution_failure,
deliverable_evidence_failure(
source_required=source_required,
report_required=report_required,
written_source_paths=written_source_paths,
written_report_paths=written_report_paths,
reports_written_after_execution=reports_written_after_execution,
verified_paths=verified_paths,
),
):
return (
"未完成：工具迭代已用尽，但交付物没有形成完整的“写入后验证”证据链。"
tests/test_agent_loop.py
+127 -1
View File
@@ -9,6 +9,7 @@ from agent_platform.auth import UserIdentity
from agent_platform.models import get_model_spec
from agent_platform.runtime.loop import (
AgentLoop,
deliverable_evidence_failure,
is_substantive_execution,
non_source_execution_error,
normalize_write_file_content,
@@ -68,6 +69,31 @@ def test_python_cannot_execute_documentation_or_data_files() -> None:
assert non_source_execution_error({"command": "python3 -c 'print(1)' "}) is None
def test_script_and_report_require_distinct_verified_artifacts() -> None:
assert (
deliverable_evidence_failure(
source_required=True,
report_required=True,
written_source_paths={"benchmark.py"},
written_report_paths=set(),
reports_written_after_execution=set(),
verified_paths={"benchmark.py"},
)
== "Create a separate report file such as `report.md`."
)
assert (
deliverable_evidence_failure(
source_required=True,
report_required=True,
written_source_paths={"benchmark.py"},
written_report_paths={"report.md"},
reports_written_after_execution={"report.md"},
verified_paths={"benchmark.py", "report.md"},
)
is None
)
def test_read_only_shell_commands_do_not_satisfy_execution_evidence() -> None:
assert not is_substantive_execution("exec", {"command": "cat report.txt"})
assert not is_substantive_execution("exec", {"command": "sed -n '1,20p' script.py"})
@@ -355,6 +381,106 @@ async def test_artifact_completion_is_rejected_until_written_and_verified(settin
await store.close()
async def test_script_and_report_completion_requires_both_files(settings) -> None:
responses = [
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [
tool_call("1", "write_file", {"path": "benchmark.py", "content": "print('1 ms')\n"})
],
},
"finish_reason": "tool_calls",
}
]
},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [tool_call("2", "read_file", {"path": "benchmark.py"})],
},
"finish_reason": "tool_calls",
}
]
},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [tool_call("3", "exec", {"command": "python3 benchmark.py"})],
},
"finish_reason": "tool_calls",
}
]
},
{"choices": [{"message": {"role": "assistant", "content": "完成"}, "finish_reason": "stop"}]},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [
tool_call("4", "write_file", {"path": "report.md", "content": "# Report\n\n1 ms\n"})
],
},
"finish_reason": "tool_calls",
}
]
},
{
"choices": [
{
"message": {
"role": "assistant",
"tool_calls": [tool_call("5", "read_file", {"path": "report.md"})],
},
"finish_reason": "tool_calls",
}
]
},
{"choices": [{"message": {"role": "assistant", "content": "已完成两个文件"}, "finish_reason": "stop"}]},
]
registry = RecordingRegistry()
store = RuntimeStore(settings.database_url)
await store.initialize()
events = []
async def callback(event_type, payload):
events.append(event_type)
try:
answer = await AgentLoop(
ScriptedProvider(responses),
registry,
store,
max_tool_output_chars=10_000,
).run(
spec=get_model_spec("work-light"),
messages=[{"role": "user", "content": "写脚本对比算法并给一个报告"}],
identity=UserIdentity("u1", "", "", "user"),
raw_user_jwt="jwt",
chat_id="script-report",
callback=callback,
)
finally:
await store.close()
assert answer == "已完成两个文件"
assert events.count("completion.rejected") == 1
assert [name for name, _ in registry.calls] == [
"write_file",
"read_file",
"exec",
"write_file",
"read_file",
]
class FailingExecutionRegistry(RecordingRegistry):
def __init__(self) -> None:
super().__init__()
@@ -441,7 +567,7 @@ async def test_failed_execution_must_be_repaired_before_completion(settings) ->
try:
answer = await AgentLoop(provider, registry, store, max_tool_output_chars=10_000).run(
spec=get_model_spec("work-light"),
messages=[{"role": "user", "content": "写脚本对比排序算法并生成报告"}],
messages=[{"role": "user", "content": "写脚本对比排序算法"}],
identity=UserIdentity("u1", "", "", "user"),
raw_user_jwt="jwt",
chat_id="c1",