fix: subprocess-based remote sync, pipeline status, skill updates
- Replace thread-based remote state sync with subprocess (sync_remote_state.py) to prevent thread pool exhaustion hanging the API - Fix pipeline showing 'waiting' when a real step is running alongside a gate - Fix watcher scanner path for linux accounts mode - Disable label-master semantic review (Step A.5 + §4.5) per user request - Correct field names: use is_model_correct_dev for accuracy, origin_predict_dev for model output (not cleaned_predict) - Add prohibition against putting test set data in relabel_candidates Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hupenglong1
+240
-143
@@ -1,9 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from typing import Any, Callable
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -3015,13 +3017,13 @@ class LocalCodingAgent:
|
||||
),
|
||||
allow_shell_commands=(
|
||||
self.runtime_config.permissions.allow_shell_commands
|
||||
and bool(arguments.get('allow_shell', False))
|
||||
and arguments.get('allow_shell', True) is not False
|
||||
),
|
||||
allow_destructive_shell_commands=False,
|
||||
)
|
||||
|
||||
# Resolve max_turns — agent definition or explicit param
|
||||
effective_max_turns = max_turns or agent_def.max_turns or min(self.runtime_config.max_turns, 6)
|
||||
effective_max_turns = max_turns or agent_def.max_turns or min(self.runtime_config.max_turns, 50)
|
||||
|
||||
child_runtime_config = replace(
|
||||
self.runtime_config,
|
||||
@@ -3064,6 +3066,8 @@ class LocalCodingAgent:
|
||||
dependency_skips = 0
|
||||
child_result = None
|
||||
stop_processing = False
|
||||
max_concurrency = int(arguments.get('max_concurrency', 4))
|
||||
use_parallel = strategy == 'topological' and max_concurrency > 1
|
||||
for batch_index, batch in enumerate(planned_batches, start=1):
|
||||
if stop_processing:
|
||||
break
|
||||
@@ -3071,6 +3075,8 @@ class LocalCodingAgent:
|
||||
batch_failed = 0
|
||||
batch_skipped = 0
|
||||
batch_labels: list[str] = []
|
||||
|
||||
runnable_subtasks: list[dict[str, object]] = []
|
||||
for subtask in batch:
|
||||
index = int(subtask.get('_delegate_index', len(child_summaries) + 1))
|
||||
subtask_label = str(subtask.get('label') or f'subtask_{index}')
|
||||
@@ -3132,160 +3138,84 @@ class LocalCodingAgent:
|
||||
stop_processing = True
|
||||
break
|
||||
continue
|
||||
# Use agent definition's system prompt if available
|
||||
child_system_prompt = agent_def.system_prompt or self.custom_system_prompt
|
||||
child_override_prompt = None
|
||||
if agent_def.system_prompt:
|
||||
child_override_prompt = agent_def.system_prompt
|
||||
else:
|
||||
child_override_prompt = self.override_system_prompt
|
||||
runnable_subtasks.append(subtask)
|
||||
|
||||
# Inject critical system reminder if agent definition has one
|
||||
child_append_prompt = self.append_system_prompt
|
||||
if agent_def.critical_system_reminder:
|
||||
reminder = f'\n\n<system-reminder>\n{agent_def.critical_system_reminder}\n</system-reminder>'
|
||||
child_append_prompt = (
|
||||
(child_append_prompt or '') + reminder
|
||||
)
|
||||
if stop_processing:
|
||||
break
|
||||
|
||||
child_agent = LocalCodingAgent(
|
||||
model_config=child_model_config,
|
||||
runtime_config=replace(
|
||||
child_runtime_config,
|
||||
max_turns=subtask.get('max_turns', child_runtime_config.max_turns),
|
||||
disable_claude_md_discovery=agent_def.omit_claude_md,
|
||||
),
|
||||
custom_system_prompt=child_system_prompt if not child_override_prompt else None,
|
||||
append_system_prompt=child_append_prompt,
|
||||
override_system_prompt=child_override_prompt,
|
||||
tool_registry=child_tools,
|
||||
agent_manager=self.agent_manager,
|
||||
parent_agent_id=self.managed_agent_id,
|
||||
managed_group_id=group_id,
|
||||
managed_child_index=index,
|
||||
managed_label=subtask_label,
|
||||
if use_parallel and len(runnable_subtasks) > 1:
|
||||
batch_results = self._run_batch_parallel(
|
||||
runnable_subtasks,
|
||||
agent_def=agent_def,
|
||||
child_model_config=child_model_config,
|
||||
child_runtime_config=child_runtime_config,
|
||||
child_tools=child_tools,
|
||||
group_id=group_id,
|
||||
batch_index=batch_index,
|
||||
include_parent_context=include_parent_context,
|
||||
prior_results=prior_results,
|
||||
delegate_preflight_messages=delegate_preflight_messages,
|
||||
max_concurrency=max_concurrency,
|
||||
)
|
||||
if self.tool_context.jupyter_runtime is not None:
|
||||
child_agent.tool_context = replace(
|
||||
child_agent.tool_context,
|
||||
jupyter_runtime=self.tool_context.jupyter_runtime,
|
||||
for br in batch_results:
|
||||
child_result = br['result']
|
||||
summary = br['summary']
|
||||
child_summaries.append(summary)
|
||||
if child_result.session_id:
|
||||
child_session_ids.append(child_result.session_id)
|
||||
prior_results.append(
|
||||
{
|
||||
'label': summary['label'],
|
||||
'output_preview': str(summary['output_preview']),
|
||||
}
|
||||
)
|
||||
if group_id is not None and child_agent.managed_agent_id is not None:
|
||||
self.agent_manager.register_group_child(
|
||||
group_id,
|
||||
child_agent.managed_agent_id,
|
||||
child_index=index,
|
||||
)
|
||||
resume_session_id = subtask.get('resume_session_id')
|
||||
child_prompt = str(subtask['prompt'])
|
||||
if agent_def.initial_prompt and not (
|
||||
isinstance(resume_session_id, str) and resume_session_id
|
||||
):
|
||||
child_prompt = f'{agent_def.initial_prompt.strip()}\n\n{child_prompt}'.strip()
|
||||
if delegate_preflight_messages:
|
||||
child_prompt = self._prepend_plugin_delegate_context(
|
||||
child_prompt,
|
||||
delegate_preflight_messages,
|
||||
)
|
||||
if include_parent_context and prior_results:
|
||||
child_prompt = self._prepend_delegate_context(child_prompt, prior_results)
|
||||
resume_used = False
|
||||
if isinstance(resume_session_id, str) and resume_session_id:
|
||||
try:
|
||||
stored_child_session = load_agent_session(
|
||||
resume_session_id,
|
||||
directory=child_runtime_config.session_directory,
|
||||
)
|
||||
except OSError:
|
||||
child_result = AgentRunResult(
|
||||
final_output=f'Unable to load delegated session {resume_session_id}.',
|
||||
turns=0,
|
||||
tool_calls=0,
|
||||
transcript=(),
|
||||
stop_reason='resume_load_error',
|
||||
session_id=resume_session_id,
|
||||
)
|
||||
if child_result.stop_reason in {'backend_error', 'budget_exceeded'}:
|
||||
failed_children += 1
|
||||
batch_failed += 1
|
||||
summary = {
|
||||
'index': index,
|
||||
'label': subtask_label,
|
||||
'session_id': resume_session_id,
|
||||
'turns': child_result.turns,
|
||||
'tool_calls': child_result.tool_calls,
|
||||
'stop_reason': child_result.stop_reason or 'resume_load_error',
|
||||
'output_preview': self._preview_text(child_result.final_output, 220),
|
||||
'resume_used': True,
|
||||
'resumed_from_session_id': resume_session_id,
|
||||
'depends_on': list(dependencies),
|
||||
'batch_index': batch_index,
|
||||
failed_labels.add(str(summary['label']))
|
||||
else:
|
||||
batch_completed += 1
|
||||
completed_labels.add(str(summary['label']))
|
||||
if isinstance(max_failures, int) and failed_children > max_failures:
|
||||
stop_processing = True
|
||||
else:
|
||||
for subtask in runnable_subtasks:
|
||||
result_info = self._run_single_subtask(
|
||||
subtask,
|
||||
agent_def=agent_def,
|
||||
child_model_config=child_model_config,
|
||||
child_runtime_config=child_runtime_config,
|
||||
child_tools=child_tools,
|
||||
group_id=group_id,
|
||||
batch_index=batch_index,
|
||||
include_parent_context=include_parent_context,
|
||||
prior_results=prior_results,
|
||||
delegate_preflight_messages=delegate_preflight_messages,
|
||||
)
|
||||
child_result = result_info['result']
|
||||
summary = result_info['summary']
|
||||
child_summaries.append(summary)
|
||||
if child_result.session_id:
|
||||
child_session_ids.append(child_result.session_id)
|
||||
prior_results.append(
|
||||
{
|
||||
'label': summary['label'],
|
||||
'output_preview': str(summary['output_preview']),
|
||||
}
|
||||
child_summaries.append(summary)
|
||||
prior_results.append(
|
||||
{
|
||||
'label': summary['label'],
|
||||
'output_preview': str(summary['output_preview']),
|
||||
}
|
||||
)
|
||||
failed_labels.add(subtask_label)
|
||||
)
|
||||
if child_result.stop_reason in {'backend_error', 'budget_exceeded'}:
|
||||
failed_children += 1
|
||||
batch_failed += 1
|
||||
failed_labels.add(str(summary['label']))
|
||||
if isinstance(max_failures, int) and failed_children > max_failures:
|
||||
stop_processing = True
|
||||
break
|
||||
if not continue_on_error:
|
||||
stop_processing = True
|
||||
break
|
||||
continue
|
||||
child_result = child_agent.resume(child_prompt, stored_child_session)
|
||||
_log_child_skill_calls(child_result, child_agent, subtask_label)
|
||||
resume_used = True
|
||||
else:
|
||||
child_result = child_agent.run(child_prompt)
|
||||
_log_child_skill_calls(child_result, child_agent, subtask_label)
|
||||
if group_id is not None and child_agent.managed_agent_id is not None:
|
||||
self.agent_manager.register_group_child(
|
||||
group_id,
|
||||
child_agent.managed_agent_id,
|
||||
child_index=index,
|
||||
)
|
||||
summary = {
|
||||
'index': index,
|
||||
'label': subtask_label,
|
||||
'session_id': child_result.session_id or '',
|
||||
'turns': child_result.turns,
|
||||
'tool_calls': child_result.tool_calls,
|
||||
'stop_reason': child_result.stop_reason or 'stop',
|
||||
'output_preview': self._preview_text(child_result.final_output, 220),
|
||||
'resume_used': resume_used,
|
||||
'resumed_from_session_id': (
|
||||
str(resume_session_id)
|
||||
if isinstance(resume_session_id, str) and resume_session_id
|
||||
else ''
|
||||
),
|
||||
'depends_on': list(dependencies),
|
||||
'batch_index': batch_index,
|
||||
}
|
||||
child_summaries.append(summary)
|
||||
if child_result.session_id:
|
||||
child_session_ids.append(child_result.session_id)
|
||||
prior_results.append(
|
||||
{
|
||||
'label': summary['label'],
|
||||
'output_preview': str(summary['output_preview']),
|
||||
}
|
||||
)
|
||||
if child_result.stop_reason in {'backend_error', 'budget_exceeded'}:
|
||||
failed_children += 1
|
||||
batch_failed += 1
|
||||
failed_labels.add(subtask_label)
|
||||
if isinstance(max_failures, int) and failed_children > max_failures:
|
||||
stop_processing = True
|
||||
break
|
||||
if not continue_on_error:
|
||||
stop_processing = True
|
||||
break
|
||||
else:
|
||||
batch_completed += 1
|
||||
completed_labels.add(subtask_label)
|
||||
else:
|
||||
batch_completed += 1
|
||||
completed_labels.add(str(summary['label']))
|
||||
batch_status = 'completed'
|
||||
if batch_failed and batch_completed:
|
||||
batch_status = 'partial'
|
||||
@@ -3466,6 +3396,173 @@ class LocalCodingAgent:
|
||||
for index, task in enumerate(subtasks[:8], start=1)
|
||||
]
|
||||
|
||||
def _run_single_subtask(
|
||||
self,
|
||||
subtask: dict[str, object],
|
||||
*,
|
||||
agent_def: 'AgentDefinition',
|
||||
child_model_config: 'ModelConfig',
|
||||
child_runtime_config: 'AgentRuntimeConfig',
|
||||
child_tools: dict[str, 'AgentTool'],
|
||||
group_id: str | None,
|
||||
batch_index: int,
|
||||
include_parent_context: bool,
|
||||
prior_results: list[dict[str, str]],
|
||||
delegate_preflight_messages: tuple[str, ...],
|
||||
) -> dict[str, object]:
|
||||
"""Run a single subtask and return {'result': AgentRunResult, 'summary': dict}."""
|
||||
index = int(subtask.get('_delegate_index', 0))
|
||||
subtask_label = str(subtask.get('label') or f'subtask_{index}')
|
||||
dependencies = tuple(
|
||||
item for item in subtask.get('depends_on', ()) if isinstance(item, str) and item
|
||||
)
|
||||
|
||||
child_system_prompt = agent_def.system_prompt or self.custom_system_prompt
|
||||
child_override_prompt = None
|
||||
if agent_def.system_prompt:
|
||||
child_override_prompt = agent_def.system_prompt
|
||||
else:
|
||||
child_override_prompt = self.override_system_prompt
|
||||
|
||||
child_append_prompt = self.append_system_prompt
|
||||
if agent_def.critical_system_reminder:
|
||||
reminder = f'\n\n<system-reminder>\n{agent_def.critical_system_reminder}\n</system-reminder>'
|
||||
child_append_prompt = (child_append_prompt or '') + reminder
|
||||
|
||||
child_agent = LocalCodingAgent(
|
||||
model_config=child_model_config,
|
||||
runtime_config=replace(
|
||||
child_runtime_config,
|
||||
max_turns=subtask.get('max_turns', child_runtime_config.max_turns),
|
||||
disable_claude_md_discovery=agent_def.omit_claude_md,
|
||||
),
|
||||
custom_system_prompt=child_system_prompt if not child_override_prompt else None,
|
||||
append_system_prompt=child_append_prompt,
|
||||
override_system_prompt=child_override_prompt,
|
||||
tool_registry=child_tools,
|
||||
agent_manager=self.agent_manager,
|
||||
parent_agent_id=self.managed_agent_id,
|
||||
managed_group_id=group_id,
|
||||
managed_child_index=index,
|
||||
managed_label=subtask_label,
|
||||
)
|
||||
if self.tool_context.jupyter_runtime is not None:
|
||||
child_agent.tool_context = replace(
|
||||
child_agent.tool_context,
|
||||
jupyter_runtime=self.tool_context.jupyter_runtime,
|
||||
)
|
||||
if group_id is not None and child_agent.managed_agent_id is not None:
|
||||
self.agent_manager.register_group_child(
|
||||
group_id,
|
||||
child_agent.managed_agent_id,
|
||||
child_index=index,
|
||||
)
|
||||
resume_session_id = subtask.get('resume_session_id')
|
||||
child_prompt = str(subtask['prompt'])
|
||||
if agent_def.initial_prompt and not (
|
||||
isinstance(resume_session_id, str) and resume_session_id
|
||||
):
|
||||
child_prompt = f'{agent_def.initial_prompt.strip()}\n\n{child_prompt}'.strip()
|
||||
if delegate_preflight_messages:
|
||||
child_prompt = self._prepend_plugin_delegate_context(
|
||||
child_prompt, delegate_preflight_messages,
|
||||
)
|
||||
if include_parent_context and prior_results:
|
||||
child_prompt = self._prepend_delegate_context(child_prompt, prior_results)
|
||||
resume_used = False
|
||||
if isinstance(resume_session_id, str) and resume_session_id:
|
||||
try:
|
||||
stored_child_session = load_agent_session(
|
||||
resume_session_id,
|
||||
directory=child_runtime_config.session_directory,
|
||||
)
|
||||
except OSError:
|
||||
child_result = AgentRunResult(
|
||||
final_output=f'Unable to load delegated session {resume_session_id}.',
|
||||
turns=0, tool_calls=0, transcript=(),
|
||||
stop_reason='resume_load_error', session_id=resume_session_id,
|
||||
)
|
||||
return {
|
||||
'result': child_result,
|
||||
'summary': {
|
||||
'index': index, 'label': subtask_label,
|
||||
'session_id': resume_session_id,
|
||||
'turns': 0, 'tool_calls': 0,
|
||||
'stop_reason': 'resume_load_error',
|
||||
'output_preview': self._preview_text(child_result.final_output, 220),
|
||||
'resume_used': True, 'resumed_from_session_id': resume_session_id,
|
||||
'depends_on': list(dependencies), 'batch_index': batch_index,
|
||||
},
|
||||
}
|
||||
child_result = child_agent.resume(child_prompt, stored_child_session)
|
||||
_log_child_skill_calls(child_result, child_agent, subtask_label)
|
||||
resume_used = True
|
||||
else:
|
||||
child_result = child_agent.run(child_prompt)
|
||||
_log_child_skill_calls(child_result, child_agent, subtask_label)
|
||||
if group_id is not None and child_agent.managed_agent_id is not None:
|
||||
self.agent_manager.register_group_child(
|
||||
group_id, child_agent.managed_agent_id, child_index=index,
|
||||
)
|
||||
summary = {
|
||||
'index': index, 'label': subtask_label,
|
||||
'session_id': child_result.session_id or '',
|
||||
'turns': child_result.turns, 'tool_calls': child_result.tool_calls,
|
||||
'stop_reason': child_result.stop_reason or 'stop',
|
||||
'output_preview': self._preview_text(child_result.final_output, 220),
|
||||
'resume_used': resume_used,
|
||||
'resumed_from_session_id': (
|
||||
str(resume_session_id) if isinstance(resume_session_id, str) and resume_session_id else ''
|
||||
),
|
||||
'depends_on': list(dependencies), 'batch_index': batch_index,
|
||||
}
|
||||
return {'result': child_result, 'summary': summary}
|
||||
|
||||
def _run_batch_parallel(
|
||||
self,
|
||||
subtasks: list[dict[str, object]],
|
||||
*,
|
||||
agent_def: 'AgentDefinition',
|
||||
child_model_config: 'ModelConfig',
|
||||
child_runtime_config: 'AgentRuntimeConfig',
|
||||
child_tools: dict[str, 'AgentTool'],
|
||||
group_id: str | None,
|
||||
batch_index: int,
|
||||
include_parent_context: bool,
|
||||
prior_results: list[dict[str, str]],
|
||||
delegate_preflight_messages: tuple[str, ...],
|
||||
max_concurrency: int = 4,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Run subtasks in a batch concurrently using threads. Returns results in original order."""
|
||||
results: list[dict[str, object] | None] = [None] * len(subtasks)
|
||||
lock = threading.Lock()
|
||||
|
||||
def _worker(idx: int, subtask: dict[str, object]) -> None:
|
||||
result_info = self._run_single_subtask(
|
||||
subtask,
|
||||
agent_def=agent_def,
|
||||
child_model_config=child_model_config,
|
||||
child_runtime_config=child_runtime_config,
|
||||
child_tools=child_tools,
|
||||
group_id=group_id,
|
||||
batch_index=batch_index,
|
||||
include_parent_context=include_parent_context,
|
||||
prior_results=prior_results,
|
||||
delegate_preflight_messages=delegate_preflight_messages,
|
||||
)
|
||||
with lock:
|
||||
results[idx] = result_info
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(max_concurrency, len(subtasks))) as executor:
|
||||
futures = {
|
||||
executor.submit(_worker, idx, subtask): idx
|
||||
for idx, subtask in enumerate(subtasks)
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
future.result()
|
||||
|
||||
return [r for r in results if r is not None]
|
||||
|
||||
def _normalize_delegate_strategy(self, strategy: object) -> str:
|
||||
if not isinstance(strategy, str) or not strategy.strip():
|
||||
return 'serial'
|
||||
|
||||
Reference in New Issue
Block a user