wuyang6
944 lines
36 KiB
Python
944 lines
36 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
from typing import Any, Iterator
|
||
from urllib import error, request
|
||
|
||
from .agent_types import (
|
||
AssistantTurn,
|
||
ModelConfig,
|
||
OutputSchemaConfig,
|
||
StreamEvent,
|
||
ToolCall,
|
||
UsageStats,
|
||
)
|
||
|
||
|
||
class OpenAICompatError(RuntimeError):
|
||
"""Raised when the local OpenAI-compatible backend returns an invalid response."""
|
||
|
||
|
||
PROVIDER_MIN_TEMPERATURES = {
|
||
# 实测这些 provider 会拒绝 temperature=0；这里做最小值兜底，
|
||
# 保持默认确定性取向的同时避免模型切换后请求直接 400。
|
||
'minimax': 0.01,
|
||
'wenxin': 0.1,
|
||
}
|
||
|
||
ANTHROPIC_MESSAGES_MODEL_PREFIXES = (
|
||
'ppio/pa/claude-',
|
||
)
|
||
|
||
ANTHROPIC_VERSION = '2023-06-01'
|
||
ANTHROPIC_MAX_TOKENS = 4096
|
||
DEFAULT_MODEL_IDLE_TIMEOUT_SECONDS = 60.0
|
||
|
||
|
||
def _join_url(base_url: str, suffix: str) -> str:
|
||
base = base_url.rstrip('/')
|
||
return f'{base}/{suffix.lstrip("/")}'
|
||
|
||
|
||
def _normalize_content(content: Any) -> str:
|
||
if content is None:
|
||
return ''
|
||
if isinstance(content, str):
|
||
return content
|
||
if isinstance(content, list):
|
||
parts: list[str] = []
|
||
for item in content:
|
||
if isinstance(item, str):
|
||
parts.append(item)
|
||
continue
|
||
if not isinstance(item, dict):
|
||
parts.append(str(item))
|
||
continue
|
||
if item.get('type') == 'text' and isinstance(item.get('text'), str):
|
||
parts.append(item['text'])
|
||
continue
|
||
if isinstance(item.get('text'), str):
|
||
parts.append(item['text'])
|
||
continue
|
||
parts.append(json.dumps(item, ensure_ascii=True))
|
||
return ''.join(parts)
|
||
return str(content)
|
||
|
||
|
||
def _parse_tool_arguments(raw_arguments: Any) -> dict[str, Any]:
|
||
if raw_arguments is None:
|
||
return {}
|
||
if isinstance(raw_arguments, dict):
|
||
return raw_arguments
|
||
if isinstance(raw_arguments, str):
|
||
raw_arguments = raw_arguments.strip()
|
||
if not raw_arguments:
|
||
return {}
|
||
try:
|
||
parsed = json.loads(raw_arguments)
|
||
except json.JSONDecodeError as exc:
|
||
preview = raw_arguments[:240].replace('\n', '\\n')
|
||
raise OpenAICompatError(
|
||
'Invalid tool arguments returned by model: '
|
||
f'line {exc.lineno} column {exc.colno}: {exc.msg}; '
|
||
f'preview={preview!r}'
|
||
) from exc
|
||
if not isinstance(parsed, dict):
|
||
raise OpenAICompatError(
|
||
f'Tool arguments must decode to an object, got {type(parsed).__name__}'
|
||
)
|
||
return parsed
|
||
raise OpenAICompatError(
|
||
f'Unsupported tool arguments payload: {type(raw_arguments).__name__}'
|
||
)
|
||
|
||
|
||
def _tool_call_ids(message: dict[str, Any]) -> list[str]:
|
||
raw_tool_calls = message.get('tool_calls')
|
||
if not isinstance(raw_tool_calls, list):
|
||
return []
|
||
ids: list[str] = []
|
||
for index, raw_call in enumerate(raw_tool_calls):
|
||
if not isinstance(raw_call, dict):
|
||
continue
|
||
call_id = raw_call.get('id')
|
||
ids.append(call_id if isinstance(call_id, str) and call_id else f'call_{index}')
|
||
return ids
|
||
|
||
|
||
def _contiguous_tool_result_ids(
|
||
messages: list[dict[str, Any]],
|
||
start_index: int,
|
||
) -> set[str]:
|
||
ids: set[str] = set()
|
||
index = start_index
|
||
while index < len(messages) and messages[index].get('role') == 'tool':
|
||
tool_call_id = messages[index].get('tool_call_id')
|
||
if isinstance(tool_call_id, str) and tool_call_id:
|
||
ids.add(tool_call_id)
|
||
index += 1
|
||
return ids
|
||
|
||
|
||
def _filter_tool_calls_by_ids(
|
||
message: dict[str, Any],
|
||
valid_ids: set[str],
|
||
) -> dict[str, Any]:
|
||
raw_tool_calls = message.get('tool_calls')
|
||
if not isinstance(raw_tool_calls, list):
|
||
return message
|
||
filtered: list[dict[str, Any]] = []
|
||
for index, raw_call in enumerate(raw_tool_calls):
|
||
if not isinstance(raw_call, dict):
|
||
continue
|
||
call_id = raw_call.get('id')
|
||
normalized_id = (
|
||
call_id if isinstance(call_id, str) and call_id else f'call_{index}'
|
||
)
|
||
if normalized_id in valid_ids:
|
||
filtered.append(raw_call)
|
||
if len(filtered) == len(raw_tool_calls):
|
||
return message
|
||
next_message = dict(message)
|
||
if filtered:
|
||
next_message['tool_calls'] = filtered
|
||
else:
|
||
next_message.pop('tool_calls', None)
|
||
return next_message
|
||
|
||
|
||
def _sanitize_anthropic_tool_messages(
|
||
messages: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
"""清理 Anthropic/Bedrock 严格要求的 tool_use/tool_result 邻接关系。
|
||
|
||
OpenAI 风格历史在异常中断、裁剪或前端恢复时可能留下 assistant tool_calls，
|
||
但下一条消息不再紧跟对应 tool 结果。Anthropic Messages API 会直接拒绝
|
||
这种历史。这里在出网关前删除缺失结果的 tool_call，并跳过没有前置
|
||
tool_use 的孤儿 tool 结果，保留普通文本上下文继续对话。
|
||
"""
|
||
sanitized: list[dict[str, Any]] = []
|
||
pending_tool_ids: set[str] = set()
|
||
|
||
for index, message in enumerate(messages):
|
||
role = message.get('role')
|
||
if role == 'assistant':
|
||
tool_ids = _tool_call_ids(message)
|
||
if tool_ids:
|
||
result_ids = _contiguous_tool_result_ids(messages, index + 1)
|
||
valid_ids = set(tool_ids).intersection(result_ids)
|
||
message = _filter_tool_calls_by_ids(message, valid_ids)
|
||
pending_tool_ids = valid_ids
|
||
else:
|
||
pending_tool_ids = set()
|
||
sanitized.append(message)
|
||
continue
|
||
|
||
if role == 'tool':
|
||
tool_call_id = message.get('tool_call_id')
|
||
if isinstance(tool_call_id, str) and tool_call_id in pending_tool_ids:
|
||
sanitized.append(message)
|
||
continue
|
||
|
||
pending_tool_ids = set()
|
||
sanitized.append(message)
|
||
|
||
return sanitized
|
||
|
||
|
||
def _optional_int(value: Any) -> int:
|
||
if isinstance(value, bool):
|
||
return 0
|
||
if isinstance(value, int):
|
||
return value
|
||
if isinstance(value, float):
|
||
return int(value)
|
||
if isinstance(value, str):
|
||
try:
|
||
return int(value)
|
||
except ValueError:
|
||
return 0
|
||
return 0
|
||
|
||
|
||
def _temperature_for_model(model: str, configured: float) -> float:
|
||
provider = model.split('/', 1)[0].strip().lower() if '/' in model else ''
|
||
minimum = PROVIDER_MIN_TEMPERATURES.get(provider)
|
||
if minimum is None:
|
||
return configured
|
||
return max(configured, minimum)
|
||
|
||
|
||
def _uses_anthropic_messages_api(model: str, base_url: str) -> bool:
|
||
normalized_model = model.strip().lower()
|
||
normalized_base = base_url.rstrip('/').lower()
|
||
return normalized_base.endswith('/anthropic') or any(
|
||
normalized_model.startswith(prefix)
|
||
for prefix in ANTHROPIC_MESSAGES_MODEL_PREFIXES
|
||
)
|
||
|
||
|
||
def _anthropic_base_url(base_url: str) -> str:
|
||
base = base_url.rstrip('/')
|
||
if base.lower().endswith('/anthropic'):
|
||
return base
|
||
if base.lower().endswith('/v1'):
|
||
return f'{base[:-3]}/anthropic'
|
||
return f'{base}/anthropic'
|
||
|
||
|
||
def _parse_usage(payload: Any) -> UsageStats:
|
||
if not isinstance(payload, dict):
|
||
return UsageStats()
|
||
completion_details = payload.get('completion_tokens_details')
|
||
if not isinstance(completion_details, dict):
|
||
completion_details = {}
|
||
return UsageStats(
|
||
input_tokens=(
|
||
_optional_int(payload.get('input_tokens'))
|
||
or _optional_int(payload.get('prompt_tokens'))
|
||
or _optional_int(payload.get('prompt_eval_count'))
|
||
),
|
||
output_tokens=(
|
||
_optional_int(payload.get('output_tokens'))
|
||
or _optional_int(payload.get('completion_tokens'))
|
||
or _optional_int(payload.get('eval_count'))
|
||
),
|
||
cache_creation_input_tokens=_optional_int(
|
||
payload.get('cache_creation_input_tokens')
|
||
),
|
||
cache_read_input_tokens=_optional_int(payload.get('cache_read_input_tokens')),
|
||
reasoning_tokens=(
|
||
_optional_int(payload.get('reasoning_tokens'))
|
||
or _optional_int(completion_details.get('reasoning_tokens'))
|
||
),
|
||
)
|
||
|
||
|
||
def _build_response_format(
|
||
schema: OutputSchemaConfig | None,
|
||
) -> dict[str, Any] | None:
|
||
if schema is None:
|
||
return None
|
||
return {
|
||
'type': 'json_schema',
|
||
'json_schema': {
|
||
'name': schema.name,
|
||
'schema': schema.schema,
|
||
'strict': schema.strict,
|
||
},
|
||
}
|
||
|
||
|
||
class OpenAICompatClient:
|
||
"""Minimal OpenAI-compatible chat client for local model servers."""
|
||
|
||
def __init__(self, config: ModelConfig) -> None:
|
||
self.config = config
|
||
|
||
def _request_timeout_seconds(self) -> float:
|
||
"""Return per-socket model timeout.
|
||
|
||
`ModelConfig.timeout_seconds` is the whole request budget used by the
|
||
product, but urllib applies it as an idle socket timeout. Keeping it at
|
||
one hour makes a bad upstream stream hold a session lock for an hour.
|
||
Use a shorter idle timeout by default; any normally streaming response
|
||
keeps extending this naturally because bytes continue to arrive.
|
||
"""
|
||
raw = os.environ.get('CLAW_MODEL_IDLE_TIMEOUT_SECONDS', '').strip()
|
||
try:
|
||
configured = float(raw) if raw else DEFAULT_MODEL_IDLE_TIMEOUT_SECONDS
|
||
except ValueError:
|
||
configured = DEFAULT_MODEL_IDLE_TIMEOUT_SECONDS
|
||
configured = max(5.0, configured)
|
||
return min(float(self.config.timeout_seconds), configured)
|
||
|
||
def complete(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
tools: list[dict[str, Any]],
|
||
*,
|
||
output_schema: OutputSchemaConfig | None = None,
|
||
) -> AssistantTurn:
|
||
if self._uses_anthropic_messages_api():
|
||
return self._complete_anthropic_messages(
|
||
messages=messages,
|
||
tools=tools,
|
||
output_schema=output_schema,
|
||
)
|
||
payload = self._request_json(
|
||
self._build_payload(
|
||
messages=messages,
|
||
tools=tools,
|
||
stream=False,
|
||
output_schema=output_schema,
|
||
)
|
||
)
|
||
choices = payload.get('choices')
|
||
if not isinstance(choices, list) or not choices:
|
||
raise OpenAICompatError('Local model backend returned no choices')
|
||
first_choice = choices[0]
|
||
if not isinstance(first_choice, dict):
|
||
raise OpenAICompatError('Local model backend returned malformed choice data')
|
||
|
||
message = first_choice.get('message')
|
||
if not isinstance(message, dict):
|
||
raise OpenAICompatError('Local model backend returned no assistant message')
|
||
|
||
content = _normalize_content(message.get('content'))
|
||
tool_calls = self._parse_tool_calls_from_message(message)
|
||
|
||
finish_reason = first_choice.get('finish_reason')
|
||
if finish_reason is not None and not isinstance(finish_reason, str):
|
||
finish_reason = str(finish_reason)
|
||
|
||
return AssistantTurn(
|
||
content=content,
|
||
tool_calls=tuple(tool_calls),
|
||
finish_reason=finish_reason,
|
||
raw_message=message,
|
||
usage=_parse_usage(payload.get('usage')),
|
||
)
|
||
|
||
def stream(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
tools: list[dict[str, Any]],
|
||
*,
|
||
output_schema: OutputSchemaConfig | None = None,
|
||
) -> Iterator[StreamEvent]:
|
||
if self._uses_anthropic_messages_api():
|
||
yield from self._stream_anthropic_messages(
|
||
messages=messages,
|
||
tools=tools,
|
||
output_schema=output_schema,
|
||
)
|
||
return
|
||
payload = self._build_payload(
|
||
messages=messages,
|
||
tools=tools,
|
||
stream=True,
|
||
output_schema=output_schema,
|
||
)
|
||
req = request.Request(
|
||
_join_url(self.config.base_url, '/chat/completions'),
|
||
data=json.dumps(payload).encode('utf-8'),
|
||
headers={
|
||
'Authorization': f'Bearer {self.config.api_key}',
|
||
'Content-Type': 'application/json',
|
||
},
|
||
method='POST',
|
||
)
|
||
try:
|
||
with request.urlopen(req, timeout=self._request_timeout_seconds()) as response:
|
||
yield StreamEvent(type='message_start')
|
||
for event_payload in self._iter_sse_payloads(response):
|
||
yield from self._parse_stream_payload(event_payload)
|
||
except error.HTTPError as exc:
|
||
detail = exc.read().decode('utf-8', errors='replace')
|
||
raise OpenAICompatError(
|
||
f'HTTP {exc.code} from local model backend: {detail}'
|
||
) from exc
|
||
except error.URLError as exc:
|
||
raise OpenAICompatError(
|
||
f'Unable to reach local model backend at {self.config.base_url}: {exc.reason}'
|
||
) from exc
|
||
|
||
def _uses_anthropic_messages_api(self) -> bool:
|
||
return _uses_anthropic_messages_api(self.config.model, self.config.base_url)
|
||
|
||
def _request_json(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
body = json.dumps(payload).encode('utf-8')
|
||
req = request.Request(
|
||
_join_url(self.config.base_url, '/chat/completions'),
|
||
data=body,
|
||
headers={
|
||
'Authorization': f'Bearer {self.config.api_key}',
|
||
'Content-Type': 'application/json',
|
||
},
|
||
method='POST',
|
||
)
|
||
try:
|
||
with request.urlopen(req, timeout=self._request_timeout_seconds()) as response:
|
||
raw = response.read()
|
||
except error.HTTPError as exc:
|
||
detail = exc.read().decode('utf-8', errors='replace')
|
||
raise OpenAICompatError(
|
||
f'HTTP {exc.code} from local model backend: {detail}'
|
||
) from exc
|
||
except error.URLError as exc:
|
||
raise OpenAICompatError(
|
||
f'Unable to reach local model backend at {self.config.base_url}: {exc.reason}'
|
||
) from exc
|
||
|
||
try:
|
||
payload = json.loads(raw.decode('utf-8'))
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenAICompatError('Local model backend returned invalid JSON') from exc
|
||
if not isinstance(payload, dict):
|
||
raise OpenAICompatError('Local model backend returned malformed JSON payload')
|
||
return payload
|
||
|
||
def _build_payload(
|
||
self,
|
||
*,
|
||
messages: list[dict[str, Any]],
|
||
tools: list[dict[str, Any]],
|
||
stream: bool,
|
||
output_schema: OutputSchemaConfig | None,
|
||
) -> dict[str, Any]:
|
||
payload: dict[str, Any] = {
|
||
'model': self.config.model,
|
||
'messages': messages,
|
||
'temperature': _temperature_for_model(self.config.model, self.config.temperature),
|
||
'stream': stream,
|
||
}
|
||
if tools:
|
||
payload['tools'] = tools
|
||
payload['tool_choice'] = 'auto'
|
||
if stream:
|
||
payload['stream_options'] = {'include_usage': True}
|
||
response_format = _build_response_format(output_schema)
|
||
if response_format is not None:
|
||
payload['response_format'] = response_format
|
||
return payload
|
||
|
||
def _build_anthropic_payload(
|
||
self,
|
||
*,
|
||
messages: list[dict[str, Any]],
|
||
tools: list[dict[str, Any]],
|
||
stream: bool,
|
||
output_schema: OutputSchemaConfig | None,
|
||
) -> dict[str, Any]:
|
||
system_parts: list[str] = []
|
||
anthropic_messages: list[dict[str, Any]] = []
|
||
for message in _sanitize_anthropic_tool_messages(messages):
|
||
role = message.get('role')
|
||
if role == 'system':
|
||
content = _normalize_content(message.get('content')).strip()
|
||
if content:
|
||
system_parts.append(content)
|
||
continue
|
||
if role == 'assistant':
|
||
blocks = self._anthropic_assistant_blocks(message)
|
||
if blocks:
|
||
self._append_anthropic_message(anthropic_messages, 'assistant', blocks)
|
||
continue
|
||
if role == 'tool':
|
||
tool_call_id = message.get('tool_call_id')
|
||
if not isinstance(tool_call_id, str) or not tool_call_id:
|
||
tool_call_id = 'toolu_unknown'
|
||
self._append_anthropic_message(
|
||
anthropic_messages,
|
||
'user',
|
||
[
|
||
{
|
||
'type': 'tool_result',
|
||
'tool_use_id': tool_call_id,
|
||
'content': _normalize_content(message.get('content')),
|
||
}
|
||
],
|
||
)
|
||
continue
|
||
if role == 'user':
|
||
blocks = self._anthropic_text_blocks(message.get('content'))
|
||
if blocks:
|
||
self._append_anthropic_message(anthropic_messages, 'user', blocks)
|
||
|
||
payload: dict[str, Any] = {
|
||
'model': self.config.model,
|
||
'messages': anthropic_messages,
|
||
'max_tokens': ANTHROPIC_MAX_TOKENS,
|
||
'stream': stream,
|
||
}
|
||
if system_parts:
|
||
payload['system'] = '\n\n'.join(system_parts)
|
||
converted_tools = self._anthropic_tools(tools)
|
||
if converted_tools:
|
||
payload['tools'] = converted_tools
|
||
if output_schema is not None:
|
||
schema_hint = (
|
||
f'请严格输出符合 JSON Schema `{output_schema.name}` 的 JSON，'
|
||
'不要输出额外解释。'
|
||
)
|
||
payload['system'] = (
|
||
f'{payload.get("system", "")}\n\n{schema_hint}'.strip()
|
||
)
|
||
return payload
|
||
|
||
def _anthropic_headers(self) -> dict[str, str]:
|
||
return {
|
||
'Authorization': f'Bearer {self.config.api_key}',
|
||
'Content-Type': 'application/json',
|
||
'anthropic-version': ANTHROPIC_VERSION,
|
||
'api-key': self.config.api_key,
|
||
'x-api-key': self.config.api_key,
|
||
}
|
||
|
||
def _anthropic_messages_url(self) -> str:
|
||
return _join_url(_anthropic_base_url(self.config.base_url), '/v1/messages')
|
||
|
||
def _complete_anthropic_messages(
|
||
self,
|
||
*,
|
||
messages: list[dict[str, Any]],
|
||
tools: list[dict[str, Any]],
|
||
output_schema: OutputSchemaConfig | None,
|
||
) -> AssistantTurn:
|
||
payload = self._build_anthropic_payload(
|
||
messages=messages,
|
||
tools=tools,
|
||
stream=False,
|
||
output_schema=output_schema,
|
||
)
|
||
body = json.dumps(payload).encode('utf-8')
|
||
req = request.Request(
|
||
self._anthropic_messages_url(),
|
||
data=body,
|
||
headers=self._anthropic_headers(),
|
||
method='POST',
|
||
)
|
||
try:
|
||
with request.urlopen(req, timeout=self._request_timeout_seconds()) as response:
|
||
raw = response.read()
|
||
except error.HTTPError as exc:
|
||
detail = exc.read().decode('utf-8', errors='replace')
|
||
raise OpenAICompatError(
|
||
f'HTTP {exc.code} from local model backend: {detail}'
|
||
) from exc
|
||
except error.URLError as exc:
|
||
raise OpenAICompatError(
|
||
f'Unable to reach local model backend at {self.config.base_url}: {exc.reason}'
|
||
) from exc
|
||
|
||
try:
|
||
response_payload = json.loads(raw.decode('utf-8'))
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenAICompatError('Local model backend returned invalid JSON') from exc
|
||
if not isinstance(response_payload, dict):
|
||
raise OpenAICompatError('Local model backend returned malformed JSON payload')
|
||
return self._parse_anthropic_message_response(response_payload)
|
||
|
||
def _stream_anthropic_messages(
|
||
self,
|
||
*,
|
||
messages: list[dict[str, Any]],
|
||
tools: list[dict[str, Any]],
|
||
output_schema: OutputSchemaConfig | None,
|
||
) -> Iterator[StreamEvent]:
|
||
payload = self._build_anthropic_payload(
|
||
messages=messages,
|
||
tools=tools,
|
||
stream=True,
|
||
output_schema=output_schema,
|
||
)
|
||
req = request.Request(
|
||
self._anthropic_messages_url(),
|
||
data=json.dumps(payload).encode('utf-8'),
|
||
headers=self._anthropic_headers(),
|
||
method='POST',
|
||
)
|
||
try:
|
||
with request.urlopen(req, timeout=self._request_timeout_seconds()) as response:
|
||
yield StreamEvent(type='message_start')
|
||
tool_block_indexes: dict[int, int] = {}
|
||
next_tool_index = 0
|
||
finish_reason: str | None = None
|
||
for event_payload in self._iter_sse_payloads(response):
|
||
event_type = event_payload.get('type')
|
||
if event_type == 'message_start':
|
||
message = event_payload.get('message')
|
||
usage = _parse_usage(
|
||
message.get('usage') if isinstance(message, dict) else None
|
||
)
|
||
if usage.total_tokens:
|
||
yield StreamEvent(
|
||
type='usage',
|
||
usage=usage,
|
||
raw_event=event_payload,
|
||
)
|
||
continue
|
||
if event_type == 'content_block_start':
|
||
block_index = event_payload.get('index')
|
||
content_block = event_payload.get('content_block')
|
||
if not isinstance(block_index, int) or not isinstance(content_block, dict):
|
||
continue
|
||
if content_block.get('type') != 'tool_use':
|
||
continue
|
||
tool_index = next_tool_index
|
||
next_tool_index += 1
|
||
tool_block_indexes[block_index] = tool_index
|
||
tool_input = content_block.get('input')
|
||
arguments = (
|
||
json.dumps(tool_input, ensure_ascii=True)
|
||
if isinstance(tool_input, dict) and tool_input
|
||
else ''
|
||
)
|
||
yield StreamEvent(
|
||
type='tool_call_delta',
|
||
tool_call_index=tool_index,
|
||
tool_call_id=(
|
||
content_block.get('id')
|
||
if isinstance(content_block.get('id'), str)
|
||
else None
|
||
),
|
||
tool_name=(
|
||
content_block.get('name')
|
||
if isinstance(content_block.get('name'), str)
|
||
else None
|
||
),
|
||
arguments_delta=arguments,
|
||
raw_event=event_payload,
|
||
)
|
||
continue
|
||
if event_type == 'content_block_delta':
|
||
delta = event_payload.get('delta')
|
||
if not isinstance(delta, dict):
|
||
continue
|
||
if delta.get('type') == 'text_delta':
|
||
text = delta.get('text')
|
||
if isinstance(text, str) and text:
|
||
yield StreamEvent(
|
||
type='content_delta',
|
||
delta=text,
|
||
raw_event=event_payload,
|
||
)
|
||
continue
|
||
if delta.get('type') == 'input_json_delta':
|
||
block_index = event_payload.get('index')
|
||
partial_json = delta.get('partial_json')
|
||
if (
|
||
isinstance(block_index, int)
|
||
and block_index in tool_block_indexes
|
||
and isinstance(partial_json, str)
|
||
and partial_json
|
||
):
|
||
yield StreamEvent(
|
||
type='tool_call_delta',
|
||
tool_call_index=tool_block_indexes[block_index],
|
||
arguments_delta=partial_json,
|
||
raw_event=event_payload,
|
||
)
|
||
continue
|
||
if event_type == 'message_delta':
|
||
delta = event_payload.get('delta')
|
||
if isinstance(delta, dict) and isinstance(delta.get('stop_reason'), str):
|
||
finish_reason = delta['stop_reason']
|
||
usage = _parse_usage(event_payload.get('usage'))
|
||
if usage.total_tokens:
|
||
yield StreamEvent(
|
||
type='usage',
|
||
usage=usage,
|
||
raw_event=event_payload,
|
||
)
|
||
continue
|
||
if event_type == 'message_stop':
|
||
yield StreamEvent(
|
||
type='message_stop',
|
||
finish_reason=finish_reason,
|
||
raw_event=event_payload,
|
||
)
|
||
except error.HTTPError as exc:
|
||
detail = exc.read().decode('utf-8', errors='replace')
|
||
raise OpenAICompatError(
|
||
f'HTTP {exc.code} from local model backend: {detail}'
|
||
) from exc
|
||
except error.URLError as exc:
|
||
raise OpenAICompatError(
|
||
f'Unable to reach local model backend at {self.config.base_url}: {exc.reason}'
|
||
) from exc
|
||
|
||
def _parse_anthropic_message_response(self, payload: dict[str, Any]) -> AssistantTurn:
|
||
raw_content = payload.get('content')
|
||
if not isinstance(raw_content, list):
|
||
raise OpenAICompatError('Anthropic backend returned no content blocks')
|
||
text_parts: list[str] = []
|
||
tool_calls: list[ToolCall] = []
|
||
for index, block in enumerate(raw_content):
|
||
if not isinstance(block, dict):
|
||
continue
|
||
if block.get('type') == 'text':
|
||
text = block.get('text')
|
||
if isinstance(text, str):
|
||
text_parts.append(text)
|
||
continue
|
||
if block.get('type') == 'tool_use':
|
||
name = block.get('name')
|
||
if not isinstance(name, str) or not name:
|
||
raise OpenAICompatError('Tool call missing function name')
|
||
call_id = block.get('id')
|
||
if not isinstance(call_id, str) or not call_id:
|
||
call_id = f'call_{index}'
|
||
arguments = block.get('input')
|
||
if not isinstance(arguments, dict):
|
||
arguments = {}
|
||
tool_calls.append(ToolCall(id=call_id, name=name, arguments=arguments))
|
||
finish_reason = payload.get('stop_reason')
|
||
if finish_reason is not None and not isinstance(finish_reason, str):
|
||
finish_reason = str(finish_reason)
|
||
return AssistantTurn(
|
||
content=''.join(text_parts),
|
||
tool_calls=tuple(tool_calls),
|
||
finish_reason=finish_reason,
|
||
raw_message=payload,
|
||
usage=_parse_usage(payload.get('usage')),
|
||
)
|
||
|
||
def _anthropic_assistant_blocks(self, message: dict[str, Any]) -> list[dict[str, Any]]:
|
||
blocks: list[dict[str, Any]] = []
|
||
content = _normalize_content(message.get('content'))
|
||
if content:
|
||
blocks.append({'type': 'text', 'text': content})
|
||
raw_tool_calls = message.get('tool_calls')
|
||
if isinstance(raw_tool_calls, list):
|
||
for index, raw_call in enumerate(raw_tool_calls):
|
||
if not isinstance(raw_call, dict):
|
||
continue
|
||
function_block = raw_call.get('function')
|
||
if not isinstance(function_block, dict):
|
||
continue
|
||
name = function_block.get('name')
|
||
if not isinstance(name, str) or not name:
|
||
continue
|
||
call_id = raw_call.get('id')
|
||
if not isinstance(call_id, str) or not call_id:
|
||
call_id = f'call_{index}'
|
||
try:
|
||
arguments = _parse_tool_arguments(function_block.get('arguments'))
|
||
except OpenAICompatError:
|
||
arguments = {}
|
||
blocks.append(
|
||
{
|
||
'type': 'tool_use',
|
||
'id': call_id,
|
||
'name': name,
|
||
'input': arguments,
|
||
}
|
||
)
|
||
return blocks
|
||
|
||
def _anthropic_text_blocks(self, content: Any) -> list[dict[str, Any]]:
|
||
normalized = _normalize_content(content)
|
||
if not normalized:
|
||
return []
|
||
return [{'type': 'text', 'text': normalized}]
|
||
|
||
def _anthropic_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
converted: list[dict[str, Any]] = []
|
||
for tool in tools:
|
||
if not isinstance(tool, dict):
|
||
continue
|
||
function_block = tool.get('function')
|
||
if not isinstance(function_block, dict):
|
||
continue
|
||
name = function_block.get('name')
|
||
if not isinstance(name, str) or not name:
|
||
continue
|
||
entry: dict[str, Any] = {
|
||
'name': name,
|
||
'input_schema': function_block.get('parameters') or {'type': 'object'},
|
||
}
|
||
description = function_block.get('description')
|
||
if isinstance(description, str) and description:
|
||
entry['description'] = description
|
||
converted.append(entry)
|
||
return converted
|
||
|
||
def _append_anthropic_message(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
role: str,
|
||
blocks: list[dict[str, Any]],
|
||
) -> None:
|
||
if not blocks:
|
||
return
|
||
if messages and messages[-1].get('role') == role:
|
||
previous = messages[-1].get('content')
|
||
if isinstance(previous, list):
|
||
previous.extend(blocks)
|
||
return
|
||
messages.append({'role': role, 'content': blocks})
|
||
|
||
def _parse_tool_calls_from_message(self, message: dict[str, Any]) -> list[ToolCall]:
|
||
tool_calls: list[ToolCall] = []
|
||
raw_tool_calls = message.get('tool_calls')
|
||
if isinstance(raw_tool_calls, list):
|
||
for idx, raw_call in enumerate(raw_tool_calls):
|
||
if not isinstance(raw_call, dict):
|
||
raise OpenAICompatError('Malformed tool call payload from model')
|
||
function_block = raw_call.get('function') or {}
|
||
if not isinstance(function_block, dict):
|
||
raise OpenAICompatError('Malformed tool call function payload from model')
|
||
name = function_block.get('name')
|
||
if not isinstance(name, str) or not name:
|
||
raise OpenAICompatError('Tool call missing function name')
|
||
call_id = raw_call.get('id')
|
||
if not isinstance(call_id, str) or not call_id:
|
||
call_id = f'call_{idx}'
|
||
arguments = _parse_tool_arguments(function_block.get('arguments'))
|
||
tool_calls.append(ToolCall(id=call_id, name=name, arguments=arguments))
|
||
elif isinstance(message.get('function_call'), dict):
|
||
function_call = message['function_call']
|
||
name = function_call.get('name')
|
||
if not isinstance(name, str) or not name:
|
||
raise OpenAICompatError('Function call missing name')
|
||
arguments = _parse_tool_arguments(function_call.get('arguments'))
|
||
tool_calls.append(ToolCall(id='call_0', name=name, arguments=arguments))
|
||
return tool_calls
|
||
|
||
def _iter_sse_payloads(self, response: Any) -> Iterator[dict[str, Any]]:
|
||
buffer: list[str] = []
|
||
while True:
|
||
line = response.readline()
|
||
if not line:
|
||
break
|
||
if isinstance(line, bytes):
|
||
text = line.decode('utf-8', errors='replace')
|
||
else:
|
||
text = str(line)
|
||
stripped = text.strip()
|
||
if not stripped:
|
||
if not buffer:
|
||
continue
|
||
joined = '\n'.join(buffer)
|
||
buffer.clear()
|
||
if joined == '[DONE]':
|
||
break
|
||
try:
|
||
payload = json.loads(joined)
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenAICompatError(
|
||
f'Invalid JSON in streaming response: {joined!r}'
|
||
) from exc
|
||
if not isinstance(payload, dict):
|
||
raise OpenAICompatError('Malformed SSE payload from model backend')
|
||
yield payload
|
||
continue
|
||
if stripped.startswith('data:'):
|
||
buffer.append(stripped[5:].strip())
|
||
|
||
if buffer:
|
||
joined = '\n'.join(buffer)
|
||
if joined != '[DONE]':
|
||
try:
|
||
payload = json.loads(joined)
|
||
except json.JSONDecodeError as exc:
|
||
raise OpenAICompatError(
|
||
f'Invalid trailing JSON in streaming response: {joined!r}'
|
||
) from exc
|
||
if not isinstance(payload, dict):
|
||
raise OpenAICompatError('Malformed trailing SSE payload from model backend')
|
||
yield payload
|
||
|
||
def _parse_stream_payload(
|
||
self,
|
||
payload: dict[str, Any],
|
||
) -> Iterator[StreamEvent]:
|
||
usage = _parse_usage(payload.get('usage'))
|
||
if usage.total_tokens:
|
||
yield StreamEvent(
|
||
type='usage',
|
||
usage=usage,
|
||
raw_event=payload,
|
||
)
|
||
|
||
choices = payload.get('choices')
|
||
if not isinstance(choices, list):
|
||
return
|
||
|
||
for choice in choices:
|
||
if not isinstance(choice, dict):
|
||
continue
|
||
delta = choice.get('delta')
|
||
if not isinstance(delta, dict):
|
||
delta = {}
|
||
content = delta.get('content')
|
||
if isinstance(content, str) and content:
|
||
yield StreamEvent(
|
||
type='content_delta',
|
||
delta=content,
|
||
raw_event=choice,
|
||
)
|
||
tool_calls = delta.get('tool_calls')
|
||
if isinstance(tool_calls, list):
|
||
for raw_tool_call in tool_calls:
|
||
if not isinstance(raw_tool_call, dict):
|
||
continue
|
||
function_block = raw_tool_call.get('function')
|
||
if not isinstance(function_block, dict):
|
||
function_block = {}
|
||
yield StreamEvent(
|
||
type='tool_call_delta',
|
||
tool_call_index=(
|
||
raw_tool_call.get('index')
|
||
if isinstance(raw_tool_call.get('index'), int)
|
||
else 0
|
||
),
|
||
tool_call_id=(
|
||
raw_tool_call.get('id')
|
||
if isinstance(raw_tool_call.get('id'), str)
|
||
else None
|
||
),
|
||
tool_name=(
|
||
function_block.get('name')
|
||
if isinstance(function_block.get('name'), str)
|
||
else None
|
||
),
|
||
arguments_delta=(
|
||
function_block.get('arguments')
|
||
if isinstance(function_block.get('arguments'), str)
|
||
else ''
|
||
),
|
||
raw_event=raw_tool_call,
|
||
)
|
||
finish_reason = choice.get('finish_reason')
|
||
if finish_reason is not None:
|
||
if not isinstance(finish_reason, str):
|
||
finish_reason = str(finish_reason)
|
||
yield StreamEvent(
|
||
type='message_stop',
|
||
finish_reason=finish_reason,
|
||
raw_event=choice,
|
||
)
|