wuyang
109 lines
3.6 KiB
Python
109 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from agent_platform.config import Settings
|
|
|
|
TRANSIENT_COMPLETE_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504, 524})
|
|
COMPLETE_MAX_ATTEMPTS = 2
|
|
MAX_RETRY_DELAY_SECONDS = 5.0
|
|
|
|
|
|
def completions_url(base_url: str) -> str:
|
|
base = base_url.rstrip("/")
|
|
if base.endswith("/v1"):
|
|
return f"{base}/chat/completions"
|
|
return f"{base}/v1/chat/completions"
|
|
|
|
|
|
def retry_delay_seconds(response: httpx.Response | None, attempt: int) -> float:
|
|
if response is not None:
|
|
retry_after = response.headers.get("Retry-After")
|
|
if retry_after:
|
|
try:
|
|
return min(MAX_RETRY_DELAY_SECONDS, max(0.0, float(retry_after)))
|
|
except ValueError:
|
|
pass
|
|
return min(MAX_RETRY_DELAY_SECONDS, float(2 ** (attempt - 1)))
|
|
|
|
|
|
class ModelProvider:
|
|
def __init__(self, settings: Settings, client: httpx.AsyncClient | None = None) -> None:
|
|
self.settings = settings
|
|
self.client = client or httpx.AsyncClient(
|
|
timeout=httpx.Timeout(settings.model_timeout_seconds),
|
|
limits=httpx.Limits(max_connections=50, max_keepalive_connections=20),
|
|
)
|
|
self._owns_client = client is None
|
|
|
|
@property
|
|
def headers(self) -> dict[str, str]:
|
|
return {
|
|
"Authorization": f"Bearer {self.settings.model_api_key}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
async def close(self) -> None:
|
|
if self._owns_client:
|
|
await self.client.aclose()
|
|
|
|
async def complete(
|
|
self,
|
|
*,
|
|
model: str,
|
|
messages: list[dict[str, Any]],
|
|
tools: list[dict[str, Any]] | None = None,
|
|
temperature: float | None = None,
|
|
) -> dict[str, Any]:
|
|
payload: dict[str, Any] = {
|
|
"model": model,
|
|
"messages": messages,
|
|
"stream": False,
|
|
}
|
|
if tools:
|
|
payload["tools"] = tools
|
|
payload["tool_choice"] = "auto"
|
|
if temperature is not None:
|
|
payload["temperature"] = temperature
|
|
for attempt in range(1, COMPLETE_MAX_ATTEMPTS + 1):
|
|
try:
|
|
response = await self.client.post(
|
|
completions_url(self.settings.model_api_base_url),
|
|
headers=self.headers,
|
|
json=payload,
|
|
)
|
|
except httpx.TransportError:
|
|
if attempt >= COMPLETE_MAX_ATTEMPTS:
|
|
raise
|
|
await asyncio.sleep(retry_delay_seconds(None, attempt))
|
|
continue
|
|
|
|
if response.status_code in TRANSIENT_COMPLETE_STATUS_CODES and attempt < COMPLETE_MAX_ATTEMPTS:
|
|
await response.aread()
|
|
await asyncio.sleep(retry_delay_seconds(response, attempt))
|
|
continue
|
|
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
if not data.get("choices"):
|
|
raise RuntimeError("Model provider returned no choices")
|
|
return data
|
|
|
|
raise RuntimeError("Model provider retry loop exited unexpectedly")
|
|
|
|
async def forward(self, payload: dict[str, Any]) -> httpx.Response:
|
|
request = self.client.build_request(
|
|
"POST",
|
|
completions_url(self.settings.model_api_base_url),
|
|
headers=self.headers,
|
|
json=payload,
|
|
)
|
|
response = await self.client.send(request, stream=bool(payload.get("stream")))
|
|
if response.status_code >= 400:
|
|
await response.aread()
|
|
response.raise_for_status()
|
|
return response
|