diff --git a/.env.example b/.env.example index 8676372..0172184 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,10 @@ INTERNAL_GATEWAY_KEY= MODEL_API_KEY= MODEL_API_BASE_URL=https://api.k1412.top +# Extreme tier provider. Keep the key only in the protected server-side .env. +DEEPSEEK_API_KEY= +DEEPSEEK_API_BASE_URL=https://api.deepseek.com + # Storage POSTGRES_USER=agent POSTGRES_PASSWORD= diff --git a/README.md b/README.md index eefbd17..5a09359 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ one Docker workspace per Open WebUI user on a dedicated execution host ``` Open WebUI is pinned and lightly patched. Its model picker is replaced by two -product-level choices: `Chat / Work` and `轻度 / 中 / 高`. Provider URLs, +product-level choices: `Chat / Work` and `轻度 / 中 / 高 / 极高`. Provider URLs, provider model IDs, API keys, tool server settings, system prompts, and runtime parameters remain server-side. @@ -43,6 +43,7 @@ The inference mapping is fixed: | 轻度 | `ChatGPT-5.6:Luna` | | 中 | `ChatGPT-5.6:Terra` | | 高 | `ChatGPT-5.6:Sol` | +| 极高 | `deepseek-v4-pro`（Thinking，`reasoning_effort=max`） | ## Local development @@ -82,6 +83,20 @@ test, with: ./scripts/verify.sh ``` +Run a live Work evaluation in a unique, disposable, network-disabled Docker +workspace with: + +```bash +set -a +source /path/to/protected/deepseek.env +set +a +.venv/bin/python scripts/eval-live-work.py --model work-extreme +``` + +The evaluator reports latency, token usage, tool counts, completion evidence, +and the generated file listing. It removes only the uniquely named evaluation +container, volume, and network unless `--keep-workspace` is supplied. + Run the disposable six-service integration stack with a deterministic fake model provider and exercise registration approval, both Agent loops, workspace isolation, and the one-way mode upgrade with: diff --git a/agent_platform/bootstrap.py b/agent_platform/bootstrap.py index 35ed7b8..9a5213a 100644 --- a/agent_platform/bootstrap.py +++ b/agent_platform/bootstrap.py @@ -158,7 +158,7 @@ async def bootstrap() -> None: ) model_response.raise_for_status() - print("Open WebUI bootstrap complete: auth policy, workspace tools, and six fixed Agent entries are ready.") + print("Open WebUI bootstrap complete: auth policy, workspace tools, and eight fixed Agent entries are ready.") def run() -> None: diff --git a/agent_platform/config.py b/agent_platform/config.py index cf9a20f..f68d910 100644 --- a/agent_platform/config.py +++ b/agent_platform/config.py @@ -26,6 +26,8 @@ def _float(name: str, default: float) -> float: class Settings: model_api_base_url: str model_api_key: str + deepseek_api_base_url: str + deepseek_api_key: str openwebui_forward_jwt_secret: str internal_provider_key: str internal_gateway_key: str @@ -49,6 +51,8 @@ class Settings: return cls( model_api_base_url=os.getenv("MODEL_API_BASE_URL", "https://api.k1412.top").rstrip("/"), model_api_key=os.getenv("MODEL_API_KEY", ""), + deepseek_api_base_url=os.getenv("DEEPSEEK_API_BASE_URL", "https://api.deepseek.com").rstrip("/"), + deepseek_api_key=os.getenv("DEEPSEEK_API_KEY", ""), openwebui_forward_jwt_secret=os.getenv("OPENWEBUI_FORWARD_JWT_SECRET", ""), internal_provider_key=os.getenv("INTERNAL_PROVIDER_KEY", ""), internal_gateway_key=os.getenv("INTERNAL_GATEWAY_KEY", ""), @@ -73,6 +77,7 @@ class Settings: name for name, value in ( ("MODEL_API_KEY", self.model_api_key), + ("DEEPSEEK_API_KEY", self.deepseek_api_key), ("OPENWEBUI_FORWARD_JWT_SECRET", self.openwebui_forward_jwt_secret), ("INTERNAL_PROVIDER_KEY", self.internal_provider_key), ("INTERNAL_GATEWAY_KEY", self.internal_gateway_key), diff --git a/agent_platform/models.py b/agent_platform/models.py index 3218c09..989b5be 100644 --- a/agent_platform/models.py +++ b/agent_platform/models.py @@ -4,7 +4,8 @@ from dataclasses import dataclass from typing import Literal Mode = Literal["chat", "work"] -Strength = Literal["light", "medium", "high"] +Strength = Literal["light", "medium", "high", "extreme"] +Provider = Literal["k1412", "deepseek"] @dataclass(frozen=True, slots=True) @@ -13,15 +14,20 @@ class ModelSpec: display_name: str mode: Mode strength: Strength + provider: Provider provider_model: str + thinking_enabled: bool + reasoning_effort: str | None + max_output_tokens: int max_iterations: int context_char_budget: int _TIERS = { - "light": ("轻度", "ChatGPT-5.6:Luna", 8, 120_000), - "medium": ("中", "ChatGPT-5.6:Terra", 16, 240_000), - "high": ("高", "ChatGPT-5.6:Sol", 24, 400_000), + "light": ("轻度", "k1412", "ChatGPT-5.6:Luna", False, None, 4_096, 8, 120_000), + "medium": ("中", "k1412", "ChatGPT-5.6:Terra", False, None, 4_096, 16, 240_000), + "high": ("高", "k1412", "ChatGPT-5.6:Sol", False, None, 4_096, 24, 400_000), + "extreme": ("极高", "deepseek", "deepseek-v4-pro", True, "max", 16_384, 32, 800_000), } MODEL_SPECS: dict[str, ModelSpec] = { @@ -30,12 +36,25 @@ MODEL_SPECS: dict[str, ModelSpec] = { display_name=f"{'Chat' if mode == 'chat' else 'Work'} · {label}", mode=mode, strength=strength, - provider_model=provider, + provider=provider, + provider_model=provider_model, + thinking_enabled=thinking_enabled, + reasoning_effort=reasoning_effort, + max_output_tokens=max_output_tokens, max_iterations=max_iterations, context_char_budget=context_budget, ) for mode in ("chat", "work") - for strength, (label, provider, max_iterations, context_budget) in _TIERS.items() + for strength, ( + label, + provider, + provider_model, + thinking_enabled, + reasoning_effort, + max_output_tokens, + max_iterations, + context_budget, + ) in _TIERS.items() } diff --git a/agent_platform/runtime/app.py b/agent_platform/runtime/app.py index 51607a5..6296b08 100644 --- a/agent_platform/runtime/app.py +++ b/agent_platform/runtime/app.py @@ -160,7 +160,13 @@ def create_app( async def forward_direct(): payload = body.model_dump(exclude_none=True) payload["model"] = spec.provider_model - response = await app.state.provider.forward(payload) + response = await app.state.provider.forward( + payload, + provider=spec.provider, + thinking_enabled=spec.thinking_enabled, + reasoning_effort=spec.reasoning_effort, + max_output_tokens=spec.max_output_tokens, + ) if body.stream: passthrough_headers = { key: value diff --git a/agent_platform/runtime/loop.py b/agent_platform/runtime/loop.py index 5f09f65..5b45aac 100644 --- a/agent_platform/runtime/loop.py +++ b/agent_platform/runtime/loop.py @@ -599,6 +599,10 @@ class AgentLoop: model=spec.provider_model, messages=messages, tools=available_specs, + provider=spec.provider, + thinking_enabled=spec.thinking_enabled, + reasoning_effort=spec.reasoning_effort, + max_output_tokens=spec.max_output_tokens, ) choice = response["choices"][0] message = choice.get("message") or {} @@ -648,9 +652,12 @@ class AgentLoop: "completion.rejected", {"reason": failure, "iteration": iteration + 1, "depth": depth}, ) + assistant_checkpoint = {"role": "assistant", "content": candidate} + if isinstance(message.get("reasoning_content"), str): + assistant_checkpoint["reasoning_content"] = message["reasoning_content"] messages.extend( [ - {"role": "assistant", "content": candidate}, + assistant_checkpoint, {"role": "user", "content": recovery_message(failure)}, ] ) @@ -683,6 +690,8 @@ class AgentLoop: "content": message.get("content"), "tool_calls": tool_calls, } + if isinstance(message.get("reasoning_content"), str): + assistant_message["reasoning_content"] = message["reasoning_content"] messages.append(assistant_message) blocked_call_reasons: dict[str, str] = {} batch_mutation_epoch = 0 @@ -801,7 +810,15 @@ class AgentLoop: ), } ) - response = await self.provider.complete(model=spec.provider_model, messages=messages, tools=None) + response = await self.provider.complete( + model=spec.provider_model, + messages=messages, + tools=None, + provider=spec.provider, + thinking_enabled=spec.thinking_enabled, + reasoning_effort=spec.reasoning_effort, + max_output_tokens=spec.max_output_tokens, + ) answer = str(response["choices"][0].get("message", {}).get("content") or "").strip() if completion_failure( artifact_required, diff --git a/agent_platform/runtime/provider.py b/agent_platform/runtime/provider.py index 1c93da7..b0b14c0 100644 --- a/agent_platform/runtime/provider.py +++ b/agent_platform/runtime/provider.py @@ -6,6 +6,7 @@ from typing import Any import httpx from agent_platform.config import Settings +from agent_platform.models import Provider TRANSIENT_COMPLETE_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504, 524}) COMPLETE_MAX_ATTEMPTS = 2 @@ -132,11 +133,37 @@ class ModelProvider: @property def headers(self) -> dict[str, str]: + return self._headers("k1412") + + def _headers(self, provider: Provider) -> dict[str, str]: + key = self.settings.deepseek_api_key if provider == "deepseek" else self.settings.model_api_key return { - "Authorization": f"Bearer {self.settings.model_api_key}", + "Authorization": f"Bearer {key}", "Content-Type": "application/json", } + def _base_url(self, provider: Provider) -> str: + return self.settings.deepseek_api_base_url if provider == "deepseek" else self.settings.model_api_base_url + + @staticmethod + def _provider_payload( + payload: dict[str, Any], + *, + provider: Provider, + thinking_enabled: bool, + reasoning_effort: str | None, + max_output_tokens: int, + ) -> dict[str, Any]: + value = dict(payload) + value.pop("thinking", None) + value.pop("reasoning_effort", None) + value["max_tokens"] = max_output_tokens + if provider == "deepseek" and thinking_enabled: + value["thinking"] = {"type": "enabled"} + if reasoning_effort: + value["reasoning_effort"] = reasoning_effort + return value + async def close(self) -> None: if self._owns_client: await self.client.aclose() @@ -148,13 +175,22 @@ class ModelProvider: messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None, temperature: float | None = None, + provider: Provider = "k1412", + thinking_enabled: bool = False, + reasoning_effort: str | None = None, + max_output_tokens: int = COMPLETE_MAX_TOKENS, ) -> dict[str, Any]: - payload: dict[str, Any] = { - "model": model, - "messages": messages, - "stream": True, - "max_tokens": COMPLETE_MAX_TOKENS, - } + payload = self._provider_payload( + { + "model": model, + "messages": messages, + "stream": True, + }, + provider=provider, + thinking_enabled=thinking_enabled, + reasoning_effort=reasoning_effort, + max_output_tokens=max_output_tokens, + ) if tools: payload["tools"] = tools payload["tool_choice"] = "auto" @@ -165,8 +201,8 @@ class ModelProvider: try: async with self.client.stream( "POST", - completions_url(self.settings.model_api_base_url), - headers=self.headers, + completions_url(self._base_url(provider)), + headers=self._headers(provider), json=payload, ) as response: if response.status_code in TRANSIENT_COMPLETE_STATUS_CODES and attempt < COMPLETE_MAX_ATTEMPTS: @@ -184,11 +220,26 @@ class ModelProvider: raise RuntimeError("Model provider retry loop exited unexpectedly") - async def forward(self, payload: dict[str, Any]) -> httpx.Response: + async def forward( + self, + payload: dict[str, Any], + *, + provider: Provider = "k1412", + thinking_enabled: bool = False, + reasoning_effort: str | None = None, + max_output_tokens: int = COMPLETE_MAX_TOKENS, + ) -> httpx.Response: + payload = self._provider_payload( + payload, + provider=provider, + thinking_enabled=thinking_enabled, + reasoning_effort=reasoning_effort, + max_output_tokens=max_output_tokens, + ) request = self.client.build_request( "POST", - completions_url(self.settings.model_api_base_url), - headers=self.headers, + completions_url(self._base_url(provider)), + headers=self._headers(provider), json=payload, ) response = await self.client.send(request, stream=bool(payload.get("stream"))) diff --git a/compose.e2e.yaml b/compose.e2e.yaml index ec006dd..9b2aa96 100644 --- a/compose.e2e.yaml +++ b/compose.e2e.yaml @@ -17,6 +17,7 @@ services: runtime: environment: MODEL_API_BASE_URL: http://stub-provider:9000 + DEEPSEEK_API_BASE_URL: http://stub-provider:9000 depends_on: stub-provider: condition: service_healthy diff --git a/compose.yaml b/compose.yaml index 869fcca..90ab97d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -123,6 +123,8 @@ services: environment: MODEL_API_BASE_URL: ${MODEL_API_BASE_URL:-https://api.k1412.top} MODEL_API_KEY: ${MODEL_API_KEY:?MODEL_API_KEY is required} + DEEPSEEK_API_BASE_URL: ${DEEPSEEK_API_BASE_URL:-https://api.deepseek.com} + DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY is required} OPENWEBUI_FORWARD_JWT_SECRET: ${OPENWEBUI_FORWARD_JWT_SECRET:?OPENWEBUI_FORWARD_JWT_SECRET is required} INTERNAL_PROVIDER_KEY: ${INTERNAL_PROVIDER_KEY:?INTERNAL_PROVIDER_KEY is required} INTERNAL_GATEWAY_KEY: ${INTERNAL_GATEWAY_KEY:?INTERNAL_GATEWAY_KEY is required} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 75af2e6..bcf3f11 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -123,6 +123,8 @@ services: environment: MODEL_API_BASE_URL: ${MODEL_API_BASE_URL:-https://api.k1412.top} MODEL_API_KEY: ${MODEL_API_KEY:?MODEL_API_KEY is required} + DEEPSEEK_API_BASE_URL: ${DEEPSEEK_API_BASE_URL:-https://api.deepseek.com} + DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:?DEEPSEEK_API_KEY is required} OPENWEBUI_FORWARD_JWT_SECRET: ${OPENWEBUI_FORWARD_JWT_SECRET:?OPENWEBUI_FORWARD_JWT_SECRET is required} INTERNAL_PROVIDER_KEY: ${INTERNAL_PROVIDER_KEY:?INTERNAL_PROVIDER_KEY is required} INTERNAL_GATEWAY_KEY: ${INTERNAL_GATEWAY_KEY:?INTERNAL_GATEWAY_KEY is required} diff --git a/docker/web/ModelSelector.svelte b/docker/web/ModelSelector.svelte index 0ab0b43..911e5d2 100644 --- a/docker/web/ModelSelector.svelte +++ b/docker/web/ModelSelector.svelte @@ -10,16 +10,19 @@ const strengths = [ { key: 'light', label: '轻度' }, { key: 'medium', label: '中' }, - { key: 'high', label: '高' } + { key: 'high', label: '高' }, + { key: 'extreme', label: '极高' } ]; const allowed = new Set([ 'chat-light', 'chat-medium', 'chat-high', + 'chat-extreme', 'work-light', 'work-medium', - 'work-high' + 'work-high', + 'work-extreme' ]); $: selected = allowed.has(selectedModels?.[0]) ? selectedModels[0] : 'chat-medium'; diff --git a/docs/architecture.md b/docs/architecture.md index db96462..27ce6f8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,6 +34,13 @@ Every run records its model tier, strategy version, scheduler version, context policy, model/tool events, failures, and completion. This event stream is the foundation for replay, evaluation, and future experiment assignment. +The light, medium, and high tiers use the internal K1412 model endpoint. The +extreme tier uses DeepSeek V4 Pro with thinking enabled and maximum reasoning +effort. Provider selection and credentials remain server-side. Thinking state +is preserved across tool turns because DeepSeek requires the assistant's +`reasoning_content` to be returned with the following tool results; Work never +publishes that hidden state as its own user-facing reasoning. + Work completion is evidence-gated. Requests for scripts, reports, code, or other concrete artifacts are not accepted as complete until a file mutation has succeeded and a later read, diff, or command has verified the latest diff --git a/docs/operations.md b/docs/operations.md index d95c205..8b197a8 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -39,6 +39,11 @@ listed above. `https://api.k1412.top` remains the external API entrypoint, but Runtime must not send long-running Agent inference through the public reverse proxy. +Set `DEEPSEEK_API_BASE_URL=https://api.deepseek.com` and place +`DEEPSEEK_API_KEY` only in the protected deployment `.env`. The key is used +only by Runtime for the extreme tier and must never be added to a browser, +Compose file, image, repository, or log. + The SSH override removes `/var/run/docker.sock` from Gateway and mounts the dedicated SSH directory read-only at `/root/.ssh`. diff --git a/e2e/verify_stack.py b/e2e/verify_stack.py index d914c2c..d4aa4dd 100644 --- a/e2e/verify_stack.py +++ b/e2e/verify_stack.py @@ -29,9 +29,11 @@ EXPECTED_MODELS = { "chat-light", "chat-medium", "chat-high", + "chat-extreme", "work-light", "work-medium", "work-high", + "work-extreme", } @@ -286,7 +288,7 @@ def main() -> None: assert downgrade.status_code == 409, downgrade.text print( - "E2E passed: auth approval, six models, Chat tools, Work evidence, file downloads, " + "E2E passed: auth approval, eight models, Chat tools, Work evidence, file downloads, " "isolation, and downgrade lock." ) diff --git a/scripts/eval-live-work.py b/scripts/eval-live-work.py new file mode 100755 index 0000000..1faa6f1 --- /dev/null +++ b/scripts/eval-live-work.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import tempfile +import time +import uuid +from dataclasses import replace +from pathlib import Path +from typing import Any + +from docker.errors import NotFound + +from agent_platform.auth import UserIdentity +from agent_platform.config import Settings +from agent_platform.gateway.provider import DockerExecutionProvider, workspace_ref +from agent_platform.models import get_model_spec +from agent_platform.runtime.loop import AgentLoop +from agent_platform.runtime.provider import ModelProvider +from agent_platform.runtime.schemas import PlanItem +from agent_platform.runtime.tools import TOOL_METADATA, ToolContext +from agent_platform.store import RuntimeStore + +EVAL_TOOL_NAMES = ( + "workspace_status", + "list_files", + "read_file", + "search_files", + "write_file", + "apply_patch", + "exec", + "update_plan", +) + + +class DirectDockerRegistry: + def __init__( + self, + execution: DockerExecutionProvider, + store: RuntimeStore, + tool_timeout_seconds: int, + ) -> None: + self.execution = execution + self.store = store + self.tool_timeout_seconds = tool_timeout_seconds + + def specs(self, *, read_only: bool = False, allow_delegate: bool = True) -> list[dict[str, Any]]: + names = [name for name in EVAL_TOOL_NAMES if not read_only or TOOL_METADATA[name].read_only] + return [TOOL_METADATA[name].openai_spec() for name in names] + + async def execute( + self, + name: str, + arguments: dict[str, Any], + context: ToolContext, + ) -> dict[str, Any]: + user_id = context.identity.user_id + if name == "workspace_status": + result = await self.execution.status(user_id) + elif name == "list_files": + result = await self.execution.list_files( + user_id, + str(arguments.get("path", ".")), + int(arguments.get("max_depth", 4)), + int(arguments.get("limit", 500)), + ) + elif name == "read_file": + result = await self.execution.read_file( + user_id, + str(arguments["path"]), + int(arguments.get("start_line", 1)), + int(arguments.get("max_lines", 1000)), + ) + elif name == "search_files": + result = await self.execution.search_files( + user_id, + str(arguments["query"]), + str(arguments.get("path", ".")), + arguments.get("glob"), + int(arguments.get("limit", 200)), + ) + elif name == "write_file": + result = await self.execution.write_file( + user_id, + str(arguments["path"]), + str(arguments["content"]), + ) + elif name == "apply_patch": + result = await self.execution.apply_patch( + user_id, + str(arguments["patch"]), + str(arguments.get("cwd", ".")), + ) + elif name == "exec": + result = await self.execution.exec( + user_id, + str(arguments["command"]), + str(arguments.get("cwd", ".")), + min( + int(arguments.get("timeout_seconds", 120)), + self.tool_timeout_seconds, + ), + ) + elif name == "update_plan": + items = [PlanItem.model_validate(item).model_dump() for item in arguments.get("items", [])] + await self.store.update_plan(user_id, context.chat_id, items) + return { + "ok": True, + "explanation": arguments.get("explanation"), + "items": items, + } + else: + raise ValueError(f"Unsupported evaluation tool: {name}") + return result.model_dump() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the real Work loop against a live provider in a disposable Docker workspace." + ) + parser.add_argument("--model", default="work-extreme") + parser.add_argument( + "--prompt", + default="写脚本对比一下几个常见排序算法，给一个报告", + ) + parser.add_argument( + "--workspace-image", + default=os.getenv("WORKSPACE_IMAGE", "k1412-agent-workspace:test"), + ) + parser.add_argument("--keep-workspace", action="store_true") + return parser.parse_args() + + +async def cleanup_workspace(execution: DockerExecutionProvider, user_id: str) -> None: + ref = workspace_ref(user_id) + try: + execution.client.containers.get(ref.container_name).remove(force=True) + except NotFound: + pass + try: + execution.client.volumes.get(ref.volume_name).remove(force=True) + except NotFound: + pass + try: + execution.client.networks.get(ref.network_name).remove() + except NotFound: + pass + + +async def main() -> None: + args = parse_args() + spec = get_model_spec(args.model) + if spec.mode != "work": + raise SystemExit("--model must select a Work tier") + if spec.provider == "deepseek" and not os.getenv("DEEPSEEK_API_KEY", "").strip(): + raise SystemExit("DEEPSEEK_API_KEY is required") + + user_id = f"live-eval-{uuid.uuid4().hex}" + chat_id = f"live-eval-{uuid.uuid4().hex}" + started = time.monotonic() + event_counts: dict[str, int] = {} + usage_totals = { + "prompt_tokens": 0, + "completion_tokens": 0, + "reasoning_tokens": 0, + "cached_tokens": 0, + "total_tokens": 0, + } + + with tempfile.TemporaryDirectory(prefix="k1412-live-eval-") as temp_dir: + settings = replace( + Settings.from_env(), + database_url=f"sqlite+aiosqlite:///{Path(temp_dir) / 'runtime.sqlite3'}", + workspace_image=args.workspace_image, + workspace_network_enabled=False, + workspace_memory_limit="1g", + workspace_cpu_limit=1.0, + workspace_pids_limit=128, + tool_timeout_seconds=180, + ) + store = RuntimeStore(settings.database_url) + await store.initialize() + provider = ModelProvider(settings) + execution = DockerExecutionProvider(settings) + registry = DirectDockerRegistry(execution, store, settings.tool_timeout_seconds) + + async def record(event_type: str, payload: dict[str, Any]) -> None: + event_counts[event_type] = event_counts.get(event_type, 0) + 1 + if event_type == "model.responded": + usage = payload.get("usage") or {} + prompt_details = usage.get("prompt_tokens_details") or {} + completion_details = usage.get("completion_tokens_details") or {} + usage_totals["prompt_tokens"] += int(usage.get("prompt_tokens", 0)) + usage_totals["completion_tokens"] += int(usage.get("completion_tokens", 0)) + usage_totals["reasoning_tokens"] += int(completion_details.get("reasoning_tokens", 0)) + usage_totals["cached_tokens"] += int( + prompt_details.get("cached_tokens", 0) or usage.get("prompt_cache_hit_tokens", 0) + ) + usage_totals["total_tokens"] += int(usage.get("total_tokens", 0)) + if event_type == "tool.completed": + print( + json.dumps( + { + "event": event_type, + "name": payload.get("name"), + "ok": payload.get("ok"), + "summary": str(payload.get("summary", ""))[:500], + }, + ensure_ascii=False, + ), + flush=True, + ) + + try: + answer = await AgentLoop( + provider, + registry, + store, + max_tool_output_chars=settings.max_tool_output_chars, + ).run( + spec=spec, + messages=[{"role": "user", "content": args.prompt}], + identity=UserIdentity(user_id, "", "Live Eval", "user"), + raw_user_jwt="local-evaluation", + chat_id=chat_id, + callback=record, + ) + listing = await execution.list_files(user_id, ".", 4, 500) + print( + json.dumps( + { + "model": spec.public_id, + "provider_model": spec.provider_model, + "elapsed_seconds": round(time.monotonic() - started, 3), + "events": event_counts, + "usage": usage_totals, + "answer": answer, + "workspace": listing.model_dump(), + "workspace_id": workspace_ref(user_id).workspace_id if args.keep_workspace else None, + }, + ensure_ascii=False, + indent=2, + ) + ) + finally: + await provider.close() + await store.close() + if not args.keep_workspace: + await cleanup_workspace(execution, user_id) + execution.client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/verify-e2e.sh b/scripts/verify-e2e.sh index 21eb8d0..d71cddb 100755 --- a/scripts/verify-e2e.sh +++ b/scripts/verify-e2e.sh @@ -18,6 +18,7 @@ export OPENWEBUI_FORWARD_JWT_SECRET="e2e-forward-jwt-secret-at-least-32-bytes" export INTERNAL_PROVIDER_KEY="e2e-provider-secret-at-least-32-bytes" export INTERNAL_GATEWAY_KEY="e2e-gateway-secret-at-least-32-bytes" export MODEL_API_KEY="e2e-model-key" +export DEEPSEEK_API_KEY="e2e-deepseek-key" export POSTGRES_PASSWORD="e2e-postgres-password" export WEB_IMAGE="k1412-agent-web:e2e" export RUNTIME_IMAGE="k1412-agent-runtime:e2e" diff --git a/scripts/verify.sh b/scripts/verify.sh index 0cb0be2..7ae3c0e 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -24,6 +24,7 @@ export OPENWEBUI_FORWARD_JWT_SECRET=verify-forward-jwt-secret-at-least-32-bytes export INTERNAL_PROVIDER_KEY=verify-provider-secret-at-least-32-bytes export INTERNAL_GATEWAY_KEY=verify-gateway-secret-at-least-32-bytes export MODEL_API_KEY=verify-model-key +export DEEPSEEK_API_KEY=verify-deepseek-key export POSTGRES_PASSWORD=verify-postgres-password export WEB_IMAGE=docker.k1412.top/wuyang/k1412-agent-web:verify export RUNTIME_IMAGE=docker.k1412.top/wuyang/k1412-agent-runtime:verify diff --git a/tests/conftest.py b/tests/conftest.py index a36c1f8..be6cb79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,8 @@ def settings(tmp_path: Path) -> Settings: return Settings( model_api_base_url="https://model.example.test", model_api_key="model-secret", + deepseek_api_base_url="https://api.deepseek.example.test", + deepseek_api_key="deepseek-secret", openwebui_forward_jwt_secret="identity-secret-with-at-least-32-bytes", # noqa: S106 internal_provider_key="provider-secret-with-at-least-32-bytes", internal_gateway_key="gateway-secret-with-at-least-32-bytes", diff --git a/tests/test_agent_loop.py b/tests/test_agent_loop.py index 0cd2909..217f241 100644 --- a/tests/test_agent_loop.py +++ b/tests/test_agent_loop.py @@ -962,3 +962,59 @@ def test_tool_events_render_one_friendly_completed_card() -> None: assert 'name="执行命令"' in rendered assert "2 passed" in rendered assert 'done="false"' not in rendered + + +async def test_extreme_tier_preserves_reasoning_state_across_tool_turns(settings) -> None: + provider = ScriptedProvider( + [ + { + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "provider-private-state", + "tool_calls": [tool_call("1", "read_file", {"path": "notes.txt"})], + }, + "finish_reason": "tool_calls", + } + ] + }, + { + "choices": [ + { + "message": {"role": "assistant", "content": "检查完成"}, + "finish_reason": "stop", + } + ] + }, + ] + ) + registry = RecordingRegistry() + store = RuntimeStore(settings.database_url) + await store.initialize() + + async def callback(event_type, payload): + return None + + try: + answer = await AgentLoop(provider, registry, store, max_tool_output_chars=10_000).run( + spec=get_model_spec("work-extreme"), + messages=[{"role": "user", "content": "读取 notes.txt 并检查内容"}], + identity=UserIdentity("u1", "", "", "user"), + raw_user_jwt="jwt", + chat_id="deepseek-reasoning", + callback=callback, + ) + finally: + await store.close() + + assert answer == "检查完成" + assert provider.requests[0]["provider"] == "deepseek" + assert provider.requests[0]["reasoning_effort"] == "max" + assistant_turn = next( + message + for message in provider.requests[1]["messages"] + if message.get("role") == "assistant" and message.get("tool_calls") + ) + assert assistant_turn["reasoning_content"] == "provider-private-state" diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 71dbb73..d48e436 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -3,7 +3,7 @@ from agent_platform.bootstrap import _model_payload def test_bootstrap_models_override_provider_ids_and_are_public() -> None: models = _model_payload() - assert len(models) == 6 + assert len(models) == 8 assert all(model["base_model_id"] is None for model in models) assert all( model["access_grants"] == [{"principal_type": "user", "principal_id": "*", "permission": "read"}] diff --git a/tests/test_config.py b/tests/test_config.py index d0b6e57..18b04b8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,3 +6,8 @@ import pytest def test_internal_secrets_must_be_at_least_32_bytes(settings) -> None: with pytest.raises(RuntimeError, match="at least 32 bytes"): replace(settings, internal_gateway_key="too-short").validate_gateway() + + +def test_runtime_requires_deepseek_key_for_extreme_tier(settings) -> None: + with pytest.raises(RuntimeError, match="DEEPSEEK_API_KEY"): + replace(settings, deepseek_api_key="").validate_runtime() diff --git a/tests/test_models.py b/tests/test_models.py index 0b27839..22369cf 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -6,11 +6,18 @@ def test_fixed_public_model_matrix() -> None: "chat-light", "chat-medium", "chat-high", + "chat-extreme", "work-light", "work-medium", "work-high", + "work-extreme", } assert get_model_spec("chat-light").provider_model == "ChatGPT-5.6:Luna" assert get_model_spec("work-medium").provider_model == "ChatGPT-5.6:Terra" assert get_model_spec("work-high").provider_model == "ChatGPT-5.6:Sol" + extreme = get_model_spec("work-extreme") + assert extreme.provider == "deepseek" + assert extreme.provider_model == "deepseek-v4-pro" + assert extreme.thinking_enabled is True + assert extreme.reasoning_effort == "max" assert {item["id"] for item in openai_model_list()["data"]} == set(MODEL_SPECS) diff --git a/tests/test_provider.py b/tests/test_provider.py index bfc5263..c671690 100644 --- a/tests/test_provider.py +++ b/tests/test_provider.py @@ -21,10 +21,15 @@ def disable_retry_sleep(monkeypatch: pytest.MonkeyPatch) -> None: async def _complete_with_handler( settings: Settings, handler: Callable[[httpx.Request], httpx.Response | Awaitable[httpx.Response]], + **kwargs, ) -> dict: async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: provider = ModelProvider(settings, client) - return await provider.complete(model="test-model", messages=[{"role": "user", "content": "hello"}]) + return await provider.complete( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + **kwargs, + ) async def test_complete_retries_transient_gateway_response(settings: Settings) -> None: @@ -158,3 +163,41 @@ async def test_complete_aggregates_streamed_tool_call(settings: Settings) -> Non } ] assert result["usage"]["total_tokens"] == 7 + + +async def test_deepseek_route_enables_thinking_and_uses_separate_credentials(settings: Settings) -> None: + body = ( + 'data: {"choices":[{"index":0,"delta":{"role":"assistant",' + '"reasoning_content":"private state","tool_calls":[{"index":0,"id":"call-1",' + '"type":"function","function":{"name":"ping","arguments":"{}"}}]},' + '"finish_reason":"tool_calls"}]}\n\n' + "data: [DONE]\n\n" + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://api.deepseek.example.test/v1/chat/completions" + assert request.headers["Authorization"] == "Bearer deepseek-secret" + payload = json.loads(request.content) + assert payload["model"] == "test-model" + assert payload["thinking"] == {"type": "enabled"} + assert payload["reasoning_effort"] == "max" + assert payload["max_tokens"] == 16_384 + return httpx.Response( + 200, + request=request, + headers={"Content-Type": "text/event-stream"}, + content=body, + ) + + result = await _complete_with_handler( + settings, + handler, + provider="deepseek", + thinking_enabled=True, + reasoning_effort="max", + max_output_tokens=16_384, + ) + + message = result["choices"][0]["message"] + assert message["reasoning_content"] == "private state" + assert message["tool_calls"][0]["function"]["name"] == "ping" diff --git a/tests/test_runtime_api.py b/tests/test_runtime_api.py index e9bc54e..fc24bc4 100644 --- a/tests/test_runtime_api.py +++ b/tests/test_runtime_api.py @@ -13,6 +13,7 @@ class FakeModelProvider: def __init__(self) -> None: self.completed = [] self.forwarded = [] + self.forward_options = [] async def complete(self, **kwargs): self.completed.append(kwargs) @@ -21,8 +22,9 @@ class FakeModelProvider: "usage": {"total_tokens": 10}, } - async def forward(self, payload): + async def forward(self, payload, **kwargs): self.forwarded.append(payload) + self.forward_options.append(kwargs) request = httpx.Request("POST", "https://model.example.test/v1/chat/completions") if payload.get("stream"): return httpx.Response( @@ -83,7 +85,7 @@ def test_model_list_requires_internal_key(settings) -> None: headers={"Authorization": f"Bearer {settings.internal_provider_key}"}, ) assert response.status_code == 200 - assert len(response.json()["data"]) == 6 + assert len(response.json()["data"]) == 8 def test_chat_maps_provider_model_and_work_upgrade_is_one_way(settings, identity_jwt) -> None: @@ -99,6 +101,20 @@ def test_chat_maps_provider_model_and_work_upgrade_is_one_way(settings, identity assert provider.forwarded[0]["model"] == "ChatGPT-5.6:Luna" assert chat.json()["model"] == "chat-light" + extreme_chat = client.post( + "/v1/chat/completions", + headers=headers(settings, identity_jwt, "extreme-chat"), + json={"model": "chat-extreme", "messages": [{"role": "user", "content": "think"}]}, + ) + assert extreme_chat.status_code == 200 + assert provider.forwarded[-1]["model"] == "deepseek-v4-pro" + assert provider.forward_options[-1] == { + "provider": "deepseek", + "thinking_enabled": True, + "reasoning_effort": "max", + "max_output_tokens": 16_384, + } + work = client.post( "/v1/chat/completions", headers=headers(settings, identity_jwt), @@ -107,6 +123,16 @@ def test_chat_maps_provider_model_and_work_upgrade_is_one_way(settings, identity assert work.status_code == 200 assert provider.completed[-1]["model"] == "ChatGPT-5.6:Sol" + extreme_work = client.post( + "/v1/chat/completions", + headers=headers(settings, identity_jwt, "extreme-work"), + json={"model": "work-extreme", "messages": [{"role": "user", "content": "do it carefully"}]}, + ) + assert extreme_work.status_code == 200 + assert provider.completed[-1]["model"] == "deepseek-v4-pro" + assert provider.completed[-1]["provider"] == "deepseek" + assert provider.completed[-1]["reasoning_effort"] == "max" + downgrade = client.post( "/v1/chat/completions", headers=headers(settings, identity_jwt),