feat: give the curator continuous context and durable idea history

app/curator.py

@@ -3,19 +3,23 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import re
 import time
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from agents import (
     Agent,
     ModelSettings,
     OpenAIChatCompletionsModel,
+    RunHooks,
     RunContextWrapper,
     Runner,
     function_tool,
     set_tracing_disabled,
 )
+from agents.exceptions import MaxTurnsExceeded
+from agents.items import ModelResponse
 from openai import AsyncOpenAI
 from openai.types.shared import Reasoning
 from pydantic import ValidationError
@@ -25,7 +29,9 @@ from .db import Database
 from .schemas import CuratorDecision
 
 logger = logging.getLogger(__name__)
-PROMPT_VERSION = "maturity-2026-07-28.v2"
+PROMPT_VERSION = "maturity-2026-07-28.v3"
+RECENT_CONTEXT_LIMIT = 12
+IDEA_CATALOG_LIMIT = 40
 
 
 CURATOR_INSTRUCTIONS = """
@@ -52,8 +58,13 @@ motion 是你对当下运动的自然语言压缩，例如“向现实探去”
 一个片段最多关联两个想法。若现在还不值得形成或归入想法，让它 standalone，原文仍会保留。
 新建想法时 idea_id 必须为 null；更新时必须使用工具返回的准确 id。
 
-先查看候选想法；遇到可能相关或可能冲突的候选时，使用 inspect_idea。最终必须只返回 JSON，
-不能用 Markdown 包裹，也不能附加解释。JSON 必须符合输入中给出的 schema。
+输入会直接提供最近的连续原文和精简想法目录。把“这个、上面、第二点、继续、刚才”等指代
+放回连续语境中理解；不要因为新片段换了关键词就丢掉思想上的延续。目录中的 id 只供系统内部
+引用。遇到可能相关或冲突的候选时，使用 inspect_idea；search_ideas 只用于目录过大或需要
+补充候选时，最多调用两次；同一个想法最多 inspect 一次。搜索无结果时不要反复改写查询，
+也绝不能用 none、unknown 等占位符调用 inspect_idea。工具探索后必须及时收敛并返回最终判断。
+
+最终必须只返回 JSON，不能用 Markdown 包裹，也不能附加解释。JSON 必须符合输入中给出的 schema。
 不要在输出中评价用户、诊断心理、说教或伪造证据。
 """.strip()
 
@@ -67,6 +78,187 @@ class CuratorContext:
     recent_fragments: list[dict[str, Any]]
     search_count: int = 0
     inspection_count: int = 0
+    model_rounds: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    reasoning_tokens: int = 0
+    cached_tokens: int = 0
+    llm_started_at: float | None = None
+    inspected_idea_ids: set[str] = field(default_factory=set)
+
+
+def _search_units(value: str) -> list[str]:
+    units: list[str] = []
+    for token in re.findall(r"[a-zA-Z0-9_-]+|[\u3400-\u9fff]+", value.lower()):
+        if token not in units:
+            units.append(token)
+        if re.fullmatch(r"[\u3400-\u9fff]+", token) and len(token) > 2:
+            for index in range(len(token) - 1):
+                pair = token[index : index + 2]
+                if pair not in units:
+                    units.append(pair)
+    return units[:24]
+
+
+def _rank_ideas(
+    ideas: list[dict[str, Any]], query: str
+) -> tuple[list[dict[str, Any]], bool]:
+    terms = _search_units(query)
+    ranked: list[tuple[int, str, dict[str, Any]]] = []
+    for idea in ideas:
+        evidence = " ".join(
+            str(fragment.get("content", ""))
+            for fragment in idea.get("recent_fragments", [])
+        )
+        haystack = " ".join(
+            str(idea.get(key, ""))
+            for key in (
+                "title",
+                "summary",
+                "position",
+                "tension",
+                "trajectory",
+            )
+        )
+        haystack = f"{haystack} {evidence}".lower()
+        score = sum(
+            (4 if term in str(idea.get("title", "")).lower() else 1)
+            * haystack.count(term)
+            for term in terms
+        )
+        if score:
+            ranked.append((score, str(idea.get("updated_at", "")), idea))
+    fallback = not ranked
+    if fallback:
+        ranked = [
+            (0, str(idea.get("updated_at", "")), idea) for idea in ideas
+        ]
+    ranked.sort(key=lambda item: (item[0], item[1]), reverse=True)
+    return [idea for _, _, idea in ranked[:8]], fallback
+
+
+class CuratorRunHooks(RunHooks[CuratorContext]):
+    async def on_llm_start(
+        self,
+        context: RunContextWrapper[CuratorContext],
+        agent: Agent[CuratorContext],
+        system_prompt: str | None,
+        input_items: list[Any],
+    ) -> None:
+        context.context.llm_started_at = time.perf_counter()
+
+    async def on_llm_end(
+        self,
+        context: RunContextWrapper[CuratorContext],
+        agent: Agent[CuratorContext],
+        response: ModelResponse,
+    ) -> None:
+        state = context.context
+        state.model_rounds += 1
+        usage = response.usage
+        input_tokens = int(usage.input_tokens or 0)
+        output_tokens = int(usage.output_tokens or 0)
+        reasoning_tokens = int(
+            usage.output_tokens_details.reasoning_tokens or 0
+        )
+        cached_tokens = int(
+            usage.input_tokens_details.cached_tokens or 0
+        )
+        state.input_tokens += input_tokens
+        state.output_tokens += output_tokens
+        state.reasoning_tokens += reasoning_tokens
+        state.cached_tokens += cached_tokens
+        duration_ms = (
+            round((time.perf_counter() - state.llm_started_at) * 1000)
+            if state.llm_started_at is not None
+            else None
+        )
+        state.db.add_agent_event(
+            state.run_id,
+            state.user_id,
+            "model_round_completed",
+            {
+                "round": state.model_rounds,
+                "agent": agent.name,
+                "input_tokens": input_tokens,
+                "output_tokens": output_tokens,
+                "reasoning_tokens": reasoning_tokens,
+                "cached_tokens": cached_tokens,
+            },
+            duration_ms=duration_ms,
+        )
+        state.llm_started_at = None
+
+
+def build_curator_prompt(
+    fragment: dict[str, Any],
+    ideas: list[dict[str, Any]],
+    recent_fragments: list[dict[str, Any]],
+) -> str:
+    prior_context = [
+        {
+            "id": item["id"],
+            "created_at": item["created_at"],
+            "content": str(item["content"])[:1_200],
+        }
+        for item in recent_fragments
+        if item["id"] != fragment["id"]
+        and item["created_at"] <= fragment["created_at"]
+    ][-RECENT_CONTEXT_LIMIT:]
+    later_context = [
+        {
+            "created_at": item["created_at"],
+            "content": str(item["content"])[:1_200],
+        }
+        for item in recent_fragments
+        if item["id"] != fragment["id"]
+        and item["created_at"] > fragment["created_at"]
+    ][-6:]
+    catalog = [
+        {
+            "id": idea["id"],
+            "title": idea["title"],
+            "summary": idea["summary"],
+            "motion": idea["motion"],
+            "position": idea["position"],
+            "tension": idea["tension"],
+            "updated_at": idea["updated_at"],
+        }
+        for idea in ideas[:IDEA_CATALOG_LIMIT]
+    ]
+    new_fragment = {
+        "id": fragment["id"],
+        "created_at": fragment["created_at"],
+        "content": fragment["content"],
+    }
+    output_schema = CuratorDecision.model_json_schema()
+    catalog_note = (
+        "目录已经覆盖全部已有想法。优先从目录直接判断候选，相关时用准确 id 调用 "
+        "inspect_idea；一般不需要 search_ideas。"
+        if len(ideas) <= IDEA_CATALOG_LIMIT
+        else "目录只包含最近更新的想法；找不到可能候选时可调用 search_ideas。"
+    )
+    delayed_note = (
+        "这是一次延迟重整。later_context 是这条记录之后才发生的输入，只用于避免把现有想法"
+        "的状态和轨迹倒退；不要把它们误当成当时的上文。应把当前片段作为迟到的证据，结合"
+        "想法目录中的最新状态作出增量判断。\n\n"
+        if later_context
+        else ""
+    )
+    return (
+        "请策展这个刚刚保存的新片段。最近原文按时间从早到晚排列，它们是理解“这个、"
+        "第二点、继续、刚才”等连续表达的第一依据；不要要求用户显式建立会话。\n\n"
+        f"<recent_context>{json.dumps(prior_context, ensure_ascii=False)}</recent_context>\n\n"
+        f"<new_fragment>{json.dumps(new_fragment, ensure_ascii=False)}</new_fragment>\n\n"
+        f"<later_context>{json.dumps(later_context, ensure_ascii=False)}</later_context>\n\n"
+        f"{delayed_note}"
+        f"<idea_catalog>{json.dumps(catalog, ensure_ascii=False)}</idea_catalog>\n\n"
+        f"当前共有 {len(ideas)} 个已有想法。{catalog_note}\n"
+        "如果新片段既验证了一个既有想法、又形成了一个新的独立问题，可以给出两个 assessment；"
+        "不要因为表面主题变化而遗漏它对上一段思想过程的反馈。\n\n"
+        "最终只输出符合以下 JSON Schema 的 JSON 对象：\n"
+        f"{json.dumps(output_schema, ensure_ascii=False)}"
+    )
 
 
 @function_tool(strict_mode=False)
@@ -76,17 +268,31 @@ async def search_ideas(
     """Search existing evolving ideas by a short concept, question, or tension."""
     ctx.context.search_count += 1
     started = time.perf_counter()
-    terms = [term.lower() for term in query.split() if term.strip()]
-    ranked: list[tuple[int, dict[str, Any]]] = []
-    for idea in ctx.context.ideas:
-        haystack = " ".join(
-            str(idea.get(key, ""))
-            for key in ("title", "summary", "position", "tension", "trajectory")
-        ).lower()
-        score = sum(haystack.count(term) for term in terms)
-        if score or not terms:
-            ranked.append((score, idea))
-    ranked.sort(key=lambda item: (item[0], item[1].get("updated_at", "")), reverse=True)
+    if ctx.context.search_count > 2:
+        candidates = [
+            {"id": item["id"], "title": item["title"]}
+            for item in ctx.context.ideas[:8]
+        ]
+        ctx.context.db.add_agent_event(
+            ctx.context.run_id,
+            ctx.context.user_id,
+            "tool_search_ideas",
+            {
+                "query": query,
+                "result_count": len(candidates),
+                "budget_exhausted": True,
+                "candidates": candidates,
+            },
+            duration_ms=round((time.perf_counter() - started) * 1000),
+        )
+        return json.dumps(
+            {
+                "notice": "搜索额度已用完。使用已有目录并立即形成最终判断。",
+                "candidates": candidates,
+            },
+            ensure_ascii=False,
+        )
+    ranked, fallback = _rank_ideas(ctx.context.ideas, query)
     compact = [
         {
             "id": idea["id"],
@@ -97,7 +303,7 @@ async def search_ideas(
             "position": idea["position"],
             "tension": idea["tension"],
         }
-        for _, idea in ranked[:8]
+        for idea in ranked
     ]
     ctx.context.db.add_agent_event(
         ctx.context.run_id,
@@ -106,6 +312,7 @@ async def search_ideas(
         {
             "query": query,
             "result_count": len(compact),
+            "fallback_to_recent": fallback,
             "candidates": [
                 {"id": item["id"], "title": item["title"]} for item in compact
             ],
@@ -134,7 +341,39 @@ async def inspect_idea(
             {"idea_id": idea_id, "found": False},
             duration_ms=round((time.perf_counter() - started) * 1000),
         )
-        return json.dumps({"error": "idea not found"})
+        return json.dumps(
+            {
+                "error": "idea not found",
+                "valid_candidates": [
+                    {"id": item["id"], "title": item["title"]}
+                    for item in ctx.context.ideas[:8]
+                ],
+            },
+            ensure_ascii=False,
+        )
+    if idea_id in ctx.context.inspected_idea_ids:
+        ctx.context.db.add_agent_event(
+            ctx.context.run_id,
+            ctx.context.user_id,
+            "tool_inspect_idea",
+            {
+                "idea_id": idea_id,
+                "found": True,
+                "reused": True,
+                "title": idea["title"],
+            },
+            duration_ms=round((time.perf_counter() - started) * 1000),
+        )
+        return json.dumps(
+            {
+                "notice": (
+                    "这个想法已经检查过。使用上一份结果并立即形成最终判断，"
+                    "不要再次调用工具。"
+                )
+            },
+            ensure_ascii=False,
+        )
+    ctx.context.inspected_idea_ids.add(idea_id)
     payload = {
         "id": idea["id"],
         "title": idea["title"],
@@ -244,22 +483,19 @@ class Curator:
                 "fragment_characters": len(fragment["content"]),
                 "idea_count": len(ideas),
                 "recent_fragment_count": len(recent_fragments),
+                "prior_fragment_count": sum(
+                    item["id"] != fragment_id
+                    and item["created_at"] <= fragment["created_at"]
+                    for item in recent_fragments
+                ),
+                "later_fragment_count": sum(
+                    item["id"] != fragment_id
+                    and item["created_at"] > fragment["created_at"]
+                    for item in recent_fragments
+                ),
             },
         )
-        output_schema = CuratorDecision.model_json_schema()
-        lookup_instruction = (
-            "必须先调用 search_ideas 搜索共享的问题或张力；如果候选可能相关，"
-            "再调用 inspect_idea 查看它的真实轨迹后判断。\n\n"
-            if ideas
-            else "目前没有已有想法，不要调用搜索工具。\n\n"
-        )
-        prompt = (
-            "请策展这个刚刚保存的新片段：\n"
-            f"<fragment id=\"{fragment_id}\">{fragment['content']}</fragment>\n\n"
-            f"当前共有 {len(ideas)} 个已有想法。{lookup_instruction}"
-            "最终只输出符合以下 JSON Schema 的 JSON 对象：\n"
-            f"{json.dumps(output_schema, ensure_ascii=False)}"
-        )
+        prompt = build_curator_prompt(fragment, ideas, recent_fragments)
 
         set_tracing_disabled(True)
         client = AsyncOpenAI(
@@ -270,63 +506,115 @@ class Curator:
             model=self.settings.deepseek_model,
             openai_client=client,
         )
+        model_settings = ModelSettings(
+            max_tokens=2_400,
+            reasoning=Reasoning(effort="high"),
+            extra_body={"thinking": {"type": "enabled"}},
+            extra_args={"response_format": {"type": "json_object"}},
+        )
         agent: Agent[CuratorContext] = Agent(
             name="私人思想策展者",
             instructions=CURATOR_INSTRUCTIONS,
             model=model,
             tools=[search_ideas, inspect_idea],
-            model_settings=ModelSettings(
-                max_tokens=2_400,
-                reasoning=Reasoning(effort="high"),
-                extra_body={"thinking": {"type": "enabled"}},
-                extra_args={"response_format": {"type": "json_object"}},
+            model_settings=model_settings,
+        )
+        repair_agent: Agent[CuratorContext] = Agent(
+            name="判断收敛器",
+            instructions=(
+                CURATOR_INSTRUCTIONS
+                + "\n\n这是收敛回合。不能调用任何工具；只使用输入中已经提供的连续原文、"
+                "想法目录和 schema，立即给出最终合法 JSON。"
             ),
+            model=model,
+            tools=[],
+            model_settings=model_settings,
         )
         decision: CuratorDecision | None = None
         last_error: Exception | None = None
         attempt_count = 0
-        model_rounds = 0
-        usage = {
-            "input_tokens": 0,
-            "output_tokens": 0,
-            "reasoning_tokens": 0,
-            "cached_tokens": 0,
-        }
+        hooks = CuratorRunHooks()
         try:
             for attempt in range(2):
                 attempt_count = attempt + 1
-                repair = (
-                    ""
-                    if attempt == 0
-                    else "\n\n上一次输出未通过结构校验。重新完整判断，并只返回合法 JSON。"
-                )
+                is_repair = attempt > 0
                 self.db.add_agent_event(
                     run_id,
                     user_id,
                     "attempt_started",
-                    {"attempt": attempt_count, "is_repair": attempt > 0},
+                    {"attempt": attempt_count, "is_repair": is_repair},
                 )
                 attempt_started = time.perf_counter()
-                result = await Runner.run(
-                    agent,
-                    input=prompt + repair,
-                    context=context,
-                    max_turns=4,
-                )
+                rounds_before = context.model_rounds
+                usage_before = {
+                    "input_tokens": context.input_tokens,
+                    "output_tokens": context.output_tokens,
+                    "reasoning_tokens": context.reasoning_tokens,
+                    "cached_tokens": context.cached_tokens,
+                }
+                try:
+                    result = await Runner.run(
+                        repair_agent if is_repair else agent,
+                        input=(
+                            prompt
+                            + (
+                                "\n\n上一次工具探索或输出没有收敛。不要再探索，"
+                                "重新完整判断并只返回合法 JSON。"
+                                if is_repair
+                                else ""
+                            )
+                        ),
+                        context=context,
+                        max_turns=2 if is_repair else 6,
+                        hooks=hooks,
+                    )
+                except MaxTurnsExceeded as exc:
+                    attempt_duration = round(
+                        (time.perf_counter() - attempt_started) * 1000
+                    )
+                    current_usage = {
+                        key: getattr(context, key) - value
+                        for key, value in usage_before.items()
+                    }
+                    self.db.add_agent_event(
+                        run_id,
+                        user_id,
+                        "model_attempt_completed",
+                        {
+                            "attempt": attempt_count,
+                            "model_rounds": (
+                                context.model_rounds - rounds_before
+                            ),
+                            "completed": False,
+                            "error_type": type(exc).__name__,
+                            **current_usage,
+                        },
+                        duration_ms=attempt_duration,
+                    )
+                    last_error = exc
+                    if not is_repair:
+                        self.db.add_agent_event(
+                            run_id,
+                            user_id,
+                            "convergence_repair_started",
+                            {"reason": "tool_loop_exceeded"},
+                        )
+                    continue
                 attempt_duration = round(
                     (time.perf_counter() - attempt_started) * 1000
                 )
-                current_usage = self._result_usage(result)
-                model_rounds += current_usage.pop("model_rounds")
-                for key in usage:
-                    usage[key] += current_usage[key]
+                current_usage = {
+                    key: getattr(context, key) - value
+                    for key, value in usage_before.items()
+                }
                 self.db.add_agent_event(
                     run_id,
                     user_id,
                     "model_attempt_completed",
                     {
                         "attempt": attempt_count,
-                        "model_rounds": len(result.raw_responses),
+                        "model_rounds": context.model_rounds - rounds_before,
+                        "completed": True,
                         **current_usage,
                     },
                     duration_ms=attempt_duration,
@@ -336,9 +624,17 @@ class Curator:
                     if not isinstance(raw, str):
                         raise TypeError("curator output was not text")
                     decision = CuratorDecision.model_validate_json(raw)
-                    if ideas and context.search_count == 0:
-                        decision = None
-                        raise ValueError("curator skipped required idea search")
+                    valid_idea_ids = {idea["id"] for idea in ideas}
+                    unknown_ids = [
+                        item.idea_id
+                        for item in decision.assessments
+                        if item.idea_id
+                        and item.idea_id not in valid_idea_ids
+                    ]
+                    if unknown_ids:
+                        raise ValueError(
+                            "curator referenced an unknown existing idea"
+                        )
                     break
                 except (ValidationError, ValueError, TypeError) as exc:
                     decision = None
@@ -358,9 +654,16 @@ class Curator:
                         fragment_id,
                         attempt_count,
                     )
+                    if not is_repair:
+                        self.db.add_agent_event(
+                            run_id,
+                            user_id,
+                            "convergence_repair_started",
+                            {"reason": "output_validation_failed"},
+                        )
             if decision is None:
                 raise RuntimeError(
-                    "curator returned invalid JSON twice"
+                    "curator could not produce a valid decision"
                 ) from last_error
             assessments = (
                 []
@@ -386,11 +689,14 @@ class Curator:
                 status="success",
                 duration_ms=round((time.perf_counter() - run_started) * 1000),
                 attempt_count=attempt_count,
-                model_rounds=model_rounds,
+                model_rounds=context.model_rounds,
                 tool_calls=context.search_count + context.inspection_count,
                 search_calls=context.search_count,
                 inspection_calls=context.inspection_count,
-                **usage,
+                input_tokens=context.input_tokens,
+                output_tokens=context.output_tokens,
+                reasoning_tokens=context.reasoning_tokens,
+                cached_tokens=context.cached_tokens,
             )
             logger.info(
                 "Curator completed fragment %s with %s search and %s inspection calls",
@@ -413,32 +719,14 @@ class Curator:
                 status="error",
                 duration_ms=round((time.perf_counter() - run_started) * 1000),
                 attempt_count=attempt_count,
-                model_rounds=model_rounds,
+                model_rounds=context.model_rounds,
                 tool_calls=context.search_count + context.inspection_count,
                 search_calls=context.search_count,
                 inspection_calls=context.inspection_count,
                 error=exc,
-                **usage,
+                input_tokens=context.input_tokens,
+                output_tokens=context.output_tokens,
+                reasoning_tokens=context.reasoning_tokens,
+                cached_tokens=context.cached_tokens,
             )
             raise
-
-    @staticmethod
-    def _result_usage(result: Any) -> dict[str, int]:
-        totals = {
-            "model_rounds": len(result.raw_responses),
-            "input_tokens": 0,
-            "output_tokens": 0,
-            "reasoning_tokens": 0,
-            "cached_tokens": 0,
-        }
-        for response in result.raw_responses:
-            response_usage = response.usage
-            totals["input_tokens"] += response_usage.input_tokens
-            totals["output_tokens"] += response_usage.output_tokens
-            totals["reasoning_tokens"] += (
-                response_usage.output_tokens_details.reasoning_tokens or 0
-            )
-            totals["cached_tokens"] += (
-                response_usage.input_tokens_details.cached_tokens or 0
-            )
-        return totals