Improve online mining record conversion

skills/online-mining-v2/scripts/online_mining_common.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 import argparse
 import importlib.util
 import json
+import os
 import re
 import sys
 from copy import deepcopy
@@ -35,7 +36,7 @@ SOURCE_CONFIGS: dict[str, dict[str, Any]] = {
     },
     "pre_processing": {
         "profile": "default",
-        "index": "pre-processing-info*",
+        "index": "pre-processing*",
         "id_field": "requestId",
         "time_field": "timestamp",
     },
@@ -282,21 +283,70 @@ def search_docs(
 
 def fetch_one_by_request_id(source: str, request_id: str, date: str | None = None) -> dict[str, Any] | None:
     config = source_config(source)
-    docs = search_docs(
-        source=source,
-        date=date,
-        size=1,
-        query_filters=[{"term": {str(config["id_field"]): request_id}}],
-    )
-    if not docs:
-        # 部分 request id 会带后缀，兜底用 wildcard。
-        docs = search_docs(
-            source=source,
-            date=date,
-            size=1,
-            query_filters=[{"wildcard": {str(config["id_field"]): {"value": f"{request_id}*"}}}],
-        )
-    return docs[0] if docs else None
+    id_field = str(config["id_field"])
+    # 精确查优先用 .keyword；部分索引字段本身就是 keyword，所以再兜底原字段。
+    attempts = [
+        {"term": {f"{id_field}.keyword": request_id}},
+        {"term": {id_field: request_id}},
+        {"wildcard": {f"{id_field}.keyword": {"value": f"{request_id}*"}}},
+        {"wildcard": {id_field: {"value": f"{request_id}*"}}},
+    ]
+    for query_filter in attempts:
+        docs = search_docs(source=source, date=date, size=1, query_filters=[query_filter])
+        if docs:
+            return docs[0]
+    return None
+
+
+def fetch_session_turns(
+    *,
+    session_id: str,
+    before_timestamp: int,
+    date: str | None = None,
+    limit: int = 10,
+) -> list[dict[str, Any]]:
+    """按主表 session_id 拉当前请求之前的多轮上下文。"""
+
+    if not session_id or not before_timestamp:
+        return []
+    filters = [
+        {"term": {"session_id.keyword": session_id}},
+        {"range": {"timestamp": {"lt": before_timestamp}}},
+    ]
+    docs = search_docs(source="main", date=date, size=max(limit * 3, limit), query_filters=filters)
+    cases = [extract_main_case(doc) for doc in docs]
+    cases = [
+        case
+        for case in cases
+        if str(case.get("session_id") or "") == session_id
+        and isinstance(case.get("timestamp"), (int, str))
+        and int(case.get("timestamp") or 0) < before_timestamp
+        and str(case.get("query") or "").strip()
+    ]
+    cases.sort(key=lambda item: int(item.get("timestamp") or 0))
+    return cases[-limit:]
+
+
+def fetch_session_turns_for_case(case: dict[str, Any], *, date: str | None = None, limit: int = 10) -> list[dict[str, Any]]:
+    session_id = str(case.get("session_id") or "").strip()
+    try:
+        timestamp = int(case.get("timestamp") or 0)
+    except (TypeError, ValueError):
+        timestamp = 0
+    return fetch_session_turns(session_id=session_id, before_timestamp=timestamp, date=date, limit=limit)
+
+
+def fetch_by_terms(source: str, *, field: str, value: str, date: str | None = None, size: int = 10) -> list[dict[str, Any]]:
+    """小范围精确字段查询，供脚本探测 session/request 字段时使用。"""
+
+    filters = [{"term": {field: value}}]
+    return search_docs(source=source, date=date, size=size, query_filters=filters)
+
+
+def fetch_one_by_request_id_legacy(source: str, request_id: str, date: str | None = None) -> dict[str, Any] | None:
+    """保留旧函数名兼容外部脚本；新代码请用 fetch_one_by_request_id。"""
+
+    return fetch_one_by_request_id(source, request_id, date)
 
 
 def write_optional_jsonl(path: str | None, rows: list[dict[str, Any]]) -> str | None:
@@ -311,8 +361,27 @@ def write_optional_jsonl(path: str | None, rows: list[dict[str, Any]]) -> str |
 
 
 def resolve_portable_path(path: str) -> Path:
+    """解析 online-mining-v2 脚本路径，并优先映射到当前会话目录。
+
+    在 ZK Data Agent 中，python_exec 会注入 PYTHON_EXEC_SCRATCHPAD。
+    用户和 skill 文档里写的 scratchpad/、output/、input/ 都应该落在
+    当前会话下，不能误写到项目根目录。
+    """
+
     raw = Path(path).expanduser()
     if raw.is_absolute():
         return raw
+    scratchpad = os.environ.get("PYTHON_EXEC_SCRATCHPAD")
+    if scratchpad:
+        session_root = Path(scratchpad).expanduser().parent
+        parts = raw.parts
+        if parts:
+            head, *tail = parts
+            tail_path = Path(*tail) if tail else Path()
+            if head in {"scratchpad", "scratch"}:
+                return (Path(scratchpad).expanduser() / tail_path).resolve()
+            if head in {"output", "outputs"}:
+                return (session_root / "output" / tail_path).resolve()
+            if head in {"input", "inputs"}:
+                return (session_root / "input" / tail_path).resolve()
     return (REPO_ROOT / raw).resolve()
-