Add online mining v2 skill

2026-05-12 17:01:43 +08:00
parent c517d93034
commit 6dad2cc4b3
10 changed files with 1105 additions and 0 deletions
@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+
+from online_mining_common import (
+    compact_text,
+    emit_error,
+    emit_success,
+    extract_case,
+    fetch_one_by_request_id,
+    load_json_payload,
+    resolve_portable_path,
+    string_values,
+    write_optional_jsonl,
+)
+
+
+def load_request_ids(payload: dict) -> list[str]:
+    ids = string_values(payload.get("request_ids"))
+    if ids:
+        return ids
+    cases_path = payload.get("cases_path")
+    if cases_path:
+        request_ids = []
+        text = resolve_portable_path(str(cases_path)).read_text(encoding="utf-8")
+        for line in text.splitlines():
+            if not line.strip():
+                continue
+            item = json.loads(line)
+            rid = item.get("request_id") or item.get("requestId")
+            if rid:
+                request_ids.append(str(rid))
+        return request_ids
+    cases = payload.get("cases") or []
+    if isinstance(cases, list):
+        return [str(item.get("request_id")) for item in cases if isinstance(item, dict) and item.get("request_id")]
+    return []
+
+
+def merge_case(request_id: str, main_case: dict | None, pre_case: dict | None) -> dict:
+    return {
+        "request_id": request_id,
+        "query": (pre_case or {}).get("query") or (main_case or {}).get("query"),
+        "timestamp": (main_case or {}).get("timestamp") or (pre_case or {}).get("timestamp"),
+        "session_id": (main_case or {}).get("session_id"),
+        "device_id": (main_case or {}).get("device_id"),
+        "device": (main_case or {}).get("device"),
+        "tts": (main_case or {}).get("text"),
+        "main": main_case,
+        "pre_processing": pre_case,
+    }
+
+
+def main() -> int:
+    try:
+        payload = load_json_payload()
+        request_ids = load_request_ids(payload)
+        if not request_ids:
+            raise ValueError("request_ids, cases or cases_path is required")
+        date = payload.get("date")
+        rows = []
+        for request_id in request_ids:
+            main_doc = fetch_one_by_request_id("main", request_id, date)
+            pre_doc = fetch_one_by_request_id("pre_processing", request_id, date)
+            main_case = extract_case("main", main_doc) if main_doc else None
+            pre_case = extract_case("pre_processing", pre_doc) if pre_doc else None
+            rows.append(merge_case(request_id, main_case, pre_case))
+        output_path = write_optional_jsonl(str(payload.get("output_path") or ""), rows)
+        emit_success(
+            {
+                "date": date or "past-48h",
+                "count": len(rows),
+                "output_path": output_path,
+                "cases": [
+                    {
+                        key: compact_text(value, 1200)
+                        for key, value in row.items()
+                        if key not in {"main", "pre_processing"} and value not in (None, "", [], {})
+                    }
+                    | {
+                        "main_domain": (row.get("main") or {}).get("domain"),
+                        "main_func": (row.get("main") or {}).get("func"),
+                        "planning_result": (row.get("pre_processing") or {}).get("planning_result"),
+                        "candidate_domains": (row.get("pre_processing") or {}).get("candidate_domains"),
+                    }
+                    for row in rows
+                ],
+            }
+        )
+        return 0
+    except Exception as exc:  # noqa: BLE001
+        emit_error(exc)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+