wuyang6
99 lines
3.5 KiB
Python
99 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from online_mining_common import (
|
|
compact_text,
|
|
emit_error,
|
|
emit_success,
|
|
extract_case,
|
|
fetch_one_by_request_id,
|
|
load_json_payload,
|
|
resolve_portable_path,
|
|
string_values,
|
|
write_optional_jsonl,
|
|
)
|
|
|
|
|
|
def load_request_ids(payload: dict) -> list[str]:
|
|
ids = string_values(payload.get("request_ids"))
|
|
if ids:
|
|
return ids
|
|
cases_path = payload.get("cases_path")
|
|
if cases_path:
|
|
request_ids = []
|
|
text = resolve_portable_path(str(cases_path)).read_text(encoding="utf-8")
|
|
for line in text.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
item = json.loads(line)
|
|
rid = item.get("request_id") or item.get("requestId")
|
|
if rid:
|
|
request_ids.append(str(rid))
|
|
return request_ids
|
|
cases = payload.get("cases") or []
|
|
if isinstance(cases, list):
|
|
return [str(item.get("request_id")) for item in cases if isinstance(item, dict) and item.get("request_id")]
|
|
return []
|
|
|
|
|
|
def merge_case(request_id: str, main_case: dict | None, pre_case: dict | None) -> dict:
|
|
return {
|
|
"request_id": request_id,
|
|
"query": (pre_case or {}).get("query") or (main_case or {}).get("query"),
|
|
"timestamp": (main_case or {}).get("timestamp") or (pre_case or {}).get("timestamp"),
|
|
"session_id": (main_case or {}).get("session_id"),
|
|
"device_id": (main_case or {}).get("device_id"),
|
|
"device": (main_case or {}).get("device"),
|
|
"tts": (main_case or {}).get("text"),
|
|
"main": main_case,
|
|
"pre_processing": pre_case,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
payload = load_json_payload()
|
|
request_ids = load_request_ids(payload)
|
|
if not request_ids:
|
|
raise ValueError("request_ids, cases or cases_path is required")
|
|
date = payload.get("date")
|
|
rows = []
|
|
for request_id in request_ids:
|
|
main_doc = fetch_one_by_request_id("main", request_id, date)
|
|
pre_doc = fetch_one_by_request_id("pre_processing", request_id, date)
|
|
main_case = extract_case("main", main_doc) if main_doc else None
|
|
pre_case = extract_case("pre_processing", pre_doc) if pre_doc else None
|
|
rows.append(merge_case(request_id, main_case, pre_case))
|
|
output_path = write_optional_jsonl(str(payload.get("output_path") or ""), rows)
|
|
emit_success(
|
|
{
|
|
"date": date or "past-48h",
|
|
"count": len(rows),
|
|
"output_path": output_path,
|
|
"cases": [
|
|
{
|
|
key: compact_text(value, 1200)
|
|
for key, value in row.items()
|
|
if key not in {"main", "pre_processing"} and value not in (None, "", [], {})
|
|
}
|
|
| {
|
|
"main_domain": (row.get("main") or {}).get("domain"),
|
|
"main_func": (row.get("main") or {}).get("func"),
|
|
"planning_result": (row.get("pre_processing") or {}).get("planning_result"),
|
|
"candidate_domains": (row.get("pre_processing") or {}).get("candidate_domains"),
|
|
}
|
|
for row in rows
|
|
],
|
|
}
|
|
)
|
|
return 0
|
|
except Exception as exc: # noqa: BLE001
|
|
emit_error(exc)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
|