Improve online mining record conversion
This commit is contained in:
wuyang6
@@ -0,0 +1,353 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""把线上 ELK case 直接转换为 product-data canonical records。
|
||||
|
||||
这个脚本承接 online-mining-v2 和 product-data：
|
||||
1. 按 request_id 拉主 NLP 表和前处理表。
|
||||
2. 用主表 session_id 补齐当前请求前最多 10 轮上下文。
|
||||
3. 从用户指定 target 或线上 planning/code 结果推断标签。
|
||||
4. 直接产出 canonical records，并可同步导出流通表格、训练 jsonl、评测 csv。
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from online_mining_common import (
|
||||
OnlineMiningError,
|
||||
compact_text,
|
||||
emit_error,
|
||||
emit_success,
|
||||
extract_case,
|
||||
fetch_one_by_request_id,
|
||||
fetch_session_turns,
|
||||
load_json_payload,
|
||||
resolve_portable_path,
|
||||
string_values,
|
||||
)
|
||||
|
||||
PRODUCT_DATA_SCRIPT_DIR = Path(__file__).resolve().parents[1].parent / "product-data" / "scripts"
|
||||
sys.path.insert(0, str(PRODUCT_DATA_SCRIPT_DIR))
|
||||
|
||||
from product_data_portable import ( # noqa: E402
|
||||
canonical_target,
|
||||
export_dataset_records,
|
||||
export_planning_eval_csv,
|
||||
export_training_jsonl,
|
||||
record_id,
|
||||
target_type,
|
||||
validate_dataset_records,
|
||||
)
|
||||
|
||||
|
||||
def load_cases(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if isinstance(payload.get("cases"), list):
|
||||
return [item for item in payload["cases"] if isinstance(item, dict)]
|
||||
cases_path = payload.get("cases_path")
|
||||
if cases_path:
|
||||
rows = []
|
||||
text = resolve_portable_path(str(cases_path)).read_text(encoding="utf-8")
|
||||
for line in text.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
item = json.loads(line)
|
||||
if isinstance(item, dict):
|
||||
rows.append(item)
|
||||
return rows
|
||||
return []
|
||||
|
||||
|
||||
def merge_case(request_id: str, main_case: dict[str, Any] | None, pre_case: dict[str, Any] | None) -> dict[str, Any]:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"query": (pre_case or {}).get("query") or (main_case or {}).get("query"),
|
||||
"timestamp": (main_case or {}).get("timestamp") or (pre_case or {}).get("timestamp"),
|
||||
"session_id": (main_case or {}).get("session_id"),
|
||||
"device_id": (main_case or {}).get("device_id"),
|
||||
"device": (main_case or {}).get("device"),
|
||||
"tts": (main_case or {}).get("text"),
|
||||
"main": main_case,
|
||||
"pre_processing": pre_case,
|
||||
}
|
||||
|
||||
|
||||
def fetch_cases_by_request_ids(request_ids: list[str], *, date: str | None) -> list[dict[str, Any]]:
|
||||
rows = []
|
||||
for request_id in request_ids:
|
||||
main_doc = fetch_one_by_request_id("main", request_id, date)
|
||||
pre_doc = fetch_one_by_request_id("pre_processing", request_id, date)
|
||||
main_case = extract_case("main", main_doc) if main_doc else None
|
||||
pre_case = extract_case("pre_processing", pre_doc) if pre_doc else None
|
||||
rows.append(merge_case(request_id, main_case, pre_case))
|
||||
return rows
|
||||
|
||||
|
||||
def case_value(case: dict[str, Any], key: str) -> Any:
|
||||
if key in case:
|
||||
return case.get(key)
|
||||
main = case.get("main") if isinstance(case.get("main"), dict) else {}
|
||||
pre = case.get("pre_processing") if isinstance(case.get("pre_processing"), dict) else {}
|
||||
return case.get(key) or pre.get(key) or main.get(key)
|
||||
|
||||
|
||||
def enrich_prev_session(case: dict[str, Any], *, date: str | None, limit: int) -> list[dict[str, Any]]:
|
||||
explicit = case.get("prev_session")
|
||||
if isinstance(explicit, list):
|
||||
return normalize_prev_session(explicit)[-limit:]
|
||||
session_id = str(case_value(case, "session_id") or "").strip()
|
||||
timestamp = optional_int(case_value(case, "timestamp"))
|
||||
if not session_id or timestamp is None:
|
||||
return []
|
||||
turns = fetch_session_turns(session_id=session_id, before_timestamp=timestamp, date=date, limit=limit)
|
||||
return [
|
||||
{
|
||||
"query": str(turn.get("query") or ""),
|
||||
"tts": str(turn.get("text") or ""),
|
||||
"timestamp": optional_int(turn.get("timestamp")) or 0,
|
||||
}
|
||||
for turn in turns
|
||||
if str(turn.get("query") or "").strip()
|
||||
][-limit:]
|
||||
|
||||
|
||||
def normalize_prev_session(items: list[Any]) -> list[dict[str, Any]]:
|
||||
rows = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
query = str(item.get("query") or "").strip()
|
||||
if not query:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"query": query,
|
||||
"tts": str(item.get("tts") or ""),
|
||||
"timestamp": optional_int(item.get("timestamp")) or 0,
|
||||
}
|
||||
)
|
||||
rows.sort(key=lambda item: int(item.get("timestamp") or 0))
|
||||
return rows
|
||||
|
||||
|
||||
def infer_target(case: dict[str, Any], explicit_target: str) -> tuple[str, str]:
|
||||
if explicit_target.strip():
|
||||
return canonical_target(explicit_target.strip()), "input.target"
|
||||
case_target = str(case_value(case, "target") or "").strip()
|
||||
if case_target:
|
||||
return canonical_target(case_target), "case.target"
|
||||
|
||||
pre = case.get("pre_processing") if isinstance(case.get("pre_processing"), dict) else {}
|
||||
main = case.get("main") if isinstance(case.get("main"), dict) else {}
|
||||
for key in ("code", "planning_result"):
|
||||
value = str(pre.get(key) or "").strip()
|
||||
if value:
|
||||
return canonical_target(value), f"pre_processing.{key}"
|
||||
|
||||
llm_agent_info = main.get("llm_agent_info")
|
||||
if isinstance(llm_agent_info, dict):
|
||||
agent_type = str(llm_agent_info.get("agentType") or "").strip()
|
||||
if agent_type:
|
||||
return f'Agent(tag="{agent_type}")', "main.intention.intent_arbitrator_info.llm_agent_info.agentType"
|
||||
|
||||
domain = str(main.get("domain") or case_value(case, "domain") or "").strip()
|
||||
if domain:
|
||||
if re.fullmatch(r"[A-Z][A-Za-z0-9_]*", domain):
|
||||
return f"{domain}()", "main.domain"
|
||||
return f'Agent(tag="{domain}")', "main.domain"
|
||||
raise OnlineMiningError("target is required because online logs do not contain code/planning_result/domain")
|
||||
|
||||
|
||||
def case_to_record(
|
||||
case: dict[str, Any],
|
||||
*,
|
||||
dataset_label: str,
|
||||
target: str,
|
||||
complex_value: bool,
|
||||
index: int,
|
||||
date: str | None,
|
||||
session_limit: int,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
query = str(case_value(case, "query") or "").strip()
|
||||
if not query:
|
||||
raise OnlineMiningError(f"case {index} has no query")
|
||||
request_id = str(case_value(case, "request_id") or "").strip()
|
||||
if not request_id:
|
||||
raise OnlineMiningError(f"case {index} has no request_id")
|
||||
timestamp = optional_int(case_value(case, "timestamp"))
|
||||
if timestamp is None:
|
||||
raise OnlineMiningError(f"case {index} has no timestamp")
|
||||
|
||||
final_target, target_source = infer_target(case, target)
|
||||
prev_session = enrich_prev_session(case, date=date, limit=session_limit)
|
||||
main = case.get("main") if isinstance(case.get("main"), dict) else {}
|
||||
pre = case.get("pre_processing") if isinstance(case.get("pre_processing"), dict) else {}
|
||||
record = {
|
||||
"record_id": record_id(source_type="online", batch_id="online", request_id=request_id, index=index),
|
||||
"source": {
|
||||
"type": "online",
|
||||
"request_id": request_id,
|
||||
"timestamp": timestamp,
|
||||
},
|
||||
"turn": {
|
||||
"query": query,
|
||||
"timestamp": timestamp,
|
||||
},
|
||||
"prev_session": prev_session,
|
||||
"context": {},
|
||||
"label": {
|
||||
"dataset_label": dataset_label,
|
||||
"target": final_target,
|
||||
"target_type": target_type(final_target),
|
||||
},
|
||||
"dimensions": {
|
||||
"complex": parse_complex(case_value(case, "complex"), default=complex_value),
|
||||
},
|
||||
"meta": {
|
||||
"session_id": str(case_value(case, "session_id") or ""),
|
||||
"device_id": str(case_value(case, "device_id") or ""),
|
||||
"device": compact_text(case_value(case, "device"), 500),
|
||||
"online_tts": str(case_value(case, "tts") or ""),
|
||||
"main_domain": str(main.get("domain") or ""),
|
||||
"main_func": str(main.get("func") or ""),
|
||||
"planning_result": str(pre.get("planning_result") or ""),
|
||||
"planning_code": str(pre.get("code") or ""),
|
||||
"hit_rules": pre.get("hit_rules") or main.get("hit_rules") or [],
|
||||
"candidate_domains": pre.get("candidate_domains") or [],
|
||||
"target_source": target_source,
|
||||
},
|
||||
}
|
||||
summary = {
|
||||
"request_id": request_id,
|
||||
"query": query,
|
||||
"target": final_target,
|
||||
"target_source": target_source,
|
||||
"prev_session_count": len(prev_session),
|
||||
"main_domain": record["meta"]["main_domain"],
|
||||
"planning_result": record["meta"]["planning_result"],
|
||||
}
|
||||
return record, summary
|
||||
|
||||
|
||||
def optional_int(value: Any) -> int | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_complex(value: Any, *, default: bool) -> bool:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
text = str(value).strip().lower()
|
||||
if text in {"true", "1", "yes", "y", "是", "复杂", "complex"}:
|
||||
return True
|
||||
if text in {"false", "0", "no", "n", "否", "不复杂", "简单", "simple"}:
|
||||
return False
|
||||
raise OnlineMiningError("complex must be true/false")
|
||||
|
||||
|
||||
def write_records_jsonl(records: list[dict[str, Any]], path: str) -> str:
|
||||
output = resolve_portable_path(path)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output.open("w", encoding="utf-8") as fh:
|
||||
for record in records:
|
||||
fh.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
return str(output)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = load_json_payload()
|
||||
dataset_label = str(payload.get("dataset_label") or "").strip()
|
||||
if not dataset_label:
|
||||
raise OnlineMiningError("dataset_label is required")
|
||||
target = str(payload.get("target") or "").strip()
|
||||
complex_value = parse_complex(payload.get("complex"), default=False)
|
||||
date = str(payload.get("date") or "").strip() or None
|
||||
session_limit = int(payload.get("session_limit") or 10)
|
||||
if session_limit < 0 or session_limit > 10:
|
||||
raise OnlineMiningError("session_limit must be between 0 and 10")
|
||||
|
||||
request_ids = string_values(payload.get("request_ids"))
|
||||
cases = load_cases(payload)
|
||||
if request_ids:
|
||||
cases = fetch_cases_by_request_ids(request_ids, date=date)
|
||||
if not cases:
|
||||
raise OnlineMiningError("request_ids, cases or cases_path is required")
|
||||
|
||||
records = []
|
||||
summaries = []
|
||||
skipped = []
|
||||
for index, case in enumerate(cases, start=1):
|
||||
try:
|
||||
record, summary = case_to_record(
|
||||
case,
|
||||
dataset_label=dataset_label,
|
||||
target=target,
|
||||
complex_value=complex_value,
|
||||
index=index,
|
||||
date=date,
|
||||
session_limit=session_limit,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
skipped.append({"index": index, "request_id": case.get("request_id"), "error": str(exc)})
|
||||
continue
|
||||
records.append(record)
|
||||
summaries.append(summary)
|
||||
|
||||
if not records:
|
||||
raise OnlineMiningError(f"no records built; skipped={skipped}")
|
||||
|
||||
records_path = write_records_jsonl(records, str(payload.get("records_output_path") or "scratchpad/online_records.jsonl"))
|
||||
validation = validate_dataset_records(records)
|
||||
output_dir = str(payload.get("output_dir") or "output")
|
||||
exports: dict[str, Any] = {}
|
||||
if bool(payload.get("export_records", True)):
|
||||
exports["records"] = export_dataset_records(
|
||||
records,
|
||||
output_path=str(resolve_portable_path(output_dir) / "records.jsonl"),
|
||||
output_format="jsonl",
|
||||
require_validation_ok=bool(payload.get("require_validation_ok", True)),
|
||||
overwrite=bool(payload.get("overwrite", True)),
|
||||
export_table=bool(payload.get("export_table", True)),
|
||||
)
|
||||
if bool(payload.get("export_training", False)):
|
||||
exports["training"] = export_training_jsonl(
|
||||
records,
|
||||
output_path=str(resolve_portable_path(output_dir) / "training.jsonl"),
|
||||
require_validation_ok=bool(payload.get("require_validation_ok", True)),
|
||||
overwrite=bool(payload.get("overwrite", True)),
|
||||
)
|
||||
if bool(payload.get("export_eval", False)):
|
||||
exports["eval"] = export_planning_eval_csv(
|
||||
records,
|
||||
output_path=str(resolve_portable_path(output_dir) / "eval_planning.csv"),
|
||||
require_validation_ok=bool(payload.get("require_validation_ok", True)),
|
||||
overwrite=bool(payload.get("overwrite", True)),
|
||||
)
|
||||
|
||||
emit_success(
|
||||
{
|
||||
"date": date or "past-48h",
|
||||
"record_count": len(records),
|
||||
"records_path": records_path,
|
||||
"validation": validation,
|
||||
"exports": exports,
|
||||
"summaries": summaries,
|
||||
"skipped": skipped,
|
||||
}
|
||||
)
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
emit_error(exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user