wuyang6
136 lines
4.9 KiB
Python
136 lines
4.9 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
|
||
from online_mining_common import (
|
||
emit_error,
|
||
emit_success,
|
||
load_json_payload,
|
||
resolve_portable_path,
|
||
)
|
||
|
||
|
||
def load_cases(payload: dict) -> list[dict]:
|
||
if isinstance(payload.get("cases"), list):
|
||
return [item for item in payload["cases"] if isinstance(item, dict)]
|
||
cases_path = payload.get("cases_path")
|
||
if not cases_path:
|
||
raise ValueError("cases or cases_path is required")
|
||
rows = []
|
||
text = resolve_portable_path(str(cases_path)).read_text(encoding="utf-8")
|
||
for line in text.splitlines():
|
||
if not line.strip():
|
||
continue
|
||
item = json.loads(line)
|
||
if isinstance(item, dict):
|
||
rows.append(item)
|
||
return rows
|
||
|
||
|
||
def case_value(case: dict, key: str):
|
||
if key in case:
|
||
return case.get(key)
|
||
main = case.get("main") if isinstance(case.get("main"), dict) else {}
|
||
pre = case.get("pre_processing") if isinstance(case.get("pre_processing"), dict) else {}
|
||
return case.get(key) or pre.get(key) or main.get(key)
|
||
|
||
|
||
def build_dataset_draft_text(
|
||
*,
|
||
dataset_label: str,
|
||
target: str,
|
||
complex_value: bool,
|
||
cases: list[dict],
|
||
) -> str:
|
||
"""把 review 后的线上 case 转成 product-data dataset draft text。
|
||
|
||
这个 draft 是中间格式，不是最终数据。后续必须再经过
|
||
product-data 的 normalize/validate/export。
|
||
"""
|
||
|
||
lines = [f"# dataset_label: {dataset_label}", ""]
|
||
for index, case in enumerate(cases, start=1):
|
||
query = str(case_value(case, "query") or "").strip()
|
||
if not query:
|
||
continue
|
||
request_id = str(case_value(case, "request_id") or "").strip()
|
||
timestamp = case_value(case, "timestamp")
|
||
tts = str(case_value(case, "tts") or "").strip()
|
||
planning_result = str(case_value(case, "planning_result") or "").strip()
|
||
case_target = str(case_value(case, "target") or target).strip()
|
||
case_complex = case_value(case, "complex")
|
||
if isinstance(case_complex, str):
|
||
case_complex_text = case_complex.strip().lower()
|
||
item_complex = case_complex_text in {"true", "1", "yes", "y", "是", "复杂", "complex"}
|
||
elif isinstance(case_complex, bool):
|
||
item_complex = case_complex
|
||
else:
|
||
item_complex = complex_value
|
||
notes = []
|
||
if planning_result:
|
||
notes.append(f"线上模型输出: {planning_result}")
|
||
main_domain = case_value(case, "domain") or case.get("main_domain")
|
||
if main_domain:
|
||
notes.append(f"线上 domain: {main_domain}")
|
||
lines.append(f"### case: online_{index:04d}")
|
||
prev_session = case_value(case, "prev_session")
|
||
if isinstance(prev_session, list):
|
||
for item in prev_session[-10:]:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
prev_query = str(item.get("query") or "").strip()
|
||
prev_tts = str(item.get("tts") or "").strip()
|
||
if prev_query and prev_tts:
|
||
lines.append(f"用户: {prev_query}")
|
||
lines.append(f"小爱: {prev_tts}")
|
||
lines.append(f"用户: {query}")
|
||
lines.append(f"complex: {'true' if item_complex else 'false'}")
|
||
lines.append(f"target: {case_target}")
|
||
if request_id:
|
||
lines.append(f"request_id: {request_id}")
|
||
if timestamp:
|
||
lines.append(f"timestamp: {timestamp}")
|
||
if tts:
|
||
lines.append(f"notes: tts={tts}" + (f"；{'；'.join(notes)}" if notes else ""))
|
||
elif notes:
|
||
lines.append(f"notes: {'；'.join(notes)}")
|
||
lines.append("")
|
||
return "\n".join(lines).strip() + "\n"
|
||
|
||
|
||
def main() -> int:
|
||
try:
|
||
payload = load_json_payload()
|
||
dataset_label = str(payload.get("dataset_label") or "").strip()
|
||
target = str(payload.get("target") or "").strip()
|
||
if not dataset_label:
|
||
raise ValueError("dataset_label is required")
|
||
if not target:
|
||
raise ValueError("target is required")
|
||
complex_value = bool(payload.get("complex", False))
|
||
cases = load_cases(payload)
|
||
if not cases:
|
||
raise ValueError("cases must not be empty")
|
||
|
||
draft_text = build_dataset_draft_text(
|
||
dataset_label=dataset_label,
|
||
target=target,
|
||
complex_value=complex_value,
|
||
cases=cases,
|
||
)
|
||
output_path = None
|
||
if payload.get("output_path"):
|
||
output = resolve_portable_path(str(payload["output_path"]))
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
output.write_text(draft_text, encoding="utf-8")
|
||
output_path = str(output)
|
||
emit_success({"case_count": draft_text.count("### case:"), "draft_text": draft_text, "output_path": output_path})
|
||
return 0
|
||
except Exception as exc: # noqa: BLE001
|
||
emit_error(exc)
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|