Add online mining v2 skill
This commit is contained in:
wuyang6
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from online_mining_common import (
|
||||
emit_error,
|
||||
emit_success,
|
||||
load_json_payload,
|
||||
resolve_portable_path,
|
||||
)
|
||||
|
||||
|
||||
def load_cases(payload: dict) -> list[dict]:
|
||||
if isinstance(payload.get("cases"), list):
|
||||
return [item for item in payload["cases"] if isinstance(item, dict)]
|
||||
cases_path = payload.get("cases_path")
|
||||
if not cases_path:
|
||||
raise ValueError("cases or cases_path is required")
|
||||
rows = []
|
||||
text = resolve_portable_path(str(cases_path)).read_text(encoding="utf-8")
|
||||
for line in text.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
item = json.loads(line)
|
||||
if isinstance(item, dict):
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
|
||||
def case_value(case: dict, key: str):
|
||||
if key in case:
|
||||
return case.get(key)
|
||||
main = case.get("main") if isinstance(case.get("main"), dict) else {}
|
||||
pre = case.get("pre_processing") if isinstance(case.get("pre_processing"), dict) else {}
|
||||
return case.get(key) or pre.get(key) or main.get(key)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = load_json_payload()
|
||||
dataset_label = str(payload.get("dataset_label") or "").strip()
|
||||
target = str(payload.get("target") or "").strip()
|
||||
if not dataset_label:
|
||||
raise ValueError("dataset_label is required")
|
||||
if not target:
|
||||
raise ValueError("target is required")
|
||||
complex_value = bool(payload.get("complex", False))
|
||||
cases = load_cases(payload)
|
||||
if not cases:
|
||||
raise ValueError("cases must not be empty")
|
||||
|
||||
lines = [f"# dataset_label: {dataset_label}", ""]
|
||||
for index, case in enumerate(cases, start=1):
|
||||
query = str(case_value(case, "query") or "").strip()
|
||||
if not query:
|
||||
continue
|
||||
request_id = str(case_value(case, "request_id") or "").strip()
|
||||
timestamp = case_value(case, "timestamp")
|
||||
tts = str(case_value(case, "tts") or "").strip()
|
||||
planning_result = str(case_value(case, "planning_result") or "").strip()
|
||||
notes = []
|
||||
if planning_result:
|
||||
notes.append(f"线上模型输出: {planning_result}")
|
||||
main_domain = case_value(case, "domain") or case.get("main_domain")
|
||||
if main_domain:
|
||||
notes.append(f"线上 domain: {main_domain}")
|
||||
lines.append(f"### case: online_{index:04d}")
|
||||
lines.append(f"用户: {query}")
|
||||
lines.append(f"complex: {'true' if complex_value else 'false'}")
|
||||
lines.append(f"target: {target}")
|
||||
if request_id:
|
||||
lines.append(f"request_id: {request_id}")
|
||||
if timestamp:
|
||||
lines.append(f"timestamp: {timestamp}")
|
||||
if tts:
|
||||
lines.append(f"notes: tts={tts}" + (f"；{'；'.join(notes)}" if notes else ""))
|
||||
elif notes:
|
||||
lines.append(f"notes: {'；'.join(notes)}")
|
||||
lines.append("")
|
||||
|
||||
draft_text = "\n".join(lines).strip() + "\n"
|
||||
output_path = None
|
||||
if payload.get("output_path"):
|
||||
output = resolve_portable_path(str(payload["output_path"]))
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(draft_text, encoding="utf-8")
|
||||
output_path = str(output)
|
||||
emit_success({"case_count": draft_text.count("### case:"), "draft_text": draft_text, "output_path": output_path})
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
emit_error(exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from online_mining_common import (
|
||||
compact_text,
|
||||
emit_error,
|
||||
emit_success,
|
||||
extract_case,
|
||||
fetch_one_by_request_id,
|
||||
load_json_payload,
|
||||
string_values,
|
||||
write_optional_jsonl,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = load_json_payload()
|
||||
request_ids = string_values(payload.get("request_ids"))
|
||||
if not request_ids:
|
||||
raise ValueError("request_ids is required")
|
||||
sources = payload.get("sources") or ["main", "pre_processing"]
|
||||
date = payload.get("date")
|
||||
rows = []
|
||||
for request_id in request_ids:
|
||||
item = {"request_id": request_id}
|
||||
for source in sources:
|
||||
doc = fetch_one_by_request_id(str(source), request_id, date)
|
||||
item[str(source)] = extract_case(str(source), doc) if doc else None
|
||||
rows.append(item)
|
||||
output_path = write_optional_jsonl(str(payload.get("output_path") or ""), rows)
|
||||
emit_success(
|
||||
{
|
||||
"date": date or "past-48h",
|
||||
"count": len(rows),
|
||||
"output_path": output_path,
|
||||
"cases": [
|
||||
{
|
||||
source: (
|
||||
{
|
||||
key: compact_text(value, 1000)
|
||||
for key, value in (case or {}).items()
|
||||
if key != "raw" and value not in (None, "", [], {})
|
||||
}
|
||||
if isinstance(case, dict)
|
||||
else case
|
||||
)
|
||||
for source, case in row.items()
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
}
|
||||
)
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
emit_error(exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from online_mining_common import (
|
||||
compact_text,
|
||||
emit_error,
|
||||
emit_success,
|
||||
extract_case,
|
||||
fetch_one_by_request_id,
|
||||
load_json_payload,
|
||||
resolve_portable_path,
|
||||
string_values,
|
||||
write_optional_jsonl,
|
||||
)
|
||||
|
||||
|
||||
def load_request_ids(payload: dict) -> list[str]:
|
||||
ids = string_values(payload.get("request_ids"))
|
||||
if ids:
|
||||
return ids
|
||||
cases_path = payload.get("cases_path")
|
||||
if cases_path:
|
||||
request_ids = []
|
||||
text = resolve_portable_path(str(cases_path)).read_text(encoding="utf-8")
|
||||
for line in text.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
item = json.loads(line)
|
||||
rid = item.get("request_id") or item.get("requestId")
|
||||
if rid:
|
||||
request_ids.append(str(rid))
|
||||
return request_ids
|
||||
cases = payload.get("cases") or []
|
||||
if isinstance(cases, list):
|
||||
return [str(item.get("request_id")) for item in cases if isinstance(item, dict) and item.get("request_id")]
|
||||
return []
|
||||
|
||||
|
||||
def merge_case(request_id: str, main_case: dict | None, pre_case: dict | None) -> dict:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"query": (pre_case or {}).get("query") or (main_case or {}).get("query"),
|
||||
"timestamp": (main_case or {}).get("timestamp") or (pre_case or {}).get("timestamp"),
|
||||
"session_id": (main_case or {}).get("session_id"),
|
||||
"device_id": (main_case or {}).get("device_id"),
|
||||
"device": (main_case or {}).get("device"),
|
||||
"tts": (main_case or {}).get("text"),
|
||||
"main": main_case,
|
||||
"pre_processing": pre_case,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = load_json_payload()
|
||||
request_ids = load_request_ids(payload)
|
||||
if not request_ids:
|
||||
raise ValueError("request_ids, cases or cases_path is required")
|
||||
date = payload.get("date")
|
||||
rows = []
|
||||
for request_id in request_ids:
|
||||
main_doc = fetch_one_by_request_id("main", request_id, date)
|
||||
pre_doc = fetch_one_by_request_id("pre_processing", request_id, date)
|
||||
main_case = extract_case("main", main_doc) if main_doc else None
|
||||
pre_case = extract_case("pre_processing", pre_doc) if pre_doc else None
|
||||
rows.append(merge_case(request_id, main_case, pre_case))
|
||||
output_path = write_optional_jsonl(str(payload.get("output_path") or ""), rows)
|
||||
emit_success(
|
||||
{
|
||||
"date": date or "past-48h",
|
||||
"count": len(rows),
|
||||
"output_path": output_path,
|
||||
"cases": [
|
||||
{
|
||||
key: compact_text(value, 1200)
|
||||
for key, value in row.items()
|
||||
if key not in {"main", "pre_processing"} and value not in (None, "", [], {})
|
||||
}
|
||||
| {
|
||||
"main_domain": (row.get("main") or {}).get("domain"),
|
||||
"main_func": (row.get("main") or {}).get("func"),
|
||||
"planning_result": (row.get("pre_processing") or {}).get("planning_result"),
|
||||
"candidate_domains": (row.get("pre_processing") or {}).get("candidate_domains"),
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
}
|
||||
)
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
emit_error(exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from online_mining_common import (
|
||||
build_client,
|
||||
compact_text,
|
||||
deep_parse,
|
||||
emit_error,
|
||||
emit_success,
|
||||
extract_case,
|
||||
load_json_payload,
|
||||
search_docs,
|
||||
source_config,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = load_json_payload()
|
||||
sources = payload.get("sources") or ["main", "pre_processing"]
|
||||
date = payload.get("date")
|
||||
sample_size = int(payload.get("sample_size") or 5)
|
||||
result: dict[str, object] = {}
|
||||
for source in sources:
|
||||
config = source_config(str(source))
|
||||
client = build_client(str(source))
|
||||
caps = client.field_caps(index=str(config["index"]), fields="*")
|
||||
fields = caps.get("fields", {})
|
||||
docs = search_docs(source=str(source), date=date, size=sample_size)
|
||||
cases = [extract_case(str(source), deep_parse(doc)) for doc in docs]
|
||||
interesting = [
|
||||
name
|
||||
for name in fields
|
||||
if any(
|
||||
token in name.lower()
|
||||
for token in [
|
||||
"request",
|
||||
"query",
|
||||
"session",
|
||||
"domain",
|
||||
"func",
|
||||
"device",
|
||||
"prompt",
|
||||
"planning",
|
||||
"dispatch",
|
||||
"excellent",
|
||||
"llm",
|
||||
"timestamp",
|
||||
]
|
||||
)
|
||||
]
|
||||
result[str(source)] = {
|
||||
"index": config["index"],
|
||||
"id_field": config["id_field"],
|
||||
"time_field": config["time_field"],
|
||||
"field_count": len(fields),
|
||||
"interesting_fields": sorted(interesting)[:200],
|
||||
"samples": [
|
||||
{
|
||||
key: compact_text(value, 600)
|
||||
for key, value in case.items()
|
||||
if key != "raw" and value not in (None, "", [], {})
|
||||
}
|
||||
for case in cases
|
||||
],
|
||||
}
|
||||
emit_success({"sources": result})
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
emit_error(exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from online_mining_common import (
|
||||
case_matches,
|
||||
compact_text,
|
||||
emit_error,
|
||||
emit_success,
|
||||
extract_case,
|
||||
load_json_payload,
|
||||
search_docs,
|
||||
source_config,
|
||||
string_values,
|
||||
write_optional_jsonl,
|
||||
)
|
||||
|
||||
|
||||
def build_index_filters(source: str, filters: dict) -> list[dict]:
|
||||
"""尽量把可索引条件下推到 ES，其余条件在客户端二次过滤。"""
|
||||
config = source_config(source)
|
||||
result: list[dict] = []
|
||||
if source == "main":
|
||||
for key in ("domain", "func"):
|
||||
values = string_values(filters.get(key))
|
||||
if values:
|
||||
result.append({"terms": {key: values}})
|
||||
contains = string_values(filters.get("query_contains"))
|
||||
if contains:
|
||||
# query 是 keyword 字段，wildcard 可用但不要放太宽，调用方要限制日期和 scan_size。
|
||||
for item in contains:
|
||||
result.append({"wildcard": {"query": {"value": f"*{item}*"}}})
|
||||
if filters.get("request_id"):
|
||||
result.append({"terms": {str(config["id_field"]): string_values(filters.get("request_id"))}})
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = load_json_payload()
|
||||
source = str(payload.get("source") or "pre_processing")
|
||||
date = payload.get("date")
|
||||
size = int(payload.get("size") or 50)
|
||||
scan_size = int(payload.get("scan_size") or max(size * 5, size))
|
||||
filters = payload.get("filters") or {}
|
||||
if not isinstance(filters, dict):
|
||||
raise ValueError("filters must be an object")
|
||||
docs = search_docs(
|
||||
source=source,
|
||||
date=date,
|
||||
size=scan_size,
|
||||
query_filters=build_index_filters(source, filters),
|
||||
)
|
||||
cases = []
|
||||
for doc in docs:
|
||||
case = extract_case(source, doc)
|
||||
if not case_matches(case, filters):
|
||||
continue
|
||||
cases.append(case)
|
||||
if len(cases) >= size:
|
||||
break
|
||||
output_path = write_optional_jsonl(str(payload.get("output_path") or ""), cases)
|
||||
emit_success(
|
||||
{
|
||||
"source": source,
|
||||
"date": date or "past-48h",
|
||||
"scanned": len(docs),
|
||||
"matched": len(cases),
|
||||
"output_path": output_path,
|
||||
"cases": [
|
||||
{
|
||||
key: compact_text(value, 1200)
|
||||
for key, value in case.items()
|
||||
if key != "raw" and value not in (None, "", [], {})
|
||||
}
|
||||
for case in cases
|
||||
],
|
||||
}
|
||||
)
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
emit_error(exc)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""online-mining-v2 可迁移脚本的共享逻辑。
|
||||
|
||||
脚本只做只读 ELK 查询和轻量格式转换，输出统一 JSON，方便被不同 Agent 调用。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class OnlineMiningError(ValueError):
|
||||
"""线上挖掘参数或数据错误。"""
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
SKILL_DIR = SCRIPT_DIR.parent
|
||||
REPO_ROOT = SKILL_DIR.parent.parent
|
||||
ELK_FETCH_DIR = SKILL_DIR.parent / "elk-fetch"
|
||||
|
||||
|
||||
SOURCE_CONFIGS: dict[str, dict[str, Any]] = {
|
||||
"main": {
|
||||
"profile": "default",
|
||||
"index": "arch-flat-nlp-log-f-*",
|
||||
"id_field": "request_id",
|
||||
"time_field": "timestamp",
|
||||
},
|
||||
"pre_processing": {
|
||||
"profile": "default",
|
||||
"index": "pre-processing-info*",
|
||||
"id_field": "requestId",
|
||||
"time_field": "timestamp",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_json_payload(argv: list[str] | None = None) -> dict[str, Any]:
|
||||
parser = argparse.ArgumentParser(add_help=True)
|
||||
parser.add_argument("--input", "-i", help="JSON 参数文件；不传则从 stdin 读取")
|
||||
args = parser.parse_args(argv)
|
||||
text = Path(args.input).expanduser().read_text(encoding="utf-8") if args.input else sys.stdin.read()
|
||||
if not text.strip():
|
||||
raise OnlineMiningError("input JSON is required")
|
||||
payload = json.loads(text)
|
||||
if not isinstance(payload, dict):
|
||||
raise OnlineMiningError("input JSON must be an object")
|
||||
return payload
|
||||
|
||||
|
||||
def emit_success(payload: dict[str, Any]) -> None:
|
||||
print(json.dumps({"ok": True, **payload}, ensure_ascii=False, separators=(",", ":")))
|
||||
|
||||
|
||||
def emit_error(exc: BaseException) -> None:
|
||||
print(json.dumps({"ok": False, "error": str(exc)}, ensure_ascii=False, separators=(",", ":")))
|
||||
|
||||
|
||||
def load_elk_query_module():
|
||||
path = ELK_FETCH_DIR / "elk_query.py"
|
||||
if not path.exists():
|
||||
raise OnlineMiningError(f"elk-fetch script not found: {path}")
|
||||
spec = importlib.util.spec_from_file_location("elk_query", path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise OnlineMiningError(f"cannot load elk_query.py from {path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def build_client(source: str):
|
||||
elk = load_elk_query_module()
|
||||
config = source_config(source)
|
||||
profiles = elk.load_profiles()
|
||||
return elk.build_client(profiles, str(config["profile"]))
|
||||
|
||||
|
||||
def source_config(source: str) -> dict[str, Any]:
|
||||
if source not in SOURCE_CONFIGS:
|
||||
raise OnlineMiningError(f"unknown source: {source}")
|
||||
return SOURCE_CONFIGS[source]
|
||||
|
||||
|
||||
def date_range_ms(date_str: str | None) -> tuple[int, int]:
|
||||
if date_str:
|
||||
date_obj = datetime.strptime(str(date_str), "%Y%m%d")
|
||||
start_ms = int(date_obj.timestamp() * 1000)
|
||||
end_ms = int((date_obj + timedelta(days=1)).timestamp() * 1000) - 1
|
||||
else:
|
||||
end = datetime.now()
|
||||
start_ms = int((end - timedelta(hours=48)).timestamp() * 1000)
|
||||
end_ms = int(end.timestamp() * 1000)
|
||||
return start_ms, end_ms
|
||||
|
||||
|
||||
def deep_parse(value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
stripped = value.strip()
|
||||
if (stripped.startswith("{") and stripped.endswith("}")) or (
|
||||
stripped.startswith("[") and stripped.endswith("]")
|
||||
):
|
||||
try:
|
||||
return deep_parse(json.loads(stripped))
|
||||
except Exception:
|
||||
return value
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return {key: deep_parse(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [deep_parse(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def get_path(obj: Any, dotted: str, default: Any = None) -> Any:
|
||||
cur = obj
|
||||
for part in dotted.split("."):
|
||||
if not part:
|
||||
continue
|
||||
if isinstance(cur, dict):
|
||||
cur = cur.get(part)
|
||||
elif isinstance(cur, list):
|
||||
try:
|
||||
cur = cur[int(part)]
|
||||
except (ValueError, IndexError):
|
||||
return default
|
||||
else:
|
||||
return default
|
||||
return default if cur is None else cur
|
||||
|
||||
|
||||
def compact_text(value: Any, limit: int = 500) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
text = json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text if len(text) <= limit else text[: limit - 3] + "..."
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
|
||||
def string_values(value: Any) -> list[str]:
|
||||
return [str(item) for item in as_list(value) if str(item).strip()]
|
||||
|
||||
|
||||
def match_text_filters(text: str, contains: Any = None, regex: str | None = None) -> bool:
|
||||
for item in string_values(contains):
|
||||
if item not in text:
|
||||
return False
|
||||
if regex and not re.search(regex, text):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def unique_strings(items: list[Any]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for item in items:
|
||||
text = str(item).strip()
|
||||
if not text or text in seen:
|
||||
continue
|
||||
seen.add(text)
|
||||
result.append(text)
|
||||
return result
|
||||
|
||||
|
||||
def extract_case(source: str, doc: dict[str, Any]) -> dict[str, Any]:
|
||||
parsed = deep_parse(doc)
|
||||
if source == "main":
|
||||
return extract_main_case(parsed)
|
||||
if source == "pre_processing":
|
||||
return extract_pre_processing_case(parsed)
|
||||
raise OnlineMiningError(f"unknown source: {source}")
|
||||
|
||||
|
||||
def extract_main_case(doc: dict[str, Any]) -> dict[str, Any]:
|
||||
device = doc.get("device")
|
||||
return {
|
||||
"request_id": doc.get("request_id"),
|
||||
"timestamp": doc.get("timestamp"),
|
||||
"query": doc.get("query"),
|
||||
"session_id": doc.get("session_id"),
|
||||
"device_id": doc.get("device_id"),
|
||||
"device": device,
|
||||
"domain": doc.get("domain"),
|
||||
"func": doc.get("func") or doc.get("func_name"),
|
||||
"text": doc.get("to_speak") or doc.get("text") or doc.get("display_text"),
|
||||
"intention": doc.get("intention"),
|
||||
"llm_agent_info": get_path(doc, "intention.intent_arbitrator_info.llm_agent_info"),
|
||||
"score_domains": get_path(doc, "intention.intent_arbitrator_info.score_domains"),
|
||||
"hit_rules": get_path(doc, "intention.intent_arbitrator_info.hit_rules"),
|
||||
"raw": doc,
|
||||
}
|
||||
|
||||
|
||||
def extract_pre_processing_case(doc: dict[str, Any]) -> dict[str, Any]:
|
||||
body = doc.get("responseBoby") or {}
|
||||
core = get_path(body, "nodes.0.core", {})
|
||||
dispatch = get_path(core, "dispatch_large_model_result", {}) or {}
|
||||
dispatch_input = get_path(dispatch, "dispatchLargeModelInput", {}) or {}
|
||||
planning_debug = get_path(dispatch, "planningDebugInfo", {}) or {}
|
||||
properties0 = get_path(body, "nodes.0.properties.0", {}) or {}
|
||||
debug = properties0.get("debug") if isinstance(properties0, dict) else {}
|
||||
excellent_domains = get_path(core, "excellent_domains_result", []) or []
|
||||
domain_infos = get_path(debug, "domain_infos", []) if isinstance(debug, dict) else []
|
||||
return {
|
||||
"request_id": doc.get("requestId"),
|
||||
"timestamp": doc.get("timestamp"),
|
||||
"query": get_path(core, "query.query"),
|
||||
"planning_result": planning_debug.get("planningOriginalResult") if isinstance(planning_debug, dict) else None,
|
||||
"code": dispatch.get("code") if isinstance(dispatch, dict) else None,
|
||||
"prompt_model": dispatch_input.get("promptModel") if isinstance(dispatch_input, dict) else None,
|
||||
"hit_rules": dispatch_input.get("hitRuleList") if isinstance(dispatch_input, dict) else None,
|
||||
"excellent_domains": excellent_domains,
|
||||
"candidate_domains": unique_strings(
|
||||
[get_path(item, "domain") for item in as_list(excellent_domains)]
|
||||
+ [get_path(item, "domain") for item in as_list(domain_infos)]
|
||||
),
|
||||
"domain_infos": domain_infos,
|
||||
"agent_llm_intent": properties0.get("agentLLMIntent") if isinstance(properties0, dict) else None,
|
||||
"prompt_string": get_path(debug, "promptString") if isinstance(debug, dict) else None,
|
||||
"planning_prompt": get_path(debug, "planningPrompt") if isinstance(debug, dict) else None,
|
||||
"raw": doc,
|
||||
}
|
||||
|
||||
|
||||
def case_matches(case: dict[str, Any], filters: dict[str, Any]) -> bool:
|
||||
query = str(case.get("query") or "")
|
||||
if not match_text_filters(query, filters.get("query_contains"), filters.get("query_regex")):
|
||||
return False
|
||||
for key in ("domain", "func", "planning_result", "code"):
|
||||
expected = string_values(filters.get(key))
|
||||
if expected and str(case.get(key) or "") not in expected:
|
||||
return False
|
||||
contains = filters.get(f"{key}_contains")
|
||||
if contains and not match_text_filters(str(case.get(key) or ""), contains):
|
||||
return False
|
||||
if filters.get("prompt_contains") and not match_text_filters(
|
||||
str(case.get("prompt_model") or case.get("prompt_string") or case.get("planning_prompt") or ""),
|
||||
filters.get("prompt_contains"),
|
||||
):
|
||||
return False
|
||||
candidate_domains = set(string_values(case.get("candidate_domains")))
|
||||
expected_candidates = set(string_values(filters.get("candidate_domain")))
|
||||
if expected_candidates and not expected_candidates.intersection(candidate_domains):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def search_docs(
|
||||
*,
|
||||
source: str,
|
||||
date: str | None,
|
||||
size: int,
|
||||
query_filters: list[dict[str, Any]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
config = source_config(source)
|
||||
client = build_client(source)
|
||||
start_ms, end_ms = date_range_ms(date)
|
||||
filters = [{"range": {str(config["time_field"]): {"gte": start_ms, "lte": end_ms}}}]
|
||||
filters.extend(deepcopy(query_filters or []))
|
||||
body = {
|
||||
"query": {"bool": {"filter": filters}},
|
||||
"size": size,
|
||||
"sort": [{str(config["time_field"]): {"order": "desc"}}],
|
||||
}
|
||||
response = client.search(index=str(config["index"]), body=body)
|
||||
return [deep_parse(hit.get("_source", {})) for hit in response.get("hits", {}).get("hits", [])]
|
||||
|
||||
|
||||
def fetch_one_by_request_id(source: str, request_id: str, date: str | None = None) -> dict[str, Any] | None:
|
||||
config = source_config(source)
|
||||
docs = search_docs(
|
||||
source=source,
|
||||
date=date,
|
||||
size=1,
|
||||
query_filters=[{"term": {str(config["id_field"]): request_id}}],
|
||||
)
|
||||
if not docs:
|
||||
# 部分 request id 会带后缀，兜底用 wildcard。
|
||||
docs = search_docs(
|
||||
source=source,
|
||||
date=date,
|
||||
size=1,
|
||||
query_filters=[{"wildcard": {str(config["id_field"]): {"value": f"{request_id}*"}}}],
|
||||
)
|
||||
return docs[0] if docs else None
|
||||
|
||||
|
||||
def write_optional_jsonl(path: str | None, rows: list[dict[str, Any]]) -> str | None:
|
||||
if not path:
|
||||
return None
|
||||
output = resolve_portable_path(path)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output.open("w", encoding="utf-8") as fh:
|
||||
for row in rows:
|
||||
fh.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
return str(output)
|
||||
|
||||
|
||||
def resolve_portable_path(path: str) -> Path:
|
||||
raw = Path(path).expanduser()
|
||||
if raw.is_absolute():
|
||||
return raw
|
||||
return (REPO_ROOT / raw).resolve()
|
||||
|
||||
Reference in New Issue
Block a user