wuyang6
86 lines
2.7 KiB
Python
86 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
from online_mining_common import (
|
|
build_client,
|
|
compact_text,
|
|
deep_parse,
|
|
emit_error,
|
|
emit_success,
|
|
extract_case,
|
|
load_json_payload,
|
|
search_docs,
|
|
source_config,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
try:
|
|
payload = load_json_payload()
|
|
sources = payload.get("sources") or ["main", "pre_processing"]
|
|
date = payload.get("date")
|
|
date_from = payload.get("date_from")
|
|
date_to = payload.get("date_to")
|
|
lookback_days = payload.get("lookback_days")
|
|
lookback_days = int(lookback_days) if lookback_days not in (None, "") else None
|
|
sample_size = int(payload.get("sample_size") or 5)
|
|
result: dict[str, object] = {}
|
|
for source in sources:
|
|
config = source_config(str(source))
|
|
client = build_client(str(source))
|
|
caps = client.field_caps(index=str(config["index"]), fields="*")
|
|
fields = caps.get("fields", {})
|
|
docs = search_docs(
|
|
source=str(source),
|
|
date=date,
|
|
size=sample_size,
|
|
date_from=date_from,
|
|
date_to=date_to,
|
|
lookback_days=lookback_days,
|
|
)
|
|
cases = [extract_case(str(source), deep_parse(doc)) for doc in docs]
|
|
interesting = [
|
|
name
|
|
for name in fields
|
|
if any(
|
|
token in name.lower()
|
|
for token in [
|
|
"request",
|
|
"query",
|
|
"session",
|
|
"domain",
|
|
"func",
|
|
"device",
|
|
"prompt",
|
|
"planning",
|
|
"dispatch",
|
|
"excellent",
|
|
"llm",
|
|
"timestamp",
|
|
]
|
|
)
|
|
]
|
|
result[str(source)] = {
|
|
"index": config["index"],
|
|
"id_field": config["id_field"],
|
|
"time_field": config["time_field"],
|
|
"field_count": len(fields),
|
|
"interesting_fields": sorted(interesting)[:200],
|
|
"samples": [
|
|
{
|
|
key: compact_text(value, 600)
|
|
for key, value in case.items()
|
|
if key != "raw" and value not in (None, "", [], {})
|
|
}
|
|
for case in cases
|
|
],
|
|
}
|
|
emit_success({"sources": result})
|
|
return 0
|
|
except Exception as exc: # noqa: BLE001
|
|
emit_error(exc)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|