wuyang/zk-data-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use prompt history for online mining records

Browse Source
This commit is contained in:
wuyang6
2026-05-14 17:42:08 +08:00
parent f1dc7e4b91
commit 87dd67c76a
10 changed files with 219 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/online-mining-v2/scripts/build_online_records.py
+82 -23
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
这个脚本承接 online-mining-v2 和 product-data：
1. 按 request_id 拉主 NLP 表和前处理表。
2. 用主表 session_id 补齐当前请求前最多 10 轮上下文。
2. 优先从前处理表 promptModel 的 [对话历史] 抽取模型真实输入 session。
3. 从用户指定 target 或线上 planning/code 结果推断标签。
4. 直接产出 canonical records，并可同步导出流通表格、训练 jsonl、评测 csv。
"""
@@ -22,7 +22,6 @@ from online_mining_common import (
emit_success,
extract_case,
fetch_one_by_request_id,
fetch_session_turns,
load_json_payload,
resolve_portable_path,
string_values,
@@ -73,11 +72,32 @@ def merge_case(request_id: str, main_case: dict[str, Any] | None, pre_case: dict
}
def fetch_cases_by_request_ids(request_ids: list[str], *, date: str | None) -> list[dict[str, Any]]:
def fetch_cases_by_request_ids(
request_ids: list[str],
*,
date: str | None,
date_from: str | None = None,
date_to: str | None = None,
lookback_days: int | None = None,
) -> list[dict[str, Any]]:
rows = []
for request_id in request_ids:
main_doc = fetch_one_by_request_id("main", request_id, date)
pre_doc = fetch_one_by_request_id("pre_processing", request_id, date)
main_doc = fetch_one_by_request_id(
"main",
request_id,
date,
date_from=date_from,
date_to=date_to,
lookback_days=lookback_days,
)
pre_doc = fetch_one_by_request_id(
"pre_processing",
request_id,
date,
date_from=date_from,
date_to=date_to,
lookback_days=lookback_days,
)
main_case = extract_case("main", main_doc) if main_doc else None
pre_case = extract_case("pre_processing", pre_doc) if pre_doc else None
rows.append(merge_case(request_id, main_case, pre_case))
@@ -92,24 +112,55 @@ def case_value(case: dict[str, Any], key: str) -> Any:
return case.get(key) or pre.get(key) or main.get(key)
def enrich_prev_session(case: dict[str, Any], *, date: str | None, limit: int) -> list[dict[str, Any]]:
def enrich_prev_session(case: dict[str, Any], *, limit: int) -> list[dict[str, Any]]:
explicit = case.get("prev_session")
if isinstance(explicit, list):
return normalize_prev_session(explicit)[-limit:]
session_id = str(case_value(case, "session_id") or "").strip()
timestamp = optional_int(case_value(case, "timestamp"))
if not session_id or timestamp is None:
# 前处理 promptModel 是中控 planning 模型真实看到的输入，里面的 [对话历史]
# 才是训练/评测应复现的 session。主表 session_id 不是同一概念，不默认使用。
prompt_history = parse_prompt_history(str(case_value(case, "prompt_model") or ""))
if prompt_history and timestamp is not None:
step_ms = 60_000
start_ts = timestamp - len(prompt_history) * step_ms
for index, item in enumerate(prompt_history):
item["timestamp"] = start_ts + index * step_ms
return prompt_history[-limit:]
def parse_prompt_history(prompt: str) -> list[dict[str, Any]]:
if not prompt:
return []
turns = fetch_session_turns(session_id=session_id, before_timestamp=timestamp, date=date, limit=limit)
return [
{
"query": str(turn.get("query") or ""),
"tts": str(turn.get("text") or ""),
"timestamp": optional_int(turn.get("timestamp")) or 0,
}
for turn in turns
if str(turn.get("query") or "").strip()
][-limit:]
history_marker = "[对话历史]"
current_marker = "[当前query]"
history_start = prompt.rfind(history_marker)
if history_start < 0:
return []
current_start = prompt.find(current_marker, history_start)
if current_start < 0:
return []
block = prompt[history_start + len(history_marker) : current_start].strip()
if not block:
return []
rows: list[dict[str, Any]] = []
pending_user: str | None = None
for raw_line in block.splitlines():
line = raw_line.strip()
if not line:
continue
user_match = re.match(r"^用户\s*[:：]\s*(.*)$", line)
if user_match:
if pending_user is not None:
rows.append({"query": pending_user, "tts": "", "timestamp": 0})
pending_user = user_match.group(1).strip()
continue
assistant_match = re.match(r"^小爱\s*[:：]\s*(.*)$", line)
if assistant_match and pending_user is not None:
rows.append({"query": pending_user, "tts": assistant_match.group(1).strip(), "timestamp": 0})
pending_user = None
if pending_user is not None:
rows.append({"query": pending_user, "tts": "", "timestamp": 0})
return [row for row in rows if row["query"]]
def normalize_prev_session(items: list[Any]) -> list[dict[str, Any]]:
@@ -166,7 +217,6 @@ def case_to_record(
target: str,
complex_value: bool,
index: int,
date: str | None,
session_limit: int,
) -> tuple[dict[str, Any], dict[str, Any]]:
query = str(case_value(case, "query") or "").strip()
@@ -180,7 +230,7 @@ def case_to_record(
raise OnlineMiningError(f"case {index} has no timestamp")
final_target, target_source = infer_target(case, target)
prev_session = enrich_prev_session(case, date=date, limit=session_limit)
prev_session = enrich_prev_session(case, limit=session_limit)
main = case.get("main") if isinstance(case.get("main"), dict) else {}
pre = case.get("pre_processing") if isinstance(case.get("pre_processing"), dict) else {}
record = {
@@ -270,6 +320,10 @@ def main() -> int:
target = str(payload.get("target") or "").strip()
complex_value = parse_complex(payload.get("complex"), default=False)
date = str(payload.get("date") or "").strip() or None
date_from = str(payload.get("date_from") or "").strip() or None
date_to = str(payload.get("date_to") or "").strip() or None
lookback_days = payload.get("lookback_days")
lookback_days = int(lookback_days) if lookback_days not in (None, "") else None
session_limit = int(payload.get("session_limit") or 10)
if session_limit < 0 or session_limit > 10:
raise OnlineMiningError("session_limit must be between 0 and 10")
@@ -277,7 +331,13 @@ def main() -> int:
request_ids = string_values(payload.get("request_ids"))
cases = load_cases(payload)
if request_ids:
cases = fetch_cases_by_request_ids(request_ids, date=date)
cases = fetch_cases_by_request_ids(
request_ids,
date=date,
date_from=date_from,
date_to=date_to,
lookback_days=lookback_days,
)
if not cases:
raise OnlineMiningError("request_ids, cases or cases_path is required")
@@ -292,7 +352,6 @@ def main() -> int:
target=target,
complex_value=complex_value,
index=index,
date=date,
session_limit=session_limit,
)
except Exception as exc: # noqa: BLE001
@@ -334,7 +393,7 @@ def main() -> int:
emit_success(
{
"date": date or "past-48h",
"date": date or (f"{date_from}..{date_to}" if date_from and date_to else f"past-{lookback_days}d" if lookback_days else "past-48h"),
"record_count": len(records),
"records_path": records_path,
"validation": validation,