wuyang/zk-data-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stabilize product data export flow

Browse Source
This commit is contained in:
wuyang6
2026-05-11 11:46:09 +08:00
parent 1752c81913
commit 0e2a00990f
19 changed files with 659 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/product-data/scripts/export_dataset_records.py
+1
View File
@@ -21,6 +21,7 @@ def main() -> int:
output_format=output_format,
require_validation_ok=bool(payload.get("require_validation_ok", True)),
overwrite=bool(payload.get("overwrite", True)),
export_table=bool(payload.get("export_table", True)),
)
emit_success(result)
return 0
skills/product-data/scripts/export_dataset_table.py
+31
View File
@@ -0,0 +1,31 @@
from __future__ import annotations
from product_data_portable import (
default_table_output_path,
emit_error,
emit_success,
export_dataset_table,
load_json_payload,
load_records,
)
def main() -> int:
try:
payload = load_json_payload()
records = load_records(payload)
result = export_dataset_table(
records,
output_path=default_table_output_path(payload),
require_validation_ok=bool(payload.get("require_validation_ok", True)),
overwrite=bool(payload.get("overwrite", True)),
)
emit_success(result)
return 0
except Exception as exc: # noqa: BLE001 - CLI 需要把错误稳定转成 JSON
emit_error(exc)
return 1
if __name__ == "__main__":
raise SystemExit(main())
skills/product-data/scripts/product_data_portable.py
+112 -1
View File
@@ -7,10 +7,12 @@ from __future__ import annotations
"""
import argparse
import csv
import json
import re
import sys
import time
from io import StringIO
from pathlib import Path
from typing import Any
@@ -88,6 +90,8 @@ def read_records_file(path: str) -> list[dict[str, Any]]:
records.append(item)
return records
decoded = json.loads(text)
if isinstance(decoded, dict) and isinstance(decoded.get("records"), list):
decoded = decoded["records"]
if not isinstance(decoded, list):
raise ProductDataError(f"{path} must contain a JSON array")
return decoded
@@ -374,6 +378,7 @@ def export_dataset_records(
output_format: str = "jsonl",
require_validation_ok: bool = True,
overwrite: bool = True,
export_table: bool = True,
) -> dict[str, Any]:
if output_format not in {"jsonl", "json"}:
raise ProductDataError("output_format must be jsonl or json")
@@ -382,8 +387,11 @@ def export_dataset_records(
raise ProductDataError(f"records failed validation with {validation['error_count']} errors")
path = Path(output_path).expanduser()
table_path = path.with_name("records.csv")
if path.exists() and not overwrite:
raise ProductDataError(f"output_path already exists: {output_path}")
if export_table and table_path.exists() and not overwrite:
raise ProductDataError("table output_path already exists: records.csv")
path.parent.mkdir(parents=True, exist_ok=True)
if output_format == "jsonl":
@@ -394,13 +402,106 @@ def export_dataset_records(
else:
content = json.dumps(records, ensure_ascii=False, separators=(",", ":")) + "\n"
path.write_text(content, encoding="utf-8")
return {
payload = {
"output_path": str(path),
"output_format": output_format,
"record_count": len(records),
"bytes_written": len(content.encode("utf-8")),
"validation": validation,
}
if export_table:
table_content = render_dataset_records_table_csv(records)
table_path.write_text(table_content, encoding="utf-8-sig")
payload.update(
{
"table_output_path": str(table_path),
"table_output_format": "csv",
"table_bytes_written": len(table_content.encode("utf-8-sig")),
}
)
return payload
def export_dataset_table(
records: list[dict[str, Any]],
*,
output_path: str,
require_validation_ok: bool = True,
overwrite: bool = True,
) -> dict[str, Any]:
validation = validate_dataset_records(records)
if require_validation_ok and not validation["ok"]:
raise ProductDataError(f"records failed validation with {validation['error_count']} errors")
path = Path(output_path).expanduser()
if path.exists() and not overwrite:
raise ProductDataError(f"output_path already exists: {output_path}")
path.parent.mkdir(parents=True, exist_ok=True)
content = render_dataset_records_table_csv(records)
path.write_text(content, encoding="utf-8-sig")
return {
"output_path": str(path),
"output_format": "csv",
"record_count": len(records),
"bytes_written": len(content.encode("utf-8-sig")),
"validation": validation,
}
def render_dataset_records_table_csv(records: list[dict[str, Any]]) -> str:
output = StringIO()
writer = csv.DictWriter(output, fieldnames=dataset_table_columns(), lineterminator="\n")
writer.writeheader()
for record in records:
writer.writerow(dataset_record_table_row(record))
return output.getvalue()
def dataset_table_columns() -> list[str]:
return [
"request_id",
"timestamp",
"query",
"prev_session",
"context",
"label",
"是否迁移Function",
"function",
]
def dataset_record_table_row(record: dict[str, Any]) -> dict[str, str]:
source = record.get("source") if isinstance(record.get("source"), dict) else {}
turn = record.get("turn") if isinstance(record.get("turn"), dict) else {}
label = record.get("label") if isinstance(record.get("label"), dict) else {}
context = record.get("context") if isinstance(record.get("context"), dict) else {}
prev_session = record.get("prev_session")
if not isinstance(prev_session, list):
prev_session = []
return {
"request_id": str(source.get("request_id") or ""),
"timestamp": str(source.get("timestamp") or turn.get("timestamp") or ""),
"query": str(turn.get("query") or ""),
"prev_session": json.dumps(table_prev_session(prev_session), ensure_ascii=False, separators=(",", ":")),
"context": json.dumps(context, ensure_ascii=False, separators=(",", ":")),
"label": str(label.get("dataset_label") or ""),
"是否迁移Function": "",
"function": str(label.get("target") or ""),
}
def table_prev_session(prev_session: list[Any]) -> list[dict[str, str]]:
rows: list[dict[str, str]] = []
for item in prev_session[-10:]:
if not isinstance(item, dict):
continue
rows.append(
{
"query": str(item.get("query") or ""),
"tts": str(item.get("tts") or ""),
"timestamp": str(item.get("timestamp") or ""),
}
)
return rows
def default_output_path(payload: dict[str, Any]) -> str:
@@ -413,6 +514,16 @@ def default_output_path(payload: dict[str, Any]) -> str:
return str(Path("output") / filename)
def default_table_output_path(payload: dict[str, Any]) -> str:
if isinstance(payload.get("output_path"), str) and payload["output_path"].strip():
return payload["output_path"].strip()
if isinstance(payload.get("output_dir"), str) and payload["output_dir"].strip():
return str(Path(payload["output_dir"]).expanduser() / "records.csv")
if isinstance(payload.get("records_path"), str) and payload["records_path"].strip():
return str(Path(payload["records_path"]).expanduser().with_name("records.csv"))
return str(Path("output") / "records.csv")
def target_type(target: str) -> str:
if re.match(r"^Agent\s*\(\s*tag\s*=", target):
return "agent"