Stabilize product data export flow

skills/product-data/scripts/product_data_portable.py

@@ -7,10 +7,12 @@ from __future__ import annotations
 """
 
 import argparse
+import csv
 import json
 import re
 import sys
 import time
+from io import StringIO
 from pathlib import Path
 from typing import Any
 
@@ -88,6 +90,8 @@ def read_records_file(path: str) -> list[dict[str, Any]]:
             records.append(item)
         return records
     decoded = json.loads(text)
+    if isinstance(decoded, dict) and isinstance(decoded.get("records"), list):
+        decoded = decoded["records"]
     if not isinstance(decoded, list):
         raise ProductDataError(f"{path} must contain a JSON array")
     return decoded
@@ -374,6 +378,7 @@ def export_dataset_records(
     output_format: str = "jsonl",
     require_validation_ok: bool = True,
     overwrite: bool = True,
+    export_table: bool = True,
 ) -> dict[str, Any]:
     if output_format not in {"jsonl", "json"}:
         raise ProductDataError("output_format must be jsonl or json")
@@ -382,8 +387,11 @@ def export_dataset_records(
         raise ProductDataError(f"records failed validation with {validation['error_count']} errors")
 
     path = Path(output_path).expanduser()
+    table_path = path.with_name("records.csv")
     if path.exists() and not overwrite:
         raise ProductDataError(f"output_path already exists: {output_path}")
+    if export_table and table_path.exists() and not overwrite:
+        raise ProductDataError("table output_path already exists: records.csv")
     path.parent.mkdir(parents=True, exist_ok=True)
 
     if output_format == "jsonl":
@@ -394,13 +402,106 @@ def export_dataset_records(
     else:
         content = json.dumps(records, ensure_ascii=False, separators=(",", ":")) + "\n"
     path.write_text(content, encoding="utf-8")
-    return {
+    payload = {
         "output_path": str(path),
         "output_format": output_format,
         "record_count": len(records),
         "bytes_written": len(content.encode("utf-8")),
         "validation": validation,
     }
+    if export_table:
+        table_content = render_dataset_records_table_csv(records)
+        table_path.write_text(table_content, encoding="utf-8-sig")
+        payload.update(
+            {
+                "table_output_path": str(table_path),
+                "table_output_format": "csv",
+                "table_bytes_written": len(table_content.encode("utf-8-sig")),
+            }
+        )
+    return payload
+
+
+def export_dataset_table(
+    records: list[dict[str, Any]],
+    *,
+    output_path: str,
+    require_validation_ok: bool = True,
+    overwrite: bool = True,
+) -> dict[str, Any]:
+    validation = validate_dataset_records(records)
+    if require_validation_ok and not validation["ok"]:
+        raise ProductDataError(f"records failed validation with {validation['error_count']} errors")
+    path = Path(output_path).expanduser()
+    if path.exists() and not overwrite:
+        raise ProductDataError(f"output_path already exists: {output_path}")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    content = render_dataset_records_table_csv(records)
+    path.write_text(content, encoding="utf-8-sig")
+    return {
+        "output_path": str(path),
+        "output_format": "csv",
+        "record_count": len(records),
+        "bytes_written": len(content.encode("utf-8-sig")),
+        "validation": validation,
+    }
+
+
+def render_dataset_records_table_csv(records: list[dict[str, Any]]) -> str:
+    output = StringIO()
+    writer = csv.DictWriter(output, fieldnames=dataset_table_columns(), lineterminator="\n")
+    writer.writeheader()
+    for record in records:
+        writer.writerow(dataset_record_table_row(record))
+    return output.getvalue()
+
+
+def dataset_table_columns() -> list[str]:
+    return [
+        "request_id",
+        "timestamp",
+        "query",
+        "prev_session",
+        "context",
+        "label",
+        "是否迁移Function",
+        "function",
+    ]
+
+
+def dataset_record_table_row(record: dict[str, Any]) -> dict[str, str]:
+    source = record.get("source") if isinstance(record.get("source"), dict) else {}
+    turn = record.get("turn") if isinstance(record.get("turn"), dict) else {}
+    label = record.get("label") if isinstance(record.get("label"), dict) else {}
+    context = record.get("context") if isinstance(record.get("context"), dict) else {}
+    prev_session = record.get("prev_session")
+    if not isinstance(prev_session, list):
+        prev_session = []
+    return {
+        "request_id": str(source.get("request_id") or ""),
+        "timestamp": str(source.get("timestamp") or turn.get("timestamp") or ""),
+        "query": str(turn.get("query") or ""),
+        "prev_session": json.dumps(table_prev_session(prev_session), ensure_ascii=False, separators=(",", ":")),
+        "context": json.dumps(context, ensure_ascii=False, separators=(",", ":")),
+        "label": str(label.get("dataset_label") or ""),
+        "是否迁移Function": "",
+        "function": str(label.get("target") or ""),
+    }
+
+
+def table_prev_session(prev_session: list[Any]) -> list[dict[str, str]]:
+    rows: list[dict[str, str]] = []
+    for item in prev_session[-10:]:
+        if not isinstance(item, dict):
+            continue
+        rows.append(
+            {
+                "query": str(item.get("query") or ""),
+                "tts": str(item.get("tts") or ""),
+                "timestamp": str(item.get("timestamp") or ""),
+            }
+        )
+    return rows
 
 
 def default_output_path(payload: dict[str, Any]) -> str:
@@ -413,6 +514,16 @@ def default_output_path(payload: dict[str, Any]) -> str:
     return str(Path("output") / filename)
 
 
+def default_table_output_path(payload: dict[str, Any]) -> str:
+    if isinstance(payload.get("output_path"), str) and payload["output_path"].strip():
+        return payload["output_path"].strip()
+    if isinstance(payload.get("output_dir"), str) and payload["output_dir"].strip():
+        return str(Path(payload["output_dir"]).expanduser() / "records.csv")
+    if isinstance(payload.get("records_path"), str) and payload["records_path"].strip():
+        return str(Path(payload["records_path"]).expanduser().with_name("records.csv"))
+    return str(Path("output") / "records.csv")
+
+
 def target_type(target: str) -> str:
     if re.match(r"^Agent\s*\(\s*tag\s*=", target):
         return "agent"