diff --git a/skills/product-data/README.md b/skills/product-data/README.md index fe7d506..06b5a64 100644 --- a/skills/product-data/README.md +++ b/skills/product-data/README.md @@ -45,8 +45,8 @@ cat input.json | python skills/product-data/scripts/.py | `scripts/validate_dataset_records.py` | 校验 canonical records | | `scripts/export_dataset_records.py` | 导出紧凑 JSONL/JSON，并默认生成同目录 `records.csv` 表格 | | `scripts/export_dataset_table.py` | 已有 canonical records 时，只补生成同事流转表格 | -| `scripts/export_training_jsonl.py` | 把 canonical records 转成训练 JSONL | -| `scripts/export_planning_eval_csv.py` | 把 canonical records 转成含 `newPrompt` 的评测 CSV | +| `scripts/export_training_jsonl.py` | 把 canonical records 转成训练 JSONL，`output` 自动组合 `complex` 和标签 | +| `scripts/export_planning_eval_csv.py` | 把 canonical records 转成含 chat-template `newPrompt` 的评测 CSV，`complex` 列来自元数据并输出为 `TRUE/FALSE` | ## 平台内优先级 diff --git a/skills/product-data/SKILL.md b/skills/product-data/SKILL.md index f7ba5fb..1e0e683 100644 --- a/skills/product-data/SKILL.md +++ b/skills/product-data/SKILL.md @@ -97,6 +97,7 @@ source_refs: - `dataset_label`：数据集或专题名称。 - `target` / `target_definitions`：最终监督标签。单标签任务用 `target`，多标签边界任务必须用 `target_definitions` 列出每个标签和判定规则。 +- `complex` 判定规则：复杂度是独立维度，必须和标签边界一起确认；如果用户没有说明复杂/不复杂，默认按“原子化单步操作=false，需要规划、分析、组合或多步骤推理=true”给出建议，并在 review 中让用户确认。 - 生成数量：总条数，以及单轮/多轮数量或比例。 - 覆盖范围：需要覆盖哪些 query 类型、意图边界或错误类型。 - 负例/排除项：哪些表达不要生成，或哪些边界容易误判。 @@ -114,12 +115,15 @@ source_refs: 如果用户在原始需求里已经写出 `Agent(tag="xxx")`、function 调用或其他完整标签表达，`target` 必须原样保留这个完整表达，不要简化成纯标签名。例如用户说 `Agent(tag="餐饮服务")`，则 `target_definitions[*].target` 和后续 draft 的 `target:` 都必须写 `Agent(tag="餐饮服务")`，不要写成 `餐饮服务`。 +如果用户给的是训练格式里的两行输出，例如 `complex=false\nAgent(tag="地图导航")`，需要拆开处理：`complex=false` 进入复杂度维度，`Agent(tag="地图导航")` 才是 `target`。不要把 `complex=...` 作为 target 的一部分写入 generation plan。 + review 展示必须简短清晰，不要重复解释工具和流程。每次 review 最多展示 6 行，格式优先如下： ```text 我先把生成目标整理好了，先确认边界，暂时不生成数据。 - 数据集：xxx - 标签：A -> Agent(tag="A")；B -> Agent(tag="B") +- 复杂度：默认 false；复杂任务按规则单独标 true - 边界：一句话说明核心判定规则 - 覆盖：一句话说明主要 case 类型 - 内部：goal_id `...`，revision `...` @@ -165,7 +169,7 @@ review 展示必须简短清晰，不要重复解释工具和流程。每次 rev 9. 用户提出修改意见时，调用 `data_agent_update_generation_plan`，再展示计划。 10. 用户明确确认当前计划版本后，调用 `data_agent_confirm_generation_plan`。 11. 生成 dataset draft text v1。 -12. 调用 `data_agent_normalize_dataset_draft`，必须传入 `confirmed_plan_id`。 +12. 调用 `data_agent_normalize_dataset_draft`，必须传入 `confirmed_plan_id`；draft 中每条 case 都必须有 `complex: true/false`。 13. 调用 `data_agent_validate_dataset_records`。 14. 如果用户要求落盘 canonical records，调用 `data_agent_export_dataset_records`，`output_path` 固定传 `output/records.jsonl`，默认导出紧凑 JSONL，并同时生成同目录 `records.csv` 表格，不要用 `write_file` 手写 JSON 或 CSV。 15. 如果用户已经明确要训练数据，调用 `data_agent_export_training_jsonl`，优先传 `records_path`，输出固定为 `output/training.jsonl`。 @@ -176,6 +180,7 @@ review 展示必须简短清晰，不要重复解释工具和流程。每次 rev - 单次 `data_agent_normalize_dataset_draft` 最多处理 8 条 case；计划数量更多时，分批生成、分批 normalize，再汇总校验和导出。 - dataset draft 中的 Agent target 推荐写成单引号形式，例如 `target: Agent(tag='地图导航')`。工具会规范化为 `Agent(tag="地图导航")`，这样可以降低 tool call JSON 里双引号转义失败的概率。 +- `complex:` 独立写一行，不要写进 `target:`；工具会在导出训练数据和流转表格时自动组合成 `complex=false\nAgent(...)`。 - 如果 draft 已经保存在文件中，优先传 `draft_path`，不要再把大段 `draft_text` 作为工具参数传入。 - 校验和导出 records 时，如果 records 已经保存在 JSON/JSONL 文件中，优先传 `records_path`。 @@ -190,6 +195,7 @@ review 展示必须简短清晰，不要重复解释工具和流程。每次 rev ### case: case名称 用户: 本轮 query +complex: false target: Agent(tag="xxx") notes: 可选，说明覆盖的问题或边界 @@ -197,6 +203,7 @@ notes: 可选，说明覆盖的问题或边界 用户: 前一轮 query 小爱: 前一轮 tts 用户: 本轮 query +complex: false target: Agent(tag="xxx") notes: 可选，说明覆盖的问题或边界 ``` @@ -207,11 +214,12 @@ notes: 可选，说明覆盖的问题或边界 - `用户:` 表示用户 query。 - `小爱:` 表示小爱回复 tts。 - 最后一个 `用户:` 是本轮 query。 +- `complex:` 是复杂度维度，必须独立填写 `true` 或 `false`；无法确定时先问用户。 - `target:` 是本轮 query 对应的监督标签，必须存在。 - 为避免工具参数 JSON 转义失败，dataset draft 里 Agent 标签优先写 `target: Agent(tag='xxx')`；转换工具会统一规范为 `Agent(tag="xxx")`。 -- 单轮数据只需要写一行 `用户:`，然后写 `target:`。 +- 单轮数据只需要写一行 `用户:`，然后写 `complex:` 和 `target:`。 - 多轮数据需要按照时间顺序写多组 `用户:` / `小爱:`。 -- 多轮数据的最后一轮只写 `用户:` 和 `target:`，不要写最后一轮 `小爱:`，因为本轮 query 不包含 tts。 +- 多轮数据的最后一轮只写 `用户:`、`complex:` 和 `target:`，不要写最后一轮 `小爱:`，因为本轮 query 不包含 tts。 - 如果 `target` 无法确定，不要编造，必须向用户确认。 - 不要手写 `record_id`、`request_id`、`timestamp`、`context`。 - 线上挖掘数据如有真实 `request_id` 和 `timestamp`，可以附加在 case 中；没有则不写。 @@ -226,7 +234,8 @@ canonical records 落盘必须使用 `data_agent_export_dataset_records`，默 - 训练数据：调用 `data_agent_export_training_jsonl`，输出 `training.jsonl`，每行包含 `system`、`instruction`、`output`。 - 评测数据：调用 `data_agent_export_planning_eval_csv`，输出 `eval_planning.csv`，字段为 `request_id,newPrompt,query,类别真实标签,code标签,complex`。 -- 两个派生格式默认复用相同 prompt 构造逻辑：`[知识注入]`、`[系统状态]`、`[对话历史]`、`[当前query]`、`[function]`。 +- 训练 `output` 和流转表格 `function` 列都会自动组合为两行：第一行 `complex=true/false`，第二行原监督标签；评测 `complex` 列直接来自 canonical record 的 `dimensions.complex`，按评测表习惯输出 `TRUE/FALSE`。 +- 训练 `instruction` 和评测 `newPrompt` 复用相同 prompt 主体：`[知识注入]`、`[系统状态]`、`[对话历史]`、`[当前query]`、`[function]`；评测 `newPrompt` 还会额外包上 `<|im_start|>system`、`<|im_start|>user`、`<|im_start|>assistant` chat template。 - 对话历史默认最多取 5 轮，且相邻轮间隔不超过 5 分钟；`context` 默认注入 `location` 和 `rag` 两个字段。 当前 canonical record v1 工作格式： @@ -256,6 +265,9 @@ canonical records 落盘必须使用 `data_agent_export_dataset_records`，默 "target": "Agent(tag=\"xxx\")", "target_type": "agent" }, + "dimensions": { + "complex": false + }, "meta": { "case_name": "case名称", "notes": "" @@ -357,7 +369,7 @@ skills/product-data/scripts/normalize_dataset_draft.py ```json { - "draft_text": "# dataset_label: 地图餐饮边界\n\n### case: 找附近美食\n用户: 附近有什么好吃的\ntarget: Agent(tag='餐饮服务')", + "draft_text": "# dataset_label: 地图餐饮边界\n\n### case: 找附近美食\n用户: 附近有什么好吃的\ncomplex: false\ntarget: Agent(tag='餐饮服务')", "batch_id": "aabbccdd", "source_type": "generated" } @@ -369,7 +381,7 @@ skills/product-data/scripts/normalize_dataset_draft.py {"draft_path": "output/dataset_draft.txt", "batch_id": "aabbccdd", "source_type": "generated"} ``` -用途：把 dataset draft text v1 转成 canonical records。 +用途：把 dataset draft text v1 转成 canonical records。每条 case 推荐包含 `complex: true/false`；旧草稿缺失时会按 `false` 兼容。 ### `product_data_validate_dataset_records` @@ -419,6 +431,8 @@ skills/product-data/scripts/export_dataset_records.py request_id,timestamp,query,prev_session,context,label,是否迁移Function,function ``` +其中 `function` 列会自动组合为 `complex=true/false` 和原监督标签两行。 + ### `product_data_export_dataset_table` 脚本： @@ -456,6 +470,7 @@ skills/product-data/scripts/export_training_jsonl.py ``` 用途：把 canonical records 转成训练 JSONL，默认文件名 `training.jsonl`。 +输出的 `output` 字段会自动组合 `complex=true/false` 和 `label.target` 两行。 ### `product_data_export_planning_eval_csv` @@ -475,6 +490,7 @@ skills/product-data/scripts/export_planning_eval_csv.py ``` 用途：把 canonical records 转成含 `newPrompt` 的评测 CSV，默认文件名 `eval_planning.csv`。 +`newPrompt` 会使用评测侧标准 chat template；`complex` 列来自 canonical records 的 `dimensions.complex`，输出为 `TRUE/FALSE`，不要用固定默认值覆盖。 ## 当前可用工具 @@ -494,11 +510,11 @@ skills/product-data/scripts/export_planning_eval_csv.py - `data_agent_show_generation_plan`：用户要求查看当前计划，或继续上下文时需要恢复计划详情时使用。 - `data_agent_update_generation_plan`：用户对计划提出修改意见后使用，更新计划并重新展示。 - `data_agent_confirm_generation_plan`：用户明确确认当前计划版本后使用，获取 `confirmed_plan_id`。 -- `data_agent_normalize_dataset_draft`：用户确认计划后，把 dataset draft text v1 转成 canonical records；必须传入 `confirmed_plan_id`；支持 `draft_text` 或 `draft_path`，大草稿优先 `draft_path`。 +- `data_agent_normalize_dataset_draft`：用户确认计划后，把 dataset draft text v1 转成 canonical records；必须传入 `confirmed_plan_id`；支持 `draft_text` 或 `draft_path`，大草稿优先 `draft_path`；每条 case 要写 `complex: true/false`。 - `data_agent_validate_dataset_records`：对 canonical records 做结构、标签、时间戳和多轮上下文校验；支持 `records` 或 `records_path`。 - `data_agent_export_dataset_records`：校验 canonical records 并落盘；支持 `records` 或 `records_path`；默认写紧凑 JSONL，一行一条，固定传 `output/records.jsonl`，并同时生成 `output/records.csv` 表格，不要再用 `write_file` 手写 records 或表格文件。 -- `data_agent_export_training_jsonl`：把 canonical records 转成训练 JSONL；支持 `records` 或 `records_path`；默认写 `output/training.jsonl`。 -- `data_agent_export_planning_eval_csv`：把 canonical records 转成评测 CSV；支持 `records` 或 `records_path`；默认写 `output/eval_planning.csv`。 +- `data_agent_export_training_jsonl`：把 canonical records 转成训练 JSONL；支持 `records` 或 `records_path`；默认写 `output/training.jsonl`；`output` 自动包含 `complex` 行。 +- `data_agent_export_planning_eval_csv`：把 canonical records 转成评测 CSV；支持 `records` 或 `records_path`；默认写 `output/eval_planning.csv`；`complex` 列从元数据生成。 ## 约束 diff --git a/skills/product-data/knowledge/canonical_record_v1.md b/skills/product-data/knowledge/canonical_record_v1.md index b38014d..351fcd0 100644 --- a/skills/product-data/knowledge/canonical_record_v1.md +++ b/skills/product-data/knowledge/canonical_record_v1.md @@ -29,6 +29,9 @@ canonical record 是产品数据生成链路的中间元数据格式。它不是 "target": "Agent(tag=\"xxx\")", "target_type": "agent" }, + "dimensions": { + "complex": false + }, "meta": { "case_name": "case名称", "notes": "" @@ -44,6 +47,7 @@ canonical record 是产品数据生成链路的中间元数据格式。它不是 - 相邻轮时间间隔超过 5 分钟时给 warning。 - `label.target` 必须有值。 - `target_type` 只能是 `agent`、`function` 或 `unknown`。 +- `dimensions.complex` 必须是布尔值，表示当前 query 是否为复杂任务；它是独立维度，不写入 `label.target`。 ## 默认导出 @@ -52,7 +56,7 @@ canonical record 是产品数据生成链路的中间元数据格式。它不是 - `records.jsonl`：canonical records，一行一条紧凑 JSON。 - `records.csv`：同事流转表格，字段为 `request_id,timestamp,query,prev_session,context,label,是否迁移Function,function`。 -`records.csv` 中的 `prev_session` 和 `context` 是紧凑 JSON 字符串；`prev_session` 中的 `timestamp` 按历史表格习惯输出为字符串。 +`records.csv` 中的 `prev_session` 和 `context` 是紧凑 JSON 字符串；`prev_session` 中的 `timestamp` 按历史表格习惯输出为字符串。`function` 列会把复杂度和监督标签组合成两行，例如 `complex=false\nAgent(tag="地图导航")`。 如果已有 `records.jsonl` 或 `records.json`，只需要补表格，可以执行 portable `export_dataset_table.py` 或等价工具；默认仍输出到同目录的 `records.csv`。 @@ -60,7 +64,7 @@ canonical record 是产品数据生成链路的中间元数据格式。它不是 训练和评测格式都从 canonical records 转换，不由模型手写： -- `training.jsonl`：每行 `{"system": "...", "instruction": "...", "output": "..."}`，由 `export_training_jsonl.py` 或 `data_agent_export_training_jsonl` 生成。 -- `eval_planning.csv`：字段为 `request_id,newPrompt,query,类别真实标签,code标签,complex`，由 `export_planning_eval_csv.py` 或 `data_agent_export_planning_eval_csv` 生成。 +- `training.jsonl`：每行 `{"system": "...", "instruction": "...", "output": "..."}`，由 `export_training_jsonl.py` 或 `data_agent_export_training_jsonl` 生成；`output` 会输出 `complex=true/false` 加监督标签两行。 +- `eval_planning.csv`：字段为 `request_id,newPrompt,query,类别真实标签,code标签,complex`，由 `export_planning_eval_csv.py` 或 `data_agent_export_planning_eval_csv` 生成；`newPrompt` 会包上 `<|im_start|>system/user/assistant` chat template；`complex` 列来自 `dimensions.complex`，按评测表习惯输出 `TRUE/FALSE`，不是固定默认值。 -`instruction` 和 `newPrompt` 使用同一套 prompt 模板，默认包含 `[知识注入]`、`[系统状态]`、`[对话历史]`、`[当前query]`、`[function]`。历史轮次默认最多取 5 轮，且相邻时间间隔不超过 5 分钟。 +`instruction` 和 `newPrompt` 使用同一套 prompt 主体，默认包含 `[知识注入]`、`[系统状态]`、`[对话历史]`、`[当前query]`、`[function]`。历史轮次默认最多取 5 轮，且相邻时间间隔不超过 5 分钟。 diff --git a/skills/product-data/knowledge/dataset_draft_v1.md b/skills/product-data/knowledge/dataset_draft_v1.md index a81ca7e..9ff0a91 100644 --- a/skills/product-data/knowledge/dataset_draft_v1.md +++ b/skills/product-data/knowledge/dataset_draft_v1.md @@ -9,6 +9,7 @@ ### case: case名称 用户: 本轮 query +complex: false target: Agent(tag='xxx') notes: 可选，说明覆盖的问题或边界 @@ -16,6 +17,7 @@ notes: 可选，说明覆盖的问题或边界 用户: 前一轮 query 小爱: 前一轮 tts 用户: 本轮 query +complex: false target: Agent(tag='xxx') notes: 可选，说明覆盖的问题或边界 ``` @@ -26,10 +28,11 @@ notes: 可选，说明覆盖的问题或边界 - `用户:` 表示用户 query。 - `小爱:` 表示小爱回复 tts。 - 最后一个 `用户:` 是本轮 query。 +- `complex:` 是复杂度维度，必须独立填写 `true` 或 `false`；无法确定时先问用户，不要把它塞进 `target`。 - `target:` 是本轮 query 对应的监督标签，必须存在。 - Agent 标签推荐写单引号形式 `Agent(tag='xxx')`，转换工具会规范成 `Agent(tag="xxx")`。 -- 单轮数据只需要写一行 `用户:`，然后写 `target:`。 +- 单轮数据只需要写一行 `用户:`，然后写 `complex:` 和 `target:`。 - 多轮数据需要按照时间顺序写多组 `用户:` / `小爱:`。 -- 多轮数据的最后一轮只写 `用户:` 和 `target:`，不要写最后一轮 `小爱:`。 +- 多轮数据的最后一轮只写 `用户:`、`complex:` 和 `target:`，不要写最后一轮 `小爱:`。 - 如果 `target` 无法确定，不要编造，必须向用户确认。 - 不要手写 `record_id`、`request_id`、`timestamp`、`context`。 diff --git a/skills/product-data/schemas/export_planning_eval_csv.input.schema.json b/skills/product-data/schemas/export_planning_eval_csv.input.schema.json index bd793e4..d0e7535 100644 --- a/skills/product-data/schemas/export_planning_eval_csv.input.schema.json +++ b/skills/product-data/schemas/export_planning_eval_csv.input.schema.json @@ -18,7 +18,15 @@ "items": {"type": "string"}, "default": ["location", "rag"] }, - "complex_default": {"type": "boolean", "default": false}, + "system_prompt": { + "type": "string", + "default": "你是小爱同学，中文智能语音助手。" + }, + "complex_default": { + "type": "boolean", + "default": false, + "description": "Fallback only for legacy records missing dimensions.complex." + }, "require_validation_ok": {"type": "boolean", "default": true}, "overwrite": {"type": "boolean", "default": true} } diff --git a/skills/product-data/schemas/normalize_dataset_draft.input.schema.json b/skills/product-data/schemas/normalize_dataset_draft.input.schema.json index f97cb4f..ce8ab5d 100644 --- a/skills/product-data/schemas/normalize_dataset_draft.input.schema.json +++ b/skills/product-data/schemas/normalize_dataset_draft.input.schema.json @@ -3,7 +3,7 @@ "properties": { "draft_text": { "type": "string", - "description": "Dataset draft text. Provide exactly one of draft_text or draft_path." + "description": "Dataset draft text. Each case should include complex: true/false. Provide exactly one of draft_text or draft_path." }, "draft_path": { "type": "string", diff --git a/skills/product-data/scripts/export_planning_eval_csv.py b/skills/product-data/scripts/export_planning_eval_csv.py index df1f3a7..c0232b1 100644 --- a/skills/product-data/scripts/export_planning_eval_csv.py +++ b/skills/product-data/scripts/export_planning_eval_csv.py @@ -1,6 +1,7 @@ from __future__ import annotations from product_data_portable import ( + DEFAULT_SYSTEM_PROMPT, default_planning_eval_output_path, emit_error, emit_success, @@ -20,6 +21,7 @@ def main() -> int: session_num=int(payload.get("session_num", 5)), session_time_minutes=int(payload.get("session_time_minutes", 5)), context_fields=payload.get("context_fields"), + system_prompt=str(payload.get("system_prompt") or DEFAULT_SYSTEM_PROMPT), complex_default=bool(payload.get("complex_default", False)), require_validation_ok=bool(payload.get("require_validation_ok", True)), overwrite=bool(payload.get("overwrite", True)), diff --git a/skills/product-data/scripts/product_data_portable.py b/skills/product-data/scripts/product_data_portable.py index 26b4944..95ac8e5 100644 --- a/skills/product-data/scripts/product_data_portable.py +++ b/skills/product-data/scripts/product_data_portable.py @@ -159,7 +159,7 @@ def parse_draft_text(draft_text: str) -> dict[str, Any]: continue field_match = re.match( - r"^(用户|小爱|target|notes|dataset_label|request_id|timestamp)\s*[:：]\s*(.*)$", + r"^(用户|小爱|target|complex|notes|dataset_label|request_id|timestamp)\s*[:：]\s*(.*)$", line, re.I, ) @@ -207,9 +207,14 @@ def case_to_record( if not current_query: raise ProductDataError(f"case {index} current 用户 line must be non-empty") - target = canonical_target(str(case.get("target") or "").strip()) + raw_target, prefixed_complex = split_complex_prefixed_target(str(case.get("target") or "").strip()) + target = canonical_target(raw_target) if not target: raise ProductDataError(f"case {index} target is required") + complex_value = parse_complex( + case.get("complex"), + default=False if prefixed_complex is None else prefixed_complex, + ) prev_session = build_prev_session(turns[:final_user_index], case_index=index) if len(prev_session) > 10: @@ -248,6 +253,9 @@ def case_to_record( "target": target, "target_type": target_type(target), }, + "dimensions": { + "complex": complex_value, + }, "meta": { "case_name": str(case.get("case_name") or "").strip(), "notes": str(case.get("notes") or "").strip(), @@ -364,6 +372,14 @@ def validate_dataset_records(records: list[dict[str, Any]]) -> dict[str, Any]: if label.get("target_type") not in {"agent", "function", "unknown"}: errors.append(issue(f"{prefix}.label.target_type", "label.target_type is invalid")) + dimensions = record.get("dimensions") + if not isinstance(dimensions, dict): + warnings.append(issue(f"{prefix}.dimensions", "dimensions.complex is missing; false will be used as fallback")) + elif "complex" not in dimensions: + warnings.append(issue(f"{prefix}.dimensions.complex", "complex is missing; false will be used as fallback")) + elif not isinstance(dimensions.get("complex"), bool): + errors.append(issue(f"{prefix}.dimensions.complex", "complex must be a boolean")) + return { "ok": not errors, "error_count": len(errors), @@ -500,6 +516,7 @@ def export_planning_eval_csv( session_num: int = 5, session_time_minutes: int = 5, context_fields: list[str] | None = None, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, complex_default: bool = False, require_validation_ok: bool = True, overwrite: bool = True, @@ -520,6 +537,7 @@ def export_planning_eval_csv( session_num=session_num, session_time_minutes=session_time_minutes, context_fields=context_fields, + system_prompt=system_prompt, complex_default=complex_default, ) path.write_text(content, encoding="utf-8-sig") @@ -547,6 +565,7 @@ def render_planning_eval_csv( session_num: int = 5, session_time_minutes: int = 5, context_fields: list[str] | None = None, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, complex_default: bool = False, ) -> str: fields = normalize_context_fields(context_fields) @@ -560,21 +579,21 @@ def render_planning_eval_csv( for record in records: source = record.get("source") if isinstance(record.get("source"), dict) else {} turn = record.get("turn") if isinstance(record.get("turn"), dict) else {} - label = record.get("label") if isinstance(record.get("label"), dict) else {} - target = str(label.get("target") or "") + target = record_target(record) writer.writerow( { "request_id": str(source.get("request_id") or ""), - "newPrompt": build_training_instruction( + "newPrompt": build_planning_prompt( record, session_num=session_num, session_time_minutes=session_time_minutes, context_fields=fields, + system_prompt=system_prompt, ), "query": str(turn.get("query") or ""), "类别真实标签": category_label_from_target(target), "code标签": target, - "complex": "true" if complex_default else "false", + "complex": eval_complex_literal(record_complex(record, default=complex_default)), } ) return output.getvalue() @@ -621,6 +640,29 @@ def build_training_instruction( return instruction +def build_planning_prompt( + record: dict[str, Any], + *, + session_num: int = 5, + session_time_minutes: int = 5, + context_fields: list[str] | None = None, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, +) -> str: + """把训练 instruction 包成评测侧使用的 chat template。""" + + instruction = build_training_instruction( + record, + session_num=session_num, + session_time_minutes=session_time_minutes, + context_fields=context_fields, + ) + return ( + f"<|im_start|>system\n{system_prompt}<|im_end|>\n" + f"<|im_start|>user\n{instruction}<|im_end|>\n" + "<|im_start|>assistant\n" + ) + + def training_jsonl_line( record: dict[str, Any], *, @@ -629,7 +671,6 @@ def training_jsonl_line( context_fields: list[str], system_prompt: str, ) -> str: - label = record.get("label") if isinstance(record.get("label"), dict) else {} payload = { "system": system_prompt, "instruction": build_training_instruction( @@ -638,7 +679,7 @@ def training_jsonl_line( session_time_minutes=session_time_minutes, context_fields=context_fields, ), - "output": str(label.get("target") or ""), + "output": combined_function_label(record), } return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) @@ -711,6 +752,7 @@ def optional_int_for_export(value: Any) -> int | None: def category_label_from_target(target: str) -> str: + target = strip_complex_prefix(target) agent_match = re.match(r'''^Agent\s*\(\s*tag\s*=\s*["']([^"']+)["']\s*\)$''', target.strip()) if agent_match: return agent_match.group(1) @@ -749,7 +791,7 @@ def dataset_record_table_row(record: dict[str, Any]) -> dict[str, str]: "context": json.dumps(context, ensure_ascii=False, separators=(",", ":")), "label": str(label.get("dataset_label") or ""), "是否迁移Function": "", - "function": str(label.get("target") or ""), + "function": combined_function_label(record), } @@ -808,7 +850,75 @@ def default_planning_eval_output_path(payload: dict[str, Any]) -> str: return str(Path("output") / "eval_planning.csv") +def normalize_target_expression(target: str) -> str: + """把可能带 complex 前缀的标签表达式收敛为纯 target。""" + + return canonical_target(strip_complex_prefix(target)) + + +def strip_complex_prefix(target: str) -> str: + stripped_target, _complex_value = split_complex_prefixed_target(target) + return stripped_target + + +def split_complex_prefixed_target(target: str) -> tuple[str, bool | None]: + lines = target.strip().splitlines() + if not lines: + return "", None + first_line = lines[0].strip() + match = re.fullmatch(r"complex\s*=\s*(.+)", first_line, flags=re.I) + if not match: + return target.strip(), None + complex_value = parse_complex(match.group(1), default=False) + return "\n".join(lines[1:]).strip(), complex_value + + +def parse_complex(value: Any, *, default: bool) -> bool: + if value is None or value == "": + return default + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if text in {"true", "1", "yes", "y", "是", "复杂", "complex"}: + return True + if text in {"false", "0", "no", "n", "否", "不复杂", "简单", "simple"}: + return False + raise ProductDataError("complex must be a boolean value such as true/false") + + +def complex_literal(value: bool) -> str: + return "true" if value else "false" + + +def eval_complex_literal(value: bool) -> str: + return "TRUE" if value else "FALSE" + + +def record_target(record: dict[str, Any]) -> str: + label = record.get("label") if isinstance(record.get("label"), dict) else {} + return normalize_target_expression(str(label.get("target") or "")) + + +def record_complex(record: dict[str, Any], *, default: bool = False) -> bool: + dimensions = record.get("dimensions") + if isinstance(dimensions, dict) and "complex" in dimensions: + return parse_complex(dimensions.get("complex"), default=default) + label = record.get("label") if isinstance(record.get("label"), dict) else {} + if "complex" in label: + return parse_complex(label.get("complex"), default=default) + _target, prefixed_complex = split_complex_prefixed_target(str(label.get("target") or "")) + if prefixed_complex is not None: + return prefixed_complex + return default + + +def combined_function_label(record: dict[str, Any], *, default_complex: bool = False) -> str: + target = record_target(record) + return f"complex={complex_literal(record_complex(record, default=default_complex))}\n{target}" + + def target_type(target: str) -> str: + target = strip_complex_prefix(target) if re.match(r"^Agent\s*\(\s*tag\s*=", target): return "agent" if target: @@ -817,6 +927,7 @@ def target_type(target: str) -> str: def canonical_target(target: str) -> str: + target = strip_complex_prefix(target) agent_match = re.match(r'''^Agent\s*\(\s*tag\s*=\s*["']([^"']+)["']\s*\)$''', target) if agent_match: return f'Agent(tag="{agent_match.group(1)}")' diff --git a/skills/product-data/tools.yaml b/skills/product-data/tools.yaml index 3707081..46469f8 100644 --- a/skills/product-data/tools.yaml +++ b/skills/product-data/tools.yaml @@ -7,7 +7,7 @@ runtime: dependencies: [] tools: - name: product_data_normalize_dataset_draft - description: Parse dataset draft text v1 into canonical records with generated ids, timestamps, source metadata, context and labels. + description: Parse dataset draft text v1 into canonical records with generated ids, timestamps, source metadata, context, labels and dimensions.complex. script: scripts/normalize_dataset_draft.py input_schema: schemas/normalize_dataset_draft.input.schema.json output_schema: schemas/normalize_dataset_draft.output.schema.json @@ -15,7 +15,7 @@ tools: command: python skills/product-data/scripts/normalize_dataset_draft.py --input stdin: true - name: product_data_validate_dataset_records - description: Validate canonical product-data records for required fields, labels, timestamp order and prompt stitching constraints. + description: Validate canonical product-data records for required fields, labels, dimensions.complex, timestamp order and prompt stitching constraints. script: scripts/validate_dataset_records.py input_schema: schemas/validate_dataset_records.input.schema.json output_schema: schemas/validate_dataset_records.output.schema.json @@ -23,7 +23,7 @@ tools: command: python skills/product-data/scripts/validate_dataset_records.py --input stdin: true - name: product_data_export_dataset_records - description: Validate canonical records and write compact JSONL or JSON files plus a sibling records.csv sharing table without asking the model to hand-write files. + description: Validate canonical records and write compact JSONL or JSON files plus a sibling records.csv sharing table whose function column combines complex and label target. script: scripts/export_dataset_records.py input_schema: schemas/export_dataset_records.input.schema.json output_schema: schemas/export_dataset_records.output.schema.json @@ -31,7 +31,7 @@ tools: command: python skills/product-data/scripts/export_dataset_records.py --input stdin: true - name: product_data_export_dataset_table - description: Convert existing canonical records into the shared records.csv table format without rewriting the metadata file. + description: Convert existing canonical records into the shared records.csv table format without rewriting the metadata file; function column combines complex and label target. script: scripts/export_dataset_table.py input_schema: schemas/export_dataset_table.input.schema.json output_schema: schemas/export_dataset_table.output.schema.json @@ -39,7 +39,7 @@ tools: command: python skills/product-data/scripts/export_dataset_table.py --input stdin: true - name: product_data_export_training_jsonl - description: Convert canonical records into training JSONL lines with system, instruction and output fields. + description: Convert canonical records into training JSONL lines with system, instruction and output fields; output combines complex and label target. script: scripts/export_training_jsonl.py input_schema: schemas/export_training_jsonl.input.schema.json output_schema: schemas/export_training_jsonl.output.schema.json @@ -47,7 +47,7 @@ tools: command: python skills/product-data/scripts/export_training_jsonl.py --input stdin: true - name: product_data_export_planning_eval_csv - description: Convert canonical records into evaluation CSV with request_id, newPrompt, query, 类别真实标签, code标签 and complex columns. + description: Convert canonical records into evaluation CSV with request_id, newPrompt, query, 类别真实标签, code标签 and complex columns; newPrompt uses chat-template tags and complex is read from dimensions.complex as TRUE/FALSE. script: scripts/export_planning_eval_csv.py input_schema: schemas/export_planning_eval_csv.input.schema.json output_schema: schemas/export_planning_eval_csv.output.schema.json diff --git a/src/agent_tool_specs/data_agent.py b/src/agent_tool_specs/data_agent.py index d3545e6..3b0f1cb 100644 --- a/src/agent_tool_specs/data_agent.py +++ b/src/agent_tool_specs/data_agent.py @@ -346,6 +346,10 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo 'type': 'string', 'description': 'Target label used for included candidates unless review_decisions provides a target.', }, + 'default_complex': { + 'type': 'boolean', + 'description': 'Default complex dimension for included candidates unless review_decisions provides complex.', + }, 'review_decisions': { 'type': 'array', 'description': 'Optional include/exclude/uncertain decisions keyed by semantic_session_id or req_id.', @@ -357,6 +361,7 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo 'matched_turn_index': {'type': 'integer'}, 'decision': {'type': 'string', 'enum': ['include', 'exclude', 'uncertain']}, 'target': {'type': 'string'}, + 'complex': {'type': 'boolean'}, 'notes': {'type': 'string'}, }, }, @@ -428,7 +433,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo name='data_agent_normalize_dataset_draft', description=( 'Parse dataset draft text written with 用户/小爱/target blocks and build canonical ' - 'data-agent records with generated record IDs, timestamps, source metadata, context, and labels. ' + 'data-agent records with generated record IDs, timestamps, source metadata, context, labels, ' + 'and dimensions.complex. ' 'Generated data requires a confirmed_plan_id from data_agent_confirm_generation_plan. ' 'Use draft_path instead of draft_text when the draft is large or contains many quoted targets.' ), @@ -437,7 +443,7 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo 'properties': { 'draft_text': { 'type': 'string', - 'description': 'Dataset draft text in dataset draft text v1 format. Provide exactly one of draft_text or draft_path.', + 'description': 'Dataset draft text in dataset draft text v1 format. Each case should include complex: true/false. Provide exactly one of draft_text or draft_path.', }, 'draft_path': { 'type': 'string', @@ -476,7 +482,7 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo name='data_agent_validate_dataset_records', description=( 'Validate canonical data-agent records for required fields, labels, source metadata, ' - 'prev_session structure, timestamp order, and 5-minute prompt stitching constraints.' + 'prev_session structure, dimensions.complex, timestamp order, and 5-minute prompt stitching constraints.' ), parameters={ 'type': 'object', @@ -501,7 +507,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo description=( 'Validate canonical data-agent records and write them to a workspace file. ' 'Defaults to compact JSONL, one canonical record per line, and also writes records.csv ' - 'in the same output directory for human sharing.' + 'in the same output directory for human sharing. The table function column combines ' + 'complex=true/false and label.target.' ), parameters={ 'type': 'object', @@ -551,7 +558,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo name='data_agent_export_training_jsonl', description=( 'Convert canonical data-agent records into training JSONL. Each line has system, instruction, ' - 'and output. The instruction contains 知识注入, 系统状态, 对话历史, 当前query, and function sections.' + 'and output. Output combines dimensions.complex and label.target as two lines. ' + 'The instruction contains 知识注入, 系统状态, 对话历史, 当前query, and function sections.' ), parameters={ 'type': 'object', @@ -600,7 +608,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo name='data_agent_export_planning_eval_csv', description=( 'Convert canonical data-agent records into evaluation CSV with columns: request_id, newPrompt, ' - 'query, 类别真实标签, code标签, complex. newPrompt uses the same prompt body as training instruction.' + 'query, 类别真实标签, code标签, complex. The complex column is read from dimensions.complex. ' + 'newPrompt wraps the prompt body with <|im_start|>system/user/assistant chat-template tags.' ), parameters={ 'type': 'object', @@ -635,9 +644,13 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo 'items': {'type': 'string'}, 'description': 'Context fields to inject. Defaults to ["location", "rag"].', }, + 'system_prompt': { + 'type': 'string', + 'description': 'System prompt wrapped into newPrompt chat template. Defaults to 你是小爱同学，中文智能语音助手。', + }, 'complex_default': { 'type': 'boolean', - 'description': 'Default complex column value. Defaults to false.', + 'description': 'Fallback complex value only for legacy records missing dimensions.complex. Defaults to false.', }, 'require_validation_ok': {'type': 'boolean'}, 'overwrite': {'type': 'boolean'}, diff --git a/src/agent_tools.py b/src/agent_tools.py index 14abb2d..51f288c 100644 --- a/src/agent_tools.py +++ b/src/agent_tools.py @@ -1417,11 +1417,15 @@ def _data_agent_convert_router_candidates_to_records_tool( include_uncertain = arguments.get('include_uncertain', False) if not isinstance(include_uncertain, bool): raise ToolExecutionError('include_uncertain must be a boolean') + default_complex = arguments.get('default_complex', False) + if not isinstance(default_complex, bool): + raise ToolExecutionError('default_complex must be a boolean') try: payload = convert_router_candidates_to_records( arguments['candidates'], dataset_label=_require_string(arguments, 'dataset_label'), default_target=_optional_string(arguments, 'default_target'), + default_complex=default_complex, review_decisions=review_decisions, batch_id=_optional_string(arguments, 'batch_id') or 'router', include_uncertain=include_uncertain, @@ -1596,6 +1600,9 @@ def _export_planning_eval_csv_tool(arguments: dict[str, Any], context: ToolExecu complex_default = arguments.get('complex_default', False) if not isinstance(complex_default, bool): raise ToolExecutionError('complex_default must be a boolean') + system_prompt = arguments.get('system_prompt', '你是小爱同学，中文智能语音助手。') + if not isinstance(system_prompt, str): + raise ToolExecutionError('system_prompt must be a string') try: records = _records_from_arguments(arguments, context) payload = export_planning_eval_csv( @@ -1609,6 +1616,7 @@ def _export_planning_eval_csv_tool(arguments: dict[str, Any], context: ToolExecu session_num=_coerce_int(arguments, 'session_num', 5), session_time_minutes=_coerce_int(arguments, 'session_time_minutes', 5), context_fields=_optional_string_list(arguments, 'context_fields'), + system_prompt=system_prompt, complex_default=complex_default, require_validation_ok=require_validation_ok, overwrite=overwrite, diff --git a/src/data_agent_records.py b/src/data_agent_records.py index d6b6c42..f8be1be 100644 --- a/src/data_agent_records.py +++ b/src/data_agent_records.py @@ -53,6 +53,7 @@ def prepare_generation_goal( if missing: raise DataRecordError('missing required goal fields: ' + ', '.join(missing)) normalized_targets = _normalize_target_definitions(target_definitions) + target = _normalize_target_expression(target) if not target.strip() and not normalized_targets: raise DataRecordError('generation goal must include target or target_definitions') state = _load_goal_state(root) @@ -63,7 +64,7 @@ def prepare_generation_goal( 'status': 'pending_confirmation', 'dataset_label': dataset_label.strip(), 'goal_summary': goal_summary.strip(), - 'target': target.strip(), + 'target': target, 'target_definitions': normalized_targets, 'plan_hint': plan_hint.strip(), 'coverage': coverage.strip(), @@ -191,13 +192,14 @@ def prepare_generation_plan( 'output_path': output_path, } ) + target = _normalize_target_expression(target) normalized_targets = _normalize_target_definitions(target_definitions) if confirmed_goal is not None and not target.strip() and not normalized_targets: normalized_targets = _normalize_target_definitions( confirmed_goal.get('target_definitions') if isinstance(confirmed_goal, dict) else None ) if not normalized_targets: - target = str(confirmed_goal.get('target') or '').strip() + target = _normalize_target_expression(str(confirmed_goal.get('target') or '')) if not target.strip() and not normalized_targets: missing.append('target') if missing: @@ -220,7 +222,7 @@ def prepare_generation_plan( 'plan_id': plan_id, 'status': 'pending_confirmation', 'dataset_label': dataset_label.strip(), - 'target': target.strip(), + 'target': target, 'target_definitions': normalized_targets, 'total_count': total_count, 'turn_mix': turn_mix.strip(), @@ -448,6 +450,14 @@ def validate_dataset_records(records: list[dict[str, Any]]) -> dict[str, Any]: if label.get('target_type') not in {'agent', 'function', 'unknown'}: errors.append(_issue(f'{prefix}.label.target_type', 'label.target_type is invalid')) + dimensions = record.get('dimensions') + if not isinstance(dimensions, dict): + warnings.append(_issue(f'{prefix}.dimensions', 'dimensions.complex is missing; false will be used as fallback')) + elif 'complex' not in dimensions: + warnings.append(_issue(f'{prefix}.dimensions.complex', 'complex is missing; false will be used as fallback')) + elif not isinstance(dimensions.get('complex'), bool): + errors.append(_issue(f'{prefix}.dimensions.complex', 'complex must be a boolean')) + return { 'ok': not errors, 'error_count': len(errors), @@ -579,6 +589,7 @@ def export_planning_eval_csv( session_num: int = 5, session_time_minutes: int = 5, context_fields: list[str] | None = None, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, complex_default: bool = False, require_validation_ok: bool = True, overwrite: bool = True, @@ -604,6 +615,7 @@ def export_planning_eval_csv( session_num=session_num, session_time_minutes=session_time_minutes, context_fields=fields, + system_prompt=system_prompt, complex_default=complex_default, ) path.write_text(content, encoding='utf-8-sig') @@ -633,6 +645,7 @@ def render_planning_eval_csv( session_num: int = 5, session_time_minutes: int = 5, context_fields: list[str] | None = None, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, complex_default: bool = False, ) -> str: """把 canonical records 转成含 newPrompt 的评测 CSV。""" @@ -648,21 +661,21 @@ def render_planning_eval_csv( for record in records: source = record.get('source') if isinstance(record.get('source'), dict) else {} turn = record.get('turn') if isinstance(record.get('turn'), dict) else {} - label = record.get('label') if isinstance(record.get('label'), dict) else {} - target = str(label.get('target') or '') + target = _record_target(record) writer.writerow( { 'request_id': str(source.get('request_id') or ''), - 'newPrompt': build_training_instruction( + 'newPrompt': build_planning_prompt( record, session_num=session_num, session_time_minutes=session_time_minutes, context_fields=fields, + system_prompt=system_prompt, ), 'query': str(turn.get('query') or ''), '类别真实标签': _category_label_from_target(target), 'code标签': target, - 'complex': 'true' if complex_default else 'false', + 'complex': _eval_complex_literal(_record_complex(record, default=complex_default)), } ) return output.getvalue() @@ -709,6 +722,29 @@ def build_training_instruction( return instruction +def build_planning_prompt( + record: dict[str, Any], + *, + session_num: int = 5, + session_time_minutes: int = 5, + context_fields: list[str] | None = None, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, +) -> str: + """把训练 instruction 包成评测侧使用的 chat template。""" + + instruction = build_training_instruction( + record, + session_num=session_num, + session_time_minutes=session_time_minutes, + context_fields=context_fields, + ) + return ( + f'<|im_start|>system\n{system_prompt}<|im_end|>\n' + f'<|im_start|>user\n{instruction}<|im_end|>\n' + '<|im_start|>assistant\n' + ) + + def _training_jsonl_line( record: dict[str, Any], *, @@ -717,7 +753,6 @@ def _training_jsonl_line( context_fields: list[str], system_prompt: str, ) -> str: - label = record.get('label') if isinstance(record.get('label'), dict) else {} payload = { 'system': system_prompt, 'instruction': build_training_instruction( @@ -726,7 +761,7 @@ def _training_jsonl_line( session_time_minutes=session_time_minutes, context_fields=context_fields, ), - 'output': str(label.get('target') or ''), + 'output': _combined_function_label(record), } return json.dumps(payload, ensure_ascii=False, separators=(',', ':')) @@ -797,6 +832,7 @@ def _optional_int_for_export(value: Any) -> int | None: def _category_label_from_target(target: str) -> str: + target = _strip_complex_prefix(target) agent_match = re.match(r'''^Agent\s*\(\s*tag\s*=\s*["']([^"']+)["']\s*\)$''', target.strip()) if agent_match: return agent_match.group(1) @@ -835,7 +871,7 @@ def _dataset_record_table_row(record: dict[str, Any]) -> dict[str, str]: 'context': json.dumps(context, ensure_ascii=False, separators=(',', ':')), 'label': str(label.get('dataset_label') or ''), '是否迁移Function': '', - 'function': str(label.get('target') or ''), + 'function': _combined_function_label(record), } @@ -875,7 +911,7 @@ def _parse_draft_text(draft_text: str) -> dict[str, Any]: if current is None: continue - field_match = re.match(r'^(用户|小爱|target|notes|dataset_label|request_id|timestamp)\s*[:：]\s*(.*)$', line, re.IGNORECASE) + field_match = re.match(r'^(用户|小爱|target|complex|notes|dataset_label|request_id|timestamp)\s*[:：]\s*(.*)$', line, re.IGNORECASE) if not field_match: continue key = field_match.group(1).lower() @@ -925,7 +961,7 @@ def _normalize_target_definitions(value: list[dict[str, Any]] | None) -> list[di if not isinstance(item, dict): raise DataRecordError(f'target_definitions[{index}] must be an object') name = str(item.get('name') or '').strip() - target = str(item.get('target') or '').strip() + target = _normalize_target_expression(str(item.get('target') or '')) rule = str(item.get('rule') or '').strip() if not target: raise DataRecordError(f'target_definitions[{index}].target is required') @@ -958,10 +994,10 @@ def _allowed_targets_for_plan(plan: dict[str, Any]) -> set[str]: if isinstance(target_definitions, list): for item in target_definitions: if isinstance(item, dict) and isinstance(item.get('target'), str) and item['target'].strip(): - allowed.add(item['target'].strip()) + allowed.add(_normalize_target_expression(item['target'])) target = plan.get('target') if isinstance(target, str) and target.strip(): - allowed.add(target.strip()) + allowed.add(_normalize_target_expression(target)) return allowed @@ -980,7 +1016,7 @@ def _validate_plan_against_goal( goal_targets = _allowed_targets_for_goal(goal) plan_targets = set() if target.strip(): - plan_targets.add(target.strip()) + plan_targets.add(_normalize_target_expression(target)) for item in target_definitions: item_target = item.get('target', '').strip() if item_target: @@ -998,9 +1034,8 @@ def _validate_records_against_plan(records: list[dict[str, Any]], plan: dict[str return unexpected: list[str] = [] for index, record in enumerate(records): - label = record.get('label') - target = label.get('target') if isinstance(label, dict) else None - if isinstance(target, str) and target.strip() in allowed_targets: + target = _record_target(record) + if target and target in allowed_targets: continue unexpected.append(f'records[{index}].label.target={target!r}') if unexpected: @@ -1148,9 +1183,14 @@ def _case_to_record( if not current_query: raise DataRecordError(f'case {index} current 用户 line must be non-empty') - target = _canonical_target(str(case.get('target') or '').strip()) + raw_target, prefixed_complex = _split_complex_prefixed_target(str(case.get('target') or '').strip()) + target = _canonical_target(raw_target) if not target: raise DataRecordError(f'case {index} target is required') + complex_value = _parse_complex( + case.get('complex'), + default=False if prefixed_complex is None else prefixed_complex, + ) prev_session = _build_prev_session(turns[:final_user_index], case_index=index) if len(prev_session) > 10: @@ -1183,6 +1223,9 @@ def _case_to_record( 'target': target, 'target_type': _target_type(target), }, + 'dimensions': { + 'complex': complex_value, + }, 'meta': { 'case_name': str(case.get('case_name') or '').strip(), 'notes': str(case.get('notes') or '').strip(), @@ -1209,7 +1252,75 @@ def _build_prev_session(turns: list[dict[str, Any]], *, case_index: int) -> list return prev_session +def _normalize_target_expression(target: str) -> str: + """把可能带 complex 前缀的标签表达式收敛为纯 target。""" + + return _canonical_target(_strip_complex_prefix(target)) + + +def _strip_complex_prefix(target: str) -> str: + stripped_target, _complex_value = _split_complex_prefixed_target(target) + return stripped_target + + +def _split_complex_prefixed_target(target: str) -> tuple[str, bool | None]: + lines = target.strip().splitlines() + if not lines: + return '', None + first_line = lines[0].strip() + match = re.fullmatch(r'complex\s*=\s*(.+)', first_line, flags=re.IGNORECASE) + if not match: + return target.strip(), None + complex_value = _parse_complex(match.group(1), default=False) + return '\n'.join(lines[1:]).strip(), complex_value + + +def _parse_complex(value: Any, *, default: bool) -> bool: + if value is None or value == '': + return default + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if text in {'true', '1', 'yes', 'y', '是', '复杂', 'complex'}: + return True + if text in {'false', '0', 'no', 'n', '否', '不复杂', '简单', 'simple'}: + return False + raise DataRecordError('complex must be a boolean value such as true/false') + + +def _complex_literal(value: bool) -> str: + return 'true' if value else 'false' + + +def _eval_complex_literal(value: bool) -> str: + return 'TRUE' if value else 'FALSE' + + +def _record_target(record: dict[str, Any]) -> str: + label = record.get('label') if isinstance(record.get('label'), dict) else {} + return _normalize_target_expression(str(label.get('target') or '')) + + +def _record_complex(record: dict[str, Any], *, default: bool = False) -> bool: + dimensions = record.get('dimensions') + if isinstance(dimensions, dict) and 'complex' in dimensions: + return _parse_complex(dimensions.get('complex'), default=default) + label = record.get('label') if isinstance(record.get('label'), dict) else {} + if 'complex' in label: + return _parse_complex(label.get('complex'), default=default) + _target, prefixed_complex = _split_complex_prefixed_target(str(label.get('target') or '')) + if prefixed_complex is not None: + return prefixed_complex + return default + + +def _combined_function_label(record: dict[str, Any], *, default_complex: bool = False) -> str: + target = _record_target(record) + return f'complex={_complex_literal(_record_complex(record, default=default_complex))}\n{target}' + + def _target_type(target: str) -> str: + target = _strip_complex_prefix(target) if re.match(r'^Agent\s*\(\s*tag\s*=', target): return 'agent' if target: @@ -1220,6 +1331,7 @@ def _target_type(target: str) -> str: def _canonical_target(target: str) -> str: """规范常见 Agent 标签写法，允许草稿里用单引号降低 JSON 转义风险。""" + target = _strip_complex_prefix(target) match = re.fullmatch(r'Agent\s*\(\s*tag\s*=\s*([\'"])(.+?)\1\s*\)', target) if not match: return target diff --git a/src/data_agent_router_sessions.py b/src/data_agent_router_sessions.py index 45d6adf..2459fed 100644 --- a/src/data_agent_router_sessions.py +++ b/src/data_agent_router_sessions.py @@ -197,6 +197,7 @@ def convert_router_candidates_to_records( *, dataset_label: str, default_target: str = '', + default_complex: bool = False, review_decisions: list[dict[str, Any]] | None = None, batch_id: str = 'router', include_uncertain: bool = False, @@ -228,11 +229,17 @@ def convert_router_candidates_to_records( raise DataAgentRouterSessionError( f'target is required for candidate {candidate.get("semantic_session_id")}#{candidate.get("matched_turn_index")}' ) + raw_target, prefixed_complex = _split_complex_prefixed_target(target) + complex_value = _parse_complex( + decision.get('complex'), + default=default_complex if prefixed_complex is None else prefixed_complex, + ) records.append( _router_candidate_to_record( candidate, dataset_label=str(decision.get('dataset_label') or dataset_label).strip(), - target=target, + target=raw_target, + complex_value=complex_value, notes=str(decision.get('notes') or '').strip(), batch_id=batch_id, index=len(records) + 1, @@ -556,6 +563,7 @@ def _router_candidate_to_record( *, dataset_label: str, target: str, + complex_value: bool, notes: str, batch_id: str, index: int, @@ -604,6 +612,9 @@ def _router_candidate_to_record( 'target': target, 'target_type': _target_type(target), }, + 'dimensions': { + 'complex': complex_value, + }, 'meta': { 'case_name': '', 'notes': notes, @@ -642,6 +653,7 @@ def _skip_item(candidate: dict[str, Any], decision: str, notes: Any) -> dict[str def _target_type(target: str) -> str: + target = _strip_complex_prefix(target) if target.startswith('Agent('): return 'agent' if target: @@ -649,6 +661,34 @@ def _target_type(target: str) -> str: return 'unknown' +def _strip_complex_prefix(target: str) -> str: + stripped_target, _complex_value = _split_complex_prefixed_target(target) + return stripped_target + + +def _split_complex_prefixed_target(target: str) -> tuple[str, bool | None]: + lines = target.strip().splitlines() + if not lines: + return '', None + match = re.fullmatch(r'complex\s*=\s*(.+)', lines[0].strip(), flags=re.IGNORECASE) + if not match: + return target.strip(), None + return '\n'.join(lines[1:]).strip(), _parse_complex(match.group(1), default=False) + + +def _parse_complex(value: Any, *, default: bool) -> bool: + if value is None or value == '': + return default + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if text in {'true', '1', 'yes', 'y', '是', '复杂', 'complex'}: + return True + if text in {'false', '0', 'no', 'n', '否', '不复杂', '简单', 'simple'}: + return False + raise DataAgentRouterSessionError('complex must be a boolean value such as true/false') + + def _safe_token(value: str) -> str: token = re.sub(r'[^A-Za-z0-9_.-]+', '_', value.strip()) return token.strip('_.-') or 'item' diff --git a/tests/test_data_agent_records.py b/tests/test_data_agent_records.py index d9b9c2f..20c9df0 100644 --- a/tests/test_data_agent_records.py +++ b/tests/test_data_agent_records.py @@ -1,6 +1,7 @@ from __future__ import annotations import csv +import io import json import tempfile import unittest @@ -57,6 +58,7 @@ notes: 多轮承接附近生活服务查询 self.assertEqual(records[0]['turn']['query'], '帮我看看附近有什么好吃的') self.assertEqual(records[0]['prev_session'], []) self.assertEqual(records[0]['label']['target_type'], 'agent') + self.assertEqual(records[0]['dimensions'], {'complex': False}) self.assertEqual(records[1]['turn']['query'], '看看附近有什么好吃的') self.assertEqual( records[1]['prev_session'], @@ -85,6 +87,23 @@ target: Agent(tag='地图导航') self.assertEqual(record['label']['target'], 'Agent(tag="地图导航")') self.assertEqual(record['label']['target_type'], 'agent') + def test_normalize_dataset_draft_keeps_complex_dimension_independent(self) -> None: + payload = normalize_dataset_draft( + ''' +# dataset_label: 地图导航复杂样例 +### case: 复杂导航 +用户: 帮我规划一条先去加油站再去公司的路线 +complex: true +target: Agent(tag='地图导航') +'''.strip(), + batch_id='demo', + base_timestamp=1_755_567_930_500, + ) + + record = payload['records'][0] + self.assertEqual(record['label']['target'], 'Agent(tag="地图导航")') + self.assertEqual(record['dimensions']['complex'], True) + def test_validate_dataset_records_reports_valid_payload(self) -> None: records = normalize_dataset_draft( ''' @@ -138,7 +157,7 @@ target: Agent(tag="life_service") output_path = Path(tmp_dir) / payload['output_path'] lines = output_path.read_text(encoding='utf-8').splitlines() table_path = Path(tmp_dir) / payload['table_output_path'] - table_rows = list(csv.DictReader(table_path.read_text(encoding='utf-8-sig').splitlines())) + table_rows = list(csv.DictReader(io.StringIO(table_path.read_text(encoding='utf-8-sig')))) self.assertEqual(payload['output_format'], 'jsonl') self.assertEqual(payload['table_output_format'], 'csv') @@ -157,7 +176,7 @@ target: Agent(tag="life_service") self.assertEqual(table_rows[0]['context'], '{}') self.assertEqual(table_rows[0]['label'], '地图和生活边界数据') self.assertEqual(table_rows[0]['是否迁移Function'], '') - self.assertEqual(table_rows[0]['function'], 'Agent(tag="life_service")') + self.assertEqual(table_rows[0]['function'], 'complex=false\nAgent(tag="life_service")') def test_render_dataset_records_table_csv_keeps_prev_session_json(self) -> None: records = normalize_dataset_draft( @@ -174,13 +193,13 @@ target: Agent(tag='地图导航') timestamp_step_ms=60_000, )['records'] - rows = list(csv.DictReader(render_dataset_records_table_csv(records).splitlines())) + rows = list(csv.DictReader(io.StringIO(render_dataset_records_table_csv(records)))) prev_session = json.loads(rows[0]['prev_session']) self.assertEqual(prev_session[0]['query'], '查一下附近停车场') self.assertEqual(prev_session[0]['tts'], '找到了附近停车场') self.assertEqual(prev_session[0]['timestamp'], '1755567870500') - self.assertEqual(rows[0]['function'], 'Agent(tag="地图导航")') + self.assertEqual(rows[0]['function'], 'complex=false\nAgent(tag="地图导航")') def test_export_training_jsonl_uses_prompt_template_and_history(self) -> None: records = normalize_dataset_draft( @@ -208,7 +227,7 @@ target: Agent(tag='地图导航') self.assertEqual(payload['output_format'], 'jsonl') self.assertEqual(line['system'], '你是小爱同学，中文智能语音助手。') - self.assertEqual(line['output'], 'Agent(tag="地图导航")') + self.assertEqual(line['output'], 'complex=false\nAgent(tag="地图导航")') self.assertIn('[知识注入]\n{\n"location": "北京",\n"rag": "地图服务可用"\n}', line['instruction']) self.assertIn('用户: 查一下附近停车场\n小爱: 找到了附近停车场', line['instruction']) self.assertIn('[当前query]\n用户: 帮我找个顺路的', line['instruction']) @@ -227,7 +246,7 @@ target: Agent(tag='地图导航') base_timestamp=1_755_567_930_500, )['records'] - rows = list(csv.DictReader(render_planning_eval_csv(records).splitlines())) + rows = list(csv.DictReader(io.StringIO(render_planning_eval_csv(records)))) self.assertEqual( list(rows[0].keys()), @@ -237,8 +256,11 @@ target: Agent(tag='地图导航') self.assertEqual(rows[0]['query'], '帮我找个顺路的') self.assertEqual(rows[0]['类别真实标签'], '地图导航') self.assertEqual(rows[0]['code标签'], 'Agent(tag="地图导航")') - self.assertEqual(rows[0]['complex'], 'false') + self.assertEqual(rows[0]['complex'], 'FALSE') + self.assertTrue(rows[0]['newPrompt'].startswith('<|im_start|>system\n你是小爱同学，中文智能语音助手。<|im_end|>')) + self.assertIn('<|im_start|>user\n请参考用户的[当前query]', rows[0]['newPrompt']) self.assertIn('[function]', rows[0]['newPrompt']) + self.assertTrue(rows[0]['newPrompt'].endswith('<|im_start|>assistant\n')) def test_export_planning_eval_csv_writes_file(self) -> None: records = normalize_dataset_draft( @@ -246,6 +268,7 @@ target: Agent(tag='地图导航') # dataset_label: 时间工具数据 ### case: 几点 用户: 现在几点 +complex: true target: CalendarQA(type="TIME") '''.strip(), batch_id='demo', @@ -256,15 +279,14 @@ target: CalendarQA(type="TIME") records, root=tmp_dir, output_path='output/eval_planning.csv', - complex_default=True, ) output_path = Path(tmp_dir) / payload['output_path'] - rows = list(csv.DictReader(output_path.read_text(encoding='utf-8-sig').splitlines())) + rows = list(csv.DictReader(io.StringIO(output_path.read_text(encoding='utf-8-sig')))) self.assertEqual(payload['output_format'], 'csv') self.assertEqual(rows[0]['类别真实标签'], 'CalendarQA') self.assertEqual(rows[0]['code标签'], 'CalendarQA(type="TIME")') - self.assertEqual(rows[0]['complex'], 'true') + self.assertEqual(rows[0]['complex'], 'TRUE') def test_normalize_online_draft_uses_real_request_metadata(self) -> None: records = normalize_dataset_draft( diff --git a/tests/test_data_agent_router_sessions.py b/tests/test_data_agent_router_sessions.py index 655af90..28724b2 100644 --- a/tests/test_data_agent_router_sessions.py +++ b/tests/test_data_agent_router_sessions.py @@ -57,6 +57,7 @@ class DataAgentRouterSessionTests(unittest.TestCase): [candidate], dataset_label='总结类边界评测集', default_target='Summarize', + default_complex=True, batch_id='summary', ) @@ -68,6 +69,7 @@ class DataAgentRouterSessionTests(unittest.TestCase): self.assertEqual(record['prev_session'], [{'query': '这篇文章讲了什么', 'tts': '', 'timestamp': 10}]) self.assertEqual(record['context']['domain'], 'QA') self.assertEqual(record['label']['target'], 'Summarize') + self.assertEqual(record['dimensions'], {'complex': True}) def test_convert_router_candidates_to_records_applies_review_decisions(self) -> None: candidates = [ @@ -84,6 +86,7 @@ class DataAgentRouterSessionTests(unittest.TestCase): 'matched_turn_index': 0, 'decision': 'include', 'target': 'Summarize', + 'complex': True, 'notes': '前文有可总结内容', }, { @@ -98,6 +101,7 @@ class DataAgentRouterSessionTests(unittest.TestCase): self.assertEqual(result['record_count'], 1) self.assertEqual(result['skipped_count'], 1) self.assertEqual(result['records'][0]['meta']['notes'], '前文有可总结内容') + self.assertEqual(result['records'][0]['dimensions']['complex'], True) @unittest.skipUnless(HAS_PYARROW, 'pyarrow is required for parquet tests') def test_profile_and_search_router_sessions_read_parquet(self) -> None: