wuyang/zk-data-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix product data eval export format

Browse Source
This commit is contained in:
wuyang6
2026-05-11 15:27:11 +08:00
parent be731caed7
commit f22de8a40b
15 changed files with 414 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent_tool_specs/data_agent.py
+20 -7
View File
@@ -346,6 +346,10 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
'type': 'string',
'description': 'Target label used for included candidates unless review_decisions provides a target.',
},
'default_complex': {
'type': 'boolean',
'description': 'Default complex dimension for included candidates unless review_decisions provides complex.',
},
'review_decisions': {
'type': 'array',
'description': 'Optional include/exclude/uncertain decisions keyed by semantic_session_id or req_id.',
@@ -357,6 +361,7 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
'matched_turn_index': {'type': 'integer'},
'decision': {'type': 'string', 'enum': ['include', 'exclude', 'uncertain']},
'target': {'type': 'string'},
'complex': {'type': 'boolean'},
'notes': {'type': 'string'},
},
},
@@ -428,7 +433,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
name='data_agent_normalize_dataset_draft',
description=(
'Parse dataset draft text written with 用户/小爱/target blocks and build canonical '
'data-agent records with generated record IDs, timestamps, source metadata, context, and labels. '
'data-agent records with generated record IDs, timestamps, source metadata, context, labels, '
'and dimensions.complex. '
'Generated data requires a confirmed_plan_id from data_agent_confirm_generation_plan. '
'Use draft_path instead of draft_text when the draft is large or contains many quoted targets.'
),
@@ -437,7 +443,7 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
'properties': {
'draft_text': {
'type': 'string',
'description': 'Dataset draft text in dataset draft text v1 format. Provide exactly one of draft_text or draft_path.',
'description': 'Dataset draft text in dataset draft text v1 format. Each case should include complex: true/false. Provide exactly one of draft_text or draft_path.',
},
'draft_path': {
'type': 'string',
@@ -476,7 +482,7 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
name='data_agent_validate_dataset_records',
description=(
'Validate canonical data-agent records for required fields, labels, source metadata, '
'prev_session structure, timestamp order, and 5-minute prompt stitching constraints.'
'prev_session structure, dimensions.complex, timestamp order, and 5-minute prompt stitching constraints.'
),
parameters={
'type': 'object',
@@ -501,7 +507,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
description=(
'Validate canonical data-agent records and write them to a workspace file. '
'Defaults to compact JSONL, one canonical record per line, and also writes records.csv '
'in the same output directory for human sharing.'
'in the same output directory for human sharing. The table function column combines '
'complex=true/false and label.target.'
),
parameters={
'type': 'object',
@@ -551,7 +558,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
name='data_agent_export_training_jsonl',
description=(
'Convert canonical data-agent records into training JSONL. Each line has system, instruction, '
'and output. The instruction contains 知识注入, 系统状态, 对话历史, 当前query, and function sections.'
'and output. Output combines dimensions.complex and label.target as two lines. '
'The instruction contains 知识注入, 系统状态, 对话历史, 当前query, and function sections.'
),
parameters={
'type': 'object',
@@ -600,7 +608,8 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
name='data_agent_export_planning_eval_csv',
description=(
'Convert canonical data-agent records into evaluation CSV with columns: request_id, newPrompt, '
'query, 类别真实标签, code标签, complex. newPrompt uses the same prompt body as training instruction.'
'query, 类别真实标签, code标签, complex. The complex column is read from dimensions.complex. '
'newPrompt wraps the prompt body with <|im_start|>system/user/assistant chat-template tags.'
),
parameters={
'type': 'object',
@@ -635,9 +644,13 @@ def build_data_agent_tools(handlers: Mapping[str, ToolHandler]) -> list[AgentToo
'items': {'type': 'string'},
'description': 'Context fields to inject. Defaults to ["location", "rag"].',
},
'system_prompt': {
'type': 'string',
'description': 'System prompt wrapped into newPrompt chat template. Defaults to 你是小爱同学，中文智能语音助手。',
},
'complex_default': {
'type': 'boolean',
'description': 'Default complex column value. Defaults to false.',
'description': 'Fallback complex value only for legacy records missing dimensions.complex. Defaults to false.',
},
'require_validation_ok': {'type': 'boolean'},
'overwrite': {'type': 'boolean'},
src/agent_tools.py
+8
View File
@@ -1417,11 +1417,15 @@ def _data_agent_convert_router_candidates_to_records_tool(
include_uncertain = arguments.get('include_uncertain', False)
if not isinstance(include_uncertain, bool):
raise ToolExecutionError('include_uncertain must be a boolean')
default_complex = arguments.get('default_complex', False)
if not isinstance(default_complex, bool):
raise ToolExecutionError('default_complex must be a boolean')
try:
payload = convert_router_candidates_to_records(
arguments['candidates'],
dataset_label=_require_string(arguments, 'dataset_label'),
default_target=_optional_string(arguments, 'default_target'),
default_complex=default_complex,
review_decisions=review_decisions,
batch_id=_optional_string(arguments, 'batch_id') or 'router',
include_uncertain=include_uncertain,
@@ -1596,6 +1600,9 @@ def _export_planning_eval_csv_tool(arguments: dict[str, Any], context: ToolExecu
complex_default = arguments.get('complex_default', False)
if not isinstance(complex_default, bool):
raise ToolExecutionError('complex_default must be a boolean')
system_prompt = arguments.get('system_prompt', '你是小爱同学，中文智能语音助手。')
if not isinstance(system_prompt, str):
raise ToolExecutionError('system_prompt must be a string')
try:
records = _records_from_arguments(arguments, context)
payload = export_planning_eval_csv(
@@ -1609,6 +1616,7 @@ def _export_planning_eval_csv_tool(arguments: dict[str, Any], context: ToolExecu
session_num=_coerce_int(arguments, 'session_num', 5),
session_time_minutes=_coerce_int(arguments, 'session_time_minutes', 5),
context_fields=_optional_string_list(arguments, 'context_fields'),
system_prompt=system_prompt,
complex_default=complex_default,
require_validation_ok=require_validation_ok,
overwrite=overwrite,
src/data_agent_records.py
+131 -19
View File
@@ -53,6 +53,7 @@ def prepare_generation_goal(
if missing:
raise DataRecordError('missing required goal fields: ' + ', '.join(missing))
normalized_targets = _normalize_target_definitions(target_definitions)
target = _normalize_target_expression(target)
if not target.strip() and not normalized_targets:
raise DataRecordError('generation goal must include target or target_definitions')
state = _load_goal_state(root)
@@ -63,7 +64,7 @@ def prepare_generation_goal(
'status': 'pending_confirmation',
'dataset_label': dataset_label.strip(),
'goal_summary': goal_summary.strip(),
'target': target.strip(),
'target': target,
'target_definitions': normalized_targets,
'plan_hint': plan_hint.strip(),
'coverage': coverage.strip(),
@@ -191,13 +192,14 @@ def prepare_generation_plan(
'output_path': output_path,
}
)
target = _normalize_target_expression(target)
normalized_targets = _normalize_target_definitions(target_definitions)
if confirmed_goal is not None and not target.strip() and not normalized_targets:
normalized_targets = _normalize_target_definitions(
confirmed_goal.get('target_definitions') if isinstance(confirmed_goal, dict) else None
)
if not normalized_targets:
target = str(confirmed_goal.get('target') or '').strip()
target = _normalize_target_expression(str(confirmed_goal.get('target') or ''))
if not target.strip() and not normalized_targets:
missing.append('target')
if missing:
@@ -220,7 +222,7 @@ def prepare_generation_plan(
'plan_id': plan_id,
'status': 'pending_confirmation',
'dataset_label': dataset_label.strip(),
'target': target.strip(),
'target': target,
'target_definitions': normalized_targets,
'total_count': total_count,
'turn_mix': turn_mix.strip(),
@@ -448,6 +450,14 @@ def validate_dataset_records(records: list[dict[str, Any]]) -> dict[str, Any]:
if label.get('target_type') not in {'agent', 'function', 'unknown'}:
errors.append(_issue(f'{prefix}.label.target_type', 'label.target_type is invalid'))
dimensions = record.get('dimensions')
if not isinstance(dimensions, dict):
warnings.append(_issue(f'{prefix}.dimensions', 'dimensions.complex is missing; false will be used as fallback'))
elif 'complex' not in dimensions:
warnings.append(_issue(f'{prefix}.dimensions.complex', 'complex is missing; false will be used as fallback'))
elif not isinstance(dimensions.get('complex'), bool):
errors.append(_issue(f'{prefix}.dimensions.complex', 'complex must be a boolean'))
return {
'ok': not errors,
'error_count': len(errors),
@@ -579,6 +589,7 @@ def export_planning_eval_csv(
session_num: int = 5,
session_time_minutes: int = 5,
context_fields: list[str] | None = None,
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
complex_default: bool = False,
require_validation_ok: bool = True,
overwrite: bool = True,
@@ -604,6 +615,7 @@ def export_planning_eval_csv(
session_num=session_num,
session_time_minutes=session_time_minutes,
context_fields=fields,
system_prompt=system_prompt,
complex_default=complex_default,
)
path.write_text(content, encoding='utf-8-sig')
@@ -633,6 +645,7 @@ def render_planning_eval_csv(
session_num: int = 5,
session_time_minutes: int = 5,
context_fields: list[str] | None = None,
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
complex_default: bool = False,
) -> str:
"""把 canonical records 转成含 newPrompt 的评测 CSV。"""
@@ -648,21 +661,21 @@ def render_planning_eval_csv(
for record in records:
source = record.get('source') if isinstance(record.get('source'), dict) else {}
turn = record.get('turn') if isinstance(record.get('turn'), dict) else {}
label = record.get('label') if isinstance(record.get('label'), dict) else {}
target = str(label.get('target') or '')
target = _record_target(record)
writer.writerow(
{
'request_id': str(source.get('request_id') or ''),
'newPrompt': build_training_instruction(
'newPrompt': build_planning_prompt(
record,
session_num=session_num,
session_time_minutes=session_time_minutes,
context_fields=fields,
system_prompt=system_prompt,
),
'query': str(turn.get('query') or ''),
'类别真实标签': _category_label_from_target(target),
'code标签': target,
'complex': 'true' if complex_default else 'false',
'complex': _eval_complex_literal(_record_complex(record, default=complex_default)),
}
)
return output.getvalue()
@@ -709,6 +722,29 @@ def build_training_instruction(
return instruction
def build_planning_prompt(
record: dict[str, Any],
*,
session_num: int = 5,
session_time_minutes: int = 5,
context_fields: list[str] | None = None,
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
) -> str:
"""把训练 instruction 包成评测侧使用的 chat template。"""
instruction = build_training_instruction(
record,
session_num=session_num,
session_time_minutes=session_time_minutes,
context_fields=context_fields,
)
return (
f'<|im_start|>system\n{system_prompt}<|im_end|>\n'
f'<|im_start|>user\n{instruction}<|im_end|>\n'
'<|im_start|>assistant\n'
)
def _training_jsonl_line(
record: dict[str, Any],
*,
@@ -717,7 +753,6 @@ def _training_jsonl_line(
context_fields: list[str],
system_prompt: str,
) -> str:
label = record.get('label') if isinstance(record.get('label'), dict) else {}
payload = {
'system': system_prompt,
'instruction': build_training_instruction(
@@ -726,7 +761,7 @@ def _training_jsonl_line(
session_time_minutes=session_time_minutes,
context_fields=context_fields,
),
'output': str(label.get('target') or ''),
'output': _combined_function_label(record),
}
return json.dumps(payload, ensure_ascii=False, separators=(',', ':'))
@@ -797,6 +832,7 @@ def _optional_int_for_export(value: Any) -> int | None:
def _category_label_from_target(target: str) -> str:
target = _strip_complex_prefix(target)
agent_match = re.match(r'''^Agent\s*\(\s*tag\s*=\s*["']([^"']+)["']\s*\)$''', target.strip())
if agent_match:
return agent_match.group(1)
@@ -835,7 +871,7 @@ def _dataset_record_table_row(record: dict[str, Any]) -> dict[str, str]:
'context': json.dumps(context, ensure_ascii=False, separators=(',', ':')),
'label': str(label.get('dataset_label') or ''),
'是否迁移Function': '',
'function': str(label.get('target') or ''),
'function': _combined_function_label(record),
}
@@ -875,7 +911,7 @@ def _parse_draft_text(draft_text: str) -> dict[str, Any]:
if current is None:
continue
field_match = re.match(r'^(用户|小爱|target|notes|dataset_label|request_id|timestamp)\s*[:：]\s*(.*)$', line, re.IGNORECASE)
field_match = re.match(r'^(用户|小爱|target|complex|notes|dataset_label|request_id|timestamp)\s*[:：]\s*(.*)$', line, re.IGNORECASE)
if not field_match:
continue
key = field_match.group(1).lower()
@@ -925,7 +961,7 @@ def _normalize_target_definitions(value: list[dict[str, Any]] | None) -> list[di
if not isinstance(item, dict):
raise DataRecordError(f'target_definitions[{index}] must be an object')
name = str(item.get('name') or '').strip()
target = str(item.get('target') or '').strip()
target = _normalize_target_expression(str(item.get('target') or ''))
rule = str(item.get('rule') or '').strip()
if not target:
raise DataRecordError(f'target_definitions[{index}].target is required')
@@ -958,10 +994,10 @@ def _allowed_targets_for_plan(plan: dict[str, Any]) -> set[str]:
if isinstance(target_definitions, list):
for item in target_definitions:
if isinstance(item, dict) and isinstance(item.get('target'), str) and item['target'].strip():
allowed.add(item['target'].strip())
allowed.add(_normalize_target_expression(item['target']))
target = plan.get('target')
if isinstance(target, str) and target.strip():
allowed.add(target.strip())
allowed.add(_normalize_target_expression(target))
return allowed
@@ -980,7 +1016,7 @@ def _validate_plan_against_goal(
goal_targets = _allowed_targets_for_goal(goal)
plan_targets = set()
if target.strip():
plan_targets.add(target.strip())
plan_targets.add(_normalize_target_expression(target))
for item in target_definitions:
item_target = item.get('target', '').strip()
if item_target:
@@ -998,9 +1034,8 @@ def _validate_records_against_plan(records: list[dict[str, Any]], plan: dict[str
return
unexpected: list[str] = []
for index, record in enumerate(records):
label = record.get('label')
target = label.get('target') if isinstance(label, dict) else None
if isinstance(target, str) and target.strip() in allowed_targets:
target = _record_target(record)
if target and target in allowed_targets:
continue
unexpected.append(f'records[{index}].label.target={target!r}')
if unexpected:
@@ -1148,9 +1183,14 @@ def _case_to_record(
if not current_query:
raise DataRecordError(f'case {index} current 用户 line must be non-empty')
target = _canonical_target(str(case.get('target') or '').strip())
raw_target, prefixed_complex = _split_complex_prefixed_target(str(case.get('target') or '').strip())
target = _canonical_target(raw_target)
if not target:
raise DataRecordError(f'case {index} target is required')
complex_value = _parse_complex(
case.get('complex'),
default=False if prefixed_complex is None else prefixed_complex,
)
prev_session = _build_prev_session(turns[:final_user_index], case_index=index)
if len(prev_session) > 10:
@@ -1183,6 +1223,9 @@ def _case_to_record(
'target': target,
'target_type': _target_type(target),
},
'dimensions': {
'complex': complex_value,
},
'meta': {
'case_name': str(case.get('case_name') or '').strip(),
'notes': str(case.get('notes') or '').strip(),
@@ -1209,7 +1252,75 @@ def _build_prev_session(turns: list[dict[str, Any]], *, case_index: int) -> list
return prev_session
def _normalize_target_expression(target: str) -> str:
"""把可能带 complex 前缀的标签表达式收敛为纯 target。"""
return _canonical_target(_strip_complex_prefix(target))
def _strip_complex_prefix(target: str) -> str:
stripped_target, _complex_value = _split_complex_prefixed_target(target)
return stripped_target
def _split_complex_prefixed_target(target: str) -> tuple[str, bool | None]:
lines = target.strip().splitlines()
if not lines:
return '', None
first_line = lines[0].strip()
match = re.fullmatch(r'complex\s*=\s*(.+)', first_line, flags=re.IGNORECASE)
if not match:
return target.strip(), None
complex_value = _parse_complex(match.group(1), default=False)
return '\n'.join(lines[1:]).strip(), complex_value
def _parse_complex(value: Any, *, default: bool) -> bool:
if value is None or value == '':
return default
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if text in {'true', '1', 'yes', 'y', '是', '复杂', 'complex'}:
return True
if text in {'false', '0', 'no', 'n', '否', '不复杂', '简单', 'simple'}:
return False
raise DataRecordError('complex must be a boolean value such as true/false')
def _complex_literal(value: bool) -> str:
return 'true' if value else 'false'
def _eval_complex_literal(value: bool) -> str:
return 'TRUE' if value else 'FALSE'
def _record_target(record: dict[str, Any]) -> str:
label = record.get('label') if isinstance(record.get('label'), dict) else {}
return _normalize_target_expression(str(label.get('target') or ''))
def _record_complex(record: dict[str, Any], *, default: bool = False) -> bool:
dimensions = record.get('dimensions')
if isinstance(dimensions, dict) and 'complex' in dimensions:
return _parse_complex(dimensions.get('complex'), default=default)
label = record.get('label') if isinstance(record.get('label'), dict) else {}
if 'complex' in label:
return _parse_complex(label.get('complex'), default=default)
_target, prefixed_complex = _split_complex_prefixed_target(str(label.get('target') or ''))
if prefixed_complex is not None:
return prefixed_complex
return default
def _combined_function_label(record: dict[str, Any], *, default_complex: bool = False) -> str:
target = _record_target(record)
return f'complex={_complex_literal(_record_complex(record, default=default_complex))}\n{target}'
def _target_type(target: str) -> str:
target = _strip_complex_prefix(target)
if re.match(r'^Agent\s*\(\s*tag\s*=', target):
return 'agent'
if target:
@@ -1220,6 +1331,7 @@ def _target_type(target: str) -> str:
def _canonical_target(target: str) -> str:
"""规范常见 Agent 标签写法，允许草稿里用单引号降低 JSON 转义风险。"""
target = _strip_complex_prefix(target)
match = re.fullmatch(r'Agent\s*\(\s*tag\s*=\s*([\'"])(.+?)\1\s*\)', target)
if not match:
return target
src/data_agent_router_sessions.py
+41 -1
View File
@@ -197,6 +197,7 @@ def convert_router_candidates_to_records(
*,
dataset_label: str,
default_target: str = '',
default_complex: bool = False,
review_decisions: list[dict[str, Any]] | None = None,
batch_id: str = 'router',
include_uncertain: bool = False,
@@ -228,11 +229,17 @@ def convert_router_candidates_to_records(
raise DataAgentRouterSessionError(
f'target is required for candidate {candidate.get("semantic_session_id")}#{candidate.get("matched_turn_index")}'
)
raw_target, prefixed_complex = _split_complex_prefixed_target(target)
complex_value = _parse_complex(
decision.get('complex'),
default=default_complex if prefixed_complex is None else prefixed_complex,
)
records.append(
_router_candidate_to_record(
candidate,
dataset_label=str(decision.get('dataset_label') or dataset_label).strip(),
target=target,
target=raw_target,
complex_value=complex_value,
notes=str(decision.get('notes') or '').strip(),
batch_id=batch_id,
index=len(records) + 1,
@@ -556,6 +563,7 @@ def _router_candidate_to_record(
*,
dataset_label: str,
target: str,
complex_value: bool,
notes: str,
batch_id: str,
index: int,
@@ -604,6 +612,9 @@ def _router_candidate_to_record(
'target': target,
'target_type': _target_type(target),
},
'dimensions': {
'complex': complex_value,
},
'meta': {
'case_name': '',
'notes': notes,
@@ -642,6 +653,7 @@ def _skip_item(candidate: dict[str, Any], decision: str, notes: Any) -> dict[str
def _target_type(target: str) -> str:
target = _strip_complex_prefix(target)
if target.startswith('Agent('):
return 'agent'
if target:
@@ -649,6 +661,34 @@ def _target_type(target: str) -> str:
return 'unknown'
def _strip_complex_prefix(target: str) -> str:
stripped_target, _complex_value = _split_complex_prefixed_target(target)
return stripped_target
def _split_complex_prefixed_target(target: str) -> tuple[str, bool | None]:
lines = target.strip().splitlines()
if not lines:
return '', None
match = re.fullmatch(r'complex\s*=\s*(.+)', lines[0].strip(), flags=re.IGNORECASE)
if not match:
return target.strip(), None
return '\n'.join(lines[1:]).strip(), _parse_complex(match.group(1), default=False)
def _parse_complex(value: Any, *, default: bool) -> bool:
if value is None or value == '':
return default
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if text in {'true', '1', 'yes', 'y', '是', '复杂', 'complex'}:
return True
if text in {'false', '0', 'no', 'n', '否', '不复杂', '简单', 'simple'}:
return False
raise DataAgentRouterSessionError('complex must be a boolean value such as true/false')
def _safe_token(value: str) -> str:
token = re.sub(r'[^A-Za-z0-9_.-]+', '_', value.strip())
return token.strip('_.-') or 'item'