feat: add editable skill evaluation versions
This commit is contained in:
@@ -178,6 +178,62 @@ class EvaluationApiTests(unittest.TestCase):
|
||||
self.assertIn('text/csv', export.headers['content-type'])
|
||||
self.assertIn('导航去公司', export.content.decode('utf-8-sig'))
|
||||
|
||||
def test_skill_version_endpoints(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
state = _build_state(root)
|
||||
with TestClient(create_app(state)) as client:
|
||||
versions_response = client.get(
|
||||
'/api/evaluations/skills/label-master/versions',
|
||||
params={'account_id': 'alice'},
|
||||
)
|
||||
self.assertEqual(versions_response.status_code, 200)
|
||||
versions = versions_response.json()
|
||||
current_id = versions['current_snapshot_id']
|
||||
|
||||
manifest = client.get(
|
||||
(
|
||||
'/api/evaluations/skills/label-master/versions/'
|
||||
f'{current_id}'
|
||||
),
|
||||
params={'account_id': 'alice'},
|
||||
)
|
||||
self.assertEqual(manifest.status_code, 200)
|
||||
self.assertTrue(
|
||||
any(
|
||||
item['path'] == 'SKILL.md'
|
||||
for item in manifest.json()['files']
|
||||
)
|
||||
)
|
||||
|
||||
file_response = client.get(
|
||||
(
|
||||
'/api/evaluations/skills/label-master/versions/'
|
||||
f'{current_id}/file'
|
||||
),
|
||||
params={'account_id': 'alice', 'path': 'SKILL.md'},
|
||||
)
|
||||
self.assertEqual(file_response.status_code, 200)
|
||||
content = file_response.json()['content'].replace(
|
||||
'Use the label catalog.',
|
||||
'Use the label catalog carefully.',
|
||||
)
|
||||
saved_response = client.post(
|
||||
'/api/evaluations/skills/label-master/versions',
|
||||
json={
|
||||
'account_id': 'alice',
|
||||
'base_snapshot_id': current_id,
|
||||
'version_name': '测试版本',
|
||||
'note': 'API 测试',
|
||||
'files': {'SKILL.md': content},
|
||||
},
|
||||
)
|
||||
self.assertEqual(saved_response.status_code, 200)
|
||||
self.assertEqual(
|
||||
saved_response.json()['version_name'],
|
||||
'测试版本',
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -169,6 +169,100 @@ class EvaluationRuntimeTests(unittest.TestCase):
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_skill_versions_are_immutable_and_account_scoped(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
_write_test_skill(root)
|
||||
runtime = EvaluationRuntime(
|
||||
root=root / 'evaluations',
|
||||
cwd_for_account=lambda _account: root,
|
||||
model_config_for=lambda _account: ModelConfig(model='test-model'),
|
||||
account_paths_for=lambda _account: {'python_env': root / '.venv'},
|
||||
max_workers=1,
|
||||
)
|
||||
try:
|
||||
versions = runtime.list_skill_versions('alice', 'label-master')
|
||||
self.assertEqual(len(versions['versions']), 1)
|
||||
current_id = versions['current_snapshot_id']
|
||||
self.assertTrue(versions['versions'][0]['is_current'])
|
||||
|
||||
manifest = runtime.get_skill_version_manifest(
|
||||
'alice',
|
||||
'label-master',
|
||||
current_id,
|
||||
)
|
||||
skill_file = next(
|
||||
item
|
||||
for item in manifest['files']
|
||||
if item['path'] == 'SKILL.md'
|
||||
)
|
||||
self.assertTrue(skill_file['editable'])
|
||||
current_file = runtime.read_skill_version_file(
|
||||
'alice',
|
||||
'label-master',
|
||||
current_id,
|
||||
'SKILL.md',
|
||||
)
|
||||
updated_content = current_file['content'].replace(
|
||||
'Read the manifest',
|
||||
'Read the manifest carefully',
|
||||
)
|
||||
saved = runtime.save_skill_version(
|
||||
'alice',
|
||||
'label-master',
|
||||
base_snapshot_id=current_id,
|
||||
version_name='边界调整 v1',
|
||||
note='测试版本',
|
||||
files={'SKILL.md': updated_content},
|
||||
)
|
||||
self.assertEqual(saved['version_name'], '边界调整 v1')
|
||||
self.assertEqual(saved['source_type'], 'local')
|
||||
self.assertNotEqual(saved['id'], current_id)
|
||||
|
||||
original = runtime.read_skill_version_file(
|
||||
'alice',
|
||||
'label-master',
|
||||
current_id,
|
||||
'SKILL.md',
|
||||
)
|
||||
forked = runtime.read_skill_version_file(
|
||||
'alice',
|
||||
'label-master',
|
||||
saved['id'],
|
||||
'SKILL.md',
|
||||
)
|
||||
self.assertNotIn('carefully', original['content'])
|
||||
self.assertIn('carefully', forked['content'])
|
||||
dataset = runtime.create_dataset(
|
||||
account_id='alice',
|
||||
name='versioned',
|
||||
filename='versioned.csv',
|
||||
rows=[{'query': '导航去公司'}],
|
||||
)
|
||||
experiment = runtime.create_experiment(
|
||||
account_id='alice',
|
||||
dataset_id=dataset['id'],
|
||||
name='local version',
|
||||
snapshot_id=saved['id'],
|
||||
)
|
||||
self.assertEqual(experiment['snapshot_id'], saved['id'])
|
||||
self.assertEqual(
|
||||
experiment['skill_version'],
|
||||
saved['content_hash'][:12],
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
'Skill 版本不存在',
|
||||
):
|
||||
runtime.read_skill_version_file(
|
||||
'bob',
|
||||
'label-master',
|
||||
saved['id'],
|
||||
'SKILL.md',
|
||||
)
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
|
||||
def test_result_normalization_and_label_equivalence(self) -> None:
|
||||
result = normalize_evaluation_output(
|
||||
json.dumps(
|
||||
|
||||
Reference in New Issue
Block a user