fix: recover interrupted evaluation cases

src/evaluation_runtime.py

@@ -37,7 +37,16 @@ DEFAULT_CONCURRENCY = 8
 MAX_CONCURRENCY = 32
 MAX_DATASET_ROWS = 20_000
 MAX_LABEL_MAPPING_VALUES = 200
-TRANSIENT_MODEL_RETRY_DELAYS_SECONDS = (5.0, 15.0, 30.0, 60.0)
+TRANSIENT_MODEL_RETRY_DELAYS_SECONDS = (
+    5.0,
+    15.0,
+    30.0,
+    60.0,
+    60.0,
+    60.0,
+    60.0,
+    60.0,
+)
 VISIBLE_SKILL_FILE_SUFFIXES = {'.json', '.md', '.py', '.txt', '.yaml', '.yml'}
 EDITABLE_SKILL_FILE_SUFFIXES = {'.json', '.md'}
 MAX_SKILL_FILE_BYTES = 512 * 1024
@@ -1006,7 +1015,9 @@ class EvaluationRuntime:
         params: list[Any] = [now, experiment_id]
         where = ["experiment_id = ?"]
         if case_ids:
-            where.append("status IN ('failed', 'cancelled', 'completed')")
+            where.append(
+                "status IN ('failed', 'cancelled', 'completed', 'running')"
+            )
             placeholders = ','.join('?' for _ in case_ids)
             where.append(f'id IN ({placeholders})')
             params.extend(case_ids)
@@ -1037,7 +1048,7 @@ class EvaluationRuntime:
             else:
                 where.append('1 = 0')
         else:
-            where.append("status IN ('failed', 'cancelled')")
+            where.append("status IN ('failed', 'cancelled', 'running')")
         self.store.execute(
             f"""
             UPDATE evaluation_cases
@@ -2918,6 +2929,19 @@ def transient_model_error_message(value: Any) -> str:
                 f'模型服务暂时不可用（{status.upper()}），'
                 '已自动退避重试，请稍后重试失败项'
             )
+    transport_markers = (
+        'incompleteread',
+        'remote end closed connection',
+        'remotedisconnected',
+        'connection reset by peer',
+        'connectionreseterror',
+        'broken pipe',
+    )
+    if any(marker in normalized for marker in transport_markers):
+        return (
+            '模型响应流意外中断，已自动退避重试；'
+            '若多次恢复失败，请稍后重试失败项'
+        )
     return ''
 
 

tests/test_evaluation_runtime.py

@@ -388,6 +388,10 @@ class EvaluationRuntimeTests(unittest.TestCase):
                 'HTTP 429 from local model backend: Too many requests'
             ),
         )
+        self.assertIn(
+            '响应流意外中断',
+            transient_model_error_message('IncompleteRead(199 bytes read)'),
+        )
 
     def test_transient_model_failure_retries_before_validation(self) -> None:
         with tempfile.TemporaryDirectory() as directory: