diff --git a/src/evaluation_runtime.py b/src/evaluation_runtime.py index 20b1489..330edec 100644 --- a/src/evaluation_runtime.py +++ b/src/evaluation_runtime.py @@ -37,7 +37,16 @@ DEFAULT_CONCURRENCY = 8 MAX_CONCURRENCY = 32 MAX_DATASET_ROWS = 20_000 MAX_LABEL_MAPPING_VALUES = 200 -TRANSIENT_MODEL_RETRY_DELAYS_SECONDS = (5.0, 15.0, 30.0, 60.0) +TRANSIENT_MODEL_RETRY_DELAYS_SECONDS = ( + 5.0, + 15.0, + 30.0, + 60.0, + 60.0, + 60.0, + 60.0, + 60.0, +) VISIBLE_SKILL_FILE_SUFFIXES = {'.json', '.md', '.py', '.txt', '.yaml', '.yml'} EDITABLE_SKILL_FILE_SUFFIXES = {'.json', '.md'} MAX_SKILL_FILE_BYTES = 512 * 1024 @@ -1006,7 +1015,9 @@ class EvaluationRuntime: params: list[Any] = [now, experiment_id] where = ["experiment_id = ?"] if case_ids: - where.append("status IN ('failed', 'cancelled', 'completed')") + where.append( + "status IN ('failed', 'cancelled', 'completed', 'running')" + ) placeholders = ','.join('?' for _ in case_ids) where.append(f'id IN ({placeholders})') params.extend(case_ids) @@ -1037,7 +1048,7 @@ class EvaluationRuntime: else: where.append('1 = 0') else: - where.append("status IN ('failed', 'cancelled')") + where.append("status IN ('failed', 'cancelled', 'running')") self.store.execute( f""" UPDATE evaluation_cases @@ -2918,6 +2929,19 @@ def transient_model_error_message(value: Any) -> str: f'模型服务暂时不可用({status.upper()}),' '已自动退避重试,请稍后重试失败项' ) + transport_markers = ( + 'incompleteread', + 'remote end closed connection', + 'remotedisconnected', + 'connection reset by peer', + 'connectionreseterror', + 'broken pipe', + ) + if any(marker in normalized for marker in transport_markers): + return ( + '模型响应流意外中断,已自动退避重试;' + '若多次恢复失败,请稍后重试失败项' + ) return '' diff --git a/tests/test_evaluation_runtime.py b/tests/test_evaluation_runtime.py index 30cfe99..8f3e86d 100644 --- a/tests/test_evaluation_runtime.py +++ b/tests/test_evaluation_runtime.py @@ -388,6 +388,10 @@ class EvaluationRuntimeTests(unittest.TestCase): 'HTTP 429 from local model backend: Too many requests' ), ) + self.assertIn( + '响应流意外中断', + transient_model_error_message('IncompleteRead(199 bytes read)'), + ) def test_transient_model_failure_retries_before_validation(self) -> None: with tempfile.TemporaryDirectory() as directory: