diff --git a/benchmarks/suites/bfcl.py b/benchmarks/suites/bfcl.py index b65d791..9df321f 100644 --- a/benchmarks/suites/bfcl.py +++ b/benchmarks/suites/bfcl.py @@ -320,6 +320,7 @@ class BFCLBenchmark(BenchmarkSuite): problem_id=pid, passed=False, error="solution.py not found" ) + output = "" test_harness = os.path.join(workspace, "test_harness.py") if os.path.exists(test_harness): code, output = self._run_shell( @@ -335,6 +336,6 @@ class BFCLBenchmark(BenchmarkSuite): return BenchmarkResult( problem_id=pid, passed=passed, - actual=output[:500] if "output" in dir() else "", - error="" if passed else (output[:500] if "output" in dir() else "function call not found"), + actual=output[:500], + error="" if passed else (output[:500] if output else "function call not found"), ) diff --git a/benchmarks/suites/mbpp.py b/benchmarks/suites/mbpp.py index 5eeecc6..755fd2b 100644 --- a/benchmarks/suites/mbpp.py +++ b/benchmarks/suites/mbpp.py @@ -90,7 +90,7 @@ _BUILTIN_PROBLEMS: list[dict[str, Any]] = [ "code": 'import re\ndef find_char_long(text):\n return (re.findall(r"\\b\\w{4,}\\b", text))', "test_list": [ 'assert find_char_long(\'Please move back to stream\') == [\'Please\', \'move\', \'back\', \'stream\']', - 'assert find_char_long(\'Joveacvber gfnbb vcdf\') == [\'Jove\', \'acvber\', \'gfnbb\', \'vcdf\']' + "", + 'assert find_char_long(\'Joveacvber gfnbb vcdf\') == [\'Jove\', \'acvber\', \'gfnbb\', \'vcdf\']', 'assert find_char_long(\'Davvede aridge\') == [\'Davvede\', \'aridge\']', ], },