Remove unused imports and increase K values

terryyz · May 27, 2024 · May 25, 2024 · May 25, 2024 · May 25, 2024 · May 25, 2024
commit 2c33a263d7c39b4167e4307c9093248ebd03b7d5
diff --git a/wildcode/eval/utils.py b/wildcode/eval/utils.py
@@ -24,12 +24,10 @@
 import faulthandler
 import io
 import os
-import sys
 import platform
 import signal
 import tempfile
 import subprocess
-import threading
 import multiprocessing
 from typing import Optional
 

diff --git a/wildcode/evaluate.py b/wildcode/evaluate.py
@@ -213,7 +213,7 @@ def stucking_checker():
 
     pass_at_k = {
         f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean()
-        for k in [1, 5, 10]
+        for k in [1, 5, 10, 25, 100]
         if total.min() >= k
     }
     cprint(f"{flags.dataset}", "green")

diff --git a/wildcode/gen/util/__init__.py b/wildcode/gen/util/__init__.py
@@ -1,11 +1,7 @@
 import time
-import os
 import sys
 import types
 import unittest
-import builtins
-
-from copy import deepcopy
 
 from wildcode.eval.utils import (
     create_tempdir,
@@ -14,14 +10,16 @@
     time_limit,
     safe_environment,
 )
+
+
 def trusted_exec(code, test_code, task_id):
     """Execute trusted code in place."""
-    
+
     with create_tempdir():
         import os
         import shutil
         import builtins
-        
+
         rmtree = shutil.rmtree
         rmdir = os.rmdir
         chdir = os.chdir
@@ -44,7 +42,8 @@ def trusted_exec(code, test_code, task_id):
         full_code = code + "\n" + test_code
 
         # Compile and execute the combined code within the new module
-        exec(compile(full_code, f"{module_name}.py", 'exec'), new_module.__dict__)
+        exec(compile(full_code, f"{module_name}.py", 'exec'),
+             new_module.__dict__)
         sys.modules[module_name] = new_module
         TestCases = getattr(new_module, 'TestCases')
         loader = unittest.TestLoader()
@@ -59,8 +58,10 @@ def trusted_exec(code, test_code, task_id):
         shutil.rmtree = rmtree
         os.rmdir = rmdir
         os.chdir = chdir
-        assert len(test_result.failures+test_result.errors) == 0, f"{task_id} failed"
-
+        assert len(
+            test_result.failures + test_result.errors
+        ) == 0, f"{task_id} failed with errors: {test_result.errors} and failures: {test_result.failures}"
+
         return time.time() - start
 
 def trusted_check_exec(code, inputs):

diff --git a/wildcode/inspect.py b/wildcode/inspect.py
@@ -1,5 +1,4 @@
 from wildcode.data import get_wildcodebench
-import sys
 import os
 import shutil
 import json

diff --git a/wildcode/lecacy_sanitize.py b/wildcode/lecacy_sanitize.py
diff --git a/wildcode/sanitize.py b/wildcode/sanitize.py
@@ -1,7 +1,6 @@
 """Post-processing LLM-generated Python code implemented using tree-sitter."""
 
 import os
-import ast
 import pathlib
 from typing import Dict, Generator, List, Optional, Set, Tuple