Skip to content

Make external compiler discovery lazy in the test suite. #8661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sklam merged 3 commits into from
Jan 11, 2023
Merged

Make external compiler discovery lazy in the test suite. #8661

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9b0f931
Make external compiler discovery lazy in the test suite.
stuartarchibald Dec 9, 2022
7dd2c39
Fix functools.lru_cache invocation on Python 3.7.
stuartarchibald Dec 9, 2022
8e43a67
Remove extra blank line.
stuartarchibald Jan 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions numba/pycc/platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,13 @@ def _gentmpfile(suffix):
else:
os.rmdir(tmpdir)

def _check_external_compiler():
# see if the external compiler bound in numpy.distutil is present
# and working

@functools.lru_cache(maxsize=1)
def external_compiler_works():
"""
Returns True if the "external compiler" bound in numpy.distutil is present
and working, False otherwise.
"""
compiler = new_compiler()
customize_compiler(compiler)
for suffix in ['.c', '.cxx']:
Expand All @@ -63,10 +67,6 @@ def _check_external_compiler():
return False
return True

# boolean on whether the externally provided compiler is present and
# functioning correctly
_external_compiler_ok = _check_external_compiler()


class _DummyExtension(object):
libraries = []
Expand All @@ -75,7 +75,7 @@ class _DummyExtension(object):
class Toolchain(object):

def __init__(self):
if not _external_compiler_ok:
if not external_compiler_works():
self._raise_external_compiler_error()

# Need to import it here since setuptools may monkeypatch it
Expand Down
18 changes: 11 additions & 7 deletions numba/tests/support.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
import unittest
from numba.core.runtime import rtsys
from numba.np import numpy_support
from numba.pycc.platform import _external_compiler_ok
from numba.core.runtime import _nrt_python as _nrt
from numba.pycc.platform import external_compiler_works


try:
Expand Down Expand Up @@ -157,12 +157,6 @@
needs_subprocess = unittest.skipUnless(_exec_cond, "needs subprocess harness")


# decorate for test needs external compilers
needs_external_compilers = unittest.skipIf(not _external_compiler_ok,
('Compatible external compilers are '
'missing'))


def ignore_internal_warnings():
"""Use in testing within a ` warnings.catch_warnings` block to filter out
warnings that are unrelated/internally generated by Numba.
Expand Down Expand Up @@ -617,6 +611,16 @@ def inner(self, *args, **kwargs):
else:
return wrapper

def skip_if_no_external_compiler(self):
"""
Call this to ensure the test is skipped if no suitable external compiler
is found. This is a method on the TestCase opposed to a stand-alone
decorator so as to make it "lazy" via runtime evaluation opposed to
running at test-discovery time.
"""
if not external_compiler_works():
self.skipTest("No suitable external compiler was found.")


class SerialMixin(object):
"""Mixin to mark test for serial execution.
Expand Down
11 changes: 6 additions & 5 deletions numba/tests/test_parallel_backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@

from numba import jit, vectorize, guvectorize, set_num_threads
from numba.tests.support import (temp_directory, override_config, TestCase, tag,
skip_parfors_unsupported, linux_only,
needs_external_compilers)
skip_parfors_unsupported, linux_only)

import queue as t_queue
from numba.testing.main import _TIMEOUT as _RUNNER_TIMEOUT
Expand Down Expand Up @@ -1064,21 +1063,23 @@ def forker():
print("OUT:", out)
print("ERR:", err)

@needs_external_compilers
@linux_only # fork required.
def test_lifetime_of_task_scheduler_handle(self):

self.skip_if_no_external_compiler() # external compiler needed

# See PR #7280 for context.
BROKEN_COMPILERS = 'SKIP: COMPILATION FAILED'
runme = """if 1:
import ctypes
import sys
import multiprocessing as mp
from tempfile import TemporaryDirectory, NamedTemporaryFile
from numba.pycc.platform import Toolchain, _external_compiler_ok
from numba.pycc.platform import Toolchain, external_compiler_works
from numba import njit, prange, threading_layer
import faulthandler
faulthandler.enable()
if not _external_compiler_ok:
if not external_compiler_works():
raise AssertionError('External compilers are not found.')
with TemporaryDirectory() as tmpdir:
with NamedTemporaryFile(dir=tmpdir) as tmpfile:
Expand Down
17 changes: 10 additions & 7 deletions numba/tests/test_pycc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
from numba.core import utils
from numba.pycc.decorators import clear_export_registry
from numba.pycc.platform import find_shared_ending, find_pyext_ending
from numba.pycc.platform import _external_compiler_ok
from numba.pycc.platform import external_compiler_works

from numba.tests.support import (TestCase, tag, import_dynamic, temp_directory,
has_blas, needs_external_compilers)
has_blas)
import unittest


Expand Down Expand Up @@ -48,12 +48,12 @@ class TestCompilerChecks(TestCase):
@_windows_only
def test_windows_compiler_validity(self):
# When inside conda-build VSINSTALLDIR should be set and windows should
# have a valid compiler available, `_external_compiler_ok` should agree
# with this. If this is not the case then error out to alert devs.
# have a valid compiler available, `external_compiler_works()` should
# agree with this. If this is not the case then error out to alert devs.
is_running_conda_build = os.environ.get('CONDA_BUILD', None) is not None
if is_running_conda_build:
if os.environ.get('VSINSTALLDIR', None) is not None:
self.assertTrue(_external_compiler_ok)
self.assertTrue(external_compiler_works())


class BasePYCCTest(TestCase):
Expand Down Expand Up @@ -84,11 +84,11 @@ def check_c_ext(self, extdir, name):
sys.modules.pop(name, None)


@needs_external_compilers
class TestCC(BasePYCCTest):

def setUp(self):
super(TestCC, self).setUp()
self.skip_if_no_external_compiler() # external compiler needed
from numba.tests import compile_with_pycc
self._test_module = compile_with_pycc
imp.reload(self._test_module)
Expand Down Expand Up @@ -259,10 +259,13 @@ def test_c_extension_usecase(self):
self.assertPreciseEqual(got, expect)


@needs_external_compilers
class TestDistutilsSupport(TestCase):


@sklam sklam Jan 9, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

empty line

@stuartarchibald stuartarchibald Jan 10, 2023

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, fixed in 8e43a67.

def setUp(self):
super().setUp()
self.skip_if_no_external_compiler() # external compiler needed

@gmarkall gmarkall Jan 3, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just checked whether it's OK to call skipTest() in a setUp() method, and it is: https://docs.python.org/3/library/unittest.html#skipping-tests-and-expected-failures


unset_macosx_deployment_target()

# Copy the test project into a temp directory to avoid
Expand Down