Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

automatic generation of type checks for overload #911

Open
vlad-perevezentsev wants to merge 12 commits into
base: numba_typing
Choose a base branch
from
Open

automatic generation of type checks for overload #911

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8bd7bc0
add automatic generation of type checks for overload_list
vlad-perevezentsev Aug 17, 2020
d3f4a5d
add exception handling to the match function and correct handling of …
vlad-perevezentsev Aug 17, 2020
b54da17
add unittests
vlad-perevezentsev Sep 4, 2020
05c745b
add default value processing and tests
vlad-perevezentsev Sep 8, 2020
b7446ca
fix comments
vlad-perevezentsev Sep 9, 2020
09289bf
fix comments and add new tests
vlad-perevezentsev Sep 14, 2020
1cb60da
remove full_match and add a signature matching check
vlad-perevezentsev Sep 15, 2020
967ae29
add a test cases generator function
vlad-perevezentsev Sep 22, 2020
716402c
fix comments and change the old tests
vlad-perevezentsev Sep 28, 2020
2096e94
fix PEP
vlad-perevezentsev Sep 28, 2020
5ec33ac
fix PEP
vlad-perevezentsev Sep 28, 2020
7da564b
add additional tests
vlad-perevezentsev Sep 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix comments and add new tests
  • Loading branch information
@vlad-perevezentsev
vlad-perevezentsev committed Sep 14, 2020
commit 09289bfb1acd0d5d25d525596c1b4ab866f1707d
100 changes: 50 additions & 50 deletions numba_typing/overload_list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
from numba.extending import overload
from type_annotations import product_annotations, get_func_annotations
import typing
from numba import NumbaDeprecationWarning, NumbaPendingDeprecationWarning
import warnings
from numba.typed import List, Dict
from inspect import getfullargspec

Expand All @@ -26,7 +24,7 @@ def wrapper(*args):
result = choose_func_by_sig(sig_list, values_dict, defaults_dict)

if result is None:
raise numba.TypingError(f'Unsupported types a={a}, b={b}')
raise TypeError(f'Unsupported types a={a}, b={b}')

@densmirn densmirn Sep 23, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks both a and b are undefined.


return result

Expand All @@ -52,28 +50,27 @@ def check_str_type(n_type):


def check_list_type(self, p_type, n_type):
res = isinstance(n_type, types.List) or isinstance(n_type, types.ListType)
if isinstance(p_type, type):
res = isinstance(n_type, (types.List, types.ListType))
if p_type == list:
return res
else:
return res and self.match(p_type.__args__[0], n_type.dtype)


def check_tuple_type(self, p_type, n_type):
res = False
if isinstance(n_type, types.Tuple):
res = True
if isinstance(p_type, type):
return res
for p_val, n_val in zip(p_type.__args__, n_type.key):
res = res and self.match(p_val, n_val)
if isinstance(n_type, types.UniTuple):
res = True
if isinstance(p_type, type):
return res
for p_val in p_type.__args__:
res = res and self.match(p_val, n_type.key[0])
return res
if not isinstance(n_type, (types.Tuple, types.UniTuple)):
return False
try:
if len(p_type.__args__) != len(n_type.types):
return False
except AttributeError: # if p_type == tuple
return True

for p_val, n_val in zip(p_type.__args__, n_type.types):
if not self.match(p_val, n_val):
return False

return True


def check_dict_type(self, p_type, n_type):
Expand All @@ -89,9 +86,7 @@ def check_dict_type(self, p_type, n_type):

class TypeChecker:

_types_dict = {int: check_int_type, float: check_float_type, bool: check_bool_type,
str: check_str_type, list: check_list_type,
tuple: check_tuple_type, dict: check_dict_type}
_types_dict: dict = {}

def __init__(self):
self._typevars_dict = {}
Expand All @@ -118,60 +113,65 @@ def _get_origin(p_obj):
return p_obj.__origin__

def match(self, p_type, n_type):
if p_type == typing.Any:
return True
try:
if p_type == typing.Any:
return True
elif self._is_generic(p_type):
if self._is_generic(p_type):
origin_type = self._get_origin(p_type)
if origin_type == typing.Generic:
return self.match_generic(p_type, n_type)
else:
return self._types_dict[origin_type](self, p_type, n_type)
elif isinstance(p_type, typing.TypeVar):

return self._types_dict[origin_type](self, p_type, n_type)

if isinstance(p_type, typing.TypeVar):
return self.match_typevar(p_type, n_type)
else:
if p_type in (list, tuple):
return self._types_dict[p_type](self, p_type, n_type)
return self._types_dict[p_type](n_type)

if p_type in (list, tuple, dict):
return self._types_dict[p_type](self, p_type, n_type)

return self._types_dict[p_type](n_type)

except KeyError:
print((f'A check for the {p_type} was not found.'))
return None
raise TypeError(f'A check for the {p_type} was not found.')

def match_typevar(self, p_type, n_type):
if not self._typevars_dict.get(p_type) and n_type not in self._typevars_dict.values():
if isinstance(n_type, types.List):
n_type = types.ListType(n_type.dtype)
if not self._typevars_dict.get(p_type):
self._typevars_dict[p_type] = n_type
return True
return self._typevars_dict.get(p_type) == n_type

@AlexanderKalistratov AlexanderKalistratov Sep 9, 2020

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe it should be self.match. E.g. list and 'types.List' are synonyms but will fail equality check ('list != types.List').
And I'm assuming you don't have such tests?


def match_generic(self, p_type, n_type):
res = True
for arg in p_type.__args__:
res = res and self.match(arg, n_type)
return res
raise SystemError


TypeChecker.add_type_check(int, check_int_type)
TypeChecker.add_type_check(float, check_float_type)
TypeChecker.add_type_check(str, check_str_type)
TypeChecker.add_type_check(bool, check_bool_type)
TypeChecker.add_type_check(list, check_list_type)
TypeChecker.add_type_check(tuple, check_tuple_type)
TypeChecker.add_type_check(dict, check_dict_type)


def choose_func_by_sig(sig_list, values_dict, defaults_dict):

@AlexanderKalistratov AlexanderKalistratov Sep 15, 2020

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's do it this way:

def choose_func_by_sig(sig_list, values_dict, defaults_dict):
    def check_signature(sig_params, types_dict):
        checker = TypeChecker()
        for name, typ in types_dict.items():  # name,type = 'a',int64
            if isinstance(typ, types.Literal):
                typ = typ.literal_type

            if not checker.match(sig_params[name], typ):
                return False
            
            return True

    for sig, func in sig_list:  # sig = (Signature,func)
        for param in sig.parameters:  # param = {'a':int,'b':int}
            if check_signature(param, values_dict):
                return func

    return None

checker = TypeChecker()
for sig, func in sig_list: # sig = (Signature,func)
for param in sig.parameters: # param = {'a':int,'b':int}
full_match = True
for name, typ in values_dict.items(): # name,type = 'a',int64
if isinstance(typ, types.Literal):
typ = typ.literal_type

full_match = full_match and checker.match(
param[name], typ.literal_type)

if sig.defaults.get(name, False):
full_match = full_match and sig.defaults[name] == typ.literal_value
else:
full_match = full_match and checker.match(param[name], typ)
full_match = checker.match(param[name], typ)

if not full_match:

@AlexanderKalistratov AlexanderKalistratov Sep 9, 2020

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we really need this full_match variable?
Can't we simply break as soon as checker.match returns false?

break

for name, val in defaults_dict.items():
if not sig.defaults.get(name) is None:
full_match = full_match and sig.defaults[name] == val
if len(param) != len(values_dict.items()):
for name, val in defaults_dict.items():
if not sig.defaults.get(name) is None:
full_match = full_match and sig.defaults[name] == val

checker.clear_typevars_dict()
if full_match:
Expand Down
81 changes: 45 additions & 36 deletions numba_typing/test_overload_list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from overload_list import types
import unittest
import typing
from numba import NumbaDeprecationWarning, NumbaPendingDeprecationWarning, njit
from numba import NumbaDeprecationWarning, NumbaPendingDeprecationWarning, njit, core
import warnings


Expand Down Expand Up @@ -39,21 +39,15 @@ def foo_tuple(a: typing.Tuple[int, float], b: tuple = (0, 0)):
def foo_dict(a: typing.Dict[str, int], b: typing.Dict[int, bool] = {0: False}):
return ('dict', a, b)

# def foo_any(a: typing.Any, b: typing.Any = None):
# return('any', a, b)

def foo_union(a: typing.Union[int, str], b: typing.Union[float, bool] = None):
return('union', a, b)

# def foo_optional(a: typing.Optional[float], b: typing.Optional[str] = None):
# return('optional', a, b)

def foo_list_in_list(a: typing.List[typing.List[int]],
b: typing.List[typing.List[typing.List[float]]] = [[[0.0, 0.0]]]):
return('list_in_list', a, b)

def foo_tuple_in_tuple(a: typing.Tuple[typing.Tuple[int]],
b: typing.Tuple[typing.Tuple[typing.Tuple[float]]] = ((0.0, 0.0))):
def foo_tuple_in_tuple(a: typing.Tuple[typing.Tuple[int, int]],
b: typing.Tuple[typing.Tuple[typing.Tuple[float, float]]] = ((0.0, 0.0))):
return('tuple_in_tuple', a, b)

def foo_typevars_T_T(a: T, b: T):
Expand Down Expand Up @@ -84,6 +78,24 @@ def jit_func(a, b):
class TestOverloadListDefault(unittest.TestCase):
maxDiff = None

def test_myfunc_literal_type_default(self):
def foo(a, b=0):
...

@overload_list.overload_list(foo)
def foo_ovld_list():

def foo_int_literal(a: int, b: int = 0):
return ('literal', a, b)

return (foo_int_literal,)

@njit
def jit_func(a):
return foo(a, 2)

self.assertEqual(jit_func(1), ('literal', 1, 2))

def test_myfunc_int_type_default(self):
def foo(a, b=0):
...
Expand Down Expand Up @@ -156,26 +168,6 @@ def jit_func(a):

self.assertEqual(jit_func('qwe'), ('str', 'qwe', '0'))

# def test_myfunc_list_type_default(self):
# warnings.simplefilter('ignore', category=NumbaDeprecationWarning)
# warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)
# L = List([0, 0])
# def foo(a,b=L):
# ...
# @overload_list.overload_list(foo)
# def foo_ovld_list():

# def foo_list(a: typing.List[int], b: typing.List[int] = [0,0]):
# return ('list', a, b)

# return (foo_list,)

# @njit
# def jit_func(a):
# return foo(a)

# self.assertEqual(jit_func([1,2]), ('list',[1,2],L))

def test_myfunc_tuple_type_default(self):
def foo(a, b=(0, 0)):
...
Expand All @@ -194,6 +186,24 @@ def jit_func(a):

self.assertEqual(jit_func((1, 2)), ('tuple', (1, 2), (0, 0)))

def test_myfunc_tuple_type_error(self):
def foo(a, b=(0, 0)):
...

@overload_list.overload_list(foo)
def foo_ovld_list():

def foo_tuple(a: typing.Tuple[int, int], b: tuple = (0, 0)):
return ('tuple_', a, b)

return (foo_tuple,)

@njit
def jit_func(a, b):
return foo(a, b)

self.assertRaises(core.errors.TypingError, jit_func, (1, 2, 3), ('3', False))


class TestOverloadList(unittest.TestCase):
maxDiff = None
Expand All @@ -214,7 +224,6 @@ def test_myfunc_list_type(self):
self.assertEqual(jit_func([1, 2], [3, 4]), ('list', [1, 2], [3, 4]))

def test_myfunc_List_typed(self):

warnings.simplefilter('ignore', category=NumbaDeprecationWarning)
warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)
L = List([1, 2, 3])
Expand All @@ -232,9 +241,6 @@ def test_myfunc_dict_type(self):
D_1[0] = False
self.assertEqual(jit_func(D, D_1), ('dict', D, D_1))

# def test_myfunc_any_typing(self):
# self.assertEqual(jit_func((1,2.0),['qaz','qwe']), ('any',(1,2.0),['qaz','qwe']))

def test_myfunc_union_typing_int_bool(self):
self.assertEqual(jit_func(1, False), ('union', 1, False))

Expand All @@ -247,9 +253,6 @@ def test_myfunc_union_typing_int_float(self):
def test_myfunc_union_typing_str_float(self):
self.assertEqual(jit_func('qwe', 2.0), ('union', 'qwe', 2.0))

# def test_myfunc_optional_typing(self):
# self.assertEqual(jit_func(1.0, 'qwe'), ('optional', 1.0, 'qwe'))

def test_myfunc_list_in_list_type(self):
L_int = List([List([1, 2])])
L_float = List([List([List([3.0, 4.0])])])
Expand All @@ -262,6 +265,12 @@ def test_myfunc_tuple_in_tuple(self):
def test_myfunc_typevar_T_T(self):
self.assertEqual(jit_func(((1, 2),), ((3, 4),)), ('TypeVars_TT', ((1, 2),), ((3, 4),)))

def test_myfunc_typevar_T_T_list(self):
warnings.simplefilter('ignore', category=NumbaDeprecationWarning)
warnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)
V = List([1.0, 2.0])
self.assertEqual(jit_func(V, [3.0, 4.0]), ('TypeVars_TT', V, [3.0, 4.0]))

def test_myfunc_typevar_T_K(self):
self.assertEqual(jit_func(1.0, 2), ('TypeVars_TK', 1.0, 2))

Expand Down