Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit b8e49a9

Browse files
committed
allow call-analysis from external modules, removing daal4py features (now in daal4py)
1 parent 2f4dcad commit b8e49a9

5 files changed

Lines changed: 15 additions & 536 deletions

File tree

‎examples/d4p_kmeans.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import daal4py
2+
import daal4py.hpat
23
import hpat
34
import numpy as np
45

5-
daal4py.daalinit()
6-
76
@hpat.jit(nopython=True)
87
def kmeans(N, D, nClusters, maxit):
98
a = np.random.ranf((N,D)) # doesn't make much sense, but ok for now
@@ -15,5 +14,3 @@ def kmeans(N, D, nClusters, maxit):
1514
print(kmeans(10000, 20, 2, 30))
1615

1716
hpat.distribution_report()
18-
19-
daal4py.daalfini()

‎examples/d4p_linreg.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import daal4py
2+
import daal4py.hpat
23
import hpat
34
import numpy as np
45

5-
daal4py.daalinit()
6-
76
@hpat.jit
87
def lr_predict(N, D, model):
98
data = np.random.ranf((N/2,D))
@@ -21,5 +20,3 @@ def lr_train(N, D):
2120
print(p_res.prediction[0], t_res.model.NumberOfBetas)
2221

2322
hpat.distribution_report()
24-
25-
daal4py.daalfini()

‎hpat/distributed_analysis.py‎

Lines changed: 13 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -31,19 +31,21 @@ class Distribution(Enum):
3131
OneD_Var = 4
3232
OneD = 5
3333

34-
try:
35-
from hpat.ml.d4p import algos as d4p_algos
36-
except:
37-
d4p_algos = []
38-
3934
_dist_analysis_result = namedtuple(
4035
'dist_analysis_result', 'array_dists,parfor_dists')
4136

4237
distributed_analysis_extensions = {}
4338
auto_rebalance = False
4439

4540
class DistributedAnalysis(object):
46-
"""analyze program for to distributed transfromation"""
41+
"""Analyze program for distributed transformation"""
42+
43+
_extra_call = []
44+
45+
@classmethod
46+
def add_call_analysis(cls, analysis_func):
47+
'external modules/packages (like daal4py) can register their own call-analysis'
48+
cls._extra_call.append(analysis_func)
4749

4850
def __init__(self, func_ir, typemap, calltypes, typingctx, metadata):
4951
self.func_ir = func_ir
@@ -442,60 +444,20 @@ def _analyze_call(self, lhs, rhs, func_var, args, array_dists):
442444
self._meet_array_dists(lhs, args[0].name, array_dists)
443445
return
444446

445-
if isinstance(func_mod, ir.Var) and self._analyze_call_d4p(lhs, func_name, self.typemap[func_mod.name], args, array_dists):
446-
return
447-
448447
# TODO: make sure assert_equiv is not generated unnecessarily
449448
# TODO: fix assert_equiv for np.stack from df.value
450449
if fdef == ('assert_equiv', 'numba.array_analysis'):
451450
return
452451

452+
# we perform call-analysis from external at the end
453+
for af in DistributedAnalysis._extra_call:
454+
if af(lhs, func_mod, func_name, self.typemap, args, array_dists):
455+
return
456+
453457
# set REP if not found
454458
self._analyze_call_set_REP(lhs, args, array_dists, fdef)
455459

456460

457-
def _analyze_call_d4p(self, lhs, func_name, mod_name, args, array_dists):
458-
'''
459-
Analyze distribution for calls to daal4py.
460-
Return True of a call for daal4py was detected and handled.
461-
We cannot simply "meet" distributions, the d4p algos accept a certain decomposition only.
462-
The required distribution/decomposition is defined in the algorithms specs.
463-
We raise an exception if the required distribution cannot be met.
464-
'''
465-
if func_name == 'compute':
466-
# every d4p algo gets executed by invoking "compute".
467-
# we need to find the algorithm that's currently called
468-
for algo in d4p_algos:
469-
if algo.all_nbtypes[algo.name] == mod_name:
470-
# handle all input arguments and set their distribution as given by the spec
471-
for i in range(len(args)):
472-
aname = args[i].name
473-
adist = algo.spec.input_types[i][2]
474-
if aname not in array_dists:
475-
array_dists[aname] = adist
476-
else:
477-
min_adist = Distribution.OneD_Var if adist == Distribution.OneD else adist
478-
assert array_dists[aname].value <= Distribution.OneD.value, "Cannot handle unknown distribution type"
479-
# bail out if there is a distribution conflict with some other use of the argument
480-
# FIXME: handle Distribution.Thread and Disribution.REP as equivalent
481-
assert array_dists[aname].value >= min_adist.value,\
482-
'Distribution of argument {} ({}) to "daal4py.{}.compute" must be "{}". '\
483-
'Some other use of it demands "{}", though.'\
484-
.format(i+1, algo.spec.input_types[i][0], algo.name, adist, array_dists[aname])
485-
# handle distribution of the result
486-
if lhs not in array_dists:
487-
array_dists[lhs] = algo.spec.result_dist
488-
else:
489-
array_dists[lhs] = Distribution(min(array_dists[lhs].value, algo.spec.result_dist.value))
490-
min_rdist = Distribution.OneD_Var if algo.spec.result_dist == Distribution.OneD else algo.spec.result_dist
491-
assert array_dists[lhs].value >= min_rdist.value,\
492-
'Distribution ({}) to "daal4py.{}.compute" must be at least "{}". '\
493-
'Some other use of it demands "{}", though.'\
494-
.format(algo.name, min_rdist, array_dists[lhs])
495-
return True
496-
return False
497-
498-
499461
def _analyze_call_np(self, lhs, func_name, args, array_dists):
500462
"""analyze distributions of numpy functions (np.func_name)
501463
"""

‎hpat/ml/__init__.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
11
from .svc import SVC
22
from .naive_bayes import MultinomialNB
33
from hpat.utils import debug_prints
4-
5-
try:
6-
from . import d4p
7-
except ImportError:
8-
if debug_prints(): # pragma: no cover
9-
print("daal4py import error")

0 commit comments

Comments
 (0)