Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Commit 7d5723c

Browse files
1e-to1e-to
authored
Df.isna impl (#636)
DataFrame.isna impl
1 parent 90b40aa commit 7d5723c

3 files changed

Lines changed: 164 additions & 0 deletions

File tree

‎examples/dataframe/dataframe_isna.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# *****************************************************************************
2+
# Copyright (c) 2020, Intel Corporation All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# Redistributions of source code must retain the above copyright notice,
8+
# this list of conditions and the following disclaimer.
9+
#
10+
# Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
18+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21+
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22+
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23+
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24+
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25+
# *****************************************************************************
26+
27+
"""
28+
Expected result:
29+
A B C
30+
0 False False True
31+
1 True False False
32+
2 False False False
33+
3 False False False
34+
"""
35+
36+
import pandas as pd
37+
import numpy as np
38+
from numba import njit
39+
40+
41+
@njit
42+
def dataframe_isna():
43+
df = pd.DataFrame({'A': [1.0, np.nan, 3.0, 1.0], 'B': [4, 5, 6, 7], 'C': [None, 'b', 'c', 'd']})
44+
45+
return df.isna()
46+
47+
48+
print(dataframe_isna())

‎sdc/datatypes/hpat_pandas_dataframe_functions.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1029,6 +1029,98 @@ def sdc_pandas_dataframe_drop_impl(df, labels=None, axis=0, index=None, columns=
10291029
return func_def, global_vars
10301030

10311031

1032+
def _dataframe_codegen_isna(func_name, columns, df):
1033+
"""
1034+
Example func_text for func_name='isna' columns=('float', 'int', 'string'):
1035+
1036+
def _df_isna_impl(df):
1037+
series_float = pandas.Series(get_dataframe_data(df, 0))
1038+
result_float = series_float.isna()
1039+
series_int = pandas.Series(get_dataframe_data(df, 1))
1040+
result_int = series_int.isna()
1041+
series_string = pandas.Series(get_dataframe_data(df, 2))
1042+
result_string = series_string.isna()
1043+
return pandas.DataFrame({"float": result_float, "int": result_int, "string": result_string},
1044+
index = df._index)
1045+
"""
1046+
results = []
1047+
func_lines = [f'def _df_{func_name}_impl(df):']
1048+
index = df_index_codegen_all(df)
1049+
for i, c in enumerate(columns):
1050+
result_c = f'result_{c}'
1051+
func_lines += [f' series_{c} = pandas.Series(get_dataframe_data(df, {i}))',
1052+
f' {result_c} = series_{c}.{func_name}()']
1053+
results.append((columns[i], result_c))
1054+
1055+
data = ', '.join(f'"{col}": {data}' for col, data in results)
1056+
func_lines += [f' return pandas.DataFrame({{{data}}}, {index})']
1057+
func_text = '\n'.join(func_lines)
1058+
global_vars = {'pandas': pandas,
1059+
'get_dataframe_data': get_dataframe_data}
1060+
1061+
return func_text, global_vars
1062+
1063+
1064+
def sdc_pandas_dataframe_isna_codegen(df, func_name):
1065+
df_func_name = f'_df_{func_name}_impl'
1066+
func_text, global_vars = _dataframe_codegen_isna(func_name, df.columns, df)
1067+
loc_vars = {}
1068+
exec(func_text, global_vars, loc_vars)
1069+
_reduce_impl = loc_vars[df_func_name]
1070+
1071+
return _reduce_impl
1072+
1073+
1074+
def df_index_codegen_all(self):
1075+
if isinstance(self.index, types.NoneType):
1076+
return ''
1077+
return 'index=df._index'
1078+
1079+
1080+
@sdc_overload_method(DataFrameType, 'isna')
1081+
def isna_overload(df):
1082+
"""
1083+
Intel Scalable Dataframe Compiler User Guide
1084+
********************************************
1085+
1086+
Pandas API: pandas.DataFrame.isna
1087+
1088+
Examples
1089+
--------
1090+
.. literalinclude:: ../../../examples/dataframe/dataframe_isna.py
1091+
:language: python
1092+
:lines: 35-
1093+
:caption: Detect missing values.
1094+
:name: ex_dataframe_isna
1095+
1096+
.. command-output:: python ./dataframe/dataframe_isna.py
1097+
:cwd: ../../../examples
1098+
1099+
.. seealso::
1100+
1101+
:ref:`DataFrame.isnull <pandas.DataFrame.isnull>`
1102+
Alias of isna.
1103+
1104+
:ref:`DataFrame.notna <pandas.DataFrame.notna>`
1105+
Boolean inverse of isna.
1106+
1107+
:ref:`DataFrame.dropna <pandas.DataFrame.dropna>`
1108+
Omit axes labels with missing values.
1109+
1110+
`pandas.absolute <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.isna.html#pandas.isna>`_
1111+
Top-level isna.
1112+
1113+
Intel Scalable Dataframe Compiler Developer Guide
1114+
*************************************************
1115+
Pandas DataFrame method :meth:`pandas.DataFrame.isna` implementation.
1116+
1117+
.. only:: developer
1118+
Test: python -m sdc.runtests -k sdc.tests.test_dataframe.TestDataFrame.test_df_isna*
1119+
"""
1120+
1121+
return sdc_pandas_dataframe_isna_codegen(df, 'isna')
1122+
1123+
10321124
@sdc_overload_method(DataFrameType, 'drop')
10331125
def sdc_pandas_dataframe_drop(df, labels=None, axis=0, index=None, columns=None, level=None, inplace=False,
10341126
errors='raise'):

‎sdc/tests/test_dataframe.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -810,6 +810,30 @@ def test_impl(df):
810810
'D': [None, 'dd', '', None]})
811811
pd.testing.assert_frame_equal(hpat_func(df), test_impl(df))
812812

813+
def test_df_isna(self):
814+
def test_impl(df):
815+
return df.isna()
816+
817+
sdc_func = sdc.jit(test_impl)
818+
indexes = [[3, 4, 2, 6, 1], ['a', 'b', 'c', 'd', 'e'], None]
819+
820+
for idx in indexes:
821+
df = pd.DataFrame({"A": [3.2, np.nan, 7.0, 3.3, np.nan],
822+
"B": [3, 4, 1, 0, 222],
823+
"C": [True, True, False, False, True],
824+
"D": ['a', 'dd', 'c', '12', None]}, index=idx)
825+
with self.subTest(index=idx):
826+
pd.testing.assert_frame_equal(sdc_func(df), test_impl(df))
827+
828+
@unittest.skip('DF with column named "bool" Segmentation fault')
829+
def test_df_bool(self):
830+
def test_impl(df):
831+
return df.isna()
832+
833+
sdc_func = sdc.jit(test_impl)
834+
df = pd.DataFrame({"bool": [True, True, False, False, True]}, index=None)
835+
pd.testing.assert_frame_equal(sdc_func(df), test_impl(df))
836+
813837
@skip_numba_jit
814838
def test_df_astype_str1(self):
815839
'''Verifies DataFrame.astype implementation converting various types to string'''

0 commit comments

Comments
 (0)