Skip to content
This repository was archived by the owner on Feb 2, 2024. It is now read-only.

Merge refactored DataFrame to master #870

Merged
densmirn merged 26 commits into from
May 30, 2020
Merged

Merge refactored DataFrame to master #870

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
4a45fa8
Turn on Azure CI for branch (#822)
akharche Apr 29, 2020
efb0c64
Redesign DataFrame structure (#817)
akharche Apr 30, 2020
682f5fc
Merge master (#840)
akharche May 15, 2020
c06c349
Re-implement df.getitem based on new structure (#845)
densmirn May 19, 2020
a5ee489
Re-implement df.values based on new structure (#846)
densmirn May 20, 2020
f5dcbc1
Re-implement df.pct_change based on new structure (#847)
densmirn May 20, 2020
645b284
Re-implement df.drop based on new structure (#848)
densmirn May 20, 2020
cb33e1e
Re-implement df.append based on new structure (#857)
densmirn May 22, 2020
ebb66e8
Re-implement df.reset_index based on new structure (#849)
densmirn May 22, 2020
401313c
Re-implement df._set_column based on new strcture (#850)
densmirn May 22, 2020
85f6e15
Re-implement df.rolling methods based on new structure (#852)
densmirn May 22, 2020
02d56d0
Re-implement df.index based on new structure (#853)
densmirn May 22, 2020
1f00705
Re-implement df.copy based on new structure (#854)
densmirn May 22, 2020
7bd330a
Re-implement df.isna based on new structure (#856)
densmirn May 22, 2020
7c8c8a3
Re-implement df.at/iat/loc/iloc based on new structure (#858)
densmirn May 22, 2020
693d893
Re-implement df.head based on new structure (#855)
densmirn May 25, 2020
3998b13
Re-implement df.groupby methods based on new structure (#859)
densmirn May 26, 2020
b5221f0
Re-implement dataframe boxing based on new structure (#861)
densmirn May 28, 2020
d780d38
Re-implement DataFrame unboxing (#860)
akharche May 29, 2020
d55933c
Fix rolling (#865)
akharche May 29, 2020
987521f
Undecorate tests on reading CSV (#866)
densmirn May 29, 2020
9b7c990
Re-implement df structure: enable rolling tests that pass (#867)
akharche May 29, 2020
46c1212
Re-implement df structure: refactor len (#868)
akharche May 29, 2020
8249bbc
Merge master to feature/dataframe_model_refactoring (#869)
densmirn May 29, 2020
73e66ad
Merge branch 'master' of https://github.com/IntelPython/sdc into merg…
densmirn May 30, 2020
b694a07
Enable CI on master
densmirn May 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Re-implement df.drop based on new structure (#848)
  • Loading branch information
@densmirn
densmirn authored May 20, 2020
commit 645b2845e1b23d222d33545d25513cc16ac14a7a
27 changes: 10 additions & 17 deletions sdc/datatypes/hpat_pandas_dataframe_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1308,22 +1308,14 @@ def isna_overload(df):

def sdc_pandas_dataframe_drop_codegen(func_name, func_args, df, drop_cols):
"""
Input:
df.drop(columns='M', errors='ignore')

Func generated:
def sdc_pandas_dataframe_drop_impl(df, labels=None, axis=0, index=None, columns=None, level=None, inplace=False,
errors="raise"):
if errors == "raise":
raise ValueError("The label M is not found in the selected axis")
new_col_A_data_df = df._data[0]
new_col_B_data_df = df._data[1]
new_col_C_data_df = df._data[2]
return pandas.DataFrame({"A": new_col_A_data_df, "B": new_col_B_data_df, "C": new_col_C_data_df})

Example of generated implementation:
def sdc_pandas_dataframe_drop_impl(df, labels=None, axis=0, index=None, columns=None,
level=None, inplace=False, errors="raise"):
new_col_0_data_df = df._data[1][0]
new_col_1_data_df = df._data[0][1]
return pandas.DataFrame({"B": new_col_0_data_df, "C": new_col_1_data_df}, index=df.index)
"""
indent = 4 * ' '
df_columns_indx = {col_name: i for i, col_name in enumerate(df.columns)}
saved_df_columns = [column for column in df.columns if column not in drop_cols]
func_definition = [f'def sdc_pandas_dataframe_{func_name}_impl({", ".join(func_args)}):']
func_text = []
Expand All @@ -1336,8 +1328,9 @@ def sdc_pandas_dataframe_drop_impl(df, labels=None, axis=0, index=None, columns=
break

for column_id, column_name in enumerate(saved_df_columns):
func_text.append(f'new_col_{column_id}_data_{"df"} = get_dataframe_data({"df"}, '
f'{df_columns_indx[column_name]})')
col_loc = df.column_loc[column_name]
type_id, col_id = col_loc.type_id, col_loc.col_id
func_text.append(f'new_col_{column_id}_data_df = df._data[{type_id}][{col_id}]')
column_list.append((f'new_col_{column_id}_data_df', column_name))

data = ', '.join(f'"{column_name}": {column}' for column, column_name in column_list)
Expand All @@ -1346,7 +1339,7 @@ def sdc_pandas_dataframe_drop_impl(df, labels=None, axis=0, index=None, columns=
func_definition.extend([indent + func_line for func_line in func_text])
func_def = '\n'.join(func_definition)

global_vars = {'pandas': pandas, 'get_dataframe_data': sdc.hiframes.pd_dataframe_ext.get_dataframe_data}
global_vars = {'pandas': pandas}

return func_def, global_vars

Expand Down
4 changes: 2 additions & 2 deletions sdc/tests/test_dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1616,7 +1616,7 @@ def test_impl(df):
index=index)
pd.testing.assert_frame_equal(sdc_func(df), test_impl(df))

@dfRefactoringNotImplemented
@dfRefactoringNotImplemented # required re-implementing DataFrame boxing
def test_df_drop_one_column(self):
def test_impl(index):
df = pd.DataFrame({'A': [1.0, 2.0, np.nan, 1.0], 'B': [4, 5, 6, 7], 'C': [1.0, 2.0, np.nan, 1.0]},
Expand Down Expand Up @@ -1658,7 +1658,7 @@ def test_impl(df):
index=index)
pd.testing.assert_frame_equal(sdc_func(df), test_impl(df))

@dfRefactoringNotImplemented
@dfRefactoringNotImplemented # required re-implementing DataFrame boxing
def test_df_drop_tuple_column(self):
def gen_test_impl(do_jit=False):
def test_impl(index):
Expand Down