Skip to content

Commit cfbbe06

Browse files
committed
FIX: MultiIndex+str column with Pandas 3.0 (fixes #1166)
1 parent 9d70c35 commit cfbbe06

3 files changed

Lines changed: 59 additions & 1 deletion

File tree

‎doc/source/changes/version_0_35_1.rst.inc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,7 @@ Miscellaneous improvements
7979
Fixes
8080
^^^^^
8181

82-
* fixed something (closes :issue:`1`).
82+
* fixed loading data files with all the following conditions when using
83+
Pandas >= 3.0: mixed type columns, some combination of labels in index
84+
columns not being present in the data and a non-NaN fill_value
85+
(closes :issue:`1166`).

‎larray/inout/pandas.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,29 @@ def cartesian_product_df(df, sort_rows=False, sort_columns=False,
8888
np.array_equal(idx.values, prod_index.values) and
8989
np.array_equal(columns.values, prod_columns.values)):
9090
return df, combined_labels
91+
import numbers
92+
if (isinstance(fill_value, numbers.Number) and not np.isnan(fill_value) and
93+
any(dt.kind == 'O' and dt.type is str for dt in df.dtypes)):
94+
df = df.copy()
95+
for col in df.columns:
96+
dt = df[col].dtype
97+
if dt.kind == 'O' and dt.type is str:
98+
# TODO: we should really output this warning, but the user
99+
# needs a way to silence it, which requires we implement
100+
# in all user-facing functions calling this function
101+
# directly or indirectly (from_series, from_frame,
102+
# read_excel, read_csv, ...) a way to specify
103+
# both a usecols argument (when the data has mixed type
104+
# but the user only needs an homogeneously typed subset)
105+
# and a dtype=object argument when the user does need
106+
# mixed types (and want to silence the warning).
107+
# warnings.warn("fill_value is not valid for all "
108+
# "columns because it is a (non-NaN) number but "
109+
# f"the '{col}' column has string dtype. That "
110+
# "column will converted to object dtype to avoid "
111+
# "errors but this may cause performance issues.",
112+
# FutureWarning, stacklevel=3)
113+
df[col] = df[col].astype(object)
91114
return df.reindex(index=prod_index, columns=prod_columns,
92115
fill_value=fill_value, **kwargs), combined_labels
93116

‎larray/tests/test_array.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4656,6 +4656,38 @@ def test_from_frame():
46564656
res = from_frame(df, sort_rows=True, sort_columns=True)
46574657
assert_larray_equal(res, expected)
46584658

4659+
# h) with a MultiIndex with some missing combination and a fill_value which
4660+
# is not valid for all columns. This pattern broke for Pandas >= 3
4661+
# See issue #1166. In the user code, this issue came from, from_frame
4662+
# is used indirectly by read_excel and users are keeping only the
4663+
# numeric column (col2 in the test below) just after reading the array.
4664+
df = pd.DataFrame({
4665+
'idx1': [0, 0, 1],
4666+
'idx2': ['a', 'b', 'a'],
4667+
'col1': ['a', 'a', 'a'],
4668+
'col2': [1, 1, 1]
4669+
}).set_index(['idx1', 'idx2'])
4670+
# TODO: we should have this warning, but before that we need more
4671+
# functionality to let users do something about it.
4672+
# See comment in inout/pandas.py:cartesian_product_df
4673+
# msg = ("fill_value is not valid for all columns because it is a "
4674+
# "(non-NaN) number but the 'col1' column has string dtype. That "
4675+
# "column will converted to object dtype to avoid errors but this "
4676+
# "may cause performance issues.")
4677+
# with must_warn(FutureWarning, msg=msg):
4678+
# from_frame(df, fill_value=-1)
4679+
res = from_frame(df, fill_value=-1)
4680+
expected_data = [[['a', 1],
4681+
['a', 1]],
4682+
[['a', 1],
4683+
[-1, -1]]]
4684+
expected_axes = [
4685+
# third axis is anonymous
4686+
Axis('idx1=0,1'), Axis('idx2=a,b'), Axis('col1,col2')
4687+
]
4688+
expected = Array(expected_data, expected_axes, dtype=object)
4689+
assert_larray_equal(res, expected)
4690+
46594691

46604692
def test_asarray():
46614693
series = pd.Series([0, 1, 2], ['a0', 'a1', 'a2'], name='a')

0 commit comments

Comments
 (0)