Skip to content

Commit 49d02ff

Browse files
alixdammanalixdamman
committed
fix larray-project#788 : convert unicode string to bytes and reversely when dumping/loading Axis and Group objects to/from HDF files (to avoid huge size hdf files)
1 parent 03b6fbf commit 49d02ff

5 files changed

Lines changed: 28 additions & 4 deletions

File tree

‎larray/core/axis.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1349,10 +1349,13 @@ def to_hdf(self, filepath, key=None):
13491349
raise ValueError("Argument key must be provided explicitly in case of anonymous axis")
13501350
key = self.name
13511351
key = _translate_group_key_hdf(key)
1352-
s = pd.Series(data=self.labels, name=self.name)
1352+
kind = self.labels.dtype.kind
1353+
data = np.char.encode(self.labels, 'utf-8') if kind == 'U' else self.labels
1354+
s = pd.Series(data=data, name=self.name)
13531355
with LHDFStore(filepath) as store:
13541356
store.put(key, s)
13551357
store.get_storer(key).attrs.type = 'Axis'
1358+
store.get_storer(key).attrs.kind = kind
13561359
store.get_storer(key).attrs.wildcard = self.iswildcard
13571360

13581361
@property

‎larray/core/group.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1462,10 +1462,15 @@ def to_hdf(self, filepath, key=None, axis_key=None):
14621462
if self.axis.name is None:
14631463
raise ValueError("Argument axis_key must be provided explicitly if the associated axis is anonymous")
14641464
axis_key = self.axis.name
1465-
s = pd.Series(data=self.eval(), name=self.name)
1465+
data = self.eval()
1466+
kind = data.dtype.kind if isinstance(data, np.ndarray) else ''
1467+
if kind == 'U':
1468+
data = np.char.encode(data, 'utf-8')
1469+
s = pd.Series(data=data, name=self.name)
14661470
with LHDFStore(filepath) as store:
14671471
store.put(key, s)
14681472
store.get_storer(key).attrs.type = 'Group'
1473+
store.get_storer(key).attrs.kind = kind
14691474
if axis_key not in store:
14701475
self.axis.to_hdf(store, key=axis_key)
14711476
store.get_storer(key).attrs.axis_key = axis_key

‎larray/inout/hdf.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,15 +88,21 @@ def read_hdf(filepath_or_buffer, key, fill_value=nan, na=nan, sort_rows=False, s
8888
name = str(pd_obj.name)
8989
if name == 'None':
9090
name = None
91-
res = Axis(labels=pd_obj.values, name=name)
91+
labels = pd_obj.values
92+
if 'kind' in attrs and attrs['kind'] == 'U':
93+
labels = np.char.decode(labels, 'utf-8')
94+
res = Axis(labels=labels, name=name)
9295
res._iswildcard = attrs['wildcard']
9396
elif _type == 'Group':
9497
if name is None:
9598
name = str(pd_obj.name)
9699
if name == 'None':
97100
name = None
101+
key = pd_obj.values
102+
if 'kind' in attrs and attrs['kind'] == 'U':
103+
key = np.char.decode(key, 'utf-8')
98104
axis = read_hdf(filepath_or_buffer, attrs['axis_key'])
99-
res = LGroup(key=pd_obj.values, name=name, axis=axis)
105+
res = LGroup(key=key, name=name, axis=axis)
100106
return res
101107

102108

‎larray/tests/test_axis.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,7 @@ def test_h5_io(tmpdir):
391391
lipro = Axis('lipro=P01..P05')
392392
anonymous = Axis(range(3))
393393
wildcard = Axis(3, 'wildcard')
394+
string_axis = Axis(['@!àéè&%µ$~', '/*-+_§()><', 'another label'], 'string_axis')
394395
fpath = os.path.join(str(tmpdir), 'axes.h5')
395396

396397
# ---- default behavior ----
@@ -410,6 +411,10 @@ def test_h5_io(tmpdir):
410411
wildcard2 = read_hdf(fpath, key=wildcard.name)
411412
assert wildcard2.iswildcard
412413
assert wildcard.equals(wildcard2)
414+
# string axis
415+
string_axis.to_hdf(fpath)
416+
string_axis2 = read_hdf(fpath, string_axis.name)
417+
assert string_axis.equals(string_axis2)
413418

414419
# ---- specific key ----
415420
# int axis

‎larray/tests/test_group.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ def test_h5_io_lgroup(tmpdir):
192192
named_axis_not_in_file = lipro['P01,P03,P05'] >> 'P_odd'
193193
anonymous = age[':5']
194194
wildcard = age_wildcard[':5'] >> 'age_w_05'
195+
string_group = Axis(['@!àéè&%µ$~', '/*-+_§()><', 'another label'], 'string_axis')[:] >> 'string_group'
195196

196197
# ---- default behavior ----
197198
# named group
@@ -209,6 +210,10 @@ def test_h5_io_lgroup(tmpdir):
209210
named_axis_not_in_file.to_hdf(fpath)
210211
named2 = read_hdf(fpath, key=named_axis_not_in_file.name)
211212
assert all(named_axis_not_in_file == named2)
213+
# string group
214+
string_group.to_hdf(fpath)
215+
string_group2 = read_hdf(fpath, key=string_group.name)
216+
assert all(string_group == string_group2)
212217

213218
# ---- specific hdf group + key ----
214219
hdf_group = 'my_groups'

0 commit comments

Comments
 (0)