Repository navigation
Expand file tree
/
Copy pathsession.py
More file actions
1682 lines (1433 loc) · 62.5 KB
/
Copy pathsession.py
File metadata and controls
1682 lines (1433 loc) · 62.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import sys
import re
import fnmatch
import warnings
from collections import OrderedDict
from pathlib import Path
import numpy as np
from typing import Any, List, Union, KeysView, ValuesView, ItemsView, Dict, Iterable
from larray.core.metadata import Metadata
from larray.core.group import Group
from larray.core.axis import Axis
from larray.core.constants import nan
from larray.core.array import Array, get_axes, ndtest, zeros, zeros_like, sequence # noqa: F401
from larray.util.misc import float_error_handler_factory, is_interactive_interpreter, renamed_to, inverseop
from larray.inout.session import ext_default_engine, get_file_handler
# XXX: inherit from OrderedDict or Array?
class Session:
r"""
Groups several objects together.
Parameters
----------
*args : str or dict of {str: object} or iterable of tuples (str, object)
Path to the file containing the session to load or
list/tuple/dictionary containing couples (name, object).
**kwargs : dict of {str: object}
* Objects to add written as name=object
* meta : list of pairs or dict or OrderedDict or Metadata
Metadata (title, description, author, creation_date, ...) associated with the array.
Keys must be strings. Values must be of type string, int, float, date, time or datetime.
Warnings
--------
Metadata is not kept when actions or methods are applied on a session
except for operations modifying a specific array, such as: `s['arr1'] = 0`.
Do not add metadata to a session if you know you will apply actions or methods
on it before dumping it.
Examples
--------
>>> # scalars
>>> i, s = 5, 'string'
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2")
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01'
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a)
create a Session by passing a list of pairs (name, object)
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)])
create a Session using keyword arguments
>>> ses = Session(i=i, s=s, a=a, b=b, a01=a01, arr1=arr1, arr2=arr2)
create a Session by passing a dictionary
>>> ses = Session({'i': i, 's': s, 'a': a, 'b': b, 'a01': a01, 'arr1': arr1, 'arr2': arr2})
load Session from file
>>> ses = Session('my_session.h5') # doctest: +SKIP
create a session with metadata
>>> ses = Session(arr1=arr1, arr2=arr2, meta=Metadata(title='my title', author='John Smith'))
>>> ses.meta
title: my title
author: John Smith
"""
def __init__(self, *args, **kwargs):
# FIXME : use a dict instead of an OrderedDict
object.__setattr__(self, '_objects', OrderedDict())
meta = kwargs.pop('meta', None)
if meta is None:
meta = Metadata()
self.meta = meta
if len(args) == 1:
assert len(kwargs) == 0
a0 = args[0]
if isinstance(a0, (str, Path)):
# assume a0 is a filename
self.load(a0)
else:
# iterable of tuple or dict-like
self.update(a0)
else:
self.add(*args, **kwargs)
@property
def meta(self) -> Metadata:
r"""Returns metadata of the session.
Returns
-------
Metadata:
Metadata of the session.
"""
return self._meta
@meta.setter
def meta(self, meta) -> None:
if not isinstance(meta, (list, dict, OrderedDict, Metadata)):
raise TypeError(f"Expected list of pairs or dict or OrderedDict or Metadata object "
f"instead of {type(meta).__name__}")
object.__setattr__(self, '_meta', meta if isinstance(meta, Metadata) else Metadata(meta))
# XXX: behave like a dict and return keys instead?
def __iter__(self) -> Iterable[Any]:
return iter(self.values())
def add(self, *args, **kwargs) -> None:
r"""
Adds objects to the current session.
Parameters
----------
*args : list of object
Objects to add. Objects must have an attribute 'name'.
**kwargs : dict of {str: object}
Objects to add written as name=array, ...
Examples
--------
>>> s = Session()
>>> axis1, axis2 = Axis('x=x0..x2'), Axis('y=y0..y2')
>>> arr1, arr2, arr3 = ndtest((2, 2)), ndtest(4), ndtest((3, 2))
>>> s.add(axis1, axis2, arr1=arr1, arr2=arr2, arr3=arr3)
>>> # print item's names in sorted order
>>> s.names
['arr1', 'arr2', 'arr3', 'x', 'y']
"""
self._update_from_iterable((arg.name, arg) for arg in args)
self._update_from_iterable(kwargs.items())
def update(self, other=None, **kwargs) -> None:
r"""
Update the session with the key/value pairs from other or passed keyword arguments, overwriting existing keys.
Note that the session is updated inplace and no new Session object is returned.
Parameters
----------
other: Session or dict-like object or iterable with key/value pairs
Object containing key/value pairs to add or modify.
**kwargs:
If keyword arguments are specified, the session is then updated with those key/value pairs
(e.g.: ses.update(pop=pop, births=births, deaths=deaths)).
Examples
--------
>>> x, y = Axis('x=x0..x2'), Axis('y=y0..y3')
>>> arr1 = ndtest((x, y))
>>> arr2 = ndtest(x)
>>> s = Session(x=x, y=y, arr1=arr1, arr2=arr2)
>>> # print item's names in sorted order
>>> s.names
['arr1', 'arr2', 'x', 'y']
>>> s.arr2
x x0 x1 x2
0 1 2
>>> # new axis and array
>>> z = Axis('z=z0..z2')
>>> arr3 = ndtest((x, z))
>>> # arr2 is modified
>>> arr2_modified = arr2.set_axes('x', z)
Passing another session
>>> s2 = Session(z=z, arr2=arr2_modified, arr3=arr3)
>>> s.names
['arr1', 'arr2', 'x', 'y']
>>> s.arr2
x x0 x1 x2
0 1 2
>>> s.update(s2)
>>> # new items have been added to the session 's'
>>> s.names
['arr1', 'arr2', 'arr3', 'x', 'y', 'z']
>>> # and array 'arr2' has been updated
>>> s.arr2
z z0 z1 z2
0 1 2
Passing a dictionary
>>> s = Session(x=x, y=y, arr1=arr1, arr2=arr2)
>>> s.names
['arr1', 'arr2', 'x', 'y']
>>> s.arr2
x x0 x1 x2
0 1 2
>>> d = {'z': z, 'arr2': arr2_modified, 'arr3': arr3}
>>> s.update(d)
>>> s.names
['arr1', 'arr2', 'arr3', 'x', 'y', 'z']
>>> s.arr2
z z0 z1 z2
0 1 2
Passing an iterable with key/value pairs
>>> s = Session(x=x, y=y, arr1=arr1, arr2=arr2)
>>> s.names
['arr1', 'arr2', 'x', 'y']
>>> s.arr2
x x0 x1 x2
0 1 2
>>> i = [('z', z), ('arr2', arr2_modified), ('arr3', arr3)]
>>> s.update(i)
>>> s.names
['arr1', 'arr2', 'arr3', 'x', 'y', 'z']
>>> s.arr2
z z0 z1 z2
0 1 2
Passing keyword arguments
>>> s = Session(x=x, y=y, arr1=arr1, arr2=arr2)
>>> s.names
['arr1', 'arr2', 'x', 'y']
>>> s.arr2
x x0 x1 x2
0 1 2
>>> s.update(z=z, arr2=arr2_modified, arr3=arr3)
>>> s.names
['arr1', 'arr2', 'arr3', 'x', 'y', 'z']
>>> s.arr2
z z0 z1 z2
0 1 2
"""
from collections.abc import Iterable
if other is None:
pass
elif hasattr(other, 'items'):
self._update_from_iterable(other.items())
elif isinstance(other, Iterable):
self._update_from_iterable(other)
else:
raise ValueError(f"Expected Session, dict-like or iterable object for 'other' argument. "
f"Got {type(other).__name__}.")
self._update_from_iterable(kwargs.items())
def _ipython_key_completions_(self) -> List[str]:
return list(self.keys())
def __getitem__(self, key) -> Union[Any, 'Session']:
if isinstance(key, int):
keys = list(self.keys())
return self._objects[keys[key]]
elif isinstance(key, Array):
assert np.issubdtype(key.dtype, np.bool_)
assert key.ndim == 1
# only keep True values
truenames = key[key].axes[0].labels
return Session([(name, self[name]) for name in truenames])
elif isinstance(key, (tuple, list)):
assert all(isinstance(k, str) for k in key)
return Session([(k, self[k]) for k in key])
else:
return self._objects[key]
def get(self, key, default=None) -> Any:
r"""
Returns the object corresponding to the key.
If the key doesn't correspond to any object, a default one can be returned.
Parameters
----------
key : str
Name of the object.
default : object, optional
Returned object if the key doesn't correspond to any object of the current session.
Returns
-------
object
Object corresponding to the given key or a default one if not found.
Examples
--------
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2")
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01'
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a)
>>> s = Session([('a', a), ('b', b), ('a01', a01), ('arr1', arr1), ('arr2', arr2)])
>>> arr = s.get('arr1')
>>> arr
a\b b0 b1 b2
a0 0 1 2
a1 3 4 5
a2 6 7 8
>>> arr = s.get('arr4', zeros('a=a0,a1;b=b0,b1', dtype=int))
>>> arr
a\b b0 b1
a0 0 0
a1 0 0
"""
try:
return self[key]
except KeyError:
return default
def __setitem__(self, key, value) -> None:
self._objects[key] = value
def __delitem__(self, key) -> None:
del self._objects[key]
def __getattr__(self, key) -> Any:
if key in self._objects:
return self._objects[key]
else:
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{key}'")
def __setattr__(self, key, value) -> None:
# the condition below is needed because, unlike __getattr__, __setattr__ is called before any property
# see https://stackoverflow.com/a/15751159
if key == 'meta':
super().__setattr__(key, value)
else:
self._objects[key] = value
def __delattr__(self, key) -> None:
del self._objects[key]
def __dir__(self) -> List[str]:
return list(set(dir(self.__class__)) | set(self.keys()))
# needed to make *un*pickling work (because otherwise, __getattr__ is called before ._objects exists, which leads to
# an infinite recursion)
def __getstate__(self):
return self.__dict__
def __setstate__(self, d):
# equivalent to self.__dict__ = d (we need this form because __setattr__ is overridden)
object.__setattr__(self, '__dict__', d)
def load(self, fname, names=None, engine='auto', display=False, **kwargs) -> None:
r"""
Load objects from a file, or several .csv files.
The Excel and CSV formats can only contain objects of Array type (plus metadata).
WARNING: never load a file using the pickle engine (.pkl or .pickle) from an untrusted source, as it can lead
to arbitrary code execution.
Parameters
----------
fname : str or Path
This can be either the path to a single file, a path to a directory containing .csv files or a pattern
representing several .csv files.
names : list of str, optional
List of objects to load.
If `fname` is None, list of paths to CSV files.
Defaults to all valid objects present in the file/directory.
engine : {'auto', 'pandas_csv', 'pandas_hdf', 'pandas_excel', 'xlwings_excel', 'pickle'}, optional
Load using `engine`. Defaults to 'auto' (use default engine for the format guessed from the file extension).
display : bool, optional
Whether or not to display which file is being worked on. Defaults to False.
Examples
--------
In one module:
>>> # scalars
>>> i, s = 5, 'string' # doctest: +SKIP
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2") # doctest: +SKIP
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01' # doctest: +SKIP
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a) # doctest: +SKIP
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)]) # doctest: +SKIP
>>> # metadata
>>> ses.meta.title = 'my title' # doctest: +SKIP
>>> ses.meta.author = 'John Smith' # doctest: +SKIP
>>> # save the session in an HDF5 file
>>> ses.save('input.h5') # doctest: +SKIP
In another module: load the whole session
>>> # the load method is automatically called when passing
>>> # the path of file to the Session constructor
>>> ses = Session('input.h5') # doctest: +SKIP
>>> ses # doctest: +SKIP
Session(a, a01, arr1, arr2, b, i, s)
>>> ses.meta # doctest: +SKIP
title: my title
author: John Smith
Load only some objects
>>> ses = Session()
>>> ses.load('input.h5', names=['s', 'a', 'b', 'arr1', 'arr2'], display=True) # doctest: +SKIP
opening input.h5
loading Axis object a ... done
loading Array object arr1 ... done
loading Array object arr2 ... done
loading Axis object b ... done
loading str object s ... done
Using .csv files (assuming the same session as above)
>>> ses.save('data') # doctest: +SKIP
>>> ses = Session() # doctest: +SKIP
>>> # load all .csv files from the 'data' directory
>>> ses.load('data', display=True) # doctest: +SKIP
opening data
loading Array object arr1 ... done
loading Array object arr2 ... done
>>> # or only arrays containing the character '1' in their names
>>> ses.load('data/*1.csv', display=True) # doctest: +SKIP
opening data/*1.csv
loading Array object arr1 ... done
"""
if display:
print("opening", fname)
if fname is None:
if all([Path(name).suffix == '.csv' for name in names]):
engine = ext_default_engine['csv']
else:
raise ValueError(f"List of paths to only CSV files expected. Got {names}")
elif isinstance(fname, str):
fname = Path(fname)
if not isinstance(fname, Path):
raise TypeError(f"Expected a string or a Path object for the 'fname' argument. "
f"Got object of type '{type(fname).__name__}' instead.")
if engine == 'auto':
ext = fname.suffix
ext = ext.strip('.') if ext else 'csv'
engine = ext_default_engine[ext]
handler_cls = get_file_handler(engine)
if engine == 'pandas_csv' and 'sep' in kwargs:
handler = handler_cls(fname, kwargs['sep'])
else:
handler = handler_cls(fname)
metadata, objects = handler.read(names, display=display, **kwargs)
self._update_from_iterable(objects.items())
self.meta = metadata
def _update_from_iterable(self, it) -> None:
for k, v in it:
self[k] = v
def save(self, fname, names=None, engine='auto', overwrite=True, display=False, **kwargs) -> None:
r"""
Dumps objects from the current session to a file, or several .csv files.
The Excel and CSV formats only dump objects of Array type (plus metadata).
Parameters
----------
fname : str or Path
Path of the file for the dump.
If objects are saved in CSV files, the path corresponds to a directory.
names : list of str or None, optional
List of names of objects to dump.
If `fname` is None, list of paths to CSV files.
Defaults to all objects present in the Session.
engine : {'auto', 'pandas_csv', 'pandas_hdf', 'pandas_excel', 'xlwings_excel', 'pickle'}, optional
Dump using `engine`. Defaults to 'auto' (use default engine for the format guessed from the file extension).
overwrite: bool, optional
Whether or not to overwrite an existing file, if any. Ignored for CSV files and 'pandas_excel' engine.
If False, file is updated. Defaults to True.
display : bool, optional
Whether or not to display which file is being worked on. Defaults to False.
Examples
--------
>>> # scalars
>>> i, s = 5, 'string' # doctest: +SKIP
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2") # doctest: +SKIP
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01' # doctest: +SKIP
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a) # doctest: +SKIP
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)]) # doctest: +SKIP
>>> # metadata
>>> ses.meta.title = 'my title' # doctest: +SKIP
>>> ses.meta.author = 'John Smith' # doctest: +SKIP
Save all objects
>>> ses.save('output.h5', display=True) # doctest: +SKIP
dumping i ... done
dumping s ... done
dumping a ... done
dumping b ... done
dumping a01 ... done
dumping arr1 ... done
dumping arr2 ... done
Save only some objects
>>> ses.save('output.h5', names=['s', 'a', 'b', 'arr1', 'arr2'], display=True) # doctest: +SKIP
dumping s ... done
dumping a ... done
dumping b ... done
dumping arr1 ... done
dumping arr2 ... done
Update file
>>> arr1, arr4 = ndtest((3, 3)), ndtest((2, 3)) # doctest: +SKIP
>>> ses2 = Session([('arr1', arr1), ('arr4', arr4)]) # doctest: +SKIP
>>> # replace arr1 and add arr4 in file output.h5
>>> ses2.save('output.h5', overwrite=False, display=True) # doctest: +SKIP
dumping arr1 ... done
dumping arr4 ... done
"""
if isinstance(fname, str):
fname = Path(fname)
if not isinstance(fname, Path):
raise TypeError(f"Expected a string or a Path object for the 'fname' argument. "
f"Got object of type '{type(fname).__name__}' instead.")
if engine == 'auto':
ext = fname.suffix
ext = ext.strip('.') if ext else 'csv'
engine = ext_default_engine[ext]
handler_cls = get_file_handler(engine)
if engine == 'pandas_csv' and 'sep' in kwargs:
handler = handler_cls(fname, overwrite, kwargs['sep'])
else:
handler = handler_cls(fname, overwrite)
meta = self.meta if overwrite else None
items = self.items()
if names is not None:
names_set = set(names)
items = [(k, v) for k, v in items if k in names_set]
handler.dump(meta, items, display=display, **kwargs)
def to_globals(self, names=None, depth=0, warn=True, inplace=False) -> None:
r"""
Create global variables out of objects in the session.
Parameters
----------
names : list of str or None, optional
List of names of objects to convert to globals. Defaults to all objects present in the Session.
depth : int, optional
depth of call stack where to create the variables. 0 is where to_globals was called, 1 the caller of
to_globals, etc. Defaults to 0.
warn : bool, optional
Whether or not to warn the user that this method should only be used in an interactive console (see below).
Defaults to True.
inplace : bool, optional
If True, to_globals will assume all arrays already exist and have the same axes and will replace their
content instead of creating new variables. Non array variables in the session will be ignored.
Defaults to False.
Notes
-----
This method should usually only be used in an interactive console and not in a script. Code editors are
confused by this kind of manipulation and will likely consider as invalid the code using variables created in
this way. Additionally, when using this method auto-completion, "show definition", "go to declaration" and
other similar code editor features will probably not work for the variables created in this way and any
variable derived from them.
Examples
--------
>>> s = Session(arr1=ndtest(3), arr2=ndtest((2, 2)))
>>> s.to_globals(warn=False)
>>> arr1
a a0 a1 a2
0 1 2
>>> arr2
a\b b0 b1
a0 0 1
a1 2 3
"""
# noinspection PyProtectedMember
if warn and not is_interactive_interpreter():
warnings.warn("Session.to_globals should usually only be used in interactive consoles and not in scripts. "
"Use warn=False to deactivate this warning.",
RuntimeWarning, stacklevel=2)
d = sys._getframe(depth + 1).f_globals
items = self.items()
if names is not None:
names_set = set(names)
items = [(k, v) for k, v in items if k in names_set]
if inplace:
for k, v in items:
if k not in d:
raise ValueError(f"'{k}' not found in current namespace. Session.to_globals(inplace=True) requires "
f"all arrays to already exist.")
if not isinstance(v, Array):
continue
if not d[k].axes == v.axes:
raise ValueError(f"Session.to_globals(inplace=True) requires the existing (destination) arrays "
f"to have the same axes than those stored in the session and this is not the case "
f"for '{k}'.\nexisting: {d[k].info}\nsession: {v.info}")
d[k][:] = v
else:
for k, v in items:
d[k] = v
def to_pickle(self, fname, names=None, overwrite=True, display=False, **kwargs) -> None:
r"""
Dumps objects from the current session to a file using pickle.
WARNING: never load a pickle file (.pkl or .pickle) from an untrusted source, as it can lead to arbitrary code
execution.
Parameters
----------
fname : str
Path for the dump.
names : list of str or None, optional
Names of objects to dump.
Defaults to all objects present in the Session.
overwrite: bool, optional
Whether or not to overwrite an existing file, if any.
If False, file is updated. Defaults to True.
display : bool, optional
Whether or not to display which file is being worked on. Defaults to False.
Examples
--------
>>> # scalars
>>> i, s = 5, 'string' # doctest: +SKIP
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2") # doctest: +SKIP
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01' # doctest: +SKIP
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a) # doctest: +SKIP
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)]) # doctest: +SKIP
>>> # metadata
>>> ses.meta.title = 'my title' # doctest: +SKIP
>>> ses.meta.author = 'John Smith' # doctest: +SKIP
Save all objects
>>> ses.to_pickle('output.pkl', display=True) # doctest: +SKIP
dumping i ... done
dumping s ... done
dumping a ... done
dumping b ... done
dumping a01 ... done
dumping arr1 ... done
dumping arr2 ... done
Save only some objects
>>> ses.to_pickle('output.pkl', names=['s', 'a', 'b', 'arr1', 'arr2'], display=True) # doctest: +SKIP
dumping s ... done
dumping a ... done
dumping b ... done
dumping arr1 ... done
dumping arr2 ... done
"""
self.save(fname, names, ext_default_engine['pkl'], overwrite, display, **kwargs)
dump = renamed_to(save, 'dump')
def to_hdf(self, fname, names=None, overwrite=True, display=False, **kwargs) -> None:
r"""
Dumps objects from the current session to an HDF file.
Parameters
----------
fname : str
Path of the file for the dump.
names : list of str or None, optional
Names of objects to dump.
Defaults to all objects present in the Session.
overwrite: bool, optional
Whether or not to overwrite an existing file, if any.
If False, file is updated. Defaults to True.
display : bool, optional
Whether or not to display which file is being worked on. Defaults to False.
Examples
--------
>>> # scalars
>>> i, s = 5, 'string' # doctest: +SKIP
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2") # doctest: +SKIP
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01' # doctest: +SKIP
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a) # doctest: +SKIP
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)]) # doctest: +SKIP
>>> # metadata
>>> ses.meta.title = 'my title' # doctest: +SKIP
>>> ses.meta.author = 'John Smith' # doctest: +SKIP
Save all objects
>>> ses.to_hdf('output.h5', display=True) # doctest: +SKIP
dumping i ... done
dumping s ... done
dumping a ... done
dumping b ... done
dumping a01 ... done
dumping arr1 ... done
dumping arr2 ... done
Save only some objects
>>> ses.to_hdf('output.h5', names=['s', 'a', 'b', 'arr1', 'arr2'], display=True) # doctest: +SKIP
dumping s ... done
dumping a ... done
dumping b ... done
dumping arr1 ... done
dumping arr2 ... done
"""
self.save(fname, names, ext_default_engine['hdf'], overwrite, display, **kwargs)
dump_hdf = renamed_to(to_hdf, 'dump_hdf')
def to_excel(self, fname, names=None, overwrite=True, display=False, **kwargs) -> None:
r"""
Dumps Array objects from the current session to an Excel file.
Parameters
----------
fname : str
Path of the file for the dump.
names : list of str or None, optional
Names of Array objects to dump.
Defaults to all Array objects present in the Session.
overwrite: bool, optional
Whether or not to overwrite an existing file, if any. If False, file is updated. Defaults to True.
display : bool, optional
Whether or not to display which file is being worked on. Defaults to False.
Notes
-----
- each array is saved in a separate sheet
- all session metadata is saved in the same sheet named __metadata__
Examples
--------
>>> # scalars
>>> i, s = 5, 'string' # doctest: +SKIP
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2") # doctest: +SKIP
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01' # doctest: +SKIP
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a) # doctest: +SKIP
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)]) # doctest: +SKIP
>>> # metadata
>>> ses.meta.title = 'my title' # doctest: +SKIP
>>> ses.meta.author = 'John Smith' # doctest: +SKIP
Save all arrays (and arrays only)
>>> ses.to_excel('output.xlsx', display=True) # doctest: +SKIP
dumping i ... Cannot dump i. int is not a supported type
dumping s ... Cannot dump s. str is not a supported type
dumping a ... Cannot dump a. Axis is not a supported type
dumping b ... Cannot dump b. Axis is not a supported type
dumping a01 ... Cannot dump a01. LGroup is not a supported type
dumping arr1 ... done
dumping arr2 ... done
Save only some arrays
>>> ses.to_excel('output.xlsx', names=['arr1'], display=True) # doctest: +SKIP
dumping arr1 ... done
"""
self.save(fname, names, ext_default_engine['xlsx'], overwrite, display, **kwargs)
dump_excel = renamed_to(to_excel, 'dump_excel')
def to_csv(self, fname, names=None, display=False, **kwargs) -> None:
r"""
Dumps Array objects from the current session to CSV files.
Parameters
----------
fname : str
Path for the directory that will contain CSV files.
names : list of str or None, optional
Names of Array objects to dump.
Defaults to all Array objects present in the Session.
display : bool, optional
Whether or not to display which file is being worked on. Defaults to False.
Notes
-----
- each array is saved in a separate file
- all session metadata is saved in the same CSV file named __metadata__.csv
Examples
--------
>>> # scalars
>>> i, s = 5, 'string' # doctest: +SKIP
>>> # axes
>>> a, b = Axis("a=a0..a2"), Axis("b=b0..b2") # doctest: +SKIP
>>> # groups
>>> a01 = a['a0,a1'] >> 'a01' # doctest: +SKIP
>>> # arrays
>>> arr1, arr2 = ndtest((a, b)), ndtest(a) # doctest: +SKIP
>>> ses = Session([('i', i), ('s', s), ('a', a), ('b', b), ('a01', a01),
... ('arr1', arr1), ('arr2', arr2)]) # doctest: +SKIP
>>> # metadata
>>> ses.meta.title = 'my title' # doctest: +SKIP
>>> ses.meta.author = 'John Smith' # doctest: +SKIP
Save all arrays (and arrays only)
>>> ses.to_csv('output', display=True) # doctest: +SKIP
dumping i ... Cannot dump i. int is not a supported type
dumping s ... Cannot dump s. str is not a supported type
dumping a ... Cannot dump a. Axis is not a supported type
dumping b ... Cannot dump b. Axis is not a supported type
dumping a01 ... Cannot dump a01. LGroup is not a supported type
dumping arr1 ... done
dumping arr2 ... done
Save only some arrays
>>> ses.to_csv('output', names=['arr1'], display=True) # doctest: +SKIP
dumping arr1 ... done
"""
self.save(fname, names, ext_default_engine['csv'], display=display, **kwargs)
dump_csv = renamed_to(to_csv, 'dump_csv')
def filter(self, pattern=None, kind=None) -> 'Session':
r"""
Returns a new session with objects which match some criteria.
Parameters
----------
pattern : str, optional
Only keep arrays whose key match `pattern`.
- `?` matches any single character
- `*` matches any number of characters
- [seq] matches any character in seq
- [!seq] matches any character not in seq
kind : (tuple of) type, optional
Only keep objects which are instances of type(s) `kind`.
Returns
-------
Session
The filtered session.
Examples
--------
>>> axis = Axis('a=a0..a2')
>>> group = axis['a0,a1'] >> 'a01'
>>> test1, zero1 = ndtest((2, 2)), zeros((3, 2))
>>> s = Session([('test1', test1), ('zero1', zero1), ('axis', axis), ('group', group)])
Filter using a pattern argument
>>> # get all items with names ending with '1'
>>> s.filter(pattern='*1').names
['test1', 'zero1']
>>> # get all items with names starting with letter in range a-k
>>> s.filter(pattern='[a-k]*').names
['axis', 'group']
Filter using kind argument
>>> s.filter(kind=Axis).names
['axis']
>>> s.filter(kind=(Axis, Group)).names
['axis', 'group']
"""
items = self._objects.items()
if pattern is not None:
regex = fnmatch.translate(pattern)
match = re.compile(regex).match
items = [(k, v) for k, v in items if match(k)]
if kind is not None:
items = [(k, v) for k, v in items if isinstance(v, kind)]
return Session(items)
@property
def names(self) -> List[str]:
r"""
Returns the list of names of the objects in the session.
The list is sorted alphabetically and does not follow the internal order.
Returns
-------
list of str
See Also
--------
Session.keys
Examples
--------
>>> axis1 = Axis("a=a0..a2")
>>> group1 = axis1['a0,a1'] >> 'a01'
>>> arr1, arr2 = ndtest((2, 2)), ndtest(4)
>>> s = Session([('arr2', arr2), ('arr1', arr1), ('group1', group1), ('axis1', axis1)])
>>> # print array's names in the alphabetical order
>>> s.names
['arr1', 'arr2', 'axis1', 'group1']
>>> # keys() follows the internal order
>>> list(s.keys())
['arr2', 'arr1', 'group1', 'axis1']
"""
return sorted(self._objects.keys())
def copy(self) -> 'Session':
r"""Returns a copy of the session.
"""
# this actually *does* a copy of the internal mapping (the mapping is not reused-as is)
return self.__class__(self._objects)
def keys(self) -> KeysView[str]:
r"""
Returns a view on the session's keys.
Returns
-------
View on the session's keys.
See Also
--------
Session.names
Examples
--------
>>> axis1 = Axis("a=a0..a2")
>>> group1 = axis1['a0,a1'] >> 'a01'
>>> arr1, arr2 = ndtest((2, 2)), ndtest(4)
>>> s = Session([('arr2', arr2), ('arr1', arr1), ('group1', group1), ('axis1', axis1)])
>>> # similar to names by follows the internal order
>>> list(s.keys())
['arr2', 'arr1', 'group1', 'axis1']
>>> # gives the names of objects in alphabetical order
>>> s.names
['arr1', 'arr2', 'axis1', 'group1']
"""
return self._objects.keys()
def values(self) -> ValuesView[Any]:
r"""
Returns a view on the session's values.
Returns
-------
View on the session's values.
Examples
--------
>>> axis1 = Axis("a=a0..a2")
>>> group1 = axis1['a0,a1'] >> 'a01'
>>> arr1, arr2 = ndtest((2, 2)), ndtest(4)
>>> s = Session([('arr2', arr2), ('arr1', arr1), ('group1', group1), ('axis1', axis1)])
>>> # assuming you know the order of objects stored in the session
>>> arr2, arr1, group1, axis1 = s.values()
>>> # otherwise, prefer the following syntax
>>> arr1, arr2, axis1, group1 = s['arr1', 'arr2', 'axis1', 'group1']
>>> arr1
a\b b0 b1
a0 0 1
a1 2 3
>>> axis1
Axis(['a0', 'a1', 'a2'], 'a')
"""
return self._objects.values()
def items(self) -> ItemsView[str, Any]:
r"""
Returns a view of the session’s items ((key, value) pairs).
Returns
-------
View on the session's items.
Examples
--------
>>> axis1 = Axis("a=a0..a2")
>>> group1 = axis1['a0,a1'] >> 'a01'
>>> arr1, arr2 = ndtest((2, 2)), ndtest(4)
>>> # make the test pass on both Windows and Linux