Skip to content

Commit 8681ff3

Browse files
alixdammanalixdamman
committed
updated read/write unittests --> use ndtest to generate data:
- updated CSV test files + test.xlsx - updated unittests test_read_csv, test_read_excel_pandas, test_read_excel_xlwings and test_to_csv
1 parent e710e74 commit 8681ff3

9 files changed

Lines changed: 106 additions & 185 deletions

File tree

‎larray/tests/data/test.xlsx‎

-807 Bytes
Binary file not shown.

‎larray/tests/data/test1d.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
time,2007,2010,2013
1+
a,a0,a1,a2
22
,0,1,2

‎larray/tests/data/test2d.csv‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1-
a\b,0,1,2
2-
0,0,1,2
3-
1,3,4,5
1+
a\b,b0,b1
2+
1,0,1
3+
2,2,3
4+
3,4,5

‎larray/tests/data/test2d_classic.csv‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
age,2007,2010,2013
2-
0,3722,3395,3347
3-
1,338,316,323
4-
2,2878,2791,2822
5-
3,1121,1037,976
6-
4,4073,4161,4429
1+
a,b0,b1,b2
2+
a0,0,1,2
3+
a1,3,4,5
4+
a2,6,7,8

‎larray/tests/data/test3d.csv‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
age,sex\time,2015,2016,2017
2-
0,F,0.5,1.5,2.5
3-
0,M,3.5,4.5,5.5
4-
1,F,6.5,7.5,8.5
5-
1,M,9.5,10.5,11.5
6-
2,F,12.5,13.5,14.5
7-
2,M,15.5,16.5,17.5
8-
3,F,18.5,19.5,20.5
9-
3,M,21.5,22.5,23.5
1+
a,b\c,c0,c1,c2
2+
1,b0,0,1,2
3+
1,b1,3,4,5
4+
2,b0,6,7,8
5+
2,b1,9,10,11
6+
3,b0,12,13,14
7+
3,b1,15,16,17

‎larray/tests/data/test5d.csv‎

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎larray/tests/data/testint_labels.csv‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
a,b\c,0,1,2
2+
0,0,0,1,2
3+
0,1,3,4,5
4+
0,2,6,7,8
5+
1,0,9,10,11
6+
1,1,12,13,14
7+
1,2,15,16,17
8+
2,0,18,19,20
9+
2,1,21,22,23
10+
2,2,24,25,26

‎larray/tests/data/testmissing_values.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
a,b\c,c0,c1,c2
2+
1,b0,0,1,2
3+
1,b1,3,4,5
4+
2,b1,9,10,11
5+
3,b0,12,13,14

‎larray/tests/test_array.py‎

Lines changed: 75 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,15 @@ def setUp(self):
8888
self.small_data = np.arange(30).reshape(2, 15)
8989
self.small = LArray(self.small_data, axes=(self.sex, self.lipro), title=self.small_title)
9090

91+
self.io_1d = ndtest(3)
92+
self.io_2d = ndtest("a=1..3; b=b0,b1")
93+
self.io_3d = ndtest("a=1..3; b=b0,b1; c=c0..c2")
94+
self.io_int_labels = ndtest("a=0..2; b=0..2; c=0..2")
95+
self.io_unsorted = ndtest("a=3..1; b=b1,b0; c=c2..c0")
96+
self.io_missing_values = ndtest("a=1..3; b=b0,b1; c=c0..c2", dtype=float)
97+
self.io_missing_values[2, 'b0'] = np.nan
98+
self.io_missing_values[3, 'b1'] = np.nan
99+
91100
@pytest.fixture(autouse=True)
92101
def setup(self, tmpdir):
93102
self.tmpdir = tmpdir.strpath
@@ -2590,27 +2599,19 @@ def test_from_string(self):
25902599

25912600
def test_read_csv(self):
25922601
res = read_csv(inputpath('test1d.csv'))
2593-
assert_array_equal(res, ndtest('time=2007,2010,2013'))
2602+
assert_array_equal(res, self.io_1d)
25942603

25952604
res = read_csv(inputpath('test2d.csv'))
2596-
assert_array_equal(res, ndtest('a=0,1;b=0,1,2'))
2605+
assert_array_equal(res, self.io_2d)
25972606

25982607
res = read_csv(inputpath('test3d.csv'))
2599-
expected = ndtest('age=0..3;sex=F,M;time=2015..2017') + 0.5
2600-
assert_array_equal(res, expected)
2608+
assert_array_equal(res, self.io_3d)
26012609

2602-
la = read_csv(inputpath('test5d.csv'))
2603-
self.assertEqual(la.ndim, 5)
2604-
self.assertEqual(la.shape, (2, 5, 2, 2, 3))
2605-
self.assertEqual(la.axes.names, ['arr', 'age', 'sex', 'nat', 'time'])
2606-
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :],
2607-
[3722, 3395, 3347])
2610+
res = read_csv(inputpath('testint_labels.csv'))
2611+
assert_array_equal(res, self.io_int_labels)
26082612

2609-
la = read_csv(inputpath('test2d_classic.csv'))
2610-
self.assertEqual(la.ndim, 2)
2611-
self.assertEqual(la.shape, (5, 3))
2612-
self.assertEqual(la.axes.names, ['age', None])
2613-
assert_array_equal(la[0, :], [3722, 3395, 3347])
2613+
res = read_csv(inputpath('test2d_classic.csv'))
2614+
assert_array_equal(res, ndtest("a=a0..a2; b0..b2"))
26142615

26152616
la = read_csv(inputpath('test1d_liam2.csv'), dialect='liam2')
26162617
self.assertEqual(la.ndim, 1)
@@ -2622,8 +2623,11 @@ def test_read_csv(self):
26222623
self.assertEqual(la.ndim, 5)
26232624
self.assertEqual(la.shape, (2, 5, 2, 2, 3))
26242625
self.assertEqual(la.axes.names, ['arr', 'age', 'sex', 'nat', 'time'])
2625-
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :],
2626-
[3722, 3395, 3347])
2626+
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :], [3722, 3395, 3347])
2627+
2628+
# missing values
2629+
res = read_csv(inputpath('testmissing_values.csv'))
2630+
assert_array_nan_equal(res, self.io_missing_values)
26272631

26282632
# test StringIO
26292633
res = read_csv(StringIO('a,1,2\n,0,1\n'))
@@ -2644,53 +2648,32 @@ def test_read_eurostat(self):
26442648

26452649
@pytest.mark.skipif(xw is None, reason="xlwings is not available")
26462650
def test_read_excel_xlwings(self):
2647-
la = read_excel(inputpath('test.xlsx'), '1d')
2648-
self.assertEqual(la.ndim, 1)
2649-
self.assertEqual(la.shape, (3,))
2650-
self.assertEqual(la.axes.names, ['time'])
2651-
assert_array_equal(la, [3722, 3395, 3347])
2651+
arr = read_excel(inputpath('test.xlsx'), '1d')
2652+
assert_array_equal(arr, self.io_1d)
26522653

2653-
la = read_excel(inputpath('test.xlsx'), '2d')
2654-
self.assertEqual(la.ndim, 2)
2655-
self.assertEqual(la.shape, (5, 3))
2656-
self.assertEqual(la.axes.names, ['age', 'time'])
2657-
assert_array_equal(la[0, :], [3722, 3395, 3347])
2654+
arr = read_excel(inputpath('test.xlsx'), '2d')
2655+
assert_array_equal(arr, self.io_2d)
26582656

2659-
la = read_excel(inputpath('test.xlsx'), '3d')
2660-
self.assertEqual(la.ndim, 3)
2661-
self.assertEqual(la.shape, (5, 2, 3))
2662-
self.assertEqual(la.axes.names, ['age', 'sex', 'time'])
2663-
assert_array_equal(la[0, 'F', :], [3722, 3395, 3347])
2657+
arr = read_excel(inputpath('test.xlsx'), '3d')
2658+
assert_array_equal(arr, self.io_3d)
26642659

2665-
la = read_excel(inputpath('test.xlsx'), '5d')
2666-
self.assertEqual(la.ndim, 5)
2667-
self.assertEqual(la.shape, (2, 5, 2, 2, 3))
2668-
self.assertEqual(la.axes.names, ['arr', 'age', 'sex', 'nat', 'time'])
2669-
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :],
2670-
[3722, 3395, 3347])
2660+
arr = read_excel(inputpath('test.xlsx'), 'int_labels')
2661+
assert_array_equal(arr, self.io_int_labels)
26712662

2672-
la = read_excel(inputpath('test.xlsx'), '2d_classic')
2673-
self.assertEqual(la.ndim, 2)
2674-
self.assertEqual(la.shape, (5, 3))
2675-
self.assertEqual(la.axes.names, ['age', None])
2676-
assert_array_equal(la[0, :], [3722, 3395, 3347])
2663+
arr = read_excel(inputpath('test.xlsx'), '2d_classic')
2664+
assert_array_equal(arr, ndtest("a=a0..a2; b0..b2"))
26772665

26782666
# passing a Group as sheet arg
26792667
axis = Axis('dim=1d,2d,3d,5d')
26802668

2681-
la = read_excel(inputpath('test.xlsx'), axis['1d'])
2682-
self.assertEqual(la.ndim, 1)
2683-
self.assertEqual(la.shape, (3,))
2684-
self.assertEqual(la.axes.names, ['time'])
2685-
assert_array_equal(la, [3722, 3395, 3347])
2669+
arr = read_excel(inputpath('test.xlsx'), axis['1d'])
2670+
assert_array_equal(arr, ndtest(3))
26862671

2687-
# fill_value argument
2688-
la = read_excel(inputpath('test.xlsx'), 'missing_values', fill_value=42)
2689-
assert la.ndim == 3
2690-
assert la.shape == (5, 2, 3)
2691-
assert la.axes.names == ['age', 'sex', 'time']
2692-
assert_array_equal(la[1, 'H', :], [42, 42, 42])
2693-
assert_array_equal(la[4, 'F', :], [42, 42, 42])
2672+
# missing rows + fill_value argument
2673+
arr = read_excel(inputpath('test.xlsx'), 'missing_values', fill_value=42)
2674+
expected = self.io_missing_values.copy()
2675+
expected[isnan(expected)] = 42
2676+
assert_array_equal(arr, expected)
26942677

26952678
# invalid keyword argument
26962679
with self.assertRaisesRegexp(TypeError, "'dtype' is an invalid keyword argument for this function when using "
@@ -2714,64 +2697,38 @@ def test_read_excel_xlwings(self):
27142697
assert_array_equal(bad4, good2)
27152698

27162699
def test_read_excel_pandas(self):
2717-
la = read_excel(inputpath('test.xlsx'), '1d', engine='xlrd')
2718-
self.assertEqual(la.ndim, 1)
2719-
self.assertEqual(la.shape, (3,))
2720-
self.assertEqual(la.axes.names, ['time'])
2721-
assert_array_equal(la, [3722, 3395, 3347])
2700+
arr = read_excel(inputpath('test.xlsx'), '1d', engine='xlrd')
2701+
assert_array_equal(arr, self.io_1d)
27222702

2723-
la = read_excel(inputpath('test.xlsx'), '2d', nb_axes=2, engine='xlrd')
2724-
self.assertEqual(la.ndim, 2)
2725-
self.assertEqual(la.shape, (5, 3))
2726-
self.assertEqual(la.axes.names, ['age', 'time'])
2727-
assert_array_equal(la[0, :], [3722, 3395, 3347])
2728-
2729-
la = read_excel(inputpath('test.xlsx'), '2d', engine='xlrd')
2730-
self.assertEqual(la.ndim, 2)
2731-
self.assertEqual(la.shape, (5, 3))
2732-
self.assertEqual(la.axes.names, ['age', 'time'])
2733-
assert_array_equal(la[0, :], [3722, 3395, 3347])
2734-
2735-
la = read_excel(inputpath('test.xlsx'), '3d', index_col=[0, 1], engine='xlrd')
2736-
self.assertEqual(la.ndim, 3)
2737-
self.assertEqual(la.shape, (5, 2, 3))
2738-
self.assertEqual(la.axes.names, ['age', 'sex', 'time'])
2739-
assert_array_equal(la[0, 'F', :], [3722, 3395, 3347])
2703+
arr = read_excel(inputpath('test.xlsx'), '2d', nb_axes=2, engine='xlrd')
2704+
assert_array_equal(arr, self.io_2d)
27402705

2741-
la = read_excel(inputpath('test.xlsx'), '3d', engine='xlrd')
2742-
self.assertEqual(la.ndim, 3)
2743-
self.assertEqual(la.shape, (5, 2, 3))
2744-
self.assertEqual(la.axes.names, ['age', 'sex', 'time'])
2745-
assert_array_equal(la[0, 'F', :], [3722, 3395, 3347])
2706+
arr = read_excel(inputpath('test.xlsx'), '2d', engine='xlrd')
2707+
assert_array_equal(arr, self.io_2d)
27462708

2747-
la = read_excel(inputpath('test.xlsx'), '5d', nb_axes=5, engine='xlrd')
2748-
self.assertEqual(la.ndim, 5)
2749-
self.assertEqual(la.shape, (2, 5, 2, 2, 3))
2750-
self.assertEqual(la.axes.names, ['arr', 'age', 'sex', 'nat', 'time'])
2751-
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :],
2752-
[3722, 3395, 3347])
2709+
arr = read_excel(inputpath('test.xlsx'), '3d', index_col=[0, 1], engine='xlrd')
2710+
assert_array_equal(arr, self.io_3d)
27532711

2754-
la = read_excel(inputpath('test.xlsx'), '5d', engine='xlrd')
2755-
self.assertEqual(la.ndim, 5)
2756-
self.assertEqual(la.shape, (2, 5, 2, 2, 3))
2757-
self.assertEqual(la.axes.names, ['arr', 'age', 'sex', 'nat', 'time'])
2758-
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :],
2759-
[3722, 3395, 3347])
2712+
arr = read_excel(inputpath('test.xlsx'), '3d', engine='xlrd')
2713+
assert_array_equal(arr, self.io_3d)
2714+
2715+
arr = read_excel(inputpath('test.xlsx'), 'int_labels', engine='xlrd')
2716+
assert_array_equal(arr, self.io_int_labels)
27602717

2761-
la = read_excel(inputpath('test.xlsx'), '2d_classic', engine='xlrd')
2762-
self.assertEqual(la.ndim, 2)
2763-
self.assertEqual(la.shape, (5, 3))
2764-
self.assertEqual(la.axes.names, ['age', None])
2765-
assert_array_equal(la[0, :], [3722, 3395, 3347])
2718+
arr = read_excel(inputpath('test.xlsx'), '2d_classic', engine='xlrd')
2719+
assert_array_equal(arr, ndtest("a=a0..a2; b0..b2"))
27662720

27672721
# passing a Group as sheet arg
27682722
axis = Axis('dim=1d,2d,3d,5d')
27692723

2770-
la = read_excel(inputpath('test.xlsx'), axis['1d'], engine='xlrd')
2771-
self.assertEqual(la.ndim, 1)
2772-
self.assertEqual(la.shape, (3,))
2773-
self.assertEqual(la.axes.names, ['time'])
2774-
assert_array_equal(la, [3722, 3395, 3347])
2724+
arr = read_excel(inputpath('test.xlsx'), axis['1d'], engine='xlrd')
2725+
assert_array_equal(arr, ndtest(3))
2726+
2727+
# missing rows + fill_value argument
2728+
arr = read_excel(inputpath('test.xlsx'), 'missing_values', fill_value=42, engine='xlrd')
2729+
expected = self.io_missing_values.copy()
2730+
expected[isnan(expected)] = 42
2731+
assert_array_equal(arr, expected)
27752732

27762733
# Excel sheet with blank cells on right/bottom border of the array to read
27772734
fpath = inputpath('test_blank_cells.xlsx')
@@ -3172,31 +3129,26 @@ def test_from_frame(self):
31723129
assert_array_equal(la[0, 'F', :], [3722, 3395, 3347])
31733130

31743131
def test_to_csv(self):
3175-
la = read_csv(inputpath('test5d.csv'))
3176-
self.assertEqual(la.ndim, 5)
3177-
self.assertEqual(la.shape, (2, 5, 2, 2, 3))
3178-
self.assertEqual(la.axes.names, ['arr', 'age', 'sex', 'nat', 'time'])
3179-
assert_array_equal(la[X.arr[1], 0, 'F', X.nat[1], :],
3180-
[3722, 3395, 3347])
3132+
arr = self.io_3d.copy()
31813133

3182-
la.to_csv(self.tmp_path('out.csv'))
3183-
result = ['arr,age,sex,nat\\time,2007,2010,2013\n',
3184-
'1,0,F,1,3722,3395,3347\n',
3185-
'1,0,F,2,338,316,323\n']
3134+
arr.to_csv(self.tmp_path('out.csv'))
3135+
result = ['a,b\\c,c0,c1,c2\n',
3136+
'1,b0,0,1,2\n',
3137+
'1,b1,3,4,5\n']
31863138
with open(self.tmp_path('out.csv')) as f:
31873139
self.assertEqual(f.readlines()[:3], result)
31883140

31893141
# stacked data (one column containing all the values and another column listing the context of the value)
3190-
la.to_csv(self.tmp_path('out.csv'), wide=False)
3191-
result = ['arr,age,sex,nat,time,value\n',
3192-
'1,0,F,1,2007,3722\n',
3193-
'1,0,F,1,2010,3395\n']
3142+
arr.to_csv(self.tmp_path('out.csv'), wide=False)
3143+
result = ['a,b,c,value\n',
3144+
'1,b0,c0,0\n',
3145+
'1,b0,c1,1\n']
31943146
with open(self.tmp_path('out.csv')) as f:
31953147
self.assertEqual(f.readlines()[:3], result)
31963148

3197-
la = ndtest([Axis('time=2015..2017')])
3198-
la.to_csv(self.tmp_path('test_out1d.csv'))
3199-
result = ['time,2015,2016,2017\n',
3149+
arr = self.io_1d.copy()
3150+
arr.to_csv(self.tmp_path('test_out1d.csv'))
3151+
result = ['a,a0,a1,a2\n',
32003152
',0,1,2\n']
32013153
with open(self.tmp_path('test_out1d.csv')) as f:
32023154
self.assertEqual(f.readlines(), result)
@@ -3521,11 +3473,9 @@ def test_open_excel(self):
35213473
assert_array_equal(res.axes[2].labels, a3.axes[2].labels)
35223474

35233475
with open_excel(inputpath('test.xlsx')) as wb:
3476+
expected = ndtest("a=a0..a2; b0..b2")
35243477
res = wb['2d_classic'].load()
3525-
assert res.ndim == 2
3526-
assert res.shape == (5, 3)
3527-
assert res.axes.names == ['age', None]
3528-
assert_array_equal(res[0, :], [3722, 3395, 3347])
3478+
assert_array_equal(res, expected)
35293479

35303480
# 3) without headers
35313481
# ==================

0 commit comments

Comments
 (0)