Skip to content

Commit 142f74e

Browse files
siddharthteotiawesm
siddharthteotia
authored and
wesm
committed
ARROW-1331: [JAVA] Refactor unit tests
[cc @jacques-n , @StevenMPhillips ] Refactored the java-vector unit tests in TestValueVector.java (1) Moved all the BitVector related tests to TestBitVector.java (2) Enhanced and added the tests for different vector types and grouped them into different categories in TestValueVector.java (3) Moved https://github.com/dremio/dremio-oss/blob/master/sabot/kernel/src/test/java/com/dremio/exec/vector/TestSplitAndTransfer.java to arrow. This currently has splitAndTransfer() test for NullableVarCharVector. (4) No tests have been removed -- they have either been enhanced or moved around for grouping. Recently we added splitAndTransfer tests for BitVector, UnionVector and ListVector in their respective test files. We should ideally move all the splitAndTransfer() related tests to TestSplitAndTransfer.java and should try to cover other vector types as well. A follow-up will address this (I want to keep the patch size small for easy review). Author: siddharth <siddharth@dremio.com> Closes apache#955 from siddharthteotia/ARROW-1331 and squashes the following commits: ead5cef [siddharth] ARROW-1331: Fixed unit tests 2174e65 [siddharth] ARROW-1331: Refactor unit tests
1 parent 6135958 commit 142f74e

3 files changed

Lines changed: 873 additions & 238 deletions

File tree

‎java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,14 @@
1919
package org.apache.arrow.vector;
2020

2121
import static org.junit.Assert.assertEquals;
22+
import static org.junit.Assert.assertNull;
23+
import static org.junit.Assert.assertTrue;
2224

2325
import org.apache.arrow.memory.BufferAllocator;
2426
import org.apache.arrow.memory.RootAllocator;
2527
import org.apache.arrow.vector.util.TransferPair;
2628
import org.junit.After;
29+
import org.junit.Assert;
2730
import org.junit.Before;
2831
import org.junit.Test;
2932

@@ -139,4 +142,121 @@ public void testSplitAndTransfer() throws Exception {
139142
sourceVector.close();
140143
}
141144
}
145+
146+
@Test
147+
public void testBitVector() {
148+
// Create a new value vector for 1024 integers
149+
try (final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
150+
final BitVector.Mutator m = vector.getMutator();
151+
vector.allocateNew(1024);
152+
m.setValueCount(1024);
153+
154+
// Put and set a few values
155+
m.set(0, 1);
156+
m.set(1, 0);
157+
m.set(100, 0);
158+
m.set(1022, 1);
159+
160+
m.setValueCount(1024);
161+
162+
final BitVector.Accessor accessor = vector.getAccessor();
163+
assertEquals(1, accessor.get(0));
164+
assertEquals(0, accessor.get(1));
165+
assertEquals(0, accessor.get(100));
166+
assertEquals(1, accessor.get(1022));
167+
168+
assertEquals(1022, accessor.getNullCount());
169+
170+
// test setting the same value twice
171+
m.set(0, 1);
172+
m.set(0, 1);
173+
m.set(1, 0);
174+
m.set(1, 0);
175+
assertEquals(1, accessor.get(0));
176+
assertEquals(0, accessor.get(1));
177+
178+
// test toggling the values
179+
m.set(0, 0);
180+
m.set(1, 1);
181+
assertEquals(0, accessor.get(0));
182+
assertEquals(1, accessor.get(1));
183+
184+
// should not change
185+
assertEquals(1022, accessor.getNullCount());
186+
187+
// Ensure unallocated space returns 0
188+
assertEquals(0, accessor.get(3));
189+
190+
// unset the previously set bits
191+
m.set(1, 0);
192+
m.set(1022, 0);
193+
// this should set all the array to 0
194+
assertEquals(1024, accessor.getNullCount());
195+
196+
// set all the array to 1
197+
for (int i = 0; i < 1024; ++i) {
198+
assertEquals(1024 - i, accessor.getNullCount());
199+
m.set(i, 1);
200+
}
201+
202+
assertEquals(0, accessor.getNullCount());
203+
204+
vector.allocateNew(1015);
205+
m.setValueCount(1015);
206+
207+
// ensure it has been zeroed
208+
assertEquals(1015, accessor.getNullCount());
209+
210+
m.set(0, 1);
211+
m.set(1014, 1); // ensure that the last item of the last byte is allocated
212+
213+
assertEquals(1013, accessor.getNullCount());
214+
215+
vector.zeroVector();
216+
assertEquals(1015, accessor.getNullCount());
217+
218+
// set all the array to 1
219+
for (int i = 0; i < 1015; ++i) {
220+
assertEquals(1015 - i, accessor.getNullCount());
221+
m.set(i, 1);
222+
}
223+
224+
assertEquals(0, accessor.getNullCount());
225+
}
226+
}
227+
228+
@Test
229+
public void testBitVectorRangeSetAllOnes() {
230+
validateRange(1000, 0, 1000);
231+
validateRange(1000, 0, 1);
232+
validateRange(1000, 1, 2);
233+
validateRange(1000, 5, 6);
234+
validateRange(1000, 5, 10);
235+
validateRange(1000, 5, 150);
236+
validateRange(1000, 5, 27);
237+
for (int i = 0; i < 8; i++) {
238+
for (int j = 0; j < 8; j++) {
239+
validateRange(1000, 10 + i, 27 + j);
240+
validateRange(1000, i, j);
241+
}
242+
}
243+
}
244+
245+
private void validateRange(int length, int start, int count) {
246+
String desc = "[" + start + ", " + (start + count) + ") ";
247+
try (BitVector bitVector = new BitVector("bits", allocator)) {
248+
bitVector.reset();
249+
bitVector.allocateNew(length);
250+
bitVector.getMutator().setRangeToOne(start, count);
251+
for (int i = 0; i < start; i++) {
252+
Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
253+
}
254+
for (int i = start; i < start + count; i++) {
255+
Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
256+
}
257+
for (int i = start + count; i < length; i++) {
258+
Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
259+
}
260+
}
261+
}
142262
}

‎java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
import static org.junit.Assert.assertArrayEquals;
20+
import static org.junit.Assert.assertFalse;
21+
import static org.junit.Assert.assertTrue;
22+
23+
import org.apache.arrow.memory.BufferAllocator;
24+
import org.apache.arrow.memory.RootAllocator;
25+
26+
import org.apache.arrow.vector.NullableVarCharVector;
27+
import org.apache.arrow.vector.NullableVarCharVector.Accessor;
28+
import org.apache.arrow.vector.util.TransferPair;
29+
30+
import org.junit.After;
31+
import org.junit.Before;
32+
import org.junit.Test;
33+
34+
public class TestSplitAndTransfer {
35+
36+
private BufferAllocator allocator;
37+
38+
@Before
39+
public void init() {
40+
allocator = new RootAllocator(Long.MAX_VALUE);
41+
}
42+
43+
@After
44+
public void terminate() throws Exception {
45+
allocator.close();
46+
}
47+
48+
@Test /* NullableVarCharVector */
49+
public void test() throws Exception {
50+
try(final NullableVarCharVector varCharVector = new NullableVarCharVector("myvector", allocator)) {
51+
varCharVector.allocateNew(10000, 1000);
52+
53+
final int valueCount = 500;
54+
final String[] compareArray = new String[valueCount];
55+
56+
final NullableVarCharVector.Mutator mutator = varCharVector.getMutator();
57+
for (int i = 0; i < valueCount; i += 3) {
58+
final String s = String.format("%010d", i);
59+
mutator.set(i, s.getBytes());
60+
compareArray[i] = s;
61+
}
62+
mutator.setValueCount(valueCount);
63+
64+
final TransferPair tp = varCharVector.getTransferPair(allocator);
65+
final NullableVarCharVector newVarCharVector = (NullableVarCharVector) tp.getTo();
66+
final Accessor accessor = newVarCharVector.getAccessor();
67+
final int[][] startLengths = {{0, 201}, {201, 200}, {401, 99}};
68+
69+
for (final int[] startLength : startLengths) {
70+
final int start = startLength[0];
71+
final int length = startLength[1];
72+
tp.splitAndTransfer(start, length);
73+
newVarCharVector.getMutator().setValueCount(length);
74+
for (int i = 0; i < length; i++) {
75+
final boolean expectedSet = ((start + i) % 3) == 0;
76+
if (expectedSet) {
77+
final byte[] expectedValue = compareArray[start + i].getBytes();
78+
assertFalse(accessor.isNull(i));
79+
assertArrayEquals(expectedValue, accessor.get(i));
80+
} else {
81+
assertTrue(accessor.isNull(i));
82+
}
83+
}
84+
newVarCharVector.clear();
85+
}
86+
}
87+
}
88+
}

0 commit comments

Comments
 (0)