Skip to content

Commit 5d88f10

Browse files
liyafan82emkornfield
liyafan82
authored and
emkornfield
committed
ARROW-8402: [Java] Support ValidateFull methods in Java
In this PR, we provide the validateFull functionalities, and refactor the validate logic. We separate the validation logic into 3 visitors: 1. Type validation: validate vector type information (O(1) time complexity). 2. Buffer validation: validate buffer count/size, etc. (O(1) time complexity). 3. Data validation: validate content of data buffers. (O(n) time complexity). For the `validate` method, we perform validations 1 and 2, with a total time complexity of O(1). For the `validateFull` method, we perform validations 1, 2, and 3, with a total time complexity of O(n). Closes apache#7248 from liyafan82/fly_0515_full Authored-by: liyafan82 <fan_li_ya@foxmail.com> Signed-off-by: Micah Kornfield <emkornfield@gmail.com>
1 parent 0d0a0cf commit 5d88f10

13 files changed

Lines changed: 1604 additions & 61 deletions

‎java/vector/src/main/codegen/templates/DenseUnionVector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ public class DenseUnionVector implements FieldVector {
122122

123123
private final FieldType fieldType;
124124

125-
private static final byte TYPE_WIDTH = 1;
125+
public static final byte TYPE_WIDTH = 1;
126126
public static final byte OFFSET_WIDTH = 4;
127127

128128
private static final FieldType INTERNAL_STRUCT_TYPE = new FieldType(/*nullable*/ false,

‎java/vector/src/main/codegen/templates/UnionVector.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public class UnionVector implements FieldVector {
108108
private final FieldType fieldType;
109109
private final Field[] typeIds = new Field[Byte.MAX_VALUE + 1];
110110

111-
private static final byte TYPE_WIDTH = 1;
111+
public static final byte TYPE_WIDTH = 1;
112112
private static final FieldType INTERNAL_STRUCT_TYPE = new FieldType(false /*nullable*/,
113113
ArrowType.Struct.INSTANCE, null /*dictionary*/, null /*metadata*/);
114114

‎java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
public final class DurationVector extends BaseFixedWidthVector {
4545
private static final byte TYPE_WIDTH = 8;
4646
private final FieldReader reader;
47+
4748
private final TimeUnit unit;
4849

4950
/**
@@ -193,6 +194,13 @@ private StringBuilder getAsStringBuilderHelper(int index) {
193194
return new StringBuilder(getObject(index).toString());
194195
}
195196

197+
/**
198+
* Gets the time unit of the duration.
199+
*/
200+
public TimeUnit getUnit() {
201+
return unit;
202+
}
203+
196204
/*----------------------------------------------------------------*
197205
| |
198206
| vector value setter methods |

‎java/vector/src/main/java/org/apache/arrow/vector/util/ValueVectorUtility.java‎

Lines changed: 49 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,24 @@
1717

1818
package org.apache.arrow.vector.util;
1919

20-
import org.apache.arrow.memory.ArrowBuf;
20+
import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow;
21+
2122
import org.apache.arrow.util.Preconditions;
2223
import org.apache.arrow.vector.BaseFixedWidthVector;
23-
import org.apache.arrow.vector.BufferLayout;
24-
import org.apache.arrow.vector.FieldVector;
25-
import org.apache.arrow.vector.TypeLayout;
2624
import org.apache.arrow.vector.ValueVector;
2725
import org.apache.arrow.vector.VectorSchemaRoot;
28-
import org.apache.arrow.vector.types.pojo.ArrowType;
29-
import org.apache.arrow.vector.validate.ValidateVectorVisitor;
26+
import org.apache.arrow.vector.validate.ValidateVectorBufferVisitor;
27+
import org.apache.arrow.vector.validate.ValidateVectorDataVisitor;
28+
import org.apache.arrow.vector.validate.ValidateVectorTypeVisitor;
3029

3130
/**
3231
* Utility methods for {@link ValueVector}.
3332
*/
3433
public class ValueVectorUtility {
3534

35+
private ValueVectorUtility() {
36+
}
37+
3638
/**
3739
* Get the toString() representation of vector suitable for debugging.
3840
* Note since vectors may have millions of values, this method only show max 20 values.
@@ -92,44 +94,57 @@ public static String getToString(ValueVector vector, int start, int end) {
9294
}
9395

9496
/**
95-
* Validate field vector.
97+
* Utility to validate vector in O(1) time.
9698
*/
97-
public static void validate(FieldVector vector) {
99+
public static void validate(ValueVector vector) {
98100
Preconditions.checkNotNull(vector);
99101

100-
ArrowType arrowType = vector.getField().getType();
101-
int typeBufferCount = TypeLayout.getTypeBufferCount(arrowType);
102-
TypeLayout typeLayout = TypeLayout.getTypeLayout(arrowType);
102+
ValidateVectorTypeVisitor typeVisitor = new ValidateVectorTypeVisitor();
103+
vector.accept(typeVisitor, null);
103104

104-
if (vector.getValueCount() < 0) {
105-
throw new IllegalArgumentException("vector valueCount is negative");
106-
}
105+
ValidateVectorBufferVisitor bufferVisitor = new ValidateVectorBufferVisitor();
106+
vector.accept(bufferVisitor, null);
107+
}
107108

108-
if (vector.getFieldBuffers().size() != typeBufferCount) {
109-
throw new IllegalArgumentException(String.format("Expected %s buffers in vector of type %s, got %s",
110-
typeBufferCount, vector.getField().getType().toString(), vector.getBufferSize()));
111-
}
109+
/**
110+
* Utility to validate vector in O(n) time, where n is the value count.
111+
*/
112+
public static void validateFull(ValueVector vector) {
113+
validate(vector);
112114

113-
for (int i = 0; i < typeBufferCount; i++) {
114-
ArrowBuf buffer = vector.getFieldBuffers().get(i);
115-
BufferLayout bufferLayout = typeLayout.getBufferLayouts().get(i);
116-
if (buffer == null) {
117-
continue;
118-
}
119-
int minBufferSize = vector.getValueCount() * bufferLayout.getTypeBitWidth();
115+
ValidateVectorDataVisitor dataVisitor = new ValidateVectorDataVisitor();
116+
vector.accept(dataVisitor, null);
117+
}
120118

121-
if (buffer.capacity() < minBufferSize / 8) {
122-
throw new IllegalArgumentException(String.format("Buffer #%s too small in vector of type %s" +
123-
"and valueCount %s : expected at least %s byte(s), got %s",
124-
i, vector.getField().getType().toString(),
125-
vector.getValueCount(), minBufferSize, buffer.capacity()));
126-
}
119+
/**
120+
* Utility to validate vector schema root in O(1) time.
121+
*/
122+
public static void validate(VectorSchemaRoot root) {
123+
Preconditions.checkNotNull(root);
124+
int valueCount = root.getRowCount();
125+
validateOrThrow(valueCount >= 0, "The row count of vector schema root %s is negative.", valueCount);
126+
for (ValueVector childVec : root.getFieldVectors()) {
127+
validateOrThrow(valueCount == childVec.getValueCount(),
128+
"Child vector and vector schema root have different value counts. " +
129+
"Child vector value count %s, vector schema root value count %s", childVec.getValueCount(), valueCount);
130+
validate(childVec);
127131
}
128-
129-
ValidateVectorVisitor visitor = new ValidateVectorVisitor();
130-
vector.accept(visitor, null);
131132
}
132133

134+
/**
135+
* Utility to validate vector in O(n) time, where n is the value count.
136+
*/
137+
public static void validateFull(VectorSchemaRoot root) {
138+
Preconditions.checkNotNull(root);
139+
int valueCount = root.getRowCount();
140+
validateOrThrow(valueCount >= 0, "The row count of vector schema root %s is negative.", valueCount);
141+
for (ValueVector childVec : root.getFieldVectors()) {
142+
validateOrThrow(valueCount == childVec.getValueCount(),
143+
"Child vector and vector schema root have different value counts. " +
144+
"Child vector value count %s, vector schema root value count %s", childVec.getValueCount(), valueCount);
145+
validateFull(childVec);
146+
}
147+
}
133148

134149
/**
135150
* Pre allocate memory for BaseFixedWidthVector.
@@ -154,5 +169,4 @@ public static void ensureCapacity(VectorSchemaRoot root, int targetCapacity) {
154169
}
155170
}
156171
}
157-
158172
}

‎java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.arrow.vector.validate;
19+
20+
/**
21+
* Utilities for vector validation.
22+
*/
23+
public class ValidateUtil {
24+
25+
private ValidateUtil() {
26+
}
27+
28+
/**
29+
* Validate the expression.
30+
* @param expression the expression to validate.
31+
* @param errorMessage the error message.
32+
* @throws ValidateException if the expression evaluates to false.
33+
*/
34+
public static void validateOrThrow(boolean expression, String errorMessage) {
35+
if (!expression) {
36+
throw new ValidateException(errorMessage);
37+
}
38+
}
39+
40+
/**
41+
* Validate the expression.
42+
* @param expression the expression to validate.
43+
* @param errorMessage the error message template.
44+
* @param args the error message arguments.
45+
* @throws ValidateException if the expression evaluates to false.
46+
*/
47+
public static void validateOrThrow(boolean expression, String errorMessage, Object... args) {
48+
if (!expression) {
49+
throw new ValidateException(String.format(errorMessage, args));
50+
}
51+
}
52+
53+
/**
54+
* A exception that is thrown when the vector validation fails.
55+
*/
56+
public static class ValidateException extends RuntimeException {
57+
public ValidateException(String message) {
58+
super(message);
59+
}
60+
}
61+
}

0 commit comments

Comments
 (0)