Skip to content

Commit 21491ec

Browse files
igor-suhorukovigor-suhorukov
authored
ARROW-17525: [Java] Read ORC files using NativeDatasetFactory (apache#13973)
Support ORC file format in java Dataset API Authored-by: igor.suhorukov <igor.suhorukov@gmail.com> Signed-off-by: David Li <li.davidm96@gmail.com>
1 parent c586b9f commit 21491ec

5 files changed

Lines changed: 108 additions & 0 deletions

File tree

‎java/dataset/pom.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,38 @@
109109
<artifactId>jackson-databind</artifactId>
110110
<scope>test</scope>
111111
</dependency>
112+
<dependency>
113+
<groupId>org.apache.arrow.orc</groupId>
114+
<artifactId>arrow-orc</artifactId>
115+
<version>${project.version}</version>
116+
<scope>test</scope>
117+
</dependency>
118+
<dependency>
119+
<groupId>org.apache.orc</groupId>
120+
<artifactId>orc-core</artifactId>
121+
<version>1.7.6</version>
122+
<scope>test</scope>
123+
<exclusions>
124+
<exclusion>
125+
<groupId>log4j</groupId>
126+
<artifactId>log4j</artifactId>
127+
</exclusion>
128+
<exclusion>
129+
<groupId>org.slf4j</groupId>
130+
<artifactId>slf4j-log4j12</artifactId>
131+
</exclusion>
132+
<exclusion>
133+
<groupId>commons-logging</groupId>
134+
<artifactId>commons-logging</artifactId>
135+
</exclusion>
136+
</exclusions>
137+
</dependency>
138+
<dependency>
139+
<groupId>org.apache.hive</groupId>
140+
<artifactId>hive-storage-api</artifactId>
141+
<version>2.8.1</version>
142+
<scope>test</scope>
143+
</dependency>
112144
</dependencies>
113145
<build>
114146
<resources>

‎java/dataset/src/main/cpp/jni_wrapper.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@ arrow::Result<std::shared_ptr<arrow::dataset::FileFormat>> GetFileFormat(
9191
return std::make_shared<arrow::dataset::ParquetFileFormat>();
9292
case 1:
9393
return std::make_shared<arrow::dataset::IpcFileFormat>();
94+
case 2:
95+
return std::make_shared<arrow::dataset::OrcFileFormat>();
9496
default:
9597
std::string error_message =
9698
"illegal file format id: " + std::to_string(file_format_id);

‎java/dataset/src/main/java/org/apache/arrow/dataset/file/FileFormat.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
public enum FileFormat {
2424
PARQUET(0),
2525
ARROW_IPC(1),
26+
ORC(2),
2627
NONE(-1);
2728

2829
private final int id;

‎java/dataset/src/test/java/org/apache/arrow/dataset/OrcWriteSupport.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.arrow.dataset;
19+
20+
import java.io.IOException;
21+
22+
import org.apache.hadoop.conf.Configuration;
23+
import org.apache.hadoop.fs.Path;
24+
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
25+
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
26+
import org.apache.orc.OrcFile;
27+
import org.apache.orc.TypeDescription;
28+
import org.apache.orc.Writer;
29+
30+
public class OrcWriteSupport {
31+
public static void writeTempFile(TypeDescription orcSchema, Path path, Integer[] values) throws IOException {
32+
Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(new Configuration()).setSchema(orcSchema));
33+
VectorizedRowBatch batch = orcSchema.createRowBatch();
34+
LongColumnVector longColumnVector = (LongColumnVector) batch.cols[0];
35+
for (int idx = 0; idx < values.length; idx++) {
36+
longColumnVector.vector[idx] = values[idx];
37+
}
38+
batch.size = values.length;
39+
writer.addRowBatch(batch);
40+
writer.close();
41+
}
42+
}

‎java/dataset/src/test/java/org/apache/arrow/dataset/file/TestFileSystemDataset.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.concurrent.Executors;
3838
import java.util.stream.Collectors;
3939

40+
import org.apache.arrow.dataset.OrcWriteSupport;
4041
import org.apache.arrow.dataset.ParquetWriteSupport;
4142
import org.apache.arrow.dataset.jni.NativeDataset;
4243
import org.apache.arrow.dataset.jni.NativeInstanceReleasedException;
@@ -59,6 +60,8 @@
5960
import org.apache.arrow.vector.types.pojo.Schema;
6061
import org.apache.avro.generic.GenericRecord;
6162
import org.apache.avro.generic.GenericRecordBuilder;
63+
import org.apache.hadoop.fs.Path;
64+
import org.apache.orc.TypeDescription;
6265
import org.junit.Assert;
6366
import org.junit.ClassRule;
6467
import org.junit.Test;
@@ -357,6 +360,34 @@ public void testBaseArrowIpcRead() throws Exception {
357360
AutoCloseables.close(factory);
358361
}
359362

363+
@Test
364+
public void testBaseOrcRead() throws Exception {
365+
String dataName = "test-orc";
366+
String basePath = TMP.getRoot().getAbsolutePath();
367+
368+
TypeDescription orcSchema = TypeDescription.fromString("struct<ints:int>");
369+
Path path = new Path(basePath, dataName);
370+
OrcWriteSupport.writeTempFile(orcSchema, path, new Integer[]{Integer.MIN_VALUE, Integer.MAX_VALUE});
371+
372+
String orcDatasetUri = new File(basePath, dataName).toURI().toString();
373+
FileSystemDatasetFactory factory = new FileSystemDatasetFactory(rootAllocator(), NativeMemoryPool.getDefault(),
374+
FileFormat.ORC, orcDatasetUri);
375+
ScanOptions options = new ScanOptions(100);
376+
Schema schema = inferResultSchemaFromFactory(factory, options);
377+
List<ArrowRecordBatch> datum = collectResultFromFactory(factory, options);
378+
379+
assertSingleTaskProduced(factory, options);
380+
assertEquals(1, datum.size());
381+
assertEquals(1, schema.getFields().size());
382+
assertEquals("ints", schema.getFields().get(0).getName());
383+
384+
String expectedJsonUnordered = "[[2147483647], [-2147483648]]";
385+
checkParquetReadResult(schema, expectedJsonUnordered, datum);
386+
387+
AutoCloseables.close(datum);
388+
AutoCloseables.close(factory);
389+
}
390+
360391
private void checkParquetReadResult(Schema schema, String expectedJson, List<ArrowRecordBatch> actual)
361392
throws IOException {
362393
final ObjectMapper json = new ObjectMapper();

0 commit comments

Comments
 (0)