Skip to content

Commit e379ee1

Browse files
roee88cdoron
authored and
kou
committed
ARROW-12965: [Java] C Data Interface implementation
Experimental C Data Interface support in Arrow Java for 64bit systems. Roundtrip example with dictionaries: ```java void roundtrip(FieldVector vector, DictionaryProvider provider) { // Consumer allocates empty structures try (ArrowSchema consumerArrowSchema = ArrowSchema.allocateNew(allocator); ArrowArray consumerArrowArray = ArrowArray.allocateNew(allocator)) { // Producer creates structures from existing memory pointers try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress()); ArrowArray arrowArray = ArrowArray.wrap(consumerArrowArray.memoryAddress())) { // Producer exports vector into the C Data Interface structures Data.exportVector(allocator, vector, provider, arrowArray, arrowSchema); } // Consumer imports vector (and dictionaries) try (CDataDictionaryProvider dictionaries = new CDataDictionaryProvider(); FieldVector imported = Data.importVector(allocator, consumerArrowArray, consumerArrowSchema, dictionaries)) { // Do something with the imported vector } } } ``` Closes apache#11067 from roee88/java-c-data-interface Lead-authored-by: Roee Shlomo <roee88@gmail.com> Co-authored-by: roee88 <roee88@gmail.com> Co-authored-by: Doron Chen <cdoron@il.ibm.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent e2b1dd9 commit e379ee1

41 files changed

Lines changed: 4566 additions & 8 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎ci/docker/linux-apt-jni.dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ ENV ARROW_BUILD_TESTS=OFF \
7373
ARROW_GANDIVA_JAVA=ON \
7474
ARROW_GANDIVA=ON \
7575
ARROW_HOME=/usr/local \
76+
ARROW_JAVA_CDATA=ON \
7677
ARROW_JNI=ON \
7778
ARROW_ORC=ON \
7879
ARROW_PARQUET=ON \

‎ci/scripts/java_build.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ set -ex
2121
arrow_dir=${1}
2222
source_dir=${1}/java
2323
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
24+
cdata_dist_dir=${2}/java/c
2425
with_docs=${3:-false}
2526

2627
if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then
@@ -84,6 +85,10 @@ if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
8485
${mvn} -Pshade-flatbuffers install
8586
fi
8687

88+
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
89+
${mvn} -Darrow.c.jni.dist.dir=${cdata_dist_dir} -Parrow-c-data install
90+
fi
91+
8792
if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
8893
${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
8994
fi

‎ci/scripts/java_cdata_build.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
set -ex
21+
22+
arrow_dir=${1}
23+
build_dir=${2}
24+
# The directory where the final binaries will be stored when scripts finish
25+
dist_dir=${3}
26+
27+
echo "=== Clear output directories and leftovers ==="
28+
# Clear output directories and leftovers
29+
rm -rf ${build_dir}
30+
31+
echo "=== Building Arrow Java C Data Interface native library ==="
32+
mkdir -p "${build_dir}"
33+
pushd "${build_dir}"
34+
35+
cmake \
36+
-DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-release} \
37+
-DCMAKE_INSTALL_LIBDIR=lib \
38+
-DCMAKE_INSTALL_PREFIX=${build_dir} \
39+
${arrow_dir}/java/c
40+
cmake --build . --target install --config ${ARROW_BUILD_TYPE:-release}
41+
popd
42+
43+
echo "=== Copying libraries to the distribution folder ==="
44+
mkdir -p "${dist_dir}"
45+
cp -L ${build_dir}/lib/*arrow_cdata_jni.* ${dist_dir}

ci/scripts/java_jni_build.sh renamed to ci/scripts/java_full_build.sh

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,17 @@
2020
set -e
2121

2222
arrow_dir=${1}
23-
cpp_lib_dir=${2}
24-
java_dist_dir=${3}
23+
dist_dir=${2}
2524

2625
export ARROW_TEST_DATA=${arrow_dir}/testing/data
2726

2827
pushd ${arrow_dir}/java
2928

3029
# build the entire project
31-
mvn clean install -P arrow-jni -Darrow.cpp.build.dir=$cpp_lib_dir
30+
mvn clean install -Parrow-c-data -Parrow-jni -Darrow.cpp.build.dir=$dist_dir -Darrow.c.jni.dist.dir=$dist_dir
3231

3332
# copy all jars and pom files to the distribution folder
34-
find . -name "*.jar" -exec echo {} \; -exec cp {} $java_dist_dir \;
35-
find . -name "*.pom" -exec echo {} \; -exec cp {} $java_dist_dir \;
33+
find . -name "*.jar" -exec echo {} \; -exec cp {} $dist_dir \;
34+
find . -name "*.pom" -exec echo {} \; -exec cp {} $dist_dir \;
3635

3736
popd

‎ci/scripts/java_test.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ set -ex
2121
arrow_dir=${1}
2222
source_dir=${1}/java
2323
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
24+
cdata_dist_dir=${2}/java/c
2425

2526
# For JNI and Plasma tests
2627
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
@@ -38,6 +39,10 @@ if [ "${ARROW_JNI}" = "ON" ]; then
3839
${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir}
3940
fi
4041

42+
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
43+
${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${cdata_dist_dir}
44+
fi
45+
4146
if [ "${ARROW_PLASMA}" = "ON" ]; then
4247
pushd ${source_dir}/plasma
4348
java -cp target/test-classes:target/classes \

‎dev/release/01-prepare-test.rb‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,9 @@ def test_version_pre_tag
207207
{ hunks: [["- <version>#{@snapshot_version}</version>",
208208
"+ <version>#{@release_version}</version>"]],
209209
path: "java/algorithm/pom.xml" },
210+
{ hunks: [["- <version>#{@snapshot_version}</version>",
211+
"+ <version>#{@release_version}</version>"]],
212+
path: "java/c/pom.xml" },
210213
{ hunks: [["- <version>#{@snapshot_version}</version>",
211214
"+ <version>#{@release_version}</version>"]],
212215
path: "java/compression/pom.xml" },
@@ -405,6 +408,9 @@ def test_version_post_tag
405408
{ hunks: [["- <version>#{@snapshot_version}</version>",
406409
"+ <version>#{@next_snapshot_version}</version>"]],
407410
path: "java/algorithm/pom.xml" },
411+
{ hunks: [["- <version>#{@snapshot_version}</version>",
412+
"+ <version>#{@next_snapshot_version}</version>"]],
413+
path: "java/c/pom.xml" },
408414
{ hunks: [["- <version>#{@snapshot_version}</version>",
409415
"+ <version>#{@next_snapshot_version}</version>"]],
410416
path: "java/compression/pom.xml" },

‎dev/tasks/java-jars/github.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@ jobs:
5151
steps:
5252
{{ macros.github_checkout_arrow()|indent }}
5353
{{ macros.github_install_archery()|indent }}
54+
- name: Build C Data Interface lib
55+
run: |
56+
set -e
57+
arrow/ci/scripts/java_cdata_build.sh \
58+
$GITHUB_WORKSPACE/arrow \
59+
$GITHUB_WORKSPACE/arrow/java-native-build \
60+
$GITHUB_WORKSPACE/arrow/java/dist
5461
- name: Build C++ Libs
5562
run: |
5663
set -e
@@ -86,17 +93,18 @@ jobs:
8693
tar -xvzf arrow-shared-libs-linux.tar.gz
8794
- name: Test that Shared Libraries Exist
8895
run: |
96+
test -f arrow/java/dist/libarrow_cdata_jni.dylib
8997
test -f arrow/java/dist/libarrow_dataset_jni.dylib
9098
test -f arrow/java/dist/libgandiva_jni.dylib
9199
test -f arrow/java/dist/libarrow_orc_jni.dylib
100+
test -f arrow/java/dist/libarrow_cdata_jni.so
92101
test -f arrow/java/dist/libarrow_dataset_jni.so
93102
test -f arrow/java/dist/libarrow_orc_jni.so
94103
test -f arrow/java/dist/libgandiva_jni.so
95104
- name: Build Bundled Jar
96105
run: |
97106
set -e
98-
arrow/ci/scripts/java_jni_build.sh \
107+
arrow/ci/scripts/java_full_build.sh \
99108
$GITHUB_WORKSPACE/arrow \
100-
$GITHUB_WORKSPACE/arrow/java/dist \
101109
$GITHUB_WORKSPACE/arrow/java/dist
102110
{{ macros.github_upload_releases(["arrow/java/dist/*.jar", "arrow/java/dist/*.pom"])|indent }}

‎docker-compose.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -875,6 +875,7 @@ services:
875875
- ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
876876
command:
877877
["pip install -e /arrow/dev/archery &&
878+
/arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java/dist &&
878879
/arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java/dist"]
879880

880881
############################## Integration #################################
@@ -1308,7 +1309,7 @@ services:
13081309
/arrow/ci/scripts/java_test.sh /arrow /build"
13091310

13101311
debian-java-jni:
1311-
# Includes plasma test and jni for gandiva and orc.
1312+
# Includes plasma test, jni for gandiva and orc, and C data interface.
13121313
# Usage:
13131314
# docker-compose build debian-java
13141315
# docker-compose build debian-java-jni
@@ -1332,6 +1333,7 @@ services:
13321333
command:
13331334
/bin/bash -c "
13341335
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
1336+
/arrow/ci/scripts/java_cdata_build.sh /arrow /build/java/c/build /build/java/c &&
13351337
/arrow/ci/scripts/java_build.sh /arrow /build &&
13361338
/arrow/ci/scripts/java_test.sh /arrow /build"
13371339

‎java/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ Makefile
2222
cmake_install.cmake
2323
install_manifest.txt
2424
?/
25+
!c/

‎java/c/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
build/**

0 commit comments

Comments
 (0)