Skip to content

Commit ab8c92c

Browse files
authored
ARROW-17080: [Java] Add a top-level CMakeLists.txt for JNI (apache#13618)
Authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 9a46977 commit ab8c92c

12 files changed

Lines changed: 151 additions & 100 deletions

File tree

‎ci/scripts/java_build.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ arrow_dir=${1}
2222
source_dir=${1}/java
2323
build_dir=${2}
2424
cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug}
25-
cdata_dist_dir=${build_dir}/java/c
25+
java_jni_dist_dir=${3}
2626

2727
: ${BUILD_DOCS_JAVA:=OFF}
2828

@@ -76,7 +76,7 @@ if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
7676
fi
7777

7878
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
79-
${mvn} -Darrow.c.jni.dist.dir=${cdata_dist_dir} -Parrow-c-data install
79+
${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data install
8080
fi
8181

8282
if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
set -ex
2121

2222
arrow_dir=${1}
23-
build_dir=${2}
23+
build_dir=${2}/java_jni
2424
# The directory where the final binaries will be stored when scripts finish
2525
dist_dir=${3}
2626

@@ -34,12 +34,9 @@ pushd "${build_dir}"
3434

3535
cmake \
3636
-DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-release} \
37-
-DCMAKE_INSTALL_LIBDIR=lib \
38-
-DCMAKE_INSTALL_PREFIX=${build_dir} \
39-
${arrow_dir}/java/c
37+
-DCMAKE_INSTALL_PREFIX=${dist_dir} \
38+
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
39+
${JAVA_JNI_CMAKE_ARGS:-} \
40+
${arrow_dir}/java
4041
cmake --build . --target install --config ${ARROW_BUILD_TYPE:-release}
4142
popd
42-
43-
echo "=== Copying libraries to the distribution folder ==="
44-
mkdir -p "${dist_dir}"
45-
cp -L ${build_dir}/lib/*arrow_cdata_jni.* ${dist_dir}

‎ci/scripts/java_jni_macos_build.sh‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,30 +46,29 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
4646
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
4747
export AWS_EC2_METADATA_DISABLED=TRUE
4848

49-
mkdir -p "${build_dir}"
50-
pushd "${build_dir}"
49+
mkdir -p "${build_dir}/cpp"
50+
pushd "${build_dir}/cpp"
5151

5252
cmake \
53-
-GNinja \
5453
-DARROW_BOOST_USE_SHARED=OFF \
5554
-DARROW_BROTLI_USE_SHARED=OFF \
5655
-DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS} \
5756
-DARROW_BUILD_UTILITIES=OFF \
5857
-DARROW_BZ2_USE_SHARED=OFF \
5958
-DARROW_DATASET=${ARROW_DATASET} \
6059
-DARROW_FILESYSTEM=${ARROW_FILESYSTEM} \
60+
-DARROW_GANDIVA=${ARROW_GANDIVA} \
6161
-DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA} \
6262
-DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \
63-
-DARROW_GANDIVA=${ARROW_GANDIVA} \
6463
-DARROW_GFLAGS_USE_SHARED=OFF \
6564
-DARROW_GRPC_USE_SHARED=OFF \
6665
-DARROW_JNI=ON \
6766
-DARROW_LZ4_USE_SHARED=OFF \
6867
-DARROW_OPENSSL_USE_SHARED=OFF \
6968
-DARROW_ORC=${ARROW_ORC} \
7069
-DARROW_PARQUET=${ARROW_PARQUET} \
71-
-DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT} \
7270
-DARROW_PLASMA=${ARROW_PLASMA} \
71+
-DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT} \
7372
-DARROW_PROTOBUF_USE_SHARED=OFF \
7473
-DARROW_PYTHON=${ARROW_PYTHON} \
7574
-DARROW_SNAPPY_USE_SHARED=OFF \
@@ -78,12 +77,13 @@ cmake \
7877
-DARROW_ZSTD_USE_SHARED=OFF \
7978
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
8079
-DCMAKE_INSTALL_LIBDIR=lib \
81-
-DCMAKE_INSTALL_PREFIX=${build_dir} \
80+
-DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
8281
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
8382
-DPARQUET_BUILD_EXAMPLES=OFF \
8483
-DPARQUET_BUILD_EXECUTABLES=OFF \
8584
-DPARQUET_REQUIRE_ENCRYPTION=OFF \
8685
-Dre2_SOURCE=BUNDLED \
86+
-GNinja \
8787
${arrow_dir}/cpp
8888
cmake --build . --target install
8989

@@ -93,24 +93,33 @@ fi
9393

9494
popd
9595

96+
97+
${arrow_dir}/ci/scripts/java_jni_build.sh \
98+
${arrow_dir} \
99+
${build_dir} \
100+
${dist_dir}
101+
102+
96103
echo "=== Copying libraries to the distribution folder ==="
97104
mkdir -p "${dist_dir}"
98-
cp -L ${build_dir}/lib/libgandiva_jni.dylib ${dist_dir}
99-
cp -L ${build_dir}/lib/libarrow_dataset_jni.dylib ${dist_dir}
100-
cp -L ${build_dir}/lib/libarrow_orc_jni.dylib ${dist_dir}
105+
cp -L ${build_dir}/cpp/lib/libgandiva_jni.dylib ${dist_dir}
106+
cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.dylib ${dist_dir}
107+
cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.dylib ${dist_dir}
101108

102109
echo "=== Checking shared dependencies for libraries ==="
103110

104111
pushd ${dist_dir}
105112
archery linking check-dependencies \
113+
--allow libSystem \
114+
--allow libarrow_cdata_jni \
106115
--allow libarrow_dataset_jni \
107116
--allow libarrow_orc_jni \
108117
--allow libc++ \
109118
--allow libgandiva_jni \
110119
--allow libncurses \
111-
--allow libSystem \
112120
--allow libz \
113-
libgandiva_jni.dylib \
121+
libarrow_cdata_jni.dylib \
114122
libarrow_dataset_jni.dylib \
115-
libarrow_orc_jni.dylib
123+
libarrow_orc_jni.dylib \
124+
libgandiva_jni.dylib
116125
popd

‎ci/scripts/java_jni_manylinux_build.sh‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ export ARROW_TEST_DATA="${arrow_dir}/testing/data"
5555
export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
5656
export AWS_EC2_METADATA_DISABLED=TRUE
5757

58-
mkdir -p "${build_dir}"
59-
pushd "${build_dir}"
58+
mkdir -p "${build_dir}/cpp"
59+
pushd "${build_dir}/cpp"
6060

6161
cmake \
6262
-DARROW_BOOST_USE_SHARED=OFF \
@@ -89,7 +89,7 @@ cmake \
8989
-DARROW_ZSTD_USE_SHARED=OFF \
9090
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
9191
-DCMAKE_INSTALL_LIBDIR=lib \
92-
-DCMAKE_INSTALL_PREFIX=${build_dir} \
92+
-DCMAKE_INSTALL_PREFIX=${build_dir}/cpp \
9393
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
9494
-DORC_SOURCE=BUNDLED \
9595
-DORC_PROTOBUF_EXECUTABLE=${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/tools/protobuf/protoc \
@@ -105,20 +105,30 @@ cmake \
105105
ninja install
106106

107107
if [ $ARROW_BUILD_TESTS = "ON" ]; then
108-
ctest \
109-
--label-regex unittest \
110-
--output-on-failure \
111-
--parallel $(nproc) \
112-
--timeout 300
108+
ctest \
109+
--label-regex unittest \
110+
--output-on-failure \
111+
--parallel $(nproc) \
112+
--timeout 300
113113
fi
114114

115115
popd
116116

117+
118+
JAVA_JNI_CMAKE_ARGS=""
119+
JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_MANIFEST_MODE=OFF"
120+
JAVA_JNI_CMAKE_ARGS="${JAVA_JNI_CMAKE_ARGS} -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}"
121+
export JAVA_JNI_CMAKE_ARGS
122+
${arrow_dir}/ci/scripts/java_jni_build.sh \
123+
${arrow_dir} \
124+
${build_dir} \
125+
${dist_dir}
126+
127+
117128
echo "=== Copying libraries to the distribution folder ==="
118-
mkdir -p "${dist_dir}"
119-
cp -L ${build_dir}/lib/libgandiva_jni.so ${dist_dir}
120-
cp -L ${build_dir}/lib/libarrow_dataset_jni.so ${dist_dir}
121-
cp -L ${build_dir}/lib/libarrow_orc_jni.so ${dist_dir}
129+
cp -L ${build_dir}/cpp/lib/libgandiva_jni.so ${dist_dir}
130+
cp -L ${build_dir}/cpp/lib/libarrow_dataset_jni.so ${dist_dir}
131+
cp -L ${build_dir}/cpp/lib/libarrow_orc_jni.so ${dist_dir}
122132

123133
echo "=== Checking shared dependencies for libraries ==="
124134

@@ -134,7 +144,8 @@ archery linking check-dependencies \
134144
--allow libstdc++ \
135145
--allow libz \
136146
--allow linux-vdso \
137-
libgandiva_jni.so \
147+
libarrow_cdata_jni.so \
138148
libarrow_dataset_jni.so \
139-
libarrow_orc_jni.so
149+
libarrow_orc_jni.so \
150+
libgandiva_jni.so
140151
popd

‎ci/scripts/java_test.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ set -ex
2121
arrow_dir=${1}
2222
source_dir=${1}/java
2323
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
24-
cdata_dist_dir=${2}/java/c
24+
java_jni_dist_dir=${3}
2525

2626
# For JNI and Plasma tests
2727
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
@@ -40,7 +40,7 @@ if [ "${ARROW_JNI}" = "ON" ]; then
4040
fi
4141

4242
if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
43-
${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${cdata_dist_dir}
43+
${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${java_jni_dist_dir}
4444
fi
4545

4646
if [ "${ARROW_PLASMA}" = "ON" ]; then

‎dev/tasks/java-jars/github.yml‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
{{ macros.github_install_archery()|indent }}
3030
- name: Build C++ Libs
3131
run: archery docker run java-jni-manylinux-2014
32-
- name: Compress into single artifact
32+
- name: Compress into single artifact to keep directory structure
3333
run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/
3434
- name: Upload Artifacts
3535
uses: actions/upload-artifact@v2
@@ -55,21 +55,14 @@ jobs:
5555
run: |
5656
brew install --overwrite git
5757
brew bundle --file=arrow/cpp/Brewfile
58-
- name: Build C Data Interface lib
59-
run: |
60-
set -e
61-
arrow/ci/scripts/java_cdata_build.sh \
62-
$GITHUB_WORKSPACE/arrow \
63-
$GITHUB_WORKSPACE/arrow/java-native-build \
64-
$GITHUB_WORKSPACE/arrow/java-dist
6558
- name: Build C++ Libs
6659
run: |
6760
set -e
6861
arrow/ci/scripts/java_jni_macos_build.sh \
6962
$GITHUB_WORKSPACE/arrow \
7063
$GITHUB_WORKSPACE/arrow/cpp-build \
7164
$GITHUB_WORKSPACE/arrow/java-dist
72-
- name: Compress into single artifact
65+
- name: Compress into single artifact to keep directory structure
7366
run: tar -cvzf arrow-shared-libs-macos.tar.gz arrow/java-dist/
7467
- name: Upload Artifacts
7568
uses: actions/upload-artifact@v2
@@ -93,10 +86,11 @@ jobs:
9386
name: macos-shared-lib
9487
- name: Descompress artifacts
9588
run: |
96-
tar -xvzf arrow-shared-libs-macos.tar.gz
9789
tar -xvzf arrow-shared-libs-linux.tar.gz
90+
tar -xvzf arrow-shared-libs-macos.tar.gz
9891
- name: Test that Shared Libraries Exist
9992
run: |
93+
set -x
10094
test -f arrow/java-dist/libarrow_cdata_jni.dylib
10195
test -f arrow/java-dist/libarrow_dataset_jni.dylib
10296
test -f arrow/java-dist/libgandiva_jni.dylib

‎docker-compose.yml‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -964,7 +964,6 @@ services:
964964
- ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
965965
command:
966966
["pip install -e /arrow/dev/archery &&
967-
/arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java-dist &&
968967
/arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"]
969968

970969
############################## Integration #################################
@@ -1117,16 +1116,19 @@ services:
11171116
ARROW_ORC: "OFF"
11181117
ARROW_PARQUET: "OFF"
11191118
ARROW_PLASMA: "OFF"
1119+
JAVA_JNI_CMAKE_ARGS: >-
1120+
-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF
1121+
-DARROW_JAVA_JNI_ENABLE_C=ON
11201122
volumes:
11211123
- .:/arrow:delegated
11221124
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
11231125
- ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
11241126
command:
11251127
[ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
11261128
/arrow/ci/scripts/python_build.sh /arrow /build &&
1127-
/arrow/ci/scripts/java_cdata_build.sh /arrow /build/java/c/build /build/java/c &&
1128-
/arrow/ci/scripts/java_build.sh /arrow /build &&
1129-
/arrow/ci/scripts/java_cdata_integration.sh /arrow" ]
1129+
/arrow/ci/scripts/java_jni_build.sh /arrow /build /tmp/dist/java &&
1130+
/arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
1131+
/arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ]
11301132

11311133
conda-python-turbodbc:
11321134
# Possible $TURBODBC parameters:
@@ -1524,9 +1526,9 @@ services:
15241526
command:
15251527
/bin/bash -c "
15261528
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
1527-
/arrow/ci/scripts/java_cdata_build.sh /arrow /build/java/c/build /build/java/c &&
1528-
/arrow/ci/scripts/java_build.sh /arrow /build &&
1529-
/arrow/ci/scripts/java_test.sh /arrow /build"
1529+
/arrow/ci/scripts/java_jni_build.sh /arrow /build /tmp/java_dist &&
1530+
/arrow/ci/scripts/java_build.sh /arrow /build /tmp/java_dist &&
1531+
/arrow/ci/scripts/java_test.sh /arrow /build /tmp/java_dist"
15301532

15311533
oracle-java:
15321534
# Usage:

‎docs/source/developers/java/building.rst‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,8 @@ To build only the C Data Interface library:
9292
$ cd java-native-c
9393
$ cmake \
9494
-DCMAKE_BUILD_TYPE=Release \
95-
-DCMAKE_INSTALL_LIBDIR=lib \
96-
-DCMAKE_INSTALL_PREFIX=../java-dist \
97-
../java/c
95+
-DCMAKE_INSTALL_PREFIX=../java-dist/lib \
96+
../java
9897
$ cmake --build . --target install
9998
$ ls -latr ../java-dist/lib
10099
|__ libarrow_cdata_jni.dylib

‎java/.gitignore‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
1-
.project
1+
*.DS_Store
2+
*.iml
3+
*.lck
4+
*.log
5+
*.patch
6+
*~
27
.buildpath
3-
.classpath
48
.checkstyle
9+
.classpath
510
.factorypath
6-
.settings/
711
.idea/
8-
TAGS
9-
*.log
10-
*.lck
11-
*.iml
12-
target/
13-
*.DS_Store
14-
*.patch
15-
*~
16-
arrow-git.properties
17-
contrib/native/client/build/
18-
contrib/native/client/build/*
12+
.project
13+
.settings/
14+
/build/
15+
/*-build/
1916
CMakeCache.txt
20-
CMakeFiles
17+
CMakeFiles/
2118
Makefile
19+
TAGS
20+
arrow-git.properties
2221
cmake_install.cmake
2322
install_manifest.txt
23+
target/
2424
?/
25-
!c/
25+
!/c/

0 commit comments

Comments
 (0)