Skip to content

Commit 275b44b

Browse files
raulcdraulcd
authored
GH-50194: [C++] Move S3 and AWS-SDK to its own libarrow_s3.so (#50195)
### Rationale for this change Trying to reduce the size of `libarrow.so` and remove AWS SDK on some builds. Allow for users to plug and play based on requirements and divide our functionality into cleaner modules. ### What changes are included in this PR? Unconditionally build S3 and the AWS SDK into a different module `libarrow_s3.so` outside of `libarrow.so`. Update bindings to link against the new `libarrow_s3.so` library. Update the Linux Package jobs to have the new module into a different package. ### Are these changes tested? Yes via CI ### Are there any user-facing changes? Yes, users will need to either link against `libarrow_s3.so` or register using `LoadFileSystemFactories` * GitHub Issue: #50194 Authored-by: Raúl Cumplido <raulcumplido@gmail.com> Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
1 parent 0bd8def commit 275b44b

24 files changed

Lines changed: 312 additions & 103 deletions

‎c_glib/arrow-glib/meson.build‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,9 @@ if not gio.found()
225225
gio = dependency('gio-2.0')
226226
endif
227227
dependencies = [arrow_acero, arrow_compute, arrow, gobject, gio]
228+
if arrow_s3.found()
229+
dependencies += arrow_s3
230+
endif
228231
libarrow_glib = library(
229232
'arrow-glib',
230233
sources: sources + enums,

‎c_glib/meson.build‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,13 @@ if arrow_cpp_build_lib_dir == ''
165165
modules: ['ArrowDataset::arrow_dataset_shared'],
166166
required: false,
167167
)
168+
arrow_s3 = dependency(
169+
'arrow-s3',
170+
'ArrowS3',
171+
kwargs: common_args,
172+
modules: ['ArrowS3::arrow_s3_shared'],
173+
required: false,
174+
)
168175
arrow_flight = dependency(
169176
'arrow-flight',
170177
'ArrowFlight',
@@ -235,6 +242,11 @@ main(void)
235242
dirs: [arrow_cpp_build_lib_dir],
236243
required: false,
237244
)
245+
arrow_s3 = cpp_compiler.find_library(
246+
'arrow_s3',
247+
dirs: [arrow_cpp_build_lib_dir],
248+
required: false,
249+
)
238250
arrow_flight = cpp_compiler.find_library(
239251
'arrow_flight',
240252
dirs: [arrow_cpp_build_lib_dir],

‎ci/docker/conda-cpp.dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ ENV ARROW_ACERO=ON \
6868
ARROW_ORC=ON \
6969
ARROW_PARQUET=ON \
7070
ARROW_S3=ON \
71-
ARROW_S3_MODULE=ON \
7271
ARROW_SUBSTRAIT=ON \
7372
ARROW_USE_CCACHE=ON \
7473
ARROW_USE_MOLD=ON \

‎ci/scripts/cpp_build.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
7676
ARROW_JEMALLOC=OFF
7777
ARROW_MIMALLOC=OFF
7878
ARROW_S3=OFF
79-
ARROW_S3_MODULE=OFF
8079
ARROW_WITH_OPENTELEMETRY=OFF
8180
fi
8281

@@ -237,7 +236,6 @@ else
237236
-DARROW_PARQUET="${ARROW_PARQUET:-OFF}" \
238237
-DARROW_RUNTIME_SIMD_LEVEL="${ARROW_RUNTIME_SIMD_LEVEL:-MAX}" \
239238
-DARROW_S3="${ARROW_S3:-OFF}" \
240-
-DARROW_S3_MODULE="${ARROW_S3_MODULE:-OFF}" \
241239
-DARROW_SIMD_LEVEL="${ARROW_SIMD_LEVEL:-DEFAULT}" \
242240
-DARROW_SUBSTRAIT="${ARROW_SUBSTRAIT:-OFF}" \
243241
-DARROW_TEST_LINKAGE="${ARROW_TEST_LINKAGE:-shared}" \

‎cpp/CMakeLists.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -483,6 +483,15 @@ if(ARROW_BUILD_STATIC)
483483
endif()
484484
set(ARROW_FLIGHT_PC_REQUIRES_PRIVATE "")
485485

486+
# For arrow-s3.pc.
487+
set(ARROW_S3_PC_CFLAGS "")
488+
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
489+
if(ARROW_BUILD_STATIC)
490+
string(APPEND ARROW_S3_PC_CFLAGS_PRIVATE " -DARROW_S3_STATIC")
491+
endif()
492+
set(ARROW_S3_PC_LIBS_PRIVATE "")
493+
set(ARROW_S3_PC_REQUIRES_PRIVATE "")
494+
486495
# For arrow-substrait.pc.
487496
set(ARROW_SUBSTRAIT_PC_CFLAGS "")
488497
set(ARROW_SUBSTRAIT_PC_CFLAGS_PRIVATE "")

‎cpp/cmake_modules/DefineOptions.cmake‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -396,17 +396,11 @@ takes precedence over ccache if a storage backend is configured" ON)
396396
ARROW_JSON)
397397

398398
define_option(ARROW_S3
399-
"Build Arrow with S3 support (requires the AWS SDK for C++)"
399+
"Build Arrow S3 Module (requires the AWS SDK for C++)"
400400
OFF
401401
DEPENDS
402402
ARROW_FILESYSTEM)
403403

404-
define_option(ARROW_S3_MODULE
405-
"Build the Arrow S3 filesystem as a dynamic module"
406-
OFF
407-
DEPENDS
408-
ARROW_S3)
409-
410404
define_option(ARROW_SUBSTRAIT
411405
"Build the Arrow Substrait Consumer Module"
412406
OFF

‎cpp/cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1409,10 +1409,13 @@ endif()
14091409
# ----------------------------------------------------------------------
14101410
# cURL
14111411

1412-
macro(find_curl)
1412+
macro(find_curl ARROW_CURL_PACKAGE_PREFIX)
14131413
if(NOT TARGET CURL::libcurl)
14141414
find_package(CURL REQUIRED)
1415-
list(APPEND ARROW_SYSTEM_DEPENDENCIES CURL)
1415+
endif()
1416+
# CURL might be needed for Arrow (GCS, OpenTelemetry) or ArrowS3
1417+
if(NOT "CURL" IN_LIST ${ARROW_CURL_PACKAGE_PREFIX}_SYSTEM_DEPENDENCIES)
1418+
list(APPEND ${ARROW_CURL_PACKAGE_PREFIX}_SYSTEM_DEPENDENCIES CURL)
14161419
endif()
14171420
endmacro()
14181421

@@ -3656,7 +3659,7 @@ if(ARROW_WITH_OPENTELEMETRY)
36563659

36573660
# cURL is required whether we build from source or use an existing installation
36583661
# (OTel's cmake files do not call find_curl for you)
3659-
find_curl()
3662+
find_curl(ARROW)
36603663
resolve_dependency(opentelemetry-cpp
36613664
COMPONENTS
36623665
exporters_ostream
@@ -3795,7 +3798,7 @@ if(ARROW_WITH_GOOGLE_CLOUD_CPP)
37953798

37963799
# curl is required on all platforms. We always use system curl to
37973800
# avoid conflict.
3798-
find_curl()
3801+
find_curl(ARROW)
37993802
resolve_dependency(google_cloud_cpp_storage PC_PACKAGE_NAMES google_cloud_cpp_storage)
38003803
get_target_property(google_cloud_cpp_storage_INCLUDE_DIR google-cloud-cpp::storage
38013804
INTERFACE_INCLUDE_DIRECTORIES)
@@ -4244,11 +4247,14 @@ endfunction()
42444247

42454248
if(ARROW_S3)
42464249
if(NOT WIN32)
4247-
# This is for adding system curl dependency.
4248-
find_curl()
4250+
find_curl(ARROW_S3)
42494251
endif()
42504252
# Keep this in sync with s3fs.cc
42514253
resolve_dependency(AWSSDK
4254+
ARROW_CMAKE_PACKAGE_NAME
4255+
ArrowS3
4256+
ARROW_PC_PACKAGE_NAME
4257+
arrow-s3
42524258
HAVE_ALT
42534259
TRUE
42544260
REQUIRED_VERSION
@@ -4260,15 +4266,15 @@ if(ARROW_S3)
42604266
if(ARROW_BUILD_STATIC)
42614267
if(${AWSSDK_SOURCE} STREQUAL "SYSTEM")
42624268
foreach(AWSSDK_LINK_LIBRARY ${AWSSDK_LINK_LIBRARIES})
4263-
string(APPEND ARROW_PC_LIBS_PRIVATE " $<TARGET_FILE:${AWSSDK_LINK_LIBRARY}>")
4269+
string(APPEND ARROW_S3_PC_LIBS_PRIVATE " $<TARGET_FILE:${AWSSDK_LINK_LIBRARY}>")
42644270
endforeach()
42654271
else()
42664272
if(UNIX)
4267-
string(APPEND ARROW_PC_REQUIRES_PRIVATE " libcurl")
4273+
string(APPEND ARROW_S3_PC_REQUIRES_PRIVATE " libcurl")
42684274
endif()
4269-
string(APPEND ARROW_PC_REQUIRES_PRIVATE " openssl")
4275+
string(APPEND ARROW_S3_PC_REQUIRES_PRIVATE " openssl")
42704276
if(APPLE)
4271-
string(APPEND ARROW_PC_LIBS_PRIVATE " -framework Security")
4277+
string(APPEND ARROW_S3_PC_LIBS_PRIVATE " -framework Security")
42724278
endif()
42734279
endif()
42744280
endif()

‎cpp/src/arrow/ArrowS3Config.cmake.in‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# This config sets the following variables in your project::
19+
#
20+
# ArrowS3_FOUND - true if Arrow S3 found on the system
21+
#
22+
# This config sets the following targets in your project::
23+
#
24+
# ArrowS3::arrow_s3_shared - for linked as shared library if shared library is built
25+
# ArrowS3::arrow_s3_static - for linked as static library if static library is built
26+
27+
@PACKAGE_INIT@
28+
29+
set(ARROW_S3_SYSTEM_DEPENDENCIES "@ARROW_S3_SYSTEM_DEPENDENCIES@")
30+
31+
include(CMakeFindDependencyMacro)
32+
find_dependency(Arrow CONFIG)
33+
34+
if(ARROW_BUILD_STATIC)
35+
arrow_find_dependencies("${ARROW_S3_SYSTEM_DEPENDENCIES}")
36+
endif()
37+
38+
include("${CMAKE_CURRENT_LIST_DIR}/ArrowS3Targets.cmake")
39+
40+
arrow_keep_backward_compatibility(ArrowS3 arrow_s3)
41+
42+
check_required_components(ArrowS3)
43+
44+
arrow_show_details(ArrowS3 ARROW_S3)

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 59 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -97,22 +97,6 @@ if(ARROW_USE_GLOG)
9797
endif()
9898
endif()
9999

100-
if(ARROW_S3)
101-
if(AWSSDK_SOURCE STREQUAL "SYSTEM")
102-
list(APPEND
103-
ARROW_STATIC_INSTALL_INTERFACE_LIBS
104-
aws-cpp-sdk-identity-management
105-
aws-cpp-sdk-sts
106-
aws-cpp-sdk-cognito-identity
107-
aws-cpp-sdk-s3
108-
aws-cpp-sdk-core)
109-
elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
110-
if(UNIX)
111-
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
112-
endif()
113-
endif()
114-
endif()
115-
116100
if(ARROW_WITH_OPENTELEMETRY)
117101
if(opentelemetry-cpp_SOURCE STREQUAL "SYSTEM")
118102
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENTELEMETRY_LIBS})
@@ -983,6 +967,12 @@ if(ARROW_WITH_OPENTELEMETRY)
983967
endforeach()
984968
endif()
985969

970+
if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT)
971+
set(ARROW_VERSION_SCRIPT_FLAGS
972+
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
973+
set(ARROW_SHARED_LINK_FLAGS ${ARROW_VERSION_SCRIPT_FLAGS})
974+
endif()
975+
986976
if(ARROW_FILESYSTEM)
987977
set(ARROW_FILESYSTEM_SRCS
988978
filesystem/filesystem.cc
@@ -1018,11 +1008,6 @@ if(ARROW_FILESYSTEM)
10181008
if(ARROW_HDFS)
10191009
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc)
10201010
endif()
1021-
if(ARROW_S3)
1022-
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/s3fs.cc)
1023-
set_source_files_properties(filesystem/s3fs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION
1024-
ON)
1025-
endif()
10261011

10271012
arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
10281013
if(ARROW_AZURE)
@@ -1042,21 +1027,61 @@ if(ARROW_FILESYSTEM)
10421027
endforeach()
10431028
endif()
10441029
if(ARROW_S3)
1045-
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
1046-
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE ${AWSSDK_LINK_LIBRARIES})
1047-
endforeach()
1048-
1049-
if(ARROW_S3_MODULE)
1050-
if(NOT ARROW_BUILD_SHARED)
1051-
message(FATAL_ERROR "ARROW_S3_MODULE without shared libarrow (-DARROW_BUILD_SHARED=ON) is not supported"
1052-
)
1030+
# If libarrow_s3.a is only built, "pkg-config --cflags --libs
1031+
# arrow-s3" outputs build flags for static linking not shared
1032+
# linking. ARROW_S3_PC_* except ARROW_S3_PC_*_PRIVATE are for the
1033+
# static linking case.
1034+
if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
1035+
string(APPEND ARROW_S3_PC_CFLAGS "${ARROW_S3_PC_CFLAGS_PRIVATE}")
1036+
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
1037+
set(ARROW_S3_PC_LIBS "${ARROW_S3_PC_LIBS_PRIVATE}")
1038+
set(ARROW_S3_PC_LIBS_PRIVATE "")
1039+
set(ARROW_S3_PC_REQUIRES "${ARROW_S3_PC_REQUIRES_PRIVATE}")
1040+
set(ARROW_S3_PC_REQUIRES_PRIVATE "")
1041+
else()
1042+
set(ARROW_S3_PC_LIBS "")
1043+
set(ARROW_S3_PC_REQUIRES "")
1044+
endif()
1045+
list(APPEND ARROW_S3_LIB_SRCS filesystem/s3fs_module.cc filesystem/s3fs.cc)
1046+
set(ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
1047+
if(AWSSDK_SOURCE STREQUAL "SYSTEM")
1048+
list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS ${AWSSDK_LINK_LIBRARIES})
1049+
elseif(AWSSDK_SOURCE STREQUAL "BUNDLED")
1050+
if(UNIX)
1051+
list(APPEND ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
10531052
endif()
1054-
1055-
add_library(arrow_s3fs MODULE filesystem/s3fs_module.cc filesystem/s3fs.cc)
1056-
target_link_libraries(arrow_s3fs PRIVATE ${AWSSDK_LINK_LIBRARIES} arrow_shared)
1057-
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
1058-
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
10591053
endif()
1054+
add_arrow_lib(arrow_s3
1055+
CMAKE_PACKAGE_NAME
1056+
ArrowS3
1057+
PKG_CONFIG_NAME
1058+
arrow-s3
1059+
SOURCES
1060+
${ARROW_S3_LIB_SRCS}
1061+
SHARED_LINK_LIBS
1062+
arrow_shared
1063+
SHARED_PRIVATE_LINK_LIBS
1064+
${AWSSDK_LINK_LIBRARIES}
1065+
SHARED_INSTALL_INTERFACE_LIBS
1066+
Arrow::arrow_shared
1067+
STATIC_LINK_LIBS
1068+
arrow_static
1069+
${AWSSDK_LINK_LIBRARIES}
1070+
STATIC_INSTALL_INTERFACE_LIBS
1071+
${ARROW_S3_STATIC_INSTALL_INTERFACE_LIBS}
1072+
SHARED_LINK_FLAGS
1073+
${ARROW_VERSION_SCRIPT_FLAGS}
1074+
OUTPUTS
1075+
ARROW_S3_LIBRARIES)
1076+
foreach(LIB_TARGET ${ARROW_S3_LIBRARIES})
1077+
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_S3_EXPORTING)
1078+
endforeach()
1079+
if(ARROW_BUILD_STATIC AND WIN32)
1080+
target_compile_definitions(arrow_s3_static PUBLIC ARROW_S3_STATIC)
1081+
endif()
1082+
1083+
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
1084+
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
10601085
endif()
10611086

10621087
list(APPEND ARROW_TESTING_SHARED_LINK_LIBS ${ARROW_GTEST_GMOCK})
@@ -1122,12 +1147,6 @@ else()
11221147
set(ARROW_ORC_TARGET_STATIC)
11231148
endif()
11241149

1125-
if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT)
1126-
set(ARROW_VERSION_SCRIPT_FLAGS
1127-
"-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map")
1128-
set(ARROW_SHARED_LINK_FLAGS ${ARROW_VERSION_SCRIPT_FLAGS})
1129-
endif()
1130-
11311150
if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS)
11321151
set(ARROW_BUILD_BUNDLED_DEPENDENCIES TRUE)
11331152
else()

‎cpp/src/arrow/arrow-s3.pc.in‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
prefix=@CMAKE_INSTALL_PREFIX@
19+
includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
20+
libdir=@ARROW_PKG_CONFIG_LIBDIR@
21+
22+
Name: Apache Arrow S3
23+
Description: Apache Arrow's S3 filesystem implementation.
24+
Version: @ARROW_VERSION@
25+
Requires: arrow@ARROW_S3_PC_REQUIRES@
26+
Requires.private:@ARROW_S3_PC_REQUIRES_PRIVATE@
27+
Libs: -L${libdir} -larrow_s3@ARROW_S3_PC_LIBS@
28+
Libs.private:@ARROW_S3_PC_LIBS_PRIVATE@
29+
Cflags:@ARROW_S3_PC_CFLAGS@
30+
Cflags.private:@ARROW_S3_PC_CFLAGS_PRIVATE@

0 commit comments

Comments
 (0)