Skip to content

Commit b07027e

Browse files
ARROW-11735: [R] Allow Parquet and Arrow Dataset to be optional components
Not implemented for Windows yet (that's ARROW-11884) Closes apache#9610 from ianmcook/ARROW-11735 Lead-authored-by: Ian Cook <ianmcook@gmail.com> Co-authored-by: Neal Richardson <neal.p.richardson@gmail.com> Signed-off-by: Neal Richardson <neal.p.richardson@gmail.com>
1 parent d6ba4f8 commit b07027e

32 files changed

Lines changed: 945 additions & 219 deletions

‎dev/tasks/conda-recipes/r-arrow/configure.win‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -euxo pipefail
44

5-
echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ARROW -DARROW_R_WITH_S3" > src/Makevars.win
5+
echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ARROW -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3" > src/Makevars.win
66
echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win
77
echo 'CXX_STD=CXX11' >> src/Makevars.win
88
echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win

‎dev/tasks/r/azure.linux.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,12 @@ jobs:
5050
# we have to export this (right?) because we need it in the build env
5151
export ARROW_R_DEV={{ not_cran }}
5252
# Note that ci/scripts/r_test.sh sets NOT_CRAN=true if ARROW_R_DEV=TRUE
53-
docker-compose run r
53+
docker-compose run \
54+
-e ARROW_DATASET={{ arrow_dataset|default("") }} \
55+
-e ARROW_PARQUET={{ arrow_parquet|default("") }} \
56+
-e ARROW_S3={{ arrow_s3|default("") }} \
57+
-e LIBARROW_MINIMAL={{ libarrow_minimal|default("") }} \
58+
r
5459
displayName: Docker run
5560
5661
- script: |

‎dev/tasks/tasks.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1789,6 +1789,19 @@ tasks:
17891789
r_image: r-base
17901790
r_tag: 3.6-opensuse42
17911791
not_cran: "TRUE"
1792+
1793+
test-r-minimal-build:
1794+
ci: azure
1795+
template: r/azure.linux.yml
1796+
params:
1797+
r_org: rocker
1798+
r_image: r-base
1799+
r_tag: latest
1800+
not_cran: "TRUE"
1801+
arrow_dataset: "OFF"
1802+
arrow_parquet: "OFF"
1803+
arrow_s3: "OFF"
1804+
libarrow_minimal: "TRUE"
17921805

17931806
test-ubuntu-18.04-r-sanitizer:
17941807
ci: azure

‎r/NAMESPACE‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@ export(Type)
167167
export(UnionDataset)
168168
export(arrow_available)
169169
export(arrow_info)
170+
export(arrow_with_dataset)
171+
export(arrow_with_parquet)
170172
export(arrow_with_s3)
171173
export(binary)
172174
export(bool)

‎r/R/arrow-package.R‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,19 +53,36 @@
5353
#' You won't generally need to call these function, but they're made available
5454
#' for diagnostic purposes.
5555
#' @return `TRUE` or `FALSE` depending on whether the package was installed
56-
#' with the Arrow C++ library (check with `arrow_available()`) or with S3
57-
#' support enabled (check with `arrow_with_s3()`).
56+
#' with:
57+
#' * The Arrow C++ library (check with `arrow_available()`)
58+
#' * Arrow Dataset support enabled (check with `arrow_with_dataset()`)
59+
#' * Parquet support enabled (check with `arrow_with_parquet()`)
60+
#' * Amazon S3 support enabled (check with `arrow_with_s3()`)
5861
#' @export
5962
#' @examples
6063
#' arrow_available()
64+
#' arrow_with_dataset()
65+
#' arrow_with_parquet()
6166
#' arrow_with_s3()
62-
#' @seealso If either of these are `FALSE`, see
67+
#' @seealso If any of these are `FALSE`, see
6368
#' `vignette("install", package = "arrow")` for guidance on reinstalling the
6469
#' package.
6570
arrow_available <- function() {
6671
.Call(`_arrow_available`)
6772
}
6873

74+
#' @rdname arrow_available
75+
#' @export
76+
arrow_with_dataset <- function() {
77+
.Call(`_dataset_available`)
78+
}
79+
80+
#' @rdname arrow_available
81+
#' @export
82+
arrow_with_parquet <- function() {
83+
.Call(`_parquet_available`)
84+
}
85+
6986
#' @rdname arrow_available
7087
#' @export
7188
arrow_with_s3 <- function() {
@@ -95,6 +112,8 @@ arrow_info <- function() {
95112
pool <- default_memory_pool()
96113
out <- c(out, list(
97114
capabilities = c(
115+
dataset = arrow_with_dataset(),
116+
parquet = arrow_with_parquet(),
98117
s3 = arrow_with_s3(),
99118
vapply(tolower(names(CompressionType)[-1]), codec_is_available, logical(1))
100119
),

‎r/R/dataset-partition.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ HivePartitioning$create <- function(schm, null_fallback = NULL) {
8989
#' @return A [HivePartitioning][Partitioning], or a `HivePartitioningFactory` if
9090
#' calling `hive_partition()` with no arguments.
9191
#' @examples
92-
#' \donttest{
92+
#' \dontrun{
9393
#' hive_partition(year = int16(), month = int8())
9494
#' }
9595
#' @export

‎r/R/parquet.R‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#' @return A [arrow::Table][Table], or a `data.frame` if `as_data_frame` is
2828
#' `TRUE` (the default).
2929
#' @examples
30-
#' \donttest{
30+
#' \dontrun{
3131
#' tf <- tempfile()
3232
#' on.exit(unlink(tf))
3333
#' write_parquet(mtcars, tf)
@@ -122,7 +122,7 @@ read_parquet <- function(file,
122122
#' @return the input `x` invisibly.
123123
#'
124124
#' @examples
125-
#' \donttest{
125+
#' \dontrun{
126126
#' tf1 <- tempfile(fileext = ".parquet")
127127
#' write_parquet(data.frame(x = 1:5), tf1)
128128
#'
@@ -449,7 +449,7 @@ ParquetFileWriter$create <- function(schema,
449449
#'
450450
#' @export
451451
#' @examples
452-
#' \donttest{
452+
#' \dontrun{
453453
#' f <- system.file("v0.7.1.parquet", package="arrow")
454454
#' pq <- ParquetFileReader$create(f)
455455
#' pq$GetSchema()

‎r/configure‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,12 @@
2626
# R CMD INSTALL --configure-vars='INCLUDE_DIR=/.../include LIB_DIR=/.../lib'
2727

2828
# Library settings
29-
PKG_CONFIG_NAME="arrow parquet arrow-dataset"
29+
PKG_CONFIG_NAME="arrow"
3030
PKG_DEB_NAME="(unsuppored)"
3131
PKG_RPM_NAME="(unsuppored)"
3232
PKG_BREW_NAME="apache-arrow"
3333
PKG_TEST_HEADER="<arrow/api.h>"
34-
# These must be the same order as $(pkg-config --libs arrow-dataset)
35-
PKG_LIBS="-larrow_dataset -lparquet -larrow"
34+
PKG_LIBS="-larrow"
3635

3736
# Make some env vars case-insensitive
3837
ARROW_R_DEV=`echo $ARROW_R_DEV | tr '[:upper:]' '[:lower:]'`
@@ -69,19 +68,22 @@ fi
6968
if [ "$INCLUDE_DIR" ] || [ "$LIB_DIR" ]; then
7069
echo "*** Using INCLUDE_DIR/LIB_DIR"
7170
PKG_CFLAGS="-I$INCLUDE_DIR $PKG_CFLAGS"
72-
PKG_LIBS="-L$LIB_DIR $PKG_LIBS"
71+
PKG_DIRS="-L$LIB_DIR"
7372
else
7473
# Use pkg-config if available and allowed
7574
pkg-config --version >/dev/null 2>&1
7675
if [ "$ARROW_USE_PKG_CONFIG" != "false" ] && [ $? -eq 0 ]; then
7776
PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}`
78-
PKGCONFIG_LIBS=`pkg-config --libs --silence-errors ${PKG_CONFIG_NAME}`
77+
PKGCONFIG_LIBS=`pkg-config --libs-only-l --silence-errors ${PKG_CONFIG_NAME}`
78+
PKGCONFIG_DIRS=`pkg-config --libs-only-L --silence-errors ${PKG_CONFIG_NAME}`
79+
# TODO: what about --libs-only-other?
7980
fi
8081

8182
if [ "$PKGCONFIG_CFLAGS" ] || [ "$PKGCONFIG_LIBS" ]; then
8283
echo "*** Arrow C++ libraries found via pkg-config"
8384
PKG_CFLAGS="$PKGCONFIG_CFLAGS"
8485
PKG_LIBS=${PKGCONFIG_LIBS}
86+
PKG_DIRS=${PKGCONFIG_DIRS}
8587

8688
# Check for version mismatch
8789
PC_LIB_VERSION=`pkg-config --modversion arrow`
@@ -98,7 +100,8 @@ else
98100
if [ "$FORCE_AUTOBREW" != "true" ] && [ "`command -v brew`" ] && [ "`brew ls --versions ${PKG_BREW_NAME}`" != "" ]; then
99101
echo "*** Using Homebrew ${PKG_BREW_NAME}"
100102
BREWDIR=`brew --prefix`
101-
PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies"
103+
PKG_LIBS="$PKG_LIBS -larrow_bundled_dependencies"
104+
PKG_DIRS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_DIRS"
102105
PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
103106
else
104107
echo "*** Downloading ${PKG_BREW_NAME}"
@@ -114,7 +117,7 @@ else
114117
if [ $? -ne 0 ]; then
115118
echo "Failed to retrieve binary for ${PKG_BREW_NAME}"
116119
fi
117-
# autobrew sets `PKG_LIBS` and `PKG_CFLAGS`
120+
# autobrew sets `PKG_LIBS`, `PKG_DIRS`, and `PKG_CFLAGS`
118121
fi
119122
else
120123
# Set some default values/backwards compatibility
@@ -135,8 +138,8 @@ else
135138

136139
LIB_DIR="libarrow/arrow-${VERSION}/lib"
137140
if [ -d "$LIB_DIR" ]; then
138-
# Enumerate the static libs and add to PKG_LIBS
139-
# (technically repeating arrow libs so they're in the right order)
141+
# Enumerate the static libs, put their -l flags in BUNDLED_LIBS,
142+
# and put their -L location in PKG_DIRS
140143
#
141144
# If tools/linuxlibs.R fails to produce libs, this dir won't exist
142145
# so don't try (the error message from `ls` would be misleading)
@@ -145,7 +148,7 @@ else
145148
# TODO: what about non-bundled deps?
146149
BUNDLED_LIBS=`cd $LIB_DIR && ls *.a`
147150
BUNDLED_LIBS=`echo $BUNDLED_LIBS | sed -E "s/lib(.*)\.a/-l\1/" | sed -e "s/\\.a lib/ -l/g"`
148-
PKG_LIBS="-L$(pwd)/$LIB_DIR $PKG_LIBS $BUNDLED_LIBS"
151+
PKG_DIRS="-L$(pwd)/$LIB_DIR"
149152
fi
150153
fi
151154
fi
@@ -182,15 +185,35 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then
182185
# Always build with arrow on macOS
183186
PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_ARROW"
184187
# Check for features
185-
LIB_DIR=`echo $PKG_LIBS | sed -e 's/ -l.*//' | sed -e 's/^-L//'`
186-
grep 'set(ARROW_S3 "ON")' $LIB_DIR/cmake/arrow/ArrowOptions.cmake >/dev/null 2>&1
188+
LIB_DIR=`echo $PKG_DIRS | sed -e 's/^-L//'`
189+
ARROW_OPTS_CMAKE="$LIB_DIR/cmake/arrow/ArrowOptions.cmake"
190+
# Check for Parquet
191+
grep 'set(ARROW_PARQUET "ON")' $ARROW_OPTS_CMAKE >/dev/null 2>&1
192+
if [ $? -eq 0 ]; then
193+
PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_PARQUET"
194+
PKG_LIBS="-lparquet $PKG_LIBS"
195+
# NOTE: parquet is assumed to have the same -L flag as arrow
196+
# so there is no need to add its location to PKG_DIRS
197+
fi
198+
# Check for Arrow Dataset subcomponent
199+
grep 'set(ARROW_DATASET "ON")' $ARROW_OPTS_CMAKE >/dev/null 2>&1
200+
if [ $? -eq 0 ]; then
201+
PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_DATASET"
202+
PKG_LIBS="-larrow_dataset $PKG_LIBS"
203+
# NOTE: arrow-dataset is assumed to have the same -L flag as arrow
204+
# so there is no need to add its location to PKG_DIRS
205+
fi
206+
# Check for S3
207+
grep 'set(ARROW_S3 "ON")' $ARROW_OPTS_CMAKE >/dev/null 2>&1
187208
if [ $? -eq 0 ]; then
188209
PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_S3"
189210
if [ "$BUNDLED_LIBS" != "" ]; then
190211
# We're depending on openssl/curl from the system, so they're not in the bundled deps
191-
PKG_LIBS="$PKG_LIBS -lssl -lcrypto -lcurl"
212+
BUNDLED_LIBS="$BUNDLED_LIBS -lssl -lcrypto -lcurl"
192213
fi
193214
fi
215+
# prepend PKG_DIRS and append BUNDLED_LIBS to PKG_LIBS
216+
PKG_LIBS="$PKG_DIRS $PKG_LIBS $BUNDLED_LIBS"
194217
echo "PKG_CFLAGS=$PKG_CFLAGS"
195218
echo "PKG_LIBS=$PKG_LIBS"
196219
else

‎r/configure.win‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-man
4949

5050
# NOTE: If you make changes to the libraries below, you should also change
5151
# ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD
52-
PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW"
52+
PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET"
5353
PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lutf8proc -lre2 -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}"
5454

5555
# S3 support only for Rtools40 (i.e. R >= 4.0)

‎r/data-raw/codegen.R‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
# #if defined(ARROW_R_WITH_FEATURE)
4242
# and each feature is written to its own set of export files.
4343

44-
features <- c("arrow", "s3")
44+
features <- c("arrow", "dataset", "parquet", "s3")
4545

4646
suppressPackageStartupMessages({
4747
library(decor)
@@ -187,27 +187,31 @@ cpp_file_header <- '// Generated by using data-raw/codegen.R -> do not edit by h
187187
#include "./arrow_types.h"
188188
'
189189

190-
arrow_exports_cpp <- glue::glue('
190+
arrow_exports_cpp <- paste0(
191+
glue::glue('
191192
{cpp_file_header}
192193
{cpp_functions_definitions}
193194
{cpp_classes_finalizers}
194-
195-
{feature_available("arrow")}
196-
197-
{feature_available("s3")}
198-
199-
static const R_CallMethodDef CallEntries[] = {{
200-
\t\t{{ "_arrow_available", (DL_FUNC)& _arrow_available, 0 }},
201-
\t\t{{ "_s3_available", (DL_FUNC)& _s3_available, 0 }},
195+
\n'),
196+
glue::glue_collapse(glue::glue('
197+
{feature_available({features})}
198+
'), sep = '\n'),
199+
'
200+
static const R_CallMethodDef CallEntries[] = {
201+
',
202+
glue::glue_collapse(glue::glue('
203+
\t\t{{ "_{features}_available", (DL_FUNC)& _{features}_available, 0 }},
204+
'), sep = '\n'),
205+
glue::glue('\n
202206
{cpp_functions_registration}
203207
{classes_finalizers_registration}
204208
\t\t{{NULL, NULL, 0}}
205209
}};
206-
207-
extern "C" void R_init_arrow(DllInfo* dll){{
210+
\n'),
211+
'extern "C" void R_init_arrow(DllInfo* dll){
208212
R_registerRoutines(dll, NULL, CallEntries, NULL, NULL);
209213
R_useDynamicSymbols(dll, FALSE);
210-
}}
214+
}
211215
\n')
212216

213217
write_if_modified(arrow_exports_cpp, "src/arrowExports.cpp")

0 commit comments

Comments
 (0)