Skip to content

Commit ef90830

Browse files
wesmwesm
committed
ARROW-222: Prototyping an IO interface for Arrow, with initial HDFS target
- Switch Travis CI back to Ubuntu trusty (old Boost in precise has issues with C++11) - Adapt SFrame libhdfs shim for arrow - Create C++ public API within arrow:io to libhdfs - Implement and test many functions in libhdfs - Start Cython wrapper interface to arrow_io. Begin Python file-like interface, unit tests - Add thirdparty hdfs.h so builds are possible without a local Hadoop distro (e.g. in Travis CI). Change-Id: I4a46e50f6c1c22787baa3749d8a542216341e630
1 parent f7ade7b commit ef90830

26 files changed

Lines changed: 3656 additions & 61 deletions

‎.travis.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
sudo: required
2-
dist: precise
2+
dist: trusty
33
addons:
44
apt:
55
sources:
@@ -12,6 +12,9 @@ addons:
1212
- ccache
1313
- cmake
1414
- valgrind
15+
- libboost-dev
16+
- libboost-filesystem-dev
17+
- libboost-system-dev
1518

1619
matrix:
1720
fast_finish: true

‎NOTICE.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Apache Arrow
2+
Copyright 2016 The Apache Software Foundation
3+
4+
This product includes software developed at
5+
The Apache Software Foundation (http://www.apache.org/).
6+
7+
This product includes software from the SFrame project (BSD, 3-clause).
8+
* Copyright (C) 2015 Dato, Inc.
9+
* Copyright (c) 2009 Carnegie Mellon University.

‎ci/travis_before_script_cpp.sh‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,21 @@ echo $GTEST_HOME
2323

2424
: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
2525

26-
CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DARROW_PARQUET=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
26+
CMAKE_COMMON_FLAGS="\
27+
-DARROW_BUILD_BENCHMARKS=ON \
28+
-DARROW_PARQUET=ON \
29+
-DARROW_HDFS=on \
30+
-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
2731

2832
if [ $TRAVIS_OS_NAME == "linux" ]; then
29-
cmake -DARROW_TEST_MEMCHECK=on $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
33+
cmake -DARROW_TEST_MEMCHECK=on \
34+
$CMAKE_COMMON_FLAGS \
35+
-DCMAKE_CXX_FLAGS="-Werror" \
36+
$CPP_DIR
3037
else
31-
cmake $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR
38+
cmake $CMAKE_COMMON_FLAGS \
39+
-DCMAKE_CXX_FLAGS="-Werror" \
40+
$CPP_DIR
3241
fi
3342

3443
make -j4

‎cpp/CMakeLists.txt‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
6262
"Build the Arrow IPC extensions"
6363
ON)
6464

65+
option(ARROW_HDFS
66+
"Build the Arrow IO extensions for the Hadoop file system"
67+
OFF)
68+
6569
option(ARROW_SSE3
6670
"Build Arrow with SSE3"
6771
ON)
@@ -454,6 +458,47 @@ if ("$ENV{GBENCHMARK_HOME}" STREQUAL "")
454458
set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed)
455459
endif()
456460

461+
# ----------------------------------------------------------------------
462+
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
463+
464+
# find boost headers and libs
465+
set(Boost_DEBUG TRUE)
466+
set(Boost_USE_MULTITHREADED ON)
467+
set(Boost_USE_STATIC_LIBS ON)
468+
find_package(Boost COMPONENTS system filesystem REQUIRED)
469+
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
470+
set(BOOST_STATIC_LIBS ${Boost_LIBRARIES})
471+
list(LENGTH BOOST_STATIC_LIBS BOOST_STATIC_LIBS_LEN)
472+
473+
# Find Boost shared libraries.
474+
set(Boost_USE_STATIC_LIBS OFF)
475+
find_package(Boost COMPONENTS system filesystem REQUIRED)
476+
set(BOOST_SHARED_LIBS ${Boost_LIBRARIES})
477+
list(LENGTH BOOST_SHARED_LIBS BOOST_SHARED_LIBS_LEN)
478+
list(SORT BOOST_SHARED_LIBS)
479+
480+
message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS})
481+
message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
482+
483+
math(EXPR LAST_IDX "${BOOST_STATIC_LIBS_LEN} - 1")
484+
foreach(IDX RANGE ${LAST_IDX})
485+
list(GET BOOST_STATIC_LIBS ${IDX} BOOST_STATIC_LIB)
486+
list(GET BOOST_SHARED_LIBS ${IDX} BOOST_SHARED_LIB)
487+
488+
# Remove the prefix/suffix from the library name.
489+
#
490+
# e.g. libboost_system-mt --> boost_system
491+
get_filename_component(LIB_NAME ${BOOST_STATIC_LIB} NAME_WE)
492+
string(REGEX REPLACE "lib([^-]*)(-mt)?" "\\1" LIB_NAME_NO_PREFIX_SUFFIX ${LIB_NAME})
493+
ADD_THIRDPARTY_LIB(${LIB_NAME_NO_PREFIX_SUFFIX}
494+
STATIC_LIB "${BOOST_STATIC_LIB}"
495+
SHARED_LIB "${BOOST_SHARED_LIB}")
496+
list(APPEND ARROW_BOOST_LIBS ${LIB_NAME_NO_PREFIX_SUFFIX})
497+
endforeach()
498+
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
499+
500+
# ----------------------------------------------------------------------
501+
# Enable / disable tests and benchmarks
457502

458503
if(ARROW_BUILD_TESTS)
459504
add_custom_target(unittest ctest -L unittest)
@@ -529,12 +574,24 @@ endif (UNIX)
529574
# "make lint" target
530575
############################################################
531576
if (UNIX)
577+
578+
file(GLOB_RECURSE LINT_FILES
579+
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
580+
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
581+
)
582+
583+
FOREACH(item ${LINT_FILES})
584+
IF(NOT (item MATCHES "_generated.h"))
585+
LIST(APPEND FILTERED_LINT_FILES ${item})
586+
ENDIF()
587+
ENDFOREACH(item ${LINT_FILES})
588+
532589
# Full lint
533590
add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py
534591
--verbose=2
535592
--linelength=90
536593
--filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/c++11,-runtime/references
537-
`find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/_generated/g'`)
594+
${FILTERED_LINT_FILES})
538595
endif (UNIX)
539596
540597
@@ -624,6 +681,7 @@ set_target_properties(arrow
624681
target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
625682
626683
add_subdirectory(src/arrow)
684+
add_subdirectory(src/arrow/io)
627685
add_subdirectory(src/arrow/util)
628686
add_subdirectory(src/arrow/types)
629687

‎cpp/doc/HDFS.md‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
## Using Arrow's HDFS (Apache Hadoop Distributed File System) interface
2+
3+
### Build requirements
4+
5+
To build the integration, pass the following option to CMake
6+
7+
```shell
8+
-DARROW_HDFS=on
9+
```
10+
11+
For convenience, we have bundled `hdfs.h` for libhdfs from Apache Hadoop in
12+
Arrow's thirdparty. If you wish to build against the `hdfs.h` in your installed
13+
Hadoop distribution, set the `$HADOOP_HOME` environment variable.
14+
15+
### Runtime requirements
16+
17+
By default, the HDFS client C++ class in `libarrow_io` uses the libhdfs JNI
18+
interface to the Java Hadoop client. This library is loaded **at runtime**
19+
(rather than at link / library load time, since the library may not be in your
20+
LD_LIBRARY_PATH), and relies on some environment variables.
21+
22+
* `HADOOP_HOME`: the root of your installed Hadoop distribution. Check in the
23+
`lib/native` directory to look for `libhdfs.so` if you have any questions
24+
about which directory you're after.
25+
* `JAVA_HOME`: the location of your Java SDK installation
26+
* `CLASSPATH`: must contain the Hadoop jars. You can set these using:
27+
28+
```shell
29+
export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
30+
```
31+
32+
#### Setting $JAVA_HOME automatically on OS X
33+
34+
The installed location of Java on OS X can vary, however the following snippet
35+
will set it automatically for you:
36+
37+
```shell
38+
export JAVA_HOME=$(/usr/libexec/java_home)
39+
```

‎cpp/src/arrow/io/CMakeLists.txt‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# ----------------------------------------------------------------------
19+
# arrow_io : Arrow IO interfaces
20+
21+
set(ARROW_IO_LINK_LIBS
22+
arrow
23+
)
24+
25+
set(ARROW_IO_PRIVATE_LINK_LIBS
26+
boost_system
27+
boost_filesystem
28+
)
29+
30+
set(ARROW_IO_TEST_LINK_LIBS
31+
arrow_io
32+
${ARROW_IO_PRIVATE_LINK_LIBS})
33+
34+
set(ARROW_IO_SRCS
35+
)
36+
37+
if(ARROW_HDFS)
38+
if(NOT THIRDPARTY_DIR)
39+
message(FATAL_ERROR "THIRDPARTY_DIR not set")
40+
endif()
41+
42+
if (DEFINED ENV{HADOOP_HOME})
43+
set(HADOOP_HOME $ENV{HADOOP_HOME})
44+
else()
45+
set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop")
46+
endif()
47+
48+
set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h")
49+
if (NOT EXISTS ${HDFS_H_PATH})
50+
message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}")
51+
endif()
52+
message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH})
53+
message(STATUS "Building libhdfs shim component")
54+
55+
include_directories(SYSTEM "${HADOOP_HOME}/include")
56+
57+
set(ARROW_HDFS_SRCS
58+
hdfs.cc
59+
libhdfs_shim.cc)
60+
61+
set_property(SOURCE ${ARROW_HDFS_SRCS}
62+
APPEND_STRING PROPERTY
63+
COMPILE_FLAGS "-DHAS_HADOOP")
64+
65+
set(ARROW_IO_SRCS
66+
${ARROW_HDFS_SRCS}
67+
${ARROW_IO_SRCS})
68+
69+
ADD_ARROW_TEST(hdfs-io-test)
70+
ARROW_TEST_LINK_LIBRARIES(hdfs-io-test
71+
${ARROW_IO_TEST_LINK_LIBS})
72+
endif()
73+
74+
add_library(arrow_io SHARED
75+
${ARROW_IO_SRCS}
76+
)
77+
target_link_libraries(arrow_io LINK_PUBLIC ${ARROW_IO_LINK_LIBS})
78+
target_link_libraries(arrow_io LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS})
79+
80+
SET_TARGET_PROPERTIES(arrow_io PROPERTIES LINKER_LANGUAGE CXX)
81+
82+
if (APPLE)
83+
set_target_properties(arrow_io
84+
PROPERTIES
85+
BUILD_WITH_INSTALL_RPATH ON
86+
INSTALL_NAME_DIR "@rpath")
87+
endif()
88+
89+
# Headers: top level
90+
install(FILES
91+
hdfs.h
92+
interfaces.h
93+
DESTINATION include/arrow/io)
94+
95+
install(TARGETS arrow_io
96+
LIBRARY DESTINATION lib
97+
ARCHIVE DESTINATION lib)

0 commit comments

Comments
 (0)