Skip to content

Commit ec66ddd

Browse files
xhochywesm
xhochy
authored and
wesm
committed
ARROW-203: Python: Basic filename based Parquet read/write
Author: Uwe L. Korn <uwelk@xhochy.com> Closes apache#83 from xhochy/arrow-203 and squashes the following commits: 405f85d [Uwe L. Korn] Remove FindParquet duplication 38d786c [Uwe L. Korn] Make code more readable by using using ec07768 [Uwe L. Korn] Set LD_LIBRARY_PATH in python build 8d90d3f [Uwe L. Korn] Do not set LD_LIBRARY_PATH in python build 000e1e3 [Uwe L. Korn] Use unique_ptr and shared_ptr from Cython 8f6010a [Uwe L. Korn] Linter fixes 0514d01 [Uwe L. Korn] Handle exceptions on RowGroupWriter::Close better 77bd21a [Uwe L. Korn] Add pandas roundtrip to tests f583b61 [Uwe L. Korn] Fix rpath for libarrow_parquet 00c1461 [Uwe L. Korn] Also ensure correct OSX compiler flags in PyArrow 4a80116 [Uwe L. Korn] Handle Python3 strings correctly 066c08a [Uwe L. Korn] Add missing functions to smart pointers 5706db2 [Uwe L. Korn] Use length and offset instead of slicing 443de8b [Uwe L. Korn] Add miniconda to the LD_LIBRARY_PATH 2dffc14 [Uwe L. Korn] Fix min mistake, use equals instead of == 2006e70 [Uwe L. Korn] Rewrite test py.test style 9520c39 [Uwe L. Korn] Use PARQUET from miniconda path cd3b9a9 [Uwe L. Korn] Also search for Parquet in PyArrow 6a41d23 [Uwe L. Korn] Re-use conda installation from C++ 81f501e [Uwe L. Korn] No need to install conda in travis_script_python anymore b505feb [Uwe L. Korn] Install parquet-cpp via conda 5d4929a [Uwe L. Korn] Add test-util.h 9b06e41 [Uwe L. Korn] Make tests templated be6415c [Uwe L. Korn] Incorportate review comments 0fbed3f [Uwe L. Korn] Remove obsolete parquet files 081db5f [Uwe L. Korn] Limit and document chunk_size 7192cfb [Uwe L. Korn] Add const to slicing parameters 0463995 [Uwe L. Korn] ARROW-203: Python: Basic filename based Parquet read/write
1 parent 8197f24 commit ec66ddd

27 files changed

Lines changed: 654 additions & 144 deletions

‎ci/travis_before_script_cpp.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
set -e
44

5+
source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
6+
conda install -y --channel apache/channel/dev parquet-cpp
7+
export PARQUET_HOME=$MINICONDA
8+
59
: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build}
610

711
mkdir $CPP_BUILD_DIR
@@ -19,7 +23,7 @@ echo $GTEST_HOME
1923

2024
: ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install}
2125

22-
CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
26+
CMAKE_COMMON_FLAGS="-DARROW_BUILD_BENCHMARKS=ON -DARROW_PARQUET=ON -DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
2327

2428
if [ $TRAVIS_OS_NAME == "linux" ]; then
2529
cmake -DARROW_TEST_MEMCHECK=on $CMAKE_COMMON_FLAGS -DCMAKE_CXX_FLAGS="-Werror" $CPP_DIR

‎ci/travis_conda_build.sh‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,7 @@
22

33
set -e
44

5-
if [ $TRAVIS_OS_NAME == "linux" ]; then
6-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
7-
else
8-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
9-
fi
10-
11-
wget -O miniconda.sh $MINICONDA_URL
12-
MINICONDA=$TRAVIS_BUILD_DIR/miniconda
13-
bash miniconda.sh -b -p $MINICONDA
14-
export PATH="$MINICONDA/bin:$PATH"
15-
conda update -y -q conda
16-
conda info -a
17-
18-
conda config --set show_channel_urls yes
19-
conda config --add channels conda-forge
20-
conda config --add channels apache
21-
22-
conda install --yes conda-build jinja2 anaconda-client
23-
24-
# faster builds, please
25-
conda install -y nomkl
5+
source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
266

277
# Build libarrow
288

‎ci/travis_install_conda.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
if [ $TRAVIS_OS_NAME == "linux" ]; then
6+
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
7+
else
8+
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
9+
fi
10+
11+
wget -O miniconda.sh $MINICONDA_URL
12+
export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
13+
bash miniconda.sh -b -p $MINICONDA
14+
export PATH="$MINICONDA/bin:$PATH"
15+
conda update -y -q conda
16+
conda info -a
17+
18+
conda config --set show_channel_urls yes
19+
conda config --add channels conda-forge
20+
conda config --add channels apache
21+
22+
conda install --yes conda-build jinja2 anaconda-client
23+
24+
# faster builds, please
25+
conda install -y nomkl
26+

‎ci/travis_script_python.sh‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,19 @@ set -e
44

55
PYTHON_DIR=$TRAVIS_BUILD_DIR/python
66

7+
# Re-use conda installation from C++
8+
export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
9+
export PATH="$MINICONDA/bin:$PATH"
10+
export LD_LIBRARY_PATH="$MINICONDA/lib:$LD_LIBRARY_PATH"
11+
export PARQUET_HOME=$MINICONDA
12+
713
# Share environment with C++
814
pushd $CPP_BUILD_DIR
915
source setup_build_env.sh
1016
popd
1117

1218
pushd $PYTHON_DIR
1319

14-
# Bootstrap a Conda Python environment
15-
16-
if [ $TRAVIS_OS_NAME == "linux" ]; then
17-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
18-
else
19-
MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh"
20-
fi
21-
22-
curl $MINICONDA_URL > miniconda.sh
23-
MINICONDA=$TRAVIS_BUILD_DIR/miniconda
24-
bash miniconda.sh -b -p $MINICONDA
25-
export PATH="$MINICONDA/bin:$PATH"
26-
conda update -y -q conda
27-
conda info -a
28-
2920
python_version_tests() {
3021
PYTHON_VERSION=$1
3122
CONDA_ENV_NAME="pyarrow-test-${PYTHON_VERSION}"

‎cpp/src/arrow/column.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ class Column {
6767

6868
int64_t null_count() const { return data_->null_count(); }
6969

70+
const std::shared_ptr<Field>& field() const { return field_; }
71+
7072
// @returns: the column's name in the passed metadata
7173
const std::string& name() const { return field_->name; }
7274

‎cpp/src/arrow/parquet/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@ add_library(arrow_parquet SHARED
3535
target_link_libraries(arrow_parquet ${PARQUET_LIBS})
3636
SET_TARGET_PROPERTIES(arrow_parquet PROPERTIES LINKER_LANGUAGE CXX)
3737

38+
if (APPLE)
39+
set_target_properties(arrow_parquet
40+
PROPERTIES
41+
BUILD_WITH_INSTALL_RPATH ON
42+
INSTALL_NAME_DIR "@rpath")
43+
endif()
44+
3845
ADD_ARROW_TEST(parquet-schema-test)
3946
ARROW_TEST_LINK_LIBRARIES(parquet-schema-test arrow_parquet)
4047

0 commit comments

Comments
 (0)