Skip to content

Commit b832853

Browse files
AlenkaFjorisvandenbossche
AlenkaF
and
jorisvandenbossche
authored
ARROW-16340: [C++][Python] Move all Python related code into PyArrow (apache#13311)
This PR moves `src/arrow/python` directory into `pyarrow` and arranges PyArrow to build it. The build on the Python side is made in two steps: 1. `_run_cmake_pyarrow_cpp()` where the C++ part of the pyarrow is build first (the part that was moved in the refactoring) 2. `_run_cmake()` where pyarrow is built as before No changes are needed in the build process from the user side to successfully build pyarrow after this refactoring. The test for PyArrow CPP will however be moved into Cython and can currently be run with: ```shell >>> pushd python/build/dist/temp >>> ctest ``` Lead-authored-by: Alenka Frim <frim.alenka@gmail.com> Co-authored-by: Joris Van den Bossche <jorisvandenbossche@gmail.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 7e7b8e1 commit b832853

103 files changed

Lines changed: 862 additions & 650 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎ci/scripts/python_test.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
set -ex
2121

2222
arrow_dir=${1}
23+
test_dir=${1}/python/build/dist
2324

2425
export ARROW_SOURCE_DIR=${arrow_dir}
2526
export ARROW_TEST_DATA=${arrow_dir}/testing/data
@@ -54,4 +55,14 @@ export PYARROW_TEST_ORC
5455
export PYARROW_TEST_PARQUET
5556
export PYARROW_TEST_S3
5657

58+
# Testing PyArrow C++
59+
if [ "${ARROW_BUILD_TESTS}" == "ON" ]; then
60+
pushd ${test_dir}
61+
ctest \
62+
--output-on-failure \
63+
--parallel ${n_jobs} \
64+
--timeout 300
65+
popd
66+
fi
67+
# Testing PyArrow
5768
pytest -r s ${PYTEST_ARGS} --pyargs pyarrow

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ export PYARROW_WITH_PLASMA=${ARROW_PLASMA}
156156
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
157157
export PYARROW_WITH_S3=${ARROW_S3}
158158
export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"
159+
export ARROW_HOME=${build_dir}/install
159160
# PyArrow build configuration
160161
export PKG_CONFIG_PATH=/usr/lib/pkgconfig:${build_dir}/install/lib/pkgconfig
161162
# Set PyArrow version explicitly

‎ci/scripts/python_wheel_manylinux_build.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
151151
export PYARROW_WITH_PLASMA=${ARROW_PLASMA}
152152
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
153153
export PYARROW_WITH_S3=${ARROW_S3}
154+
export ARROW_HOME=/tmp/arrow-dist
154155
# PyArrow build configuration
155156
export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/tmp/arrow-dist/lib/pkgconfig
156157

‎cpp/cmake_modules/FindArrowPython.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ find_package(Arrow ${find_package_arguments})
4747

4848
if(ARROW_FOUND)
4949
arrow_find_package(ARROW_PYTHON
50-
"${ARROW_HOME}"
50+
"${PYARROW_CPP_HOME}"
5151
arrow_python
5252
arrow/python/api.h
5353
ArrowPython

‎cpp/cmake_modules/FindArrowPythonFlight.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ find_package(ArrowPython ${find_package_arguments})
5050

5151
if(ARROW_PYTHON_FOUND AND ARROW_FLIGHT_FOUND)
5252
arrow_find_package(ARROW_PYTHON_FLIGHT
53-
"${ARROW_HOME}"
53+
"${PYARROW_CPP_HOME}"
5454
arrow_python_flight
5555
arrow/python/flight.h
5656
ArrowPythonFlight

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -805,10 +805,6 @@ if(ARROW_ORC)
805805
add_subdirectory(adapters/orc)
806806
endif()
807807

808-
if(ARROW_PYTHON)
809-
add_subdirectory(python)
810-
endif()
811-
812808
if(ARROW_TENSORFLOW)
813809
add_subdirectory(adapters/tensorflow)
814810
endif()

‎cpp/src/arrow/public_api_test.cc‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,6 @@
5050
#include "arrow/json/api.h" // IWYU pragma: keep
5151
#endif
5252

53-
#ifdef ARROW_PYTHON
54-
#include "arrow/python/api.h" // IWYU pragma: keep
55-
#endif
56-
5753
#ifdef DCHECK
5854
#error "DCHECK should not be visible from Arrow public headers."
5955
#endif

‎cpp/src/arrow/python/CMakeLists.txt‎

Lines changed: 0 additions & 208 deletions
This file was deleted.

‎dev/release/01-prepare-test.rb‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,13 @@ def test_version_pre_tag
197197
"+set(MLARROW_VERSION \"#{@release_version}\")"],
198198
],
199199
},
200+
{
201+
path: "python/pyarrow/src/CMakeLists.txt",
202+
hunks: [
203+
["-set(ARROW_PYTHON_VERSION \"#{@snapshot_version}\")",
204+
"+set(ARROW_PYTHON_VERSION \"#{@release_version}\")"],
205+
],
206+
},
200207
{
201208
path: "python/setup.py",
202209
hunks: [

‎dev/release/post-11-bump-versions-test.rb‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@ def test_version_post_tag
144144
"+set(MLARROW_VERSION \"#{@next_snapshot_version}\")"],
145145
],
146146
},
147+
{
148+
path: "python/pyarrow/src/CMakeLists.txt",
149+
hunks: [
150+
["-set(ARROW_PYTHON_VERSION \"#{@snapshot_version}\")",
151+
"+set(ARROW_PYTHON_VERSION \"#{@next_snapshot_version}\")"],
152+
],
153+
},
147154
{
148155
path: "python/setup.py",
149156
hunks: [

0 commit comments

Comments
 (0)