Skip to content

Commit e21d5aa

Browse files
raulcdraulcd
authored
ARROW-17487: [Python][Packaging][CI] Add support for Python 3.11 (apache#14499)
This PR adds jobs to build pyarrow wheels for Python 3.11. Authored-by: Raúl Cumplido <raulcumplido@gmail.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 8e3a1e1 commit e21d5aa

12 files changed

Lines changed: 40 additions & 118 deletions

‎ci/docker/python-wheel-windows-test-vs2017.dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
3737
ARG python=3.8
3838
RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
3939
(if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
40-
(if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
41-
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
40+
(if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
41+
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
42+
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts")
4243
RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
4344
RUN python -m pip install -U pip setuptools

‎ci/docker/python-wheel-windows-vs2017.dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
8080
ARG python=3.8
8181
RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
8282
(if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
83-
(if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
84-
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
83+
(if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
84+
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
85+
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts")
8586
RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
8687
RUN python -m pip install -U pip setuptools
8788

‎ci/scripts/install_gcs_testbench.sh‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,9 @@ case "$(uname -m)" in
3434
;;
3535
esac
3636

37-
case "$(uname -s)-$(uname -m)" in
38-
Darwin-arm64)
39-
# Workaround for https://github.com/grpc/grpc/issues/28387 .
40-
# Build grpcio instead of using wheel.
41-
# storage-testbench 0.27.0 pins grpcio to 1.46.1.
42-
${PYTHON:-python3} -m pip install --no-binary :all: "grpcio==1.46.1"
43-
;;
44-
esac
45-
4637
version=$1
4738
if [[ "${version}" -eq "default" ]]; then
48-
version="v0.27.0"
39+
version="v0.32.0"
4940
fi
5041

5142
${PYTHON:-python3} -m pip install \

‎ci/scripts/install_python.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ platforms=([windows]=Windows
2727
declare -A versions
2828
versions=([3.7]=3.7.9
2929
[3.8]=3.8.10
30-
[3.9]=3.9.9
31-
[3.10]=3.10.1)
30+
[3.9]=3.9.13
31+
[3.10]=3.10.8
32+
[3.11]=3.11.0)
3233

3334
if [ "$#" -ne 2 ]; then
3435
echo "Usage: $0 <platform> <version>"
@@ -45,7 +46,7 @@ full_version=${versions[$2]}
4546
if [ $platform = "MacOSX" ]; then
4647
echo "Downloading Python installer..."
4748

48-
if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ]; then
49+
if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then
4950
fname="python-${full_version}-macos11.pkg"
5051
else
5152
fname="python-${full_version}-macosx10.9.pkg"

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,6 @@ if [ $arch = "arm64" ]; then
4141
export CMAKE_OSX_ARCHITECTURES="arm64"
4242
elif [ $arch = "x86_64" ]; then
4343
export CMAKE_OSX_ARCHITECTURES="x86_64"
44-
elif [ $arch = "universal2" ]; then
45-
export CMAKE_OSX_ARCHITECTURES="x86_64;arm64"
4644
else
4745
echo "Unexpected architecture: $arch"
4846
exit 1
@@ -58,7 +56,7 @@ pip install \
5856
--target $PIP_SITE_PACKAGES \
5957
--platform $PIP_TARGET_PLATFORM \
6058
-r ${source_dir}/python/requirements-wheel-build.txt
61-
pip install "delocate>=0.9"
59+
pip install "delocate>=0.10.3"
6260

6361
echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
6462
: ${ARROW_DATASET:=ON}

‎dev/release/verify-release-candidate.sh‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,7 +1019,7 @@ test_linux_wheels() {
10191019
local arch="x86_64"
10201020
fi
10211021

1022-
local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10}"
1022+
local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10 3.11}"
10231023
local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}"
10241024

10251025
for python in ${python_versions}; do
@@ -1041,11 +1041,11 @@ test_macos_wheels() {
10411041

10421042
# apple silicon processor
10431043
if [ "$(uname -m)" = "arm64" ]; then
1044-
local python_versions="3.8 3.9 3.10"
1044+
local python_versions="3.8 3.9 3.10 3.11"
10451045
local platform_tags="macosx_11_0_arm64"
10461046
local check_flight=OFF
10471047
else
1048-
local python_versions="3.7m 3.8 3.9 3.10"
1048+
local python_versions="3.7m 3.8 3.9 3.10 3.11"
10491049
local platform_tags="macosx_10_14_x86_64"
10501050
fi
10511051

@@ -1067,26 +1067,6 @@ test_macos_wheels() {
10671067
${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR}
10681068
done
10691069
done
1070-
1071-
# verify arm64 and universal2 wheels using an universal2 python binary
1072-
# the interpreter should be installed from python.org:
1073-
# https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg
1074-
if [ "$(uname -m)" = "arm64" ]; then
1075-
for pyver in 3.9 3.10; do
1076-
local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}"
1077-
1078-
# create and activate a virtualenv for testing as arm64
1079-
for arch in "arm64" "x86_64"; do
1080-
show_header "Testing Python ${pyver} universal2 wheel on ${arch}"
1081-
VENV_ENV=wheel-${pyver}-universal2-${arch} PYTHON=${python} maybe_setup_virtualenv || continue
1082-
# install pyarrow's universal2 wheel
1083-
pip install pyarrow-${VERSION}-cp${pyver/.}-cp${pyver/.}-macosx_11_0_universal2.whl
1084-
# check the imports and execute the unittests
1085-
INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} \
1086-
arch -${arch} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR}
1087-
done
1088-
done
1089-
fi
10901070
}
10911071

10921072
test_wheels() {

‎dev/tasks/python-wheels/github.osx.arm64.yml‎

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -87,46 +87,6 @@ jobs:
8787
pip install --upgrade pip wheel
8888
arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build
8989
90-
{% if arch == "universal2" %}
91-
- name: Install AMD64 Packages
92-
env:
93-
VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
94-
run: |
95-
vcpkg install \
96-
--clean-after-build \
97-
--x-install-root=${VCPKG_ROOT}/installed \
98-
--x-manifest-root=arrow/ci/vcpkg \
99-
--x-feature=flight \
100-
--x-feature=gcs \
101-
--x-feature=json \
102-
--x-feature=parquet \
103-
--x-feature=s3
104-
105-
- name: Build AMD64 Wheel
106-
env:
107-
ARROW_SIMD_LEVEL: "NONE"
108-
VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
109-
run: |
110-
$PYTHON -m venv build-amd64-env
111-
source build-amd64-env/bin/activate
112-
pip install --upgrade pip wheel
113-
arch -x86_64 arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build
114-
115-
- name: Fuse AMD64 and ARM64 wheels
116-
run: |
117-
source build-amd64-env/bin/activate
118-
pip install delocate
119-
120-
amd64_wheel=$(ls arrow/python/repaired_wheels/pyarrow*x86_64.whl)
121-
arm64_wheel=$(ls arrow/python/repaired_wheels/pyarrow*arm64.whl)
122-
echo "Fusing ${amd64_wheel} and ${arm64_wheel} into an universal2 wheel..."
123-
delocate-fuse $amd64_wheel $arm64_wheel -w .
124-
125-
fused_wheel=$(ls *x86_64.whl)
126-
rm arrow/python/repaired_wheels/*.whl
127-
mv $fused_wheel arrow/python/repaired_wheels/${fused_wheel/x86_64/universal2}
128-
{% endif %}
129-
13090
- uses: actions/upload-artifact@v3
13191
with:
13292
name: wheel
@@ -148,20 +108,6 @@ jobs:
148108
PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default
149109
arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
150110
151-
{% if arch == "universal2" %}
152-
- name: Test Wheel on AMD64
153-
shell: bash
154-
env:
155-
PYTEST_ADDOPTS: "-k 'not test_cancellation'"
156-
run: |
157-
$PYTHON -m venv test-amd64-env
158-
source test-amd64-env/bin/activate
159-
pip install --upgrade pip wheel
160-
arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt
161-
PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default
162-
arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
163-
{% endif %}
164-
165111
- name: Upload artifacts
166112
shell: bash
167113
run: |

‎dev/tasks/tasks.yml‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,8 @@ tasks:
427427
{% for python_version, python_tag, abi_tag in [("3.7", "cp37", "cp37m"),
428428
("3.8", "cp38", "cp38"),
429429
("3.9", "cp39", "cp39"),
430-
("3.10", "cp310", "cp310")] %}
430+
("3.10", "cp310", "cp310"),
431+
("3.11", "cp311", "cp311")] %}
431432

432433
{############################## Wheel Linux ##################################}
433434

@@ -488,7 +489,7 @@ tasks:
488489
artifacts:
489490
- pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl
490491

491-
{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %}
492+
{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %}
492493
wheel-macos-big-sur-{{ python_tag }}-arm64:
493494
ci: github
494495
template: python-wheels/github.osx.arm64.yml
@@ -498,16 +499,6 @@ tasks:
498499
macos_deployment_target: "11.0"
499500
artifacts:
500501
- pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl
501-
502-
wheel-macos-big-sur-{{ python_tag }}-universal2:
503-
ci: github
504-
template: python-wheels/github.osx.arm64.yml
505-
params:
506-
arch: universal2
507-
python_version: "{{ python_version }}"
508-
macos_deployment_target: "10.14"
509-
artifacts:
510-
- pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_10_14_universal2.whl
511502
{% endfor %}
512503

513504
{############################ Python sdist ####################################}
@@ -1193,7 +1184,7 @@ tasks:
11931184
UBUNTU: 20.04
11941185
image: ubuntu-cpp-thread-sanitizer
11951186

1196-
{% for python_version in ["3.7", "3.8", "3.9", "3.10"] %}
1187+
{% for python_version in ["3.7", "3.8", "3.9", "3.10", "3.11"] %}
11971188
test-conda-python-{{ python_version }}:
11981189
ci: github
11991190
template: docker-tests/github.linux.yml

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -953,7 +953,7 @@ services:
953953
args:
954954
arch: ${ARCH}
955955
arch_short: ${ARCH_SHORT}
956-
base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
956+
base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779
957957
vcpkg: ${VCPKG}
958958
python: ${PYTHON}
959959
manylinux: 2014

‎python/pyarrow/_fs.pyx‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,12 @@ cdef CFileType _unwrap_file_type(FileType ty) except *:
7878
assert 0
7979

8080

81+
def _file_type_to_string(ty):
82+
# Python 3.11 changed str(IntEnum) to return the string representation
83+
# of the integer value: https://github.com/python/cpython/issues/94763
84+
return f"{ty.__class__.__name__}.{ty._name_}"
85+
86+
8187
cdef class FileInfo(_Weakrefable):
8288
"""
8389
FileSystem entry info.
@@ -185,9 +191,10 @@ cdef class FileInfo(_Weakrefable):
185191
except ValueError:
186192
return ''
187193

188-
s = '<FileInfo for {!r}: type={}'.format(self.path, str(self.type))
194+
s = (f'<FileInfo for {self.path!r}: '
195+
f'type={_file_type_to_string(self.type)}')
189196
if self.is_file:
190-
s += ', size={}'.format(self.size)
197+
s += f', size={self.size}'
191198
s += '>'
192199
return s
193200

0 commit comments

Comments
 (0)