Skip to content

Commit 4e1ba57

Browse files
andersonm-ibmroee88pitrou
committed
ARROW-9947: [Python] High-level Python API for Parquet encryption of files.
Exposes in PyArrow the high-level C++ API for Parquet encryption that was added in apache#8023. Design document: https://docs.google.com/document/d/1i1M5f5azLEmASj9XQZ_aQLl5Fr5F0CvnyPPVu1xaD9U A test is added for writing and reading encrypted parquet files using a simple in-memory KMS for testing, that is not to be used as an example of a KMS client. In addition, there is an example KMS client using Vault KMS. This PR handles only the file-level encryption and decryption. Dataset is handled in separate PRs. The investigation of the multithreading model of PME is separated into a separate issue, independent of this one. Closes apache#10450 from andersonm-ibm/encryption Lead-authored-by: Maya Anderson <mayaa@il.ibm.com> Co-authored-by: andersonm-ibm <63074550+andersonm-ibm@users.noreply.github.com> Co-authored-by: roee88 <roee88@gmail.com> Co-authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Antoine Pitrou <antoine@python.org>
1 parent 69682ec commit 4e1ba57

36 files changed

Lines changed: 2075 additions & 82 deletions

‎.github/workflows/python.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ jobs:
126126
ARROW_JEMALLOC: ON
127127
ARROW_ORC: ON
128128
ARROW_PARQUET: ON
129+
PARQUET_REQUIRE_ENCRYPTION: ON
129130
ARROW_PLASMA: ON
130131
ARROW_PYTHON: ON
131132
ARROW_S3: ON

‎ci/appveyor-cpp-build.bat‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ set PYARROW_WITH_DATASET=ON
146146
set PYARROW_WITH_FLIGHT=%ARROW_BUILD_FLIGHT%
147147
set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
148148
set PYARROW_WITH_PARQUET=ON
149+
set PYARROW_WITH_PARQUET_ENCRYPTION=ON
149150
set PYARROW_WITH_S3=%ARROW_S3%
150151
set PYARROW_WITH_STATIC_BOOST=ON
151152

‎ci/docker/conda-python-hdfs.dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ ENV CC=gcc \
4646
ARROW_GANDIVA=OFF \
4747
ARROW_PLASMA=OFF \
4848
ARROW_PARQUET=ON \
49+
PARQUET_REQUIRE_ENCRYPTION=ON \
4950
ARROW_ORC=OFF \
5051
ARROW_HDFS=ON \
5152
ARROW_PYTHON=ON \

‎ci/scripts/python_build.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
5757
export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF}
5858
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
5959
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
60+
export PYARROW_WITH_PARQUET_ENCRYPTION=${ARROW_PARQUET:-OFF}
6061
export PYARROW_WITH_DATASET=${ARROW_DATASET:-OFF}
6162

6263
export PYARROW_PARALLEL=${n_jobs}

‎ci/scripts/python_sdist_test.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
3535
export PYARROW_WITH_PLASMA=${ARROW_PLASMA:-OFF}
3636
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
3737
export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
38+
export PYARROW_WITH_PARQUET_ENCRYPTION=${ARROW_PARQUET:-OFF}
3839
export PYARROW_WITH_DATASET=${ARROW_DATASET:-OFF}
3940

4041
# TODO: Users should not require ARROW_HOME and pkg-config to find Arrow C++.

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
7070
: ${ARROW_MIMALLOC:=ON}
7171
: ${ARROW_ORC:=ON}
7272
: ${ARROW_PARQUET:=ON}
73+
: ${PARQUET_REQUIRE_ENCRYPTION:=ON}
7374
: ${ARROW_PLASMA:=ON}
7475
: ${ARROW_S3:=ON}
7576
: ${ARROW_SIMD_LEVEL:="SSE4_2"}
@@ -106,6 +107,7 @@ cmake \
106107
-DARROW_ORC=${ARROW_ORC} \
107108
-DARROW_PACKAGE_KIND="python-wheel-macos" \
108109
-DARROW_PARQUET=${ARROW_PARQUET} \
110+
-DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
109111
-DARROW_PLASMA=${ARROW_PLASMA} \
110112
-DARROW_PYTHON=ON \
111113
-DARROW_RPATH_ORIGIN=ON \
@@ -147,6 +149,7 @@ export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
147149
export PYARROW_WITH_HDFS=${ARROW_HDFS}
148150
export PYARROW_WITH_ORC=${ARROW_ORC}
149151
export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
152+
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
150153
export PYARROW_WITH_PLASMA=${ARROW_PLASMA}
151154
export PYARROW_WITH_S3=${ARROW_S3}
152155
export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"

‎ci/scripts/python_wheel_manylinux_build.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
5757
: ${ARROW_MIMALLOC:=ON}
5858
: ${ARROW_ORC:=ON}
5959
: ${ARROW_PARQUET:=ON}
60+
: ${PARQUET_REQUIRE_ENCRYPTION:=ON}
6061
: ${ARROW_PLASMA:=ON}
6162
: ${ARROW_S3:=ON}
6263
: ${ARROW_TENSORFLOW:=ON}
@@ -100,6 +101,7 @@ cmake \
100101
-DARROW_ORC=${ARROW_ORC} \
101102
-DARROW_PACKAGE_KIND="python-wheel-manylinux${MANYLINUX_VERSION}" \
102103
-DARROW_PARQUET=${ARROW_PARQUET} \
104+
-DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION} \
103105
-DARROW_PLASMA=${ARROW_PLASMA} \
104106
-DARROW_PYTHON=ON \
105107
-DARROW_RPATH_ORIGIN=ON \
@@ -143,6 +145,7 @@ export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
143145
export PYARROW_WITH_HDFS=${ARROW_HDFS}
144146
export PYARROW_WITH_ORC=${ARROW_ORC}
145147
export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
148+
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
146149
export PYARROW_WITH_PLASMA=${ARROW_PLASMA}
147150
export PYARROW_WITH_S3=${ARROW_S3}
148151
# PyArrow build configuration

‎ci/scripts/python_wheel_windows_build.bat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ set ARROW_GANDIVA=OFF
3636
set ARROW_HDFS=ON
3737
set ARROW_ORC=OFF
3838
set ARROW_PARQUET=ON
39+
set PARQUET_REQUIRE_ENCRYPTION=ON
3940
set ARROW_MIMALLOC=ON
4041
set ARROW_S3=ON
4142
set ARROW_TENSORFLOW=ON
@@ -68,6 +69,7 @@ cmake ^
6869
-DARROW_ORC=%ARROW_ORC% ^
6970
-DARROW_PACKAGE_KIND="python-wheel-windows" ^
7071
-DARROW_PARQUET=%ARROW_PARQUET% ^
72+
-DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^
7173
-DARROW_PYTHON=ON ^
7274
-DARROW_S3=%ARROW_S3% ^
7375
-DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^
@@ -102,6 +104,7 @@ set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
102104
set PYARROW_WITH_HDFS=%ARROW_HDFS%
103105
set PYARROW_WITH_ORC=%ARROW_ORC%
104106
set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
107+
set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
105108
set PYARROW_WITH_S3=%ARROW_S3%
106109
set ARROW_HOME=C:\arrow-dist
107110

‎cpp/src/arrow/python/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ if(ARROW_FILESYSTEM)
5757
list(APPEND ARROW_PYTHON_SRCS filesystem.cc)
5858
endif()
5959

60+
if(PARQUET_REQUIRE_ENCRYPTION)
61+
list(APPEND ARROW_PYTHON_SRCS parquet_encryption.cc)
62+
endif()
63+
6064
set(ARROW_PYTHON_DEPENDENCIES arrow_dependencies)
6165

6266
if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
@@ -70,6 +74,10 @@ if(WIN32)
7074
list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS})
7175
endif()
7276

77+
if(PARQUET_REQUIRE_ENCRYPTION)
78+
list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS parquet_shared)
79+
endif()
80+
7381
set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})
7482

7583
add_arrow_lib(arrow_python
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "arrow/python/parquet_encryption.h"
19+
#include "parquet/exception.h"
20+
21+
namespace arrow {
22+
namespace py {
23+
namespace parquet {
24+
namespace encryption {
25+
26+
PyKmsClient::PyKmsClient(PyObject* handler, PyKmsClientVtable vtable)
27+
: handler_(handler), vtable_(std::move(vtable)) {
28+
Py_INCREF(handler);
29+
}
30+
31+
PyKmsClient::~PyKmsClient() {}
32+
33+
std::string PyKmsClient::WrapKey(const std::string& key_bytes,
34+
const std::string& master_key_identifier) {
35+
std::string wrapped;
36+
auto st = SafeCallIntoPython([&]() -> Status {
37+
vtable_.wrap_key(handler_.obj(), key_bytes, master_key_identifier, &wrapped);
38+
return CheckPyError();
39+
});
40+
if (!st.ok()) {
41+
throw ::parquet::ParquetStatusException(st);
42+
}
43+
return wrapped;
44+
}
45+
46+
std::string PyKmsClient::UnwrapKey(const std::string& wrapped_key,
47+
const std::string& master_key_identifier) {
48+
std::string unwrapped;
49+
auto st = SafeCallIntoPython([&]() -> Status {
50+
vtable_.unwrap_key(handler_.obj(), wrapped_key, master_key_identifier, &unwrapped);
51+
return CheckPyError();
52+
});
53+
if (!st.ok()) {
54+
throw ::parquet::ParquetStatusException(st);
55+
}
56+
return unwrapped;
57+
}
58+
59+
PyKmsClientFactory::PyKmsClientFactory(PyObject* handler, PyKmsClientFactoryVtable vtable)
60+
: handler_(handler), vtable_(std::move(vtable)) {
61+
Py_INCREF(handler);
62+
}
63+
64+
PyKmsClientFactory::~PyKmsClientFactory() {}
65+
66+
std::shared_ptr<::parquet::encryption::KmsClient> PyKmsClientFactory::CreateKmsClient(
67+
const ::parquet::encryption::KmsConnectionConfig& kms_connection_config) {
68+
std::shared_ptr<::parquet::encryption::KmsClient> kms_client;
69+
auto st = SafeCallIntoPython([&]() -> Status {
70+
vtable_.create_kms_client(handler_.obj(), kms_connection_config, &kms_client);
71+
return CheckPyError();
72+
});
73+
if (!st.ok()) {
74+
throw ::parquet::ParquetStatusException(st);
75+
}
76+
return kms_client;
77+
}
78+
79+
arrow::Result<std::shared_ptr<::parquet::FileEncryptionProperties>>
80+
PyCryptoFactory::SafeGetFileEncryptionProperties(
81+
const ::parquet::encryption::KmsConnectionConfig& kms_connection_config,
82+
const ::parquet::encryption::EncryptionConfiguration& encryption_config) {
83+
PARQUET_CATCH_AND_RETURN(
84+
this->GetFileEncryptionProperties(kms_connection_config, encryption_config));
85+
}
86+
87+
arrow::Result<std::shared_ptr<::parquet::FileDecryptionProperties>>
88+
PyCryptoFactory::SafeGetFileDecryptionProperties(
89+
const ::parquet::encryption::KmsConnectionConfig& kms_connection_config,
90+
const ::parquet::encryption::DecryptionConfiguration& decryption_config) {
91+
PARQUET_CATCH_AND_RETURN(
92+
this->GetFileDecryptionProperties(kms_connection_config, decryption_config));
93+
}
94+
95+
} // namespace encryption
96+
} // namespace parquet
97+
} // namespace py
98+
} // namespace arrow

0 commit comments

Comments
 (0)