Skip to content

Commit 7fb2e18

Browse files
timsaucerclaude
timsaucer
and
claude
committed
feat: add plain-C scan ABI crate over Arrow C Data/Stream
Introduce datafusion-scan-ffi: a cdylib exposing a DataFusion TableProvider scan through extern "C" entrypoints that speak only C primitives and the standard Arrow C Data/Stream interface (ArrowSchema/ArrowArrayStream). No JVM/JNI dependency, so the surface is consumable from Java (via a thin shim or FFM), Python, Go, or Rust, and is a candidate to live closer to DataFusion proper. This is the JNI-free reshaping of PR apache#103's scan logic per review feedback on PR apache#104: providers are compiled in and registered by name (approach A), filters cross as datafusion.LogicalExprNode protobufs (shared vocabulary with datafusion-ffi/Comet), and each scanned partition is handed back as a zero-copy FFI_ArrowArrayStream. - abi.rs: df_scan_{schema,create,partition_count,execute_partition, execute,close}, df_error_free, df_scan_abi_version - scan.rs: build -> register -> project -> filter -> plan core - registry.rs: name-keyed provider builders - reader.rs: SendableRecordBatchStream -> panic-safe RecordBatchReader - include/datafusion_scan.h: the C header - tests/roundtrip.rs: drives the ABI and re-imports the stream via the Arrow C Stream interface, no JVM involved (6 tests) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 8054c23 commit 7fb2e18

14 files changed

Lines changed: 1353 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ resolver = "2"
2020
members = [
2121
"native",
2222
"native-common",
23+
"native-ffi",
2324
]
2425

2526
# Shared package metadata so every crate moves in lock step. Members inherit

‎native-ffi/Cargo.toml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
[package]
19+
name = "datafusion-scan-ffi"
20+
version.workspace = true
21+
edition.workspace = true
22+
license.workspace = true
23+
repository.workspace = true
24+
# Not published yet; this is the in-tree home of the plain-C scan ABI while it
25+
# stabilizes. The intent is for this surface to eventually live in DataFusion
26+
# proper (it has no JVM/JNI dependency), so keep it free of anything
27+
# Java-specific.
28+
publish = false
29+
30+
[lib]
31+
# `cdylib` -> the shippable plain-C shared library (`libdatafusion_scan_ffi`).
32+
# `rlib` -> lets a downstream cdylib statically link this crate, register
33+
# its own providers, and re-export the `df_scan_*` symbols; also
34+
# gives `cargo test` a Rust harness that round-trips the ABI with
35+
# no JVM in sight.
36+
crate-type = ["cdylib", "rlib"]
37+
38+
[features]
39+
# A built-in in-memory provider builder registered under `datafusion.memory`,
40+
# used by the round-trip tests and handy as a reference builder. Off by default
41+
# so a production cdylib only carries the providers it registers itself.
42+
demo-providers = []
43+
44+
[dependencies]
45+
# The arrow C Data / C Stream interface types are the entire data plane of this
46+
# ABI. `ffi` pulls in both `arrow::ffi` (FFI_ArrowSchema/Array) and
47+
# `arrow::ffi_stream` (FFI_ArrowArrayStream). Same crate+version DataFusion
48+
# links, so the types unify.
49+
arrow = { workspace = true }
50+
datafusion = { workspace = true }
51+
# Pushed filters arrive as serialized `datafusion.LogicalExprNode` protobufs --
52+
# the same vocabulary `datafusion-ffi` already uses, so the encoder is shared
53+
# with any future Comet path.
54+
datafusion-proto = { workspace = true }
55+
futures = { workspace = true }
56+
prost = { workspace = true }
57+
tokio = { workspace = true }
58+
59+
[dev-dependencies]
60+
# Round-trip tests import the produced FFI_ArrowArrayStream back into Rust via
61+
# the same C Stream interface a Java/Python/Go consumer would use.
62+
datafusion-scan-ffi = { path = ".", features = ["demo-providers"] }

‎native-ffi/include/datafusion_scan.h‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
// Plain-C scan ABI over the Arrow C Data / C Stream interface.
19+
//
20+
// The only "rich" types crossing this boundary are the standard Arrow C
21+
// structs `ArrowSchema` and `ArrowArrayStream` (from Arrow's abi.h), which any
22+
// Arrow implementation can produce/consume. Everything else is C primitives
23+
// and borrowed (ptr, len) views. No JVM/JNI types appear here, by design.
24+
25+
#ifndef DATAFUSION_SCAN_H
26+
#define DATAFUSION_SCAN_H
27+
28+
#include <stddef.h>
29+
#include <stdint.h>
30+
31+
#include "arrow/c/abi.h" // struct ArrowSchema, struct ArrowArrayStream
32+
33+
#ifdef __cplusplus
34+
extern "C" {
35+
#endif
36+
37+
// --- Status codes ----------------------------------------------------------
38+
// 0 on success; nonzero classifies the failure. On error the call also writes
39+
// a malloc'd, NUL-terminated message to *out_err (free with df_error_free).
40+
typedef enum {
41+
DF_OK = 0,
42+
DF_INVALID_ARGUMENT = 1,
43+
DF_UNKNOWN_PROVIDER = 2,
44+
DF_PROVIDER_BUILD = 3,
45+
DF_PLANNING = 4,
46+
DF_EXECUTION = 5,
47+
DF_PANIC = 6,
48+
DF_INTERNAL = 7
49+
} DfStatus;
50+
51+
// --- Borrowed input views (caller owns the memory) -------------------------
52+
typedef struct {
53+
const uint8_t* ptr; // UTF-8, not NUL-terminated; may be null if len == 0
54+
size_t len;
55+
} DfStr;
56+
57+
typedef struct {
58+
const uint8_t* ptr; // may be null if len == 0
59+
size_t len;
60+
} DfBytes;
61+
62+
typedef struct {
63+
DfStr key;
64+
DfStr value;
65+
} DfKeyValue;
66+
67+
// Opaque planned-scan handle.
68+
typedef struct DfScanHandle DfScanHandle;
69+
70+
// --- Lifecycle / versioning ------------------------------------------------
71+
72+
// ABI major version; compare before any other call.
73+
uint64_t df_scan_abi_version(void);
74+
75+
// Free a message previously written to an out_err argument (null-safe).
76+
void df_error_free(char* err);
77+
78+
// --- Scan API --------------------------------------------------------------
79+
80+
// Probe a provider's output schema into the caller-allocated out_schema.
81+
int32_t df_scan_schema(DfStr provider, DfBytes options, DfBytes partition,
82+
struct ArrowSchema* out_schema, char** out_err);
83+
84+
// Plan a scan. On success writes an owned handle to *out_handle (release with
85+
// df_scan_close). projection is an array of column-name DfStr (empty = all);
86+
// filters is an array of serialized datafusion.LogicalExprNode DfBytes;
87+
// target_partitions / batch_size <= 0 keep DataFusion defaults.
88+
int32_t df_scan_create(DfStr provider, DfBytes options, DfBytes partition,
89+
int32_t target_partitions, int32_t batch_size,
90+
const DfKeyValue* config_overrides, size_t config_overrides_len,
91+
const DfStr* projection, size_t projection_len,
92+
const DfBytes* filters, size_t filters_len,
93+
DfScanHandle** out_handle, char** out_err);
94+
95+
// Output partition count of the planned scan.
96+
int32_t df_scan_partition_count(const DfScanHandle* handle, int32_t* out_count,
97+
char** out_err);
98+
99+
// Execute one partition into the caller-allocated Arrow C Stream.
100+
int32_t df_scan_execute_partition(const DfScanHandle* handle, int32_t partition,
101+
struct ArrowArrayStream* out_stream, char** out_err);
102+
103+
// Execute the whole plan as a single coalesced Arrow C Stream.
104+
int32_t df_scan_execute(const DfScanHandle* handle,
105+
struct ArrowArrayStream* out_stream, char** out_err);
106+
107+
// Drop a planned scan (null-safe). Must not race an in-flight execute on the
108+
// same handle.
109+
void df_scan_close(DfScanHandle* handle);
110+
111+
#ifdef __cplusplus
112+
} // extern "C"
113+
#endif
114+
115+
#endif // DATAFUSION_SCAN_H

0 commit comments

Comments
 (0)