Skip to content

Apache DataFusion SQL Query Engine

feat: Allow datafusion-ffi to opt out of proto parquet (#22951)

Xuanwo34200e4b49ea
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index b317dbbf62..7ce7251542 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -162,6 +162,26 @@ jobs:
- name: Check datafusion-proto (avro)
run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro
+ # Check datafusion-ffi features
+ #
+ # Ensure via `cargo check` that the crate can be built with a
+ # subset of the features packages enabled.
+ linux-datafusion-ffi-features:
+ name: cargo check datafusion-ffi features
+ needs: linux-build-lib
+ runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
+ container:
+ image: amd64/rust
+ steps:
+ - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Check datafusion-ffi (no-default-features)
+ run: cargo check --profile ci --no-default-features -p datafusion-ffi
+
# Check datafusion crate features
#
diff --git a/Cargo.toml b/Cargo.toml
index 769e5bd710..24a4c7a5a8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -151,7 +151,7 @@ datafusion-physical-expr-adapter = { path = "datafusion/physical-expr-adapter",
datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "54.0.0", default-features = false }
datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "54.0.0" }
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "54.0.0" }
-datafusion-proto = { path = "datafusion/proto", version = "54.0.0" }
+datafusion-proto = { path = "datafusion/proto", version = "54.0.0", default-features = false }
datafusion-proto-common = { path = "datafusion/proto-common", version = "54.0.0" }
datafusion-proto-models = { path = "datafusion/proto-models", version = "54.0.0" }
datafusion-pruning = { path = "datafusion/pruning", version = "54.0.0" }
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index afe3401654..5dae70761f 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -64,7 +64,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }
tokio-util = { version = "0.7.17" }
[dev-dependencies]
-datafusion-proto = { workspace = true }
+datafusion-proto = { workspace = true, features = ["parquet"] }
tempfile = { workspace = true }
[[bench]]
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index bb8a92dbe0..5f66412e7d 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -53,7 +53,7 @@ dashmap = { workspace = true }
base64 = "0.22.1"
datafusion-expr = { workspace = true }
datafusion-physical-expr-adapter = { workspace = true }
-datafusion-proto = { workspace = true }
+datafusion-proto = { workspace = true, features = ["parquet"] }
datafusion-sql = { workspace = true }
env_logger = { workspace = true }
futures = { workspace = true }
diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.toml
index 37023d21c4..e50530c868 100644
--- a/datafusion/ffi/Cargo.toml
+++ b/datafusion/ffi/Cargo.toml
@@ -63,7 +63,7 @@ datafusion-physical-expr = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-optimizer = { workspace = true }
datafusion-physical-plan = { workspace = true }
-datafusion-proto = { workspace = true }
+datafusion-proto = { workspace = true, default-features = false }
datafusion-proto-common = { workspace = true }
datafusion-session = { workspace = true }
futures = { workspace = true }
@@ -83,10 +83,12 @@ datafusion-functions-window = { workspace = true }
doc-comment = { workspace = true }
[features]
+default = ["parquet"]
integration-tests = [
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-table",
"datafusion-functions-window",
]
+parquet = ["datafusion-proto/parquet"]
tarpaulin_include = [] # Exists only to prevent warnings on stable and still have accurate coverage
diff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rs
index 6ddb879feb..6ab6f0dd4e 100644
--- a/datafusion/ffi/src/session/mod.rs
+++ b/datafusion/ffi/src/session/mod.rs
@@ -258,6 +258,7 @@ fn table_options_to_rhash(mut options: TableOptions) -> SVec<(SString, SString)>
"datafusion_ffi.table_current_format".into(),
match current_format {
ConfigFileType::JSON => "json",
+ #[cfg(feature = "parquet")]
ConfigFileType::PARQUET => "parquet",
ConfigFileType::CSV => "csv",
}
@@ -476,6 +477,7 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOption
let formats = [
ConfigFileType::CSV,
ConfigFileType::JSON,
+ #[cfg(feature = "parquet")]
ConfigFileType::PARQUET,
];
for format in formats {
@@ -483,6 +485,7 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOption
// included in the formats list above and in the extension check below.
let format_name = match &format {
ConfigFileType::CSV => "csv",
+ #[cfg(feature = "parquet")]
ConfigFileType::PARQUET => "parquet",
ConfigFileType::JSON => "json",
};
@@ -504,7 +507,6 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOption
.unwrap_or_else(|err| log::warn!("Error parsing table options: {err}"));
}
}
-
let extension_options: HashMap<String, String> = options
.iter()
.filter_map(|(k, v)| {
@@ -525,6 +527,7 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOption
table_options.current_format =
current_format.and_then(|format| match format.as_str() {
"csv" => Some(ConfigFileType::CSV),
+ #[cfg(feature = "parquet")]
"parquet" => Some(ConfigFileType::PARQUET),
"json" => Some(ConfigFileType::JSON),
_ => None,
@@ -661,7 +664,10 @@ mod tests {
let mut table_options = TableOptions::default();
table_options.csv.has_header = Some(true);
table_options.json.schema_infer_max_rec = Some(10);
- table_options.parquet.global.coerce_int96 = Some("123456789".into());
+ #[cfg(feature = "parquet")]
+ {
+ table_options.parquet.global.coerce_int96 = Some("123456789".into());
+ }
table_options.current_format = Some(ConfigFileType::JSON);
let state = SessionStateBuilder::new_from_existing(ctx.state())
diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rs
index 8e71cc9268..8940b16bf8 100644
--- a/datafusion/proto/src/logical_plan/file_formats.rs
+++ b/datafusion/proto/src/logical_plan/file_formats.rs
@@ -18,7 +18,9 @@
use std::sync::Arc;
use super::LogicalExtensionCodec;
-use crate::convert::{FromProto, TryFromProto};
+use crate::convert::FromProto;
+#[cfg(feature = "parquet")]
+use crate::convert::TryFromProto;
use crate::protobuf::{
CsvOptions as CsvOptionsProto, CsvQuoteStyle as CsvQuoteStyleProto,
JsonOptions as JsonOptionsProto,