Apache DataFusion SQL Query Engine
feat: Allow datafusion-ffi to opt out of proto parquet (#22951)
Xuanwo34200e4b49ea
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex b317dbbf62..7ce7251542 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -162,6 +162,26 @@ jobs:- name: Check datafusion-proto (avro)run: cargo check --profile ci --no-default-features -p datafusion-proto --features=avro+ # Check datafusion-ffi features+ #+ # Ensure via `cargo check` that the crate can be built with a+ # subset of the features packages enabled.+ linux-datafusion-ffi-features:+ name: cargo check datafusion-ffi features+ needs: linux-build-lib+ runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}+ container:+ image: amd64/rust+ steps:+ - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-builder+ with:+ rust-version: stable+ - name: Check datafusion-ffi (no-default-features)+ run: cargo check --profile ci --no-default-features -p datafusion-ffi+# Check datafusion crate features#diff --git a/Cargo.toml b/Cargo.tomlindex 769e5bd710..24a4c7a5a8 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -151,7 +151,7 @@ datafusion-physical-expr-adapter = { path = "datafusion/physical-expr-adapter",datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "54.0.0", default-features = false }datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "54.0.0" }datafusion-physical-plan = { path = "datafusion/physical-plan", version = "54.0.0" }-datafusion-proto = { path = "datafusion/proto", version = "54.0.0" }+datafusion-proto = { path = "datafusion/proto", version = "54.0.0", default-features = false }datafusion-proto-common = { path = "datafusion/proto-common", version = "54.0.0" }datafusion-proto-models = { path = "datafusion/proto-models", version = "54.0.0" }datafusion-pruning = { path = "datafusion/pruning", version = "54.0.0" }diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.tomlindex afe3401654..5dae70761f 100644--- a/benchmarks/Cargo.toml+++ b/benchmarks/Cargo.toml@@ -64,7 +64,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }tokio-util = { version = "0.7.17" }[dev-dependencies]-datafusion-proto = { workspace = true }+datafusion-proto = { workspace = true, features = ["parquet"] }tempfile = { workspace = true }[[bench]]diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.tomlindex bb8a92dbe0..5f66412e7d 100644--- a/datafusion-examples/Cargo.toml+++ b/datafusion-examples/Cargo.toml@@ -53,7 +53,7 @@ dashmap = { workspace = true }base64 = "0.22.1"datafusion-expr = { workspace = true }datafusion-physical-expr-adapter = { workspace = true }-datafusion-proto = { workspace = true }+datafusion-proto = { workspace = true, features = ["parquet"] }datafusion-sql = { workspace = true }env_logger = { workspace = true }futures = { workspace = true }diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.tomlindex 37023d21c4..e50530c868 100644--- a/datafusion/ffi/Cargo.toml+++ b/datafusion/ffi/Cargo.toml@@ -63,7 +63,7 @@ datafusion-physical-expr = { workspace = true }datafusion-physical-expr-common = { workspace = true }datafusion-physical-optimizer = { workspace = true }datafusion-physical-plan = { workspace = true }-datafusion-proto = { workspace = true }+datafusion-proto = { workspace = true, default-features = false }datafusion-proto-common = { workspace = true }datafusion-session = { workspace = true }futures = { workspace = true }@@ -83,10 +83,12 @@ datafusion-functions-window = { workspace = true }doc-comment = { workspace = true }[features]+default = ["parquet"]integration-tests = ["datafusion-functions","datafusion-functions-aggregate","datafusion-functions-table","datafusion-functions-window",]+parquet = ["datafusion-proto/parquet"]tarpaulin_include = [] # Exists only to prevent warnings on stable and still have accurate coveragediff --git a/datafusion/ffi/src/session/mod.rs b/datafusion/ffi/src/session/mod.rsindex 6ddb879feb..6ab6f0dd4e 100644--- a/datafusion/ffi/src/session/mod.rs+++ b/datafusion/ffi/src/session/mod.rs@@ -258,6 +258,7 @@ fn table_options_to_rhash(mut options: TableOptions) -> SVec<(SString, SString)>"datafusion_ffi.table_current_format".into(),match current_format {ConfigFileType::JSON => "json",+ #[cfg(feature = "parquet")]ConfigFileType::PARQUET => "parquet",ConfigFileType::CSV => "csv",}@@ -476,6 +477,7 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOptionlet formats = [ConfigFileType::CSV,ConfigFileType::JSON,+ #[cfg(feature = "parquet")]ConfigFileType::PARQUET,];for format in formats {@@ -483,6 +485,7 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOption// included in the formats list above and in the extension check below.let format_name = match &format {ConfigFileType::CSV => "csv",+ #[cfg(feature = "parquet")]ConfigFileType::PARQUET => "parquet",ConfigFileType::JSON => "json",};@@ -504,7 +507,6 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOption.unwrap_or_else(|err| log::warn!("Error parsing table options: {err}"));}}-let extension_options: HashMap<String, String> = options.iter().filter_map(|(k, v)| {@@ -525,6 +527,7 @@ fn table_options_from_rhashmap(options: SVec<(SString, SString)>) -> TableOptiontable_options.current_format =current_format.and_then(|format| match format.as_str() {"csv" => Some(ConfigFileType::CSV),+ #[cfg(feature = "parquet")]"parquet" => Some(ConfigFileType::PARQUET),"json" => Some(ConfigFileType::JSON),_ => None,@@ -661,7 +664,10 @@ mod tests {let mut table_options = TableOptions::default();table_options.csv.has_header = Some(true);table_options.json.schema_infer_max_rec = Some(10);- table_options.parquet.global.coerce_int96 = Some("123456789".into());+ #[cfg(feature = "parquet")]+ {+ table_options.parquet.global.coerce_int96 = Some("123456789".into());+ }table_options.current_format = Some(ConfigFileType::JSON);let state = SessionStateBuilder::new_from_existing(ctx.state())diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 8e71cc9268..8940b16bf8 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -18,7 +18,9 @@use std::sync::Arc;use super::LogicalExtensionCodec;-use crate::convert::{FromProto, TryFromProto};+use crate::convert::FromProto;+#[cfg(feature = "parquet")]+use crate::convert::TryFromProto;use crate::protobuf::{CsvOptions as CsvOptionsProto, CsvQuoteStyle as CsvQuoteStyleProto,JsonOptions as JsonOptionsProto,