Apache DataFusion SQL Query Engine
Explicitly declare spill codec dependency in `physical-plan` (#21917)
kosiewc134a848289c
diff --git a/Cargo.lock b/Cargo.lockindex af52588e53..833839d57e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2416,6 +2416,7 @@ version = "53.1.0"dependencies = ["arrow","arrow-data",+ "arrow-ipc","arrow-ord","arrow-schema","async-trait",diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.tomlindex 414b8c6444..baf8e2c297 100644--- a/datafusion-cli/Cargo.toml+++ b/datafusion-cli/Cargo.toml@@ -82,7 +82,7 @@ testcontainers-modules = { workspace = true, features = ["minio"] }# feature unification with dependenciesserde_json = { workspace = true, features = ["preserve_order"] }-# Required because we pull serde_json with a feature to get consistent pg display,-# but its not directly used.+# serde_json is pulled with a feature to get+# consistent pg display, but is not directly used in this crate.[package.metadata.cargo-machete]-ignored = "serde_json"+ignored = ["serde_json"]diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 374fc275a0..7c52ee6e17 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -49,6 +49,10 @@ name = "datafusion_physical_plan"[dependencies]arrow = { workspace = true }arrow-data = { workspace = true }+# Spill IPC writes require lz4 and zstd codec support. Keep these features in+# sync with the SpillCompression variants in datafusion-common so codec+# availability is explicit in the crate that owns spill handling.+arrow-ipc = { workspace = true, features = ["lz4", "zstd"] }arrow-ord = { workspace = true }arrow-schema = { workspace = true }async-trait = { workspace = true }diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 51e59318e2..3c95a1da5b 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -48,6 +48,7 @@ use arrow::ipc::{};use arrow::record_batch::RecordBatch;use arrow_data::ArrayDataBuilder;+use arrow_ipc::CompressionType;use datafusion_common::config::SpillCompression;use datafusion_common::{DataFusionError, Result, exec_datafusion_err, exec_err};@@ -290,10 +291,21 @@ struct IPCStreamWriter {impl IPCStreamWriter {/// Create new writer+ ///+ /// # Codec contract+ ///+ /// `arrow-ipc` must be compiled with the `lz4` and `zstd` features+ /// (declared explicitly in `datafusion-physical-plan/Cargo.toml`). If+ /// those features are absent, `try_with_compression` will return an+ /// error at runtime for [`SpillCompression::Lz4Frame`] and+ /// [`SpillCompression::Zstd`] variants. The Cargo dependency keeps this+ /// contract local and build-visible during Cargo feature resolution,+ /// rather than relying solely on workspace-level feature unification;+ /// see #21917.pub fn new(path: &Path,schema: &Schema,- compression_type: SpillCompression,+ spill_compression: SpillCompression,) -> Result<Self> {let file = File::create(path).map_err(|e| {exec_datafusion_err!("(Hint: you may increase the file descriptor limit with shell command 'ulimit -n 4096') Failed to create partition file at {path:?}: {e:?}")@@ -308,7 +320,8 @@ impl IPCStreamWriter {let alignment = get_max_alignment_for_schema(schema);let mut write_options =IpcWriteOptions::try_new(alignment, false, metadata_version)?;- write_options = write_options.try_with_compression(compression_type.into())?;+ let compression_type = Option::<CompressionType>::from(spill_compression);+ write_options = write_options.try_with_compression(compression_type)?;let writer = StreamWriter::try_new_with_options(file, schema, write_options)?;Ok(Self {diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.tomlindex 1159b7f3b7..e2ffe1415a 100644--- a/datafusion/sqllogictest/Cargo.toml+++ b/datafusion/sqllogictest/Cargo.toml@@ -94,7 +94,7 @@ harness = falsename = "sqllogictests"path = "bin/sqllogictests.rs"-# Required because we pull serde_json with a feature to get consistent pg display,-# but its not directly used.+# serde_json is pulled with a feature to get+# consistent pg display, but is not directly used in this crate.[package.metadata.cargo-machete]-ignored = "serde_json"+ignored = ["serde_json"]