Skip to content

Apache DataFusion SQL Query Engine

Explicitly declare spill codec dependency in `physical-plan` (#21917)

kosiewc134a848289c
diff --git a/Cargo.lock b/Cargo.lock
index af52588e53..833839d57e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2416,6 +2416,7 @@ version = "53.1.0"
dependencies = [
"arrow",
"arrow-data",
+ "arrow-ipc",
"arrow-ord",
"arrow-schema",
"async-trait",
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index 414b8c6444..baf8e2c297 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -82,7 +82,7 @@ testcontainers-modules = { workspace = true, features = ["minio"] }
# feature unification with dependencies
serde_json = { workspace = true, features = ["preserve_order"] }
-# Required because we pull serde_json with a feature to get consistent pg display,
-# but its not directly used.
+# serde_json is pulled with a feature to get
+# consistent pg display, but is not directly used in this crate.
[package.metadata.cargo-machete]
-ignored = "serde_json"
+ignored = ["serde_json"]
diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml
index 374fc275a0..7c52ee6e17 100644
--- a/datafusion/physical-plan/Cargo.toml
+++ b/datafusion/physical-plan/Cargo.toml
@@ -49,6 +49,10 @@ name = "datafusion_physical_plan"
[dependencies]
arrow = { workspace = true }
arrow-data = { workspace = true }
+# Spill IPC writes require lz4 and zstd codec support. Keep these features in
+# sync with the SpillCompression variants in datafusion-common so codec
+# availability is explicit in the crate that owns spill handling.
+arrow-ipc = { workspace = true, features = ["lz4", "zstd"] }
arrow-ord = { workspace = true }
arrow-schema = { workspace = true }
async-trait = { workspace = true }
diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs
index 51e59318e2..3c95a1da5b 100644
--- a/datafusion/physical-plan/src/spill/mod.rs
+++ b/datafusion/physical-plan/src/spill/mod.rs
@@ -48,6 +48,7 @@ use arrow::ipc::{
};
use arrow::record_batch::RecordBatch;
use arrow_data::ArrayDataBuilder;
+use arrow_ipc::CompressionType;
use datafusion_common::config::SpillCompression;
use datafusion_common::{DataFusionError, Result, exec_datafusion_err, exec_err};
@@ -290,10 +291,21 @@ struct IPCStreamWriter {
impl IPCStreamWriter {
/// Create new writer
+ ///
+ /// # Codec contract
+ ///
+ /// `arrow-ipc` must be compiled with the `lz4` and `zstd` features
+ /// (declared explicitly in `datafusion-physical-plan/Cargo.toml`). If
+ /// those features are absent, `try_with_compression` will return an
+ /// error at runtime for [`SpillCompression::Lz4Frame`] and
+ /// [`SpillCompression::Zstd`] variants. The Cargo dependency keeps this
+ /// contract local and build-visible during Cargo feature resolution,
+ /// rather than relying solely on workspace-level feature unification;
+ /// see #21917.
pub fn new(
path: &Path,
schema: &Schema,
- compression_type: SpillCompression,
+ spill_compression: SpillCompression,
) -> Result<Self> {
let file = File::create(path).map_err(|e| {
exec_datafusion_err!("(Hint: you may increase the file descriptor limit with shell command 'ulimit -n 4096') Failed to create partition file at {path:?}: {e:?}")
@@ -308,7 +320,8 @@ impl IPCStreamWriter {
let alignment = get_max_alignment_for_schema(schema);
let mut write_options =
IpcWriteOptions::try_new(alignment, false, metadata_version)?;
- write_options = write_options.try_with_compression(compression_type.into())?;
+ let compression_type = Option::<CompressionType>::from(spill_compression);
+ write_options = write_options.try_with_compression(compression_type)?;
let writer = StreamWriter::try_new_with_options(file, schema, write_options)?;
Ok(Self {
diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml
index 1159b7f3b7..e2ffe1415a 100644
--- a/datafusion/sqllogictest/Cargo.toml
+++ b/datafusion/sqllogictest/Cargo.toml
@@ -94,7 +94,7 @@ harness = false
name = "sqllogictests"
path = "bin/sqllogictests.rs"
-# Required because we pull serde_json with a feature to get consistent pg display,
-# but its not directly used.
+# serde_json is pulled with a feature to get
+# consistent pg display, but is not directly used in this crate.
[package.metadata.cargo-machete]
-ignored = "serde_json"
+ignored = ["serde_json"]