Skip to content

Apache DataFusion SQL Query Engine

fix: Enable `arrow-ipc/zstd` in `datasource-arrow` to make `test_spill_compression` pass in every config (#21504)

Adam Gutglick14c18ece4a17
diff --git a/Cargo.toml b/Cargo.toml
index 59707ba8e3..82081c1e42 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -104,8 +104,11 @@ arrow-data = { version = "58.1.0", default-features = false }
arrow-flight = { version = "58.1.0", features = [
"flight-sql-experimental",
] }
+# Both codecs are required here to make sure that code paths like
+# file-spilling have access to all compression codecs.
arrow-ipc = { version = "58.1.0", default-features = false, features = [
"lz4",
+ "zstd",
] }
arrow-ord = { version = "58.1.0", default-features = false }
arrow-schema = { version = "58.1.0", default-features = false }
diff --git a/datafusion/datasource-arrow/Cargo.toml b/datafusion/datasource-arrow/Cargo.toml
index fbadc8708c..2718e424c6 100644
--- a/datafusion/datasource-arrow/Cargo.toml
+++ b/datafusion/datasource-arrow/Cargo.toml
@@ -62,6 +62,6 @@ name = "datafusion_datasource_arrow"
path = "src/mod.rs"
[features]
-compression = [
- "arrow-ipc/zstd",
-]
+# This feature is deprecated, as core functionality in the SpillManager requires all features
+# it enabled, and will be removed in a future version.
+compression = []
diff --git a/datafusion/datasource-arrow/src/mod.rs b/datafusion/datasource-arrow/src/mod.rs
index 4816a45942..0e3b0fdb5e 100644
--- a/datafusion/datasource-arrow/src/mod.rs
+++ b/datafusion/datasource-arrow/src/mod.rs
@@ -21,6 +21,10 @@
#![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]
//! [`ArrowFormat`]: Apache Arrow file format abstractions
+//!
+//! Note: As of DataFusion 54.0.0, the `compression` feature of this crate
+//! is a no-op, only kept for backwards compatibility purposes, and it will
+//! be removed in a future release.
pub mod file_format;
pub mod source;