Skip to content

Apache DataFusion SQL Query Engine

Switch from xz2 to liblzma to reduce duplicate dependencies (#17509)

Tim Saucer5765a08c842e
diff --git a/Cargo.lock b/Cargo.lock
index 6a749e8a68..1afa1e3491 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -184,15 +184,16 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
[[package]]
name = "apache-avro"
-version = "0.20.0"
+version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7"
+checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
dependencies = [
"bigdecimal",
"bon",
- "bzip2 0.6.1",
+ "bzip2",
"crc32fast",
"digest",
+ "liblzma",
"log",
"miniz_oxide",
"num-bigint",
@@ -207,7 +208,6 @@ dependencies = [
"strum_macros",
"thiserror",
"uuid",
- "xz2",
"zstd",
]
@@ -529,19 +529,15 @@ dependencies = [
[[package]]
name = "async-compression"
-version = "0.4.19"
+version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c"
+checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2"
dependencies = [
- "bzip2 0.5.2",
- "flate2",
+ "compression-codecs",
+ "compression-core",
"futures-core",
- "memchr",
"pin-project-lite",
"tokio",
- "xz2",
- "zstd",
- "zstd-safe",
]
[[package]]
@@ -1327,15 +1323,6 @@ dependencies = [
"either",
]
-[[package]]
-name = "bzip2"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47"
-dependencies = [
- "bzip2-sys",
-]
-
[[package]]
name = "bzip2"
version = "0.6.1"
@@ -1345,16 +1332,6 @@ dependencies = [
"libbz2-rs-sys",
]
-[[package]]
-name = "bzip2-sys"
-version = "0.1.13+1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14"
-dependencies = [
- "cc",
- "pkg-config",
-]
-
[[package]]
name = "cast"
version = "0.3.0"
@@ -1541,6 +1518,27 @@ dependencies = [
"unicode-width 0.2.2",
]
+[[package]]
+name = "compression-codecs"
+version = "0.4.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b"
+dependencies = [
+ "bzip2",
+ "compression-core",
+ "flate2",
+ "liblzma",
+ "memchr",
+ "zstd",
+ "zstd-safe",
+]
+
+[[package]]
+name = "compression-core"
+version = "0.4.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582"
+
[[package]]
name = "console"
version = "0.15.11"
@@ -1845,7 +1843,7 @@ dependencies = [
"arrow-schema",
"async-trait",
"bytes",
- "bzip2 0.6.1",
+ "bzip2",
"chrono",
"criterion",
"ctor",
@@ -1886,6 +1884,7 @@ dependencies = [
"glob",
"insta",
"itertools 0.14.0",
+ "liblzma",
"log",
"nix",
"object_store",
@@ -1905,7 +1904,6 @@ dependencies = [
"tokio",
"url",
"uuid",
- "xz2",
"zstd",
]
@@ -2055,7 +2053,7 @@ dependencies = [
"async-compression",
"async-trait",
"bytes",
- "bzip2 0.6.1",
+ "bzip2",
"chrono",
"criterion",
"datafusion-common",
@@ -2071,6 +2069,7 @@ dependencies = [
"futures",
"glob",
"itertools 0.14.0",
+ "liblzma",
"log",
"object_store",
"rand 0.9.2",
@@ -2078,7 +2077,6 @@ dependencies = [
"tokio",
"tokio-util",
"url",
- "xz2",
"zstd",
]
@@ -4001,6 +3999,26 @@ dependencies = [
"windows-link 0.2.1",
]
+[[package]]
+name = "liblzma"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648"
+dependencies = [
+ "liblzma-sys",
+]
+
+[[package]]
+name = "liblzma-sys"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
[[package]]
name = "libm"
version = "0.2.15"
@@ -4092,17 +4110,6 @@ dependencies = [
"twox-hash",
]
-[[package]]
-name = "lzma-sys"
-version = "0.1.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
-]
-
[[package]]
name = "matchit"
version = "0.8.4"
@@ -7491,15 +7498,6 @@ version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
-[[package]]
-name = "xz2"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
-dependencies = [
- "lzma-sys",
-]
-
[[package]]
name = "yansi"
version = "1.0.1"
diff --git a/Cargo.toml b/Cargo.toml
index 5a79c7fe2b..e5acbd2022 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -90,7 +90,7 @@ version = "51.0.0"
ahash = { version = "0.8", default-features = false, features = [
"runtime-rng",
] }
-apache-avro = { version = "0.20", default-features = false }
+apache-avro = { version = "0.21", default-features = false }
arrow = { version = "57.0.0", features = [
"prettyprint",
"chrono-tz",
@@ -158,6 +158,7 @@ hex = { version = "0.4.3" }
indexmap = "2.12.0"
insta = { version = "1.43.2", features = ["glob", "filters"] }
itertools = "0.14"
+liblzma = { version = "0.4.4", features = ["static"] }
log = "^0.4"
num-traits = { version = "0.2" }
object_store = { version = "0.12.4", default-features = false }
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index b222ae12b9..030c53101c 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -56,7 +56,7 @@ sql = ["sqlparser"]
[dependencies]
ahash = { workspace = true }
-apache-avro = { version = "0.20", default-features = false, features = [
+apache-avro = { workspace = true, features = [
"bzip",
"snappy",
"xz",
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index 67a73ac6f6..494baa187c 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -46,7 +46,7 @@ array_expressions = ["nested_expressions"]
avro = ["datafusion-common/avro", "datafusion-datasource-avro"]
backtrace = ["datafusion-common/backtrace"]
compression = [
- "xz2",
+ "liblzma",
"bzip2",
"flate2",
"zstd",
@@ -146,6 +146,7 @@ datafusion-sql = { workspace = true, optional = true }
flate2 = { version = "1.1.4", optional = true }
futures = { workspace = true }
itertools = { workspace = true }
+liblzma = { workspace = true, optional = true }
log = { workspace = true }
object_store = { workspace = true }
parking_lot = { workspace = true }
@@ -159,7 +160,6 @@ tempfile = { workspace = true }
tokio = { workspace = true }
url = { workspace = true }
uuid = { version = "1.18", features = ["v4", "js"] }
-xz2 = { version = "0.1", optional = true, features = ["static"] }
zstd = { version = "0.13", optional = true, default-features = false }
[dev-dependencies]
diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rs
index 3a50167594..2bf9c89576 100644
--- a/datafusion/core/src/test/mod.rs
+++ b/datafusion/core/src/test/mod.rs
@@ -56,9 +56,9 @@ use datafusion_datasource_csv::partitioned_csv_config;
use flate2::write::GzEncoder;
#[cfg(feature = "compression")]
use flate2::Compression as GzCompression;
-use object_store::local_unpartitioned_file;
#[cfg(feature = "compression")]
-use xz2::write::XzEncoder;
+use liblzma::write::XzEncoder;
+use object_store::local_unpartitioned_file;
#[cfg(feature = "compression")]
use zstd::Encoder as ZstdEncoder;
diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml
index 19b247829d..e37752aa54 100644
--- a/datafusion/datasource/Cargo.toml
+++ b/datafusion/datasource/Cargo.toml
@@ -31,12 +31,12 @@ version.workspace = true
all-features = true
[features]
-compression = ["async-compression", "xz2", "bzip2", "flate2", "zstd", "tokio-util"]
+compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
default = ["compression"]
[dependencies]
arrow = { workspace = true }
-async-compression = { version = "0.4.19", features = [
+async-compression = { version = "0.4.30", features = [
"bzip2",
"gzip",
"xz",
@@ -60,6 +60,7 @@ flate2 = { version = "1.1.4", optional = true }
futures = { workspace = true }
glob = "0.3.0"
itertools = { workspace = true }
+liblzma = { workspace = true, optional = true }
log = { workspace = true }
object_store = { workspace = true }
rand = { workspace = true }
@@ -67,7 +68,6 @@ tempfile = { workspace = true, optional = true }
tokio = { workspace = true }
tokio-util = { version = "0.7.16", features = ["io"], optional = true }
url = { workspace = true }
-xz2 = { version = "0.1", optional = true, features = ["static"] }
zstd = { version = "0.13", optional = true, default-features = false }
[dev-dependencies]
diff --git a/datafusion/datasource/src/file_compression_type.rs b/datafusion/datasource/src/file_compression_type.rs
index 7cc3142564..9ca5d8763b 100644
--- a/datafusion/datasource/src/file_compression_type.rs
+++ b/datafusion/datasource/src/file_compression_type.rs
@@ -43,13 +43,13 @@ use futures::stream::BoxStream;
use futures::StreamExt;
#[cfg(feature = "compression")]
use futures::TryStreamExt;
+#[cfg(feature = "compression")]
+use liblzma::read::XzDecoder;
use object_store::buffered::BufWriter;
use tokio::io::AsyncWrite;
#[cfg(feature = "compression")]
use tokio_util::io::{ReaderStream, StreamReader};
#[cfg(feature = "compression")]
-use xz2::read::XzDecoder;
-#[cfg(feature = "compression")]
use zstd::Decoder as ZstdDecoder;
/// Readable file compression type