Apache DataFusion SQL Query Engine
Switch from xz2 to liblzma to reduce duplicate dependencies (#17509)
Tim Saucer5765a08c842e
diff --git a/Cargo.lock b/Cargo.lockindex 6a749e8a68..1afa1e3491 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -184,15 +184,16 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"[[package]]name = "apache-avro"-version = "0.20.0"+version = "0.21.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7"+checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"dependencies = ["bigdecimal","bon",- "bzip2 0.6.1",+ "bzip2","crc32fast","digest",+ "liblzma","log","miniz_oxide","num-bigint",@@ -207,7 +208,6 @@ dependencies = ["strum_macros","thiserror","uuid",- "xz2","zstd",]@@ -529,19 +529,15 @@ dependencies = [[[package]]name = "async-compression"-version = "0.4.19"+version = "0.4.33"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c"+checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2"dependencies = [- "bzip2 0.5.2",- "flate2",+ "compression-codecs",+ "compression-core","futures-core",- "memchr","pin-project-lite","tokio",- "xz2",- "zstd",- "zstd-safe",][[package]]@@ -1327,15 +1323,6 @@ dependencies = ["either",]-[[package]]-name = "bzip2"-version = "0.5.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47"-dependencies = [- "bzip2-sys",-]-[[package]]name = "bzip2"version = "0.6.1"@@ -1345,16 +1332,6 @@ dependencies = ["libbz2-rs-sys",]-[[package]]-name = "bzip2-sys"-version = "0.1.13+1.0.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14"-dependencies = [- "cc",- "pkg-config",-]-[[package]]name = "cast"version = "0.3.0"@@ -1541,6 +1518,27 @@ dependencies = ["unicode-width 0.2.2",]+[[package]]+name = "compression-codecs"+version = "0.4.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b"+dependencies = [+ "bzip2",+ "compression-core",+ "flate2",+ "liblzma",+ "memchr",+ "zstd",+ "zstd-safe",+]++[[package]]+name = "compression-core"+version = "0.4.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582"+[[package]]name = "console"version = "0.15.11"@@ -1845,7 +1843,7 @@ dependencies = ["arrow-schema","async-trait","bytes",- "bzip2 0.6.1",+ "bzip2","chrono","criterion","ctor",@@ -1886,6 +1884,7 @@ dependencies = ["glob","insta","itertools 0.14.0",+ "liblzma","log","nix","object_store",@@ -1905,7 +1904,6 @@ dependencies = ["tokio","url","uuid",- "xz2","zstd",]@@ -2055,7 +2053,7 @@ dependencies = ["async-compression","async-trait","bytes",- "bzip2 0.6.1",+ "bzip2","chrono","criterion","datafusion-common",@@ -2071,6 +2069,7 @@ dependencies = ["futures","glob","itertools 0.14.0",+ "liblzma","log","object_store","rand 0.9.2",@@ -2078,7 +2077,6 @@ dependencies = ["tokio","tokio-util","url",- "xz2","zstd",]@@ -4001,6 +3999,26 @@ dependencies = ["windows-link 0.2.1",]+[[package]]+name = "liblzma"+version = "0.4.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648"+dependencies = [+ "liblzma-sys",+]++[[package]]+name = "liblzma-sys"+version = "0.4.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736"+dependencies = [+ "cc",+ "libc",+ "pkg-config",+]+[[package]]name = "libm"version = "0.2.15"@@ -4092,17 +4110,6 @@ dependencies = ["twox-hash",]-[[package]]-name = "lzma-sys"-version = "0.1.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"-dependencies = [- "cc",- "libc",- "pkg-config",-]-[[package]]name = "matchit"version = "0.8.4"@@ -7491,15 +7498,6 @@ version = "0.13.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"-[[package]]-name = "xz2"-version = "0.1.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"-dependencies = [- "lzma-sys",-]-[[package]]name = "yansi"version = "1.0.1"diff --git a/Cargo.toml b/Cargo.tomlindex 5a79c7fe2b..e5acbd2022 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -90,7 +90,7 @@ version = "51.0.0"ahash = { version = "0.8", default-features = false, features = ["runtime-rng",] }-apache-avro = { version = "0.20", default-features = false }+apache-avro = { version = "0.21", default-features = false }arrow = { version = "57.0.0", features = ["prettyprint","chrono-tz",@@ -158,6 +158,7 @@ hex = { version = "0.4.3" }indexmap = "2.12.0"insta = { version = "1.43.2", features = ["glob", "filters"] }itertools = "0.14"+liblzma = { version = "0.4.4", features = ["static"] }log = "^0.4"num-traits = { version = "0.2" }object_store = { version = "0.12.4", default-features = false }diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.tomlindex b222ae12b9..030c53101c 100644--- a/datafusion/common/Cargo.toml+++ b/datafusion/common/Cargo.toml@@ -56,7 +56,7 @@ sql = ["sqlparser"][dependencies]ahash = { workspace = true }-apache-avro = { version = "0.20", default-features = false, features = [+apache-avro = { workspace = true, features = ["bzip","snappy","xz",diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex 67a73ac6f6..494baa187c 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -46,7 +46,7 @@ array_expressions = ["nested_expressions"]avro = ["datafusion-common/avro", "datafusion-datasource-avro"]backtrace = ["datafusion-common/backtrace"]compression = [- "xz2",+ "liblzma","bzip2","flate2","zstd",@@ -146,6 +146,7 @@ datafusion-sql = { workspace = true, optional = true }flate2 = { version = "1.1.4", optional = true }futures = { workspace = true }itertools = { workspace = true }+liblzma = { workspace = true, optional = true }log = { workspace = true }object_store = { workspace = true }parking_lot = { workspace = true }@@ -159,7 +160,6 @@ tempfile = { workspace = true }tokio = { workspace = true }url = { workspace = true }uuid = { version = "1.18", features = ["v4", "js"] }-xz2 = { version = "0.1", optional = true, features = ["static"] }zstd = { version = "0.13", optional = true, default-features = false }[dev-dependencies]diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rsindex 3a50167594..2bf9c89576 100644--- a/datafusion/core/src/test/mod.rs+++ b/datafusion/core/src/test/mod.rs@@ -56,9 +56,9 @@ use datafusion_datasource_csv::partitioned_csv_config;use flate2::write::GzEncoder;#[cfg(feature = "compression")]use flate2::Compression as GzCompression;-use object_store::local_unpartitioned_file;#[cfg(feature = "compression")]-use xz2::write::XzEncoder;+use liblzma::write::XzEncoder;+use object_store::local_unpartitioned_file;#[cfg(feature = "compression")]use zstd::Encoder as ZstdEncoder;diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.tomlindex 19b247829d..e37752aa54 100644--- a/datafusion/datasource/Cargo.toml+++ b/datafusion/datasource/Cargo.toml@@ -31,12 +31,12 @@ version.workspace = trueall-features = true[features]-compression = ["async-compression", "xz2", "bzip2", "flate2", "zstd", "tokio-util"]+compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]default = ["compression"][dependencies]arrow = { workspace = true }-async-compression = { version = "0.4.19", features = [+async-compression = { version = "0.4.30", features = ["bzip2","gzip","xz",@@ -60,6 +60,7 @@ flate2 = { version = "1.1.4", optional = true }futures = { workspace = true }glob = "0.3.0"itertools = { workspace = true }+liblzma = { workspace = true, optional = true }log = { workspace = true }object_store = { workspace = true }rand = { workspace = true }@@ -67,7 +68,6 @@ tempfile = { workspace = true, optional = true }tokio = { workspace = true }tokio-util = { version = "0.7.16", features = ["io"], optional = true }url = { workspace = true }-xz2 = { version = "0.1", optional = true, features = ["static"] }zstd = { version = "0.13", optional = true, default-features = false }[dev-dependencies]diff --git a/datafusion/datasource/src/file_compression_type.rs b/datafusion/datasource/src/file_compression_type.rsindex 7cc3142564..9ca5d8763b 100644--- a/datafusion/datasource/src/file_compression_type.rs+++ b/datafusion/datasource/src/file_compression_type.rs@@ -43,13 +43,13 @@ use futures::stream::BoxStream;use futures::StreamExt;#[cfg(feature = "compression")]use futures::TryStreamExt;+#[cfg(feature = "compression")]+use liblzma::read::XzDecoder;use object_store::buffered::BufWriter;use tokio::io::AsyncWrite;#[cfg(feature = "compression")]use tokio_util::io::{ReaderStream, StreamReader};#[cfg(feature = "compression")]-use xz2::read::XzDecoder;-#[cfg(feature = "compression")]use zstd::Decoder as ZstdDecoder;/// Readable file compression type