Apache DataFusion SQL Query Engine
chore: Update to arrow/parquet 59.0.0 (#22744)
Andrew Lambdae03ee062b2
diff --git a/Cargo.lock b/Cargo.lockindex def7f88448..ed90dd25bd 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -164,9 +164,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"[[package]]name = "arrow"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"+checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7"dependencies = ["arrow-arith","arrow-array",@@ -187,9 +187,9 @@ dependencies = [[[package]]name = "arrow-arith"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"+checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc"dependencies = ["arrow-array","arrow-buffer",@@ -201,9 +201,9 @@ dependencies = [[[package]]name = "arrow-array"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"+checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137"dependencies = ["ahash","arrow-buffer",@@ -220,9 +220,9 @@ dependencies = [[[package]]name = "arrow-avro"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"+checksum = "2835d67df2b69bf5de251ee6d289f85650d41b8169dcee0f950ea88747812c32"dependencies = ["arrow-array","arrow-buffer",@@ -244,9 +244,9 @@ dependencies = [[[package]]name = "arrow-buffer"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"+checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a"dependencies = ["bytes","half",@@ -256,9 +256,9 @@ dependencies = [[[package]]name = "arrow-cast"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"+checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9"dependencies = ["arrow-array","arrow-buffer",@@ -278,9 +278,9 @@ dependencies = [[[package]]name = "arrow-csv"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"+checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7"dependencies = ["arrow-array","arrow-cast",@@ -293,9 +293,9 @@ dependencies = [[[package]]name = "arrow-data"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"+checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab"dependencies = ["arrow-buffer","arrow-schema",@@ -306,9 +306,9 @@ dependencies = [[[package]]name = "arrow-flight"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704"+checksum = "68365401e834743d708094927e2ca727a32d639fe900df04b936e07a36701b74"dependencies = ["arrow-arith","arrow-array",@@ -334,9 +334,9 @@ dependencies = [[[package]]name = "arrow-ipc"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"+checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5"dependencies = ["arrow-array","arrow-buffer",@@ -350,9 +350,9 @@ dependencies = [[[package]]name = "arrow-json"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"+checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed"dependencies = ["arrow-array","arrow-buffer",@@ -375,9 +375,9 @@ dependencies = [[[package]]name = "arrow-ord"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"+checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983"dependencies = ["arrow-array","arrow-buffer",@@ -388,9 +388,9 @@ dependencies = [[[package]]name = "arrow-row"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"+checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879"dependencies = ["arrow-array","arrow-buffer",@@ -401,9 +401,9 @@ dependencies = [[[package]]name = "arrow-schema"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"+checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1"dependencies = ["bitflags","serde",@@ -413,9 +413,9 @@ dependencies = [[[package]]name = "arrow-select"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"+checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624"dependencies = ["ahash","arrow-array",@@ -427,9 +427,9 @@ dependencies = [[[package]]name = "arrow-string"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"+checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d"dependencies = ["arrow-array","arrow-buffer",@@ -3730,12 +3730,6 @@ dependencies = ["serde_json",]-[[package]]-name = "integer-encoding"-version = "3.0.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"-[[package]]name = "ipnet"version = "2.12.0"@@ -4389,15 +4383,6 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"-[[package]]-name = "ordered-float"-version = "2.10.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"-dependencies = [- "num-traits",-]-[[package]]name = "outref"version = "0.5.2"@@ -4445,9 +4430,9 @@ dependencies = [[[package]]name = "parquet"-version = "58.3.0"+version = "59.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"+checksum = "970dff83e97d953c827ae8176f6bf4e9f77bf62daacc01ec5df348ec5eacd913"dependencies = ["ahash","arrow-array",@@ -4474,7 +4459,6 @@ dependencies = ["seq-macro","simdutf8","snap",- "thrift","tokio","twox-hash","zstd",@@ -4834,7 +4818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"dependencies = ["heck",- "itertools 0.14.0",+ "itertools 0.13.0","log","multimap","petgraph",@@ -4853,7 +4837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"dependencies = ["anyhow",- "itertools 0.14.0",+ "itertools 0.13.0","proc-macro2","quote","syn 2.0.117",@@ -6147,17 +6131,6 @@ dependencies = ["cfg-if",]-[[package]]-name = "thrift"-version = "0.17.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"-dependencies = [- "byteorder",- "integer-encoding",- "ordered-float",-]-[[package]]name = "time"version = "0.3.47"diff --git a/Cargo.toml b/Cargo.tomlindex 61b999d818..773a38ac50 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -89,30 +89,30 @@ version = "54.0.0"## See for more details: https://github.com/rust-lang/cargo/issues/11329apache-avro = { version = "0.21", default-features = false }-arrow = { version = "58.3.0", features = [+arrow = { version = "59.0.0", features = ["prettyprint","chrono-tz",] }-arrow-avro = { version = "58.3.0", default-features = false, features = [+arrow-avro = { version = "59.0.0", default-features = false, features = ["deflate","snappy","zstd","bzip2","xz",] }-arrow-buffer = { version = "58.3.0", default-features = false }-arrow-data = { version = "58.3.0", default-features = false }-arrow-flight = { version = "58.3.0", features = [+arrow-buffer = { version = "59.0.0", default-features = false }+arrow-data = { version = "59.0.0", default-features = false }+arrow-flight = { version = "59.0.0", features = ["flight-sql-experimental",] }# Both codecs are required here to make sure that code paths like# file-spilling have access to all compression codecs.-arrow-ipc = { version = "58.3.0", default-features = false, features = [+arrow-ipc = { version = "59.0.0", default-features = false, features = ["lz4","zstd",] }-arrow-ord = { version = "58.3.0", default-features = false }-arrow-schema = { version = "58.3.0", default-features = false }+arrow-ord = { version = "59.0.0", default-features = false }+arrow-schema = { version = "59.0.0", default-features = false }async-trait = "0.1.89"bigdecimal = "0.4.8"bytes = "1.11"@@ -178,7 +178,7 @@ memchr = "2.8.1"num-traits = { version = "0.2" }object_store = { version = "0.13.2", default-features = false }parking_lot = "0.12"-parquet = { version = "58.3.0", default-features = false, features = [+parquet = { version = "59.0.0", default-features = false, features = ["arrow","async","object_store",diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rsindex 935bf0a974..0d8ada1367 100644--- a/datafusion-cli/src/main.rs+++ b/datafusion-cli/src/main.rs@@ -613,9 +613,9 @@ mod tests {+-----------------------------------+-----------------+---------------------+------+------------------+| filename | file_size_bytes | metadata_size_bytes | hits | extra |+-----------------------------------+-----------------+---------------------+------+------------------+- | alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false |- | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |- | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 2 | page_index=false |+ | alltypes_plain.parquet | 1851 | 8794 | 2 | page_index=false |+ | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |+ | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false |+-----------------------------------+-----------------+---------------------+------+------------------+");@@ -644,9 +644,9 @@ mod tests {+-----------------------------------+-----------------+---------------------+------+------------------+| filename | file_size_bytes | metadata_size_bytes | hits | extra |+-----------------------------------+-----------------+---------------------+------+------------------+- | alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false |- | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |- | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 3 | page_index=false |+ | alltypes_plain.parquet | 1851 | 8794 | 5 | page_index=false |+ | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |+ | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 3 | page_index=false |+-----------------------------------+-----------------+---------------------+------+------------------+");diff --git a/datafusion/common/src/file_options/mod.rs b/datafusion/common/src/file_options/mod.rsindex 5d2abd2317..97b4a44f03 100644--- a/datafusion/common/src/file_options/mod.rs+++ b/datafusion/common/src/file_options/mod.rs@@ -114,14 +114,14 @@ mod tests {properties.bloom_filter_properties(&ColumnPath::from("")).expect("expected bloom properties!")- .fpp,+ .fpp(),0.123);assert_eq!(properties.bloom_filter_properties(&ColumnPath::from("")).expect("expected bloom properties!")- .ndv,+ .ndv(),123);@@ -242,7 +242,7 @@ mod tests {properties.bloom_filter_properties(&col1).expect("expected bloom properties!")- .fpp,+ .fpp(),0.123);@@ -250,7 +250,7 @@ mod tests {properties.bloom_filter_properties(&col2_nested).expect("expected bloom properties!")- .fpp,+ .fpp(),0.456);@@ -258,7 +258,7 @@ mod tests {properties.bloom_filter_properties(&col1).expect("expected bloom properties!")- .ndv,+ .ndv(),123);@@ -266,7 +266,7 @@ mod tests {properties.bloom_filter_properties(&col2_nested).expect("expected bloom properties!")- .ndv,+ .ndv(),456);diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rsindex a5b270a8f5..320bfcf33e 100644--- a/datafusion/common/src/file_options/parquet_writer.rs+++ b/datafusion/common/src/file_options/parquet_writer.rs@@ -157,8 +157,8 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {}if let Some(bloom_filter_ndv) = options.bloom_filter_ndv {- builder =- builder.set_column_bloom_filter_ndv(path.clone(), bloom_filter_ndv);+ builder = builder+ .set_column_bloom_filter_max_ndv(path.clone(), bloom_filter_ndv);}}@@ -273,7 +273,7 @@ impl ParquetOptions {builder = builder.set_bloom_filter_fpp(*bloom_filter_fpp);};if let Some(bloom_filter_ndv) = bloom_filter_ndv {- builder = builder.set_bloom_filter_ndv(*bloom_filter_ndv);+ builder = builder.set_bloom_filter_max_ndv(*bloom_filter_ndv);};if let Some(dictionary_enabled) = dictionary_enabled {builder = builder.set_dictionary_enabled(*dictionary_enabled);@@ -534,8 +534,8 @@ mod tests {}.into(),),- bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp),- bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv),+ bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp()),+ bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv()),}}@@ -830,10 +830,12 @@ mod tests {);assert_eq!(default_writer_props.bloom_filter_properties(&"default".into()),- Some(&BloomFilterProperties {- fpp: 0.42,- ndv: DEFAULT_BLOOM_FILTER_NDV- }),+ Some(+ &BloomFilterProperties::builder()+ .with_fpp(0.42)+ .with_max_ndv(DEFAULT_BLOOM_FILTER_NDV)+ .build()+ ),"should have only the fpp set, and the ndv at default",);}@@ -937,7 +939,7 @@ mod tests {// the WriterProperties::default, with only ndv setlet default_writer_props = WriterProperties::builder().set_bloom_filter_enabled(true)- .set_bloom_filter_ndv(42)+ .set_bloom_filter_max_ndv(42).build();assert_eq!(@@ -947,10 +949,12 @@ mod tests {);assert_eq!(default_writer_props.bloom_filter_properties(&"default".into()),- Some(&BloomFilterProperties {- fpp: DEFAULT_BLOOM_FILTER_FPP,- ndv: 42- }),+ Some(+ &BloomFilterProperties::builder()+ .with_fpp(DEFAULT_BLOOM_FILTER_FPP)+ .with_max_ndv(42)+ .build()+ ),"should have only the ndv set, and the fpp at default",);}diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rsindex 83b84f6f92..25150ae284 100644--- a/datafusion/core/tests/datasource/object_store_access.rs+++ b/datafusion/core/tests/datasource/object_store_access.rs@@ -904,7 +904,7 @@ async fn query_single_parquet_file_with_single_predicate() {RequestCountingObjectStore()Total Requests: 2- GET (opts) path=parquet_table.parquet head=true- - GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124+ - GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124");}@@ -928,8 +928,8 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() {RequestCountingObjectStore()Total Requests: 3- GET (opts) path=parquet_table.parquet head=true- - GET (ranges) path=parquet_table.parquet ranges=4-421,421-534,534-951,951-1064- - GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124+ - GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064+ - GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124");}diff --git a/datafusion/core/tests/extension_types/pretty_printing.rs b/datafusion/core/tests/extension_types/pretty_printing.rsindex c0796887b8..f097b5bec9 100644--- a/datafusion/core/tests/extension_types/pretty_printing.rs+++ b/datafusion/core/tests/extension_types/pretty_printing.rs@@ -40,10 +40,16 @@ async fn create_test_table() -> Result<DataFrame> {// define data.let batch = RecordBatch::try_new(schema,- vec![Arc::new(FixedSizeBinaryArray::from(vec![- &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],- &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],- ]))],+ vec![Arc::new(+ FixedSizeBinaryArray::try_from_iter(+ vec![+ &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],+ &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],+ ]+ .into_iter(),+ )+ .unwrap(),+ )],)?;let state = SessionStateBuilder::default()diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rsindex 0e936a79eb..12296f8498 100644--- a/datafusion/core/tests/parquet/mod.rs+++ b/datafusion/core/tests/parquet/mod.rs@@ -726,11 +726,11 @@ fn make_bytearray_batch(let name: StringArray = std::iter::repeat_n(Some(name), num_rows).collect();let service_string: StringArray = string_values.iter().map(Some).collect();let service_binary: BinaryArray = binary_values.iter().map(Some).collect();- let service_fixedsize: FixedSizeBinaryArray = fixedsize_values- .iter()- .map(|value| Some(value.as_slice()))- .collect::<Vec<_>>()- .into();+ let service_fixedsize = FixedSizeBinaryArray::try_from_sparse_iter_with_size(+ fixedsize_values.iter().map(|value| Some(value.as_slice())),+ 3,+ )+ .unwrap();let service_large_binary: LargeBinaryArray =large_binary_values.iter().map(Some).collect();diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 07f4fe92cf..1e9b0636e5 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -955,10 +955,7 @@ mod tests {let schema =Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 2), false)]));let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)- .with_logical_type(LogicalType::Decimal {- scale: 2,- precision: 9,- })+ .with_logical_type(LogicalType::decimal(2, 9)).with_scale(2).with_precision(9);let schema_descr = get_test_schema_descr(vec![field]);@@ -1023,10 +1020,7 @@ mod tests {Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 0), false)]));let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)- .with_logical_type(LogicalType::Decimal {- scale: 0,- precision: 9,- })+ .with_logical_type(LogicalType::decimal(0, 9)).with_scale(0).with_precision(9);let schema_descr = get_test_schema_descr(vec![field]);@@ -1118,10 +1112,7 @@ mod tests {false,)]));let field = PrimitiveTypeField::new("c1", PhysicalType::INT64)- .with_logical_type(LogicalType::Decimal {- scale: 2,- precision: 18,- })+ .with_logical_type(LogicalType::decimal(2, 18)).with_scale(2).with_precision(18);let schema_descr = get_test_schema_descr(vec![field]);@@ -1176,10 +1167,7 @@ mod tests {false,)]));let field = PrimitiveTypeField::new("c1", PhysicalType::FIXED_LEN_BYTE_ARRAY)- .with_logical_type(LogicalType::Decimal {- scale: 2,- precision: 18,- })+ .with_logical_type(LogicalType::decimal(2, 18)).with_scale(2).with_precision(18).with_byte_len(16);@@ -1254,10 +1242,7 @@ mod tests {false,)]));let field = PrimitiveTypeField::new("c1", PhysicalType::BYTE_ARRAY)- .with_logical_type(LogicalType::Decimal {- scale: 2,- precision: 18,- })+ .with_logical_type(LogicalType::decimal(2, 18)).with_scale(2).with_precision(18).with_byte_len(16);diff --git a/datafusion/physical-expr-common/src/utils.rs b/datafusion/physical-expr-common/src/utils.rsindex e469885f83..117da23df2 100644--- a/datafusion/physical-expr-common/src/utils.rs+++ b/datafusion/physical-expr-common/src/utils.rs@@ -614,11 +614,9 @@ mod tests {#[test]fn scatter_fixed_size_binary_test() -> Result<()> {- let truthy = Arc::new(FixedSizeBinaryArray::from(vec![- &[1u8, 2][..],- &[3, 4][..],- &[5, 6][..],- ]));+ let truthy = Arc::new(FixedSizeBinaryArray::try_from_iter(+ vec![&[1u8, 2][..], &[3, 4][..], &[5, 6][..]].into_iter(),+ )?);let mask = BooleanArray::from(vec![true, false, true, false, true]);let result = scatter(&mask, truthy.as_ref())?;diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rsindex 5c4922fdcf..3eff1f5cf3 100644--- a/datafusion/physical-expr/benches/in_list_strategy.rs+++ b/datafusion/physical-expr/benches/in_list_strategy.rs@@ -993,7 +993,7 @@ fn bench_fixed_size_binary_inner(.collect();let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();- let array = FixedSizeBinaryArray::from(refs);+ let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap();let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]);let exprs: Vec<_> = haystackdiff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex c4377b3189..b1fdf3ddab 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -175,7 +175,7 @@ fn build_fixed_size_binary_table(let batch = RecordBatch::try_new(Arc::new(schema),vec![- Arc::new(FixedSizeBinaryArray::from(a.1.clone())),+ Arc::new(FixedSizeBinaryArray::try_from_iter(a.1.iter().copied()).unwrap()),Arc::new(Int32Array::from(b.1.clone())),Arc::new(Int32Array::from(c.1.clone())),],diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rsindex 5dca47bcb8..9d02a51b22 100644--- a/datafusion/spark/src/function/hash/xxhash64.rs+++ b/datafusion/spark/src/function/hash/xxhash64.rs@@ -363,12 +363,17 @@ mod tests {#[test]fn test_xxhash64_fixed_size_binary() {- let array = FixedSizeBinaryArray::from(vec![- Some(&[0x01, 0x02, 0x03, 0x04][..]),- Some(&[0x05, 0x06, 0x07, 0x08][..]),- None,- Some(&[0x00, 0x00, 0x00, 0x00][..]),- ]);+ let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(+ vec![+ Some(&[0x01, 0x02, 0x03, 0x04][..]),+ Some(&[0x05, 0x06, 0x07, 0x08][..]),+ None,+ Some(&[0x00, 0x00, 0x00, 0x00][..]),+ ]+ .into_iter(),+ 4,+ )+ .unwrap();let array_ref: ArrayRef = Arc::new(array);let mut hashes = vec![DEFAULT_SEED; 4];