Apache DataFusion SQL Query Engine
Update to `arrow`, `parquet` to `57.1.0` (#18820)
Andrew Lamba06073913ba8
diff --git a/Cargo.lock b/Cargo.lockindex b25e60eb6f..6415fa65c0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -234,9 +234,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"[[package]]name = "arrow"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"+checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3"dependencies = ["arrow-arith","arrow-array",@@ -257,9 +257,9 @@ dependencies = [[[package]]name = "arrow-arith"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"+checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049"dependencies = ["arrow-array","arrow-buffer",@@ -271,9 +271,9 @@ dependencies = [[[package]]name = "arrow-array"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"+checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002"dependencies = ["ahash 0.8.12","arrow-buffer",@@ -290,9 +290,9 @@ dependencies = [[[package]]name = "arrow-buffer"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27"+checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626"dependencies = ["bytes","half",@@ -302,13 +302,14 @@ dependencies = [[[package]]name = "arrow-cast"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"+checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271"dependencies = ["arrow-array","arrow-buffer","arrow-data",+ "arrow-ord","arrow-schema","arrow-select","atoi",@@ -323,9 +324,9 @@ dependencies = [[[package]]name = "arrow-csv"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"+checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b"dependencies = ["arrow-array","arrow-cast",@@ -338,9 +339,9 @@ dependencies = [[[package]]name = "arrow-data"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"+checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3"dependencies = ["arrow-buffer","arrow-schema",@@ -351,9 +352,9 @@ dependencies = [[[package]]name = "arrow-flight"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9"+checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3"dependencies = ["arrow-arith","arrow-array",@@ -379,9 +380,9 @@ dependencies = [[[package]]name = "arrow-ipc"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"+checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3"dependencies = ["arrow-array","arrow-buffer",@@ -395,9 +396,9 @@ dependencies = [[[package]]name = "arrow-json"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"+checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0"dependencies = ["arrow-array","arrow-buffer",@@ -419,9 +420,9 @@ dependencies = [[[package]]name = "arrow-ord"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"+checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521"dependencies = ["arrow-array","arrow-buffer",@@ -432,9 +433,9 @@ dependencies = [[[package]]name = "arrow-row"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"+checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f"dependencies = ["arrow-array","arrow-buffer",@@ -445,9 +446,9 @@ dependencies = [[[package]]name = "arrow-schema"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"+checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0"dependencies = ["bitflags 2.9.4","serde",@@ -457,9 +458,9 @@ dependencies = [[[package]]name = "arrow-select"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"+checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010"dependencies = ["ahash 0.8.12","arrow-array",@@ -471,9 +472,9 @@ dependencies = [[[package]]name = "arrow-string"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"+checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534"dependencies = ["arrow-array","arrow-buffer",@@ -2813,7 +2814,7 @@ dependencies = ["libc","option-ext","redox_users",- "windows-sys 0.61.0",+ "windows-sys 0.60.2",][[package]]@@ -2957,7 +2958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"dependencies = ["libc",- "windows-sys 0.61.0",+ "windows-sys 0.52.0",][[package]]@@ -3003,7 +3004,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"dependencies = ["cfg-if","rustix",- "windows-sys 0.59.0",+ "windows-sys 0.52.0",][[package]]@@ -3597,7 +3598,7 @@ dependencies = ["js-sys","log","wasm-bindgen",- "windows-core 0.62.0",+ "windows-core",][[package]]@@ -4080,9 +4081,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"[[package]]name = "lz4_flex"-version = "0.11.5"+version = "0.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"+checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"dependencies = ["twox-hash",]@@ -4439,9 +4440,9 @@ dependencies = [[[package]]name = "parquet"-version = "57.0.0"+version = "57.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a"+checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89"dependencies = ["ahash 0.8.12","arrow-array",@@ -4835,7 +4836,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"dependencies = ["heck 0.5.0",- "itertools 0.14.0",+ "itertools 0.13.0","log","multimap","once_cell",@@ -4855,7 +4856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"dependencies = ["anyhow",- "itertools 0.14.0",+ "itertools 0.13.0","proc-macro2","quote","syn 2.0.111",@@ -5395,7 +5396,7 @@ dependencies = ["errno","libc","linux-raw-sys",- "windows-sys 0.61.0",+ "windows-sys 0.52.0",][[package]]@@ -5931,7 +5932,7 @@ dependencies = ["cfg-if","libc","psm",- "windows-sys 0.59.0",+ "windows-sys 0.52.0",][[package]]@@ -6149,7 +6150,7 @@ dependencies = ["getrandom 0.3.4","once_cell","rustix",- "windows-sys 0.61.0",+ "windows-sys 0.52.0",][[package]]@@ -7046,7 +7047,7 @@ version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"dependencies = [- "windows-sys 0.61.0",+ "windows-sys 0.52.0",][[package]]@@ -7062,7 +7063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"dependencies = ["windows-collections",- "windows-core 0.61.2",+ "windows-core","windows-future","windows-link 0.1.3","windows-numerics",@@ -7074,7 +7075,7 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"dependencies = [- "windows-core 0.61.2",+ "windows-core",][[package]]@@ -7086,21 +7087,8 @@ dependencies = ["windows-implement","windows-interface","windows-link 0.1.3",- "windows-result 0.3.4",- "windows-strings 0.4.2",-]--[[package]]-name = "windows-core"-version = "0.62.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c"-dependencies = [- "windows-implement",- "windows-interface",- "windows-link 0.2.0",- "windows-result 0.4.0",- "windows-strings 0.5.0",+ "windows-result",+ "windows-strings",][[package]]@@ -7109,7 +7097,7 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"dependencies = [- "windows-core 0.61.2",+ "windows-core","windows-link 0.1.3","windows-threading",]@@ -7154,7 +7142,7 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"dependencies = [- "windows-core 0.61.2",+ "windows-core","windows-link 0.1.3",]@@ -7167,15 +7155,6 @@ dependencies = ["windows-link 0.1.3",]-[[package]]-name = "windows-result"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f"-dependencies = [- "windows-link 0.2.0",-]-[[package]]name = "windows-strings"version = "0.4.2"@@ -7185,15 +7164,6 @@ dependencies = ["windows-link 0.1.3",]-[[package]]-name = "windows-strings"-version = "0.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda"-dependencies = [- "windows-link 0.2.0",-]-[[package]]name = "windows-sys"version = "0.52.0"diff --git a/Cargo.toml b/Cargo.tomlindex 846fd297e0..dd6543db98 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -91,19 +91,19 @@ ahash = { version = "0.8", default-features = false, features = ["runtime-rng",] }apache-avro = { version = "0.21", default-features = false }-arrow = { version = "57.0.0", features = [+arrow = { version = "57.1.0", features = ["prettyprint","chrono-tz",] }-arrow-buffer = { version = "57.0.0", default-features = false }-arrow-flight = { version = "57.0.0", features = [+arrow-buffer = { version = "57.1.0", default-features = false }+arrow-flight = { version = "57.1.0", features = ["flight-sql-experimental",] }-arrow-ipc = { version = "57.0.0", default-features = false, features = [+arrow-ipc = { version = "57.1.0", default-features = false, features = ["lz4",] }-arrow-ord = { version = "57.0.0", default-features = false }-arrow-schema = { version = "57.0.0", default-features = false }+arrow-ord = { version = "57.1.0", default-features = false }+arrow-schema = { version = "57.1.0", default-features = false }async-trait = "0.1.89"bigdecimal = "0.4.8"bytes = "1.11"@@ -166,7 +166,7 @@ log = "^0.4"num-traits = { version = "0.2" }object_store = { version = "0.12.4", default-features = false }parking_lot = "0.12"-parquet = { version = "57.0.0", default-features = false, features = [+parquet = { version = "57.1.0", default-features = false, features = ["arrow","async","object_store",diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rsindex 09fa8ef15a..de666fced7 100644--- a/datafusion-cli/src/main.rs+++ b/datafusion-cli/src/main.rs@@ -592,9 +592,9 @@ mod tests {+-----------------------------------+-----------------+---------------------+------+------------------+| filename | file_size_bytes | metadata_size_bytes | hits | extra |+-----------------------------------+-----------------+---------------------+------+------------------+- | alltypes_plain.parquet | 1851 | 6957 | 2 | page_index=false |- | alltypes_tiny_pages.parquet | 454233 | 267014 | 2 | page_index=true |- | lz4_raw_compressed_larger.parquet | 380836 | 996 | 2 | page_index=false |+ | alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false |+ | alltypes_tiny_pages.parquet | 454233 | 269266 | 2 | page_index=true |+ | lz4_raw_compressed_larger.parquet | 380836 | 1347 | 2 | page_index=false |+-----------------------------------+-----------------+---------------------+------+------------------+");@@ -623,9 +623,9 @@ mod tests {+-----------------------------------+-----------------+---------------------+------+------------------+| filename | file_size_bytes | metadata_size_bytes | hits | extra |+-----------------------------------+-----------------+---------------------+------+------------------+- | alltypes_plain.parquet | 1851 | 6957 | 5 | page_index=false |- | alltypes_tiny_pages.parquet | 454233 | 267014 | 2 | page_index=true |- | lz4_raw_compressed_larger.parquet | 380836 | 996 | 3 | page_index=false |+ | alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false |+ | alltypes_tiny_pages.parquet | 454233 | 269266 | 2 | page_index=true |+ | lz4_raw_compressed_larger.parquet | 380836 | 1347 | 3 | page_index=false |+-----------------------------------+-----------------+---------------------+------+------------------+");diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rsindex c58c33568c..cf951b17ac 100644--- a/datafusion/common/src/error.rs+++ b/datafusion/common/src/error.rs@@ -1233,7 +1233,6 @@ mod test {// To pass the test the environment variable RUST_BACKTRACE should be set to 1 to enforce backtrace#[cfg(feature = "backtrace")]#[test]- #[expect(clippy::unnecessary_literal_unwrap)]fn test_enabled_backtrace() {match std::env::var("RUST_BACKTRACE") {Ok(val) if val == "1" => {}diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 6f0b201185..aea530d67f 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -3109,7 +3109,7 @@ mod tests {assert_contains!(&e,- r#"Error during planning: Can not find compatible types to compare Boolean with [Struct("foo": Boolean), Utf8]"#+ r#"Error during planning: Can not find compatible types to compare Boolean with [Struct("foo": non-null Boolean), Utf8]"#);Ok(())diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rsindex 265862ff9a..56cdd78d70 100644--- a/datafusion/core/tests/dataframe/dataframe_functions.rs+++ b/datafusion/core/tests/dataframe/dataframe_functions.rs@@ -313,10 +313,10 @@ async fn test_fn_arrow_typeof() -> Result<()> {+----------------------+| arrow_typeof(test.l) |+----------------------+- | List(nullable Int32) |- | List(nullable Int32) |- | List(nullable Int32) |- | List(nullable Int32) |+ | List(Int32) |+ | List(Int32) |+ | List(Int32) |+ | List(Int32) |+----------------------+");diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rsindex 966f251613..d371f86253 100644--- a/datafusion/core/tests/parquet/filter_pushdown.rs+++ b/datafusion/core/tests/parquet/filter_pushdown.rs@@ -638,7 +638,9 @@ async fn predicate_cache_pushdown_default() -> datafusion_common::Result<()> {// The cache is on by default, and used when filter pushdown is enabledPredicateCacheTest {expected_inner_records: 8,- expected_records: 4,+ // reads more than necessary from the cache as then another bitmap is applied+ // See https://github.com/apache/datafusion/pull/18820 for setting and workaround+ expected_records: 7,}.run(&ctx).awaitdiff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rsindex 84899137e5..5a51451461 100644--- a/datafusion/core/tests/sql/select.rs+++ b/datafusion/core/tests/sql/select.rs@@ -222,10 +222,10 @@ async fn test_parameter_invalid_types() -> Result<()> {.await;assert_snapshot!(results.unwrap_err().strip_backtrace(),@r"- type_coercion- caused by- Error during planning: Cannot infer common argument type for comparison operation List(nullable Int32) = Int32- ");+ type_coercion+ caused by+ Error during planning: Cannot infer common argument type for comparison operation List(Int32) = Int32+ ");Ok(())}diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 4fb0f8553b..a557d3356d 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -2465,7 +2465,7 @@ mod test {assert_analyzed_plan_eq!(plan,@r#"- Projection: a = CAST(CAST(a AS Map("key_value": Struct("key": Utf8, "value": nullable Float64), unsorted)) AS Map("entries": Struct("key": Utf8, "value": nullable Float64), unsorted))+ Projection: a = CAST(CAST(a AS Map("key_value": non-null Struct("key": non-null Utf8, "value": Float64), unsorted)) AS Map("entries": non-null Struct("key": non-null Utf8, "value": Float64), unsorted))EmptyRelation: rows=0"#)diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.sltindex 77197721e1..15b50bef11 100644--- a/datafusion/sqllogictest/test_files/array.slt+++ b/datafusion/sqllogictest/test_files/array.slt@@ -710,13 +710,13 @@ selectquery TTTselect arrow_typeof(column1), arrow_typeof(column2), arrow_typeof(column3) from arrays;-----List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)-List(nullable List(nullable Int64)) List(nullable Float64) List(nullable Utf8)+List(List(Int64)) List(Float64) List(Utf8)+List(List(Int64)) List(Float64) List(Utf8)+List(List(Int64)) List(Float64) List(Utf8)+List(List(Int64)) List(Float64) List(Utf8)+List(List(Int64)) List(Float64) List(Utf8)+List(List(Int64)) List(Float64) List(Utf8)+List(List(Int64)) List(Float64) List(Utf8)# arrays tablequery ???@@ -1182,7 +1182,7 @@ select make_array(make_array(1), arrow_cast(make_array(-1), 'LargeList(Int8)'))query Tselect arrow_typeof(make_array(make_array(1), arrow_cast(make_array(-1), 'LargeList(Int8)')));-----List(nullable LargeList(nullable Int64))+List(LargeList(Int64))query ???@@ -1978,11 +1978,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), 0,----[1, 2, 3, 4, 5] [h, e, l, l, o]-# TODO: Enable once arrow_cast supports ListView types.+# TODO make error message nicer: https://github.com/apache/datafusion/issues/19004# Expected output (once supported):# ----# [1, 2, 3, 4, 5] [h, e, l, l, o]-query error DataFusion error: Execution error: Unsupported type 'ListView\(Int64\)'. Must be a supported arrow type name such as 'Int32' or 'Timestamp\(ns\)'. Error unknown token: ListView+query error Failed to coerce arguments to satisfy a call to 'array_slice' function:select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'ListView(Int64)'), 0, 6),array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'), 'ListView(Utf8)'), 0, 5);@@ -2025,11 +2025,11 @@ select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeList(Int64)'), 2,----[2, 3, 4, 5] [l, l, o]-# TODO: Enable once arrow_cast supports LargeListView types.+# TODO: Enable once array_slice supports LargeListView types.# Expected output (once supported):# ----# [2, 3, 4, 5] [l, l, o]-query error DataFusion error: Execution error: Unsupported type 'LargeListView\(Int64\)'. Must be a supported arrow type name such as 'Int32' or 'Timestamp\(ns\)'. Error unknown token: LargeListView+query error Failed to coerce arguments to satisfy a call to 'array_slice' function:select array_slice(arrow_cast(make_array(1, 2, 3, 4, 5), 'LargeListView(Int64)'), 2, 6),array_slice(arrow_cast(make_array('h', 'e', 'l', 'l', 'o'), 'LargeListView(Utf8)'), 3, 7);@@ -3321,7 +3321,7 @@ selectarray_concat([arrow_cast('1', 'Utf8'), arrow_cast('2', 'Utf8')], [arrow_cast('3', 'Utf8View')]),arrow_typeof(array_concat([arrow_cast('1', 'Utf8'), arrow_cast('2', 'Utf8')], [arrow_cast('3', 'Utf8View')]));-----[1, 2, 3] List(nullable Utf8View)+[1, 2, 3] List(Utf8View)# array_concat errorquery error DataFusion error: Error during planning: Execution error: Function 'array_concat' user-defined coercion failed with "Error during planning: array_concat does not support type Int64"@@ -4614,7 +4614,7 @@ NULL [baz] bazquery TSELECT arrow_typeof(make_array(arrow_cast('a', 'Utf8View'), 'b', 'c', 'd'));-----List(nullable Utf8View)+List(Utf8View)# expect a,b,c,d. make_array forces all types to be of a common type (see above)query T@@ -7708,8 +7708,8 @@ CREATE EXTERNAL TABLE fixed_size_list_array STORED AS PARQUET LOCATION '../core/query Tselect arrow_typeof(f0) from fixed_size_list_array;-----FixedSizeList(2 x nullable Int64)-FixedSizeList(2 x nullable Int64)+FixedSizeList(2 x Int64)+FixedSizeList(2 x Int64)query ?select * from fixed_size_list_array;@@ -7738,8 +7738,8 @@ select make_array(arrow_cast(f0, 'List(Int64)')) from fixed_size_list_arrayquery Tselect arrow_typeof(make_array(arrow_cast(f0, 'List(Int64)'))) from fixed_size_list_array-----List(nullable List(nullable Int64))-List(nullable List(nullable Int64))+List(List(Int64))+List(List(Int64))query ?select make_array(f0) from fixed_size_list_array@@ -7750,8 +7750,8 @@ select make_array(f0) from fixed_size_list_arrayquery Tselect arrow_typeof(make_array(f0)) from fixed_size_list_array-----List(nullable FixedSizeList(2 x nullable Int64))-List(nullable FixedSizeList(2 x nullable Int64))+List(FixedSizeList(2 x Int64))+List(FixedSizeList(2 x Int64))query ?select array_concat(column1, [7]) from arrays_values_v2;@@ -7798,7 +7798,7 @@ select flatten(arrow_cast(make_array([1], [2, 3], [null], make_array(4, null, 5)arrow_typeof(flatten(arrow_cast(make_array([1], [2, 3], [null], make_array(4, null, 5)), 'FixedSizeList(4, LargeList(Int64))'))),arrow_typeof(flatten(arrow_cast(make_array([[1.1], [2.2]], [[3.3], [4.4]]), 'List(LargeList(FixedSizeList(1, Float64)))')));-----[1, 2, 3, NULL, 4, NULL, 5] [[1.1], [2.2], [3.3], [4.4]] LargeList(nullable Int64) LargeList(nullable FixedSizeList(1 x nullable Float64))+[1, 2, 3, NULL, 4, NULL, 5] [[1.1], [2.2], [3.3], [4.4]] LargeList(Int64) LargeList(FixedSizeList(1 x Float64))# flatten with column valuesquery ????@@ -8338,19 +8338,19 @@ select * from test_create_array_table;query Tselect arrow_typeof(a) from test_create_array_table;-----List(nullable Int32)+List(Int32)query Tselect arrow_typeof(c) from test_create_array_table;-----List(nullable List(nullable Int32))+List(List(Int32))# Test casting to array types# issue: https://github.com/apache/datafusion/issues/9440query ??Tselect [1,2,3]::int[], [['1']]::int[][], arrow_typeof([]::text[]);-----[1, 2, 3] [[1]] List(nullable Utf8View)+[1, 2, 3] [[1]] List(Utf8View)# test empty arrays return length# issue: https://github.com/apache/datafusion/pull/12459@@ -8370,8 +8370,8 @@ create table fixed_size_col_table (a int[3]) as values ([1,2,3]), ([4,5,6]);query Tselect arrow_typeof(a) from fixed_size_col_table;-----FixedSizeList(3 x nullable Int32)-FixedSizeList(3 x nullable Int32)+FixedSizeList(3 x Int32)+FixedSizeList(3 x Int32)query ? rowsortSELECT DISTINCT a FROM fixed_size_col_table@@ -8407,7 +8407,6 @@ select array_contains(a, b) from array_has order by 1 nulls last;trueNULL-# TODO: Enable once arrow_cast supports ListView types.# Expected output (once supported):# ----# [5, 4, 3, 2, 1]diff --git a/datafusion/sqllogictest/test_files/arrow_typeof.slt b/datafusion/sqllogictest/test_files/arrow_typeof.sltindex 5ba62be687..c213f2abf7 100644--- a/datafusion/sqllogictest/test_files/arrow_typeof.slt+++ b/datafusion/sqllogictest/test_files/arrow_typeof.slt@@ -357,12 +357,12 @@ select arrow_cast(make_array(1, 2, 3), 'List(Int64)');query Tselect arrow_typeof(arrow_cast(make_array(1, 2, 3), 'List(Int64)'));-----List(nullable Int64)+List(Int64)query Tselect arrow_typeof(arrow_cast(arrow_cast(make_array([1, 2, 3]), 'LargeList(LargeList(Int64))'), 'List(List(Int64))'));-----List(nullable List(nullable Int64))+List(List(Int64))## LargeList@@ -380,12 +380,12 @@ select arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)');query Tselect arrow_typeof(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'));-----LargeList(nullable Int64)+LargeList(Int64)query Tselect arrow_typeof(arrow_cast(make_array([1, 2, 3]), 'LargeList(LargeList(Int64))'));-----LargeList(nullable LargeList(nullable Int64))+LargeList(LargeList(Int64))## FixedSizeList@@ -417,7 +417,7 @@ select arrow_cast(make_array(1, 2, 3), 'FixedSizeList(3, Int64)');query Tselect arrow_typeof(arrow_cast(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 'FixedSizeList(3, Int64)'));-----FixedSizeList(3 x nullable Int64)+FixedSizeList(3 x Int64)query ?select arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)');diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.sltindex e34a601851..9e5b71b871 100644--- a/datafusion/sqllogictest/test_files/coalesce.slt+++ b/datafusion/sqllogictest/test_files/coalesce.slt@@ -199,14 +199,14 @@ selectcoalesce(array[1, 2], array[3, 4]),arrow_typeof(coalesce(array[1, 2], array[3, 4]));-----[1, 2] List(nullable Int64)+[1, 2] List(Int64)query ?Tselectcoalesce(null, array[3, 4]),arrow_typeof(coalesce(array[1, 2], array[3, 4]));-----[3, 4] List(nullable Int64)+[3, 4] List(Int64)# coalesce with arrayquery ?T@@ -214,7 +214,7 @@ selectcoalesce(array[1, 2], array[arrow_cast(3, 'Int32'), arrow_cast(4, 'Int32')]),arrow_typeof(coalesce(array[1, 2], array[arrow_cast(3, 'Int32'), arrow_cast(4, 'Int32')]));-----[1, 2] List(nullable Int64)+[1, 2] List(Int64)# test dict(int32, utf8)statement okdiff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.sltindex 21fa72ad86..7ea54464d3 100644--- a/datafusion/sqllogictest/test_files/map.slt+++ b/datafusion/sqllogictest/test_files/map.slt@@ -43,8 +43,8 @@ LOCATION '../core/tests/data/parquet_map.parquet';query TTTdescribe data;-----ints Map("entries": Struct("key": Utf8, "value": Int64), unsorted) NO-strings Map("entries": Struct("key": Utf8, "value": Utf8), unsorted) NO+ints Map("entries": non-null Struct("key": non-null Utf8, "value": non-null Int64), unsorted) NO+strings Map("entries": non-null Struct("key": non-null Utf8, "value": non-null Utf8), unsorted) NOtimestamp Utf8View NOquery ??Tdiff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex dce5fe036b..0989dd382d 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -53,9 +53,9 @@ select * from struct_values;query TTselect arrow_typeof(s1), arrow_typeof(s2) from struct_values;-----Struct("c0": nullable Int32) Struct("a": nullable Int32, "b": nullable Utf8View)-Struct("c0": nullable Int32) Struct("a": nullable Int32, "b": nullable Utf8View)-Struct("c0": nullable Int32) Struct("a": nullable Int32, "b": nullable Utf8View)+Struct("c0": Int32) Struct("a": Int32, "b": Utf8View)+Struct("c0": Int32) Struct("a": Int32, "b": Utf8View)+Struct("c0": Int32) Struct("a": Int32, "b": Utf8View)# struct[i]@@ -229,12 +229,12 @@ select named_struct('field_a', 1, 'field_b', 2);query Tselect arrow_typeof(named_struct('first', 1, 'second', 2, 'third', 3));-----Struct("first": nullable Int64, "second": nullable Int64, "third": nullable Int64)+Struct("first": Int64, "second": Int64, "third": Int64)query Tselect arrow_typeof({'first': 1, 'second': 2, 'third': 3});-----Struct("first": nullable Int64, "second": nullable Int64, "third": nullable Int64)+Struct("first": Int64, "second": Int64, "third": Int64)# test nested struct literalquery ?@@ -413,7 +413,7 @@ create table t(a struct<r varchar, c int>, b struct<r varchar, c float>) as valuquery Tselect arrow_typeof([a, b]) from t;-----List(nullable Struct("r": nullable Utf8View, "c": nullable Float32))+List(Struct("r": Utf8View, "c": Float32))query ?select [a, b] from t;@@ -464,12 +464,12 @@ select * from t;query Tselect arrow_typeof(c1) from t;-----Struct("r": nullable Utf8View, "b": nullable Int32)+Struct("r": Utf8View, "b": Int32)query Tselect arrow_typeof(c2) from t;-----Struct("r": nullable Utf8View, "b": nullable Float32)+Struct("r": Utf8View, "b": Float32)statement okdrop table t;@@ -486,8 +486,8 @@ select * from t;query Tselect arrow_typeof(column1) from t;-----Struct("r": nullable Utf8, "c": nullable Float64)-Struct("r": nullable Utf8, "c": nullable Float64)+Struct("r": Utf8, "c": Float64)+Struct("r": Utf8, "c": Float64)statement okdrop table t;@@ -519,9 +519,9 @@ select coalesce(s1) from t;query Tselect arrow_typeof(coalesce(s1, s2)) from t;-----Struct("a": nullable Float32, "b": nullable Utf8View)-Struct("a": nullable Float32, "b": nullable Utf8View)-Struct("a": nullable Float32, "b": nullable Utf8View)+Struct("a": Float32, "b": Utf8View)+Struct("a": Float32, "b": Utf8View)+Struct("a": Float32, "b": Utf8View)statement okdrop table t;@@ -546,9 +546,9 @@ select coalesce(s1, s2) from t;query Tselect arrow_typeof(coalesce(s1, s2)) from t;-----Struct("a": nullable Float32, "b": nullable Utf8View)-Struct("a": nullable Float32, "b": nullable Utf8View)-Struct("a": nullable Float32, "b": nullable Utf8View)+Struct("a": Float32, "b": Utf8View)+Struct("a": Float32, "b": Utf8View)+Struct("a": Float32, "b": Utf8View)statement okdrop table t;@@ -583,7 +583,7 @@ create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as valuquery Tselect arrow_typeof([a, b]) from t;-----List(nullable Struct("r": nullable Utf8View, "c": nullable Float32))+List(Struct("r": Utf8View, "c": Float32))statement okdrop table t;@@ -606,13 +606,13 @@ create table t(a struct(r varchar, c int, g float), b struct(r varchar, c float,query Tselect arrow_typeof(a) from t;-----Struct("r": nullable Utf8View, "c": nullable Int32, "g": nullable Float32)+Struct("r": Utf8View, "c": Int32, "g": Float32)# type of each column should not coerced but perserve as it isquery Tselect arrow_typeof(b) from t;-----Struct("r": nullable Utf8View, "c": nullable Float32, "g": nullable Int32)+Struct("r": Utf8View, "c": Float32, "g": Int32)statement okdrop table t;diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex acb2f07e01..818c861695 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -5906,7 +5906,7 @@ LIMIT 5----DataFusion error: type_coercioncaused by-Error during planning: Cannot infer common argument type for comparison operation Int64 >= List(nullable Null)+Error during planning: Cannot infer common argument type for comparison operation Int64 >= List(Null)@@ -5934,7 +5934,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[c1@2 as c1, c2@3 as c2, sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as sum1, sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as sum2, count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as count1, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as array_agg1, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 as array_agg2]02)--GlobalLimitExec: skip=0, fetch=5-03)----BoundedWindowAggExec: wdw=[sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable List(nullable Int64) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable List(nullable Int64) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----BoundedWindowAggExec: wdw=[sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "count(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable List(Int64) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "array_agg(test.c2) FILTER (WHERE test.c2 >= Int64(2) AND test.c2 < Int64(4) AND test.c1 > Int64(0)) ORDER BY [test.c1 ASC NULLS LAST, test.c2 ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable List(Int64) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]04)------SortPreservingMergeExec: [c1@2 ASC NULLS LAST, c2@3 ASC NULLS LAST], fetch=505)--------SortExec: TopK(fetch=5), expr=[c1@2 ASC NULLS LAST, c2@3 ASC NULLS LAST], preserve_partitioning=[true]06)----------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-0.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-1.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-2.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-3.csv]]}, projection=[c2@1 >= 2 as __common_expr_1, c2@1 >= 2 AND c2@1 < 4 AND c1@0 > 0 as __common_expr_2, c1, c2], file_type=csv, has_header=false