Apache DataFusion SQL Query Engine
chore: Update to arrow/parquet `60.1.0` (#26867)
Andrew Lamb6a9a1298e6f2
diff --git a/Cargo.lock b/Cargo.lockindex b581b3c8d4..68170c4f61 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -158,9 +158,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"[[package]]name = "arrow"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14"+checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999"dependencies = ["arrow-arith","arrow-array",@@ -181,9 +181,9 @@ dependencies = [[[package]]name = "arrow-arith"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720"+checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923"dependencies = ["arrow-array","arrow-buffer",@@ -195,9 +195,9 @@ dependencies = [[[package]]name = "arrow-array"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e"+checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"dependencies = ["ahash","arrow-buffer",@@ -215,9 +215,9 @@ dependencies = [[[package]]name = "arrow-avro"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "edad137a3609e2c6a4af0b5ff1c5c049b7af532649d235c7e22e9a9cad933b35"+checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6"dependencies = ["arrow-array","arrow-buffer",@@ -239,9 +239,9 @@ dependencies = [[[package]]name = "arrow-buffer"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5"+checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"dependencies = ["bytes","half",@@ -251,9 +251,9 @@ dependencies = [[[package]]name = "arrow-cast"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e"+checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf"dependencies = ["arrow-array","arrow-buffer",@@ -273,9 +273,9 @@ dependencies = [[[package]]name = "arrow-cmp"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687"+checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"dependencies = ["arrow-array","arrow-buffer",@@ -284,9 +284,9 @@ dependencies = [[[package]]name = "arrow-csv"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2ad83176cc677f996096d694a3cccd7310069faeee4b3df9e2dabf92fdcf8b4b"+checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28"dependencies = ["arrow-array","arrow-cast",@@ -299,9 +299,9 @@ dependencies = [[[package]]name = "arrow-data"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f"+checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"dependencies = ["arrow-buffer","arrow-schema",@@ -312,9 +312,9 @@ dependencies = [[[package]]name = "arrow-flight"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fbc77d8656483656cbf966e04b62f193f74111ff7d9cc15f58a549314a3b061e"+checksum = "8709a798bd0825cb6207861aad09093133d64314c25569132dbc7f9f083879a5"dependencies = ["arrow-arith","arrow-array",@@ -330,7 +330,6 @@ dependencies = ["base64 0.23.1","bytes","futures",- "once_cell","prost","prost-types","tonic",@@ -339,9 +338,9 @@ dependencies = [[[package]]name = "arrow-ipc"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2"+checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"dependencies = ["arrow-array","arrow-buffer",@@ -355,9 +354,9 @@ dependencies = [[[package]]name = "arrow-json"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "342fc6600a93a48038d853431803c941436b7b51f756763e76cad9784b7e3718"+checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06"dependencies = ["arrow-array","arrow-buffer",@@ -380,9 +379,9 @@ dependencies = [[[package]]name = "arrow-ord"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698"+checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4"dependencies = ["arrow-array","arrow-buffer",@@ -394,9 +393,9 @@ dependencies = [[[package]]name = "arrow-row"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a"+checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e"dependencies = ["arrow-array","arrow-buffer",@@ -407,9 +406,9 @@ dependencies = [[[package]]name = "arrow-schema"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338"+checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"dependencies = ["bitflags","serde",@@ -419,9 +418,9 @@ dependencies = [[[package]]name = "arrow-select"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977"+checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"dependencies = ["ahash","arrow-array",@@ -434,9 +433,9 @@ dependencies = [[[package]]name = "arrow-string"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1"+checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862"dependencies = ["arrow-array","arrow-buffer",@@ -529,9 +528,9 @@ dependencies = [[[package]]name = "atoi"-version = "3.1.0"+version = "3.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be"+checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993"dependencies = ["num-traits",]@@ -4666,9 +4665,9 @@ dependencies = [[[package]]name = "parquet"-version = "60.0.0"+version = "60.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba"+checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"dependencies = ["ahash","arrow-array",@@ -6816,7 +6815,7 @@ version = "2.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"dependencies = [- "rand 0.10.1",+ "rand 0.9.4",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 3070ed33f3..007ca5401e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -90,31 +90,31 @@ version = "55.1.0"## See for more details: https://github.com/rust-lang/cargo/issues/11329apache-avro = { version = "0.21", default-features = false }-arrow = { version = "60.0.0", features = [+arrow = { version = "60.1.0", features = ["prettyprint","chrono-tz",] }-arrow-avro = { version = "60.0.0", default-features = false, features = [+arrow-avro = { version = "60.1.0", default-features = false, features = ["deflate","snappy","zstd","bzip2","xz",] }-arrow-buffer = { version = "60.0.0", default-features = false }-arrow-data = { version = "60.0.0", default-features = false }-arrow-flight = { version = "60.0.0", features = [+arrow-buffer = { version = "60.1.0", default-features = false }+arrow-data = { version = "60.1.0", default-features = false }+arrow-flight = { version = "60.1.0", features = ["flight-sql",] }# Both codecs are required here to make sure that code paths like# file-spilling have access to all compression codecs.-arrow-ipc = { version = "60.0.0", default-features = false, features = [+arrow-ipc = { version = "60.1.0", default-features = false, features = ["lz4","zstd",] }-arrow-ord = { version = "60.0.0", default-features = false }-arrow-schema = { version = "60.0.0", default-features = false }-arrow-select = { version = "60.0.0", default-features = false }+arrow-ord = { version = "60.1.0", default-features = false }+arrow-schema = { version = "60.1.0", default-features = false }+arrow-select = { version = "60.1.0", default-features = false }async-trait = "0.1.89"bigdecimal = "0.4.8"bytes = "1.11"@@ -181,7 +181,7 @@ memchr = "2.8.1"num-traits = { version = "0.2" }object_store = { version = "0.14.2", default-features = false }parking_lot = "0.13"-parquet = { version = "60.0.0", default-features = false, features = [+parquet = { version = "60.1.0", default-features = false, features = ["arrow","async","object_store",diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex ab57d87dce..842105c99d 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -819,8 +819,8 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {/// The IPC writer writes every data buffer of an array in full, once per/// entry, and `gc()` copies the bytes of every view separately. Neither/// handles views that share bytes, as the views of a dictionary-encoded-/// Parquet column do: after `concat` or `interleave` the dictionary is often-/// listed once per input batch, and `gc()` copies it once per row+/// Parquet column do: after `concat` the dictionary is listed once per input+/// batch, and `gc()` copies it once per row/// (<https://github.com/apache/datafusion/issues/23564>).////// Repeated buffer entries are merged first. `gc()` then runs only if it@@ -1985,15 +1985,15 @@ mod tests {// After appending each batch, spilled_rows and spilled_bytes should increase incrementally,// while spill_file_count remains 1 (since we're writing to the same file)in_progress_file.append_batch(&batch1)?;- verify_metrics(&in_progress_file, 1, 440, 3)?;+ verify_metrics(&in_progress_file, 1, 424, 3)?;in_progress_file.append_batch(&batch2)?;- verify_metrics(&in_progress_file, 1, 704, 6)?;+ verify_metrics(&in_progress_file, 1, 672, 6)?;let completed_file = in_progress_file.finish()?;assert!(completed_file.is_some());assert!(completed_file.unwrap().path().unwrap().exists());- verify_metrics(&in_progress_file, 1, 712, 6)?;+ verify_metrics(&in_progress_file, 1, 680, 6)?;// Double finish produce errorlet result = in_progress_file.finish();assert!(result.is_err());@@ -2128,7 +2128,7 @@ mod tests {// Metrics should be updated immediately (at least schema and first batch)let bytes_after_batch1 = metrics.spilled_bytes.value();- assert_eq!(bytes_after_batch1, 440);+ assert_eq!(bytes_after_batch1, 424);assert_eq!(metrics.spill_file_count.value(), 1);// Check global progress@@ -2493,8 +2493,8 @@ mod tests {}/// Arrays that each hold views into one shared dictionary buffer, as the- /// Parquet reader produces for a dictionary-encoded column, combined the- /// way a sort combines its input batches. Each output row cycles through+ /// Parquet reader produces for a dictionary-encoded column, combined with+ /// `concat` as a sort combines its buffered batches. The rows cycle through/// the dictionary (<https://github.com/apache/datafusion/issues/23564>).fn shared_dictionary_views(dictionary_len: u32,@@ -2502,7 +2502,7 @@ mod tests {rows_per_input: u32,) -> Result<(StringViewArray, StringViewArray)> {use arrow::array::{AsArray, UInt32Array};- use arrow::compute::{interleave, take};+ use arrow::compute::{concat, take};let dictionary = dictionary_array(dictionary_len)?;let arrays = (0..inputs)@@ -2515,17 +2515,14 @@ mod tests {}).collect::<std::result::Result<Vec<_>, _>>()?;let arrays: Vec<&dyn Array> = arrays.iter().map(|a| a.as_ref()).collect();- let indices: Vec<(usize, usize)> = (0..rows_per_input as usize)- .flat_map(|row| (0..inputs as usize).map(move |input| (input, row)))- .collect();- let merged = interleave(&arrays, &indices)?;+ let merged = concat(&arrays)?;Ok((dictionary, merged.as_string_view().clone()))}#[test]fn test_gc_keeps_shared_dictionary_buffer() -> Result<()> {let (dictionary, merged) = shared_dictionary_views(1000, 8, 1024)?;- // `interleave` lists the dictionary once per input+ // `concat` lists the dictionary once per inputassert_eq!(merged.data_buffers().len(), 8);// `gc()` would copy 8192 values of 63 bytes, 8x the dictionaryassert!(merged.total_buffer_bytes_used() > data_buffers_size(&dictionary));diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rsindex 6d50245ea6..48e67e72d2 100644--- a/datafusion/physical-plan/src/spill/spill_pool.rs+++ b/datafusion/physical-plan/src/spill/spill_pool.rs@@ -1005,7 +1005,7 @@ mod tests {);assert_eq!(metrics.spilled_bytes.value(),- 320,+ 312,"Spilled bytes should reflect data written (header + 1 batch)");assert_eq!(@@ -1480,7 +1480,7 @@ mod tests {// Check metrics before drop - spilled_bytes already reflects written datalet spilled_bytes_before = metrics.spilled_bytes.value();assert_eq!(- spilled_bytes_before, 1088,+ spilled_bytes_before, 1048,"Spilled bytes should reflect data written (header + 5 batches)");diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex a5e57c745a..8dbbed5558 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -1214,7 +1214,7 @@ impl ListArrayType for FixedSizeListArray {}fn value_offsets(&self, row: usize) -> (i64, i64) {- let start = self.value_offset(row) as i64;+ let start = self.value_offset_at(row) as i64;(start, start + self.value_length() as i64)}}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 112e257677..605e54d154 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -1732,7 +1732,7 @@ SELECT approx_distinct(c9) AS a, approx_distinct(c9) AS b FROM aggregate_test_10query IIIIISELECT approx_distinct(c14) AS a, approx_distinct(c15) AS b, approx_distinct(arrow_cast(c15, 'Date64')), approx_distinct(arrow_cast(c15, 'Time32(Second)')) as c, approx_distinct(arrow_cast(c15, 'Time64(Nanosecond)')) AS d FROM aggregate_test_100-----18 60 60 60 60+18 60 7 60 60# approx_distinct over Boolean: exact count via flag-pair accumulator (0..=2).statement okdiff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.sltindex 8ff029712c..b8ebe00717 100644--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt@@ -1069,12 +1069,14 @@ FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))28 28500 28500000 28500000000NULL NULL NULL NULL-# timestamps outside the range arrow represents are NULL, like the other parts+# The second is computed directly from the stored value (i64::MAX % 60 = 7). The+# sub-second parts of a timestamp outside the range arrow represents are NULL,+# like the other parts.query IIIISELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts)-----NULL NULL NULL NULL+7 NULL NULL NULLNULL NULL NULL NULL# test_extract_interval@@ -1603,7 +1605,7 @@ NULL# Non-null inputs can give NULL results, so the result is nullable.# Reproducers from https://github.com/apache/datafusion/issues/26685query I-SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts)+SELECT extract(millisecond from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts)----NULLdiff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.sltindex 747433b4d6..ce525005de 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt@@ -1377,13 +1377,9 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z############ SECTION 16: DST boundaries##-## KNOWN WRONG, and the single largest divergence in this file: DataFusion-## cannot represent an ambiguous or non-existent local time in a named zone at-## all. Every route into a named zone -- a column cast, `AT TIME ZONE`, a-## string literal, or `::timestamptz` under a session zone -- fails with an-## error, where PostgreSQL resolves the value.+## Casting a naive timestamp into a named zone resolves an ambiguous or+## non-existent local time like PostgreSQL.##-## PostgreSQL's answers, for reference:## ambiguous 2024-11-03 01:30:00 America/Denver -> 2024-11-03 01:30:00-07## (the second, standard-time occurrence)## nonexistent 2024-03-10 02:30:00 America/Denver -> 2024-03-10 03:30:00-06@@ -1391,19 +1387,14 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z## ambiguous 2024-10-27 02:30:00 Europe/Brussels -> 2024-10-27 02:30:00+01## nonexistent 2024-03-31 02:30:00 Europe/Brussels -> 2024-03-31 03:30:00+02##-## Note that the column path and the literal path fail with *different* errors-## even though the value is identical: the column path reports-## "Cannot cast timezone to different timezone" from the Arrow cast kernel,-## while the literal path is folded by `simplify_expressions` and reports-## "error computing timezone offset" from the Arrow timestamp parser.-##-## See https://github.com/apache/datafusion/issues/25084 and the arrow-rs fix-## https://github.com/apache/arrow-rs/pull/11038+## The *parsing* routes into a named zone with a session zone+## still fail with "error computing timezone offset". See+## See https://github.com/apache/datafusion/issues/25084########### --- US fall back: 2024-11-03 01:30:00 America/Denver occurs twice ----# The naive value itself is fine; only the move into a named zone fails.+# The naive value itself is finequery PSELECT '2024-11-03 01:30:00'::timestamp----@@ -1412,25 +1403,29 @@ SELECT '2024-11-03 01:30:00'::timestampstatement okCREATE TABLE dst_ambiguous AS VALUES ('2024-11-03 01:30:00'::timestamp)-# These expectations match only the stable `Arrow error: ...` tail. The wrapper-# chain above it (optimizer rule, field context) is not what these cases pin,-# and it changed in https://github.com/apache/datafusion/pull/24920.-query error Arrow error: Cast error: Cannot cast timezone to different timezone+# The ambiguous wall clock resolves to the second (standard time) occurrence+query PSELECT column1 AT TIME ZONE 'America/Denver' FROM dst_ambiguous+----+2024-11-03T01:30:00-07:00-query error Arrow error: Cast error: Cannot cast timezone to different timezone+query PSELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM dst_ambiguous+----+2024-11-03T01:30:00-07:00-# The same wall clock arriving as a string literal instead of a column+# String literal parsing fails.+# https://github.com/apache/arrow-rs/issues/11039query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-11\-03T01:30:00':\ error\ computing\ timezone\ offsetSELECT arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))')-query error Arrow\ error:\ Cast\ error:\ Cannot\ cast\ timezone\ to\ different\ timezone+query PSELECT TIMESTAMP '2024-11-03 01:30:00' AT TIME ZONE 'America/Denver'+----+2024-11-03T01:30:00-07:00-# Both instants are representable when the offset is spelled out explicitly ---# it is only the *resolution* of a bare wall clock that fails. Note the two-# rows differ solely in the rendered offset.+# Both instants are representable when the offset is spelled out explicitly.+# Note the two rows differ solely in the rendered offset.query PPSELECT arrow_cast('2024-11-03T07:30:00Z', 'Timestamp(Nanosecond, Some("America/Denver"))'),arrow_cast('2024-11-03T08:30:00Z', 'Timestamp(Nanosecond, Some("America/Denver"))')@@ -1442,17 +1437,23 @@ SELECT arrow_cast('2024-11-03T07:30:00Z', 'Timestamp(Nanosecond, Some("America/Dstatement okCREATE TABLE dst_nonexistent AS VALUES ('2024-03-10 02:30:00'::timestamp)-query error Arrow error: Cast error: Cannot cast timezone to different timezone+query PSELECT column1 AT TIME ZONE 'America/Denver' FROM dst_nonexistent+----+2024-03-10T03:30:00-06:00-query error Arrow error: Cast error: Cannot cast timezone to different timezone+query PSELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM dst_nonexistent+----+2024-03-10T03:30:00-06:00query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-03\-10T02:30:00':\ error\ computing\ timezone\ offsetSELECT arrow_cast('2024-03-10T02:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))')-query error Arrow\ error:\ Cast\ error:\ Cannot\ cast\ timezone\ to\ different\ timezone+query PSELECT TIMESTAMP '2024-03-10 02:30:00' AT TIME ZONE 'America/Denver'+----+2024-03-10T03:30:00-06:00statement okSET datafusion.execution.time_zone = 'America/Denver'@@ -1471,11 +1472,17 @@ RESET datafusion.execution.time_zonestatement okCREATE TABLE dst_eu AS VALUES ('2024-10-27 02:30:00'::timestamp), ('2024-03-31 02:30:00'::timestamp)-query error Arrow error: Cast error: Cannot cast timezone to different timezone+query PSELECT column1 AT TIME ZONE 'Europe/Brussels' FROM dst_eu+----+2024-10-27T02:30:00+01:00+2024-03-31T03:30:00+02:00-query error Arrow error: Cast error: Cannot cast timezone to different timezone+query PSELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Europe/Brussels"))') FROM dst_eu+----+2024-10-27T02:30:00+01:00+2024-03-31T03:30:00+02:00query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-10\-27T02:30:00':\ error\ computing\ timezone\ offsetSELECT arrow_cast('2024-10-27T02:30:00', 'Timestamp(Nanosecond, Some("Europe/Brussels"))')@@ -1496,9 +1503,7 @@ statement okRESET datafusion.execution.time_zone# A tz-aware column crossing the fall-back boundary. Constructing the values-# from explicit UTC instants sidesteps the failure above, so the behaviour of-# date_trunc / date_bin / date_part / interval arithmetic across a repeated-# hour can still be pinned down.+# from explicit UTC instants covers a repeated hourstatement okCREATE TABLE dst_cross ASSELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') AS tsdiff --git a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt b/datafusion/sqllogictest/test_files/nested_loop_join_spill.sltindex 5dfdfff6d5..adcd7b949e 100644--- a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt+++ b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt@@ -302,7 +302,7 @@ RIGHT SEMI JOIN existence_spill_right r ON l.threshold > r.kPlan with Metrics01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]-03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=128, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]+03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=128, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]04)------ProjectionExec: expr=[CASE WHEN value@0 = 4097 THEN 130 WHEN value@0 % 65 = 0 THEN 65 WHEN value@0 % 65 = 1 THEN 97 ELSE 114 END as threshold], metrics=[<slt:ignore>]05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]06)------ProjectionExec: expr=[CASE WHEN __common_expr_1@0 = 128 THEN NULL ELSE __common_expr_1@0 END as k], metrics=[<slt:ignore>]@@ -317,7 +317,7 @@ RIGHT ANTI JOIN existence_spill_right r ON l.threshold > r.kPlan with Metrics01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]-03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=1, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]+03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=1, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]04)------ProjectionExec: expr=[CASE WHEN value@0 = 4097 THEN 130 WHEN value@0 % 65 = 0 THEN 65 WHEN value@0 % 65 = 1 THEN 97 ELSE 114 END as threshold], metrics=[<slt:ignore>]05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]06)------ProjectionExec: expr=[CASE WHEN __common_expr_1@0 = 128 THEN NULL ELSE __common_expr_1@0 END as k], metrics=[<slt:ignore>]@@ -339,7 +339,7 @@ RIGHT SEMI JOIN range(129) r ON l.threshold > r.value----129 8256 0 128-# Both joins must spill and replay the two right batches across four left chunks.+# Both joins must spill and replay the two right batches across three left chunks.query TTEXPLAIN ANALYZE SELECT count(*)FROM existence_spill_early_left l@@ -348,7 +348,7 @@ RIGHT SEMI JOIN range(129) r ON l.threshold > r.valuePlan with Metrics01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]-03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=129, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]+03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=129, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]04)------ProjectionExec: expr=[CASE WHEN value@0 = 0 THEN 129 ELSE 0 END as threshold], metrics=[<slt:ignore>]05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]06)------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=129, batch_size=65], metrics=[<slt:ignore>]@@ -361,7 +361,7 @@ RIGHT ANTI JOIN range(129) r ON l.threshold > r.valuePlan with Metrics01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]-03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=0, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]+03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=0, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]04)------ProjectionExec: expr=[CASE WHEN value@0 = 0 THEN 129 ELSE 0 END as threshold], metrics=[<slt:ignore>]05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]06)------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=129, batch_size=65], metrics=[<slt:ignore>]diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 6b33b5eded..4665bfa52e 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -96,16 +96,16 @@ Aggregate plans now display `group_clustering_mode=Full` or`group_clustering_mode=Partial(indices)` instead of `ordering_mode=Sorted` or`ordering_mode=PartiallySorted(indices)`.-### Upgrade arrow/parquet to 60.0.0 and object_store to 0.14.2+### Upgrade arrow/parquet to 60 and object_store to 0.14-DataFusion 56.0.0 uses `arrow` and `parquet` 60.0.0, and `object_store` 0.14.2.+DataFusion 56.0.0 uses `arrow` and `parquet` 60, and `object_store` 0.14.This may require updates to your Cargo.toml if you have direct dependencies onthese crates.-See the [Arrow 60.0.0 release notes] and the [object_store 0.14.2 upgrade guide] for details on breaking changes in those versions.+See the [Arrow 60.0.0 release notes] and the [object_store 0.14.0 changelog] for details on breaking changes in those versions.[arrow 60.0.0 release notes]: https://github.com/apache/arrow-rs/releases/tag/60.0.0-[object_store 0.14.2 upgrade guide]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG.md+[object_store 0.14.0 changelog]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG-old.md#v0140-2026-06-18#### Field and schema metadata now use arrow's `Metadata` type