Skip to content

Apache DataFusion SQL Query Engine

chore: Update to arrow/parquet `60.1.0` (#26867)

Andrew Lamb6a9a1298e6f2
diff --git a/Cargo.lock b/Cargo.lock
index b581b3c8d4..68170c4f61 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -158,9 +158,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14"
+checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -181,9 +181,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720"
+checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -195,9 +195,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e"
+checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4"
dependencies = [
"ahash",
"arrow-buffer",
@@ -215,9 +215,9 @@ dependencies = [
[[package]]
name = "arrow-avro"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edad137a3609e2c6a4af0b5ff1c5c049b7af532649d235c7e22e9a9cad933b35"
+checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -239,9 +239,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5"
+checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade"
dependencies = [
"bytes",
"half",
@@ -251,9 +251,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e"
+checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -273,9 +273,9 @@ dependencies = [
[[package]]
name = "arrow-cmp"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687"
+checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -284,9 +284,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ad83176cc677f996096d694a3cccd7310069faeee4b3df9e2dabf92fdcf8b4b"
+checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -299,9 +299,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f"
+checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -312,9 +312,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbc77d8656483656cbf966e04b62f193f74111ff7d9cc15f58a549314a3b061e"
+checksum = "8709a798bd0825cb6207861aad09093133d64314c25569132dbc7f9f083879a5"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -330,7 +330,6 @@ dependencies = [
"base64 0.23.1",
"bytes",
"futures",
- "once_cell",
"prost",
"prost-types",
"tonic",
@@ -339,9 +338,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2"
+checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -355,9 +354,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "342fc6600a93a48038d853431803c941436b7b51f756763e76cad9784b7e3718"
+checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -380,9 +379,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698"
+checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -394,9 +393,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a"
+checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -407,9 +406,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338"
+checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3"
dependencies = [
"bitflags",
"serde",
@@ -419,9 +418,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977"
+checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30"
dependencies = [
"ahash",
"arrow-array",
@@ -434,9 +433,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1"
+checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -529,9 +528,9 @@ dependencies = [
[[package]]
name = "atoi"
-version = "3.1.0"
+version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be"
+checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993"
dependencies = [
"num-traits",
]
@@ -4666,9 +4665,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "60.0.0"
+version = "60.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba"
+checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4"
dependencies = [
"ahash",
"arrow-array",
@@ -6816,7 +6815,7 @@ version = "2.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2"
dependencies = [
- "rand 0.10.1",
+ "rand 0.9.4",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 3070ed33f3..007ca5401e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -90,31 +90,31 @@ version = "55.1.0"
#
# See for more details: https://github.com/rust-lang/cargo/issues/11329
apache-avro = { version = "0.21", default-features = false }
-arrow = { version = "60.0.0", features = [
+arrow = { version = "60.1.0", features = [
"prettyprint",
"chrono-tz",
] }
-arrow-avro = { version = "60.0.0", default-features = false, features = [
+arrow-avro = { version = "60.1.0", default-features = false, features = [
"deflate",
"snappy",
"zstd",
"bzip2",
"xz",
] }
-arrow-buffer = { version = "60.0.0", default-features = false }
-arrow-data = { version = "60.0.0", default-features = false }
-arrow-flight = { version = "60.0.0", features = [
+arrow-buffer = { version = "60.1.0", default-features = false }
+arrow-data = { version = "60.1.0", default-features = false }
+arrow-flight = { version = "60.1.0", features = [
"flight-sql",
] }
# Both codecs are required here to make sure that code paths like
# file-spilling have access to all compression codecs.
-arrow-ipc = { version = "60.0.0", default-features = false, features = [
+arrow-ipc = { version = "60.1.0", default-features = false, features = [
"lz4",
"zstd",
] }
-arrow-ord = { version = "60.0.0", default-features = false }
-arrow-schema = { version = "60.0.0", default-features = false }
-arrow-select = { version = "60.0.0", default-features = false }
+arrow-ord = { version = "60.1.0", default-features = false }
+arrow-schema = { version = "60.1.0", default-features = false }
+arrow-select = { version = "60.1.0", default-features = false }
async-trait = "0.1.89"
bigdecimal = "0.4.8"
bytes = "1.11"
@@ -181,7 +181,7 @@ memchr = "2.8.1"
num-traits = { version = "0.2" }
object_store = { version = "0.14.2", default-features = false }
parking_lot = "0.13"
-parquet = { version = "60.0.0", default-features = false, features = [
+parquet = { version = "60.1.0", default-features = false, features = [
"arrow",
"async",
"object_store",
diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs
index ab57d87dce..842105c99d 100644
--- a/datafusion/physical-plan/src/spill/mod.rs
+++ b/datafusion/physical-plan/src/spill/mod.rs
@@ -819,8 +819,8 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
/// The IPC writer writes every data buffer of an array in full, once per
/// entry, and `gc()` copies the bytes of every view separately. Neither
/// handles views that share bytes, as the views of a dictionary-encoded
-/// Parquet column do: after `concat` or `interleave` the dictionary is often
-/// listed once per input batch, and `gc()` copies it once per row
+/// Parquet column do: after `concat` the dictionary is listed once per input
+/// batch, and `gc()` copies it once per row
/// (<https://github.com/apache/datafusion/issues/23564>).
///
/// Repeated buffer entries are merged first. `gc()` then runs only if it
@@ -1985,15 +1985,15 @@ mod tests {
// After appending each batch, spilled_rows and spilled_bytes should increase incrementally,
// while spill_file_count remains 1 (since we're writing to the same file)
in_progress_file.append_batch(&batch1)?;
- verify_metrics(&in_progress_file, 1, 440, 3)?;
+ verify_metrics(&in_progress_file, 1, 424, 3)?;
in_progress_file.append_batch(&batch2)?;
- verify_metrics(&in_progress_file, 1, 704, 6)?;
+ verify_metrics(&in_progress_file, 1, 672, 6)?;
let completed_file = in_progress_file.finish()?;
assert!(completed_file.is_some());
assert!(completed_file.unwrap().path().unwrap().exists());
- verify_metrics(&in_progress_file, 1, 712, 6)?;
+ verify_metrics(&in_progress_file, 1, 680, 6)?;
// Double finish produce error
let result = in_progress_file.finish();
assert!(result.is_err());
@@ -2128,7 +2128,7 @@ mod tests {
// Metrics should be updated immediately (at least schema and first batch)
let bytes_after_batch1 = metrics.spilled_bytes.value();
- assert_eq!(bytes_after_batch1, 440);
+ assert_eq!(bytes_after_batch1, 424);
assert_eq!(metrics.spill_file_count.value(), 1);
// Check global progress
@@ -2493,8 +2493,8 @@ mod tests {
}
/// Arrays that each hold views into one shared dictionary buffer, as the
- /// Parquet reader produces for a dictionary-encoded column, combined the
- /// way a sort combines its input batches. Each output row cycles through
+ /// Parquet reader produces for a dictionary-encoded column, combined with
+ /// `concat` as a sort combines its buffered batches. The rows cycle through
/// the dictionary (<https://github.com/apache/datafusion/issues/23564>).
fn shared_dictionary_views(
dictionary_len: u32,
@@ -2502,7 +2502,7 @@ mod tests {
rows_per_input: u32,
) -> Result<(StringViewArray, StringViewArray)> {
use arrow::array::{AsArray, UInt32Array};
- use arrow::compute::{interleave, take};
+ use arrow::compute::{concat, take};
let dictionary = dictionary_array(dictionary_len)?;
let arrays = (0..inputs)
@@ -2515,17 +2515,14 @@ mod tests {
})
.collect::<std::result::Result<Vec<_>, _>>()?;
let arrays: Vec<&dyn Array> = arrays.iter().map(|a| a.as_ref()).collect();
- let indices: Vec<(usize, usize)> = (0..rows_per_input as usize)
- .flat_map(|row| (0..inputs as usize).map(move |input| (input, row)))
- .collect();
- let merged = interleave(&arrays, &indices)?;
+ let merged = concat(&arrays)?;
Ok((dictionary, merged.as_string_view().clone()))
}
#[test]
fn test_gc_keeps_shared_dictionary_buffer() -> Result<()> {
let (dictionary, merged) = shared_dictionary_views(1000, 8, 1024)?;
- // `interleave` lists the dictionary once per input
+ // `concat` lists the dictionary once per input
assert_eq!(merged.data_buffers().len(), 8);
// `gc()` would copy 8192 values of 63 bytes, 8x the dictionary
assert!(merged.total_buffer_bytes_used() > data_buffers_size(&dictionary));
diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rs
index 6d50245ea6..48e67e72d2 100644
--- a/datafusion/physical-plan/src/spill/spill_pool.rs
+++ b/datafusion/physical-plan/src/spill/spill_pool.rs
@@ -1005,7 +1005,7 @@ mod tests {
);
assert_eq!(
metrics.spilled_bytes.value(),
- 320,
+ 312,
"Spilled bytes should reflect data written (header + 1 batch)"
);
assert_eq!(
@@ -1480,7 +1480,7 @@ mod tests {
// Check metrics before drop - spilled_bytes already reflects written data
let spilled_bytes_before = metrics.spilled_bytes.value();
assert_eq!(
- spilled_bytes_before, 1088,
+ spilled_bytes_before, 1048,
"Spilled bytes should reflect data written (header + 5 batches)"
);
diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs
index a5e57c745a..8dbbed5558 100644
--- a/datafusion/physical-plan/src/unnest.rs
+++ b/datafusion/physical-plan/src/unnest.rs
@@ -1214,7 +1214,7 @@ impl ListArrayType for FixedSizeListArray {
}
fn value_offsets(&self, row: usize) -> (i64, i64) {
- let start = self.value_offset(row) as i64;
+ let start = self.value_offset_at(row) as i64;
(start, start + self.value_length() as i64)
}
}
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 112e257677..605e54d154 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -1732,7 +1732,7 @@ SELECT approx_distinct(c9) AS a, approx_distinct(c9) AS b FROM aggregate_test_10
query IIIII
SELECT approx_distinct(c14) AS a, approx_distinct(c15) AS b, approx_distinct(arrow_cast(c15, 'Date64')), approx_distinct(arrow_cast(c15, 'Time32(Second)')) as c, approx_distinct(arrow_cast(c15, 'Time64(Nanosecond)')) AS d FROM aggregate_test_100
----
-18 60 60 60 60
+18 60 7 60 60
# approx_distinct over Boolean: exact count via flag-pair accumulator (0..=2).
statement ok
diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt
index 8ff029712c..b8ebe00717 100644
--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt
+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt
@@ -1069,12 +1069,14 @@ FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia"))
28 28500 28500000 28500000000
NULL NULL NULL NULL
-# timestamps outside the range arrow represents are NULL, like the other parts
+# The second is computed directly from the stored value (i64::MAX % 60 = 7). The
+# sub-second parts of a timestamp outside the range arrow represents are NULL,
+# like the other parts.
query IIII
SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts)
FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts)
----
-NULL NULL NULL NULL
+7 NULL NULL NULL
NULL NULL NULL NULL
# test_extract_interval
@@ -1603,7 +1605,7 @@ NULL
# Non-null inputs can give NULL results, so the result is nullable.
# Reproducers from https://github.com/apache/datafusion/issues/26685
query I
-SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts)
+SELECT extract(millisecond from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts)
----
NULL
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
index 747433b4d6..ce525005de 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
@@ -1377,13 +1377,9 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z
##########
## SECTION 16: DST boundaries
##
-## KNOWN WRONG, and the single largest divergence in this file: DataFusion
-## cannot represent an ambiguous or non-existent local time in a named zone at
-## all. Every route into a named zone -- a column cast, `AT TIME ZONE`, a
-## string literal, or `::timestamptz` under a session zone -- fails with an
-## error, where PostgreSQL resolves the value.
+## Casting a naive timestamp into a named zone resolves an ambiguous or
+## non-existent local time like PostgreSQL.
##
-## PostgreSQL's answers, for reference:
## ambiguous 2024-11-03 01:30:00 America/Denver -> 2024-11-03 01:30:00-07
## (the second, standard-time occurrence)
## nonexistent 2024-03-10 02:30:00 America/Denver -> 2024-03-10 03:30:00-06
@@ -1391,19 +1387,14 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z
## ambiguous 2024-10-27 02:30:00 Europe/Brussels -> 2024-10-27 02:30:00+01
## nonexistent 2024-03-31 02:30:00 Europe/Brussels -> 2024-03-31 03:30:00+02
##
-## Note that the column path and the literal path fail with *different* errors
-## even though the value is identical: the column path reports
-## "Cannot cast timezone to different timezone" from the Arrow cast kernel,
-## while the literal path is folded by `simplify_expressions` and reports
-## "error computing timezone offset" from the Arrow timestamp parser.
-##
-## See https://github.com/apache/datafusion/issues/25084 and the arrow-rs fix
-## https://github.com/apache/arrow-rs/pull/11038
+## The *parsing* routes into a named zone with a session zone
+## still fail with "error computing timezone offset". See
+## See https://github.com/apache/datafusion/issues/25084
##########
# --- US fall back: 2024-11-03 01:30:00 America/Denver occurs twice ---
-# The naive value itself is fine; only the move into a named zone fails.
+# The naive value itself is fine
query P
SELECT '2024-11-03 01:30:00'::timestamp
----
@@ -1412,25 +1403,29 @@ SELECT '2024-11-03 01:30:00'::timestamp
statement ok
CREATE TABLE dst_ambiguous AS VALUES ('2024-11-03 01:30:00'::timestamp)
-# These expectations match only the stable `Arrow error: ...` tail. The wrapper
-# chain above it (optimizer rule, field context) is not what these cases pin,
-# and it changed in https://github.com/apache/datafusion/pull/24920.
-query error Arrow error: Cast error: Cannot cast timezone to different timezone
+# The ambiguous wall clock resolves to the second (standard time) occurrence
+query P
SELECT column1 AT TIME ZONE 'America/Denver' FROM dst_ambiguous
+----
+2024-11-03T01:30:00-07:00
-query error Arrow error: Cast error: Cannot cast timezone to different timezone
+query P
SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM dst_ambiguous
+----
+2024-11-03T01:30:00-07:00
-# The same wall clock arriving as a string literal instead of a column
+# String literal parsing fails.
+# https://github.com/apache/arrow-rs/issues/11039
query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-11\-03T01:30:00':\ error\ computing\ timezone\ offset
SELECT arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))')
-query error Arrow\ error:\ Cast\ error:\ Cannot\ cast\ timezone\ to\ different\ timezone
+query P
SELECT TIMESTAMP '2024-11-03 01:30:00' AT TIME ZONE 'America/Denver'
+----
+2024-11-03T01:30:00-07:00
-# Both instants are representable when the offset is spelled out explicitly --
-# it is only the *resolution* of a bare wall clock that fails. Note the two
-# rows differ solely in the rendered offset.
+# Both instants are representable when the offset is spelled out explicitly.
+# Note the two rows differ solely in the rendered offset.
query PP
SELECT arrow_cast('2024-11-03T07:30:00Z', 'Timestamp(Nanosecond, Some("America/Denver"))'),
arrow_cast('2024-11-03T08:30:00Z', 'Timestamp(Nanosecond, Some("America/Denver"))')
@@ -1442,17 +1437,23 @@ SELECT arrow_cast('2024-11-03T07:30:00Z', 'Timestamp(Nanosecond, Some("America/D
statement ok
CREATE TABLE dst_nonexistent AS VALUES ('2024-03-10 02:30:00'::timestamp)
-query error Arrow error: Cast error: Cannot cast timezone to different timezone
+query P
SELECT column1 AT TIME ZONE 'America/Denver' FROM dst_nonexistent
+----
+2024-03-10T03:30:00-06:00
-query error Arrow error: Cast error: Cannot cast timezone to different timezone
+query P
SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM dst_nonexistent
+----
+2024-03-10T03:30:00-06:00
query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-03\-10T02:30:00':\ error\ computing\ timezone\ offset
SELECT arrow_cast('2024-03-10T02:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))')
-query error Arrow\ error:\ Cast\ error:\ Cannot\ cast\ timezone\ to\ different\ timezone
+query P
SELECT TIMESTAMP '2024-03-10 02:30:00' AT TIME ZONE 'America/Denver'
+----
+2024-03-10T03:30:00-06:00
statement ok
SET datafusion.execution.time_zone = 'America/Denver'
@@ -1471,11 +1472,17 @@ RESET datafusion.execution.time_zone
statement ok
CREATE TABLE dst_eu AS VALUES ('2024-10-27 02:30:00'::timestamp), ('2024-03-31 02:30:00'::timestamp)
-query error Arrow error: Cast error: Cannot cast timezone to different timezone
+query P
SELECT column1 AT TIME ZONE 'Europe/Brussels' FROM dst_eu
+----
+2024-10-27T02:30:00+01:00
+2024-03-31T03:30:00+02:00
-query error Arrow error: Cast error: Cannot cast timezone to different timezone
+query P
SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Europe/Brussels"))') FROM dst_eu
+----
+2024-10-27T02:30:00+01:00
+2024-03-31T03:30:00+02:00
query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-10\-27T02:30:00':\ error\ computing\ timezone\ offset
SELECT arrow_cast('2024-10-27T02:30:00', 'Timestamp(Nanosecond, Some("Europe/Brussels"))')
@@ -1496,9 +1503,7 @@ statement ok
RESET datafusion.execution.time_zone
# A tz-aware column crossing the fall-back boundary. Constructing the values
-# from explicit UTC instants sidesteps the failure above, so the behaviour of
-# date_trunc / date_bin / date_part / interval arithmetic across a repeated
-# hour can still be pinned down.
+# from explicit UTC instants covers a repeated hour
statement ok
CREATE TABLE dst_cross AS
SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') AS ts
diff --git a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt
index 5dfdfff6d5..adcd7b949e 100644
--- a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt
+++ b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt
@@ -302,7 +302,7 @@ RIGHT SEMI JOIN existence_spill_right r ON l.threshold > r.k
Plan with Metrics
01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]
02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]
-03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=128, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]
+03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=128, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]
04)------ProjectionExec: expr=[CASE WHEN value@0 = 4097 THEN 130 WHEN value@0 % 65 = 0 THEN 65 WHEN value@0 % 65 = 1 THEN 97 ELSE 114 END as threshold], metrics=[<slt:ignore>]
05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]
06)------ProjectionExec: expr=[CASE WHEN __common_expr_1@0 = 128 THEN NULL ELSE __common_expr_1@0 END as k], metrics=[<slt:ignore>]
@@ -317,7 +317,7 @@ RIGHT ANTI JOIN existence_spill_right r ON l.threshold > r.k
Plan with Metrics
01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]
02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]
-03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=1, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]
+03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=1, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]
04)------ProjectionExec: expr=[CASE WHEN value@0 = 4097 THEN 130 WHEN value@0 % 65 = 0 THEN 65 WHEN value@0 % 65 = 1 THEN 97 ELSE 114 END as threshold], metrics=[<slt:ignore>]
05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]
06)------ProjectionExec: expr=[CASE WHEN __common_expr_1@0 = 128 THEN NULL ELSE __common_expr_1@0 END as k], metrics=[<slt:ignore>]
@@ -339,7 +339,7 @@ RIGHT SEMI JOIN range(129) r ON l.threshold > r.value
----
129 8256 0 128
-# Both joins must spill and replay the two right batches across four left chunks.
+# Both joins must spill and replay the two right batches across three left chunks.
query TT
EXPLAIN ANALYZE SELECT count(*)
FROM existence_spill_early_left l
@@ -348,7 +348,7 @@ RIGHT SEMI JOIN range(129) r ON l.threshold > r.value
Plan with Metrics
01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]
02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]
-03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=129, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]
+03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=129, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]
04)------ProjectionExec: expr=[CASE WHEN value@0 = 0 THEN 129 ELSE 0 END as threshold], metrics=[<slt:ignore>]
05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]
06)------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=129, batch_size=65], metrics=[<slt:ignore>]
@@ -361,7 +361,7 @@ RIGHT ANTI JOIN range(129) r ON l.threshold > r.value
Plan with Metrics
01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]
02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]
-03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=0, <slt:ignore> spill_count=2, <slt:ignore> input_batches=8, input_rows=516, <slt:ignore>]
+03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=0, <slt:ignore> spill_count=2, <slt:ignore> input_batches=6, input_rows=387, <slt:ignore>]
04)------ProjectionExec: expr=[CASE WHEN value@0 = 0 THEN 129 ELSE 0 END as threshold], metrics=[<slt:ignore>]
05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[<slt:ignore>]
06)------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=129, batch_size=65], metrics=[<slt:ignore>]
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 6b33b5eded..4665bfa52e 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -96,16 +96,16 @@ Aggregate plans now display `group_clustering_mode=Full` or
`group_clustering_mode=Partial(indices)` instead of `ordering_mode=Sorted` or
`ordering_mode=PartiallySorted(indices)`.
-### Upgrade arrow/parquet to 60.0.0 and object_store to 0.14.2
+### Upgrade arrow/parquet to 60 and object_store to 0.14
-DataFusion 56.0.0 uses `arrow` and `parquet` 60.0.0, and `object_store` 0.14.2.
+DataFusion 56.0.0 uses `arrow` and `parquet` 60, and `object_store` 0.14.
This may require updates to your Cargo.toml if you have direct dependencies on
these crates.
-See the [Arrow 60.0.0 release notes] and the [object_store 0.14.2 upgrade guide] for details on breaking changes in those versions.
+See the [Arrow 60.0.0 release notes] and the [object_store 0.14.0 changelog] for details on breaking changes in those versions.
[arrow 60.0.0 release notes]: https://github.com/apache/arrow-rs/releases/tag/60.0.0
-[object_store 0.14.2 upgrade guide]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG.md
+[object_store 0.14.0 changelog]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG-old.md#v0140-2026-06-18
#### Field and schema metadata now use arrow's `Metadata` type