Skip to content

Apache DataFusion SQL Query Engine

fix: prune statistics only through casts that keep order (#26982)

Jay Zhan4bf32cb46df9
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index d605f0cf49..23291099ec 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -1663,22 +1663,25 @@ mod tests {
fn bloom_filter_pruning_error_retains_row_group_without_match() {
let schema =
Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
- let cast_expr: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(
- Arc::new(PhysicalColumn::new("c1", 0)),
- DataType::FixedSizeBinary(4),
+ // c1 = CAST('abc' AS Int32): constant folding fails, so the
+ // simplifier keeps the cast, and the cast fails at evaluation.
+ let value: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(
+ Arc::new(Literal::new(ScalarValue::from("abc"))),
+ DataType::Int32,
None,
));
- let literal: Arc<dyn PhysicalExpr> = Arc::new(Literal::new(
- ScalarValue::FixedSizeBinary(4, Some(vec![0, 0, 0, 1])),
+ let expr = Arc::new(BinaryExpr::new(
+ Arc::new(PhysicalColumn::new("c1", 0)),
+ Operator::Eq,
+ value,
));
- let expr = Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal));
let pruning_predicate = PruningPredicateBuilder::new()
.with_file_schema(schema)
.try_build(expr)
.unwrap();
// A nonempty map bypasses the no-filter path. Evaluating the predicate
- // then fails because Arrow cannot cast Int32 statistics to FixedSizeBinary.
+ // then fails because 'abc' is not an Int32.
let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap();
sbbf.insert(&1_i32);
let mut bloom_statistics = BloomFilterStatistics::new();
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
index 7fbed9187b..a832ce0662 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -379,38 +379,67 @@ fn is_fixed_offset(tz: &str) -> bool {
tz == "UTC" || tz.starts_with(['+', '-'])
}
-/// Whether successful casts preserve order when conversion failures return errors.
-/// Unlike `check_bigger_cast`, this allows precision loss and is not sufficient
-/// for propagating distinct counts or the ordering of subsequent sort keys.
-fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool {
- use arrow::datatypes::TimeUnit::*;
+fn is_binary(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ Binary | LargeBinary | BinaryView | FixedSizeBinary(_)
+ )
+}
+
+/// Temporal types stored as a count of units since the epoch (or midnight),
+/// which a cast from or to a number reads or writes as that count.
+fn is_temporal_count(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ Date32 | Date64 | Timestamp(_, _) | Time32(_) | Time64(_) | Duration(_)
+ )
+}
+
+/// Whether the cast never reverses the order of two values that both convert:
+/// `a <= b` implies `cast(a) <= cast(b)`. Precision loss may merge values, so
+/// this is not sufficient for propagating distinct counts or the ordering of
+/// subsequent sort keys (see [`CastExpr::check_bigger_cast`] for that).
+///
+/// Values that fail to convert are not covered: callers must account for the
+/// error or NULL a failure produces.
+pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool {
if source_type == target_type
|| (source_type.is_numeric() || *source_type == Boolean)
&& target_type.is_numeric()
+ // Changing the string or binary encoding keeps the byte order.
+ || source_type.is_string() && target_type.is_string()
+ || is_binary(source_type) && is_binary(target_type)
+ || source_type.is_numeric() && is_temporal_count(target_type)
+ || is_temporal_count(source_type) && target_type.is_numeric()
{
return true;
}
// Temporal casts are not generally monotonic: extracting time-of-day wraps
// at midnight, and timezone transitions can reverse the local date.
match (source_type, target_type) {
- (Date32 | Date64, Date32 | Date64)
- | (Date32 | Date64, Timestamp(_, None))
- | (Timestamp(_, None), Date32) => true,
- (Timestamp(_, Some(tz)), Date32) => is_fixed_offset(tz),
- // Arrow converts timestamps to Date64 by scaling the epoch value,
- // whereas Date32 extracts the date in the timestamp's timezone.
- (Timestamp(_, _), Date64) => true,
- // Only admit scaling that cannot wrap on overflow. Time64 widening
- // and narrowing to Time32 do not currently check overflow in Arrow.
- (Time32(Second | Millisecond), Time32(Second | Millisecond))
- | (Time32(Second | Millisecond), Time64(Microsecond | Nanosecond))
- | (Time64(Nanosecond), Time64(Microsecond))
- | (Duration(_), Duration(_)) => true,
- (Timestamp(_, from_tz), Timestamp(_, to_tz)) => {
- // Adding a timezone to naive timestamps interprets local times;
- // other timezone changes only change metadata on the epoch value.
- from_tz.is_some() || to_tz.as_deref().is_none_or(is_fixed_offset)
+ (Date32 | Date64, Date32 | Date64) | (Timestamp(_, None), Date32 | Date64) => {
+ true
+ }
+ // Both date types take the date in the timestamp's timezone.
+ (Timestamp(_, Some(tz)), Date32 | Date64) => is_fixed_offset(tz),
+ // Arrow checks every scaling that can overflow, so a value that does
+ // not fit fails to convert instead of wrapping.
+ (Time32(_) | Time64(_), Time32(_) | Time64(_)) | (Duration(_), Duration(_)) => {
+ true
}
+ // Changing the timezone only changes metadata on the epoch value.
+ (Timestamp(_, Some(_)), Timestamp(_, _)) => true,
+ // Adding a timezone reads a naive timestamp (or a Date64, which may
+ // hold any time of day) as a local time. Arrow moves a local time that
+ // a transition skips forward by the length of the gap, past the local
+ // times just after the gap, so only a fixed offset keeps the order.
+ (Timestamp(_, None) | Date64, Timestamp(_, tz)) => {
+ tz.as_deref().is_none_or(is_fixed_offset)
+ }
+ // A Date32 reads as local midnight. A skipped midnight moves forward
+ // by the gap, and no gap is longer than a day, so it never passes the
+ // next midnight.
+ (Date32, Timestamp(_, _)) => true,
_ => false,
}
}
@@ -2186,11 +2215,16 @@ mod tests {
(Date64, Date32, true),
(Date32, Timestamp(Nanosecond, None), true),
(Date64, Timestamp(Second, None), true),
+ (Date32, Timestamp(Second, timezone.clone()), true),
+ (Date64, Timestamp(Second, Some("+08:00".into())), true),
+ (Date64, Timestamp(Second, timezone.clone()), false),
(Timestamp(Second, None), Date32, true),
+ (Timestamp(Second, None), Date64, true),
(Timestamp(Second, Some("UTC".into())), Date32, true),
(Timestamp(Second, Some("+08:00".into())), Date32, true),
+ (Timestamp(Second, Some("+08:00".into())), Date64, true),
(Timestamp(Second, timezone.clone()), Date32, false),
- (Timestamp(Second, timezone.clone()), Date64, true),
+ (Timestamp(Second, timezone.clone()), Date64, false),
(Timestamp(Second, None), Timestamp(Nanosecond, None), true),
(Timestamp(Nanosecond, None), Timestamp(Second, None), true),
(
@@ -2219,8 +2253,8 @@ mod tests {
(Time32(Millisecond), Time32(Second), true),
(Time32(Second), Time64(Nanosecond), true),
(Time64(Nanosecond), Time64(Microsecond), true),
- (Time64(Microsecond), Time64(Nanosecond), false),
- (Time64(Nanosecond), Time32(Second), false),
+ (Time64(Microsecond), Time64(Nanosecond), true),
+ (Time64(Nanosecond), Time32(Second), true),
(Duration(Second), Duration(Nanosecond), true),
(Duration(Nanosecond), Duration(Second), true),
(Null, Timestamp(Second, None), false),
@@ -2263,6 +2297,250 @@ mod tests {
}
}
+ /// Sorted, distinct, non-null sample values of `data_type`, obtained by
+ /// casting a fixed set of seeds and keeping the values that convert.
+ fn order_samples(data_type: &DataType) -> Option<ArrayRef> {
+ use arrow::array::{
+ BooleanArray, FixedSizeBinaryArray, Float64Array, TimestampSecondArray,
+ UInt32Array, make_comparator,
+ };
+ use arrow::compute::{
+ SortOptions, concat, filter, is_not_null, sort_to_indices, take,
+ };
+
+ let seeds: Vec<ArrayRef> = vec![
+ Arc::new(Int64Array::from(vec![
+ i64::MIN,
+ -2_147_483_649,
+ -129,
+ -1,
+ 0,
+ 1,
+ 9,
+ 10,
+ 127,
+ 128,
+ 255,
+ 256,
+ // As dates, 2011-12-29 to 2011-12-31: Pacific/Apia skipped
+ // all of 2011-12-30.
+ 15_337,
+ 15_338,
+ 15_339,
+ 86_399,
+ // America/Goose_Bay fell back from 00:01 to 23:01 the previous
+ // day at this instant, so the local date goes backwards.
+ 562_129_259,
+ 562_129_260,
+ 2_147_483_647,
+ i64::MAX,
+ ])),
+ Arc::new(Float64Array::from(vec![
+ f64::NEG_INFINITY,
+ -1.5,
+ -0.5,
+ 0.0,
+ 0.5,
+ 1.7,
+ 1e300,
+ f64::INFINITY,
+ ])),
+ Arc::new(StringArray::from(vec!["", "-1", "0", "1", "10", "9", "a"])),
+ Arc::new(BooleanArray::from(vec![false, true])),
+ Arc::new(
+ FixedSizeBinaryArray::try_from_iter(
+ ["abcd", "abce", "zzzz"].into_iter().map(str::as_bytes),
+ )
+ .expect("valid fixed size binary"),
+ ),
+ Arc::new(TimestampSecondArray::from(vec![
+ // 23:00, 00:00 and 01:00 around midnight
+ 82_800,
+ 86_400,
+ 90_000,
+ // Local times around the America/New_York transitions of
+ // 2024: before, inside and after the skipped hour in March and
+ // the repeated hour in November.
+ 1_710_035_999,
+ 1_710_037_800,
+ 1_710_039_600,
+ 1_730_595_599,
+ 1_730_597_400,
+ 1_730_599_200,
+ ])),
+ ];
+ // Cast each seed on its own, so one value that fails to convert does
+ // not discard the others.
+ let parts = seeds
+ .iter()
+ .flat_map(|seed| (0..seed.len()).map(|i| seed.slice(i, 1)))
+ .filter_map(|seed| arrow::compute::cast(&seed, data_type).ok())
+ .map(|array| filter(&array, &is_not_null(&array).unwrap()).unwrap())
+ .filter(|array| !array.is_empty())
+ .collect::<Vec<_>>();
+ if parts.is_empty() {
+ return None;
+ }
+ let parts = parts.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
+ let all = concat(&parts).unwrap();
+ let sorted =
+ take(&all, &sort_to_indices(&all, None, None).unwrap(), None).unwrap();
+ let cmp = make_comparator(&sorted, &sorted, SortOptions::default()).unwrap();
+ let distinct = (0..sorted.len())
+ .filter(|&i| i == 0 || cmp(i - 1, i).is_ne())
+ .map(|i| i as u32)
+ .collect::<UInt32Array>();
+ Some(take(&sorted, &distinct, None).unwrap())
+ }
+
+ /// Casts the sorted `samples` to `target` with the strict cast that
+ /// `CastExpr` runs by default (`safe = false`) or the cast that
+ /// `TryCastExpr` runs (`safe = true`), and returns the first pair of
+ /// converted values whose order the cast reverses. Each value is cast on
+ /// its own, and values that fail to convert are skipped:
+ /// `is_order_preserving_cast` does not cover them.
+ fn order_violation(
+ samples: &ArrayRef,
+ target: &DataType,
+ safe: bool,
+ ) -> Option<String> {
+ use arrow::array::make_comparator;
+ use arrow::compute::SortOptions;
+
+ let options = CastOptions {
+ safe,
+ format_options: DEFAULT_FORMAT_OPTIONS,
+ };
+ let converted = (0..samples.len())
+ .filter_map(|i| {
+ let value = ColumnarValue::Array(samples.slice(i, 1));
+ match value.cast_to(target, Some(&options)) {
+ Ok(ColumnarValue::Array(output)) if output.is_valid(0) => {
+ Some((i, output))
+ }
+ _ => None,
+ }
+ })
+ .collect::<Vec<_>>();
+ converted.windows(2).find_map(|w| {
+ let ((i, left), (j, right)) = (&w[0], &w[1]);
+ let cmp = make_comparator(left, right, SortOptions::default()).unwrap();
+ cmp(0, 0).is_gt().then(|| {
+ let value = |array: &ArrayRef, i| {
+ ScalarValue::try_from_array(array, i).unwrap().to_string()
+ };
+ format!(
+ "{} -> {}, but {} -> {}",
+ value(samples, *i),
+ value(left, 0),
+ value(samples, *j),
+ value(right, 0),
+ )
+ })
+ })
+ }
+
+ /// Checks `is_order_preserving_cast` against the casts that `CastExpr` and
+ /// `TryCastExpr` run: every pair it accepts must keep the order of the
+ /// sample values that convert.
+ /// Pruning (for both casts) and sort elimination (for the strict cast) rely
+ /// on this rule, so a wrong entry drops matching rows or returns unsorted
+ /// output.
+ #[test]
+ fn test_order_preserving_cast_matches_cast_kernel() {
+ use TimeUnit::*;
+ use arrow::array::{AsArray, Date32Array};
+
+ let tz = |unit, tz: &str| Timestamp(unit, Some(tz.into()));
+ let types = [
+ Boolean,
+ Int8,
+ Int16,
+ Int32,
+ Int64,
+ UInt8,
+ UInt16,
+ UInt32,
+ UInt64,
+ Float32,
+ Float64,
+ Decimal128(10, 2),
+ Decimal256(40, 2),
+ Utf8,
+ LargeUtf8,
+ Utf8View,
+ Binary,
+ LargeBinary,
+ BinaryView,
+ FixedSizeBinary(4),
+ Date32,
+ Date64,
+ Timestamp(Second, None),
+ Timestamp(Nanosecond, None),
+ tz(Millisecond, "UTC"),
+ tz(Second, "+05:00"),
+ tz(Second, "America/New_York"),
+ tz(Second, "America/Goose_Bay"),
+ tz(Second, "Pacific/Apia"),
+ Time32(Second),
+ Time32(Millisecond),
+ Time64(Microsecond),
+ Time64(Nanosecond),
+ Duration(Second),
+ Duration(Nanosecond),
+ ];
+ let mut violations = vec![];
+ for source in &types {
+ let Some(samples) = order_samples(source) else {
+ continue;
+ };
+ for target in &types {
+ if !can_cast_types(source, target)
+ || !is_order_preserving_cast(source, target)
+ {
+ continue;
+ }
+ for safe in [false, true] {
+ if let Some(e) = order_violation(&samples, target, safe) {
+ violations
+ .push(format!("{source} -> {target} (safe={safe}): {e}"));
+ }
+ }
+ }
+ }
+ assert!(violations.is_empty(), "{}", violations.join("\n"));
+
+ // The samples are able to expose the casts the rule rejects.
+ for (source, target) in [
+ (Utf8, Int32),
+ (Int32, Boolean),
+ (Int16, Binary),
+ (Timestamp(Second, None), Time32(Second)),
+ (tz(Second, "+05:00"), Time64(Nanosecond)),
+ (tz(Second, "America/Goose_Bay"), Date32),
+ (tz(Second, "America/Goose_Bay"), Date64),
+ (Timestamp(Second, None), tz(Second, "America/New_York")),
+ ] {
+ assert!(!is_order_preserving_cast(&source, &target));
+ let samples = order_samples(&source).unwrap();
+ assert!(
+ order_violation(&samples, &target, false).is_some(),
+ "{source} -> {target}: no sample exposes the reversed order"
+ );
+ }
+
+ // The samples reach the edge of the Date32 entry: a skipped day starts
+ // at the same instant as the day after it.
+ let skipped =
+ ColumnarValue::Array(Arc::new(Date32Array::from(vec![15_338, 15_339])))
+ .cast_to(&tz(Second, "Pacific/Apia"), None)
+ .unwrap()
+ .into_array(2)
+ .unwrap();
+ let skipped = skipped.as_primitive::<TimestampSecondType>();
+ assert_eq!(skipped.value(0), skipped.value(1));
+ }
+
#[test]
fn test_cast_ordering_with_null_on_failure() {
use arrow::array::TimestampSecondArray;
diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs
index 767c0b4ce4..bb5c818ece 100644
--- a/datafusion/physical-expr/src/expressions/mod.rs
+++ b/datafusion/physical-expr/src/expressions/mod.rs
@@ -30,7 +30,7 @@ pub use crate::aggregate::stats::StatsType;
pub use binary::{BinaryExpr, binary, similar_to};
pub use case::{CaseExpr, case};
-pub use cast::{CastExpr, cast};
+pub use cast::{CastExpr, cast, is_order_preserving_cast};
pub use column::{Column, col, with_new_schema};
pub use datafusion_expr::utils::format_state_name;
pub use dynamic_filters::{
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index 2e3c84673f..b7325d90d5 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -1331,10 +1331,10 @@ fn is_compare_op(op: Operator) -> bool {
)
}
-// The pruning logic is based on the comparing the min/max bounds.
-// Must make sure the two type has order.
-// For example, casts from string to numbers is not correct.
-// Because the "13" is less than "3" with UTF8 comparison order.
+// Pruning casts the container's min/max statistics and compares them with the
+// literal, which is only sound when the cast keeps the order of the values,
+// e.g. not for strings to numbers ("13" < "3") or timestamps to time of day
+// (wraps at midnight). Ask the same rule that sort elimination uses.
fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Result<()> {
// Dictionary casts are always supported as long as the value types are supported
let from_type = match from_type {
@@ -1349,20 +1349,7 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re
}
_ => to_type,
};
- // Casting a timestamp to a time-of-day wraps at midnight and does not
- // preserve ordering across day boundaries. Casting timestamp min/max
- // statistics could therefore produce invalid bounds and prune matches.
- if matches!(from_type, DataType::Timestamp(_, _))
- && matches!(to_type, DataType::Time32(_) | DataType::Time64(_))
- {
- return plan_err!(
- "Try Cast/Cast with from type {from_type} to type {to_type} is not supported"
- );
- }
- // If both types are strings or both are not strings (number, timestamp, etc)
- // then we can compare them.
- // PruningPredicate does not support casting of strings to numbers and such.
- if from_type.is_string() == to_type.is_string() {
+ if phys_expr::is_order_preserving_cast(from_type, to_type) {
Ok(())
} else {
plan_err!(
@@ -5095,6 +5082,99 @@ mod tests {
Ok(())
}
+ /// Pruning casts the min/max statistics, so it may only use casts that
+ /// keep the order of the values.
+ #[test]
+ fn row_group_predicate_cast_must_preserve_order() {
+ use TimeUnit::*;
+
+ let new_york = Some(Arc::from("America/New_York"));
+ let dictionary = |value: DataType| {
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(value))
+ };
+ let cases = [
+ // 0 is false while -1 and 1 are true
+ (DataType::Int32, ScalarValue::Boolean(Some(false)), false),
+ (DataType::Float64, ScalarValue::Boolean(Some(false)), false),
+ (
+ dictionary(DataType::Int64),
+ ScalarValue::Boolean(Some(false)),
+ false,
+ ),
+ // little-endian bytes: 256 is [0, 1, ..] and 255 is [255, 0, ..]
+ (DataType::Int64, ScalarValue::Binary(Some(vec![5])), false),
+ // a named timezone can move the local date backwards
+ (
+ DataType::Timestamp(Nanosecond, new_york.clone()),
+ ScalarValue::Date32(Some(0)),
+ false,
+ ),
+ (
+ DataType::Timestamp(Nanosecond, new_york.clone()),
+ ScalarValue::Date64(Some(0)),
+ false,
+ ),
+ // a local time that a transition skips moves past later ones
+ (
+ DataType::Timestamp(Nanosecond, None),
+ ScalarValue::TimestampNanosecond(Some(0), new_york.clone()),
+ false,
+ ),
+ (
+ DataType::Int64,
+ ScalarValue::TimestampNanosecond(Some(0), None),
+ true,
+ ),
+ (DataType::Int32, ScalarValue::Date32(Some(0)), true),
+ // Parquet TIME(MICROS) read as a TIME column
+ (
+ DataType::Time64(Microsecond),
+ ScalarValue::Time64Nanosecond(Some(0)),
+ true,
+ ),
+ (
+ DataType::Utf8,
+ ScalarValue::Utf8View(Some("a".into())),
+ true,
+ ),
+ (dictionary(DataType::Utf8), ScalarValue::from("a"), true),
+ (
+ DataType::FixedSizeBinary(1),
+ ScalarValue::Binary(Some(vec![5])),
+ true,
+ ),
+ (
+ DataType::Timestamp(Nanosecond, None),
+ ScalarValue::TimestampNanosecond(Some(0), Some(Arc::from("-05:00"))),
+ true,
+ ),
+ (
+ DataType::Date32,
+ ScalarValue::TimestampSecond(Some(0), new_york),
+ true,
+ ),
+ ];
+ for (source, literal, prunable) in cases {
+ let target = literal.data_type();
+ let schema = Schema::new(vec![Field::new("c1", source.clone(), false)]);
+ for expr in [
+ cast(col("c1"), target.clone()).eq(lit(literal.clone())),
+ try_cast(col("c1"), target.clone()).eq(lit(literal.clone())),
+ ] {
+ let predicate_expr = test_build_predicate_expression(
+ &expr,
+ &schema,
+ &mut RequiredColumns::new(),
+ );
+ assert_eq!(
+ predicate_expr.to_string() != "true",
+ prunable,
+ "{source} -> {target}: {predicate_expr}"
+ );
+ }
+ }
+ }
+
#[test]
fn row_group_predicate_date_date() -> Result<()> {
let schema = Schema::new(vec![Field::new("c1", DataType::Date32, false)]);
diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.slt
index 3d11899c93..6dedfa39c8 100644
--- a/datafusion/sqllogictest/test_files/cast.slt
+++ b/datafusion/sqllogictest/test_files/cast.slt
@@ -142,6 +142,19 @@ ORDER BY d;
562129260 1987-10-24
562129259 1987-10-25
+# Date64 also takes the local date, so it reverses the same way.
+query ID
+SELECT arrow_cast(ts, 'Int64') AS epoch, arrow_cast(ts, 'Date64') AS d
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second, Some("America/Goose_Bay"))') AS ts
+ FROM (VALUES (562129259::bigint), (562129260::bigint))
+ ORDER BY ts LIMIT 2
+)
+ORDER BY d;
+----
+562129260 1987-10-24T00:00:00
+562129259 1987-10-25T00:00:00
+
# Reducing timestamp precision remains order-preserving, even when values merge.
statement ok
SET datafusion.explain.physical_plan_only = true;
@@ -164,6 +177,35 @@ physical_plan
02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
+# An integer read as a count of seconds keeps its order, so the outer sort is
+# removed.
+query TT
+EXPLAIN SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts
+FROM (
+ SELECT column1 AS i
+ FROM (VALUES (3::bigint), (1::bigint), (2::bigint))
+ ORDER BY i LIMIT 3
+)
+ORDER BY ts;
+----
+physical_plan
+01)ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
+02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+query P
+SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts
+FROM (
+ SELECT column1 AS i
+ FROM (VALUES (3::bigint), (1::bigint), (2::bigint))
+ ORDER BY i LIMIT 3
+)
+ORDER BY ts;
+----
+1970-01-01T00:00:01
+1970-01-01T00:00:02
+1970-01-01T00:00:03
+
statement ok
SET datafusion.explain.physical_plan_only = false;
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index ea61f0be6c..3e5a35e0fe 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -1602,3 +1602,117 @@ RESET datafusion.execution.parquet.pushdown_filters;
statement ok
RESET datafusion.explain.physical_plan_only;
+
+##########
+# Pruning casts the min/max statistics of each row group, so it may only use a
+# cast that keeps the order of the values. Each predicate below runs against
+# an in-memory table, which is never pruned, and the same rows in one Parquet
+# row group. The two must agree.
+##########
+
+statement ok
+CREATE TABLE cast_order_src(i INT, ts TIMESTAMP) AS VALUES
+ (-1, TIMESTAMP '2024-03-10 00:00:00'),
+ (0, TIMESTAMP '2024-03-10 01:00:00'),
+ (1, TIMESTAMP '2024-03-10 02:30:00'),
+ (255, TIMESTAMP '2024-03-10 02:45:00'),
+ (256, TIMESTAMP '2024-03-10 03:00:00');
+
+statement ok
+COPY cast_order_src
+TO 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE cast_order
+STORED AS PARQUET
+LOCATION 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet';
+
+# -1 and 1 are true and 0 is false, so the bounds -1 and 256 both cast to true.
+query I
+SELECT i FROM cast_order_src WHERE CAST(i AS BOOLEAN) <= false;
+----
+0
+
+query I
+SELECT i FROM cast_order WHERE CAST(i AS BOOLEAN) <= false;
+----
+0
+
+# An integer casts to its little-endian bytes: -1 is X'FFFFFFFF' and 256 is
+# X'00010000', so the bounds no longer bracket the values in between.
+query I
+SELECT i FROM cast_order_src WHERE CAST(i AS BYTEA) < X'05' ORDER BY i;
+----
+0
+1
+256
+
+query I
+SELECT i FROM cast_order WHERE CAST(i AS BYTEA) < X'05' ORDER BY i;
+----
+0
+1
+256
+
+# Comparing a timestamp without a timezone to one with a timezone casts the
+# column to that timezone, reading the values as local times. New York skipped
+# 02:00 to 03:00 on 2024-03-10, and Arrow reads 02:30 as 03:30 (07:30 UTC),
+# after 03:00 (07:00 UTC). The bound 03:00 is no longer the largest value after
+# the cast, so this cast must not prune.
+statement ok
+SET datafusion.execution.time_zone = 'America/New_York';
+
+query I
+SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;
+----
+1
+255
+
+query I
+SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;
+----
+1
+255
+
+# A fixed offset skips no local times, so that cast keeps the order and still
+# prunes.
+statement ok
+SET datafusion.execution.time_zone = '-05:00';
+
+statement ok
+SET datafusion.explain.physical_plan_only = true;
+
+query TT
+EXPLAIN SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00';
+----
+physical_plan
+01)FilterExec: CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, projection=[i@0]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/cast_order/data.parquet]]}, projection=[i, ts], file_type=parquet, predicate=CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, pruning_predicate=ts_null_count@1 != row_count@2 AND CAST(ts_max@0 AS Timestamp(ns, "-05:00");N) > 1710054900000000000, required_guarantees=[]
+
+query I
+SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;
+----
+1
+255
+256
+
+query I
+SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;
+----
+1
+255
+256
+
+statement ok
+RESET datafusion.explain.physical_plan_only;
+
+statement ok
+RESET datafusion.execution.time_zone;
+
+statement ok
+DROP TABLE cast_order;
+
+statement ok
+DROP TABLE cast_order_src;