Apache DataFusion SQL Query Engine
fix: prune statistics only through casts that keep order (#26982)
Jay Zhan4bf32cb46df9
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex d605f0cf49..23291099ec 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -1663,22 +1663,25 @@ mod tests {fn bloom_filter_pruning_error_retains_row_group_without_match() {let schema =Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));- let cast_expr: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(- Arc::new(PhysicalColumn::new("c1", 0)),- DataType::FixedSizeBinary(4),+ // c1 = CAST('abc' AS Int32): constant folding fails, so the+ // simplifier keeps the cast, and the cast fails at evaluation.+ let value: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(+ Arc::new(Literal::new(ScalarValue::from("abc"))),+ DataType::Int32,None,));- let literal: Arc<dyn PhysicalExpr> = Arc::new(Literal::new(- ScalarValue::FixedSizeBinary(4, Some(vec![0, 0, 0, 1])),+ let expr = Arc::new(BinaryExpr::new(+ Arc::new(PhysicalColumn::new("c1", 0)),+ Operator::Eq,+ value,));- let expr = Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal));let pruning_predicate = PruningPredicateBuilder::new().with_file_schema(schema).try_build(expr).unwrap();// A nonempty map bypasses the no-filter path. Evaluating the predicate- // then fails because Arrow cannot cast Int32 statistics to FixedSizeBinary.+ // then fails because 'abc' is not an Int32.let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap();sbbf.insert(&1_i32);let mut bloom_statistics = BloomFilterStatistics::new();diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rsindex 7fbed9187b..a832ce0662 100644--- a/datafusion/physical-expr/src/expressions/cast.rs+++ b/datafusion/physical-expr/src/expressions/cast.rs@@ -379,38 +379,67 @@ fn is_fixed_offset(tz: &str) -> bool {tz == "UTC" || tz.starts_with(['+', '-'])}-/// Whether successful casts preserve order when conversion failures return errors.-/// Unlike `check_bigger_cast`, this allows precision loss and is not sufficient-/// for propagating distinct counts or the ordering of subsequent sort keys.-fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool {- use arrow::datatypes::TimeUnit::*;+fn is_binary(data_type: &DataType) -> bool {+ matches!(+ data_type,+ Binary | LargeBinary | BinaryView | FixedSizeBinary(_)+ )+}++/// Temporal types stored as a count of units since the epoch (or midnight),+/// which a cast from or to a number reads or writes as that count.+fn is_temporal_count(data_type: &DataType) -> bool {+ matches!(+ data_type,+ Date32 | Date64 | Timestamp(_, _) | Time32(_) | Time64(_) | Duration(_)+ )+}++/// Whether the cast never reverses the order of two values that both convert:+/// `a <= b` implies `cast(a) <= cast(b)`. Precision loss may merge values, so+/// this is not sufficient for propagating distinct counts or the ordering of+/// subsequent sort keys (see [`CastExpr::check_bigger_cast`] for that).+///+/// Values that fail to convert are not covered: callers must account for the+/// error or NULL a failure produces.+pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool {if source_type == target_type|| (source_type.is_numeric() || *source_type == Boolean)&& target_type.is_numeric()+ // Changing the string or binary encoding keeps the byte order.+ || source_type.is_string() && target_type.is_string()+ || is_binary(source_type) && is_binary(target_type)+ || source_type.is_numeric() && is_temporal_count(target_type)+ || is_temporal_count(source_type) && target_type.is_numeric(){return true;}// Temporal casts are not generally monotonic: extracting time-of-day wraps// at midnight, and timezone transitions can reverse the local date.match (source_type, target_type) {- (Date32 | Date64, Date32 | Date64)- | (Date32 | Date64, Timestamp(_, None))- | (Timestamp(_, None), Date32) => true,- (Timestamp(_, Some(tz)), Date32) => is_fixed_offset(tz),- // Arrow converts timestamps to Date64 by scaling the epoch value,- // whereas Date32 extracts the date in the timestamp's timezone.- (Timestamp(_, _), Date64) => true,- // Only admit scaling that cannot wrap on overflow. Time64 widening- // and narrowing to Time32 do not currently check overflow in Arrow.- (Time32(Second | Millisecond), Time32(Second | Millisecond))- | (Time32(Second | Millisecond), Time64(Microsecond | Nanosecond))- | (Time64(Nanosecond), Time64(Microsecond))- | (Duration(_), Duration(_)) => true,- (Timestamp(_, from_tz), Timestamp(_, to_tz)) => {- // Adding a timezone to naive timestamps interprets local times;- // other timezone changes only change metadata on the epoch value.- from_tz.is_some() || to_tz.as_deref().is_none_or(is_fixed_offset)+ (Date32 | Date64, Date32 | Date64) | (Timestamp(_, None), Date32 | Date64) => {+ true+ }+ // Both date types take the date in the timestamp's timezone.+ (Timestamp(_, Some(tz)), Date32 | Date64) => is_fixed_offset(tz),+ // Arrow checks every scaling that can overflow, so a value that does+ // not fit fails to convert instead of wrapping.+ (Time32(_) | Time64(_), Time32(_) | Time64(_)) | (Duration(_), Duration(_)) => {+ true}+ // Changing the timezone only changes metadata on the epoch value.+ (Timestamp(_, Some(_)), Timestamp(_, _)) => true,+ // Adding a timezone reads a naive timestamp (or a Date64, which may+ // hold any time of day) as a local time. Arrow moves a local time that+ // a transition skips forward by the length of the gap, past the local+ // times just after the gap, so only a fixed offset keeps the order.+ (Timestamp(_, None) | Date64, Timestamp(_, tz)) => {+ tz.as_deref().is_none_or(is_fixed_offset)+ }+ // A Date32 reads as local midnight. A skipped midnight moves forward+ // by the gap, and no gap is longer than a day, so it never passes the+ // next midnight.+ (Date32, Timestamp(_, _)) => true,_ => false,}}@@ -2186,11 +2215,16 @@ mod tests {(Date64, Date32, true),(Date32, Timestamp(Nanosecond, None), true),(Date64, Timestamp(Second, None), true),+ (Date32, Timestamp(Second, timezone.clone()), true),+ (Date64, Timestamp(Second, Some("+08:00".into())), true),+ (Date64, Timestamp(Second, timezone.clone()), false),(Timestamp(Second, None), Date32, true),+ (Timestamp(Second, None), Date64, true),(Timestamp(Second, Some("UTC".into())), Date32, true),(Timestamp(Second, Some("+08:00".into())), Date32, true),+ (Timestamp(Second, Some("+08:00".into())), Date64, true),(Timestamp(Second, timezone.clone()), Date32, false),- (Timestamp(Second, timezone.clone()), Date64, true),+ (Timestamp(Second, timezone.clone()), Date64, false),(Timestamp(Second, None), Timestamp(Nanosecond, None), true),(Timestamp(Nanosecond, None), Timestamp(Second, None), true),(@@ -2219,8 +2253,8 @@ mod tests {(Time32(Millisecond), Time32(Second), true),(Time32(Second), Time64(Nanosecond), true),(Time64(Nanosecond), Time64(Microsecond), true),- (Time64(Microsecond), Time64(Nanosecond), false),- (Time64(Nanosecond), Time32(Second), false),+ (Time64(Microsecond), Time64(Nanosecond), true),+ (Time64(Nanosecond), Time32(Second), true),(Duration(Second), Duration(Nanosecond), true),(Duration(Nanosecond), Duration(Second), true),(Null, Timestamp(Second, None), false),@@ -2263,6 +2297,250 @@ mod tests {}}+ /// Sorted, distinct, non-null sample values of `data_type`, obtained by+ /// casting a fixed set of seeds and keeping the values that convert.+ fn order_samples(data_type: &DataType) -> Option<ArrayRef> {+ use arrow::array::{+ BooleanArray, FixedSizeBinaryArray, Float64Array, TimestampSecondArray,+ UInt32Array, make_comparator,+ };+ use arrow::compute::{+ SortOptions, concat, filter, is_not_null, sort_to_indices, take,+ };++ let seeds: Vec<ArrayRef> = vec![+ Arc::new(Int64Array::from(vec![+ i64::MIN,+ -2_147_483_649,+ -129,+ -1,+ 0,+ 1,+ 9,+ 10,+ 127,+ 128,+ 255,+ 256,+ // As dates, 2011-12-29 to 2011-12-31: Pacific/Apia skipped+ // all of 2011-12-30.+ 15_337,+ 15_338,+ 15_339,+ 86_399,+ // America/Goose_Bay fell back from 00:01 to 23:01 the previous+ // day at this instant, so the local date goes backwards.+ 562_129_259,+ 562_129_260,+ 2_147_483_647,+ i64::MAX,+ ])),+ Arc::new(Float64Array::from(vec![+ f64::NEG_INFINITY,+ -1.5,+ -0.5,+ 0.0,+ 0.5,+ 1.7,+ 1e300,+ f64::INFINITY,+ ])),+ Arc::new(StringArray::from(vec!["", "-1", "0", "1", "10", "9", "a"])),+ Arc::new(BooleanArray::from(vec![false, true])),+ Arc::new(+ FixedSizeBinaryArray::try_from_iter(+ ["abcd", "abce", "zzzz"].into_iter().map(str::as_bytes),+ )+ .expect("valid fixed size binary"),+ ),+ Arc::new(TimestampSecondArray::from(vec![+ // 23:00, 00:00 and 01:00 around midnight+ 82_800,+ 86_400,+ 90_000,+ // Local times around the America/New_York transitions of+ // 2024: before, inside and after the skipped hour in March and+ // the repeated hour in November.+ 1_710_035_999,+ 1_710_037_800,+ 1_710_039_600,+ 1_730_595_599,+ 1_730_597_400,+ 1_730_599_200,+ ])),+ ];+ // Cast each seed on its own, so one value that fails to convert does+ // not discard the others.+ let parts = seeds+ .iter()+ .flat_map(|seed| (0..seed.len()).map(|i| seed.slice(i, 1)))+ .filter_map(|seed| arrow::compute::cast(&seed, data_type).ok())+ .map(|array| filter(&array, &is_not_null(&array).unwrap()).unwrap())+ .filter(|array| !array.is_empty())+ .collect::<Vec<_>>();+ if parts.is_empty() {+ return None;+ }+ let parts = parts.iter().map(|a| a.as_ref()).collect::<Vec<_>>();+ let all = concat(&parts).unwrap();+ let sorted =+ take(&all, &sort_to_indices(&all, None, None).unwrap(), None).unwrap();+ let cmp = make_comparator(&sorted, &sorted, SortOptions::default()).unwrap();+ let distinct = (0..sorted.len())+ .filter(|&i| i == 0 || cmp(i - 1, i).is_ne())+ .map(|i| i as u32)+ .collect::<UInt32Array>();+ Some(take(&sorted, &distinct, None).unwrap())+ }++ /// Casts the sorted `samples` to `target` with the strict cast that+ /// `CastExpr` runs by default (`safe = false`) or the cast that+ /// `TryCastExpr` runs (`safe = true`), and returns the first pair of+ /// converted values whose order the cast reverses. Each value is cast on+ /// its own, and values that fail to convert are skipped:+ /// `is_order_preserving_cast` does not cover them.+ fn order_violation(+ samples: &ArrayRef,+ target: &DataType,+ safe: bool,+ ) -> Option<String> {+ use arrow::array::make_comparator;+ use arrow::compute::SortOptions;++ let options = CastOptions {+ safe,+ format_options: DEFAULT_FORMAT_OPTIONS,+ };+ let converted = (0..samples.len())+ .filter_map(|i| {+ let value = ColumnarValue::Array(samples.slice(i, 1));+ match value.cast_to(target, Some(&options)) {+ Ok(ColumnarValue::Array(output)) if output.is_valid(0) => {+ Some((i, output))+ }+ _ => None,+ }+ })+ .collect::<Vec<_>>();+ converted.windows(2).find_map(|w| {+ let ((i, left), (j, right)) = (&w[0], &w[1]);+ let cmp = make_comparator(left, right, SortOptions::default()).unwrap();+ cmp(0, 0).is_gt().then(|| {+ let value = |array: &ArrayRef, i| {+ ScalarValue::try_from_array(array, i).unwrap().to_string()+ };+ format!(+ "{} -> {}, but {} -> {}",+ value(samples, *i),+ value(left, 0),+ value(samples, *j),+ value(right, 0),+ )+ })+ })+ }++ /// Checks `is_order_preserving_cast` against the casts that `CastExpr` and+ /// `TryCastExpr` run: every pair it accepts must keep the order of the+ /// sample values that convert.+ /// Pruning (for both casts) and sort elimination (for the strict cast) rely+ /// on this rule, so a wrong entry drops matching rows or returns unsorted+ /// output.+ #[test]+ fn test_order_preserving_cast_matches_cast_kernel() {+ use TimeUnit::*;+ use arrow::array::{AsArray, Date32Array};++ let tz = |unit, tz: &str| Timestamp(unit, Some(tz.into()));+ let types = [+ Boolean,+ Int8,+ Int16,+ Int32,+ Int64,+ UInt8,+ UInt16,+ UInt32,+ UInt64,+ Float32,+ Float64,+ Decimal128(10, 2),+ Decimal256(40, 2),+ Utf8,+ LargeUtf8,+ Utf8View,+ Binary,+ LargeBinary,+ BinaryView,+ FixedSizeBinary(4),+ Date32,+ Date64,+ Timestamp(Second, None),+ Timestamp(Nanosecond, None),+ tz(Millisecond, "UTC"),+ tz(Second, "+05:00"),+ tz(Second, "America/New_York"),+ tz(Second, "America/Goose_Bay"),+ tz(Second, "Pacific/Apia"),+ Time32(Second),+ Time32(Millisecond),+ Time64(Microsecond),+ Time64(Nanosecond),+ Duration(Second),+ Duration(Nanosecond),+ ];+ let mut violations = vec![];+ for source in &types {+ let Some(samples) = order_samples(source) else {+ continue;+ };+ for target in &types {+ if !can_cast_types(source, target)+ || !is_order_preserving_cast(source, target)+ {+ continue;+ }+ for safe in [false, true] {+ if let Some(e) = order_violation(&samples, target, safe) {+ violations+ .push(format!("{source} -> {target} (safe={safe}): {e}"));+ }+ }+ }+ }+ assert!(violations.is_empty(), "{}", violations.join("\n"));++ // The samples are able to expose the casts the rule rejects.+ for (source, target) in [+ (Utf8, Int32),+ (Int32, Boolean),+ (Int16, Binary),+ (Timestamp(Second, None), Time32(Second)),+ (tz(Second, "+05:00"), Time64(Nanosecond)),+ (tz(Second, "America/Goose_Bay"), Date32),+ (tz(Second, "America/Goose_Bay"), Date64),+ (Timestamp(Second, None), tz(Second, "America/New_York")),+ ] {+ assert!(!is_order_preserving_cast(&source, &target));+ let samples = order_samples(&source).unwrap();+ assert!(+ order_violation(&samples, &target, false).is_some(),+ "{source} -> {target}: no sample exposes the reversed order"+ );+ }++ // The samples reach the edge of the Date32 entry: a skipped day starts+ // at the same instant as the day after it.+ let skipped =+ ColumnarValue::Array(Arc::new(Date32Array::from(vec![15_338, 15_339])))+ .cast_to(&tz(Second, "Pacific/Apia"), None)+ .unwrap()+ .into_array(2)+ .unwrap();+ let skipped = skipped.as_primitive::<TimestampSecondType>();+ assert_eq!(skipped.value(0), skipped.value(1));+ }+#[test]fn test_cast_ordering_with_null_on_failure() {use arrow::array::TimestampSecondArray;diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rsindex 767c0b4ce4..bb5c818ece 100644--- a/datafusion/physical-expr/src/expressions/mod.rs+++ b/datafusion/physical-expr/src/expressions/mod.rs@@ -30,7 +30,7 @@ pub use crate::aggregate::stats::StatsType;pub use binary::{BinaryExpr, binary, similar_to};pub use case::{CaseExpr, case};-pub use cast::{CastExpr, cast};+pub use cast::{CastExpr, cast, is_order_preserving_cast};pub use column::{Column, col, with_new_schema};pub use datafusion_expr::utils::format_state_name;pub use dynamic_filters::{diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rsindex 2e3c84673f..b7325d90d5 100644--- a/datafusion/pruning/src/pruning_predicate.rs+++ b/datafusion/pruning/src/pruning_predicate.rs@@ -1331,10 +1331,10 @@ fn is_compare_op(op: Operator) -> bool {)}-// The pruning logic is based on the comparing the min/max bounds.-// Must make sure the two type has order.-// For example, casts from string to numbers is not correct.-// Because the "13" is less than "3" with UTF8 comparison order.+// Pruning casts the container's min/max statistics and compares them with the+// literal, which is only sound when the cast keeps the order of the values,+// e.g. not for strings to numbers ("13" < "3") or timestamps to time of day+// (wraps at midnight). Ask the same rule that sort elimination uses.fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Result<()> {// Dictionary casts are always supported as long as the value types are supportedlet from_type = match from_type {@@ -1349,20 +1349,7 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re}_ => to_type,};- // Casting a timestamp to a time-of-day wraps at midnight and does not- // preserve ordering across day boundaries. Casting timestamp min/max- // statistics could therefore produce invalid bounds and prune matches.- if matches!(from_type, DataType::Timestamp(_, _))- && matches!(to_type, DataType::Time32(_) | DataType::Time64(_))- {- return plan_err!(- "Try Cast/Cast with from type {from_type} to type {to_type} is not supported"- );- }- // If both types are strings or both are not strings (number, timestamp, etc)- // then we can compare them.- // PruningPredicate does not support casting of strings to numbers and such.- if from_type.is_string() == to_type.is_string() {+ if phys_expr::is_order_preserving_cast(from_type, to_type) {Ok(())} else {plan_err!(@@ -5095,6 +5082,99 @@ mod tests {Ok(())}+ /// Pruning casts the min/max statistics, so it may only use casts that+ /// keep the order of the values.+ #[test]+ fn row_group_predicate_cast_must_preserve_order() {+ use TimeUnit::*;++ let new_york = Some(Arc::from("America/New_York"));+ let dictionary = |value: DataType| {+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(value))+ };+ let cases = [+ // 0 is false while -1 and 1 are true+ (DataType::Int32, ScalarValue::Boolean(Some(false)), false),+ (DataType::Float64, ScalarValue::Boolean(Some(false)), false),+ (+ dictionary(DataType::Int64),+ ScalarValue::Boolean(Some(false)),+ false,+ ),+ // little-endian bytes: 256 is [0, 1, ..] and 255 is [255, 0, ..]+ (DataType::Int64, ScalarValue::Binary(Some(vec![5])), false),+ // a named timezone can move the local date backwards+ (+ DataType::Timestamp(Nanosecond, new_york.clone()),+ ScalarValue::Date32(Some(0)),+ false,+ ),+ (+ DataType::Timestamp(Nanosecond, new_york.clone()),+ ScalarValue::Date64(Some(0)),+ false,+ ),+ // a local time that a transition skips moves past later ones+ (+ DataType::Timestamp(Nanosecond, None),+ ScalarValue::TimestampNanosecond(Some(0), new_york.clone()),+ false,+ ),+ (+ DataType::Int64,+ ScalarValue::TimestampNanosecond(Some(0), None),+ true,+ ),+ (DataType::Int32, ScalarValue::Date32(Some(0)), true),+ // Parquet TIME(MICROS) read as a TIME column+ (+ DataType::Time64(Microsecond),+ ScalarValue::Time64Nanosecond(Some(0)),+ true,+ ),+ (+ DataType::Utf8,+ ScalarValue::Utf8View(Some("a".into())),+ true,+ ),+ (dictionary(DataType::Utf8), ScalarValue::from("a"), true),+ (+ DataType::FixedSizeBinary(1),+ ScalarValue::Binary(Some(vec![5])),+ true,+ ),+ (+ DataType::Timestamp(Nanosecond, None),+ ScalarValue::TimestampNanosecond(Some(0), Some(Arc::from("-05:00"))),+ true,+ ),+ (+ DataType::Date32,+ ScalarValue::TimestampSecond(Some(0), new_york),+ true,+ ),+ ];+ for (source, literal, prunable) in cases {+ let target = literal.data_type();+ let schema = Schema::new(vec![Field::new("c1", source.clone(), false)]);+ for expr in [+ cast(col("c1"), target.clone()).eq(lit(literal.clone())),+ try_cast(col("c1"), target.clone()).eq(lit(literal.clone())),+ ] {+ let predicate_expr = test_build_predicate_expression(+ &expr,+ &schema,+ &mut RequiredColumns::new(),+ );+ assert_eq!(+ predicate_expr.to_string() != "true",+ prunable,+ "{source} -> {target}: {predicate_expr}"+ );+ }+ }+ }+#[test]fn row_group_predicate_date_date() -> Result<()> {let schema = Schema::new(vec![Field::new("c1", DataType::Date32, false)]);diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.sltindex 3d11899c93..6dedfa39c8 100644--- a/datafusion/sqllogictest/test_files/cast.slt+++ b/datafusion/sqllogictest/test_files/cast.slt@@ -142,6 +142,19 @@ ORDER BY d;562129260 1987-10-24562129259 1987-10-25+# Date64 also takes the local date, so it reverses the same way.+query ID+SELECT arrow_cast(ts, 'Int64') AS epoch, arrow_cast(ts, 'Date64') AS d+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second, Some("America/Goose_Bay"))') AS ts+ FROM (VALUES (562129259::bigint), (562129260::bigint))+ ORDER BY ts LIMIT 2+)+ORDER BY d;+----+562129260 1987-10-24T00:00:00+562129259 1987-10-25T00:00:00+# Reducing timestamp precision remains order-preserving, even when values merge.statement okSET datafusion.explain.physical_plan_only = true;@@ -164,6 +177,35 @@ physical_plan02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]03)----DataSourceExec: partitions=1, partition_sizes=[1]+# An integer read as a count of seconds keeps its order, so the outer sort is+# removed.+query TT+EXPLAIN SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts+FROM (+ SELECT column1 AS i+ FROM (VALUES (3::bigint), (1::bigint), (2::bigint))+ ORDER BY i LIMIT 3+)+ORDER BY ts;+----+physical_plan+01)ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]+02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[1]++query P+SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts+FROM (+ SELECT column1 AS i+ FROM (VALUES (3::bigint), (1::bigint), (2::bigint))+ ORDER BY i LIMIT 3+)+ORDER BY ts;+----+1970-01-01T00:00:01+1970-01-01T00:00:02+1970-01-01T00:00:03+statement okSET datafusion.explain.physical_plan_only = false;diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex ea61f0be6c..3e5a35e0fe 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -1602,3 +1602,117 @@ RESET datafusion.execution.parquet.pushdown_filters;statement okRESET datafusion.explain.physical_plan_only;++##########+# Pruning casts the min/max statistics of each row group, so it may only use a+# cast that keeps the order of the values. Each predicate below runs against+# an in-memory table, which is never pruned, and the same rows in one Parquet+# row group. The two must agree.+##########++statement ok+CREATE TABLE cast_order_src(i INT, ts TIMESTAMP) AS VALUES+ (-1, TIMESTAMP '2024-03-10 00:00:00'),+ (0, TIMESTAMP '2024-03-10 01:00:00'),+ (1, TIMESTAMP '2024-03-10 02:30:00'),+ (255, TIMESTAMP '2024-03-10 02:45:00'),+ (256, TIMESTAMP '2024-03-10 03:00:00');++statement ok+COPY cast_order_src+TO 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE cast_order+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet';++# -1 and 1 are true and 0 is false, so the bounds -1 and 256 both cast to true.+query I+SELECT i FROM cast_order_src WHERE CAST(i AS BOOLEAN) <= false;+----+0++query I+SELECT i FROM cast_order WHERE CAST(i AS BOOLEAN) <= false;+----+0++# An integer casts to its little-endian bytes: -1 is X'FFFFFFFF' and 256 is+# X'00010000', so the bounds no longer bracket the values in between.+query I+SELECT i FROM cast_order_src WHERE CAST(i AS BYTEA) < X'05' ORDER BY i;+----+0+1+256++query I+SELECT i FROM cast_order WHERE CAST(i AS BYTEA) < X'05' ORDER BY i;+----+0+1+256++# Comparing a timestamp without a timezone to one with a timezone casts the+# column to that timezone, reading the values as local times. New York skipped+# 02:00 to 03:00 on 2024-03-10, and Arrow reads 02:30 as 03:30 (07:30 UTC),+# after 03:00 (07:00 UTC). The bound 03:00 is no longer the largest value after+# the cast, so this cast must not prune.+statement ok+SET datafusion.execution.time_zone = 'America/New_York';++query I+SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;+----+1+255++query I+SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;+----+1+255++# A fixed offset skips no local times, so that cast keeps the order and still+# prunes.+statement ok+SET datafusion.execution.time_zone = '-05:00';++statement ok+SET datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00';+----+physical_plan+01)FilterExec: CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, projection=[i@0]+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/cast_order/data.parquet]]}, projection=[i, ts], file_type=parquet, predicate=CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, pruning_predicate=ts_null_count@1 != row_count@2 AND CAST(ts_max@0 AS Timestamp(ns, "-05:00");N) > 1710054900000000000, required_guarantees=[]++query I+SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;+----+1+255+256++query I+SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i;+----+1+255+256++statement ok+RESET datafusion.explain.physical_plan_only;++statement ok+RESET datafusion.execution.time_zone;++statement ok+DROP TABLE cast_order;++statement ok+DROP TABLE cast_order_src;