Apache DataFusion SQL Query Engine
fix: avoid pruning timestamp-to-time casts (#26452)
Rishabh Gupta6110f74f5f4c
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rsindex d24424f444..b002d1023f 100644--- a/datafusion/core/tests/parquet/row_group_pruning.rs+++ b/datafusion/core/tests/parquet/row_group_pruning.rs@@ -7,8 +7,8 @@//! expected.use std::sync::Arc;-use arrow::array::{ArrayRef, Int32Array, RecordBatch};-use arrow_schema::{DataType, Field, Schema};+use arrow::array::{ArrayRef, Int32Array, RecordBatch, TimestampSecondArray};+use arrow_schema::{DataType, Field, Schema, TimeUnit};use datafusion::prelude::SessionConfig;use datafusion_common::{DataFusionError, ScalarValue};use itertools::Itertools;@@ -293,6 +293,51 @@ async fn prune_timestamps_seconds() {.await;}+#[tokio::test]+async fn timestamp_to_time_cast_across_midnight_is_not_pruned() {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "ts",+ DataType::Timestamp(TimeUnit::Second, None),+ false,+ )]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(TimestampSecondArray::from(vec![+ 82_800, 86_400, 90_000,+ ]))],+ )+ .unwrap();++ for (time_type, op, literal, expected_rows) in [+ ("Time32(Second)", "=", "00:00:00", 1),+ ("Time32(Second)", "<", "02:00:00", 2),+ ("Time64(Nanosecond)", "=", "00:00:00", 1),+ ("Time64(Nanosecond)", "<", "02:00:00", 2),+ ] {+ let query = format!(+ "SELECT ts FROM t WHERE arrow_cast(ts, '{time_type}') {op} \+ arrow_cast('{literal}', '{time_type}')"+ );++ RowGroupPruningTest::new()+ .with_scenario(Scenario::Timestamps)+ .with_query(&query)+ .with_expected_errors(Some(0))+ .with_matched_by_stats(Some(1))+ .with_fully_matched_by_stats(Some(0))+ .with_pruned_by_stats(Some(0))+ .with_pruned_files(Some(0))+ .with_limit_pruned_row_groups(Some(0))+ .with_expected_rows(expected_rows)+ .test_row_group_prune_with_custom_data(+ Arc::clone(&schema),+ vec![batch.clone()],+ 3,+ )+ .await;+ }+}+#[tokio::test]async fn prune_date32() {RowGroupPruningTest::new()diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rsindex c8508ed3b5..aa48b75ae7 100644--- a/datafusion/pruning/src/pruning_predicate.rs+++ b/datafusion/pruning/src/pruning_predicate.rs@@ -1349,6 +1349,16 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re}_ => to_type,};+ // Casting a timestamp to a time-of-day wraps at midnight and does not+ // preserve ordering across day boundaries. Casting timestamp min/max+ // statistics could therefore produce invalid bounds and prune matches.+ if matches!(from_type, DataType::Timestamp(_, _))+ && matches!(to_type, DataType::Time32(_) | DataType::Time64(_))+ {+ return plan_err!(+ "Try Cast/Cast with from type {from_type} to type {to_type} is not supported"+ );+ }// If both types are strings or both are not strings (number, timestamp, etc)// then we can compare them.// PruningPredicate does not support casting of strings to numbers and such.@@ -4971,6 +4981,56 @@ mod tests {Ok(())}+ #[test]+ fn row_group_predicate_cast_timestamp_to_time() {+ // A timezone shifts where the day boundary falls but does not remove+ // it, so a timezone-aware timestamp is just as unsafe to prune on.+ for timestamp_type in [+ DataType::Timestamp(TimeUnit::Second, None),+ DataType::Timestamp(TimeUnit::Second, Some("+05:00".into())),+ DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),+ ] {+ assert_timestamp_to_time_cast_is_not_prunable(timestamp_type);+ }+ }++ /// Asserts that no comparison of `CAST`/`TRY_CAST` from `timestamp_type` to+ /// a time-of-day produces a pruning expression.+ fn assert_timestamp_to_time_cast_is_not_prunable(timestamp_type: DataType) {+ let schema = Schema::new(vec![Field::new("ts", timestamp_type, false)]);++ for (time_type, literal) in [+ (+ DataType::Time32(TimeUnit::Second),+ ScalarValue::Time32Second(Some(0)),+ ),+ (+ DataType::Time32(TimeUnit::Millisecond),+ ScalarValue::Time32Millisecond(Some(0)),+ ),+ (+ DataType::Time64(TimeUnit::Microsecond),+ ScalarValue::Time64Microsecond(Some(0)),+ ),+ (+ DataType::Time64(TimeUnit::Nanosecond),+ ScalarValue::Time64Nanosecond(Some(0)),+ ),+ ] {+ for expr in [+ cast(col("ts"), time_type.clone()).eq(lit(literal.clone())),+ try_cast(col("ts"), time_type).eq(lit(literal)),+ ] {+ let predicate_expr = test_build_predicate_expression(+ &expr,+ &schema,+ &mut RequiredColumns::new(),+ );+ assert_eq!(predicate_expr.to_string(), "true");+ }+ }+ }+#[test]fn row_group_predicate_cast_string_string() -> Result<()> {let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]);diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex bc9fa4f5b9..ea61f0be6c 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -1333,6 +1333,168 @@ RESET datafusion.execution.parquet.pushdown_filters;statement okdrop table t;+##########+# Casting a timestamp to a time-of-day wraps at midnight+#+# https://github.com/apache/datafusion/issues/26451+##########++# Pruning must never change a query's answer. A cast whose output does not+# preserve the ordering of its input can invert the bounds derived from+# statistics, which makes pruning discard row groups that do contain matching+# rows. Every predicate below is therefore run twice: once against an+# in-memory table, which is never pruned and so gives the ground truth, and+# once against the same rows stored as Parquet. The two must agree.++# Three timestamps at 23:00, 00:00 and 01:00, written as a single row group so+# that the row group's min/max bracket midnight. Casting those bounds to+# time-of-day turns them into 23:00:00 and 01:00:00, which excludes every value+# that actually lies between them.+statement ok+CREATE TABLE source_data(ts TIMESTAMP) AS VALUES+ (TIMESTAMP '2024-03-10 23:00:00'),+ (TIMESTAMP '2024-03-11 00:00:00'),+ (TIMESTAMP '2024-03-11 01:00:00');++statement ok+COPY (SELECT ts FROM source_data)+TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '3');++statement ok+CREATE EXTERNAL TABLE timestamps+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet';++# Equality against the row at midnight.+query P+SELECT ts FROM source_data WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;+----+2024-03-11T00:00:00++query P+SELECT ts FROM timestamps WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;+----+2024-03-11T00:00:00++# Range predicate spanning the wrap point.+query P+SELECT ts FROM source_data WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;+----+2024-03-11T00:00:00+2024-03-11T01:00:00++query P+SELECT ts FROM timestamps WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;+----+2024-03-11T00:00:00+2024-03-11T01:00:00++# The same predicates spelled with arrow_cast, which also reaches Time32.+query P+SELECT ts FROM source_data+WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)')+ORDER BY ts;+----+2024-03-11T00:00:00++query P+SELECT ts FROM timestamps+WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)')+ORDER BY ts;+----+2024-03-11T00:00:00++query P+SELECT ts FROM source_data+WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)')+ORDER BY ts;+----+2024-03-11T00:00:00+2024-03-11T01:00:00++query P+SELECT ts FROM timestamps+WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)')+ORDER BY ts;+----+2024-03-11T00:00:00+2024-03-11T01:00:00++statement ok+DROP TABLE timestamps;++##########+# The same wrap happens at the local midnight of a timezone-aware column+##########++# The same three local wall-clock times, tagged +05:00, so the instants+# actually stored are 18:00, 19:00 and 20:00 UTC on 2024-03-10. Those are+# monotonic and fall inside a single UTC day, so the min/max bounds only+# invert if the cast honours the column's timezone.+statement ok+CREATE TABLE source_data_tz AS+SELECT arrow_cast(ts, 'Timestamp(Nanosecond, Some("+05:00"))') AS ts+FROM source_data;++statement ok+DROP TABLE source_data;++statement ok+COPY (SELECT ts FROM source_data_tz)+TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '3');++statement ok+CREATE EXTERNAL TABLE timestamps_tz+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet';++# Pin the shape of the data this section depends on: the local times have to+# straddle a local midnight while the stored instants stay monotonic within a+# single UTC day. Without this, a change to how a timezone is attached would+# leave the queries below passing while no longer exercising the wrap at all.+query TIT+SELECT+ ts::string AS local_time,+ to_unixtime(ts) AS stored_instant,+ CAST(ts AS TIME)::string AS time_of_day+FROM timestamps_tz ORDER BY ts;+----+2024-03-10T23:00:00+05:00 1710093600 23:00:00+2024-03-11T00:00:00+05:00 1710097200 00:00:00+2024-03-11T01:00:00+05:00 1710100800 01:00:00++query P+SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;+----+2024-03-11T00:00:00+05:00++query P+SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;+----+2024-03-11T00:00:00+05:00++query P+SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;+----+2024-03-11T00:00:00+05:00+2024-03-11T01:00:00+05:00++query P+SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;+----+2024-03-11T00:00:00+05:00+2024-03-11T01:00:00+05:00++statement ok+DROP TABLE timestamps_tz;++statement ok+DROP TABLE source_data_tz;+# A predicate that the Parquet scan keeps only for pruning# (pushdown_filters = false) must not make the scan claim equivalences.# Each file is sorted by (a, b). If the scan claims that `a` is constant,