Skip to content

Apache DataFusion SQL Query Engine

fix: avoid pruning timestamp-to-time casts (#26452)

Rishabh Gupta6110f74f5f4c
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index d24424f444..b002d1023f 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -7,8 +7,8 @@
//! expected.
use std::sync::Arc;
-use arrow::array::{ArrayRef, Int32Array, RecordBatch};
-use arrow_schema::{DataType, Field, Schema};
+use arrow::array::{ArrayRef, Int32Array, RecordBatch, TimestampSecondArray};
+use arrow_schema::{DataType, Field, Schema, TimeUnit};
use datafusion::prelude::SessionConfig;
use datafusion_common::{DataFusionError, ScalarValue};
use itertools::Itertools;
@@ -293,6 +293,51 @@ async fn prune_timestamps_seconds() {
.await;
}
+#[tokio::test]
+async fn timestamp_to_time_cast_across_midnight_is_not_pruned() {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "ts",
+ DataType::Timestamp(TimeUnit::Second, None),
+ false,
+ )]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(TimestampSecondArray::from(vec![
+ 82_800, 86_400, 90_000,
+ ]))],
+ )
+ .unwrap();
+
+ for (time_type, op, literal, expected_rows) in [
+ ("Time32(Second)", "=", "00:00:00", 1),
+ ("Time32(Second)", "<", "02:00:00", 2),
+ ("Time64(Nanosecond)", "=", "00:00:00", 1),
+ ("Time64(Nanosecond)", "<", "02:00:00", 2),
+ ] {
+ let query = format!(
+ "SELECT ts FROM t WHERE arrow_cast(ts, '{time_type}') {op} \
+ arrow_cast('{literal}', '{time_type}')"
+ );
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Timestamps)
+ .with_query(&query)
+ .with_expected_errors(Some(0))
+ .with_matched_by_stats(Some(1))
+ .with_fully_matched_by_stats(Some(0))
+ .with_pruned_by_stats(Some(0))
+ .with_pruned_files(Some(0))
+ .with_limit_pruned_row_groups(Some(0))
+ .with_expected_rows(expected_rows)
+ .test_row_group_prune_with_custom_data(
+ Arc::clone(&schema),
+ vec![batch.clone()],
+ 3,
+ )
+ .await;
+ }
+}
+
#[tokio::test]
async fn prune_date32() {
RowGroupPruningTest::new()
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index c8508ed3b5..aa48b75ae7 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -1349,6 +1349,16 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re
}
_ => to_type,
};
+ // Casting a timestamp to a time-of-day wraps at midnight and does not
+ // preserve ordering across day boundaries. Casting timestamp min/max
+ // statistics could therefore produce invalid bounds and prune matches.
+ if matches!(from_type, DataType::Timestamp(_, _))
+ && matches!(to_type, DataType::Time32(_) | DataType::Time64(_))
+ {
+ return plan_err!(
+ "Try Cast/Cast with from type {from_type} to type {to_type} is not supported"
+ );
+ }
// If both types are strings or both are not strings (number, timestamp, etc)
// then we can compare them.
// PruningPredicate does not support casting of strings to numbers and such.
@@ -4971,6 +4981,56 @@ mod tests {
Ok(())
}
+ #[test]
+ fn row_group_predicate_cast_timestamp_to_time() {
+ // A timezone shifts where the day boundary falls but does not remove
+ // it, so a timezone-aware timestamp is just as unsafe to prune on.
+ for timestamp_type in [
+ DataType::Timestamp(TimeUnit::Second, None),
+ DataType::Timestamp(TimeUnit::Second, Some("+05:00".into())),
+ DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
+ ] {
+ assert_timestamp_to_time_cast_is_not_prunable(timestamp_type);
+ }
+ }
+
+ /// Asserts that no comparison of `CAST`/`TRY_CAST` from `timestamp_type` to
+ /// a time-of-day produces a pruning expression.
+ fn assert_timestamp_to_time_cast_is_not_prunable(timestamp_type: DataType) {
+ let schema = Schema::new(vec![Field::new("ts", timestamp_type, false)]);
+
+ for (time_type, literal) in [
+ (
+ DataType::Time32(TimeUnit::Second),
+ ScalarValue::Time32Second(Some(0)),
+ ),
+ (
+ DataType::Time32(TimeUnit::Millisecond),
+ ScalarValue::Time32Millisecond(Some(0)),
+ ),
+ (
+ DataType::Time64(TimeUnit::Microsecond),
+ ScalarValue::Time64Microsecond(Some(0)),
+ ),
+ (
+ DataType::Time64(TimeUnit::Nanosecond),
+ ScalarValue::Time64Nanosecond(Some(0)),
+ ),
+ ] {
+ for expr in [
+ cast(col("ts"), time_type.clone()).eq(lit(literal.clone())),
+ try_cast(col("ts"), time_type).eq(lit(literal)),
+ ] {
+ let predicate_expr = test_build_predicate_expression(
+ &expr,
+ &schema,
+ &mut RequiredColumns::new(),
+ );
+ assert_eq!(predicate_expr.to_string(), "true");
+ }
+ }
+ }
+
#[test]
fn row_group_predicate_cast_string_string() -> Result<()> {
let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]);
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index bc9fa4f5b9..ea61f0be6c 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -1333,6 +1333,168 @@ RESET datafusion.execution.parquet.pushdown_filters;
statement ok
drop table t;
+##########
+# Casting a timestamp to a time-of-day wraps at midnight
+#
+# https://github.com/apache/datafusion/issues/26451
+##########
+
+# Pruning must never change a query's answer. A cast whose output does not
+# preserve the ordering of its input can invert the bounds derived from
+# statistics, which makes pruning discard row groups that do contain matching
+# rows. Every predicate below is therefore run twice: once against an
+# in-memory table, which is never pruned and so gives the ground truth, and
+# once against the same rows stored as Parquet. The two must agree.
+
+# Three timestamps at 23:00, 00:00 and 01:00, written as a single row group so
+# that the row group's min/max bracket midnight. Casting those bounds to
+# time-of-day turns them into 23:00:00 and 01:00:00, which excludes every value
+# that actually lies between them.
+statement ok
+CREATE TABLE source_data(ts TIMESTAMP) AS VALUES
+ (TIMESTAMP '2024-03-10 23:00:00'),
+ (TIMESTAMP '2024-03-11 00:00:00'),
+ (TIMESTAMP '2024-03-11 01:00:00');
+
+statement ok
+COPY (SELECT ts FROM source_data)
+TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet'
+STORED AS PARQUET
+OPTIONS ('format.max_row_group_size' '3');
+
+statement ok
+CREATE EXTERNAL TABLE timestamps
+STORED AS PARQUET
+LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet';
+
+# Equality against the row at midnight.
+query P
+SELECT ts FROM source_data WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00
+
+query P
+SELECT ts FROM timestamps WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00
+
+# Range predicate spanning the wrap point.
+query P
+SELECT ts FROM source_data WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00
+2024-03-11T01:00:00
+
+query P
+SELECT ts FROM timestamps WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00
+2024-03-11T01:00:00
+
+# The same predicates spelled with arrow_cast, which also reaches Time32.
+query P
+SELECT ts FROM source_data
+WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)')
+ORDER BY ts;
+----
+2024-03-11T00:00:00
+
+query P
+SELECT ts FROM timestamps
+WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)')
+ORDER BY ts;
+----
+2024-03-11T00:00:00
+
+query P
+SELECT ts FROM source_data
+WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)')
+ORDER BY ts;
+----
+2024-03-11T00:00:00
+2024-03-11T01:00:00
+
+query P
+SELECT ts FROM timestamps
+WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)')
+ORDER BY ts;
+----
+2024-03-11T00:00:00
+2024-03-11T01:00:00
+
+statement ok
+DROP TABLE timestamps;
+
+##########
+# The same wrap happens at the local midnight of a timezone-aware column
+##########
+
+# The same three local wall-clock times, tagged +05:00, so the instants
+# actually stored are 18:00, 19:00 and 20:00 UTC on 2024-03-10. Those are
+# monotonic and fall inside a single UTC day, so the min/max bounds only
+# invert if the cast honours the column's timezone.
+statement ok
+CREATE TABLE source_data_tz AS
+SELECT arrow_cast(ts, 'Timestamp(Nanosecond, Some("+05:00"))') AS ts
+FROM source_data;
+
+statement ok
+DROP TABLE source_data;
+
+statement ok
+COPY (SELECT ts FROM source_data_tz)
+TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet'
+STORED AS PARQUET
+OPTIONS ('format.max_row_group_size' '3');
+
+statement ok
+CREATE EXTERNAL TABLE timestamps_tz
+STORED AS PARQUET
+LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet';
+
+# Pin the shape of the data this section depends on: the local times have to
+# straddle a local midnight while the stored instants stay monotonic within a
+# single UTC day. Without this, a change to how a timezone is attached would
+# leave the queries below passing while no longer exercising the wrap at all.
+query TIT
+SELECT
+ ts::string AS local_time,
+ to_unixtime(ts) AS stored_instant,
+ CAST(ts AS TIME)::string AS time_of_day
+FROM timestamps_tz ORDER BY ts;
+----
+2024-03-10T23:00:00+05:00 1710093600 23:00:00
+2024-03-11T00:00:00+05:00 1710097200 00:00:00
+2024-03-11T01:00:00+05:00 1710100800 01:00:00
+
+query P
+SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00+05:00
+
+query P
+SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00+05:00
+
+query P
+SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00+05:00
+2024-03-11T01:00:00+05:00
+
+query P
+SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts;
+----
+2024-03-11T00:00:00+05:00
+2024-03-11T01:00:00+05:00
+
+statement ok
+DROP TABLE timestamps_tz;
+
+statement ok
+DROP TABLE source_data_tz;
+
# A predicate that the Parquet scan keeps only for pruning
# (pushdown_filters = false) must not make the scan claim equivalences.
# Each file is sorted by (a, b). If the scan claims that `a` is constant,