Apache DataFusion SQL Query Engine
fix: preserve date_trunc sorting for timezone-aware timestamps (#25409)
Malingshuefd36e7b66a1
diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rsindex e61637c093..2c677213a4 100644--- a/datafusion/functions/src/datetime/date_trunc.rs+++ b/datafusion/functions/src/datetime/date_trunc.rs@@ -431,11 +431,15 @@ impl ScalarUDFImpl for DateTruncFunc {}fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {- // The DATE_TRUNC function preserves the order of its second argument.let precision = &input[0];let date_value = &input[1];- if precision.sort_properties.eq(&SortProperties::Singleton) {+ let order_safe_input = matches!(+ date_value.range.data_type(),+ Timestamp(_, None) | Time32(_) | Time64(_)+ );++ if precision.sort_properties == SortProperties::Singleton && order_safe_input {Ok(date_value.sort_properties)} else {Ok(SortProperties::Unordered)@@ -889,13 +893,55 @@ mod tests {TimestampNanosecondArray, TimestampSecondArray,};use arrow::buffer::NullBuffer;- use arrow::compute::DatePart;use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos;+ use arrow::compute::{DatePart, SortOptions};use arrow::datatypes::{DataType, Field, TimeUnit};use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;+ use datafusion_expr::interval_arithmetic::Interval;+ use datafusion_expr::sort_properties::{ExprProperties, SortProperties};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};+ #[test]+ fn output_ordering_respects_timestamp_timezone() {+ let precision_value = ScalarValue::Utf8(Some("hour".into()));+ let precision = ExprProperties::new_unknown()+ .with_order(SortProperties::Singleton)+ .with_range(+ Interval::try_new(precision_value.clone(), precision_value).unwrap(),+ );+ let ordered = SortProperties::Ordered(SortOptions::default());+ let date_value = |data_type| {+ ExprProperties::new_unknown()+ .with_order(ordered)+ .with_range(Interval::make_unbounded(&data_type).unwrap())+ };+ let function = DateTruncFunc::new();++ let timestamp = date_value(DataType::Timestamp(TimeUnit::Second, None));+ assert_eq!(+ function+ .output_ordering(&[precision.clone(), timestamp])+ .unwrap(),+ ordered+ );+ let timestamp_with_timezone = date_value(DataType::Timestamp(+ TimeUnit::Second,+ Some("America/Goose_Bay".into()),+ ));+ assert_eq!(+ function+ .output_ordering(&[precision.clone(), timestamp_with_timezone])+ .unwrap(),+ SortProperties::Unordered+ );+ let unknown = ExprProperties::new_unknown().with_order(ordered);+ assert_eq!(+ function.output_ordering(&[precision, unknown]).unwrap(),+ SortProperties::Unordered+ );+ }+#[test]fn date_trunc_test() {let cases = vec![diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex fa4159c1dc..05622c22da 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -1655,7 +1655,53 @@ SELECT DATE_TRUNC('day', arrow_cast(TIMESTAMP '2023-08-03 14:38:50Z', 'Timestamp2023-08-03T14:38:502023-08-03T14:38:50-# date_trunc with data with timezones+# date_trunc must not propagate ordering across named timezone transitions+query II+SELECT+ arrow_cast(ts, 'Int64') AS input_epoch,+ arrow_cast(date_trunc('hour', ts), 'Int64') AS truncated+FROM (+ SELECT+ arrow_cast(+ column1,+ 'Timestamp(Second, Some("America/Goose_Bay"))'+ ) AS ts+ FROM (+ VALUES+ (562129260::bigint),+ (562129259::bigint)+ )+ ORDER BY ts+ LIMIT 2+)+ORDER BY date_trunc('hour', ts);+----+562129260 562125600+562129259 562129200++# An unknown range from a child function may hide a timezone-aware timestamp+query II+SELECT+ c,+ arrow_cast(+ date_trunc('hour', from_unixtime(c, 'America/Goose_Bay')),+ 'Int64'+ ) AS truncated+FROM (+ SELECT column1 AS c+ FROM (+ VALUES+ (562129260::bigint),+ (562129259::bigint)+ )+ ORDER BY c+ LIMIT 2+)+ORDER BY date_trunc('hour', from_unixtime(c, 'America/Goose_Bay'));+----+562129260 562125600+562129259 562129200+statement okCREATE TABLE timestamp_strings(ts varchar)AS VALUES