Skip to content

Apache DataFusion SQL Query Engine

fix: preserve date_trunc sorting for timezone-aware timestamps (#25409)

Malingshuefd36e7b66a1
diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs
index e61637c093..2c677213a4 100644
--- a/datafusion/functions/src/datetime/date_trunc.rs
+++ b/datafusion/functions/src/datetime/date_trunc.rs
@@ -431,11 +431,15 @@ impl ScalarUDFImpl for DateTruncFunc {
}
fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
- // The DATE_TRUNC function preserves the order of its second argument.
let precision = &input[0];
let date_value = &input[1];
- if precision.sort_properties.eq(&SortProperties::Singleton) {
+ let order_safe_input = matches!(
+ date_value.range.data_type(),
+ Timestamp(_, None) | Time32(_) | Time64(_)
+ );
+
+ if precision.sort_properties == SortProperties::Singleton && order_safe_input {
Ok(date_value.sort_properties)
} else {
Ok(SortProperties::Unordered)
@@ -889,13 +893,55 @@ mod tests {
TimestampNanosecondArray, TimestampSecondArray,
};
use arrow::buffer::NullBuffer;
- use arrow::compute::DatePart;
use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos;
+ use arrow::compute::{DatePart, SortOptions};
use arrow::datatypes::{DataType, Field, TimeUnit};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
+ use datafusion_expr::interval_arithmetic::Interval;
+ use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+ #[test]
+ fn output_ordering_respects_timestamp_timezone() {
+ let precision_value = ScalarValue::Utf8(Some("hour".into()));
+ let precision = ExprProperties::new_unknown()
+ .with_order(SortProperties::Singleton)
+ .with_range(
+ Interval::try_new(precision_value.clone(), precision_value).unwrap(),
+ );
+ let ordered = SortProperties::Ordered(SortOptions::default());
+ let date_value = |data_type| {
+ ExprProperties::new_unknown()
+ .with_order(ordered)
+ .with_range(Interval::make_unbounded(&data_type).unwrap())
+ };
+ let function = DateTruncFunc::new();
+
+ let timestamp = date_value(DataType::Timestamp(TimeUnit::Second, None));
+ assert_eq!(
+ function
+ .output_ordering(&[precision.clone(), timestamp])
+ .unwrap(),
+ ordered
+ );
+ let timestamp_with_timezone = date_value(DataType::Timestamp(
+ TimeUnit::Second,
+ Some("America/Goose_Bay".into()),
+ ));
+ assert_eq!(
+ function
+ .output_ordering(&[precision.clone(), timestamp_with_timezone])
+ .unwrap(),
+ SortProperties::Unordered
+ );
+ let unknown = ExprProperties::new_unknown().with_order(ordered);
+ assert_eq!(
+ function.output_ordering(&[precision, unknown]).unwrap(),
+ SortProperties::Unordered
+ );
+ }
+
#[test]
fn date_trunc_test() {
let cases = vec![
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index fa4159c1dc..05622c22da 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -1655,7 +1655,53 @@ SELECT DATE_TRUNC('day', arrow_cast(TIMESTAMP '2023-08-03 14:38:50Z', 'Timestamp
2023-08-03T14:38:50
2023-08-03T14:38:50
-# date_trunc with data with timezones
+# date_trunc must not propagate ordering across named timezone transitions
+query II
+SELECT
+ arrow_cast(ts, 'Int64') AS input_epoch,
+ arrow_cast(date_trunc('hour', ts), 'Int64') AS truncated
+FROM (
+ SELECT
+ arrow_cast(
+ column1,
+ 'Timestamp(Second, Some("America/Goose_Bay"))'
+ ) AS ts
+ FROM (
+ VALUES
+ (562129260::bigint),
+ (562129259::bigint)
+ )
+ ORDER BY ts
+ LIMIT 2
+)
+ORDER BY date_trunc('hour', ts);
+----
+562129260 562125600
+562129259 562129200
+
+# An unknown range from a child function may hide a timezone-aware timestamp
+query II
+SELECT
+ c,
+ arrow_cast(
+ date_trunc('hour', from_unixtime(c, 'America/Goose_Bay')),
+ 'Int64'
+ ) AS truncated
+FROM (
+ SELECT column1 AS c
+ FROM (
+ VALUES
+ (562129260::bigint),
+ (562129259::bigint)
+ )
+ ORDER BY c
+ LIMIT 2
+)
+ORDER BY date_trunc('hour', from_unixtime(c, 'America/Goose_Bay'));
+----
+562129260 562125600
+562129259 562129200
+
statement ok
CREATE TABLE timestamp_strings(ts varchar)
AS VALUES