Skip to content

Apache DataFusion SQL Query Engine

fix: avoid invalid date_bin ordering after overflow (#25692)

Liang-Chi Hsiehf50c17dfb089
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rs
index 909a7dbb75..2696916b06 100644
--- a/datafusion/functions/src/datetime/date_bin.rs
+++ b/datafusion/functions/src/datetime/date_bin.rs
@@ -24,7 +24,7 @@ use arrow::array::types::{
TimestampSecondType,
};
use arrow::array::{ArrayRef, AsArray, PrimitiveArray};
-use arrow::datatypes::DataType::{Time32, Time64, Timestamp};
+use arrow::datatypes::DataType::{Null, Time32, Time64, Timestamp};
use arrow::datatypes::IntervalUnit::{DayTime, MonthDayNano};
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
@@ -269,14 +269,23 @@ impl ScalarUDFImpl for DateBinFunc {
}
fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
- // The DATE_BIN function preserves the order of its second argument.
let step = &input[0];
let date_value = &input[1];
let reference = input.get(2);
- if step.sort_properties.eq(&SortProperties::Singleton)
+ // Scaling these representations to nanoseconds can overflow and turn
+ // otherwise valid input rows into NULL. Unknown ranges use the Null
+ // type and can hide one of these representations. The generated NULLs
+ // need not have the same placement as the source ordering.
+ let scale_can_overflow = matches!(
+ date_value.range.data_type(),
+ Null | Timestamp(Second | Millisecond | Microsecond, _) | Time64(Microsecond)
+ );
+
+ if !scale_can_overflow
+ && step.sort_properties == SortProperties::Singleton
&& reference
- .map(|r| r.sort_properties.eq(&SortProperties::Singleton))
+ .map(|r| r.sort_properties == SortProperties::Singleton)
.unwrap_or(true)
{
Ok(date_value.sort_properties)
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index 951c50e706..c2be17579a 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -1728,7 +1728,7 @@ physical_plan
02)--SortExec: TopK(fetch=2), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
-# Typed bounds allow date_trunc to propagate ordering through timezone-free date_bin
+# date_bin remains conservatively unordered because per-row failures become NULL
query TT
EXPLAIN SELECT date_trunc('hour', date_bin(INTERVAL '1 minute', ts)) AS truncated
FROM (
@@ -1750,10 +1750,51 @@ logical_plan
04)------Projection: CAST(column1 AS Timestamp(s)) AS ts
05)--------Values: (Int64(7200)), (Int64(3600))
physical_plan
-01)ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]
-02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
-03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
-04)------DataSourceExec: partitions=1, partition_sizes=[1]
+01)SortExec: expr=[truncated@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]
+03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
+04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# date_bin can return NULL for out-of-range values, so its output does not
+# necessarily preserve the source's NULL ordering.
+query P
+SELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (-10000000000::bigint),
+ (0::bigint),
+ (10000000000::bigint)
+ )
+ ORDER BY ts
+ LIMIT 3
+)
+ORDER BY b ASC NULLS LAST
+----
+1970-01-01T00:00:00
+NULL
+NULL
+
+# An ordered expression can retain its ordering while its range type is unknown.
+# date_bin must not assume that such an input is safe to scale to nanoseconds.
+query P
+SELECT date_bin(INTERVAL '1 second', date_trunc('second', ts)) AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (-10000000000::bigint),
+ (0::bigint),
+ (10000000000::bigint)
+ )
+ ORDER BY ts
+ LIMIT 3
+)
+ORDER BY b ASC NULLS LAST
+----
+1970-01-01T00:00:00
+NULL
+NULL
statement ok
CREATE TABLE timestamp_strings(ts varchar)