Apache DataFusion SQL Query Engine
fix: avoid invalid date_bin ordering after overflow (#25692)
Liang-Chi Hsiehf50c17dfb089
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rsindex 909a7dbb75..2696916b06 100644--- a/datafusion/functions/src/datetime/date_bin.rs+++ b/datafusion/functions/src/datetime/date_bin.rs@@ -24,7 +24,7 @@ use arrow::array::types::{TimestampSecondType,};use arrow::array::{ArrayRef, AsArray, PrimitiveArray};-use arrow::datatypes::DataType::{Time32, Time64, Timestamp};+use arrow::datatypes::DataType::{Null, Time32, Time64, Timestamp};use arrow::datatypes::IntervalUnit::{DayTime, MonthDayNano};use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};use arrow::datatypes::{@@ -269,14 +269,23 @@ impl ScalarUDFImpl for DateBinFunc {}fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {- // The DATE_BIN function preserves the order of its second argument.let step = &input[0];let date_value = &input[1];let reference = input.get(2);- if step.sort_properties.eq(&SortProperties::Singleton)+ // Scaling these representations to nanoseconds can overflow and turn+ // otherwise valid input rows into NULL. Unknown ranges use the Null+ // type and can hide one of these representations. The generated NULLs+ // need not have the same placement as the source ordering.+ let scale_can_overflow = matches!(+ date_value.range.data_type(),+ Null | Timestamp(Second | Millisecond | Microsecond, _) | Time64(Microsecond)+ );++ if !scale_can_overflow+ && step.sort_properties == SortProperties::Singleton&& reference- .map(|r| r.sort_properties.eq(&SortProperties::Singleton))+ .map(|r| r.sort_properties == SortProperties::Singleton).unwrap_or(true){Ok(date_value.sort_properties)diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex 951c50e706..c2be17579a 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -1728,7 +1728,7 @@ physical_plan02)--SortExec: TopK(fetch=2), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]03)----DataSourceExec: partitions=1, partition_sizes=[1]-# Typed bounds allow date_trunc to propagate ordering through timezone-free date_bin+# date_bin remains conservatively unordered because per-row failures become NULLquery TTEXPLAIN SELECT date_trunc('hour', date_bin(INTERVAL '1 minute', ts)) AS truncatedFROM (@@ -1750,10 +1750,51 @@ logical_plan04)------Projection: CAST(column1 AS Timestamp(s)) AS ts05)--------Values: (Int64(7200)), (Int64(3600))physical_plan-01)ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]-02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]-03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]-04)------DataSourceExec: partitions=1, partition_sizes=[1]+01)SortExec: expr=[truncated@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]+03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]+04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]+05)--------DataSourceExec: partitions=1, partition_sizes=[1]++# date_bin can return NULL for out-of-range values, so its output does not+# necessarily preserve the source's NULL ordering.+query P+SELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (-10000000000::bigint),+ (0::bigint),+ (10000000000::bigint)+ )+ ORDER BY ts+ LIMIT 3+)+ORDER BY b ASC NULLS LAST+----+1970-01-01T00:00:00+NULL+NULL++# An ordered expression can retain its ordering while its range type is unknown.+# date_bin must not assume that such an input is safe to scale to nanoseconds.+query P+SELECT date_bin(INTERVAL '1 second', date_trunc('second', ts)) AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (-10000000000::bigint),+ (0::bigint),+ (10000000000::bigint)+ )+ ORDER BY ts+ LIMIT 3+)+ORDER BY b ASC NULLS LAST+----+1970-01-01T00:00:00+NULL+NULLstatement okCREATE TABLE timestamp_strings(ts varchar)