Apache DataFusion SQL Query Engine
fix: bin coarse date_bin timestamps in i128 and restore ordering (#25815)
Liang-Chi Hsiehc0e872fb4d02
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rsindex dc3da309da..1e04c6c057 100644--- a/datafusion/functions/src/datetime/date_bin.rs+++ b/datafusion/functions/src/datetime/date_bin.rs@@ -24,7 +24,7 @@ use arrow::array::types::{TimestampSecondType,};use arrow::array::{ArrayRef, AsArray, PrimitiveArray};-use arrow::datatypes::DataType::{Null, Time32, Time64, Timestamp};+use arrow::datatypes::DataType::{Time32, Time64, Timestamp};use arrow::datatypes::IntervalUnit::{DayTime, MonthDayNano};use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};use arrow::datatypes::{@@ -272,16 +272,19 @@ impl ScalarUDFImpl for DateBinFunc {let date_value = &input[1];let reference = input.get(2);- // Scaling these representations to nanoseconds can overflow and turn- // otherwise valid input rows into NULL. Unknown ranges use the Null- // type and can hide one of these representations. The generated NULLs- // need not have the same placement as the source ordering.- let scale_can_overflow = matches!(- date_value.range.data_type(),- Null | Timestamp(Second | Millisecond | Microsecond, _) | Time64(Microsecond)- );+ // DATE_BIN preserves the order of its second argument.+ //+ // A negative month stride can move a bin past its source (see+ // `bin_months`), so its output is not monotonic, even for ordinary+ // dates. See https://github.com/apache/datafusion/issues/25856+ let monotonic_stride =+ matches!(step.range.lower(), ScalarValue::IntervalDayTime(Some(_)))+ || matches!(+ step.range.lower(),+ ScalarValue::IntervalMonthDayNano(Some(v)) if v.months >= 0+ );- if !scale_can_overflow+ if monotonic_stride&& step.sort_properties == SortProperties::Singleton&& reference.map(|r| r.sort_properties == SortProperties::Singleton)@@ -310,6 +313,21 @@ const NANOS_PER_SEC: i64 = NANOSECONDS;////// Returns: Binned timestamp in nanoseconds, or error if out of rangetype BinFunction = fn(i64, i64, i64) -> Result<i64>;++/// Same as [`BinFunction`], but with an `i128` source and result so that+/// coarse-precision values beyond the `i64` nanosecond range can be binned.+///+/// Returns: Binned timestamp in nanoseconds, or `None` if out of range+type WideBinFunction = fn(i64, i128, i64) -> Option<i128>;++/// The `i64` binning function, used first, and its `i128` counterpart, used+/// only when the `i64` computation overflows.+#[derive(Clone, Copy)]+struct BinFunctions {+ narrow: BinFunction,+ wide: WideBinFunction,+}+enum Interval {Nanoseconds(i64),Months(i64),@@ -319,15 +337,27 @@ impl Interval {/// Returns (`stride_nanos`, `fn`) where////// 1. `stride_nanos` is a width, in nanoseconds- /// 2. `fn` is a function that takes (stride_nanos, source, origin)+ /// 2. `fn` holds functions that take (stride_nanos, source, origin)////// `source` is the timestamp being binned////// `origin` is the time, in nanoseconds, where windows are measured from- fn bin_fn(&self) -> (i64, BinFunction) {+ fn bin_fn(&self) -> (i64, BinFunctions) {match self {- Interval::Nanoseconds(nanos) => (*nanos, date_bin_nanos_interval),- Interval::Months(months) => (*months, date_bin_months_interval),+ Interval::Nanoseconds(nanos) => (+ *nanos,+ BinFunctions {+ narrow: date_bin_nanos_interval,+ wide: date_bin_nanos_interval_wide,+ },+ ),+ Interval::Months(months) => (+ *months,+ BinFunctions {+ narrow: date_bin_months_interval,+ wide: date_bin_months_interval_wide,+ },+ ),}}}@@ -377,6 +407,33 @@ fn compute_distance(time_diff: i64, stride: i64) -> Result<i64> {}}+// `date_bin_nanos_interval` in i128, which cannot overflow for an i64 source+// scaled to nanoseconds.+fn date_bin_nanos_interval_wide(+ stride_nanos: i64,+ source: i128,+ origin: i64,+) -> Option<i128> {+ let origin = i128::from(origin);+ let time_delta = compute_distance_wide(source - origin, i128::from(stride_nanos));+ Some(origin + time_delta)+}++// `compute_distance` in i128. `stride` is non-zero, and `time_diff` is far+// from i128::MIN, so none of these operations can overflow.+fn compute_distance_wide(time_diff: i128, stride: i128) -> i128 {+ let time_delta = time_diff - time_diff % stride;+ // `%` rounds toward zero, so a negative `time_diff` between two bins is+ // rounded up to the later bin; move back one bin. This must match+ // `compute_distance`, so it does not use `rem_euclid`, which rounds+ // negative strides differently.+ if time_diff < 0 && stride > 1 && time_delta != time_diff {+ time_delta - stride+ } else {+ time_delta+ }+}+// Shift `origin_date` by `month_delta` months, mapping an out-of-range result to// the same error the binning paths reported when this was written inline.fn shift_months(origin_date: DateTime<Utc>, month_delta: i64) -> Result<DateTime<Utc>> {@@ -399,6 +456,39 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> Reslet source_date = to_utc_date_time(source)?;let origin_date = to_utc_date_time(origin)?;+ let bin_time = bin_months(stride_months, source_date, origin_date)?;+ match bin_time.timestamp_nanos_opt() {+ Some(nanos) => Ok(nanos),+ None => exec_err!("DATE_BIN result timestamp out of range"),+ }+}++// `date_bin_months_interval` with i128 nanoseconds, limited by the range of+// `DateTime<Utc>` instead of i64 nanoseconds.+fn date_bin_months_interval_wide(+ stride_months: i64,+ source: i128,+ origin: i64,+) -> Option<i128> {+ let nanos_per_sec = i128::from(NANOS_PER_SEC);+ let secs = i64::try_from(source.div_euclid(nanos_per_sec)).ok()?;+ let nsec = source.rem_euclid(nanos_per_sec) as u32;+ let source_date = DateTime::from_timestamp(secs, nsec)?;+ let origin_date = to_utc_date_time(origin).ok()?;++ let bin_time = bin_months(stride_months, source_date, origin_date).ok()?;+ Some(+ i128::from(bin_time.timestamp()) * nanos_per_sec+ + i128::from(bin_time.timestamp_subsec_nanos()),+ )+}++// return the start of the month bin that `source_date` falls into+fn bin_months(+ stride_months: i64,+ source_date: DateTime<Utc>,+ origin_date: DateTime<Utc>,+) -> Result<DateTime<Utc>> {// calculate the number of months between the source and originlet month_diff = (source_date.year() - origin_date.year()) * 12+ source_date.month() as i32@@ -415,10 +505,7 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> Reslet month_delta = month_delta - stride_months;bin_time = shift_months(origin_date, month_delta)?;}- match bin_time.timestamp_nanos_opt() {- Some(nanos) => Ok(nanos),- None => exec_err!("DATE_BIN result timestamp out of range"),- }+ Ok(bin_time)}fn to_utc_date_time(nanos: i64) -> Result<DateTime<Utc>> {@@ -469,11 +556,31 @@ fn date_bin_timestamp_value<T: ArrowTimestampType>(value: i64,origin: i64,stride: i64,- stride_fn: BinFunction,+ stride_fn: BinFunctions,) -> Option<i64> {let scale = timestamp_scale::<T>();- scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn)- .map(|binned| binned / scale)+ match scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn.narrow) {+ Some(binned) => Some(binned / scale),+ None => {+ date_bin_timestamp_value_wide(value, scale, origin, stride, stride_fn.wide)+ }+ }+}++// Slow path for values whose i64 nanosecond computation overflows. Binning in+// i128 means that only a result outside the source type's range becomes NULL,+// instead of any value outside the i64 nanosecond range.+#[cold]+#[inline(never)]+fn date_bin_timestamp_value_wide(+ value: i64,+ scale: i64,+ origin: i64,+ stride: i64,+ stride_fn: WideBinFunction,+) -> Option<i64> {+ let binned = stride_fn(stride, i128::from(value) * i128::from(scale), origin)?;+ i64::try_from(binned / i128::from(scale)).ok()}// Per-row TIME binning shared by scalar and array paths.@@ -484,10 +591,26 @@ fn date_bin_time_value(scale: i64,origin: i64,stride: i64,- stride_fn: BinFunction,+ stride_fn: BinFunctions,) -> Option<i64> {- scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn)- .map(|binned| (binned % NANOSECONDS_IN_DAY) / scale)+ match scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn.narrow) {+ Some(binned) => Some((binned % NANOSECONDS_IN_DAY) / scale),+ None => date_bin_time_value_wide(value, scale, origin, stride, stride_fn.wide),+ }+}++// Slow path of `date_bin_time_value`, like `date_bin_timestamp_value_wide`.+#[cold]+#[inline(never)]+fn date_bin_time_value_wide(+ value: i64,+ scale: i64,+ origin: i64,+ stride: i64,+ stride_fn: WideBinFunction,+) -> Option<i64> {+ let binned = stride_fn(stride, i128::from(value) * i128::from(scale), origin)?;+ i64::try_from((binned % i128::from(NANOSECONDS_IN_DAY)) / i128::from(scale)).ok()}fn validate_time_stride(stride: &Interval) -> Result<()> {@@ -693,7 +816,7 @@ fn date_bin_impl(fn transform_array_with_stride<T>(origin: i64,stride: i64,- stride_fn: BinFunction,+ stride_fn: BinFunctions,array: &ArrayRef,tz_opt: Option<&Arc<str>>,) -> Result<ColumnarValue>@@ -852,21 +975,34 @@ mod tests {DateBinFunc::new().invoke_with_args(args)}- fn assert_null_scalar(value: ColumnarValue, expected_type: DataType) {- let ColumnarValue::Scalar(value) = value else {- panic!("expected scalar, got {value:?}");- };- assert_eq!(value.data_type(), expected_type);- assert!(value.is_null(), "expected NULL, got {value:?}");- }-- fn assert_array_null_then_valid(value: ColumnarValue, expected_type: DataType) {- let ColumnarValue::Array(array) = value else {- panic!("expected array, got {value:?}");- };- assert_eq!(array.data_type(), &expected_type);- assert!(array.is_null(0), "expected NULL at row 0");- assert!(array.is_valid(1), "expected valid value at row 1");+ // Check `date_bin` against `expected` for a scalar source and for the+ // same source as a one-row array.+ fn assert_date_bin(+ stride: ScalarValue,+ source: ScalarValue,+ origin: ScalarValue,+ expected: ScalarValue,+ ) {+ let return_field = Arc::new(Field::new("f", source.data_type(), true));+ let sources = [+ ColumnarValue::Scalar(source.clone()),+ ColumnarValue::Array(source.to_array().unwrap()),+ ];+ for source in sources {+ let args = vec![+ ColumnarValue::Scalar(stride.clone()),+ source,+ ColumnarValue::Scalar(origin.clone()),+ ];+ let actual = match invoke_date_bin_with_args(args, 1, &return_field).unwrap()+ {+ ColumnarValue::Scalar(value) => value,+ ColumnarValue::Array(array) => {+ ScalarValue::try_from_array(&array, 0).unwrap()+ }+ };+ assert_eq!(actual, expected, "stride {stride:?}");+ }}fn assert_overflow_error(result: Result<ColumnarValue, DataFusionError>) {@@ -877,6 +1013,51 @@ mod tests {);}+ #[test]+ fn output_ordering_requires_monotonic_stride() {+ use arrow::compute::SortOptions;+ use datafusion_expr::sort_properties::{ExprProperties, SortProperties};++ let literal = |value: ScalarValue| {+ ExprProperties::new_unknown()+ .with_order(SortProperties::Singleton)+ .with_range(value.into())+ };+ let ordered = SortProperties::Ordered(SortOptions::default());+ let source = ExprProperties::new_unknown().with_order(ordered);+ let ordering = |stride: ExprProperties| {+ DateBinFunc::new()+ .output_ordering(&[stride, source.clone()])+ .unwrap()+ };++ assert_eq!(+ ordering(literal(ScalarValue::new_interval_dt(0, 1000))),+ ordered+ );+ assert_eq!(+ ordering(literal(ScalarValue::new_interval_dt(-1, 0))),+ ordered+ );+ assert_eq!(+ ordering(literal(ScalarValue::new_interval_mdn(1, 0, 0))),+ ordered+ );+ assert_eq!(+ ordering(literal(ScalarValue::new_interval_mdn(0, -1, 0))),+ ordered+ );+ assert_eq!(+ ordering(literal(ScalarValue::new_interval_mdn(-1, 0, 0))),+ SortProperties::Unordered+ );+ // A constant stride whose value is unknown may be a negative month.+ assert_eq!(+ ordering(ExprProperties::new_unknown().with_order(SortProperties::Singleton)),+ SortProperties::Unordered+ );+ }+#[test]fn test_date_bin() {let return_field = &Arc::new(Field::new(@@ -1390,53 +1571,86 @@ mod tests {}#[test]- fn test_date_bin_scale_overflow_returns_null() {- // Scaling non-nanosecond timestamps to nanoseconds can overflow.- use arrow::array::{- ArrayRef, TimestampMicrosecondArray, TimestampMillisecondArray,- TimestampSecondArray,+ fn test_date_bin_beyond_nanosecond_range() {+ // Values whose nanosecond scaling or binning overflows i64 are binned+ // in i128, so they keep a valid result in their own precision.+ let day = || ScalarValue::new_interval_dt(1, 0);+ let second = || ScalarValue::new_interval_dt(0, 1000);+ let month = || ScalarValue::new_interval_mdn(1, 0, 0);+ let epoch = || ScalarValue::TimestampNanosecond(Some(0), None);+ let secs = |v| ScalarValue::TimestampSecond(Some(v), None);+ let month_start = |year, month| {+ chrono::NaiveDate::from_ymd_opt(year, month, 1)+ .unwrap()+ .and_hms_opt(0, 0, 0)+ .unwrap()+ .and_utc()+ .timestamp()};- let scalar_cases = [- ScalarValue::TimestampSecond(Some(i64::MAX), None),- ScalarValue::TimestampMillisecond(Some(i64::MAX), None),- ScalarValue::TimestampMicrosecond(Some(i64::MAX), None),+ let cases = [+ (day(), secs(i64::MAX), secs(i64::MAX - i64::MAX % 86_400)),+ (+ day(),+ ScalarValue::TimestampMillisecond(Some(i64::MAX), None),+ ScalarValue::TimestampMillisecond(+ Some(i64::MAX - i64::MAX % 86_400_000),+ None,+ ),+ ),+ (+ day(),+ ScalarValue::TimestampMicrosecond(Some(i64::MAX), None),+ ScalarValue::TimestampMicrosecond(+ Some(i64::MAX - i64::MAX % 86_400_000_000),+ None,+ ),+ ),+ // Around 1653 and 2286, outside the nanosecond range.+ (second(), secs(-10_000_000_000), secs(-10_000_000_000)),+ (second(), secs(10_000_000_000), secs(10_000_000_000)),+ (day(), secs(-10_000_000_001), secs(-10_000_022_400)),+ (month(), secs(-10_000_000_000), secs(month_start(1653, 2))),+ (month(), secs(10_000_000_000), secs(month_start(2286, 11))),+ // An exact bin at the minimum value.+ (second(), secs(i64::MIN), secs(i64::MIN)),];- for source in scalar_cases {- let expected_type = source.data_type();- let return_field = Arc::new(Field::new("f", expected_type.clone(), true));- let args = vec![- ColumnarValue::Scalar(ScalarValue::new_interval_dt(1, 0)),- ColumnarValue::Scalar(source),- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(0), None)),- ];- let result = invoke_date_bin_with_args(args, 1, &return_field)- .unwrap_or_else(|e| panic!("expected Ok for {expected_type}, got {e:?}"));- assert_null_scalar(result, expected_type);+ for (stride, source, expected) in cases {+ assert_date_bin(stride, source, epoch(), expected);}- let array_cases: Vec<ArrayRef> = vec![- Arc::new(TimestampSecondArray::from(vec![Some(i64::MAX), Some(0)])),- Arc::new(TimestampMillisecondArray::from(vec![- Some(i64::MAX),- Some(0),- ])),- Arc::new(TimestampMicrosecondArray::from(vec![- Some(i64::MAX),- Some(0),- ])),+ // `source - origin` overflows i64 nanoseconds.+ let hour = ScalarValue::new_interval_dt(0, 3_600_000);+ assert_date_bin(+ hour,+ ScalarValue::TimestampNanosecond(Some(-9_000_000_000_000_000_000 + 1), None),+ ScalarValue::TimestampNanosecond(Some(3_600_000_000_000_000_000), None),+ ScalarValue::TimestampNanosecond(Some(-9_000_000_000_000_000_000), None),+ );+ }++ #[test]+ fn test_date_bin_unrepresentable_result_returns_null() {+ let epoch = || ScalarValue::TimestampNanosecond(Some(0), None);+ let cases = [+ // The bin starts before the minimum value.+ (+ ScalarValue::new_interval_dt(1, 0),+ ScalarValue::TimestampSecond(Some(i64::MIN), None),+ ),+ (+ ScalarValue::new_interval_mdn(0, 0, 3),+ ScalarValue::TimestampNanosecond(Some(i64::MIN), None),+ ),+ // Month bins are limited by the range of `DateTime<Utc>`.+ (+ ScalarValue::new_interval_mdn(1, 0, 0),+ ScalarValue::TimestampSecond(Some(i64::MAX), None),+ ),];- for array in array_cases {- let dt = array.data_type().clone();- let return_field = Arc::new(Field::new("f", dt.clone(), true));- let args = vec![- ColumnarValue::Scalar(ScalarValue::new_interval_dt(1, 0)),- ColumnarValue::Array(array),- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(0), None)),- ];- let result = invoke_date_bin_with_args(args, 2, &return_field)- .unwrap_or_else(|e| panic!("expected Ok for {dt:?}, got {e:?}"));- assert_array_null_then_valid(result, dt);+ for (stride, source) in cases {+ let expected = ScalarValue::try_from(source.data_type()).unwrap();+ assert_date_bin(stride, source, epoch(), expected);}}@@ -1448,21 +1662,17 @@ mod tests {let data_type = DataType::Time64(TimeUnit::Microsecond);let return_field = &Arc::new(Field::new("f", data_type.clone(), true));let stride = || ColumnarValue::Scalar(ScalarValue::new_interval_dt(0, 1000));- let origin = || ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(0)));- // Out-of-range source values are per-row data, so they become NULL.- let args = vec![- stride(),- ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(i64::MAX))),- origin(),- ];- let result = invoke_date_bin_with_args(args, 1, return_field).unwrap();- assert_null_scalar(result, data_type.clone());-- let array = Arc::new(Time64MicrosecondArray::from(vec![Some(i64::MAX), Some(0)]));- let args = vec![stride(), ColumnarValue::Array(array), origin()];- let result = invoke_date_bin_with_args(args, 2, return_field).unwrap();- assert_array_null_then_valid(result, data_type);+ // Out-of-range source values are binned in i128 and reduced to a day,+ // like in-range values.+ assert_date_bin(+ ScalarValue::new_interval_dt(0, 1000),+ ScalarValue::Time64Microsecond(Some(i64::MAX)),+ ScalarValue::Time64Microsecond(Some(0)),+ ScalarValue::Time64Microsecond(Some(+ (i64::MAX - i64::MAX % 1_000_000) % 86_400_000_000,+ )),+ );let bad_origin =|| ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(i64::MAX)));diff --git a/datafusion/sqllogictest/test_files/date_bin_errors.slt b/datafusion/sqllogictest/test_files/date_bin_errors.sltindex 53cba506de..cb26023c46 100644--- a/datafusion/sqllogictest/test_files/date_bin_errors.slt+++ b/datafusion/sqllogictest/test_files/date_bin_errors.slt@@ -73,7 +73,7 @@ select date_bin(----NULL-# Extreme timestamp overflow: source - origin overflows i64 (should return NULL, not panic)+# source - origin overflows i64 nanosecondsquery Pselect date_bin(interval '1 nanosecond',@@ -81,7 +81,7 @@ select date_bin(arrow_cast(-9223372036854775808, 'Timestamp(Nanosecond, None)'));-----NULL+2262-04-11T23:47:16.854775807# compute_distance overflow: source at i64::MIN nanoseconds previously panicked# inside compute_distance; it must return NULL through the SQL execution path@@ -93,17 +93,17 @@ select date_bin(----NULL-# Source timestamp scaling to nanoseconds overflows: should return NULL, not panic+# Scaling the source to nanoseconds overflows i64query Pselect date_bin(interval '1 nanosecond',- arrow_cast(9223372036854775807, 'Timestamp(Second, None)'),+ arrow_cast(10000000000, 'Timestamp(Second, None)'),timestamp '1970-01-01 00:00:00');-----NULL+2286-11-20T17:46:40-# Source timestamp scaling to nanoseconds overflows in array path: should return NULL, not panic+# Same in the array pathquery Pselect date_bin(interval '1 nanosecond',@@ -111,7 +111,7 @@ select date_bin(timestamp '1970-01-01 00:00:00')from (- values (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))+ values (arrow_cast(10000000000, 'Timestamp(Second, None)'))) as t(ts);-----NULL+2286-11-20T17:46:40diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex 28a4269488..71118ddcab 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -1728,7 +1728,7 @@ physical_plan02)--SortExec: TopK(fetch=2), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]03)----DataSourceExec: partitions=1, partition_sizes=[1]-# date_bin remains conservatively unordered because per-row failures become NULL+# Typed bounds allow date_trunc to propagate ordering through timezone-free date_binquery TTEXPLAIN SELECT date_trunc('hour', date_bin(INTERVAL '1 minute', ts)) AS truncatedFROM (@@ -1750,14 +1750,39 @@ logical_plan04)------Projection: CAST(column1 AS Timestamp(s)) AS ts05)--------Values: (Int64(7200)), (Int64(3600))physical_plan-01)SortExec: expr=[truncated@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]-03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]-04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]-05)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]+02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++# Coarse-precision values outside the nanosecond range are binned without+# overflowing, so date_bin keeps both their values and their ordering.+query TT+EXPLAIN SELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (-10000000000::bigint),+ (0::bigint),+ (10000000000::bigint)+ )+ ORDER BY ts+ LIMIT 3+)+ORDER BY b ASC NULLS LAST+----+logical_plan+01)Sort: b ASC NULLS LAST+02)--Projection: date_trunc(Utf8("second"), date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 1000000000 }"), ts)) AS b+03)----Sort: ts ASC NULLS LAST, fetch=3+04)------Projection: CAST(column1 AS Timestamp(s)) AS ts+05)--------Values: (Int64(-10000000000) AS (- Int64(10000000000))), (Int64(0)), (Int64(10000000000))+physical_plan+01)ProjectionExec: expr=[date_trunc(second, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 1000000000 }, ts@0)) as b]+02)--SortExec: TopK(fetch=3), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]+04)------DataSourceExec: partitions=1, partition_sizes=[1]-# date_bin can return NULL for out-of-range values, so its output does not-# necessarily preserve the source's NULL ordering.query PSELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS bFROM (@@ -1772,12 +1797,12 @@ FROM ()ORDER BY b ASC NULLS LAST----+1653-02-10T06:13:201970-01-01T00:00:00-NULL-NULL+2286-11-20T17:46:40-# An ordered expression can retain its ordering while its range type is unknown.-# date_bin must not assume that such an input is safe to scale to nanoseconds.+# The same holds when the source is an ordered expression with an unknown+# range type.query PSELECT date_bin(INTERVAL '1 second', date_trunc('second', ts)) AS bFROM (@@ -1792,9 +1817,113 @@ FROM ()ORDER BY b ASC NULLS LAST----+1653-02-10T06:13:201970-01-01T00:00:00-NULL-NULL+2286-11-20T17:46:40++# A negative month stride does not shift every bin by the same amount: here+# 2023-01-01 bins forward to 2023-02-28, while 2023-01-31 bins to itself. The+# output is not sorted, so date_bin keeps the final sort for it.+query TT+EXPLAIN SELECT date_bin(INTERVAL '-1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (TIMESTAMP '2023-01-01 00:00:00'),+ (TIMESTAMP '2023-01-31 00:00:00')+ )+ ORDER BY ts+ LIMIT 2+)+ORDER BY b+----+logical_plan+01)Sort: b ASC NULLS LAST+02)--Projection: date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: -1, days: 0, nanoseconds: 0 }"), ts, TimestampNanosecond(1675123200000000000, None)) AS b+03)----Sort: ts ASC NULLS LAST, fetch=2+04)------Projection: CAST(column1 AS Timestamp(s)) AS ts+05)--------Values: (TimestampNanosecond(1672531200000000000, None) AS Utf8("2023-01-01 00:00:00")), (TimestampNanosecond(1675123200000000000, None) AS Utf8("2023-01-31 00:00:00"))+physical_plan+01)SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--ProjectionExec: expr=[date_bin(IntervalMonthDayNano { months: -1, days: 0, nanoseconds: 0 }, ts@0, 1675123200000000000) as b]+03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]+04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]+05)--------DataSourceExec: partitions=1, partition_sizes=[1]++query P+SELECT date_bin(INTERVAL '-1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (TIMESTAMP '2023-01-01 00:00:00'),+ (TIMESTAMP '2023-01-31 00:00:00')+ )+ ORDER BY ts+ LIMIT 2+)+ORDER BY b+----+2023-01-31T00:00:00+2023-02-28T00:00:00++# Same for nanosecond timestamps.+query P+SELECT date_bin(INTERVAL '-1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b+FROM (+ SELECT column1 AS ts+ FROM (VALUES+ (TIMESTAMP '2023-01-01 00:00:00'),+ (TIMESTAMP '2023-01-31 00:00:00')+ )+ ORDER BY ts+ LIMIT 2+)+ORDER BY b+----+2023-01-31T00:00:00+2023-02-28T00:00:00++# A positive month stride is monotonic, so the final sort is removed.+query TT+EXPLAIN SELECT date_bin(INTERVAL '1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (TIMESTAMP '2023-01-01 00:00:00'),+ (TIMESTAMP '2023-01-31 00:00:00')+ )+ ORDER BY ts+ LIMIT 2+)+ORDER BY b+----+logical_plan+01)Sort: b ASC NULLS LAST+02)--Projection: date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 1, days: 0, nanoseconds: 0 }"), ts, TimestampNanosecond(1675123200000000000, None)) AS b+03)----Sort: ts ASC NULLS LAST, fetch=2+04)------Projection: CAST(column1 AS Timestamp(s)) AS ts+05)--------Values: (TimestampNanosecond(1672531200000000000, None) AS Utf8("2023-01-01 00:00:00")), (TimestampNanosecond(1675123200000000000, None) AS Utf8("2023-01-31 00:00:00"))+physical_plan+01)ProjectionExec: expr=[date_bin(IntervalMonthDayNano { months: 1, days: 0, nanoseconds: 0 }, ts@0, 1675123200000000000) as b]+02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]+04)------DataSourceExec: partitions=1, partition_sizes=[1]++query P+SELECT date_bin(INTERVAL '1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b+FROM (+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts+ FROM (VALUES+ (TIMESTAMP '2023-01-01 00:00:00'),+ (TIMESTAMP '2023-01-31 00:00:00')+ )+ ORDER BY ts+ LIMIT 2+)+ORDER BY b+----+2022-12-31T00:00:00+2023-01-31T00:00:00# Default bounds from nested functions retain their resolved output type.statement ok