Skip to content

Apache DataFusion SQL Query Engine

fix: bin coarse date_bin timestamps in i128 and restore ordering (#25815)

Liang-Chi Hsiehc0e872fb4d02
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rs
index dc3da309da..1e04c6c057 100644
--- a/datafusion/functions/src/datetime/date_bin.rs
+++ b/datafusion/functions/src/datetime/date_bin.rs
@@ -24,7 +24,7 @@ use arrow::array::types::{
TimestampSecondType,
};
use arrow::array::{ArrayRef, AsArray, PrimitiveArray};
-use arrow::datatypes::DataType::{Null, Time32, Time64, Timestamp};
+use arrow::datatypes::DataType::{Time32, Time64, Timestamp};
use arrow::datatypes::IntervalUnit::{DayTime, MonthDayNano};
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
@@ -272,16 +272,19 @@ impl ScalarUDFImpl for DateBinFunc {
let date_value = &input[1];
let reference = input.get(2);
- // Scaling these representations to nanoseconds can overflow and turn
- // otherwise valid input rows into NULL. Unknown ranges use the Null
- // type and can hide one of these representations. The generated NULLs
- // need not have the same placement as the source ordering.
- let scale_can_overflow = matches!(
- date_value.range.data_type(),
- Null | Timestamp(Second | Millisecond | Microsecond, _) | Time64(Microsecond)
- );
+ // DATE_BIN preserves the order of its second argument.
+ //
+ // A negative month stride can move a bin past its source (see
+ // `bin_months`), so its output is not monotonic, even for ordinary
+ // dates. See https://github.com/apache/datafusion/issues/25856
+ let monotonic_stride =
+ matches!(step.range.lower(), ScalarValue::IntervalDayTime(Some(_)))
+ || matches!(
+ step.range.lower(),
+ ScalarValue::IntervalMonthDayNano(Some(v)) if v.months >= 0
+ );
- if !scale_can_overflow
+ if monotonic_stride
&& step.sort_properties == SortProperties::Singleton
&& reference
.map(|r| r.sort_properties == SortProperties::Singleton)
@@ -310,6 +313,21 @@ const NANOS_PER_SEC: i64 = NANOSECONDS;
///
/// Returns: Binned timestamp in nanoseconds, or error if out of range
type BinFunction = fn(i64, i64, i64) -> Result<i64>;
+
+/// Same as [`BinFunction`], but with an `i128` source and result so that
+/// coarse-precision values beyond the `i64` nanosecond range can be binned.
+///
+/// Returns: Binned timestamp in nanoseconds, or `None` if out of range
+type WideBinFunction = fn(i64, i128, i64) -> Option<i128>;
+
+/// The `i64` binning function, used first, and its `i128` counterpart, used
+/// only when the `i64` computation overflows.
+#[derive(Clone, Copy)]
+struct BinFunctions {
+ narrow: BinFunction,
+ wide: WideBinFunction,
+}
+
enum Interval {
Nanoseconds(i64),
Months(i64),
@@ -319,15 +337,27 @@ impl Interval {
/// Returns (`stride_nanos`, `fn`) where
///
/// 1. `stride_nanos` is a width, in nanoseconds
- /// 2. `fn` is a function that takes (stride_nanos, source, origin)
+ /// 2. `fn` holds functions that take (stride_nanos, source, origin)
///
/// `source` is the timestamp being binned
///
/// `origin` is the time, in nanoseconds, where windows are measured from
- fn bin_fn(&self) -> (i64, BinFunction) {
+ fn bin_fn(&self) -> (i64, BinFunctions) {
match self {
- Interval::Nanoseconds(nanos) => (*nanos, date_bin_nanos_interval),
- Interval::Months(months) => (*months, date_bin_months_interval),
+ Interval::Nanoseconds(nanos) => (
+ *nanos,
+ BinFunctions {
+ narrow: date_bin_nanos_interval,
+ wide: date_bin_nanos_interval_wide,
+ },
+ ),
+ Interval::Months(months) => (
+ *months,
+ BinFunctions {
+ narrow: date_bin_months_interval,
+ wide: date_bin_months_interval_wide,
+ },
+ ),
}
}
}
@@ -377,6 +407,33 @@ fn compute_distance(time_diff: i64, stride: i64) -> Result<i64> {
}
}
+// `date_bin_nanos_interval` in i128, which cannot overflow for an i64 source
+// scaled to nanoseconds.
+fn date_bin_nanos_interval_wide(
+ stride_nanos: i64,
+ source: i128,
+ origin: i64,
+) -> Option<i128> {
+ let origin = i128::from(origin);
+ let time_delta = compute_distance_wide(source - origin, i128::from(stride_nanos));
+ Some(origin + time_delta)
+}
+
+// `compute_distance` in i128. `stride` is non-zero, and `time_diff` is far
+// from i128::MIN, so none of these operations can overflow.
+fn compute_distance_wide(time_diff: i128, stride: i128) -> i128 {
+ let time_delta = time_diff - time_diff % stride;
+ // `%` rounds toward zero, so a negative `time_diff` between two bins is
+ // rounded up to the later bin; move back one bin. This must match
+ // `compute_distance`, so it does not use `rem_euclid`, which rounds
+ // negative strides differently.
+ if time_diff < 0 && stride > 1 && time_delta != time_diff {
+ time_delta - stride
+ } else {
+ time_delta
+ }
+}
+
// Shift `origin_date` by `month_delta` months, mapping an out-of-range result to
// the same error the binning paths reported when this was written inline.
fn shift_months(origin_date: DateTime<Utc>, month_delta: i64) -> Result<DateTime<Utc>> {
@@ -399,6 +456,39 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> Res
let source_date = to_utc_date_time(source)?;
let origin_date = to_utc_date_time(origin)?;
+ let bin_time = bin_months(stride_months, source_date, origin_date)?;
+ match bin_time.timestamp_nanos_opt() {
+ Some(nanos) => Ok(nanos),
+ None => exec_err!("DATE_BIN result timestamp out of range"),
+ }
+}
+
+// `date_bin_months_interval` with i128 nanoseconds, limited by the range of
+// `DateTime<Utc>` instead of i64 nanoseconds.
+fn date_bin_months_interval_wide(
+ stride_months: i64,
+ source: i128,
+ origin: i64,
+) -> Option<i128> {
+ let nanos_per_sec = i128::from(NANOS_PER_SEC);
+ let secs = i64::try_from(source.div_euclid(nanos_per_sec)).ok()?;
+ let nsec = source.rem_euclid(nanos_per_sec) as u32;
+ let source_date = DateTime::from_timestamp(secs, nsec)?;
+ let origin_date = to_utc_date_time(origin).ok()?;
+
+ let bin_time = bin_months(stride_months, source_date, origin_date).ok()?;
+ Some(
+ i128::from(bin_time.timestamp()) * nanos_per_sec
+ + i128::from(bin_time.timestamp_subsec_nanos()),
+ )
+}
+
+// return the start of the month bin that `source_date` falls into
+fn bin_months(
+ stride_months: i64,
+ source_date: DateTime<Utc>,
+ origin_date: DateTime<Utc>,
+) -> Result<DateTime<Utc>> {
// calculate the number of months between the source and origin
let month_diff = (source_date.year() - origin_date.year()) * 12
+ source_date.month() as i32
@@ -415,10 +505,7 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> Res
let month_delta = month_delta - stride_months;
bin_time = shift_months(origin_date, month_delta)?;
}
- match bin_time.timestamp_nanos_opt() {
- Some(nanos) => Ok(nanos),
- None => exec_err!("DATE_BIN result timestamp out of range"),
- }
+ Ok(bin_time)
}
fn to_utc_date_time(nanos: i64) -> Result<DateTime<Utc>> {
@@ -469,11 +556,31 @@ fn date_bin_timestamp_value<T: ArrowTimestampType>(
value: i64,
origin: i64,
stride: i64,
- stride_fn: BinFunction,
+ stride_fn: BinFunctions,
) -> Option<i64> {
let scale = timestamp_scale::<T>();
- scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn)
- .map(|binned| binned / scale)
+ match scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn.narrow) {
+ Some(binned) => Some(binned / scale),
+ None => {
+ date_bin_timestamp_value_wide(value, scale, origin, stride, stride_fn.wide)
+ }
+ }
+}
+
+// Slow path for values whose i64 nanosecond computation overflows. Binning in
+// i128 means that only a result outside the source type's range becomes NULL,
+// instead of any value outside the i64 nanosecond range.
+#[cold]
+#[inline(never)]
+fn date_bin_timestamp_value_wide(
+ value: i64,
+ scale: i64,
+ origin: i64,
+ stride: i64,
+ stride_fn: WideBinFunction,
+) -> Option<i64> {
+ let binned = stride_fn(stride, i128::from(value) * i128::from(scale), origin)?;
+ i64::try_from(binned / i128::from(scale)).ok()
}
// Per-row TIME binning shared by scalar and array paths.
@@ -484,10 +591,26 @@ fn date_bin_time_value(
scale: i64,
origin: i64,
stride: i64,
- stride_fn: BinFunction,
+ stride_fn: BinFunctions,
) -> Option<i64> {
- scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn)
- .map(|binned| (binned % NANOSECONDS_IN_DAY) / scale)
+ match scale_and_bin_to_nanos(value, scale, origin, stride, stride_fn.narrow) {
+ Some(binned) => Some((binned % NANOSECONDS_IN_DAY) / scale),
+ None => date_bin_time_value_wide(value, scale, origin, stride, stride_fn.wide),
+ }
+}
+
+// Slow path of `date_bin_time_value`, like `date_bin_timestamp_value_wide`.
+#[cold]
+#[inline(never)]
+fn date_bin_time_value_wide(
+ value: i64,
+ scale: i64,
+ origin: i64,
+ stride: i64,
+ stride_fn: WideBinFunction,
+) -> Option<i64> {
+ let binned = stride_fn(stride, i128::from(value) * i128::from(scale), origin)?;
+ i64::try_from((binned % i128::from(NANOSECONDS_IN_DAY)) / i128::from(scale)).ok()
}
fn validate_time_stride(stride: &Interval) -> Result<()> {
@@ -693,7 +816,7 @@ fn date_bin_impl(
fn transform_array_with_stride<T>(
origin: i64,
stride: i64,
- stride_fn: BinFunction,
+ stride_fn: BinFunctions,
array: &ArrayRef,
tz_opt: Option<&Arc<str>>,
) -> Result<ColumnarValue>
@@ -852,21 +975,34 @@ mod tests {
DateBinFunc::new().invoke_with_args(args)
}
- fn assert_null_scalar(value: ColumnarValue, expected_type: DataType) {
- let ColumnarValue::Scalar(value) = value else {
- panic!("expected scalar, got {value:?}");
- };
- assert_eq!(value.data_type(), expected_type);
- assert!(value.is_null(), "expected NULL, got {value:?}");
- }
-
- fn assert_array_null_then_valid(value: ColumnarValue, expected_type: DataType) {
- let ColumnarValue::Array(array) = value else {
- panic!("expected array, got {value:?}");
- };
- assert_eq!(array.data_type(), &expected_type);
- assert!(array.is_null(0), "expected NULL at row 0");
- assert!(array.is_valid(1), "expected valid value at row 1");
+ // Check `date_bin` against `expected` for a scalar source and for the
+ // same source as a one-row array.
+ fn assert_date_bin(
+ stride: ScalarValue,
+ source: ScalarValue,
+ origin: ScalarValue,
+ expected: ScalarValue,
+ ) {
+ let return_field = Arc::new(Field::new("f", source.data_type(), true));
+ let sources = [
+ ColumnarValue::Scalar(source.clone()),
+ ColumnarValue::Array(source.to_array().unwrap()),
+ ];
+ for source in sources {
+ let args = vec![
+ ColumnarValue::Scalar(stride.clone()),
+ source,
+ ColumnarValue::Scalar(origin.clone()),
+ ];
+ let actual = match invoke_date_bin_with_args(args, 1, &return_field).unwrap()
+ {
+ ColumnarValue::Scalar(value) => value,
+ ColumnarValue::Array(array) => {
+ ScalarValue::try_from_array(&array, 0).unwrap()
+ }
+ };
+ assert_eq!(actual, expected, "stride {stride:?}");
+ }
}
fn assert_overflow_error(result: Result<ColumnarValue, DataFusionError>) {
@@ -877,6 +1013,51 @@ mod tests {
);
}
+ #[test]
+ fn output_ordering_requires_monotonic_stride() {
+ use arrow::compute::SortOptions;
+ use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
+
+ let literal = |value: ScalarValue| {
+ ExprProperties::new_unknown()
+ .with_order(SortProperties::Singleton)
+ .with_range(value.into())
+ };
+ let ordered = SortProperties::Ordered(SortOptions::default());
+ let source = ExprProperties::new_unknown().with_order(ordered);
+ let ordering = |stride: ExprProperties| {
+ DateBinFunc::new()
+ .output_ordering(&[stride, source.clone()])
+ .unwrap()
+ };
+
+ assert_eq!(
+ ordering(literal(ScalarValue::new_interval_dt(0, 1000))),
+ ordered
+ );
+ assert_eq!(
+ ordering(literal(ScalarValue::new_interval_dt(-1, 0))),
+ ordered
+ );
+ assert_eq!(
+ ordering(literal(ScalarValue::new_interval_mdn(1, 0, 0))),
+ ordered
+ );
+ assert_eq!(
+ ordering(literal(ScalarValue::new_interval_mdn(0, -1, 0))),
+ ordered
+ );
+ assert_eq!(
+ ordering(literal(ScalarValue::new_interval_mdn(-1, 0, 0))),
+ SortProperties::Unordered
+ );
+ // A constant stride whose value is unknown may be a negative month.
+ assert_eq!(
+ ordering(ExprProperties::new_unknown().with_order(SortProperties::Singleton)),
+ SortProperties::Unordered
+ );
+ }
+
#[test]
fn test_date_bin() {
let return_field = &Arc::new(Field::new(
@@ -1390,53 +1571,86 @@ mod tests {
}
#[test]
- fn test_date_bin_scale_overflow_returns_null() {
- // Scaling non-nanosecond timestamps to nanoseconds can overflow.
- use arrow::array::{
- ArrayRef, TimestampMicrosecondArray, TimestampMillisecondArray,
- TimestampSecondArray,
+ fn test_date_bin_beyond_nanosecond_range() {
+ // Values whose nanosecond scaling or binning overflows i64 are binned
+ // in i128, so they keep a valid result in their own precision.
+ let day = || ScalarValue::new_interval_dt(1, 0);
+ let second = || ScalarValue::new_interval_dt(0, 1000);
+ let month = || ScalarValue::new_interval_mdn(1, 0, 0);
+ let epoch = || ScalarValue::TimestampNanosecond(Some(0), None);
+ let secs = |v| ScalarValue::TimestampSecond(Some(v), None);
+ let month_start = |year, month| {
+ chrono::NaiveDate::from_ymd_opt(year, month, 1)
+ .unwrap()
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ .and_utc()
+ .timestamp()
};
- let scalar_cases = [
- ScalarValue::TimestampSecond(Some(i64::MAX), None),
- ScalarValue::TimestampMillisecond(Some(i64::MAX), None),
- ScalarValue::TimestampMicrosecond(Some(i64::MAX), None),
+ let cases = [
+ (day(), secs(i64::MAX), secs(i64::MAX - i64::MAX % 86_400)),
+ (
+ day(),
+ ScalarValue::TimestampMillisecond(Some(i64::MAX), None),
+ ScalarValue::TimestampMillisecond(
+ Some(i64::MAX - i64::MAX % 86_400_000),
+ None,
+ ),
+ ),
+ (
+ day(),
+ ScalarValue::TimestampMicrosecond(Some(i64::MAX), None),
+ ScalarValue::TimestampMicrosecond(
+ Some(i64::MAX - i64::MAX % 86_400_000_000),
+ None,
+ ),
+ ),
+ // Around 1653 and 2286, outside the nanosecond range.
+ (second(), secs(-10_000_000_000), secs(-10_000_000_000)),
+ (second(), secs(10_000_000_000), secs(10_000_000_000)),
+ (day(), secs(-10_000_000_001), secs(-10_000_022_400)),
+ (month(), secs(-10_000_000_000), secs(month_start(1653, 2))),
+ (month(), secs(10_000_000_000), secs(month_start(2286, 11))),
+ // An exact bin at the minimum value.
+ (second(), secs(i64::MIN), secs(i64::MIN)),
];
- for source in scalar_cases {
- let expected_type = source.data_type();
- let return_field = Arc::new(Field::new("f", expected_type.clone(), true));
- let args = vec![
- ColumnarValue::Scalar(ScalarValue::new_interval_dt(1, 0)),
- ColumnarValue::Scalar(source),
- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(0), None)),
- ];
- let result = invoke_date_bin_with_args(args, 1, &return_field)
- .unwrap_or_else(|e| panic!("expected Ok for {expected_type}, got {e:?}"));
- assert_null_scalar(result, expected_type);
+ for (stride, source, expected) in cases {
+ assert_date_bin(stride, source, epoch(), expected);
}
- let array_cases: Vec<ArrayRef> = vec![
- Arc::new(TimestampSecondArray::from(vec![Some(i64::MAX), Some(0)])),
- Arc::new(TimestampMillisecondArray::from(vec![
- Some(i64::MAX),
- Some(0),
- ])),
- Arc::new(TimestampMicrosecondArray::from(vec![
- Some(i64::MAX),
- Some(0),
- ])),
+ // `source - origin` overflows i64 nanoseconds.
+ let hour = ScalarValue::new_interval_dt(0, 3_600_000);
+ assert_date_bin(
+ hour,
+ ScalarValue::TimestampNanosecond(Some(-9_000_000_000_000_000_000 + 1), None),
+ ScalarValue::TimestampNanosecond(Some(3_600_000_000_000_000_000), None),
+ ScalarValue::TimestampNanosecond(Some(-9_000_000_000_000_000_000), None),
+ );
+ }
+
+ #[test]
+ fn test_date_bin_unrepresentable_result_returns_null() {
+ let epoch = || ScalarValue::TimestampNanosecond(Some(0), None);
+ let cases = [
+ // The bin starts before the minimum value.
+ (
+ ScalarValue::new_interval_dt(1, 0),
+ ScalarValue::TimestampSecond(Some(i64::MIN), None),
+ ),
+ (
+ ScalarValue::new_interval_mdn(0, 0, 3),
+ ScalarValue::TimestampNanosecond(Some(i64::MIN), None),
+ ),
+ // Month bins are limited by the range of `DateTime<Utc>`.
+ (
+ ScalarValue::new_interval_mdn(1, 0, 0),
+ ScalarValue::TimestampSecond(Some(i64::MAX), None),
+ ),
];
- for array in array_cases {
- let dt = array.data_type().clone();
- let return_field = Arc::new(Field::new("f", dt.clone(), true));
- let args = vec![
- ColumnarValue::Scalar(ScalarValue::new_interval_dt(1, 0)),
- ColumnarValue::Array(array),
- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(0), None)),
- ];
- let result = invoke_date_bin_with_args(args, 2, &return_field)
- .unwrap_or_else(|e| panic!("expected Ok for {dt:?}, got {e:?}"));
- assert_array_null_then_valid(result, dt);
+ for (stride, source) in cases {
+ let expected = ScalarValue::try_from(source.data_type()).unwrap();
+ assert_date_bin(stride, source, epoch(), expected);
}
}
@@ -1448,21 +1662,17 @@ mod tests {
let data_type = DataType::Time64(TimeUnit::Microsecond);
let return_field = &Arc::new(Field::new("f", data_type.clone(), true));
let stride = || ColumnarValue::Scalar(ScalarValue::new_interval_dt(0, 1000));
- let origin = || ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(0)));
- // Out-of-range source values are per-row data, so they become NULL.
- let args = vec![
- stride(),
- ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(i64::MAX))),
- origin(),
- ];
- let result = invoke_date_bin_with_args(args, 1, return_field).unwrap();
- assert_null_scalar(result, data_type.clone());
-
- let array = Arc::new(Time64MicrosecondArray::from(vec![Some(i64::MAX), Some(0)]));
- let args = vec![stride(), ColumnarValue::Array(array), origin()];
- let result = invoke_date_bin_with_args(args, 2, return_field).unwrap();
- assert_array_null_then_valid(result, data_type);
+ // Out-of-range source values are binned in i128 and reduced to a day,
+ // like in-range values.
+ assert_date_bin(
+ ScalarValue::new_interval_dt(0, 1000),
+ ScalarValue::Time64Microsecond(Some(i64::MAX)),
+ ScalarValue::Time64Microsecond(Some(0)),
+ ScalarValue::Time64Microsecond(Some(
+ (i64::MAX - i64::MAX % 1_000_000) % 86_400_000_000,
+ )),
+ );
let bad_origin =
|| ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(i64::MAX)));
diff --git a/datafusion/sqllogictest/test_files/date_bin_errors.slt b/datafusion/sqllogictest/test_files/date_bin_errors.slt
index 53cba506de..cb26023c46 100644
--- a/datafusion/sqllogictest/test_files/date_bin_errors.slt
+++ b/datafusion/sqllogictest/test_files/date_bin_errors.slt
@@ -73,7 +73,7 @@ select date_bin(
----
NULL
-# Extreme timestamp overflow: source - origin overflows i64 (should return NULL, not panic)
+# source - origin overflows i64 nanoseconds
query P
select date_bin(
interval '1 nanosecond',
@@ -81,7 +81,7 @@ select date_bin(
arrow_cast(-9223372036854775808, 'Timestamp(Nanosecond, None)')
);
----
-NULL
+2262-04-11T23:47:16.854775807
# compute_distance overflow: source at i64::MIN nanoseconds previously panicked
# inside compute_distance; it must return NULL through the SQL execution path
@@ -93,17 +93,17 @@ select date_bin(
----
NULL
-# Source timestamp scaling to nanoseconds overflows: should return NULL, not panic
+# Scaling the source to nanoseconds overflows i64
query P
select date_bin(
interval '1 nanosecond',
- arrow_cast(9223372036854775807, 'Timestamp(Second, None)'),
+ arrow_cast(10000000000, 'Timestamp(Second, None)'),
timestamp '1970-01-01 00:00:00'
);
----
-NULL
+2286-11-20T17:46:40
-# Source timestamp scaling to nanoseconds overflows in array path: should return NULL, not panic
+# Same in the array path
query P
select date_bin(
interval '1 nanosecond',
@@ -111,7 +111,7 @@ select date_bin(
timestamp '1970-01-01 00:00:00'
)
from (
- values (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))
+ values (arrow_cast(10000000000, 'Timestamp(Second, None)'))
) as t(ts);
----
-NULL
+2286-11-20T17:46:40
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index 28a4269488..71118ddcab 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -1728,7 +1728,7 @@ physical_plan
02)--SortExec: TopK(fetch=2), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
-# date_bin remains conservatively unordered because per-row failures become NULL
+# Typed bounds allow date_trunc to propagate ordering through timezone-free date_bin
query TT
EXPLAIN SELECT date_trunc('hour', date_bin(INTERVAL '1 minute', ts)) AS truncated
FROM (
@@ -1750,14 +1750,39 @@ logical_plan
04)------Projection: CAST(column1 AS Timestamp(s)) AS ts
05)--------Values: (Int64(7200)), (Int64(3600))
physical_plan
-01)SortExec: expr=[truncated@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]
-03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
-04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
-05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+01)ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]
+02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Coarse-precision values outside the nanosecond range are binned without
+# overflowing, so date_bin keeps both their values and their ordering.
+query TT
+EXPLAIN SELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (-10000000000::bigint),
+ (0::bigint),
+ (10000000000::bigint)
+ )
+ ORDER BY ts
+ LIMIT 3
+)
+ORDER BY b ASC NULLS LAST
+----
+logical_plan
+01)Sort: b ASC NULLS LAST
+02)--Projection: date_trunc(Utf8("second"), date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 1000000000 }"), ts)) AS b
+03)----Sort: ts ASC NULLS LAST, fetch=3
+04)------Projection: CAST(column1 AS Timestamp(s)) AS ts
+05)--------Values: (Int64(-10000000000) AS (- Int64(10000000000))), (Int64(0)), (Int64(10000000000))
+physical_plan
+01)ProjectionExec: expr=[date_trunc(second, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 1000000000 }, ts@0)) as b]
+02)--SortExec: TopK(fetch=3), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
-# date_bin can return NULL for out-of-range values, so its output does not
-# necessarily preserve the source's NULL ordering.
query P
SELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS b
FROM (
@@ -1772,12 +1797,12 @@ FROM (
)
ORDER BY b ASC NULLS LAST
----
+1653-02-10T06:13:20
1970-01-01T00:00:00
-NULL
-NULL
+2286-11-20T17:46:40
-# An ordered expression can retain its ordering while its range type is unknown.
-# date_bin must not assume that such an input is safe to scale to nanoseconds.
+# The same holds when the source is an ordered expression with an unknown
+# range type.
query P
SELECT date_bin(INTERVAL '1 second', date_trunc('second', ts)) AS b
FROM (
@@ -1792,9 +1817,113 @@ FROM (
)
ORDER BY b ASC NULLS LAST
----
+1653-02-10T06:13:20
1970-01-01T00:00:00
-NULL
-NULL
+2286-11-20T17:46:40
+
+# A negative month stride does not shift every bin by the same amount: here
+# 2023-01-01 bins forward to 2023-02-28, while 2023-01-31 bins to itself. The
+# output is not sorted, so date_bin keeps the final sort for it.
+query TT
+EXPLAIN SELECT date_bin(INTERVAL '-1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (TIMESTAMP '2023-01-01 00:00:00'),
+ (TIMESTAMP '2023-01-31 00:00:00')
+ )
+ ORDER BY ts
+ LIMIT 2
+)
+ORDER BY b
+----
+logical_plan
+01)Sort: b ASC NULLS LAST
+02)--Projection: date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: -1, days: 0, nanoseconds: 0 }"), ts, TimestampNanosecond(1675123200000000000, None)) AS b
+03)----Sort: ts ASC NULLS LAST, fetch=2
+04)------Projection: CAST(column1 AS Timestamp(s)) AS ts
+05)--------Values: (TimestampNanosecond(1672531200000000000, None) AS Utf8("2023-01-01 00:00:00")), (TimestampNanosecond(1675123200000000000, None) AS Utf8("2023-01-31 00:00:00"))
+physical_plan
+01)SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--ProjectionExec: expr=[date_bin(IntervalMonthDayNano { months: -1, days: 0, nanoseconds: 0 }, ts@0, 1675123200000000000) as b]
+03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
+04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query P
+SELECT date_bin(INTERVAL '-1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (TIMESTAMP '2023-01-01 00:00:00'),
+ (TIMESTAMP '2023-01-31 00:00:00')
+ )
+ ORDER BY ts
+ LIMIT 2
+)
+ORDER BY b
+----
+2023-01-31T00:00:00
+2023-02-28T00:00:00
+
+# Same for nanosecond timestamps.
+query P
+SELECT date_bin(INTERVAL '-1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b
+FROM (
+ SELECT column1 AS ts
+ FROM (VALUES
+ (TIMESTAMP '2023-01-01 00:00:00'),
+ (TIMESTAMP '2023-01-31 00:00:00')
+ )
+ ORDER BY ts
+ LIMIT 2
+)
+ORDER BY b
+----
+2023-01-31T00:00:00
+2023-02-28T00:00:00
+
+# A positive month stride is monotonic, so the final sort is removed.
+query TT
+EXPLAIN SELECT date_bin(INTERVAL '1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (TIMESTAMP '2023-01-01 00:00:00'),
+ (TIMESTAMP '2023-01-31 00:00:00')
+ )
+ ORDER BY ts
+ LIMIT 2
+)
+ORDER BY b
+----
+logical_plan
+01)Sort: b ASC NULLS LAST
+02)--Projection: date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 1, days: 0, nanoseconds: 0 }"), ts, TimestampNanosecond(1675123200000000000, None)) AS b
+03)----Sort: ts ASC NULLS LAST, fetch=2
+04)------Projection: CAST(column1 AS Timestamp(s)) AS ts
+05)--------Values: (TimestampNanosecond(1672531200000000000, None) AS Utf8("2023-01-01 00:00:00")), (TimestampNanosecond(1675123200000000000, None) AS Utf8("2023-01-31 00:00:00"))
+physical_plan
+01)ProjectionExec: expr=[date_bin(IntervalMonthDayNano { months: 1, days: 0, nanoseconds: 0 }, ts@0, 1675123200000000000) as b]
+02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query P
+SELECT date_bin(INTERVAL '1 month', ts, TIMESTAMP '2023-01-31 00:00:00') AS b
+FROM (
+ SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
+ FROM (VALUES
+ (TIMESTAMP '2023-01-01 00:00:00'),
+ (TIMESTAMP '2023-01-31 00:00:00')
+ )
+ ORDER BY ts
+ LIMIT 2
+)
+ORDER BY b
+----
+2022-12-31T00:00:00
+2023-01-31T00:00:00
# Default bounds from nested functions retain their resolved output type.
statement ok