Skip to content

Apache DataFusion SQL Query Engine

Update date_bin to support Time32 and Time64 data types (#19341)

Bruce Ritchie4a1f69f9bcf8
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rs
index 3fd2fd3fe2..6c67fbad34 100644
--- a/datafusion/functions/src/datetime/date_bin.rs
+++ b/datafusion/functions/src/datetime/date_bin.rs
@@ -24,12 +24,15 @@ use arrow::array::types::{
TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
TimestampSecondType,
};
-use arrow::array::{ArrayRef, PrimitiveArray};
-use arrow::datatypes::DataType::{Null, Timestamp, Utf8};
+use arrow::array::{ArrayRef, AsArray, PrimitiveArray};
+use arrow::datatypes::DataType::{Time32, Time64, Timestamp};
use arrow::datatypes::IntervalUnit::{DayTime, MonthDayNano};
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
-use arrow::datatypes::{DataType, TimeUnit};
-
+use arrow::datatypes::{
+ DataType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
+ Time64NanosecondType, TimeUnit,
+};
+use arrow::temporal_conversions::NANOSECONDS_IN_DAY;
use datafusion_common::cast::as_primitive_array;
use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err, plan_err};
use datafusion_expr::TypeSignature::Exact;
@@ -71,6 +74,17 @@ FROM VALUES ('2023-01-01T18:18:18Z'), ('2023-01-03T19:00:03Z') t(time);
| 2023-01-03T03:00:00 |
+---------------------+
2 row(s) fetched.
+
+-- Bin the time into 15 minute intervals starting at 1 min
+> SELECT date_bin(interval '15 minutes', time, TIME '00:01:00') as bin
+FROM VALUES (TIME '02:18:18'), (TIME '19:00:03') t(time);
++----------+
+| bin |
++----------+
+| 02:16:00 |
+| 18:46:00 |
++----------+
+2 row(s) fetched.
```"#,
argument(name = "interval", description = "Bin interval."),
argument(
@@ -109,7 +123,7 @@ impl Default for DateBinFunc {
impl DateBinFunc {
pub fn new() -> Self {
let base_sig = |array_type: TimeUnit| {
- vec![
+ let mut v = vec![
Exact(vec![
DataType::Interval(MonthDayNano),
Timestamp(array_type, None),
@@ -146,7 +160,44 @@ impl DateBinFunc {
DataType::Interval(DayTime),
Timestamp(array_type, Some(TIMEZONE_WILDCARD.into())),
]),
- ]
+ ];
+
+ match array_type {
+ Second | Millisecond => {
+ v.append(&mut vec![
+ Exact(vec![
+ DataType::Interval(MonthDayNano),
+ Time32(array_type),
+ Time32(array_type),
+ ]),
+ Exact(vec![DataType::Interval(MonthDayNano), Time32(array_type)]),
+ Exact(vec![
+ DataType::Interval(DayTime),
+ Time32(array_type),
+ Time32(array_type),
+ ]),
+ Exact(vec![DataType::Interval(DayTime), Time32(array_type)]),
+ ]);
+ }
+ Microsecond | Nanosecond => {
+ v.append(&mut vec![
+ Exact(vec![
+ DataType::Interval(DayTime),
+ Time64(array_type),
+ Time64(array_type),
+ ]),
+ Exact(vec![DataType::Interval(DayTime), Time64(array_type)]),
+ Exact(vec![
+ DataType::Interval(MonthDayNano),
+ Time64(array_type),
+ Time64(array_type),
+ ]),
+ Exact(vec![DataType::Interval(MonthDayNano), Time64(array_type)]),
+ ]);
+ }
+ }
+
+ v
};
let full_sig = [Nanosecond, Microsecond, Millisecond, Second]
@@ -176,13 +227,11 @@ impl ScalarUDFImpl for DateBinFunc {
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
match &arg_types[1] {
- Timestamp(Nanosecond, None) | Utf8 | Null => Ok(Timestamp(Nanosecond, None)),
- Timestamp(Nanosecond, tz_opt) => Ok(Timestamp(Nanosecond, tz_opt.clone())),
- Timestamp(Microsecond, tz_opt) => Ok(Timestamp(Microsecond, tz_opt.clone())),
- Timestamp(Millisecond, tz_opt) => Ok(Timestamp(Millisecond, tz_opt.clone())),
- Timestamp(Second, tz_opt) => Ok(Timestamp(Second, tz_opt.clone())),
+ Timestamp(tu, tz_opt) => Ok(Timestamp(*tu, tz_opt.clone())),
+ Time32(tu) => Ok(Time32(*tu)),
+ Time64(tu) => Ok(Time64(*tu)),
_ => plan_err!(
- "The date_bin function can only accept timestamp as the second arg."
+ "The date_bin function can only accept timestamp or time as the second arg."
),
}
}
@@ -193,11 +242,27 @@ impl ScalarUDFImpl for DateBinFunc {
) -> Result<ColumnarValue> {
let args = &args.args;
if args.len() == 2 {
- // Default to unix EPOCH
- let origin = ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
- Some(0),
- Some("+00:00".into()),
- ));
+ let origin = match args[1].data_type() {
+ Time32(Second) => {
+ ColumnarValue::Scalar(ScalarValue::Time32Second(Some(0)))
+ }
+ Time32(Millisecond) => {
+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(Some(0)))
+ }
+ Time64(Microsecond) => {
+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(0)))
+ }
+ Time64(Nanosecond) => {
+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(Some(0)))
+ }
+ _ => {
+ // Default to unix EPOCH
+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ Some(0),
+ Some("+00:00".into()),
+ ))
+ }
+ };
date_bin_impl(&args[0], &args[1], &origin)
} else if args.len() == 3 {
date_bin_impl(&args[0], &args[1], &args[2])
@@ -227,6 +292,10 @@ impl ScalarUDFImpl for DateBinFunc {
}
}
+const NANOS_PER_MICRO: i64 = 1_000;
+const NANOS_PER_MILLI: i64 = 1_000_000;
+const NANOS_PER_SEC: i64 = NANOSECONDS;
+
enum Interval {
Nanoseconds(i64),
Months(i64),
@@ -306,8 +375,8 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> i64
}
fn to_utc_date_time(nanos: i64) -> DateTime<Utc> {
- let secs = nanos / 1_000_000_000;
- let nsec = (nanos % 1_000_000_000) as u32;
+ let secs = nanos / NANOS_PER_SEC;
+ let nsec = (nanos % NANOS_PER_SEC) as u32;
DateTime::from_timestamp(secs, nsec).unwrap()
}
@@ -370,11 +439,93 @@ fn date_bin_impl(
}
};
- let origin = match origin {
- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(v), _)) => *v,
+ let (origin, is_time) = match origin {
+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(v), _)) => {
+ (*v, false)
+ }
+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(Some(v))) => {
+ match stride {
+ Interval::Months(m) => {
+ if m > 0 {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ Interval::Nanoseconds(ns) => {
+ if ns >= NANOSECONDS_IN_DAY {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ }
+
+ (*v as i64 * NANOS_PER_MILLI, true)
+ }
+ ColumnarValue::Scalar(ScalarValue::Time32Second(Some(v))) => {
+ match stride {
+ Interval::Months(m) => {
+ if m > 0 {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ Interval::Nanoseconds(ns) => {
+ if ns >= NANOSECONDS_IN_DAY {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ }
+
+ (*v as i64 * NANOS_PER_SEC, true)
+ }
+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(v))) => {
+ match stride {
+ Interval::Months(m) => {
+ if m > 0 {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ Interval::Nanoseconds(ns) => {
+ if ns >= NANOSECONDS_IN_DAY {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ }
+
+ (*v * NANOS_PER_MICRO, true)
+ }
+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(Some(v))) => {
+ match stride {
+ Interval::Months(m) => {
+ if m > 0 {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ Interval::Nanoseconds(ns) => {
+ if ns >= NANOSECONDS_IN_DAY {
+ return exec_err!(
+ "DATE_BIN stride for TIME input must be less than 1 day"
+ );
+ }
+ }
+ }
+
+ (*v, true)
+ }
ColumnarValue::Scalar(v) => {
return exec_err!(
- "DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision but got {}",
+ "DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision or a TIME but got {}",
v.data_type()
);
}
@@ -399,8 +550,8 @@ fn date_bin_impl(
) -> impl Fn(i64) -> i64 {
let scale = match T::UNIT {
Nanosecond => 1,
- Microsecond => NANOSECONDS / 1_000_000,
- Millisecond => NANOSECONDS / 1_000,
+ Microsecond => NANOS_PER_MICRO,
+ Millisecond => NANOS_PER_MILLI,
Second => NANOSECONDS,
};
move |x: i64| stride_fn(stride, x * scale, origin) / scale
@@ -439,7 +590,49 @@ fn date_bin_impl(
tz_opt.clone(),
))
}
-
+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(v)) => {
+ if !is_time {
+ return exec_err!("DATE_BIN with Time32 source requires Time32 origin");
+ }
+ let apply_stride_fn = move |x: i32| {
+ let binned_nanos = stride_fn(stride, x as i64 * NANOS_PER_MILLI, origin);
+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
+ (nanos / NANOS_PER_MILLI) as i32
+ };
+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(v.map(apply_stride_fn)))
+ }
+ ColumnarValue::Scalar(ScalarValue::Time32Second(v)) => {
+ if !is_time {
+ return exec_err!("DATE_BIN with Time32 source requires Time32 origin");
+ }
+ let apply_stride_fn = move |x: i32| {
+ let binned_nanos = stride_fn(stride, x as i64 * NANOS_PER_SEC, origin);
+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
+ (nanos / NANOS_PER_SEC) as i32
+ };
+ ColumnarValue::Scalar(ScalarValue::Time32Second(v.map(apply_stride_fn)))
+ }
+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(v)) => {
+ if !is_time {
+ return exec_err!("DATE_BIN with Time64 source requires Time64 origin");
+ }
+ let apply_stride_fn = move |x: i64| {
+ let binned_nanos = stride_fn(stride, x, origin);
+ binned_nanos % (NANOSECONDS_IN_DAY)
+ };
+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(v.map(apply_stride_fn)))
+ }
+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(v)) => {
+ if !is_time {
+ return exec_err!("DATE_BIN with Time64 source requires Time64 origin");
+ }
+ let apply_stride_fn = move |x: i64| {
+ let binned_nanos = stride_fn(stride, x * NANOS_PER_MICRO, origin);
+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
+ nanos / NANOS_PER_MICRO
+ };
+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(v.map(apply_stride_fn)))
+ }
ColumnarValue::Array(array) => {
fn transform_array_with_stride<T>(
origin: i64,
@@ -481,9 +674,74 @@ fn date_bin_impl(
origin, stride, stride_fn, array, tz_opt,
)?
}
+ Time32(Millisecond) => {
+ if !is_time {
+ return exec_err!(
+ "DATE_BIN with Time32 source requires Time32 origin"
+ );
+ }
+ let array = array.as_primitive::<Time32MillisecondType>();
+ let apply_stride_fn = move |x: i32| {
+ let binned_nanos =
+ stride_fn(stride, x as i64 * NANOS_PER_MILLI, origin);
+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
+ (nanos / NANOS_PER_MILLI) as i32
+ };
+ let array: PrimitiveArray<Time32MillisecondType> =
+ array.unary(apply_stride_fn);
+ ColumnarValue::Array(Arc::new(array))
+ }
+ Time32(Second) => {
+ if !is_time {
+ return exec_err!(
+ "DATE_BIN with Time32 source requires Time32 origin"
+ );
+ }
+ let array = array.as_primitive::<Time32SecondType>();
+ let apply_stride_fn = move |x: i32| {
+ let binned_nanos =
+ stride_fn(stride, x as i64 * NANOS_PER_SEC, origin);
+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
+ (nanos / NANOS_PER_SEC) as i32
+ };
+ let array: PrimitiveArray<Time32SecondType> =
+ array.unary(apply_stride_fn);
+ ColumnarValue::Array(Arc::new(array))
+ }
+ Time64(Microsecond) => {
+ if !is_time {
+ return exec_err!(
+ "DATE_BIN with Time64 source requires Time64 origin"
+ );
+ }
+ let array = array.as_primitive::<Time64MicrosecondType>();
+ let apply_stride_fn = move |x: i64| {
+ let binned_nanos = stride_fn(stride, x * NANOS_PER_MICRO, origin);
+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);
+ nanos / NANOS_PER_MICRO
+ };
+ let array: PrimitiveArray<Time64MicrosecondType> =
+ array.unary(apply_stride_fn);
+ ColumnarValue::Array(Arc::new(array))
+ }
+ Time64(Nanosecond) => {
+ if !is_time {
+ return exec_err!(
+ "DATE_BIN with Time64 source requires Time64 origin"
+ );
+ }
+ let array = array.as_primitive::<Time64NanosecondType>();
+ let apply_stride_fn = move |x: i64| {
+ let binned_nanos = stride_fn(stride, x, origin);
+ binned_nanos % (NANOSECONDS_IN_DAY)
+ };
+ let array: PrimitiveArray<Time64NanosecondType> =
+ array.unary(apply_stride_fn);
+ ColumnarValue::Array(Arc::new(array))
+ }
_ => {
return exec_err!(
- "DATE_BIN expects source argument to be a TIMESTAMP but got {}",
+ "DATE_BIN expects source argument to be a TIMESTAMP or TIME but got {}",
array.data_type()
);
}
@@ -491,7 +749,7 @@ fn date_bin_impl(
}
_ => {
return exec_err!(
- "DATE_BIN expects source argument to be a TIMESTAMP scalar or array"
+ "DATE_BIN expects source argument to be a TIMESTAMP or TIME scalar or array"
);
}
})
@@ -687,7 +945,7 @@ mod tests {
let res = invoke_date_bin_with_args(args, 1, return_field);
assert_eq!(
res.err().unwrap().strip_backtrace(),
- "Execution error: DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision but got Timestamp(µs)"
+ "Execution error: DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision or a TIME but got Timestamp(µs)"
);
args = vec![
diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt
index 05d379c110..67f44312e5 100644
--- a/datafusion/sqllogictest/test_files/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -791,6 +791,81 @@ FROM (
) as t (time, val)
group by time;
+query D
+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '00:00:00')
+----
+14:30:00
+
+# Supports Month-Day-Nano nanosecond interval
+query D
+SELECT DATE_BIN(INTERVAL '10 nanoseconds', TIME '14:38:50.000000016', TIME '00:00:00')
+----
+14:38:50.000000010
+
+# Supports Month-Day-Nano nanosecond interval via fractions
+query D
+SELECT DATE_BIN(INTERVAL '0.000000010 seconds', TIME '14:38:50.000000016', TIME '00:00:00')
+----
+14:38:50.000000010
+
+# Supports Month-Day-Nano microsecond interval
+query D
+SELECT DATE_BIN(INTERVAL '5 microseconds', TIME '14:38:50.000006', TIME '00:00:00')
+----
+14:38:50.000005
+
+# stride by 7 days
+query error DataFusion error: Execution error: DATE_BIN stride for TIME input must be less than 1 day
+SELECT DATE_BIN(INTERVAL '7 days', TIME '14:38:50', TIME '00:00:00')
+
+# stride by 25 hours
+query error DataFusion error: Execution error: DATE_BIN stride for TIME input must be less than 1 day
+SELECT DATE_BIN(INTERVAL '25 hours', TIME '14:38:50', TIME '00:00:00')
+
+# stride by 23 hours, 59 minutes 59 seconds
+query D
+SELECT DATE_BIN(INTERVAL '23 hours 59 minutes 59 seconds', TIME '14:38:50', TIME '00:00:00')
+----
+00:00:00
+
+# mixed types
+query error Failed to coerce arguments to satisfy a call to 'date_bin' function:
+SELECT DATE_BIN(INTERVAL '23 hours', TIME '14:38:50', TIMESTAMP '2022-08-03 14:38:50.000000006Z')
+
+# mixed types
+query error Failed to coerce arguments to satisfy a call to 'date_bin' function:
+SELECT DATE_BIN(INTERVAL '23 hours', TIMESTAMP '2022-08-03 14:38:50.000000006Z', TIME '14:38:50')
+
+# Can coerce all string arguments
+query D
+SELECT DATE_BIN('15 minutes', '14:38:50'::time, '00:00:00'::time)
+----
+14:30:00
+
+# Call in two arguments (should be the same as the above query)
+query B
+SELECT DATE_BIN('15 minutes', '14:38:50'::time) = DATE_BIN('15 minutes', '14:38:50'::time, '00:00:00'::time)
+----
+true
+
+# Shift forward by 5 minutes
+query D
+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '00:05:00')
+----
+14:35:00
+
+# Shift backward by 5 minutes
+query D
+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '23:55:00')
+----
+14:25:00
+
+# origin after source, TIME in previous bucket
+query D
+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '14:40:00')
+----
+14:25:00
+
query P
SELECT DATE_BIN(INTERVAL '15 minutes', TIMESTAMP '2022-08-03 14:38:50Z', TIMESTAMP '1970-01-01T00:00:00Z')
----
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index ce7ea85a81..3f0939d00d 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -2488,6 +2488,17 @@ FROM VALUES ('2023-01-01T18:18:18Z'), ('2023-01-03T19:00:03Z') t(time);
| 2023-01-03T03:00:00 |
+---------------------+
2 row(s) fetched.
+
+-- Bin the time into 15 minute intervals starting at 1 min
+> SELECT date_bin(interval '15 minutes', time, TIME '00:01:00') as bin
+FROM VALUES (TIME '02:18:18'), (TIME '19:00:03') t(time);
++----------+
+| bin |
++----------+
+| 02:16:00 |
+| 18:46:00 |
++----------+
+2 row(s) fetched.
```
### `date_format`