Apache DataFusion SQL Query Engine
Update date_bin to support Time32 and Time64 data types (#19341)
Bruce Ritchie4a1f69f9bcf8
diff --git a/datafusion/functions/src/datetime/date_bin.rs b/datafusion/functions/src/datetime/date_bin.rsindex 3fd2fd3fe2..6c67fbad34 100644--- a/datafusion/functions/src/datetime/date_bin.rs+++ b/datafusion/functions/src/datetime/date_bin.rs@@ -24,12 +24,15 @@ use arrow::array::types::{TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,TimestampSecondType,};-use arrow::array::{ArrayRef, PrimitiveArray};-use arrow::datatypes::DataType::{Null, Timestamp, Utf8};+use arrow::array::{ArrayRef, AsArray, PrimitiveArray};+use arrow::datatypes::DataType::{Time32, Time64, Timestamp};use arrow::datatypes::IntervalUnit::{DayTime, MonthDayNano};use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};-use arrow::datatypes::{DataType, TimeUnit};-+use arrow::datatypes::{+ DataType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,+ Time64NanosecondType, TimeUnit,+};+use arrow::temporal_conversions::NANOSECONDS_IN_DAY;use datafusion_common::cast::as_primitive_array;use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err, plan_err};use datafusion_expr::TypeSignature::Exact;@@ -71,6 +74,17 @@ FROM VALUES ('2023-01-01T18:18:18Z'), ('2023-01-03T19:00:03Z') t(time);| 2023-01-03T03:00:00 |+---------------------+2 row(s) fetched.++-- Bin the time into 15 minute intervals starting at 1 min+> SELECT date_bin(interval '15 minutes', time, TIME '00:01:00') as bin+FROM VALUES (TIME '02:18:18'), (TIME '19:00:03') t(time);++----------++| bin |++----------++| 02:16:00 |+| 18:46:00 |++----------++2 row(s) fetched.```"#,argument(name = "interval", description = "Bin interval."),argument(@@ -109,7 +123,7 @@ impl Default for DateBinFunc {impl DateBinFunc {pub fn new() -> Self {let base_sig = |array_type: TimeUnit| {- vec![+ let mut v = vec![Exact(vec![DataType::Interval(MonthDayNano),Timestamp(array_type, None),@@ -146,7 +160,44 @@ impl DateBinFunc {DataType::Interval(DayTime),Timestamp(array_type, Some(TIMEZONE_WILDCARD.into())),]),- ]+ ];++ match array_type {+ Second | Millisecond => {+ v.append(&mut vec![+ Exact(vec![+ DataType::Interval(MonthDayNano),+ Time32(array_type),+ Time32(array_type),+ ]),+ Exact(vec![DataType::Interval(MonthDayNano), Time32(array_type)]),+ Exact(vec![+ DataType::Interval(DayTime),+ Time32(array_type),+ Time32(array_type),+ ]),+ Exact(vec![DataType::Interval(DayTime), Time32(array_type)]),+ ]);+ }+ Microsecond | Nanosecond => {+ v.append(&mut vec![+ Exact(vec![+ DataType::Interval(DayTime),+ Time64(array_type),+ Time64(array_type),+ ]),+ Exact(vec![DataType::Interval(DayTime), Time64(array_type)]),+ Exact(vec![+ DataType::Interval(MonthDayNano),+ Time64(array_type),+ Time64(array_type),+ ]),+ Exact(vec![DataType::Interval(MonthDayNano), Time64(array_type)]),+ ]);+ }+ }++ v};let full_sig = [Nanosecond, Microsecond, Millisecond, Second]@@ -176,13 +227,11 @@ impl ScalarUDFImpl for DateBinFunc {fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {match &arg_types[1] {- Timestamp(Nanosecond, None) | Utf8 | Null => Ok(Timestamp(Nanosecond, None)),- Timestamp(Nanosecond, tz_opt) => Ok(Timestamp(Nanosecond, tz_opt.clone())),- Timestamp(Microsecond, tz_opt) => Ok(Timestamp(Microsecond, tz_opt.clone())),- Timestamp(Millisecond, tz_opt) => Ok(Timestamp(Millisecond, tz_opt.clone())),- Timestamp(Second, tz_opt) => Ok(Timestamp(Second, tz_opt.clone())),+ Timestamp(tu, tz_opt) => Ok(Timestamp(*tu, tz_opt.clone())),+ Time32(tu) => Ok(Time32(*tu)),+ Time64(tu) => Ok(Time64(*tu)),_ => plan_err!(- "The date_bin function can only accept timestamp as the second arg."+ "The date_bin function can only accept timestamp or time as the second arg."),}}@@ -193,11 +242,27 @@ impl ScalarUDFImpl for DateBinFunc {) -> Result<ColumnarValue> {let args = &args.args;if args.len() == 2 {- // Default to unix EPOCH- let origin = ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(- Some(0),- Some("+00:00".into()),- ));+ let origin = match args[1].data_type() {+ Time32(Second) => {+ ColumnarValue::Scalar(ScalarValue::Time32Second(Some(0)))+ }+ Time32(Millisecond) => {+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(Some(0)))+ }+ Time64(Microsecond) => {+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(0)))+ }+ Time64(Nanosecond) => {+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(Some(0)))+ }+ _ => {+ // Default to unix EPOCH+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(+ Some(0),+ Some("+00:00".into()),+ ))+ }+ };date_bin_impl(&args[0], &args[1], &origin)} else if args.len() == 3 {date_bin_impl(&args[0], &args[1], &args[2])@@ -227,6 +292,10 @@ impl ScalarUDFImpl for DateBinFunc {}}+const NANOS_PER_MICRO: i64 = 1_000;+const NANOS_PER_MILLI: i64 = 1_000_000;+const NANOS_PER_SEC: i64 = NANOSECONDS;+enum Interval {Nanoseconds(i64),Months(i64),@@ -306,8 +375,8 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> i64}fn to_utc_date_time(nanos: i64) -> DateTime<Utc> {- let secs = nanos / 1_000_000_000;- let nsec = (nanos % 1_000_000_000) as u32;+ let secs = nanos / NANOS_PER_SEC;+ let nsec = (nanos % NANOS_PER_SEC) as u32;DateTime::from_timestamp(secs, nsec).unwrap()}@@ -370,11 +439,93 @@ fn date_bin_impl(}};- let origin = match origin {- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(v), _)) => *v,+ let (origin, is_time) = match origin {+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(v), _)) => {+ (*v, false)+ }+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(Some(v))) => {+ match stride {+ Interval::Months(m) => {+ if m > 0 {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ Interval::Nanoseconds(ns) => {+ if ns >= NANOSECONDS_IN_DAY {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ }++ (*v as i64 * NANOS_PER_MILLI, true)+ }+ ColumnarValue::Scalar(ScalarValue::Time32Second(Some(v))) => {+ match stride {+ Interval::Months(m) => {+ if m > 0 {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ Interval::Nanoseconds(ns) => {+ if ns >= NANOSECONDS_IN_DAY {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ }++ (*v as i64 * NANOS_PER_SEC, true)+ }+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(Some(v))) => {+ match stride {+ Interval::Months(m) => {+ if m > 0 {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ Interval::Nanoseconds(ns) => {+ if ns >= NANOSECONDS_IN_DAY {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ }++ (*v * NANOS_PER_MICRO, true)+ }+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(Some(v))) => {+ match stride {+ Interval::Months(m) => {+ if m > 0 {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ Interval::Nanoseconds(ns) => {+ if ns >= NANOSECONDS_IN_DAY {+ return exec_err!(+ "DATE_BIN stride for TIME input must be less than 1 day"+ );+ }+ }+ }++ (*v, true)+ }ColumnarValue::Scalar(v) => {return exec_err!(- "DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision but got {}",+ "DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision or a TIME but got {}",v.data_type());}@@ -399,8 +550,8 @@ fn date_bin_impl() -> impl Fn(i64) -> i64 {let scale = match T::UNIT {Nanosecond => 1,- Microsecond => NANOSECONDS / 1_000_000,- Millisecond => NANOSECONDS / 1_000,+ Microsecond => NANOS_PER_MICRO,+ Millisecond => NANOS_PER_MILLI,Second => NANOSECONDS,};move |x: i64| stride_fn(stride, x * scale, origin) / scale@@ -439,7 +590,49 @@ fn date_bin_impl(tz_opt.clone(),))}-+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(v)) => {+ if !is_time {+ return exec_err!("DATE_BIN with Time32 source requires Time32 origin");+ }+ let apply_stride_fn = move |x: i32| {+ let binned_nanos = stride_fn(stride, x as i64 * NANOS_PER_MILLI, origin);+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);+ (nanos / NANOS_PER_MILLI) as i32+ };+ ColumnarValue::Scalar(ScalarValue::Time32Millisecond(v.map(apply_stride_fn)))+ }+ ColumnarValue::Scalar(ScalarValue::Time32Second(v)) => {+ if !is_time {+ return exec_err!("DATE_BIN with Time32 source requires Time32 origin");+ }+ let apply_stride_fn = move |x: i32| {+ let binned_nanos = stride_fn(stride, x as i64 * NANOS_PER_SEC, origin);+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);+ (nanos / NANOS_PER_SEC) as i32+ };+ ColumnarValue::Scalar(ScalarValue::Time32Second(v.map(apply_stride_fn)))+ }+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(v)) => {+ if !is_time {+ return exec_err!("DATE_BIN with Time64 source requires Time64 origin");+ }+ let apply_stride_fn = move |x: i64| {+ let binned_nanos = stride_fn(stride, x, origin);+ binned_nanos % (NANOSECONDS_IN_DAY)+ };+ ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(v.map(apply_stride_fn)))+ }+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(v)) => {+ if !is_time {+ return exec_err!("DATE_BIN with Time64 source requires Time64 origin");+ }+ let apply_stride_fn = move |x: i64| {+ let binned_nanos = stride_fn(stride, x * NANOS_PER_MICRO, origin);+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);+ nanos / NANOS_PER_MICRO+ };+ ColumnarValue::Scalar(ScalarValue::Time64Microsecond(v.map(apply_stride_fn)))+ }ColumnarValue::Array(array) => {fn transform_array_with_stride<T>(origin: i64,@@ -481,9 +674,74 @@ fn date_bin_impl(origin, stride, stride_fn, array, tz_opt,)?}+ Time32(Millisecond) => {+ if !is_time {+ return exec_err!(+ "DATE_BIN with Time32 source requires Time32 origin"+ );+ }+ let array = array.as_primitive::<Time32MillisecondType>();+ let apply_stride_fn = move |x: i32| {+ let binned_nanos =+ stride_fn(stride, x as i64 * NANOS_PER_MILLI, origin);+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);+ (nanos / NANOS_PER_MILLI) as i32+ };+ let array: PrimitiveArray<Time32MillisecondType> =+ array.unary(apply_stride_fn);+ ColumnarValue::Array(Arc::new(array))+ }+ Time32(Second) => {+ if !is_time {+ return exec_err!(+ "DATE_BIN with Time32 source requires Time32 origin"+ );+ }+ let array = array.as_primitive::<Time32SecondType>();+ let apply_stride_fn = move |x: i32| {+ let binned_nanos =+ stride_fn(stride, x as i64 * NANOS_PER_SEC, origin);+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);+ (nanos / NANOS_PER_SEC) as i32+ };+ let array: PrimitiveArray<Time32SecondType> =+ array.unary(apply_stride_fn);+ ColumnarValue::Array(Arc::new(array))+ }+ Time64(Microsecond) => {+ if !is_time {+ return exec_err!(+ "DATE_BIN with Time64 source requires Time64 origin"+ );+ }+ let array = array.as_primitive::<Time64MicrosecondType>();+ let apply_stride_fn = move |x: i64| {+ let binned_nanos = stride_fn(stride, x * NANOS_PER_MICRO, origin);+ let nanos = binned_nanos % (NANOSECONDS_IN_DAY);+ nanos / NANOS_PER_MICRO+ };+ let array: PrimitiveArray<Time64MicrosecondType> =+ array.unary(apply_stride_fn);+ ColumnarValue::Array(Arc::new(array))+ }+ Time64(Nanosecond) => {+ if !is_time {+ return exec_err!(+ "DATE_BIN with Time64 source requires Time64 origin"+ );+ }+ let array = array.as_primitive::<Time64NanosecondType>();+ let apply_stride_fn = move |x: i64| {+ let binned_nanos = stride_fn(stride, x, origin);+ binned_nanos % (NANOSECONDS_IN_DAY)+ };+ let array: PrimitiveArray<Time64NanosecondType> =+ array.unary(apply_stride_fn);+ ColumnarValue::Array(Arc::new(array))+ }_ => {return exec_err!(- "DATE_BIN expects source argument to be a TIMESTAMP but got {}",+ "DATE_BIN expects source argument to be a TIMESTAMP or TIME but got {}",array.data_type());}@@ -491,7 +749,7 @@ fn date_bin_impl(}_ => {return exec_err!(- "DATE_BIN expects source argument to be a TIMESTAMP scalar or array"+ "DATE_BIN expects source argument to be a TIMESTAMP or TIME scalar or array");}})@@ -687,7 +945,7 @@ mod tests {let res = invoke_date_bin_with_args(args, 1, return_field);assert_eq!(res.err().unwrap().strip_backtrace(),- "Execution error: DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision but got Timestamp(µs)"+ "Execution error: DATE_BIN expects origin argument to be a TIMESTAMP with nanosecond precision or a TIME but got Timestamp(µs)");args = vec![diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.sltindex 05d379c110..67f44312e5 100644--- a/datafusion/sqllogictest/test_files/timestamps.slt+++ b/datafusion/sqllogictest/test_files/timestamps.slt@@ -791,6 +791,81 @@ FROM () as t (time, val)group by time;+query D+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '00:00:00')+----+14:30:00++# Supports Month-Day-Nano nanosecond interval+query D+SELECT DATE_BIN(INTERVAL '10 nanoseconds', TIME '14:38:50.000000016', TIME '00:00:00')+----+14:38:50.000000010++# Supports Month-Day-Nano nanosecond interval via fractions+query D+SELECT DATE_BIN(INTERVAL '0.000000010 seconds', TIME '14:38:50.000000016', TIME '00:00:00')+----+14:38:50.000000010++# Supports Month-Day-Nano microsecond interval+query D+SELECT DATE_BIN(INTERVAL '5 microseconds', TIME '14:38:50.000006', TIME '00:00:00')+----+14:38:50.000005++# stride by 7 days+query error DataFusion error: Execution error: DATE_BIN stride for TIME input must be less than 1 day+SELECT DATE_BIN(INTERVAL '7 days', TIME '14:38:50', TIME '00:00:00')++# stride by 25 hours+query error DataFusion error: Execution error: DATE_BIN stride for TIME input must be less than 1 day+SELECT DATE_BIN(INTERVAL '25 hours', TIME '14:38:50', TIME '00:00:00')++# stride by 23 hours, 59 minutes 59 seconds+query D+SELECT DATE_BIN(INTERVAL '23 hours 59 minutes 59 seconds', TIME '14:38:50', TIME '00:00:00')+----+00:00:00++# mixed types+query error Failed to coerce arguments to satisfy a call to 'date_bin' function:+SELECT DATE_BIN(INTERVAL '23 hours', TIME '14:38:50', TIMESTAMP '2022-08-03 14:38:50.000000006Z')++# mixed types+query error Failed to coerce arguments to satisfy a call to 'date_bin' function:+SELECT DATE_BIN(INTERVAL '23 hours', TIMESTAMP '2022-08-03 14:38:50.000000006Z', TIME '14:38:50')++# Can coerce all string arguments+query D+SELECT DATE_BIN('15 minutes', '14:38:50'::time, '00:00:00'::time)+----+14:30:00++# Call in two arguments (should be the same as the above query)+query B+SELECT DATE_BIN('15 minutes', '14:38:50'::time) = DATE_BIN('15 minutes', '14:38:50'::time, '00:00:00'::time)+----+true++# Shift forward by 5 minutes+query D+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '00:05:00')+----+14:35:00++# Shift backward by 5 minutes+query D+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '23:55:00')+----+14:25:00++# origin after source, TIME in previous bucket+query D+SELECT DATE_BIN(INTERVAL '15 minutes', TIME '14:38:50', TIME '14:40:00')+----+14:25:00+query PSELECT DATE_BIN(INTERVAL '15 minutes', TIMESTAMP '2022-08-03 14:38:50Z', TIMESTAMP '1970-01-01T00:00:00Z')----diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex ce7ea85a81..3f0939d00d 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -2488,6 +2488,17 @@ FROM VALUES ('2023-01-01T18:18:18Z'), ('2023-01-03T19:00:03Z') t(time);| 2023-01-03T03:00:00 |+---------------------+2 row(s) fetched.++-- Bin the time into 15 minute intervals starting at 1 min+> SELECT date_bin(interval '15 minutes', time, TIME '00:01:00') as bin+FROM VALUES (TIME '02:18:18'), (TIME '19:00:03') t(time);++----------++| bin |++----------++| 02:16:00 |+| 18:46:00 |++----------++2 row(s) fetched.```### `date_format`