Apache DataFusion SQL Query Engine
Refactor `to_local_time()` signature away from user_defined (#18707)
Jeffrey Vo473cbdab83e0
diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rsindex 6e0a150b0a..0495a4841f 100644--- a/datafusion/functions/src/datetime/to_local_time.rs+++ b/datafusion/functions/src/datetime/to_local_time.rs@@ -20,7 +20,7 @@ use std::ops::Add;use std::sync::Arc;use arrow::array::timezone::Tz;-use arrow::array::{Array, ArrayRef, PrimitiveBuilder};+use arrow::array::{ArrayRef, PrimitiveBuilder};use arrow::datatypes::DataType::Timestamp;use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};use arrow::datatypes::{@@ -31,11 +31,12 @@ use chrono::{DateTime, MappedLocalTime, Offset, TimeDelta, TimeZone, Utc};use datafusion_common::cast::as_primitive_array;use datafusion_common::{- exec_err, internal_datafusion_err, plan_err, utils::take_function_args, Result,+ exec_err, internal_datafusion_err, internal_err, utils::take_function_args, Result,ScalarValue,};use datafusion_expr::{- ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass,+ Volatility,};use datafusion_macros::user_doc;@@ -111,133 +112,163 @@ impl Default for ToLocalTimeFunc {impl ToLocalTimeFunc {pub fn new() -> Self {Self {- signature: Signature::user_defined(Volatility::Immutable),+ signature: Signature::coercible(+ vec![Coercion::new_exact(TypeSignatureClass::Timestamp)],+ Volatility::Immutable,+ ),}}+}- fn to_local_time(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {- let [time_value] = take_function_args(self.name(), args)?;+impl ScalarUDFImpl for ToLocalTimeFunc {+ fn as_any(&self) -> &dyn Any {+ self+ }- let arg_type = time_value.data_type();- match arg_type {- Timestamp(_, None) => {- // if no timezone specified, just return the input- Ok(time_value.clone())- }- // If has timezone, adjust the underlying time value. The current time value- // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore,- // we need to adjust the time value to the local time. See [`adjust_to_local_time`]- // for more details.- //- // Then remove the timezone in return type, i.e. return None- Timestamp(_, Some(timezone)) => {- let tz: Tz = timezone.parse()?;-- match time_value {- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(- Some(ts),- Some(_),- )) => {- let adjusted_ts =- adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?;- Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(- Some(adjusted_ts),- None,- )))- }- ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(- Some(ts),- Some(_),- )) => {- let adjusted_ts =- adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?;- Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(- Some(adjusted_ts),- None,- )))- }- ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(- Some(ts),- Some(_),- )) => {- let adjusted_ts =- adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?;- Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(- Some(adjusted_ts),- None,- )))- }- ColumnarValue::Scalar(ScalarValue::TimestampSecond(- Some(ts),- Some(_),- )) => {- let adjusted_ts =- adjust_to_local_time::<TimestampSecondType>(*ts, tz)?;- Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond(- Some(adjusted_ts),- None,- )))- }- ColumnarValue::Array(array) => {- fn transform_array<T: ArrowTimestampType>(- array: &ArrayRef,- tz: Tz,- ) -> Result<ColumnarValue> {- let mut builder = PrimitiveBuilder::<T>::new();-- let primitive_array = as_primitive_array::<T>(array)?;- for ts_opt in primitive_array.iter() {- match ts_opt {- None => builder.append_null(),- Some(ts) => {- let adjusted_ts: i64 =- adjust_to_local_time::<T>(ts, tz)?;- builder.append_value(adjusted_ts)- }- }- }-- Ok(ColumnarValue::Array(Arc::new(builder.finish())))- }-- match array.data_type() {- Timestamp(_, None) => {- // if no timezone specified, just return the input- Ok(time_value.clone())- }- Timestamp(Nanosecond, Some(_)) => {- transform_array::<TimestampNanosecondType>(array, tz)- }- Timestamp(Microsecond, Some(_)) => {- transform_array::<TimestampMicrosecondType>(array, tz)- }- Timestamp(Millisecond, Some(_)) => {- transform_array::<TimestampMillisecondType>(array, tz)- }- Timestamp(Second, Some(_)) => {- transform_array::<TimestampSecondType>(array, tz)- }- _ => {- exec_err!("to_local_time function requires timestamp argument in array, got {:?}", array.data_type())- }- }- }- _ => {- exec_err!(- "to_local_time function requires timestamp argument, got {:?}",- time_value.data_type()- )- }- }- }- _ => {- exec_err!(- "to_local_time function requires timestamp argument, got {:?}",- arg_type- )+ fn name(&self) -> &str {+ "to_local_time"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {+ match &arg_types[0] {+ DataType::Null => Ok(Timestamp(Nanosecond, None)),+ Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)),+ dt => internal_err!(+ "The to_local_time function can only accept timestamp as the arg, got {dt}"+ ),+ }+ }++ fn invoke_with_args(+ &self,+ args: datafusion_expr::ScalarFunctionArgs,+ ) -> Result<ColumnarValue> {+ let [time_value] = take_function_args(self.name(), &args.args)?;+ to_local_time(time_value)+ }++ fn documentation(&self) -> Option<&Documentation> {+ self.doc()+ }+}++fn transform_array<T: ArrowTimestampType>(+ array: &ArrayRef,+ tz: Tz,+) -> Result<ColumnarValue> {+ let primitive_array = as_primitive_array::<T>(array)?;+ let mut builder = PrimitiveBuilder::<T>::with_capacity(primitive_array.len());+ for ts_opt in primitive_array.iter() {+ match ts_opt {+ None => builder.append_null(),+ Some(ts) => {+ let adjusted_ts: i64 = adjust_to_local_time::<T>(ts, tz)?;+ builder.append_value(adjusted_ts)}}}++ Ok(ColumnarValue::Array(Arc::new(builder.finish())))+}++fn to_local_time(time_value: &ColumnarValue) -> Result<ColumnarValue> {+ let arg_type = time_value.data_type();++ let tz: Tz = match &arg_type {+ Timestamp(_, Some(timezone)) => timezone.parse()?,+ Timestamp(_, None) => {+ // if no timezone specified, just return the input+ return Ok(time_value.clone());+ }+ DataType::Null => {+ return Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(+ None, None,+ )));+ }+ dt => {+ return internal_err!(+ "to_local_time function requires timestamp argument, got {dt}"+ )+ }+ };++ // If has timezone, adjust the underlying time value. The current time value+ // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore,+ // we need to adjust the time value to the local time. See [`adjust_to_local_time`]+ // for more details.+ //+ // Then remove the timezone in return type, i.e. return None+ match time_value {+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, Some(_))) => Ok(+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, None)),+ ),+ ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, Some(_))) => Ok(+ ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, None)),+ ),+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, Some(_))) => Ok(+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None)),+ ),+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, Some(_))) => Ok(+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, None)),+ ),+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(ts), Some(_))) => {+ let adjusted_ts = adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?;+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(+ Some(adjusted_ts),+ None,+ )))+ }+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(ts), Some(_))) => {+ let adjusted_ts = adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?;+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(+ Some(adjusted_ts),+ None,+ )))+ }+ ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(Some(ts), Some(_))) => {+ let adjusted_ts = adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?;+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(+ Some(adjusted_ts),+ None,+ )))+ }+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(ts), Some(_))) => {+ let adjusted_ts = adjust_to_local_time::<TimestampSecondType>(*ts, tz)?;+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond(+ Some(adjusted_ts),+ None,+ )))+ }+ ColumnarValue::Array(array)+ if matches!(array.data_type(), Timestamp(Nanosecond, Some(_))) =>+ {+ transform_array::<TimestampNanosecondType>(array, tz)+ }+ ColumnarValue::Array(array)+ if matches!(array.data_type(), Timestamp(Microsecond, Some(_))) =>+ {+ transform_array::<TimestampMicrosecondType>(array, tz)+ }+ ColumnarValue::Array(array)+ if matches!(array.data_type(), Timestamp(Millisecond, Some(_))) =>+ {+ transform_array::<TimestampMillisecondType>(array, tz)+ }+ ColumnarValue::Array(array)+ if matches!(array.data_type(), Timestamp(Second, Some(_))) =>+ {+ transform_array::<TimestampSecondType>(array, tz)+ }+ _ => {+ internal_err!(+ "to_local_time function requires timestamp argument, got {arg_type}"+ )+ }+ }}/// This function converts a timestamp with a timezone to a timestamp without a timezone.@@ -343,68 +374,6 @@ fn adjust_to_local_time<T: ArrowTimestampType>(ts: i64, tz: Tz) -> Result<i64> {}}-impl ScalarUDFImpl for ToLocalTimeFunc {- fn as_any(&self) -> &dyn Any {- self- }-- fn name(&self) -> &str {- "to_local_time"- }-- fn signature(&self) -> &Signature {- &self.signature- }-- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- let [time_value] = take_function_args(self.name(), arg_types)?;-- match time_value {- Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)),- _ => exec_err!(- "The to_local_time function can only accept timestamp as the arg, got {:?}", time_value- )- }- }-- fn invoke_with_args(- &self,- args: datafusion_expr::ScalarFunctionArgs,- ) -> Result<ColumnarValue> {- let [time_value] = take_function_args(self.name(), args.args)?;-- self.to_local_time(std::slice::from_ref(&time_value))- }-- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {- if arg_types.len() != 1 {- return plan_err!(- "to_local_time function requires 1 argument, got {:?}",- arg_types.len()- );- }-- let first_arg = arg_types[0].clone();- match &first_arg {- DataType::Null => Ok(vec![Timestamp(Nanosecond, None)]),- Timestamp(Nanosecond, timezone) => {- Ok(vec![Timestamp(Nanosecond, timezone.clone())])- }- Timestamp(Microsecond, timezone) => {- Ok(vec![Timestamp(Microsecond, timezone.clone())])- }- Timestamp(Millisecond, timezone) => {- Ok(vec![Timestamp(Millisecond, timezone.clone())])- }- Timestamp(Second, timezone) => Ok(vec![Timestamp(Second, timezone.clone())]),- _ => plan_err!("The to_local_time function can only accept Timestamp as the arg got {first_arg}"),- }- }- fn documentation(&self) -> Option<&Documentation> {- self.doc()- }-}-#[cfg(test)]mod tests {use std::sync::Arc;diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.sltindex 5c365b056d..3a1a26257b 100644--- a/datafusion/sqllogictest/test_files/timestamps.slt+++ b/datafusion/sqllogictest/test_files/timestamps.slt@@ -2250,7 +2250,7 @@ SET TIME ZONE = '+05:00'statement okCREATE TABLE foo (time TIMESTAMPTZ) AS VALUES- ('2020-01-01T00:00:00+05:00'),+ ('2020-01-01T00:00:00+05:00'),('2020-01-01T01:00:00+05:00'),('2020-01-01T02:00:00+05:00'),('2020-01-01T03:00:00+05:00')@@ -2335,17 +2335,17 @@ NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:5# verify timestamp syntax styles are consistentquery BBBBBBBBBBBBBSELECT to_timestamp(null) is null as c1,- null::timestamp is null as c2,- cast(null as timestamp) is null as c3,- to_timestamp(0) = 0::timestamp as c4,- to_timestamp(1926632005) = 1926632005::timestamp as c5,- to_timestamp(1) = 1::timestamp as c6,- to_timestamp(-1) = -1::timestamp as c7,+ null::timestamp is null as c2,+ cast(null as timestamp) is null as c3,+ to_timestamp(0) = 0::timestamp as c4,+ to_timestamp(1926632005) = 1926632005::timestamp as c5,+ to_timestamp(1) = 1::timestamp as c6,+ to_timestamp(-1) = -1::timestamp as c7,to_timestamp(0-1) = (0-1)::timestamp as c8,- to_timestamp(0) = cast(0 as timestamp) as c9,- to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10,- to_timestamp(1) = cast(1 as timestamp) as c11,- to_timestamp(-1) = cast(-1 as timestamp) as c12,+ to_timestamp(0) = cast(0 as timestamp) as c9,+ to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10,+ to_timestamp(1) = cast(1 as timestamp) as c11,+ to_timestamp(-1) = cast(-1 as timestamp) as c12,to_timestamp(0-1) = cast(0-1 as timestamp) as c13----true true true true true true true true true true true true true@@ -2358,10 +2358,10 @@ Timestamp(ns) Timestamp(ns) Timestamp(ns)# verify timestamp output types using timestamp literal syntaxquery BBBBBB-SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1,+SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1,arrow_typeof(to_timestamp(null)) = arrow_typeof(null::timestamp) as c2,arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof('2023-01-10 12:34:56.000'::timestamp) as c3,- arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4,+ arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4,arrow_typeof(to_timestamp(null)) = arrow_typeof(cast(null as timestamp)) as c5,arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp)) as c6----@@ -2605,13 +2605,13 @@ drop table table_a##########statement ok-create table table_a (ts timestamp) as values- ('2020-09-08T11:42:29Z'::timestamp),+create table table_a (ts timestamp) as values+ ('2020-09-08T11:42:29Z'::timestamp),('2020-09-08T12:42:29Z'::timestamp),('2020-09-08T13:42:29Z'::timestamp)statement ok-create table table_b (ts timestamp) as values+create table table_b (ts timestamp) as values('2020-09-08T11:42:29.190Z'::timestamp),('2020-09-08T13:42:29.190Z'::timestamp),('2020-09-08T12:42:29.190Z'::timestamp)@@ -2733,8 +2733,8 @@ statement okdrop table t1statement ok-create table table_a (val int, ts1 timestamp, ts2 timestamp) as values- (1, '2018-07-01T06:00:00'::timestamp, '2018-07-01T07:00:00'::timestamp),+create table table_a (val int, ts1 timestamp, ts2 timestamp) as values+ (1, '2018-07-01T06:00:00'::timestamp, '2018-07-01T07:00:00'::timestamp),(2, '2018-07-01T07:00:00'::timestamp, '2018-07-01T08:00:00'::timestamp)query I?@@ -3038,7 +3038,7 @@ NULLquery TSELECT to_char(date_column, '%Y-%m-%d')-FROM (VALUES+FROM (VALUES(DATE '2020-09-01'),(NULL)) AS t(date_column);@@ -3048,7 +3048,7 @@ NULLquery TSELECT to_char(date_column, '%Y-%m-%d')-FROM (VALUES+FROM (VALUES(NULL),(DATE '2020-09-01')) AS t(date_column);@@ -3249,7 +3249,7 @@ statement errorselect to_local_time('2024-04-01T00:00:20Z'::timestamp, 'some string');# invalid argument data type-statement error The to_local_time function can only accept Timestamp as the arg got Utf8+statement error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Timestamp but received NativeType::String, DataType: Utf8select to_local_time('2024-04-01T00:00:20Z');# invalid timezone@@ -3277,6 +3277,13 @@ select to_local_time(NULL);----NULL+query PT+select+ to_local_time(arrow_cast(null, 'Timestamp(s, "Asia/Tokyo")')),+ arrow_typeof(to_local_time(arrow_cast(null, 'Timestamp(s, "Asia/Tokyo")')));+----+NULL Timestamp(s)+query PTPTselecttime,