Skip to content

Apache DataFusion SQL Query Engine

Refactor `to_local_time()` signature away from user_defined (#18707)

Jeffrey Vo473cbdab83e0
diff --git a/datafusion/functions/src/datetime/to_local_time.rs b/datafusion/functions/src/datetime/to_local_time.rs
index 6e0a150b0a..0495a4841f 100644
--- a/datafusion/functions/src/datetime/to_local_time.rs
+++ b/datafusion/functions/src/datetime/to_local_time.rs
@@ -20,7 +20,7 @@ use std::ops::Add;
use std::sync::Arc;
use arrow::array::timezone::Tz;
-use arrow::array::{Array, ArrayRef, PrimitiveBuilder};
+use arrow::array::{ArrayRef, PrimitiveBuilder};
use arrow::datatypes::DataType::Timestamp;
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
@@ -31,11 +31,12 @@ use chrono::{DateTime, MappedLocalTime, Offset, TimeDelta, TimeZone, Utc};
use datafusion_common::cast::as_primitive_array;
use datafusion_common::{
- exec_err, internal_datafusion_err, plan_err, utils::take_function_args, Result,
+ exec_err, internal_datafusion_err, internal_err, utils::take_function_args, Result,
ScalarValue,
};
use datafusion_expr::{
- ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,
+ Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass,
+ Volatility,
};
use datafusion_macros::user_doc;
@@ -111,133 +112,163 @@ impl Default for ToLocalTimeFunc {
impl ToLocalTimeFunc {
pub fn new() -> Self {
Self {
- signature: Signature::user_defined(Volatility::Immutable),
+ signature: Signature::coercible(
+ vec![Coercion::new_exact(TypeSignatureClass::Timestamp)],
+ Volatility::Immutable,
+ ),
}
}
+}
- fn to_local_time(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {
- let [time_value] = take_function_args(self.name(), args)?;
+impl ScalarUDFImpl for ToLocalTimeFunc {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
- let arg_type = time_value.data_type();
- match arg_type {
- Timestamp(_, None) => {
- // if no timezone specified, just return the input
- Ok(time_value.clone())
- }
- // If has timezone, adjust the underlying time value. The current time value
- // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore,
- // we need to adjust the time value to the local time. See [`adjust_to_local_time`]
- // for more details.
- //
- // Then remove the timezone in return type, i.e. return None
- Timestamp(_, Some(timezone)) => {
- let tz: Tz = timezone.parse()?;
-
- match time_value {
- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
- Some(ts),
- Some(_),
- )) => {
- let adjusted_ts =
- adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?;
- Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
- Some(adjusted_ts),
- None,
- )))
- }
- ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
- Some(ts),
- Some(_),
- )) => {
- let adjusted_ts =
- adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?;
- Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
- Some(adjusted_ts),
- None,
- )))
- }
- ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
- Some(ts),
- Some(_),
- )) => {
- let adjusted_ts =
- adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?;
- Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
- Some(adjusted_ts),
- None,
- )))
- }
- ColumnarValue::Scalar(ScalarValue::TimestampSecond(
- Some(ts),
- Some(_),
- )) => {
- let adjusted_ts =
- adjust_to_local_time::<TimestampSecondType>(*ts, tz)?;
- Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond(
- Some(adjusted_ts),
- None,
- )))
- }
- ColumnarValue::Array(array) => {
- fn transform_array<T: ArrowTimestampType>(
- array: &ArrayRef,
- tz: Tz,
- ) -> Result<ColumnarValue> {
- let mut builder = PrimitiveBuilder::<T>::new();
-
- let primitive_array = as_primitive_array::<T>(array)?;
- for ts_opt in primitive_array.iter() {
- match ts_opt {
- None => builder.append_null(),
- Some(ts) => {
- let adjusted_ts: i64 =
- adjust_to_local_time::<T>(ts, tz)?;
- builder.append_value(adjusted_ts)
- }
- }
- }
-
- Ok(ColumnarValue::Array(Arc::new(builder.finish())))
- }
-
- match array.data_type() {
- Timestamp(_, None) => {
- // if no timezone specified, just return the input
- Ok(time_value.clone())
- }
- Timestamp(Nanosecond, Some(_)) => {
- transform_array::<TimestampNanosecondType>(array, tz)
- }
- Timestamp(Microsecond, Some(_)) => {
- transform_array::<TimestampMicrosecondType>(array, tz)
- }
- Timestamp(Millisecond, Some(_)) => {
- transform_array::<TimestampMillisecondType>(array, tz)
- }
- Timestamp(Second, Some(_)) => {
- transform_array::<TimestampSecondType>(array, tz)
- }
- _ => {
- exec_err!("to_local_time function requires timestamp argument in array, got {:?}", array.data_type())
- }
- }
- }
- _ => {
- exec_err!(
- "to_local_time function requires timestamp argument, got {:?}",
- time_value.data_type()
- )
- }
- }
- }
- _ => {
- exec_err!(
- "to_local_time function requires timestamp argument, got {:?}",
- arg_type
- )
+ fn name(&self) -> &str {
+ "to_local_time"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ match &arg_types[0] {
+ DataType::Null => Ok(Timestamp(Nanosecond, None)),
+ Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)),
+ dt => internal_err!(
+ "The to_local_time function can only accept timestamp as the arg, got {dt}"
+ ),
+ }
+ }
+
+ fn invoke_with_args(
+ &self,
+ args: datafusion_expr::ScalarFunctionArgs,
+ ) -> Result<ColumnarValue> {
+ let [time_value] = take_function_args(self.name(), &args.args)?;
+ to_local_time(time_value)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+fn transform_array<T: ArrowTimestampType>(
+ array: &ArrayRef,
+ tz: Tz,
+) -> Result<ColumnarValue> {
+ let primitive_array = as_primitive_array::<T>(array)?;
+ let mut builder = PrimitiveBuilder::<T>::with_capacity(primitive_array.len());
+ for ts_opt in primitive_array.iter() {
+ match ts_opt {
+ None => builder.append_null(),
+ Some(ts) => {
+ let adjusted_ts: i64 = adjust_to_local_time::<T>(ts, tz)?;
+ builder.append_value(adjusted_ts)
}
}
}
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
+}
+
+fn to_local_time(time_value: &ColumnarValue) -> Result<ColumnarValue> {
+ let arg_type = time_value.data_type();
+
+ let tz: Tz = match &arg_type {
+ Timestamp(_, Some(timezone)) => timezone.parse()?,
+ Timestamp(_, None) => {
+ // if no timezone specified, just return the input
+ return Ok(time_value.clone());
+ }
+ DataType::Null => {
+ return Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ None, None,
+ )));
+ }
+ dt => {
+ return internal_err!(
+ "to_local_time function requires timestamp argument, got {dt}"
+ )
+ }
+ };
+
+ // If has timezone, adjust the underlying time value. The current time value
+ // is stored as i64 in UTC, even though the timezone may not be in UTC. Therefore,
+ // we need to adjust the time value to the local time. See [`adjust_to_local_time`]
+ // for more details.
+ //
+ // Then remove the timezone in return type, i.e. return None
+ match time_value {
+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, Some(_))) => Ok(
+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(None, None)),
+ ),
+ ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, Some(_))) => Ok(
+ ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(None, None)),
+ ),
+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, Some(_))) => Ok(
+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, None)),
+ ),
+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, Some(_))) => Ok(
+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(None, None)),
+ ),
+ ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(ts), Some(_))) => {
+ let adjusted_ts = adjust_to_local_time::<TimestampNanosecondType>(*ts, tz)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ Some(adjusted_ts),
+ None,
+ )))
+ }
+ ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(Some(ts), Some(_))) => {
+ let adjusted_ts = adjust_to_local_time::<TimestampMicrosecondType>(*ts, tz)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(
+ Some(adjusted_ts),
+ None,
+ )))
+ }
+ ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(Some(ts), Some(_))) => {
+ let adjusted_ts = adjust_to_local_time::<TimestampMillisecondType>(*ts, tz)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampMillisecond(
+ Some(adjusted_ts),
+ None,
+ )))
+ }
+ ColumnarValue::Scalar(ScalarValue::TimestampSecond(Some(ts), Some(_))) => {
+ let adjusted_ts = adjust_to_local_time::<TimestampSecondType>(*ts, tz)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampSecond(
+ Some(adjusted_ts),
+ None,
+ )))
+ }
+ ColumnarValue::Array(array)
+ if matches!(array.data_type(), Timestamp(Nanosecond, Some(_))) =>
+ {
+ transform_array::<TimestampNanosecondType>(array, tz)
+ }
+ ColumnarValue::Array(array)
+ if matches!(array.data_type(), Timestamp(Microsecond, Some(_))) =>
+ {
+ transform_array::<TimestampMicrosecondType>(array, tz)
+ }
+ ColumnarValue::Array(array)
+ if matches!(array.data_type(), Timestamp(Millisecond, Some(_))) =>
+ {
+ transform_array::<TimestampMillisecondType>(array, tz)
+ }
+ ColumnarValue::Array(array)
+ if matches!(array.data_type(), Timestamp(Second, Some(_))) =>
+ {
+ transform_array::<TimestampSecondType>(array, tz)
+ }
+ _ => {
+ internal_err!(
+ "to_local_time function requires timestamp argument, got {arg_type}"
+ )
+ }
+ }
}
/// This function converts a timestamp with a timezone to a timestamp without a timezone.
@@ -343,68 +374,6 @@ fn adjust_to_local_time<T: ArrowTimestampType>(ts: i64, tz: Tz) -> Result<i64> {
}
}
-impl ScalarUDFImpl for ToLocalTimeFunc {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn name(&self) -> &str {
- "to_local_time"
- }
-
- fn signature(&self) -> &Signature {
- &self.signature
- }
-
- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- let [time_value] = take_function_args(self.name(), arg_types)?;
-
- match time_value {
- Timestamp(timeunit, _) => Ok(Timestamp(*timeunit, None)),
- _ => exec_err!(
- "The to_local_time function can only accept timestamp as the arg, got {:?}", time_value
- )
- }
- }
-
- fn invoke_with_args(
- &self,
- args: datafusion_expr::ScalarFunctionArgs,
- ) -> Result<ColumnarValue> {
- let [time_value] = take_function_args(self.name(), args.args)?;
-
- self.to_local_time(std::slice::from_ref(&time_value))
- }
-
- fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
- if arg_types.len() != 1 {
- return plan_err!(
- "to_local_time function requires 1 argument, got {:?}",
- arg_types.len()
- );
- }
-
- let first_arg = arg_types[0].clone();
- match &first_arg {
- DataType::Null => Ok(vec![Timestamp(Nanosecond, None)]),
- Timestamp(Nanosecond, timezone) => {
- Ok(vec![Timestamp(Nanosecond, timezone.clone())])
- }
- Timestamp(Microsecond, timezone) => {
- Ok(vec![Timestamp(Microsecond, timezone.clone())])
- }
- Timestamp(Millisecond, timezone) => {
- Ok(vec![Timestamp(Millisecond, timezone.clone())])
- }
- Timestamp(Second, timezone) => Ok(vec![Timestamp(Second, timezone.clone())]),
- _ => plan_err!("The to_local_time function can only accept Timestamp as the arg got {first_arg}"),
- }
- }
- fn documentation(&self) -> Option<&Documentation> {
- self.doc()
- }
-}
-
#[cfg(test)]
mod tests {
use std::sync::Arc;
diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt
index 5c365b056d..3a1a26257b 100644
--- a/datafusion/sqllogictest/test_files/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -2250,7 +2250,7 @@ SET TIME ZONE = '+05:00'
statement ok
CREATE TABLE foo (time TIMESTAMPTZ) AS VALUES
- ('2020-01-01T00:00:00+05:00'),
+ ('2020-01-01T00:00:00+05:00'),
('2020-01-01T01:00:00+05:00'),
('2020-01-01T02:00:00+05:00'),
('2020-01-01T03:00:00+05:00')
@@ -2335,17 +2335,17 @@ NULL 1970-01-01T00:00:00 2031-01-19T23:33:25 1970-01-01T00:00:01 1969-12-31T23:5
# verify timestamp syntax styles are consistent
query BBBBBBBBBBBBB
SELECT to_timestamp(null) is null as c1,
- null::timestamp is null as c2,
- cast(null as timestamp) is null as c3,
- to_timestamp(0) = 0::timestamp as c4,
- to_timestamp(1926632005) = 1926632005::timestamp as c5,
- to_timestamp(1) = 1::timestamp as c6,
- to_timestamp(-1) = -1::timestamp as c7,
+ null::timestamp is null as c2,
+ cast(null as timestamp) is null as c3,
+ to_timestamp(0) = 0::timestamp as c4,
+ to_timestamp(1926632005) = 1926632005::timestamp as c5,
+ to_timestamp(1) = 1::timestamp as c6,
+ to_timestamp(-1) = -1::timestamp as c7,
to_timestamp(0-1) = (0-1)::timestamp as c8,
- to_timestamp(0) = cast(0 as timestamp) as c9,
- to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10,
- to_timestamp(1) = cast(1 as timestamp) as c11,
- to_timestamp(-1) = cast(-1 as timestamp) as c12,
+ to_timestamp(0) = cast(0 as timestamp) as c9,
+ to_timestamp(1926632005) = cast(1926632005 as timestamp) as c10,
+ to_timestamp(1) = cast(1 as timestamp) as c11,
+ to_timestamp(-1) = cast(-1 as timestamp) as c12,
to_timestamp(0-1) = cast(0-1 as timestamp) as c13
----
true true true true true true true true true true true true true
@@ -2358,10 +2358,10 @@ Timestamp(ns) Timestamp(ns) Timestamp(ns)
# verify timestamp output types using timestamp literal syntax
query BBBBBB
-SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1,
+SELECT arrow_typeof(to_timestamp(1)) = arrow_typeof(1::timestamp) as c1,
arrow_typeof(to_timestamp(null)) = arrow_typeof(null::timestamp) as c2,
arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof('2023-01-10 12:34:56.000'::timestamp) as c3,
- arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4,
+ arrow_typeof(to_timestamp(1)) = arrow_typeof(cast(1 as timestamp)) as c4,
arrow_typeof(to_timestamp(null)) = arrow_typeof(cast(null as timestamp)) as c5,
arrow_typeof(to_timestamp('2023-01-10 12:34:56.000')) = arrow_typeof(cast('2023-01-10 12:34:56.000' as timestamp)) as c6
----
@@ -2605,13 +2605,13 @@ drop table table_a
##########
statement ok
-create table table_a (ts timestamp) as values
- ('2020-09-08T11:42:29Z'::timestamp),
+create table table_a (ts timestamp) as values
+ ('2020-09-08T11:42:29Z'::timestamp),
('2020-09-08T12:42:29Z'::timestamp),
('2020-09-08T13:42:29Z'::timestamp)
statement ok
-create table table_b (ts timestamp) as values
+create table table_b (ts timestamp) as values
('2020-09-08T11:42:29.190Z'::timestamp),
('2020-09-08T13:42:29.190Z'::timestamp),
('2020-09-08T12:42:29.190Z'::timestamp)
@@ -2733,8 +2733,8 @@ statement ok
drop table t1
statement ok
-create table table_a (val int, ts1 timestamp, ts2 timestamp) as values
- (1, '2018-07-01T06:00:00'::timestamp, '2018-07-01T07:00:00'::timestamp),
+create table table_a (val int, ts1 timestamp, ts2 timestamp) as values
+ (1, '2018-07-01T06:00:00'::timestamp, '2018-07-01T07:00:00'::timestamp),
(2, '2018-07-01T07:00:00'::timestamp, '2018-07-01T08:00:00'::timestamp)
query I?
@@ -3038,7 +3038,7 @@ NULL
query T
SELECT to_char(date_column, '%Y-%m-%d')
-FROM (VALUES
+FROM (VALUES
(DATE '2020-09-01'),
(NULL)
) AS t(date_column);
@@ -3048,7 +3048,7 @@ NULL
query T
SELECT to_char(date_column, '%Y-%m-%d')
-FROM (VALUES
+FROM (VALUES
(NULL),
(DATE '2020-09-01')
) AS t(date_column);
@@ -3249,7 +3249,7 @@ statement error
select to_local_time('2024-04-01T00:00:20Z'::timestamp, 'some string');
# invalid argument data type
-statement error The to_local_time function can only accept Timestamp as the arg got Utf8
+statement error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Timestamp but received NativeType::String, DataType: Utf8
select to_local_time('2024-04-01T00:00:20Z');
# invalid timezone
@@ -3277,6 +3277,13 @@ select to_local_time(NULL);
----
NULL
+query PT
+select
+ to_local_time(arrow_cast(null, 'Timestamp(s, "Asia/Tokyo")')),
+ arrow_typeof(to_local_time(arrow_cast(null, 'Timestamp(s, "Asia/Tokyo")')));
+----
+NULL Timestamp(s)
+
query PTPT
select
time,