Apache DataFusion SQL Query Engine
feat: `to_time` function (#19540)
Kumar Ujjawal818706ab78a1
diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rsindex 6ebea3c1a5..9872db3faf 100644--- a/datafusion/functions/src/datetime/mod.rs+++ b/datafusion/functions/src/datetime/mod.rs@@ -35,6 +35,7 @@ pub mod planner;pub mod to_char;pub mod to_date;pub mod to_local_time;+pub mod to_time;pub mod to_timestamp;pub mod to_unixtime;@@ -50,6 +51,7 @@ make_udf_function!(from_unixtime::FromUnixtimeFunc, from_unixtime);make_udf_function!(to_char::ToCharFunc, to_char);make_udf_function!(to_date::ToDateFunc, to_date);make_udf_function!(to_local_time::ToLocalTimeFunc, to_local_time);+make_udf_function!(to_time::ToTimeFunc, to_time);make_udf_function!(to_unixtime::ToUnixtimeFunc, to_unixtime);make_udf_function!(to_timestamp::ToTimestampFunc, to_timestamp);make_udf_function!(to_timestamp::ToTimestampSecondsFunc, to_timestamp_seconds);@@ -110,6 +112,10 @@ pub mod expr_fn {to_unixtime,"converts a value to seconds since the unix epoch",args,+ ),(+ to_time,+ "converts a string and optional formats to a `Time64(Nanoseconds)`",+ args,),(to_timestamp,"converts a string and optional formats to a `Timestamp(Nanoseconds, None)`",@@ -278,6 +284,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {to_char(),to_date(),to_local_time(),+ to_time(),to_unixtime(),to_timestamp(),to_timestamp_seconds(),diff --git a/datafusion/functions/src/datetime/to_time.rs b/datafusion/functions/src/datetime/to_time.rsnew file mode 100644index 0000000000..086126409e--- /dev/null+++ b/datafusion/functions/src/datetime/to_time.rs@@ -0,0 +1,252 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use crate::datetime::common::*;+use arrow::array::builder::PrimitiveBuilder;+use arrow::array::cast::AsArray;+use arrow::array::temporal_conversions::time_to_time64ns;+use arrow::array::types::Time64NanosecondType;+use arrow::array::{Array, PrimitiveArray, StringArrayType};+use arrow::datatypes::DataType;+use arrow::datatypes::DataType::*;+use chrono::NaiveTime;+use datafusion_common::{Result, ScalarValue, exec_err};+use datafusion_expr::{+ ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,+};+use datafusion_macros::user_doc;+use std::any::Any;+use std::sync::Arc;++/// Default time formats to try when parsing without an explicit format+const DEFAULT_TIME_FORMATS: &[&str] = &[+ "%H:%M:%S%.f", // 12:30:45.123456789+ "%H:%M:%S", // 12:30:45+ "%H:%M", // 12:30+];++#[user_doc(+ doc_section(label = "Time and Date Functions"),+ description = r"Converts a value to a time (`HH:MM:SS.nnnnnnnnn`).+Supports strings and timestamps as input.+Strings are parsed as `HH:MM:SS`, `HH:MM:SS.nnnnnnnnn`, or `HH:MM` if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided.+Timestamps will have the time portion extracted.+Returns the corresponding time.++Note: `to_time` returns Time64(Nanosecond), which represents the time of day in nanoseconds since midnight.",+ syntax_example = "to_time('12:30:45', '%H:%M:%S')",+ sql_example = r#"```sql+> select to_time('12:30:45');++---------------------------++| to_time(Utf8("12:30:45")) |++---------------------------++| 12:30:45 |++---------------------------++> select to_time('12-30-45', '%H-%M-%S');++--------------------------------------------++| to_time(Utf8("12-30-45"),Utf8("%H-%M-%S")) |++--------------------------------------------++| 12:30:45 |++--------------------------------------------++> select to_time('2024-01-15 14:30:45'::timestamp);++--------------------------------------------------++| to_time(Utf8("2024-01-15 14:30:45")) |++--------------------------------------------------++| 14:30:45 |++--------------------------------------------------++```++Additional examples can be found [here](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/builtin_functions/date_time.rs)+"#,+ standard_argument(name = "expression", prefix = "String or Timestamp"),+ argument(+ name = "format_n",+ description = r"Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order+ they appear with the first successful one being returned. If none of the formats successfully parse the expression+ an error will be returned."+ )+)]+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct ToTimeFunc {+ signature: Signature,+}++impl Default for ToTimeFunc {+ fn default() -> Self {+ Self::new()+ }+}++impl ToTimeFunc {+ pub fn new() -> Self {+ Self {+ signature: Signature::variadic_any(Volatility::Immutable),+ }+ }+}++impl ScalarUDFImpl for ToTimeFunc {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "to_time"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(Time64(arrow::datatypes::TimeUnit::Nanosecond))+ }++ fn invoke_with_args(+ &self,+ args: datafusion_expr::ScalarFunctionArgs,+ ) -> Result<ColumnarValue> {+ let args = args.args;+ if args.is_empty() {+ return exec_err!("to_time function requires 1 or more arguments, got 0");+ }++ // validate that any args after the first one are Utf8+ if args.len() > 1 {+ validate_data_types(&args, "to_time")?;+ }++ match args[0].data_type() {+ Utf8View | LargeUtf8 | Utf8 => string_to_time(&args),+ Null => Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(None))),+ // Support timestamp input by extracting time portion using Arrow cast+ Timestamp(_, _) => timestamp_to_time(&args[0]),+ other => {+ exec_err!("Unsupported data type {} for function to_time", other)+ }+ }+ }++ fn documentation(&self) -> Option<&Documentation> {+ self.doc()+ }+}++/// Convert string arguments to time (standalone function, not a method on ToTimeFunc)+fn string_to_time(args: &[ColumnarValue]) -> Result<ColumnarValue> {+ let formats = collect_formats(args)?;++ match &args[0] {+ ColumnarValue::Scalar(ScalarValue::Utf8(s))+ | ColumnarValue::Scalar(ScalarValue::LargeUtf8(s))+ | ColumnarValue::Scalar(ScalarValue::Utf8View(s)) => {+ let result = s+ .as_ref()+ .map(|s| parse_time_with_formats(s, &formats))+ .transpose()?;+ Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(result)))+ }+ ColumnarValue::Array(array) => {+ let result = match array.data_type() {+ Utf8 => parse_time_array(&array.as_string::<i32>(), &formats)?,+ LargeUtf8 => parse_time_array(&array.as_string::<i64>(), &formats)?,+ Utf8View => parse_time_array(&array.as_string_view(), &formats)?,+ other => return exec_err!("Unsupported type for to_time: {other}"),+ };+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ other => exec_err!("Unsupported argument for to_time: {other:?}"),+ }+}++/// Collect format strings from arguments, erroring on non-scalar inputs+fn collect_formats(args: &[ColumnarValue]) -> Result<Vec<&str>> {+ if args.len() <= 1 {+ return Ok(DEFAULT_TIME_FORMATS.to_vec());+ }++ let mut formats = Vec::with_capacity(args.len() - 1);+ for (i, arg) in args[1..].iter().enumerate() {+ match arg {+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(s)))+ | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(s)))+ | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(s))) => {+ formats.push(s.as_str());+ }+ ColumnarValue::Scalar(ScalarValue::Utf8(None))+ | ColumnarValue::Scalar(ScalarValue::LargeUtf8(None))+ | ColumnarValue::Scalar(ScalarValue::Utf8View(None)) => {+ // Skip null format strings+ }+ ColumnarValue::Array(_) => {+ return exec_err!(+ "to_time format argument {} must be a scalar, not an array",+ i + 2 // argument position (1-indexed, +1 for the first arg)+ );+ }+ other => {+ return exec_err!(+ "to_time format argument {} has unsupported type: {:?}",+ i + 2,+ other.data_type()+ );+ }+ }+ }+ Ok(formats)+}++/// Extract time portion from timestamp using Arrow cast kernel+fn timestamp_to_time(arg: &ColumnarValue) -> Result<ColumnarValue> {+ arg.cast_to(&Time64(arrow::datatypes::TimeUnit::Nanosecond), None)+}++/// Parse time array using the provided formats+fn parse_time_array<'a, A: StringArrayType<'a>>(+ array: &A,+ formats: &[&str],+) -> Result<PrimitiveArray<Time64NanosecondType>> {+ let mut builder: PrimitiveBuilder<Time64NanosecondType> =+ PrimitiveArray::builder(array.len());++ for i in 0..array.len() {+ if array.is_null(i) {+ builder.append_null();+ } else {+ let s = array.value(i);+ let nanos = parse_time_with_formats(s, formats)?;+ builder.append_value(nanos);+ }+ }++ Ok(builder.finish())+}++/// Parse time string using provided formats+fn parse_time_with_formats(s: &str, formats: &[&str]) -> Result<i64> {+ for format in formats {+ if let Ok(time) = NaiveTime::parse_from_str(s, format) {+ // Use Arrow's time_to_time64ns function instead of custom implementation+ return Ok(time_to_time64ns(time));+ }+ }+ exec_err!(+ "Error parsing '{}' as time. Tried formats: {:?}",+ s,+ formats+ )+}diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex 90accbce38..5749b1c53d 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -3285,6 +3285,221 @@ select make_time(22, 1, '');query error Expect TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\) but received NativeType::Float64, DataType: Float64select make_time(arrow_cast(22, 'Float64'), 1, '');+##########+## to_time tests+##########++# Basic time parsing++query D+select to_time('12:30:45');+----+12:30:45++query D+select to_time('00:00:00');+----+00:00:00++query D+select to_time('23:59:59');+----+23:59:59++query D+select to_time('08:15:30');+----+08:15:30++# Time with fractional seconds++query D+select to_time('12:30:45.123');+----+12:30:45.123++query D+select to_time('12:30:45.123456789');+----+12:30:45.123456789++# Time with custom format++query D+select to_time('12-30-45', '%H-%M-%S');+----+12:30:45++query D+select to_time('14/25/30', '%H/%M/%S');+----+14:25:30++query D+select to_time('02:30:45 PM', '%I:%M:%S %p');+----+14:30:45++# Null handling++query D+select to_time(null);+----+NULL++# Return type check++query T+select arrow_typeof(to_time('12:30:45'));+----+Time64(ns)++# Table input++statement ok+create table time_strings (time_str varchar) as values+ ('12:30:45'),+ ('23:59:59'),+ ('00:00:00');++query D+select to_time(time_str) from time_strings;+----+12:30:45+23:59:59+00:00:00++statement ok+drop table time_strings;++# Error cases++query error Error parsing 'not_a_time' as time+select to_time('not_a_time');++query error Error parsing '25:00:00' as time+select to_time('25:00:00');++# Out of range minutes+query error Error parsing '12:60:00' as time+select to_time('12:60:00');++# Out of range seconds (61 is invalid, 60 is allowed as leap second)+query error Error parsing '12:30:61' as time+select to_time('12:30:61');++query error does not support zero arguments+select to_time();++# StringView type support++query D+select to_time(arrow_cast('08:15:30', 'Utf8View'));+----+08:15:30++# LargeUtf8 type support++query D+select to_time(arrow_cast('14:45:00', 'LargeUtf8'));+----+14:45:00++# HH:MM default parsing (no seconds)++query D+select to_time('14:30');+----+14:30:00++query D+select to_time('09:05');+----+09:05:00++query D+select to_time('00:00');+----+00:00:00++# Timestamp input - extract time portion++query D+select to_time(to_timestamp('2024-01-15 14:30:45'));+----+14:30:45++query D+select to_time('2024-03-20 09:15:30'::timestamp);+----+09:15:30++query D+select to_time('2024-06-15 23:59:59.123456789'::timestamp);+----+23:59:59.123456789++query D+select to_time('2024-01-01 00:00:00'::timestamp);+----+00:00:00++# Timestamp with timezone++query D+select to_time(to_timestamp('2024-01-15T14:30:45+00:00'));+----+14:30:45++# Null timestamp++query D+select to_time(null::timestamp);+----+NULL++# Return type check with timestamp input++query T+select arrow_typeof(to_time(to_timestamp('2024-01-15 12:30:45')));+----+Time64(ns)++# Timestamp with timezone offset - to_timestamp parses and normalizes to UTC+# 14:30:45 in UTC-5 = 19:30:45 in UTC, and Arrow stores as UTC internally++query D+select to_time(to_timestamp('2024-01-15T14:30:45-05:00'));+----+19:30:45++# Timestamp without timezone - time portion is extracted as-is++query D+select to_time('2024-03-20 09:15:30'::timestamp);+----+09:15:30++# Timestamp with timezone (timestamptz) - time is extracted as stored+# Note: AT TIME ZONE labels the naive timestamp but doesn't convert the time value++query D+select to_time('2024-03-20 09:15:30'::timestamp AT TIME ZONE 'America/Los_Angeles');+----+09:15:30++# Timestamp before epoch (1969-12-31 23:30:00 UTC)++query D+select to_time(to_timestamp('1969-12-31T23:30:00+00:00'));+----+23:30:00++query D+select to_time(to_timestamp('1960-06-15T08:45:30+00:00'));+----+08:45:30+############ to_char tests##########diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex ddf32a5066..360311552f 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -2394,6 +2394,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo- [to_char](#to_char)- [to_date](#to_date)- [to_local_time](#to_local_time)+- [to_time](#to_time)- [to_timestamp](#to_timestamp)- [to_timestamp_micros](#to_timestamp_micros)- [to_timestamp_millis](#to_timestamp_millis)@@ -2818,6 +2819,52 @@ FROM (+---------------------------+```+### `to_time`++Converts a value to a time (`HH:MM:SS.nnnnnnnnn`).+Supports strings and timestamps as input.+Strings are parsed as `HH:MM:SS`, `HH:MM:SS.nnnnnnnnn`, or `HH:MM` if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided.+Timestamps will have the time portion extracted.+Returns the corresponding time.++Note: `to_time` returns Time64(Nanosecond), which represents the time of day in nanoseconds since midnight.++```sql+to_time('12:30:45', '%H:%M:%S')+```++#### Arguments++- **expression**: String or Timestamp expression to operate on. Can be a constant, column, or function, and any combination of operators.+- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order+ they appear with the first successful one being returned. If none of the formats successfully parse the expression+ an error will be returned.++#### Example++```sql+> select to_time('12:30:45');++---------------------------++| to_time(Utf8("12:30:45")) |++---------------------------++| 12:30:45 |++---------------------------++> select to_time('12-30-45', '%H-%M-%S');++--------------------------------------------++| to_time(Utf8("12-30-45"),Utf8("%H-%M-%S")) |++--------------------------------------------++| 12:30:45 |++--------------------------------------------++> select to_time('2024-01-15 14:30:45'::timestamp);++--------------------------------------------------++| to_time(Utf8("2024-01-15 14:30:45")) |++--------------------------------------------------++| 14:30:45 |++--------------------------------------------------++```++Additional examples can be found [here](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/builtin_functions/date_time.rs)+### `to_timestamp`Converts a value to a timestamp (`YYYY-MM-DDT00:00:00Z`). Supports strings, integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats] are provided. Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.