Skip to content

Apache DataFusion SQL Query Engine

feat: `to_time` function (#19540)

Kumar Ujjawal818706ab78a1
diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs
index 6ebea3c1a5..9872db3faf 100644
--- a/datafusion/functions/src/datetime/mod.rs
+++ b/datafusion/functions/src/datetime/mod.rs
@@ -35,6 +35,7 @@ pub mod planner;
pub mod to_char;
pub mod to_date;
pub mod to_local_time;
+pub mod to_time;
pub mod to_timestamp;
pub mod to_unixtime;
@@ -50,6 +51,7 @@ make_udf_function!(from_unixtime::FromUnixtimeFunc, from_unixtime);
make_udf_function!(to_char::ToCharFunc, to_char);
make_udf_function!(to_date::ToDateFunc, to_date);
make_udf_function!(to_local_time::ToLocalTimeFunc, to_local_time);
+make_udf_function!(to_time::ToTimeFunc, to_time);
make_udf_function!(to_unixtime::ToUnixtimeFunc, to_unixtime);
make_udf_function!(to_timestamp::ToTimestampFunc, to_timestamp);
make_udf_function!(to_timestamp::ToTimestampSecondsFunc, to_timestamp_seconds);
@@ -110,6 +112,10 @@ pub mod expr_fn {
to_unixtime,
"converts a value to seconds since the unix epoch",
args,
+ ),(
+ to_time,
+ "converts a string and optional formats to a `Time64(Nanoseconds)`",
+ args,
),(
to_timestamp,
"converts a string and optional formats to a `Timestamp(Nanoseconds, None)`",
@@ -278,6 +284,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
to_char(),
to_date(),
to_local_time(),
+ to_time(),
to_unixtime(),
to_timestamp(),
to_timestamp_seconds(),
diff --git a/datafusion/functions/src/datetime/to_time.rs b/datafusion/functions/src/datetime/to_time.rs
new file mode 100644
index 0000000000..086126409e
--- /dev/null
+++ b/datafusion/functions/src/datetime/to_time.rs
@@ -0,0 +1,252 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::datetime::common::*;
+use arrow::array::builder::PrimitiveBuilder;
+use arrow::array::cast::AsArray;
+use arrow::array::temporal_conversions::time_to_time64ns;
+use arrow::array::types::Time64NanosecondType;
+use arrow::array::{Array, PrimitiveArray, StringArrayType};
+use arrow::datatypes::DataType;
+use arrow::datatypes::DataType::*;
+use chrono::NaiveTime;
+use datafusion_common::{Result, ScalarValue, exec_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+use std::any::Any;
+use std::sync::Arc;
+
+/// Default time formats to try when parsing without an explicit format
+const DEFAULT_TIME_FORMATS: &[&str] = &[
+ "%H:%M:%S%.f", // 12:30:45.123456789
+ "%H:%M:%S", // 12:30:45
+ "%H:%M", // 12:30
+];
+
+#[user_doc(
+ doc_section(label = "Time and Date Functions"),
+ description = r"Converts a value to a time (`HH:MM:SS.nnnnnnnnn`).
+Supports strings and timestamps as input.
+Strings are parsed as `HH:MM:SS`, `HH:MM:SS.nnnnnnnnn`, or `HH:MM` if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided.
+Timestamps will have the time portion extracted.
+Returns the corresponding time.
+
+Note: `to_time` returns Time64(Nanosecond), which represents the time of day in nanoseconds since midnight.",
+ syntax_example = "to_time('12:30:45', '%H:%M:%S')",
+ sql_example = r#"```sql
+> select to_time('12:30:45');
++---------------------------+
+| to_time(Utf8("12:30:45")) |
++---------------------------+
+| 12:30:45 |
++---------------------------+
+> select to_time('12-30-45', '%H-%M-%S');
++--------------------------------------------+
+| to_time(Utf8("12-30-45"),Utf8("%H-%M-%S")) |
++--------------------------------------------+
+| 12:30:45 |
++--------------------------------------------+
+> select to_time('2024-01-15 14:30:45'::timestamp);
++--------------------------------------------------+
+| to_time(Utf8("2024-01-15 14:30:45")) |
++--------------------------------------------------+
+| 14:30:45 |
++--------------------------------------------------+
+```
+
+Additional examples can be found [here](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/builtin_functions/date_time.rs)
+"#,
+ standard_argument(name = "expression", prefix = "String or Timestamp"),
+ argument(
+ name = "format_n",
+ description = r"Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order
+ they appear with the first successful one being returned. If none of the formats successfully parse the expression
+ an error will be returned."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct ToTimeFunc {
+ signature: Signature,
+}
+
+impl Default for ToTimeFunc {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl ToTimeFunc {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for ToTimeFunc {
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn name(&self) -> &str {
+ "to_time"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(Time64(arrow::datatypes::TimeUnit::Nanosecond))
+ }
+
+ fn invoke_with_args(
+ &self,
+ args: datafusion_expr::ScalarFunctionArgs,
+ ) -> Result<ColumnarValue> {
+ let args = args.args;
+ if args.is_empty() {
+ return exec_err!("to_time function requires 1 or more arguments, got 0");
+ }
+
+ // validate that any args after the first one are Utf8
+ if args.len() > 1 {
+ validate_data_types(&args, "to_time")?;
+ }
+
+ match args[0].data_type() {
+ Utf8View | LargeUtf8 | Utf8 => string_to_time(&args),
+ Null => Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(None))),
+ // Support timestamp input by extracting time portion using Arrow cast
+ Timestamp(_, _) => timestamp_to_time(&args[0]),
+ other => {
+ exec_err!("Unsupported data type {} for function to_time", other)
+ }
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Convert string arguments to time (standalone function, not a method on ToTimeFunc)
+fn string_to_time(args: &[ColumnarValue]) -> Result<ColumnarValue> {
+ let formats = collect_formats(args)?;
+
+ match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Utf8(s))
+ | ColumnarValue::Scalar(ScalarValue::LargeUtf8(s))
+ | ColumnarValue::Scalar(ScalarValue::Utf8View(s)) => {
+ let result = s
+ .as_ref()
+ .map(|s| parse_time_with_formats(s, &formats))
+ .transpose()?;
+ Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(result)))
+ }
+ ColumnarValue::Array(array) => {
+ let result = match array.data_type() {
+ Utf8 => parse_time_array(&array.as_string::<i32>(), &formats)?,
+ LargeUtf8 => parse_time_array(&array.as_string::<i64>(), &formats)?,
+ Utf8View => parse_time_array(&array.as_string_view(), &formats)?,
+ other => return exec_err!("Unsupported type for to_time: {other}"),
+ };
+ Ok(ColumnarValue::Array(Arc::new(result)))
+ }
+ other => exec_err!("Unsupported argument for to_time: {other:?}"),
+ }
+}
+
+/// Collect format strings from arguments, erroring on non-scalar inputs
+fn collect_formats(args: &[ColumnarValue]) -> Result<Vec<&str>> {
+ if args.len() <= 1 {
+ return Ok(DEFAULT_TIME_FORMATS.to_vec());
+ }
+
+ let mut formats = Vec::with_capacity(args.len() - 1);
+ for (i, arg) in args[1..].iter().enumerate() {
+ match arg {
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some(s)))
+ | ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(s)))
+ | ColumnarValue::Scalar(ScalarValue::Utf8View(Some(s))) => {
+ formats.push(s.as_str());
+ }
+ ColumnarValue::Scalar(ScalarValue::Utf8(None))
+ | ColumnarValue::Scalar(ScalarValue::LargeUtf8(None))
+ | ColumnarValue::Scalar(ScalarValue::Utf8View(None)) => {
+ // Skip null format strings
+ }
+ ColumnarValue::Array(_) => {
+ return exec_err!(
+ "to_time format argument {} must be a scalar, not an array",
+ i + 2 // argument position (1-indexed, +1 for the first arg)
+ );
+ }
+ other => {
+ return exec_err!(
+ "to_time format argument {} has unsupported type: {:?}",
+ i + 2,
+ other.data_type()
+ );
+ }
+ }
+ }
+ Ok(formats)
+}
+
+/// Extract time portion from timestamp using Arrow cast kernel
+fn timestamp_to_time(arg: &ColumnarValue) -> Result<ColumnarValue> {
+ arg.cast_to(&Time64(arrow::datatypes::TimeUnit::Nanosecond), None)
+}
+
+/// Parse time array using the provided formats
+fn parse_time_array<'a, A: StringArrayType<'a>>(
+ array: &A,
+ formats: &[&str],
+) -> Result<PrimitiveArray<Time64NanosecondType>> {
+ let mut builder: PrimitiveBuilder<Time64NanosecondType> =
+ PrimitiveArray::builder(array.len());
+
+ for i in 0..array.len() {
+ if array.is_null(i) {
+ builder.append_null();
+ } else {
+ let s = array.value(i);
+ let nanos = parse_time_with_formats(s, formats)?;
+ builder.append_value(nanos);
+ }
+ }
+
+ Ok(builder.finish())
+}
+
+/// Parse time string using provided formats
+fn parse_time_with_formats(s: &str, formats: &[&str]) -> Result<i64> {
+ for format in formats {
+ if let Ok(time) = NaiveTime::parse_from_str(s, format) {
+ // Use Arrow's time_to_time64ns function instead of custom implementation
+ return Ok(time_to_time64ns(time));
+ }
+ }
+ exec_err!(
+ "Error parsing '{}' as time. Tried formats: {:?}",
+ s,
+ formats
+ )
+}
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index 90accbce38..5749b1c53d 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -3285,6 +3285,221 @@ select make_time(22, 1, '');
query error Expect TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\) but received NativeType::Float64, DataType: Float64
select make_time(arrow_cast(22, 'Float64'), 1, '');
+##########
+## to_time tests
+##########
+
+# Basic time parsing
+
+query D
+select to_time('12:30:45');
+----
+12:30:45
+
+query D
+select to_time('00:00:00');
+----
+00:00:00
+
+query D
+select to_time('23:59:59');
+----
+23:59:59
+
+query D
+select to_time('08:15:30');
+----
+08:15:30
+
+# Time with fractional seconds
+
+query D
+select to_time('12:30:45.123');
+----
+12:30:45.123
+
+query D
+select to_time('12:30:45.123456789');
+----
+12:30:45.123456789
+
+# Time with custom format
+
+query D
+select to_time('12-30-45', '%H-%M-%S');
+----
+12:30:45
+
+query D
+select to_time('14/25/30', '%H/%M/%S');
+----
+14:25:30
+
+query D
+select to_time('02:30:45 PM', '%I:%M:%S %p');
+----
+14:30:45
+
+# Null handling
+
+query D
+select to_time(null);
+----
+NULL
+
+# Return type check
+
+query T
+select arrow_typeof(to_time('12:30:45'));
+----
+Time64(ns)
+
+# Table input
+
+statement ok
+create table time_strings (time_str varchar) as values
+ ('12:30:45'),
+ ('23:59:59'),
+ ('00:00:00');
+
+query D
+select to_time(time_str) from time_strings;
+----
+12:30:45
+23:59:59
+00:00:00
+
+statement ok
+drop table time_strings;
+
+# Error cases
+
+query error Error parsing 'not_a_time' as time
+select to_time('not_a_time');
+
+query error Error parsing '25:00:00' as time
+select to_time('25:00:00');
+
+# Out of range minutes
+query error Error parsing '12:60:00' as time
+select to_time('12:60:00');
+
+# Out of range seconds (61 is invalid, 60 is allowed as leap second)
+query error Error parsing '12:30:61' as time
+select to_time('12:30:61');
+
+query error does not support zero arguments
+select to_time();
+
+# StringView type support
+
+query D
+select to_time(arrow_cast('08:15:30', 'Utf8View'));
+----
+08:15:30
+
+# LargeUtf8 type support
+
+query D
+select to_time(arrow_cast('14:45:00', 'LargeUtf8'));
+----
+14:45:00
+
+# HH:MM default parsing (no seconds)
+
+query D
+select to_time('14:30');
+----
+14:30:00
+
+query D
+select to_time('09:05');
+----
+09:05:00
+
+query D
+select to_time('00:00');
+----
+00:00:00
+
+# Timestamp input - extract time portion
+
+query D
+select to_time(to_timestamp('2024-01-15 14:30:45'));
+----
+14:30:45
+
+query D
+select to_time('2024-03-20 09:15:30'::timestamp);
+----
+09:15:30
+
+query D
+select to_time('2024-06-15 23:59:59.123456789'::timestamp);
+----
+23:59:59.123456789
+
+query D
+select to_time('2024-01-01 00:00:00'::timestamp);
+----
+00:00:00
+
+# Timestamp with timezone
+
+query D
+select to_time(to_timestamp('2024-01-15T14:30:45+00:00'));
+----
+14:30:45
+
+# Null timestamp
+
+query D
+select to_time(null::timestamp);
+----
+NULL
+
+# Return type check with timestamp input
+
+query T
+select arrow_typeof(to_time(to_timestamp('2024-01-15 12:30:45')));
+----
+Time64(ns)
+
+# Timestamp with timezone offset - to_timestamp parses and normalizes to UTC
+# 14:30:45 in UTC-5 = 19:30:45 in UTC, and Arrow stores as UTC internally
+
+query D
+select to_time(to_timestamp('2024-01-15T14:30:45-05:00'));
+----
+19:30:45
+
+# Timestamp without timezone - time portion is extracted as-is
+
+query D
+select to_time('2024-03-20 09:15:30'::timestamp);
+----
+09:15:30
+
+# Timestamp with timezone (timestamptz) - time is extracted as stored
+# Note: AT TIME ZONE labels the naive timestamp but doesn't convert the time value
+
+query D
+select to_time('2024-03-20 09:15:30'::timestamp AT TIME ZONE 'America/Los_Angeles');
+----
+09:15:30
+
+# Timestamp before epoch (1969-12-31 23:30:00 UTC)
+
+query D
+select to_time(to_timestamp('1969-12-31T23:30:00+00:00'));
+----
+23:30:00
+
+query D
+select to_time(to_timestamp('1960-06-15T08:45:30+00:00'));
+----
+08:45:30
+
##########
## to_char tests
##########
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index ddf32a5066..360311552f 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -2394,6 +2394,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
- [to_char](#to_char)
- [to_date](#to_date)
- [to_local_time](#to_local_time)
+- [to_time](#to_time)
- [to_timestamp](#to_timestamp)
- [to_timestamp_micros](#to_timestamp_micros)
- [to_timestamp_millis](#to_timestamp_millis)
@@ -2818,6 +2819,52 @@ FROM (
+---------------------------+
```
+### `to_time`
+
+Converts a value to a time (`HH:MM:SS.nnnnnnnnn`).
+Supports strings and timestamps as input.
+Strings are parsed as `HH:MM:SS`, `HH:MM:SS.nnnnnnnnn`, or `HH:MM` if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided.
+Timestamps will have the time portion extracted.
+Returns the corresponding time.
+
+Note: `to_time` returns Time64(Nanosecond), which represents the time of day in nanoseconds since midnight.
+
+```sql
+to_time('12:30:45', '%H:%M:%S')
+```
+
+#### Arguments
+
+- **expression**: String or Timestamp expression to operate on. Can be a constant, column, or function, and any combination of operators.
+- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order
+ they appear with the first successful one being returned. If none of the formats successfully parse the expression
+ an error will be returned.
+
+#### Example
+
+```sql
+> select to_time('12:30:45');
++---------------------------+
+| to_time(Utf8("12:30:45")) |
++---------------------------+
+| 12:30:45 |
++---------------------------+
+> select to_time('12-30-45', '%H-%M-%S');
++--------------------------------------------+
+| to_time(Utf8("12-30-45"),Utf8("%H-%M-%S")) |
++--------------------------------------------+
+| 12:30:45 |
++--------------------------------------------+
+> select to_time('2024-01-15 14:30:45'::timestamp);
++--------------------------------------------------+
+| to_time(Utf8("2024-01-15 14:30:45")) |
++--------------------------------------------------+
+| 14:30:45 |
++--------------------------------------------------+
+```
+
+Additional examples can be found [here](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/builtin_functions/date_time.rs)
+
### `to_timestamp`
Converts a value to a timestamp (`YYYY-MM-DDT00:00:00Z`). Supports strings, integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats] are provided. Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.