Skip to content

Apache DataFusion SQL Query Engine

Update `to_unixtime` udf function to support a consistent set of argument types (#19442)

Kumar Ujjawal62740802f648
diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs
index f9b4b5a3c9..6ebea3c1a5 100644
--- a/datafusion/functions/src/datetime/mod.rs
+++ b/datafusion/functions/src/datetime/mod.rs
@@ -108,7 +108,7 @@ pub mod expr_fn {
),
(
to_unixtime,
- "converts a string and optional formats to a Unixtime",
+ "converts a value to seconds since the unix epoch",
args,
),(
to_timestamp,
diff --git a/datafusion/functions/src/datetime/to_unixtime.rs b/datafusion/functions/src/datetime/to_unixtime.rs
index a62ace9ccb..fb6315d8f0 100644
--- a/datafusion/functions/src/datetime/to_unixtime.rs
+++ b/datafusion/functions/src/datetime/to_unixtime.rs
@@ -27,7 +27,7 @@ use std::any::Any;
#[user_doc(
doc_section(label = "Time and Date Functions"),
- description = "Converts a value to seconds since the unix epoch (`1970-01-01T00:00:00Z`). Supports strings, dates, timestamps and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided.",
+ description = "Converts a value to seconds since the unix epoch (`1970-01-01T00:00:00`). Supports strings, dates, timestamps, integer, unsigned integer, and float types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers, unsigned integers, and floats are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00`).",
syntax_example = "to_unixtime(expression[, ..., format_n])",
sql_example = r#"
```sql
@@ -101,22 +101,44 @@ impl ScalarUDFImpl for ToUnixtimeFunc {
// validate that any args after the first one are Utf8
if arg_args.len() > 1 {
- validate_data_types(arg_args, "to_unixtime")?;
+ // Format arguments only make sense for string inputs
+ match arg_args[0].data_type() {
+ DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => {
+ validate_data_types(arg_args, "to_unixtime")?;
+ }
+ _ => {
+ return exec_err!(
+ "to_unixtime function only accepts format arguments with string input, got {} arguments",
+ arg_args.len()
+ );
+ }
+ }
}
match arg_args[0].data_type() {
- DataType::Int32 | DataType::Int64 | DataType::Null | DataType::Float64 => {
- arg_args[0].cast_to(&DataType::Int64, None)
- }
+ DataType::Int8
+ | DataType::Int16
+ | DataType::Int32
+ | DataType::Int64
+ | DataType::UInt8
+ | DataType::UInt16
+ | DataType::UInt32
+ | DataType::UInt64
+ | DataType::Float16
+ | DataType::Float32
+ | DataType::Float64
+ | DataType::Null => arg_args[0].cast_to(&DataType::Int64, None),
DataType::Date64 | DataType::Date32 => arg_args[0]
.cast_to(&DataType::Timestamp(TimeUnit::Second, None), None)?
.cast_to(&DataType::Int64, None),
DataType::Timestamp(_, tz) => arg_args[0]
.cast_to(&DataType::Timestamp(TimeUnit::Second, tz), None)?
.cast_to(&DataType::Int64, None),
- DataType::Utf8 => ToTimestampSecondsFunc::new()
- .invoke_with_args(args)?
- .cast_to(&DataType::Int64, None),
+ DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => {
+ ToTimestampSecondsFunc::new()
+ .invoke_with_args(args)?
+ .cast_to(&DataType::Int64, None)
+ }
other => {
exec_err!("Unsupported data type {} for function to_unixtime", other)
}
diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.slt
index 67f44312e5..90accbce38 100644
--- a/datafusion/sqllogictest/test_files/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -3526,6 +3526,64 @@ select to_unixtime(arrow_cast(1599523200.414, 'Float64'));
----
1599523200
+query I
+select to_unixtime(arrow_cast(-1, 'Int8'));
+----
+-1
+
+query I
+select to_unixtime(arrow_cast(null, 'Int8'));
+----
+NULL
+
+query I
+select to_unixtime(arrow_cast(1000, 'Int16'));
+----
+1000
+
+query I
+select to_unixtime(arrow_cast(255, 'UInt8'));
+----
+255
+
+query I
+select to_unixtime(arrow_cast(65535, 'UInt16'));
+----
+65535
+
+query I
+select to_unixtime(arrow_cast(1599523200, 'UInt32'));
+----
+1599523200
+
+query I
+select to_unixtime(arrow_cast(1599523200, 'UInt64'));
+----
+1599523200
+
+query error DataFusion error: Arrow error: Cast error: Can't cast value 18446744073709551615 to type Int64
+select to_unixtime(arrow_cast(18446744073709551615, 'UInt64'));
+
+query I
+select to_unixtime(arrow_cast(1000.12, 'Float16'));
+----
+1000
+
+query I
+select to_unixtime(arrow_cast(1000.414, 'Float32'));
+----
+1000
+
+query I
+select to_unixtime(arrow_cast('2020-09-08T12:00:00+00:00', 'Utf8View'));
+----
+1599566400
+
+query I
+select to_unixtime(arrow_cast('2020-09-08T12:00:00+00:00', 'LargeUtf8'));
+----
+1599566400
+
##########
## Tests for the "AT TIME ZONE" clause
##########
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index e454788003..cf35b9f3c3 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -2982,7 +2982,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
### `to_unixtime`
-Converts a value to seconds since the unix epoch (`1970-01-01T00:00:00Z`). Supports strings, dates, timestamps and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided.
+Converts a value to seconds since the unix epoch (`1970-01-01T00:00:00`). Supports strings, dates, timestamps, integer, unsigned integer, and float types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers, unsigned integers, and floats are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00`).
```sql
to_unixtime(expression[, ..., format_n])