Skip to content

Apache DataFusion SQL Query Engine

fix: handle null date and timestamp format arguments (#23641)

linfeng8a37a07d5ef1
diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs
index 2db64beafa..9a7f94bd59 100644
--- a/datafusion/functions/src/datetime/common.rs
+++ b/datafusion/functions/src/datetime/common.rs
@@ -32,7 +32,7 @@ use chrono::{DateTime, TimeZone, Utc};
use datafusion_common::cast::as_generic_string_array;
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err,
- internal_datafusion_err, unwrap_or_internal_err,
+ internal_datafusion_err,
};
use datafusion_expr::ColumnarValue;
@@ -353,9 +353,9 @@ where
// if the first argument is a scalar utf8 all arguments are expected to be scalar utf8
ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {
Some(a) => {
- let a = a.as_ref();
- // ASK: Why do we trust `a` to be non-null at this point?
- let a = unwrap_or_internal_err!(a);
+ let Some(a) = a.as_ref() else {
+ return Ok(ColumnarValue::Scalar(scalar_value(dt, None)?));
+ };
let mut ret = None;
@@ -384,7 +384,10 @@ where
}
}
- unwrap_or_internal_err!(ret)
+ match ret {
+ Some(ret) => ret,
+ None => Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)),
+ }
}
other => {
exec_err!("Unsupported data type {other:?} for function {name}")
@@ -483,12 +486,21 @@ where
if let Some(x) = x {
for arg in args {
let v = match arg {
- ColumnarValue::Array(a) => match a.data_type() {
- DataType::Utf8View => Ok(a.as_string_view().value(pos)),
- DataType::LargeUtf8 => Ok(a.as_string::<i64>().value(pos)),
- DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)),
- other => exec_err!("Unexpected type encountered '{other}'"),
- },
+ ColumnarValue::Array(a) => {
+ if a.is_null(pos) {
+ continue;
+ }
+ match a.data_type() {
+ DataType::Utf8View => Ok(a.as_string_view().value(pos)),
+ DataType::LargeUtf8 => {
+ Ok(a.as_string::<i64>().value(pos))
+ }
+ DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)),
+ other => {
+ exec_err!("Unexpected type encountered '{other}'")
+ }
+ }
+ }
ColumnarValue::Scalar(s) => match s.try_as_str() {
Some(Some(v)) => Ok(v),
Some(None) => continue, // null string
diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.slt
index d2a7360b12..68d87eceed 100644
--- a/datafusion/sqllogictest/test_files/datetime/dates.slt
+++ b/datafusion/sqllogictest/test_files/datetime/dates.slt
@@ -298,6 +298,27 @@ SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+')
query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input
SELECT to_date('2020-09-08 12/00/00+00:00', '%q')
+# NULL string scalar inputs and all-NULL scalar formats return NULL
+query DD
+SELECT
+ to_date(NULL::VARCHAR, '%Y-%m-%d'),
+ to_date('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+----
+NULL NULL
+
+# NULL array formats are skipped for each row; rows with no usable format return NULL
+query ID
+SELECT id, to_date(value, format1, format2)
+FROM (
+ VALUES
+ (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'),
+ (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+) AS t(id, value, format1, format2)
+ORDER BY id
+----
+1 2020-09-08
+2 NULL
+
statement ok
create table ts_utf8_data(ts varchar(100), format varchar(100)) as values
('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'),
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index 8ba095ef93..9ac00e72b4 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -518,6 +518,35 @@ SELECT COUNT(*) FROM ts_data_secs where ts > to_timestamp_seconds('2020-09-08 12
----
2
+# NULL string scalar inputs and all-NULL scalar formats return NULL
+query PP
+SELECT
+ to_timestamp(NULL::VARCHAR, '%Y-%m-%d'),
+ to_timestamp('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+----
+NULL NULL
+
+# NULL array formats are skipped for each row; rows with no usable format return NULL
+query IP
+SELECT id, to_timestamp(value, format1, format2)
+FROM (
+ VALUES
+ (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'),
+ (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+) AS t(id, value, format1, format2)
+ORDER BY id
+----
+1 2020-09-08T00:00:00
+2 NULL
+
+# to_unixtime uses the same formatted string parsing path
+query II
+SELECT
+ to_unixtime(NULL::VARCHAR, '%Y-%m-%d'),
+ to_unixtime('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)
+----
+NULL NULL
+
# to_timestamp float inputs
query PPP