Apache DataFusion SQL Query Engine
fix: handle null date and timestamp format arguments (#23641)
linfeng8a37a07d5ef1
diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rsindex 2db64beafa..9a7f94bd59 100644--- a/datafusion/functions/src/datetime/common.rs+++ b/datafusion/functions/src/datetime/common.rs@@ -32,7 +32,7 @@ use chrono::{DateTime, TimeZone, Utc};use datafusion_common::cast::as_generic_string_array;use datafusion_common::{DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err,- internal_datafusion_err, unwrap_or_internal_err,+ internal_datafusion_err,};use datafusion_expr::ColumnarValue;@@ -353,9 +353,9 @@ where// if the first argument is a scalar utf8 all arguments are expected to be scalar utf8ColumnarValue::Scalar(scalar) => match scalar.try_as_str() {Some(a) => {- let a = a.as_ref();- // ASK: Why do we trust `a` to be non-null at this point?- let a = unwrap_or_internal_err!(a);+ let Some(a) = a.as_ref() else {+ return Ok(ColumnarValue::Scalar(scalar_value(dt, None)?));+ };let mut ret = None;@@ -384,7 +384,10 @@ where}}- unwrap_or_internal_err!(ret)+ match ret {+ Some(ret) => ret,+ None => Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)),+ }}other => {exec_err!("Unsupported data type {other:?} for function {name}")@@ -483,12 +486,21 @@ whereif let Some(x) = x {for arg in args {let v = match arg {- ColumnarValue::Array(a) => match a.data_type() {- DataType::Utf8View => Ok(a.as_string_view().value(pos)),- DataType::LargeUtf8 => Ok(a.as_string::<i64>().value(pos)),- DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)),- other => exec_err!("Unexpected type encountered '{other}'"),- },+ ColumnarValue::Array(a) => {+ if a.is_null(pos) {+ continue;+ }+ match a.data_type() {+ DataType::Utf8View => Ok(a.as_string_view().value(pos)),+ DataType::LargeUtf8 => {+ Ok(a.as_string::<i64>().value(pos))+ }+ DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)),+ other => {+ exec_err!("Unexpected type encountered '{other}'")+ }+ }+ }ColumnarValue::Scalar(s) => match s.try_as_str() {Some(Some(v)) => Ok(v),Some(None) => continue, // null stringdiff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.sltindex d2a7360b12..68d87eceed 100644--- a/datafusion/sqllogictest/test_files/datetime/dates.slt+++ b/datafusion/sqllogictest/test_files/datetime/dates.slt@@ -298,6 +298,27 @@ SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+')query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing inputSELECT to_date('2020-09-08 12/00/00+00:00', '%q')+# NULL string scalar inputs and all-NULL scalar formats return NULL+query DD+SELECT+ to_date(NULL::VARCHAR, '%Y-%m-%d'),+ to_date('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)+----+NULL NULL++# NULL array formats are skipped for each row; rows with no usable format return NULL+query ID+SELECT id, to_date(value, format1, format2)+FROM (+ VALUES+ (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'),+ (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR)+) AS t(id, value, format1, format2)+ORDER BY id+----+1 2020-09-08+2 NULL+statement okcreate table ts_utf8_data(ts varchar(100), format varchar(100)) as values('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'),diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex 8ba095ef93..9ac00e72b4 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -518,6 +518,35 @@ SELECT COUNT(*) FROM ts_data_secs where ts > to_timestamp_seconds('2020-09-08 12----2+# NULL string scalar inputs and all-NULL scalar formats return NULL+query PP+SELECT+ to_timestamp(NULL::VARCHAR, '%Y-%m-%d'),+ to_timestamp('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)+----+NULL NULL++# NULL array formats are skipped for each row; rows with no usable format return NULL+query IP+SELECT id, to_timestamp(value, format1, format2)+FROM (+ VALUES+ (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'),+ (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR)+) AS t(id, value, format1, format2)+ORDER BY id+----+1 2020-09-08T00:00:00+2 NULL++# to_unixtime uses the same formatted string parsing path+query II+SELECT+ to_unixtime(NULL::VARCHAR, '%Y-%m-%d'),+ to_unixtime('2020-09-08', NULL::VARCHAR, NULL::VARCHAR)+----+NULL NULL+# to_timestamp float inputsquery PPP