Apache DataFusion SQL Query Engine
Simplify make_date & fix null handling (#19296)
Jeffrey Vo94d9c8e68c01
diff --git a/datafusion-examples/examples/builtin_functions/date_time.rs b/datafusion-examples/examples/builtin_functions/date_time.rsindex 7fb2c25f56..08d4bc6e29 100644--- a/datafusion-examples/examples/builtin_functions/date_time.rs+++ b/datafusion-examples/examples/builtin_functions/date_time.rs@@ -181,12 +181,13 @@ async fn query_make_date() -> Result<()> {// invalid column values will result in an errorlet result = ctx- .sql("select make_date(2024, null, 23)")+ .sql("select make_date(2024, '', 23)").await?.collect().await;- let expected = "Execution error: Unable to parse date from null/empty value";+ let expected =+ "Arrow error: Cast error: Cannot cast string '' to value of Int32 type";assert_contains!(result.unwrap_err().to_string(), expected);// invalid date values will also result in an error@@ -196,7 +197,7 @@ async fn query_make_date() -> Result<()> {.collect().await;- let expected = "Execution error: Unable to parse date from 2024, 1, 32";+ let expected = "Execution error: Day value '32' is out of range";assert_contains!(result.unwrap_err().to_string(), expected);Ok(())diff --git a/datafusion/functions/src/datetime/make_date.rs b/datafusion/functions/src/datetime/make_date.rsindex fbe28b8c23..8a87809c43 100644--- a/datafusion/functions/src/datetime/make_date.rs+++ b/datafusion/functions/src/datetime/make_date.rs@@ -18,17 +18,19 @@use std::any::Any;use std::sync::Arc;-use arrow::array::PrimitiveArray;use arrow::array::builder::PrimitiveBuilder;use arrow::array::cast::AsArray;use arrow::array::types::{Date32Type, Int32Type};+use arrow::array::{Array, PrimitiveArray};use arrow::datatypes::DataType;-use arrow::datatypes::DataType::{Date32, Int32, Int64, UInt32, UInt64, Utf8, Utf8View};+use arrow::datatypes::DataType::Date32;use chrono::prelude::*;+use datafusion_common::types::{NativeType, logical_int32, logical_string};use datafusion_common::{Result, ScalarValue, exec_err, utils::take_function_args};use datafusion_expr::{- ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, Documentation, ScalarUDFImpl, Signature, TypeSignatureClass,+ Volatility,};use datafusion_macros::user_doc;@@ -79,12 +81,16 @@ impl Default for MakeDateFunc {impl MakeDateFunc {pub fn new() -> Self {+ let int = Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int32()),+ vec![+ TypeSignatureClass::Integer,+ TypeSignatureClass::Native(logical_string()),+ ],+ NativeType::Int32,+ );Self {- signature: Signature::uniform(- 3,- vec![Int32, Int64, UInt32, UInt64, Utf8, Utf8View],- Volatility::Immutable,- ),+ signature: Signature::coercible(vec![int; 3], Volatility::Immutable),}}}@@ -110,87 +116,59 @@ impl ScalarUDFImpl for MakeDateFunc {&self,args: datafusion_expr::ScalarFunctionArgs,) -> Result<ColumnarValue> {- // first, identify if any of the arguments is an Array. If yes, store its `len`,- // as any scalar will need to be converted to an array of len `len`.- let args = args.args;- let len = args- .iter()- .fold(Option::<usize>::None, |acc, arg| match arg {- ColumnarValue::Scalar(_) => acc,- ColumnarValue::Array(a) => Some(a.len()),- });+ let [years, months, days] = take_function_args(self.name(), args.args)?;- let [years, months, days] = take_function_args(self.name(), args)?;-- if matches!(years, ColumnarValue::Scalar(ScalarValue::Null))- || matches!(months, ColumnarValue::Scalar(ScalarValue::Null))- || matches!(days, ColumnarValue::Scalar(ScalarValue::Null))- {- return Ok(ColumnarValue::Scalar(ScalarValue::Null));- }-- let years = years.cast_to(&Int32, None)?;- let months = months.cast_to(&Int32, None)?;- let days = days.cast_to(&Int32, None)?;-- let scalar_value_fn = |col: &ColumnarValue| -> Result<i32> {- let ColumnarValue::Scalar(s) = col else {- return exec_err!("Expected scalar value");- };- let ScalarValue::Int32(Some(i)) = s else {- return exec_err!("Unable to parse date from null/empty value");- };- Ok(*i)- };-- let value = if let Some(array_size) = len {- let to_primitive_array_fn =- |col: &ColumnarValue| -> PrimitiveArray<Int32Type> {- match col {- ColumnarValue::Array(a) => {- a.as_primitive::<Int32Type>().to_owned()- }- _ => {- let v = scalar_value_fn(col).unwrap();- PrimitiveArray::<Int32Type>::from_value(v, array_size)- }+ match (years, months, days) {+ (ColumnarValue::Scalar(y), _, _) if y.is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Date32(None)))+ }+ (_, ColumnarValue::Scalar(m), _) if m.is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Date32(None)))+ }+ (_, _, ColumnarValue::Scalar(d)) if d.is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Date32(None)))+ }+ (+ ColumnarValue::Scalar(ScalarValue::Int32(Some(years))),+ ColumnarValue::Scalar(ScalarValue::Int32(Some(months))),+ ColumnarValue::Scalar(ScalarValue::Int32(Some(days))),+ ) => {+ let mut value = 0;+ make_date_inner(years, months, days, |days: i32| value = days)?;+ Ok(ColumnarValue::Scalar(ScalarValue::Date32(Some(value))))+ }+ (years, months, days) => {+ let len = args.number_rows;+ let years = years.into_array(len)?;+ let months = months.into_array(len)?;+ let days = days.into_array(len)?;++ let years = years.as_primitive::<Int32Type>();+ let months = months.as_primitive::<Int32Type>();+ let days = days.as_primitive::<Int32Type>();++ let mut builder: PrimitiveBuilder<Date32Type> =+ PrimitiveArray::builder(len);++ for i in 0..len {+ // match postgresql behaviour which returns null for any null input+ if years.is_null(i) || months.is_null(i) || days.is_null(i) {+ builder.append_null();+ } else {+ make_date_inner(+ years.value(i),+ months.value(i),+ days.value(i),+ |days: i32| builder.append_value(days),+ )?;}- };-- let years = to_primitive_array_fn(&years);- let months = to_primitive_array_fn(&months);- let days = to_primitive_array_fn(&days);+ }- let mut builder: PrimitiveBuilder<Date32Type> =- PrimitiveArray::builder(array_size);- for i in 0..array_size {- make_date_inner(- years.value(i),- months.value(i),- days.value(i),- |days: i32| builder.append_value(days),- )?;+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))}-- let arr = builder.finish();-- ColumnarValue::Array(Arc::new(arr))- } else {- // For scalar only columns the operation is faster without using the PrimitiveArray.- // Also, keep the output as scalar since all inputs are scalar.- let mut value = 0;- make_date_inner(- scalar_value_fn(&years)?,- scalar_value_fn(&months)?,- scalar_value_fn(&days)?,- |days: i32| value = days,- )?;-- ColumnarValue::Scalar(ScalarValue::Date32(Some(value)))- };-- Ok(value)+ }}+fn documentation(&self) -> Option<&Documentation> {self.doc()}@@ -204,11 +182,13 @@ fn make_date_inner<F: FnMut(i32)>(day: i32,mut date_consumer_fn: F,) -> Result<()> {- let Ok(m) = u32::try_from(month) else {- return exec_err!("Month value '{month:?}' is out of range");+ let m = match month {+ 1..=12 => month as u32,+ _ => return exec_err!("Month value '{month:?}' is out of range"),};- let Ok(d) = u32::try_from(day) else {- return exec_err!("Day value '{day:?}' is out of range");+ let d = match day {+ 1..=31 => day as u32,+ _ => return exec_err!("Day value '{day:?}' is out of range"),};if let Some(date) = NaiveDate::from_ymd_opt(year, m, d) {@@ -225,180 +205,3 @@ fn make_date_inner<F: FnMut(i32)>(exec_err!("Unable to parse date from {year}, {month}, {day}")}}--#[cfg(test)]-mod tests {- use crate::datetime::make_date::MakeDateFunc;- use arrow::array::{Array, Date32Array, Int32Array, Int64Array, UInt32Array};- use arrow::datatypes::{DataType, Field};- use datafusion_common::config::ConfigOptions;- use datafusion_common::{DataFusionError, ScalarValue};- use datafusion_expr::{ColumnarValue, ScalarUDFImpl};- use std::sync::Arc;-- fn invoke_make_date_with_args(- args: Vec<ColumnarValue>,- number_rows: usize,- ) -> Result<ColumnarValue, DataFusionError> {- let arg_fields = args- .iter()- .map(|arg| Field::new("a", arg.data_type(), true).into())- .collect::<Vec<_>>();- let args = datafusion_expr::ScalarFunctionArgs {- args,- arg_fields,- number_rows,- return_field: Field::new("f", DataType::Date32, true).into(),- config_options: Arc::new(ConfigOptions::default()),- };- MakeDateFunc::new().invoke_with_args(args)- }-- #[test]- fn test_make_date() {- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::Int32(Some(2024))),- ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),- ColumnarValue::Scalar(ScalarValue::UInt32(Some(14))),- ],- 1,- )- .expect("that make_date parsed values without error");-- if let ColumnarValue::Scalar(ScalarValue::Date32(date)) = res {- assert_eq!(19736, date.unwrap());- } else {- panic!("Expected a scalar value")- }-- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::Int64(Some(2024))),- ColumnarValue::Scalar(ScalarValue::UInt64(Some(1))),- ColumnarValue::Scalar(ScalarValue::UInt32(Some(14))),- ],- 1,- )- .expect("that make_date parsed values without error");-- if let ColumnarValue::Scalar(ScalarValue::Date32(date)) = res {- assert_eq!(19736, date.unwrap());- } else {- panic!("Expected a scalar value")- }-- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::Utf8(Some("2024".to_string()))),- ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("1".to_string()))),- ColumnarValue::Scalar(ScalarValue::Utf8(Some("14".to_string()))),- ],- 1,- )- .expect("that make_date parsed values without error");-- if let ColumnarValue::Scalar(ScalarValue::Date32(date)) = res {- assert_eq!(19736, date.unwrap());- } else {- panic!("Expected a scalar value")- }-- let years = Arc::new((2021..2025).map(Some).collect::<Int64Array>());- let months = Arc::new((1..5).map(Some).collect::<Int32Array>());- let days = Arc::new((11..15).map(Some).collect::<UInt32Array>());- let batch_len = years.len();- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Array(years),- ColumnarValue::Array(months),- ColumnarValue::Array(days),- ],- batch_len,- )- .unwrap();-- if let ColumnarValue::Array(array) = res {- assert_eq!(array.len(), 4);- let mut builder = Date32Array::builder(4);- builder.append_value(18_638);- builder.append_value(19_035);- builder.append_value(19_429);- builder.append_value(19_827);- assert_eq!(&builder.finish() as &dyn Array, array.as_ref());- } else {- panic!("Expected a columnar array")- }-- //- // Fallible test cases- //-- // invalid number of arguments- let res = invoke_make_date_with_args(- vec![ColumnarValue::Scalar(ScalarValue::Int32(Some(1)))],- 1,- );- assert_eq!(- res.err().unwrap().strip_backtrace(),- "Execution error: make_date function requires 3 arguments, got 1"- );-- // invalid type- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::IntervalYearMonth(Some(1))),- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)),- ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(Some(1), None)),- ],- 1,- );- assert_eq!(- res.err().unwrap().strip_backtrace(),- "Arrow error: Cast error: Casting from Interval(YearMonth) to Int32 not supported"- );-- // overflow of month- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::Int32(Some(2023))),- ColumnarValue::Scalar(ScalarValue::UInt64(Some(u64::MAX))),- ColumnarValue::Scalar(ScalarValue::Int32(Some(22))),- ],- 1,- );- assert_eq!(- res.err().unwrap().strip_backtrace(),- "Arrow error: Cast error: Can't cast value 18446744073709551615 to type Int32"- );-- // overflow of day- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::Int32(Some(2023))),- ColumnarValue::Scalar(ScalarValue::Int32(Some(22))),- ColumnarValue::Scalar(ScalarValue::UInt32(Some(u32::MAX))),- ],- 1,- );- assert_eq!(- res.err().unwrap().strip_backtrace(),- "Arrow error: Cast error: Can't cast value 4294967295 to type Int32"- );- }-- #[test]- fn test_make_date_null_param() {- let res = invoke_make_date_with_args(- vec![- ColumnarValue::Scalar(ScalarValue::Null),- ColumnarValue::Scalar(ScalarValue::Int64(Some(1))),- ColumnarValue::Scalar(ScalarValue::UInt32(Some(14))),- ],- 1,- )- .expect("that make_date parsed values without error");-- assert!(matches!(res, ColumnarValue::Scalar(ScalarValue::Null)));- }-}diff --git a/datafusion/functions/src/datetime/make_time.rs b/datafusion/functions/src/datetime/make_time.rsindex 7083bf3185..5775ce851a 100644--- a/datafusion/functions/src/datetime/make_time.rs+++ b/datafusion/functions/src/datetime/make_time.rs@@ -90,10 +90,7 @@ impl MakeTimeFunc {NativeType::Int32,);Self {- signature: Signature::coercible(- vec![int.clone(), int.clone(), int.clone()],- Volatility::Immutable,- ),+ signature: Signature::coercible(vec![int; 3], Volatility::Immutable),}}}diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.sltindex 6bed34e0a2..6ad6c799f2 100644--- a/datafusion/sqllogictest/test_files/timestamps.slt+++ b/datafusion/sqllogictest/test_files/timestamps.slt@@ -2847,8 +2847,12 @@ select make_date(t.year, t.month, '4') from table_nums t;statement okinsert into table_nums values (2024, null, 23);-query error DataFusion error: Execution error: Unable to parse date from 2024, 0, 23+query Dselect make_date(t.year, t.month, t.day) from table_nums t;+----+2024-01-23+2023-11-30+NULLstatement okdrop table table_nums;@@ -2867,48 +2871,71 @@ select make_date(t.year, t.month, t.day) from table_strings t;statement okinsert into table_strings values (2024, null, 23);-query error DataFusion error: Execution error: Unable to parse date from 2024, 0, 23+query Dselect make_date(t.year, t.month, t.day) from table_strings t;+----+2024-01-23+2023-11-30+NULLstatement okdrop table table_strings;-query error DataFusion error: Execution error: Unable to parse date from 2024, 13, 23+query error DataFusion error: Execution error: Month value '13' is out of rangeselect make_date(2024, 13, 23);-query error DataFusion error: Execution error: Unable to parse date from 2024, 1, 32-select make_date(2024, 01, 32);+query error DataFusion error: Execution error: Day value '32' is out of range+select make_date(2024, 1, 32);-query error DataFusion error: Execution error: Unable to parse date from 2024, 0, 23+query error DataFusion error: Execution error: Month value '0' is out of rangeselect make_date(2024, 0, 23);query error DataFusion error: Execution error: Month value '\-1' is out of rangeselect make_date(2024, -1, 23);-query error DataFusion error: Execution error: Unable to parse date from 2024, 12, 0+query error DataFusion error: Execution error: Day value '0' is out of rangeselect make_date(2024, 12, 0);-query error DataFusion error: Execution error: Day value '\-1' is out of range+query error DataFusion error: Execution error: Month value '13' is out of rangeselect make_date(2024, 13, -1);-query error DataFusion error: Execution error: Unable to parse date from null/empty value+query Dselect make_date(null, 1, 23);+----+NULL-query error DataFusion error: Arrow error: Cast error: Cannot cast string '' to value of Int32 type+query error Cast error: Cannot cast string '' to value of Int32 typeselect make_date('', 1, 23);-query error DataFusion error: Execution error: Unable to parse date from null/empty value+query Dselect make_date(2024, null, 23);+----+NULL-query error DataFusion error: Arrow error: Cast error: Cannot cast string '' to value of Int32 type+query error Arrow error: Cast error: Cannot cast string '' to value of Int32 typeselect make_date(2024, '', 27);-query error DataFusion error: Execution error: Unable to parse date from null/empty value+query Dselect make_date(2024, 1, null);+----+NULL-query error DataFusion error: Arrow error: Cast error: Cannot cast string '' to value of Int32 type+query error Arrow error: Cast error: Cannot cast string '' to value of Int32 typeselect make_date(2024, 1, '');+query error DataFusion error: Execution error: Unable to parse date from 2024, 11, 31+select make_date(2024, 11, 31);++query D+select make_date(null, 1::bigint, 14::bigint unsigned);+----+NULL++query error DataFusion error: Error during planning: Function 'make_date' expects 3 arguments but received 1+select make_date(1);++query error Expect TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\) but received NativeType::Interval\(MonthDayNano\), DataType: Interval\(MonthDayNano\)+select make_date(interval '1 day', '2001-05-21'::timestamp, '2001-05-21'::timestamp);############ make time tests