Skip to content

Apache DataFusion SQL Query Engine

fix(spark): accept STRING and TIMESTAMP start dates in next_day (#26127)

lazzypanda7d2bd93cc09ed
diff --git a/datafusion/spark/src/function/datetime/next_day.rs b/datafusion/spark/src/function/datetime/next_day.rs
index 5cd52edcf4..92623367a1 100644
--- a/datafusion/spark/src/function/datetime/next_day.rs
+++ b/datafusion/spark/src/function/datetime/next_day.rs
@@ -7,10 +7,11 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, AsArray, Date32Array, StringArrayType};
use arrow::datatypes::{DataType, Date32Type, Field, FieldRef};
use chrono::{Datelike, Weekday};
+use datafusion_common::types::{NativeType, logical_date, logical_string};
use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
use datafusion_expr::{
- ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- Volatility,
+ Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
+ Signature, TypeSignatureClass, Volatility,
};
/// <https://spark.apache.org/docs/latest/api/sql/index.html#next_day>
@@ -28,8 +29,24 @@ impl Default for SparkNextDay {
impl SparkNextDay {
pub fn new() -> Self {
Self {
- signature: Signature::exact(
- vec![DataType::Date32, DataType::Utf8],
+ // Spark casts the start date to DATE, so STRING and TIMESTAMP are accepted
+ signature: Signature::coercible(
+ vec![
+ Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_date()),
+ vec![
+ TypeSignatureClass::Native(logical_string()),
+ TypeSignatureClass::Timestamp,
+ ],
+ NativeType::Date,
+ ),
+ Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_string()),
+ // Preserve the previous Exact signature's weekday-to-string coercion.
+ vec![TypeSignatureClass::Any],
+ NativeType::String,
+ ),
+ ],
Volatility::Immutable,
),
}
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/next_day.slt b/datafusion/sqllogictest/test_files/spark/datetime/next_day.slt
index fa78c3d2fc..435226050b 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/next_day.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/next_day.slt
@@ -29,22 +29,22 @@ SELECT next_day('2015-01-14'::DATE, ' MO '::string);
----
NULL
-query error Failed to coerce arguments to satisfy a call to 'next_day' function
+query error Function 'next_day' expects 2 arguments but received 1
SELECT next_day('2015-07-27'::DATE);
-query error Failed to coerce arguments to satisfy a call to 'next_day' function
+query error Function 'next_day' expects 2 arguments but received 1
SELECT next_day('Sun'::string);
query error 'next_day' does not support zero arguments
SELECT next_day();
-query error Failed to coerce arguments to satisfy a call to 'next_day' function
+query error Function 'next_day' requires Date, but received Int32
SELECT next_day(1::int, 'Sun'::string);
-query error Failed to coerce arguments to satisfy a call to 'next_day' function
+query error Function 'next_day' expects 2 arguments but received 3
SELECT next_day('2015-07-27'::DATE, 'Sat'::string, 'Sun'::string);
-query error Failed to coerce arguments to satisfy a call to 'next_day' function
+query error Cannot cast string 'invalid_date' to value of Date32 type
SELECT next_day('invalid_date'::string, 'Mon'::string);
query D
@@ -52,6 +52,17 @@ SELECT next_day('2000-01-01'::DATE, 2.0::float);
----
NULL
+# Preserve implicit weekday-to-string conversions.
+query D
+SELECT next_day(DATE '2015-07-23', arrow_cast('Mon', 'Binary'));
+----
+2015-07-27
+
+query D
+SELECT next_day(DATE '2015-07-23', true);
+----
+NULL
+
query D
SELECT next_day('2020-01-01'::DATE, 'invalid_day'::string);
----
@@ -87,3 +98,69 @@ query I
SELECT arrow_cast(next_day(arrow_cast(95026230, 'Date32'), 'Tue'::string), 'Int32');
----
95026237
+
+# https://github.com/apache/datafusion/issues/23889
+# Spark casts the start date to DATE, so STRING and TIMESTAMP are accepted
+query D
+SELECT next_day('2015-07-23'::string, 'Mon'::string);
+----
+2015-07-27
+
+query D
+SELECT next_day('2015-07-23 12:12:12'::string, 'Mon'::string);
+----
+2015-07-27
+
+query D
+SELECT next_day('2015-07-23 12:12:12'::timestamp, 'Mon'::string);
+----
+2015-07-27
+
+# Timestamp-to-date conversion must round down before the Unix epoch.
+query D
+SELECT next_day(TIMESTAMP '1969-12-31 23:59:59', 'Thu');
+----
+1970-01-01
+
+query D
+SELECT next_day(a, 'Mon'::string)
+FROM VALUES
+ ('2015-07-23'::string),
+ (NULL::string) as t(a);
+----
+2015-07-27
+NULL
+
+# Timestamp column with a day-of-week column
+query D
+SELECT next_day(a, b)
+FROM VALUES
+ ('2015-07-23 12:12:12'::timestamp, 'Mon'::string),
+ ('2015-07-23 23:59:59'::timestamp, 'Fri'::string),
+ (NULL::timestamp, 'Mon'::string) as t(a, b);
+----
+2015-07-27
+2015-07-24
+NULL
+
+# LargeUtf8 / Utf8View start dates
+query D
+SELECT next_day(arrow_cast('2015-07-23', 'LargeUtf8'), 'Mon');
+----
+2015-07-27
+
+query D
+SELECT next_day(arrow_cast('2015-07-23', 'Utf8View'), 'Mon');
+----
+2015-07-27
+
+# String start date and LargeUtf8 / Utf8View day-of-week
+query D
+SELECT next_day('2015-07-23', arrow_cast('Mon', 'LargeUtf8'));
+----
+2015-07-27
+
+query D
+SELECT next_day('2015-07-23', arrow_cast('Mon', 'Utf8View'));
+----
+2015-07-27