Apache DataFusion SQL Query Engine
feat(spark): implement add_months function (#19711)
cht4235ebe7955e6b
diff --git a/datafusion/spark/src/function/datetime/add_months.rs b/datafusion/spark/src/function/datetime/add_months.rsnew file mode 100644index 0000000000..fa9f6fa8db--- /dev/null+++ b/datafusion/spark/src/function/datetime/add_months.rs@@ -0,0 +1,95 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::any::Any;+use std::ops::Add;+use std::sync::Arc;++use arrow::datatypes::{DataType, Field, FieldRef, IntervalUnit};+use datafusion_common::utils::take_function_args;+use datafusion_common::{Result, internal_err};+use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};+use datafusion_expr::{+ ColumnarValue, Expr, ExprSchemable, ReturnFieldArgs, ScalarFunctionArgs,+ ScalarUDFImpl, Signature, Volatility,+};++/// <https://spark.apache.org/docs/latest/api/sql/index.html#add_months>+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct SparkAddMonths {+ signature: Signature,+}++impl Default for SparkAddMonths {+ fn default() -> Self {+ Self::new()+ }+}++impl SparkAddMonths {+ pub fn new() -> Self {+ Self {+ signature: Signature::exact(+ vec![DataType::Date32, DataType::Int32],+ Volatility::Immutable,+ ),+ }+ }+}++impl ScalarUDFImpl for SparkAddMonths {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "add_months"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());++ Ok(Arc::new(Field::new(+ self.name(),+ DataType::Date32,+ nullable,+ )))+ }++ fn simplify(+ &self,+ args: Vec<Expr>,+ info: &SimplifyContext,+ ) -> Result<ExprSimplifyResult> {+ let [date_arg, months_arg] = take_function_args("add_months", args)?;+ let interval = months_arg+ .cast_to(&DataType::Interval(IntervalUnit::YearMonth), info.schema())?;+ Ok(ExprSimplifyResult::Simplified(date_arg.add(interval)))+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ internal_err!("invoke should not be called on a simplified add_months() function")+ }+}diff --git a/datafusion/spark/src/function/datetime/mod.rs b/datafusion/spark/src/function/datetime/mod.rsindex 7c270c1440..7d6c9e9493 100644--- a/datafusion/spark/src/function/datetime/mod.rs+++ b/datafusion/spark/src/function/datetime/mod.rs@@ -15,6 +15,7 @@// specific language governing permissions and limitations// under the License.+pub mod add_months;pub mod date_add;pub mod date_diff;pub mod date_part;@@ -32,6 +33,7 @@ use datafusion_expr::ScalarUDF;use datafusion_functions::make_udf_function;use std::sync::Arc;+make_udf_function!(add_months::SparkAddMonths, add_months);make_udf_function!(date_add::SparkDateAdd, date_add);make_udf_function!(date_diff::SparkDateDiff, date_diff);make_udf_function!(date_part::SparkDatePart, date_part);@@ -50,6 +52,11 @@ make_udf_function!(trunc::SparkTrunc, trunc);pub mod expr_fn {use datafusion_functions::export_functions;+ export_functions!((+ add_months,+ "Returns the date that is months months after start. The function returns NULL if at least one of the input parameters is NULL.",+ arg1 arg2+ ));export_functions!((date_add,"Returns the date that is days days after start. The function returns NULL if at least one of the input parameters is NULL.",@@ -122,6 +129,7 @@ pub mod expr_fn {pub fn functions() -> Vec<Arc<ScalarUDF>> {vec![+ add_months(),date_add(),date_diff(),date_part(),diff --git a/datafusion/sqllogictest/test_files/spark/datetime/add_months.slt b/datafusion/sqllogictest/test_files/spark/datetime/add_months.sltindex cae9b21dd4..55a493ffef 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/add_months.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/add_months.slt@@ -15,13 +15,45 @@# specific language governing permissions and limitations# under the License.-# This file was originally created by a porting script from:-# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function-# This file is part of the implementation of the datafusion-spark function library.-# For more information, please see:-# https://github.com/apache/datafusion/issues/15914--## Original Query: SELECT add_months('2016-08-31', 1);-## PySpark 3.5.5 Result: {'add_months(2016-08-31, 1)': datetime.date(2016, 9, 30), 'typeof(add_months(2016-08-31, 1))': 'date', 'typeof(2016-08-31)': 'string', 'typeof(1)': 'int'}-#query-#SELECT add_months('2016-08-31'::string, 1::int);+query D+SELECT add_months('2016-07-30'::date, 1::int);+----+2016-08-30++query D+SELECT add_months('2016-07-30'::date, 0::int);+----+2016-07-30++query D+SELECT add_months('2016-07-30'::date, 10000::int);+----+2849-11-30++# Test integer overflow+# TODO: Enable with next arrow upgrade (>=58.0.0)+# query D+# SELECT add_months('2016-07-30'::date, 2147483647::int);+# ----+# NULL++query D+SELECT add_months('2016-07-30'::date, -5::int);+----+2016-02-29++# Test with NULL values+query D+SELECT add_months(NULL::date, 1::int);+----+NULL++query D+SELECT add_months('2016-07-30'::date, NULL::int);+----+NULL++query D+SELECT add_months(NULL::date, NULL::int);+----+NULLdiff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_add.sltindex a2ac7cf2ed..cb407a6453 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt@@ -41,7 +41,7 @@ SELECT date_add('2016-07-30'::date, arrow_cast(1, 'Int8'));2016-07-31query D-SELECT date_sub('2016-07-30'::date, 0::int);+SELECT date_add('2016-07-30'::date, 0::int);----2016-07-30@@ -51,20 +51,15 @@ SELECT date_add('2016-07-30'::date, 2147483647::int)::int;-2147466637query I-SELECT date_sub('1969-01-01'::date, 2147483647::int)::int;+SELECT date_add('1969-01-01'::date, 2147483647::int)::int;-----2147483284+2147483282query DSELECT date_add('2016-07-30'::date, 100000::int);----2290-05-15-query D-SELECT date_sub('2016-07-30'::date, 100000::int);------1742-10-15-# Test with negative day values (should subtract days)query DSELECT date_add('2016-07-30'::date, -5::int);diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_sub.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_sub.sltindex cb5e77c3b4..bf36ebd867 100644--- a/datafusion/sqllogictest/test_files/spark/datetime/date_sub.slt+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_sub.slt@@ -45,6 +45,16 @@ SELECT date_sub('2016-07-30'::date, 0::int);----2016-07-30+query I+SELECT date_sub('1969-01-01'::date, 2147483647::int)::int;+----+2147483284++query D+SELECT date_sub('2016-07-30'::date, 100000::int);+----+1742-10-15+# Test with negative day values (should add days)query DSELECT date_sub('2016-07-30'::date, -1::int);