Apache DataFusion SQL Query Engine
feat(spark): Adds negative spark function (#20006)
Subham Singhald544735a5fb0
diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rsindex bf212a8219..92d8e90ac3 100644--- a/datafusion/spark/src/function/math/mod.rs+++ b/datafusion/spark/src/function/math/mod.rs@@ -20,6 +20,7 @@ pub mod expm1;pub mod factorial;pub mod hex;pub mod modulus;+pub mod negative;pub mod rint;pub mod trigonometry;pub mod unhex;@@ -40,6 +41,7 @@ make_udf_function!(unhex::SparkUnhex, unhex);make_udf_function!(width_bucket::SparkWidthBucket, width_bucket);make_udf_function!(trigonometry::SparkCsc, csc);make_udf_function!(trigonometry::SparkSec, sec);+make_udf_function!(negative::SparkNegative, negative);pub mod expr_fn {use datafusion_functions::export_functions;@@ -63,6 +65,11 @@ pub mod expr_fn {export_functions!((width_bucket, "Returns the bucket number into which the value of this expression would fall after being evaluated.", arg1 arg2 arg3 arg4));export_functions!((csc, "Returns the cosecant of expr.", arg1));export_functions!((sec, "Returns the secant of expr.", arg1));+ export_functions!((+ negative,+ "Returns the negation of expr (unary minus).",+ arg1+ ));}pub fn functions() -> Vec<Arc<ScalarUDF>> {@@ -78,5 +85,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {width_bucket(),csc(),sec(),+ negative(),]}diff --git a/datafusion/spark/src/function/math/negative.rs b/datafusion/spark/src/function/math/negative.rsnew file mode 100644index 0000000000..f1803d2d77--- /dev/null+++ b/datafusion/spark/src/function/math/negative.rs@@ -0,0 +1,293 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::types::*;+use arrow::array::*;+use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit};+use bigdecimal::num_traits::WrappingNeg;+use datafusion_common::utils::take_function_args;+use datafusion_common::{Result, ScalarValue, not_impl_err};+use datafusion_expr::{+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,+ Volatility,+};+use std::any::Any;+use std::sync::Arc;++/// Spark-compatible `negative` expression+/// <https://spark.apache.org/docs/latest/api/sql/index.html#negative>+///+/// Returns the negation of input (equivalent to unary minus)+/// Returns NULL if input is NULL, returns NaN if input is NaN.+///+/// ANSI mode support see (<https://github.com/apache/datafusion/issues/20034>):+/// - Spark's ANSI-compliant dialect, when off (i.e. `spark.sql.ansi.enabled=false`),+/// negating the minimal value of a signed integer wraps around.+/// For example: negative(i32::MIN) returns i32::MIN (wraps instead of error).+/// This is the current implementation (legacy mode only).+/// - Spark's ANSI mode (when `spark.sql.ansi.enabled=true`) should throw an+/// ARITHMETIC_OVERFLOW error on integer overflow instead of wrapping.+/// This is not yet implemented - all operations currently use wrapping behavior.+///+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct SparkNegative {+ signature: Signature,+}++impl Default for SparkNegative {+ fn default() -> Self {+ Self::new()+ }+}++impl SparkNegative {+ pub fn new() -> Self {+ Self {+ signature: Signature {+ type_signature: TypeSignature::OneOf(vec![+ // Numeric types: signed integers, float, decimals+ TypeSignature::Numeric(1),+ // Interval types: YearMonth, DayTime, MonthDayNano+ TypeSignature::Uniform(+ 1,+ vec![+ DataType::Interval(IntervalUnit::YearMonth),+ DataType::Interval(IntervalUnit::DayTime),+ DataType::Interval(IntervalUnit::MonthDayNano),+ ],+ ),+ ]),+ volatility: Volatility::Immutable,+ parameter_names: None,+ },+ }+ }+}++impl ScalarUDFImpl for SparkNegative {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "negative"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {+ Ok(arg_types[0].clone())+ }++ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ spark_negative(&args.args)+ }+}++/// Core implementation of Spark's negative function+fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {+ let [arg] = take_function_args("negative", args)?;++ match arg {+ ColumnarValue::Array(array) => match array.data_type() {+ DataType::Null => Ok(arg.clone()),++ // Signed integers - use wrapping negation (Spark legacy mode behavior)+ DataType::Int8 => {+ let array = array.as_primitive::<Int8Type>();+ let result: PrimitiveArray<Int8Type> = array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Int16 => {+ let array = array.as_primitive::<Int16Type>();+ let result: PrimitiveArray<Int16Type> = array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Int32 => {+ let array = array.as_primitive::<Int32Type>();+ let result: PrimitiveArray<Int32Type> = array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Int64 => {+ let array = array.as_primitive::<Int64Type>();+ let result: PrimitiveArray<Int64Type> = array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }++ // Floating point - simple negation (no overflow possible)+ DataType::Float16 => {+ let array = array.as_primitive::<Float16Type>();+ let result: PrimitiveArray<Float16Type> = array.unary(|x| -x);+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Float32 => {+ let array = array.as_primitive::<Float32Type>();+ let result: PrimitiveArray<Float32Type> = array.unary(|x| -x);+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Float64 => {+ let array = array.as_primitive::<Float64Type>();+ let result: PrimitiveArray<Float64Type> = array.unary(|x| -x);+ Ok(ColumnarValue::Array(Arc::new(result)))+ }++ // Decimal types - wrapping negation+ DataType::Decimal32(_, _) => {+ let array = array.as_primitive::<Decimal32Type>();+ let result: PrimitiveArray<Decimal32Type> =+ array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Decimal64(_, _) => {+ let array = array.as_primitive::<Decimal64Type>();+ let result: PrimitiveArray<Decimal64Type> =+ array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Decimal128(_, _) => {+ let array = array.as_primitive::<Decimal128Type>();+ let result: PrimitiveArray<Decimal128Type> =+ array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Decimal256(_, _) => {+ let array = array.as_primitive::<Decimal256Type>();+ let result: PrimitiveArray<Decimal256Type> =+ array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }++ // interval type+ DataType::Interval(IntervalUnit::YearMonth) => {+ let array = array.as_primitive::<IntervalYearMonthType>();+ let result: PrimitiveArray<IntervalYearMonthType> =+ array.unary(|x| x.wrapping_neg());+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Interval(IntervalUnit::DayTime) => {+ let array = array.as_primitive::<IntervalDayTimeType>();+ let result: PrimitiveArray<IntervalDayTimeType> =+ array.unary(|x| IntervalDayTime {+ days: x.days.wrapping_neg(),+ milliseconds: x.milliseconds.wrapping_neg(),+ });+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ DataType::Interval(IntervalUnit::MonthDayNano) => {+ let array = array.as_primitive::<IntervalMonthDayNanoType>();+ let result: PrimitiveArray<IntervalMonthDayNanoType> =+ array.unary(|x| IntervalMonthDayNano {+ months: x.months.wrapping_neg(),+ days: x.days.wrapping_neg(),+ nanoseconds: x.nanoseconds.wrapping_neg(),+ });+ Ok(ColumnarValue::Array(Arc::new(result)))+ }++ dt => not_impl_err!("Not supported datatype for Spark negative(): {dt}"),+ },+ ColumnarValue::Scalar(sv) => match sv {+ ScalarValue::Null => Ok(arg.clone()),+ _ if sv.is_null() => Ok(arg.clone()),++ // Signed integers - wrapping negation+ ScalarValue::Int8(Some(v)) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Int8(Some(result))))+ }+ ScalarValue::Int16(Some(v)) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Int16(Some(result))))+ }+ ScalarValue::Int32(Some(v)) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(result))))+ }+ ScalarValue::Int64(Some(v)) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some(result))))+ }++ // Floating point - simple negation+ ScalarValue::Float16(Some(v)) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Float16(Some(-v))))+ }+ ScalarValue::Float32(Some(v)) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Float32(Some(-v))))+ }+ ScalarValue::Float64(Some(v)) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Float64(Some(-v))))+ }++ // Decimal types - wrapping negation+ ScalarValue::Decimal32(Some(v), precision, scale) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Decimal32(+ Some(result),+ *precision,+ *scale,+ )))+ }+ ScalarValue::Decimal64(Some(v), precision, scale) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Decimal64(+ Some(result),+ *precision,+ *scale,+ )))+ }+ ScalarValue::Decimal128(Some(v), precision, scale) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Decimal128(+ Some(result),+ *precision,+ *scale,+ )))+ }+ ScalarValue::Decimal256(Some(v), precision, scale) => {+ let result = v.wrapping_neg();+ Ok(ColumnarValue::Scalar(ScalarValue::Decimal256(+ Some(result),+ *precision,+ *scale,+ )))+ }++ //interval type+ ScalarValue::IntervalYearMonth(Some(v)) => Ok(ColumnarValue::Scalar(+ ScalarValue::IntervalYearMonth(Some(v.wrapping_neg())),+ )),+ ScalarValue::IntervalDayTime(Some(v)) => Ok(ColumnarValue::Scalar(+ ScalarValue::IntervalDayTime(Some(IntervalDayTime {+ days: v.days.wrapping_neg(),+ milliseconds: v.milliseconds.wrapping_neg(),+ })),+ )),+ ScalarValue::IntervalMonthDayNano(Some(v)) => Ok(ColumnarValue::Scalar(+ ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano {+ months: v.months.wrapping_neg(),+ days: v.days.wrapping_neg(),+ nanoseconds: v.nanoseconds.wrapping_neg(),+ })),+ )),++ dt => not_impl_err!("Not supported datatype for Spark negative(): {dt}"),+ },+ }+}diff --git a/datafusion/sqllogictest/test_files/spark/math/negative.slt b/datafusion/sqllogictest/test_files/spark/math/negative.sltindex aa8e558e98..c62267e496 100644--- a/datafusion/sqllogictest/test_files/spark/math/negative.slt+++ b/datafusion/sqllogictest/test_files/spark/math/negative.slt@@ -23,5 +23,257 @@## Original Query: SELECT negative(1);## PySpark 3.5.5 Result: {'negative(1)': -1, 'typeof(negative(1))': 'int', 'typeof(1)': 'int'}-#query-#SELECT negative(1::int);++# Test negative with integer+query I+SELECT negative(1::int);+----+-1++# Test negative with positive integer+query I+SELECT negative(42::int);+----+-42++# Test negative with negative integer+query I+SELECT negative(-10::int);+----+10++# Test negative with zero+query I+SELECT negative(0::int);+----+0++# Test negative with bigint+query I+SELECT negative(9223372036854775807::bigint);+----+-9223372036854775807++# Test negative with negative bigint+query I+SELECT negative(-100::bigint);+----+100++# Test negative with smallint+query I+SELECT negative(32767::smallint);+----+-32767++# Test negative with float+query R+SELECT negative(3.14::float);+----+-3.14++# Test negative with negative float+query R+SELECT negative(-2.5::float);+----+2.5++# Test negative with double+query R+SELECT negative(3.14159265358979::double);+----+-3.14159265358979++# Test negative with negative double+query R+SELECT negative(-1.5::double);+----+1.5++# Test negative with decimal+query R+SELECT negative(123.456::decimal(10,3));+----+-123.456++# Test negative with negative decimal+query R+SELECT negative(-99.99::decimal(10,2));+----+99.99++# Test negative with NULL+query I+SELECT negative(NULL::int);+----+NULL++# Test negative with column values+statement ok+CREATE TABLE test_negative (id int, value int) AS VALUES (1, 10), (2, -20), (3, 0), (4, NULL);++query II rowsort+SELECT id, negative(value) FROM test_negative;+----+1 -10+2 20+3 0+4 NULL++statement ok+DROP TABLE test_negative;++# Test negative in expressions+query I+SELECT negative(5) + 3;+----+-2++# Test nested negative+query I+SELECT negative(negative(7));+----+7++# Test negative with large numbers+query R+SELECT negative(1234567890.123456::double);+----+-1234567890.123456++# Test wrap-around: negative of minimum int (should wrap to same value)+# Using table to avoid constant folding overflow during optimization+statement ok+CREATE TABLE min_values_int AS VALUES (-2147483648);++query I+SELECT negative(column1::int) FROM min_values_int;+----+-2147483648++statement ok+DROP TABLE min_values_int;++# Test wrap-around: negative of minimum bigint (should wrap to same value)+statement ok+CREATE TABLE min_values_bigint AS VALUES (-9223372036854775808);++query I+SELECT negative(column1::bigint) FROM min_values_bigint;+----+-9223372036854775808++statement ok+DROP TABLE min_values_bigint;++# Test wrap-around: negative of minimum smallint (should wrap to same value)+statement ok+CREATE TABLE min_values_smallint AS VALUES (-32768);++query I+SELECT negative(column1::smallint) FROM min_values_smallint;+----+-32768++statement ok+DROP TABLE min_values_smallint;++# Test wrap-around: negative of minimum tinyint (should wrap to same value)+statement ok+CREATE TABLE min_values_tinyint AS VALUES (-128);++query I+SELECT negative(column1::tinyint) FROM min_values_tinyint;+----+-128++statement ok+DROP TABLE min_values_tinyint;++# Test overflow: negative of positive infinity (float)+query R+SELECT negative('Infinity'::float);+----+-Infinity++# Test overflow: negative of negative infinity (float)+query R+SELECT negative('-Infinity'::float);+----+Infinity++# Test overflow: negative of positive infinity (double)+query R+SELECT negative('Infinity'::double);+----+-Infinity++# Test overflow: negative of negative infinity (double)+query R+SELECT negative('-Infinity'::double);+----+Infinity++# Test overflow: negative of NaN (float)+query R+SELECT negative('NaN'::float);+----+NaN++# Test overflow: negative of NaN (double)+query R+SELECT negative('NaN'::double);+----+NaN++# Test overflow: negative of maximum float value+query R+SELECT negative(3.4028235e38::float);+----+-340282350000000000000000000000000000000++# Test overflow: negative of minimum float value+query R+SELECT negative(-3.4028235e38::float);+----+340282350000000000000000000000000000000++# Test overflow: negative of maximum double value+query R+SELECT negative(1.7976931348623157e308::double);+----+-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000++# Test overflow: negative of minimum double value+query R+SELECT negative(-1.7976931348623157e308::double);+----+179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000++# Test negative with CalendarIntervalType (IntervalMonthDayNano)+# Spark make_interval creates CalendarInterval+query ?+SELECT negative(make_interval(1, 2, 3, 4, 5, 6, 7.5));+----+-14 mons -25 days -5 hours -6 mins -7.500000000 secs++# Test negative with negative CalendarIntervalType+query ?+SELECT negative(make_interval(-2, -5, -1, -10, -3, -30, -15.25));+----+29 mons 17 days 3 hours 30 mins 15.250000000 secs++# Test negative with CalendarInterval from table+statement ok+CREATE TABLE interval_test AS VALUES+ (make_interval(1, 2, 0, 5, 0, 0, 0.0)),+ (make_interval(-3, -1, 0, -2, 0, 0, 0.0));++query ? rowsort+SELECT negative(column1) FROM interval_test;+----+-14 mons -5 days+37 mons 2 days++statement ok+DROP TABLE interval_test;