Apache DataFusion SQL Query Engine
Adds support for ANSI mode in negative function (#20189)
Subham Singhal98cc75361fb0
diff --git a/datafusion/spark/src/function/math/negative.rs b/datafusion/spark/src/function/math/negative.rsindex f1803d2d77..2df71b709d 100644--- a/datafusion/spark/src/function/math/negative.rs+++ b/datafusion/spark/src/function/math/negative.rs@@ -20,7 +20,7 @@ use arrow::array::*;use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit};use bigdecimal::num_traits::WrappingNeg;use datafusion_common::utils::take_function_args;-use datafusion_common::{Result, ScalarValue, not_impl_err};+use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err};use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,Volatility,@@ -34,14 +34,12 @@ use std::sync::Arc;/// Returns the negation of input (equivalent to unary minus)/// Returns NULL if input is NULL, returns NaN if input is NaN.///-/// ANSI mode support see (<https://github.com/apache/datafusion/issues/20034>):-/// - Spark's ANSI-compliant dialect, when off (i.e. `spark.sql.ansi.enabled=false`),-/// negating the minimal value of a signed integer wraps around.-/// For example: negative(i32::MIN) returns i32::MIN (wraps instead of error).-/// This is the current implementation (legacy mode only).-/// - Spark's ANSI mode (when `spark.sql.ansi.enabled=true`) should throw an-/// ARITHMETIC_OVERFLOW error on integer overflow instead of wrapping.-/// This is not yet implemented - all operations currently use wrapping behavior.+/// ANSI mode support:+/// - When ANSI mode is disabled (`spark.sql.ansi.enabled=false`), negating the minimal+/// value of a signed integer wraps around. For example: negative(i32::MIN) returns+/// i32::MIN (wraps instead of error).+/// - When ANSI mode is enabled (`spark.sql.ansi.enabled=true`), overflow conditions+/// throw an ARITHMETIC_OVERFLOW error instead of wrapping.///#[derive(Debug, PartialEq, Eq, Hash)]pub struct SparkNegative {@@ -96,107 +94,231 @@ impl ScalarUDFImpl for SparkNegative {}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- spark_negative(&args.args)+ spark_negative(&args.args, args.config_options.execution.enable_ansi_mode)}}+/// Macro to implement negation for integer array types+macro_rules! impl_integer_array_negative {+ ($array:expr, $type:ty, $type_name:expr, $enable_ansi_mode:expr) => {{+ let array = $array.as_primitive::<$type>();+ let result: PrimitiveArray<$type> = if $enable_ansi_mode {+ array.try_unary(|x| {+ x.checked_neg().ok_or_else(|| {+ (exec_err!("{} overflow on negative({x})", $type_name)+ as Result<(), _>)+ .unwrap_err()+ })+ })?+ } else {+ array.unary(|x| x.wrapping_neg())+ };+ Ok(ColumnarValue::Array(Arc::new(result)))+ }};+}++/// Macro to implement negation for float array types+macro_rules! impl_float_array_negative {+ ($array:expr, $type:ty) => {{+ let array = $array.as_primitive::<$type>();+ let result: PrimitiveArray<$type> = array.unary(|x| -x);+ Ok(ColumnarValue::Array(Arc::new(result)))+ }};+}++/// Macro to implement negation for decimal array types+macro_rules! impl_decimal_array_negative {+ ($array:expr, $type:ty, $type_name:expr, $enable_ansi_mode:expr) => {{+ let array = $array.as_primitive::<$type>();+ let result: PrimitiveArray<$type> = if $enable_ansi_mode {+ array+ .try_unary(|x| {+ x.checked_neg().ok_or_else(|| {+ (exec_err!("{} overflow on negative({x})", $type_name)+ as Result<(), _>)+ .unwrap_err()+ })+ })?+ .with_data_type(array.data_type().clone())+ } else {+ array.unary(|x| x.wrapping_neg())+ };+ Ok(ColumnarValue::Array(Arc::new(result)))+ }};+}++/// Macro to implement negation for integer scalar types+macro_rules! impl_integer_scalar_negative {+ ($v:expr, $type_name:expr, $variant:ident, $enable_ansi_mode:expr) => {{+ let result = if $enable_ansi_mode {+ $v.checked_neg().ok_or_else(|| {+ (exec_err!("{} overflow on negative({})", $type_name, $v)+ as Result<(), _>)+ .unwrap_err()+ })?+ } else {+ $v.wrapping_neg()+ };+ Ok(ColumnarValue::Scalar(ScalarValue::$variant(Some(result))))+ }};+}++/// Macro to implement negation for decimal scalar types+macro_rules! impl_decimal_scalar_negative {+ ($v:expr, $precision:expr, $scale:expr, $type_name:expr, $variant:ident, $enable_ansi_mode:expr) => {{+ let result = if $enable_ansi_mode {+ $v.checked_neg().ok_or_else(|| {+ (exec_err!("{} overflow on negative({})", $type_name, $v)+ as Result<(), _>)+ .unwrap_err()+ })?+ } else {+ $v.wrapping_neg()+ };+ Ok(ColumnarValue::Scalar(ScalarValue::$variant(+ Some(result),+ *$precision,+ *$scale,+ )))+ }};+}+/// Core implementation of Spark's negative function-fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {+fn spark_negative(+ args: &[ColumnarValue],+ enable_ansi_mode: bool,+) -> Result<ColumnarValue> {let [arg] = take_function_args("negative", args)?;match arg {ColumnarValue::Array(array) => match array.data_type() {DataType::Null => Ok(arg.clone()),- // Signed integers - use wrapping negation (Spark legacy mode behavior)+ // Signed integers - use checked negation in ANSI mode, wrapping in legacy modeDataType::Int8 => {- let array = array.as_primitive::<Int8Type>();- let result: PrimitiveArray<Int8Type> = array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))+ impl_integer_array_negative!(array, Int8Type, "Int8", enable_ansi_mode)}DataType::Int16 => {- let array = array.as_primitive::<Int16Type>();- let result: PrimitiveArray<Int16Type> = array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))+ impl_integer_array_negative!(array, Int16Type, "Int16", enable_ansi_mode)}DataType::Int32 => {- let array = array.as_primitive::<Int32Type>();- let result: PrimitiveArray<Int32Type> = array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))+ impl_integer_array_negative!(array, Int32Type, "Int32", enable_ansi_mode)}DataType::Int64 => {- let array = array.as_primitive::<Int64Type>();- let result: PrimitiveArray<Int64Type> = array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))+ impl_integer_array_negative!(array, Int64Type, "Int64", enable_ansi_mode)}// Floating point - simple negation (no overflow possible)- DataType::Float16 => {- let array = array.as_primitive::<Float16Type>();- let result: PrimitiveArray<Float16Type> = array.unary(|x| -x);- Ok(ColumnarValue::Array(Arc::new(result)))- }- DataType::Float32 => {- let array = array.as_primitive::<Float32Type>();- let result: PrimitiveArray<Float32Type> = array.unary(|x| -x);- Ok(ColumnarValue::Array(Arc::new(result)))- }- DataType::Float64 => {- let array = array.as_primitive::<Float64Type>();- let result: PrimitiveArray<Float64Type> = array.unary(|x| -x);- Ok(ColumnarValue::Array(Arc::new(result)))- }+ DataType::Float16 => impl_float_array_negative!(array, Float16Type),+ DataType::Float32 => impl_float_array_negative!(array, Float32Type),+ DataType::Float64 => impl_float_array_negative!(array, Float64Type),- // Decimal types - wrapping negation- DataType::Decimal32(_, _) => {- let array = array.as_primitive::<Decimal32Type>();- let result: PrimitiveArray<Decimal32Type> =- array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))- }- DataType::Decimal64(_, _) => {- let array = array.as_primitive::<Decimal64Type>();- let result: PrimitiveArray<Decimal64Type> =- array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))- }- DataType::Decimal128(_, _) => {- let array = array.as_primitive::<Decimal128Type>();- let result: PrimitiveArray<Decimal128Type> =- array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))- }- DataType::Decimal256(_, _) => {- let array = array.as_primitive::<Decimal256Type>();- let result: PrimitiveArray<Decimal256Type> =- array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))- }+ // Decimal types - use checked negation in ANSI mode, wrapping in legacy mode+ DataType::Decimal32(_, _) => impl_decimal_array_negative!(+ array,+ Decimal32Type,+ "Decimal32",+ enable_ansi_mode+ ),+ DataType::Decimal64(_, _) => impl_decimal_array_negative!(+ array,+ Decimal64Type,+ "Decimal64",+ enable_ansi_mode+ ),+ DataType::Decimal128(_, _) => impl_decimal_array_negative!(+ array,+ Decimal128Type,+ "Decimal128",+ enable_ansi_mode+ ),+ DataType::Decimal256(_, _) => impl_decimal_array_negative!(+ array,+ Decimal256Type,+ "Decimal256",+ enable_ansi_mode+ ),- // interval type+ // interval type - use checked negation in ANSI mode, wrapping in legacy modeDataType::Interval(IntervalUnit::YearMonth) => {- let array = array.as_primitive::<IntervalYearMonthType>();- let result: PrimitiveArray<IntervalYearMonthType> =- array.unary(|x| x.wrapping_neg());- Ok(ColumnarValue::Array(Arc::new(result)))+ impl_integer_array_negative!(+ array,+ IntervalYearMonthType,+ "IntervalYearMonth",+ enable_ansi_mode+ )}DataType::Interval(IntervalUnit::DayTime) => {let array = array.as_primitive::<IntervalDayTimeType>();- let result: PrimitiveArray<IntervalDayTimeType> =+ let result: PrimitiveArray<IntervalDayTimeType> = if enable_ansi_mode {+ array.try_unary(|x| {+ let days = x.days.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalDayTime overflow on negative (days: {})",+ x.days+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ let milliseconds =+ x.milliseconds.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalDayTime overflow on negative (milliseconds: {})",+ x.milliseconds+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ Ok::<_, arrow::error::ArrowError>(IntervalDayTime {+ days,+ milliseconds,+ })+ })?+ } else {array.unary(|x| IntervalDayTime {days: x.days.wrapping_neg(),milliseconds: x.milliseconds.wrapping_neg(),- });+ })+ };Ok(ColumnarValue::Array(Arc::new(result)))}DataType::Interval(IntervalUnit::MonthDayNano) => {let array = array.as_primitive::<IntervalMonthDayNanoType>();- let result: PrimitiveArray<IntervalMonthDayNanoType> =+ let result: PrimitiveArray<IntervalMonthDayNanoType> = if enable_ansi_mode+ {+ array.try_unary(|x| {+ let months = x.months.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalMonthDayNano overflow on negative (months: {})",+ x.months+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ let days = x.days.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalMonthDayNano overflow on negative (days: {})",+ x.days+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ let nanoseconds = x.nanoseconds.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalMonthDayNano overflow on negative (nanoseconds: {})",+ x.nanoseconds+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ Ok::<_, arrow::error::ArrowError>(IntervalMonthDayNano {+ months,+ days,+ nanoseconds,+ })+ })?+ } else {array.unary(|x| IntervalMonthDayNano {months: x.months.wrapping_neg(),days: x.days.wrapping_neg(),nanoseconds: x.nanoseconds.wrapping_neg(),- });+ })+ };Ok(ColumnarValue::Array(Arc::new(result)))}@@ -206,22 +328,18 @@ fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {ScalarValue::Null => Ok(arg.clone()),_ if sv.is_null() => Ok(arg.clone()),- // Signed integers - wrapping negation+ // Signed integers - use checked negation in ANSI mode, wrapping in legacy modeScalarValue::Int8(Some(v)) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Int8(Some(result))))+ impl_integer_scalar_negative!(v, "Int8", Int8, enable_ansi_mode)}ScalarValue::Int16(Some(v)) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Int16(Some(result))))+ impl_integer_scalar_negative!(v, "Int16", Int16, enable_ansi_mode)}ScalarValue::Int32(Some(v)) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(result))))+ impl_integer_scalar_negative!(v, "Int32", Int32, enable_ansi_mode)}ScalarValue::Int64(Some(v)) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some(result))))+ impl_integer_scalar_negative!(v, "Int64", Int64, enable_ansi_mode)}// Floating point - simple negation@@ -235,58 +353,124 @@ fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {Ok(ColumnarValue::Scalar(ScalarValue::Float64(Some(-v))))}- // Decimal types - wrapping negation+ // Decimal types - use checked negation in ANSI mode, wrapping in legacy modeScalarValue::Decimal32(Some(v), precision, scale) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Decimal32(- Some(result),- *precision,- *scale,- )))+ impl_decimal_scalar_negative!(+ v,+ precision,+ scale,+ "Decimal32",+ Decimal32,+ enable_ansi_mode+ )}ScalarValue::Decimal64(Some(v), precision, scale) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Decimal64(- Some(result),- *precision,- *scale,- )))+ impl_decimal_scalar_negative!(+ v,+ precision,+ scale,+ "Decimal64",+ Decimal64,+ enable_ansi_mode+ )}ScalarValue::Decimal128(Some(v), precision, scale) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Decimal128(- Some(result),- *precision,- *scale,- )))+ impl_decimal_scalar_negative!(+ v,+ precision,+ scale,+ "Decimal128",+ Decimal128,+ enable_ansi_mode+ )}ScalarValue::Decimal256(Some(v), precision, scale) => {- let result = v.wrapping_neg();- Ok(ColumnarValue::Scalar(ScalarValue::Decimal256(+ impl_decimal_scalar_negative!(+ v,+ precision,+ scale,+ "Decimal256",+ Decimal256,+ enable_ansi_mode+ )+ }++ //interval type - use checked negation in ANSI mode, wrapping in legacy mode+ ScalarValue::IntervalYearMonth(Some(v)) => {+ impl_integer_scalar_negative!(+ v,+ "IntervalYearMonth",+ IntervalYearMonth,+ enable_ansi_mode+ )+ }+ ScalarValue::IntervalDayTime(Some(v)) => {+ let result = if enable_ansi_mode {+ let days = v.days.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalDayTime overflow on negative (days: {})",+ v.days+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ let milliseconds = v.milliseconds.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalDayTime overflow on negative (milliseconds: {})",+ v.milliseconds+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ IntervalDayTime { days, milliseconds }+ } else {+ IntervalDayTime {+ days: v.days.wrapping_neg(),+ milliseconds: v.milliseconds.wrapping_neg(),+ }+ };+ Ok(ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(+ result,+ ))))+ }+ ScalarValue::IntervalMonthDayNano(Some(v)) => {+ let result = if enable_ansi_mode {+ let months = v.months.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalMonthDayNano overflow on negative (months: {})",+ v.months+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ let days = v.days.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalMonthDayNano overflow on negative (days: {})",+ v.days+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ let nanoseconds = v.nanoseconds.checked_neg().ok_or_else(|| {+ (exec_err!(+ "IntervalMonthDayNano overflow on negative (nanoseconds: {})",+ v.nanoseconds+ ) as Result<(), _>)+ .unwrap_err()+ })?;+ IntervalMonthDayNano {+ months,+ days,+ nanoseconds,+ }+ } else {+ IntervalMonthDayNano {+ months: v.months.wrapping_neg(),+ days: v.days.wrapping_neg(),+ nanoseconds: v.nanoseconds.wrapping_neg(),+ }+ };+ Ok(ColumnarValue::Scalar(ScalarValue::IntervalMonthDayNano(Some(result),- *precision,- *scale,)))}- //interval type- ScalarValue::IntervalYearMonth(Some(v)) => Ok(ColumnarValue::Scalar(- ScalarValue::IntervalYearMonth(Some(v.wrapping_neg())),- )),- ScalarValue::IntervalDayTime(Some(v)) => Ok(ColumnarValue::Scalar(- ScalarValue::IntervalDayTime(Some(IntervalDayTime {- days: v.days.wrapping_neg(),- milliseconds: v.milliseconds.wrapping_neg(),- })),- )),- ScalarValue::IntervalMonthDayNano(Some(v)) => Ok(ColumnarValue::Scalar(- ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano {- months: v.months.wrapping_neg(),- days: v.days.wrapping_neg(),- nanoseconds: v.nanoseconds.wrapping_neg(),- })),- )),-dt => not_impl_err!("Not supported datatype for Spark negative(): {dt}"),},}diff --git a/datafusion/sqllogictest/test_files/spark/math/negative.slt b/datafusion/sqllogictest/test_files/spark/math/negative.sltindex c62267e496..40bfaf791f 100644--- a/datafusion/sqllogictest/test_files/spark/math/negative.slt+++ b/datafusion/sqllogictest/test_files/spark/math/negative.slt@@ -277,3 +277,55 @@ SELECT negative(column1) FROM interval_test;statement okDROP TABLE interval_test;++## ANSI mode tests: overflow detection+statement ok+set datafusion.execution.enable_ansi_mode = true;++# Test ANSI mode: negative of minimum values should error (overflow)+query error DataFusion error: Execution error: Int8 overflow on negative\(\-128\)+SELECT negative((-128)::tinyint);++query error DataFusion error: Execution error: Int16 overflow on negative\(\-32768\)+SELECT negative((-32768)::smallint);++query error DataFusion error: Execution error: Int32 overflow on negative\(\-2147483648\)+SELECT negative((-2147483648)::int);++query error DataFusion error: Execution error: Int64 overflow on negative\(\-9223372036854775808\)+SELECT negative((-9223372036854775808)::bigint);++# Test ANSI mode: negative of (MIN+1) should succeed (boundary test)+query I+SELECT negative((-127)::tinyint);+----+127++query I+SELECT negative((-32767)::smallint);+----+32767++query I+SELECT negative((-2147483647)::int);+----+2147483647++query I+SELECT negative((-9223372036854775807)::bigint);+----+9223372036854775807++# Test ANSI mode: array with MIN value should error+statement ok+CREATE TABLE min_values_ansi AS VALUES (-2147483648);++query error DataFusion error: Execution error: Int32 overflow on negative\(\-2147483648\)+SELECT negative(column1::int) FROM min_values_ansi;++statement ok+DROP TABLE min_values_ansi;++# Reset ANSI mode to false+statement ok+set datafusion.execution.enable_ansi_mode = false;