Skip to content

Apache DataFusion SQL Query Engine

Adds support for ANSI mode in negative function (#20189)

Subham Singhal98cc75361fb0
diff --git a/datafusion/spark/src/function/math/negative.rs b/datafusion/spark/src/function/math/negative.rs
index f1803d2d77..2df71b709d 100644
--- a/datafusion/spark/src/function/math/negative.rs
+++ b/datafusion/spark/src/function/math/negative.rs
@@ -20,7 +20,7 @@ use arrow::array::*;
use arrow::datatypes::{DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit};
use bigdecimal::num_traits::WrappingNeg;
use datafusion_common::utils::take_function_args;
-use datafusion_common::{Result, ScalarValue, not_impl_err};
+use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
Volatility,
@@ -34,14 +34,12 @@ use std::sync::Arc;
/// Returns the negation of input (equivalent to unary minus)
/// Returns NULL if input is NULL, returns NaN if input is NaN.
///
-/// ANSI mode support see (<https://github.com/apache/datafusion/issues/20034>):
-/// - Spark's ANSI-compliant dialect, when off (i.e. `spark.sql.ansi.enabled=false`),
-/// negating the minimal value of a signed integer wraps around.
-/// For example: negative(i32::MIN) returns i32::MIN (wraps instead of error).
-/// This is the current implementation (legacy mode only).
-/// - Spark's ANSI mode (when `spark.sql.ansi.enabled=true`) should throw an
-/// ARITHMETIC_OVERFLOW error on integer overflow instead of wrapping.
-/// This is not yet implemented - all operations currently use wrapping behavior.
+/// ANSI mode support:
+/// - When ANSI mode is disabled (`spark.sql.ansi.enabled=false`), negating the minimal
+/// value of a signed integer wraps around. For example: negative(i32::MIN) returns
+/// i32::MIN (wraps instead of error).
+/// - When ANSI mode is enabled (`spark.sql.ansi.enabled=true`), overflow conditions
+/// throw an ARITHMETIC_OVERFLOW error instead of wrapping.
///
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkNegative {
@@ -96,107 +94,231 @@ impl ScalarUDFImpl for SparkNegative {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- spark_negative(&args.args)
+ spark_negative(&args.args, args.config_options.execution.enable_ansi_mode)
}
}
+/// Macro to implement negation for integer array types
+macro_rules! impl_integer_array_negative {
+ ($array:expr, $type:ty, $type_name:expr, $enable_ansi_mode:expr) => {{
+ let array = $array.as_primitive::<$type>();
+ let result: PrimitiveArray<$type> = if $enable_ansi_mode {
+ array.try_unary(|x| {
+ x.checked_neg().ok_or_else(|| {
+ (exec_err!("{} overflow on negative({x})", $type_name)
+ as Result<(), _>)
+ .unwrap_err()
+ })
+ })?
+ } else {
+ array.unary(|x| x.wrapping_neg())
+ };
+ Ok(ColumnarValue::Array(Arc::new(result)))
+ }};
+}
+
+/// Macro to implement negation for float array types
+macro_rules! impl_float_array_negative {
+ ($array:expr, $type:ty) => {{
+ let array = $array.as_primitive::<$type>();
+ let result: PrimitiveArray<$type> = array.unary(|x| -x);
+ Ok(ColumnarValue::Array(Arc::new(result)))
+ }};
+}
+
+/// Macro to implement negation for decimal array types
+macro_rules! impl_decimal_array_negative {
+ ($array:expr, $type:ty, $type_name:expr, $enable_ansi_mode:expr) => {{
+ let array = $array.as_primitive::<$type>();
+ let result: PrimitiveArray<$type> = if $enable_ansi_mode {
+ array
+ .try_unary(|x| {
+ x.checked_neg().ok_or_else(|| {
+ (exec_err!("{} overflow on negative({x})", $type_name)
+ as Result<(), _>)
+ .unwrap_err()
+ })
+ })?
+ .with_data_type(array.data_type().clone())
+ } else {
+ array.unary(|x| x.wrapping_neg())
+ };
+ Ok(ColumnarValue::Array(Arc::new(result)))
+ }};
+}
+
+/// Macro to implement negation for integer scalar types
+macro_rules! impl_integer_scalar_negative {
+ ($v:expr, $type_name:expr, $variant:ident, $enable_ansi_mode:expr) => {{
+ let result = if $enable_ansi_mode {
+ $v.checked_neg().ok_or_else(|| {
+ (exec_err!("{} overflow on negative({})", $type_name, $v)
+ as Result<(), _>)
+ .unwrap_err()
+ })?
+ } else {
+ $v.wrapping_neg()
+ };
+ Ok(ColumnarValue::Scalar(ScalarValue::$variant(Some(result))))
+ }};
+}
+
+/// Macro to implement negation for decimal scalar types
+macro_rules! impl_decimal_scalar_negative {
+ ($v:expr, $precision:expr, $scale:expr, $type_name:expr, $variant:ident, $enable_ansi_mode:expr) => {{
+ let result = if $enable_ansi_mode {
+ $v.checked_neg().ok_or_else(|| {
+ (exec_err!("{} overflow on negative({})", $type_name, $v)
+ as Result<(), _>)
+ .unwrap_err()
+ })?
+ } else {
+ $v.wrapping_neg()
+ };
+ Ok(ColumnarValue::Scalar(ScalarValue::$variant(
+ Some(result),
+ *$precision,
+ *$scale,
+ )))
+ }};
+}
+
/// Core implementation of Spark's negative function
-fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {
+fn spark_negative(
+ args: &[ColumnarValue],
+ enable_ansi_mode: bool,
+) -> Result<ColumnarValue> {
let [arg] = take_function_args("negative", args)?;
match arg {
ColumnarValue::Array(array) => match array.data_type() {
DataType::Null => Ok(arg.clone()),
- // Signed integers - use wrapping negation (Spark legacy mode behavior)
+ // Signed integers - use checked negation in ANSI mode, wrapping in legacy mode
DataType::Int8 => {
- let array = array.as_primitive::<Int8Type>();
- let result: PrimitiveArray<Int8Type> = array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
+ impl_integer_array_negative!(array, Int8Type, "Int8", enable_ansi_mode)
}
DataType::Int16 => {
- let array = array.as_primitive::<Int16Type>();
- let result: PrimitiveArray<Int16Type> = array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
+ impl_integer_array_negative!(array, Int16Type, "Int16", enable_ansi_mode)
}
DataType::Int32 => {
- let array = array.as_primitive::<Int32Type>();
- let result: PrimitiveArray<Int32Type> = array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
+ impl_integer_array_negative!(array, Int32Type, "Int32", enable_ansi_mode)
}
DataType::Int64 => {
- let array = array.as_primitive::<Int64Type>();
- let result: PrimitiveArray<Int64Type> = array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
+ impl_integer_array_negative!(array, Int64Type, "Int64", enable_ansi_mode)
}
// Floating point - simple negation (no overflow possible)
- DataType::Float16 => {
- let array = array.as_primitive::<Float16Type>();
- let result: PrimitiveArray<Float16Type> = array.unary(|x| -x);
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
- DataType::Float32 => {
- let array = array.as_primitive::<Float32Type>();
- let result: PrimitiveArray<Float32Type> = array.unary(|x| -x);
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
- DataType::Float64 => {
- let array = array.as_primitive::<Float64Type>();
- let result: PrimitiveArray<Float64Type> = array.unary(|x| -x);
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
+ DataType::Float16 => impl_float_array_negative!(array, Float16Type),
+ DataType::Float32 => impl_float_array_negative!(array, Float32Type),
+ DataType::Float64 => impl_float_array_negative!(array, Float64Type),
- // Decimal types - wrapping negation
- DataType::Decimal32(_, _) => {
- let array = array.as_primitive::<Decimal32Type>();
- let result: PrimitiveArray<Decimal32Type> =
- array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
- DataType::Decimal64(_, _) => {
- let array = array.as_primitive::<Decimal64Type>();
- let result: PrimitiveArray<Decimal64Type> =
- array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
- DataType::Decimal128(_, _) => {
- let array = array.as_primitive::<Decimal128Type>();
- let result: PrimitiveArray<Decimal128Type> =
- array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
- DataType::Decimal256(_, _) => {
- let array = array.as_primitive::<Decimal256Type>();
- let result: PrimitiveArray<Decimal256Type> =
- array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
- }
+ // Decimal types - use checked negation in ANSI mode, wrapping in legacy mode
+ DataType::Decimal32(_, _) => impl_decimal_array_negative!(
+ array,
+ Decimal32Type,
+ "Decimal32",
+ enable_ansi_mode
+ ),
+ DataType::Decimal64(_, _) => impl_decimal_array_negative!(
+ array,
+ Decimal64Type,
+ "Decimal64",
+ enable_ansi_mode
+ ),
+ DataType::Decimal128(_, _) => impl_decimal_array_negative!(
+ array,
+ Decimal128Type,
+ "Decimal128",
+ enable_ansi_mode
+ ),
+ DataType::Decimal256(_, _) => impl_decimal_array_negative!(
+ array,
+ Decimal256Type,
+ "Decimal256",
+ enable_ansi_mode
+ ),
- // interval type
+ // interval type - use checked negation in ANSI mode, wrapping in legacy mode
DataType::Interval(IntervalUnit::YearMonth) => {
- let array = array.as_primitive::<IntervalYearMonthType>();
- let result: PrimitiveArray<IntervalYearMonthType> =
- array.unary(|x| x.wrapping_neg());
- Ok(ColumnarValue::Array(Arc::new(result)))
+ impl_integer_array_negative!(
+ array,
+ IntervalYearMonthType,
+ "IntervalYearMonth",
+ enable_ansi_mode
+ )
}
DataType::Interval(IntervalUnit::DayTime) => {
let array = array.as_primitive::<IntervalDayTimeType>();
- let result: PrimitiveArray<IntervalDayTimeType> =
+ let result: PrimitiveArray<IntervalDayTimeType> = if enable_ansi_mode {
+ array.try_unary(|x| {
+ let days = x.days.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalDayTime overflow on negative (days: {})",
+ x.days
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ let milliseconds =
+ x.milliseconds.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalDayTime overflow on negative (milliseconds: {})",
+ x.milliseconds
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ Ok::<_, arrow::error::ArrowError>(IntervalDayTime {
+ days,
+ milliseconds,
+ })
+ })?
+ } else {
array.unary(|x| IntervalDayTime {
days: x.days.wrapping_neg(),
milliseconds: x.milliseconds.wrapping_neg(),
- });
+ })
+ };
Ok(ColumnarValue::Array(Arc::new(result)))
}
DataType::Interval(IntervalUnit::MonthDayNano) => {
let array = array.as_primitive::<IntervalMonthDayNanoType>();
- let result: PrimitiveArray<IntervalMonthDayNanoType> =
+ let result: PrimitiveArray<IntervalMonthDayNanoType> = if enable_ansi_mode
+ {
+ array.try_unary(|x| {
+ let months = x.months.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalMonthDayNano overflow on negative (months: {})",
+ x.months
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ let days = x.days.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalMonthDayNano overflow on negative (days: {})",
+ x.days
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ let nanoseconds = x.nanoseconds.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalMonthDayNano overflow on negative (nanoseconds: {})",
+ x.nanoseconds
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ Ok::<_, arrow::error::ArrowError>(IntervalMonthDayNano {
+ months,
+ days,
+ nanoseconds,
+ })
+ })?
+ } else {
array.unary(|x| IntervalMonthDayNano {
months: x.months.wrapping_neg(),
days: x.days.wrapping_neg(),
nanoseconds: x.nanoseconds.wrapping_neg(),
- });
+ })
+ };
Ok(ColumnarValue::Array(Arc::new(result)))
}
@@ -206,22 +328,18 @@ fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {
ScalarValue::Null => Ok(arg.clone()),
_ if sv.is_null() => Ok(arg.clone()),
- // Signed integers - wrapping negation
+ // Signed integers - use checked negation in ANSI mode, wrapping in legacy mode
ScalarValue::Int8(Some(v)) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Int8(Some(result))))
+ impl_integer_scalar_negative!(v, "Int8", Int8, enable_ansi_mode)
}
ScalarValue::Int16(Some(v)) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Int16(Some(result))))
+ impl_integer_scalar_negative!(v, "Int16", Int16, enable_ansi_mode)
}
ScalarValue::Int32(Some(v)) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(result))))
+ impl_integer_scalar_negative!(v, "Int32", Int32, enable_ansi_mode)
}
ScalarValue::Int64(Some(v)) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some(result))))
+ impl_integer_scalar_negative!(v, "Int64", Int64, enable_ansi_mode)
}
// Floating point - simple negation
@@ -235,58 +353,124 @@ fn spark_negative(args: &[ColumnarValue]) -> Result<ColumnarValue> {
Ok(ColumnarValue::Scalar(ScalarValue::Float64(Some(-v))))
}
- // Decimal types - wrapping negation
+ // Decimal types - use checked negation in ANSI mode, wrapping in legacy mode
ScalarValue::Decimal32(Some(v), precision, scale) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Decimal32(
- Some(result),
- *precision,
- *scale,
- )))
+ impl_decimal_scalar_negative!(
+ v,
+ precision,
+ scale,
+ "Decimal32",
+ Decimal32,
+ enable_ansi_mode
+ )
}
ScalarValue::Decimal64(Some(v), precision, scale) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Decimal64(
- Some(result),
- *precision,
- *scale,
- )))
+ impl_decimal_scalar_negative!(
+ v,
+ precision,
+ scale,
+ "Decimal64",
+ Decimal64,
+ enable_ansi_mode
+ )
}
ScalarValue::Decimal128(Some(v), precision, scale) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Decimal128(
- Some(result),
- *precision,
- *scale,
- )))
+ impl_decimal_scalar_negative!(
+ v,
+ precision,
+ scale,
+ "Decimal128",
+ Decimal128,
+ enable_ansi_mode
+ )
}
ScalarValue::Decimal256(Some(v), precision, scale) => {
- let result = v.wrapping_neg();
- Ok(ColumnarValue::Scalar(ScalarValue::Decimal256(
+ impl_decimal_scalar_negative!(
+ v,
+ precision,
+ scale,
+ "Decimal256",
+ Decimal256,
+ enable_ansi_mode
+ )
+ }
+
+ //interval type - use checked negation in ANSI mode, wrapping in legacy mode
+ ScalarValue::IntervalYearMonth(Some(v)) => {
+ impl_integer_scalar_negative!(
+ v,
+ "IntervalYearMonth",
+ IntervalYearMonth,
+ enable_ansi_mode
+ )
+ }
+ ScalarValue::IntervalDayTime(Some(v)) => {
+ let result = if enable_ansi_mode {
+ let days = v.days.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalDayTime overflow on negative (days: {})",
+ v.days
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ let milliseconds = v.milliseconds.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalDayTime overflow on negative (milliseconds: {})",
+ v.milliseconds
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ IntervalDayTime { days, milliseconds }
+ } else {
+ IntervalDayTime {
+ days: v.days.wrapping_neg(),
+ milliseconds: v.milliseconds.wrapping_neg(),
+ }
+ };
+ Ok(ColumnarValue::Scalar(ScalarValue::IntervalDayTime(Some(
+ result,
+ ))))
+ }
+ ScalarValue::IntervalMonthDayNano(Some(v)) => {
+ let result = if enable_ansi_mode {
+ let months = v.months.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalMonthDayNano overflow on negative (months: {})",
+ v.months
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ let days = v.days.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalMonthDayNano overflow on negative (days: {})",
+ v.days
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ let nanoseconds = v.nanoseconds.checked_neg().ok_or_else(|| {
+ (exec_err!(
+ "IntervalMonthDayNano overflow on negative (nanoseconds: {})",
+ v.nanoseconds
+ ) as Result<(), _>)
+ .unwrap_err()
+ })?;
+ IntervalMonthDayNano {
+ months,
+ days,
+ nanoseconds,
+ }
+ } else {
+ IntervalMonthDayNano {
+ months: v.months.wrapping_neg(),
+ days: v.days.wrapping_neg(),
+ nanoseconds: v.nanoseconds.wrapping_neg(),
+ }
+ };
+ Ok(ColumnarValue::Scalar(ScalarValue::IntervalMonthDayNano(
Some(result),
- *precision,
- *scale,
)))
}
- //interval type
- ScalarValue::IntervalYearMonth(Some(v)) => Ok(ColumnarValue::Scalar(
- ScalarValue::IntervalYearMonth(Some(v.wrapping_neg())),
- )),
- ScalarValue::IntervalDayTime(Some(v)) => Ok(ColumnarValue::Scalar(
- ScalarValue::IntervalDayTime(Some(IntervalDayTime {
- days: v.days.wrapping_neg(),
- milliseconds: v.milliseconds.wrapping_neg(),
- })),
- )),
- ScalarValue::IntervalMonthDayNano(Some(v)) => Ok(ColumnarValue::Scalar(
- ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano {
- months: v.months.wrapping_neg(),
- days: v.days.wrapping_neg(),
- nanoseconds: v.nanoseconds.wrapping_neg(),
- })),
- )),
-
dt => not_impl_err!("Not supported datatype for Spark negative(): {dt}"),
},
}
diff --git a/datafusion/sqllogictest/test_files/spark/math/negative.slt b/datafusion/sqllogictest/test_files/spark/math/negative.slt
index c62267e496..40bfaf791f 100644
--- a/datafusion/sqllogictest/test_files/spark/math/negative.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/negative.slt
@@ -277,3 +277,55 @@ SELECT negative(column1) FROM interval_test;
statement ok
DROP TABLE interval_test;
+
+## ANSI mode tests: overflow detection
+statement ok
+set datafusion.execution.enable_ansi_mode = true;
+
+# Test ANSI mode: negative of minimum values should error (overflow)
+query error DataFusion error: Execution error: Int8 overflow on negative\(\-128\)
+SELECT negative((-128)::tinyint);
+
+query error DataFusion error: Execution error: Int16 overflow on negative\(\-32768\)
+SELECT negative((-32768)::smallint);
+
+query error DataFusion error: Execution error: Int32 overflow on negative\(\-2147483648\)
+SELECT negative((-2147483648)::int);
+
+query error DataFusion error: Execution error: Int64 overflow on negative\(\-9223372036854775808\)
+SELECT negative((-9223372036854775808)::bigint);
+
+# Test ANSI mode: negative of (MIN+1) should succeed (boundary test)
+query I
+SELECT negative((-127)::tinyint);
+----
+127
+
+query I
+SELECT negative((-32767)::smallint);
+----
+32767
+
+query I
+SELECT negative((-2147483647)::int);
+----
+2147483647
+
+query I
+SELECT negative((-9223372036854775807)::bigint);
+----
+9223372036854775807
+
+# Test ANSI mode: array with MIN value should error
+statement ok
+CREATE TABLE min_values_ansi AS VALUES (-2147483648);
+
+query error DataFusion error: Execution error: Int32 overflow on negative\(\-2147483648\)
+SELECT negative(column1::int) FROM min_values_ansi;
+
+statement ok
+DROP TABLE min_values_ansi;
+
+# Reset ANSI mode to false
+statement ok
+set datafusion.execution.enable_ansi_mode = false;