Apache DataFusion SQL Query Engine
fix(spark): derive pmod's decimal result type from the declared arguments (#24409)
Amit Vijapur262936eef507
diff --git a/datafusion/spark/src/function/math/modulus.rs b/datafusion/spark/src/function/math/modulus.rsindex c37513c12c..92f037dfff 100644--- a/datafusion/spark/src/function/math/modulus.rs+++ b/datafusion/spark/src/function/math/modulus.rs@@ -15,7 +15,10 @@// specific language governing permissions and limitations// under the License.+use std::sync::Arc;+use arrow::array::{ArrayRef, BooleanArray, Scalar, new_null_array};+use arrow::compute::kernels::cast::{CastOptions, cast_with_options};use arrow::compute::kernels::numeric::add;use arrow::compute::kernels::{boolean::{and, is_not_null, or},@@ -23,11 +26,12 @@ use arrow::compute::kernels::{numeric::{neg, rem},zip::zip,};-use arrow::datatypes::DataType;+use arrow::datatypes::{DECIMAL128_MAX_PRECISION, DataType};use arrow::error::ArrowError;-use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err};+use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err, exec_err};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,+ TypeSignatureClass, Volatility, binary::decimal_coercion,};/// Returns a one element array holding negative zero, for the floating point@@ -101,23 +105,87 @@ pub fn spark_mod(Ok(ColumnarValue::Array(result))}+/// Spark derives the decimal result type of `pmod` with `Pmod.resultDecimalType`,+/// which follows the `Remainder` rule:+///+/// ```text+/// scale = max(s1, s2)+/// precision = min(p1 - s1, p2 - s2) + scale+/// ```+///+/// The rule is applied to the input argument types.+fn pmod_decimal_result_type(p1: u8, s1: i8, p2: u8, s2: i8) -> DataType {+ let scale = s1.max(s2);+ let whole_digits = (i32::from(p1) - i32::from(s1)).min(i32::from(p2) - i32::from(s2));+ let precision =+ (whole_digits + i32::from(scale)).clamp(1, i32::from(DECIMAL128_MAX_PRECISION));+ DataType::Decimal128(precision as u8, scale)+}+/// Spark-compatible `pmod` function/// In ANSI mode, division by zero throws an error./// In legacy mode, division by zero returns NULL (Spark behavior).pub fn spark_pmod(args: &[ColumnarValue],enable_ansi_mode: bool,+ result_type: &DataType,) -> Result<ColumnarValue> {assert_eq_or_internal_err!(args.len(), 2, "pmod expects exactly two arguments");let args = ColumnarValue::values_to_arrays(args)?;- let left = &args[0];- let right = &args[1];++ // Need to handle nulls separately as they are pass through by the signature+ if args.iter().any(|arg| arg.data_type() == &DataType::Null) {+ return Ok(ColumnarValue::Scalar(ScalarValue::try_new_null(+ result_type,+ )?));+ }++ let (left, right): (ArrayRef, ArrayRef) =+ if args[0].data_type() == args[1].data_type() {+ (Arc::clone(&args[0]), Arc::clone(&args[1]))+ } else {+ let Some(computation_type) =+ decimal_coercion(args[0].data_type(), args[1].data_type())+ else {+ return exec_err!(+ "pmod does not support ({}, {})",+ args[0].data_type(),+ args[1].data_type()+ );+ };+ let widen = CastOptions {+ safe: false,+ ..Default::default()+ };+ (+ cast_with_options(&args[0], &computation_type, &widen)?,+ cast_with_options(&args[1], &computation_type, &widen)?,+ )+ };++ let left = &left;+ let right = &right;let zero = ScalarValue::new_zero(left.data_type())?.to_array_of_size(left.len())?;let result = try_rem(left, right, enable_ansi_mode)?;let neg = lt(&result, &zero)?;let plus = zip(&neg, right, &zero)?;let result = add(&plus, &result)?;let result = try_rem(&result, right, enable_ansi_mode)?;++ // The remainder is bounded by the divisor, but the result type only carries+ // `min(p1 - s1, p2 - s2)` integer digits, so a remainder approaching a+ // divisor wider than the dividend does not always fit. Spark wraps decimal+ // arithmetic in `CheckOverflow(nullOnOverflow = !ansiEnabled)`, so an+ // overflow here is NULL in legacy mode and an error under ANSI.+ let result = if result.data_type() == result_type {+ result+ } else {+ let narrow = CastOptions {+ safe: !enable_ansi_mode,+ ..Default::default()+ };+ cast_with_options(&result, result_type, &narrow)?+ };Ok(ColumnarValue::Array(result))}@@ -182,7 +250,19 @@ impl Default for SparkPmod {impl SparkPmod {pub fn new() -> Self {Self {- signature: Signature::numeric(2, Volatility::Immutable),+ signature: Signature::one_of(+ vec![+ // A decimal pair must reach `return_type` with the+ // precision and scale as written, since Spark defines+ // `Pmod.resultDecimalType` on the declared arguments.+ TypeSignature::Coercible(vec![+ Coercion::new_exact(TypeSignatureClass::Decimal),+ Coercion::new_exact(TypeSignatureClass::Decimal),+ ]),+ TypeSignature::Numeric(2),+ ],+ Volatility::Immutable,+ ),}}}@@ -203,13 +283,24 @@ impl ScalarUDFImpl for SparkPmod {"pmod expects exactly two arguments");- // Return the same type as the first argument for simplicity- // Arrow's rem function handles type promotion internally- Ok(arg_types[0].clone())+ match (&arg_types[0], &arg_types[1]) {+ (DataType::Decimal128(p1, s1), DataType::Decimal128(p2, s2)) => {+ Ok(pmod_decimal_result_type(*p1, *s1, *p2, *s2))+ }+ // Need to handle nulls explicitly, see: https://github.com/apache/datafusion/issues/19458+ // We align with the behaviour of `mod`+ (DataType::Null, DataType::Null) => Ok(DataType::Float64),+ (DataType::Null, other) | (other, DataType::Null) => Ok(other.clone()),+ _ => Ok(arg_types[0].clone()),+ }}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- spark_pmod(&args.args, args.config_options.execution.enable_ansi_mode)+ spark_pmod(+ &args.args,+ args.config_options.execution.enable_ansi_mode,+ args.return_type(),+ )}}@@ -558,7 +649,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int32 =@@ -581,7 +673,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int64 =@@ -625,7 +718,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_float64 = result_array@@ -683,7 +777,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_float32 = result_array@@ -718,7 +813,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int32 =@@ -737,7 +833,8 @@ mod test {let left = Int32Array::from(vec![Some(10)]);let left_value = ColumnarValue::Array(Arc::new(left));- let result = spark_pmod(&[left_value], false);+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value], false, &return_type);assert!(result.is_err());}@@ -750,7 +847,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int32 =@@ -772,7 +870,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], true);+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], true, &return_type);assert!(result.is_err());}@@ -788,7 +887,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], true);+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], true, &return_type);assert!(result.is_err());}@@ -802,7 +902,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_float64 = result_array@@ -825,7 +926,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], true);+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], true, &return_type);assert!(result.is_err());}@@ -840,7 +942,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], true).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], true, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int32 =@@ -858,7 +961,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], true).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], true, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_float64 = result_array@@ -881,7 +985,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int32 =@@ -919,7 +1024,8 @@ mod test {let left_value = ColumnarValue::Array(Arc::new(left));let right_value = ColumnarValue::Array(Arc::new(right));- let result = spark_pmod(&[left_value, right_value], false).unwrap();+ let return_type = left_value.data_type();+ let result = spark_pmod(&[left_value, right_value], false, &return_type).unwrap();if let ColumnarValue::Array(result_array) = result {let result_int32 =@@ -935,4 +1041,35 @@ mod test {panic!("Expected array result");}}++ /// Spark's `Pmod.resultDecimalType`: scale = max(s1, s2),+ /// precision = min(p1 - s1, p2 - s2) + scale.+ #[test]+ fn test_pmod_decimal_result_type() {+ // Equal scales: the narrower argument decides the precision.+ assert_eq!(+ pmod_decimal_result_type(3, 1, 2, 1),+ DataType::Decimal128(2, 1)+ );+ // The divisor is wider, so the dividend bounds the result.+ assert_eq!(+ pmod_decimal_result_type(5, 2, 4, 1),+ DataType::Decimal128(5, 2)+ );+ // Differing scales: the wider scale wins.+ assert_eq!(+ pmod_decimal_result_type(6, 3, 4, 1),+ DataType::Decimal128(6, 3)+ );+ // Integral decimals keep a zero scale.+ assert_eq!(+ pmod_decimal_result_type(10, 0, 5, 0),+ DataType::Decimal128(5, 0)+ );+ // The result never exceeds the maximum precision arrow can represent.+ assert_eq!(+ pmod_decimal_result_type(38, 0, 38, 38),+ DataType::Decimal128(38, 38)+ );+ }}diff --git a/datafusion/sqllogictest/test_files/spark/math/pmod.slt b/datafusion/sqllogictest/test_files/spark/math/pmod.sltindex 672ac370d6..5b9c84bfec 100644--- a/datafusion/sqllogictest/test_files/spark/math/pmod.slt+++ b/datafusion/sqllogictest/test_files/spark/math/pmod.slt@@ -98,6 +98,10 @@ SELECT pmod(10.5::float8, 0.0::float8);statement error DataFusion error: Arrow error: Divide by zero errorSELECT pmod(10.5::float8, -0.0::float8);+# A decimal result that does not fit raises instead of returning NULL+statement error DataFusion error: Arrow error: Invalid argument error: 9999\.8 is too large to store in a Decimal128 of precision 3+SELECT pmod(-0.1::decimal(3,1), 9999.9::decimal(5,1));+# A NULL dividend short-circuits to NULL before the divisor is validatedquery ISELECT pmod(NULL::int, 0::int) as pmod_null_dividend_ansi;@@ -128,6 +132,21 @@ SELECT pmod(NULL::int, NULL::int) as pmod_null_3;----NULL+query T+SELECT arrow_typeof(pmod(NULL, NULL));+----+Float64++query T+SELECT arrow_typeof(pmod(2.5::decimal(3,1), NULL));+----+Decimal128(3, 1)++# An untyped NULL beside a typed non-decimal argument takes the Numeric path,+# which cannot coerce the pair. `mod` rejects it the same way.+statement error DataFusion error: Error during planning: Internal error: Function 'pmod' failed to match any signature+SELECT pmod(NULL, 3::int);+# PMOD tests with large integersquery ISELECT pmod(100::int, 30::int) as pmod_large_1;@@ -305,6 +324,53 @@ SELECT pmod(-10.0::decimal(3,1), 3.0::decimal(2,1)) as pmod_decimal_4;----2.0+# The decimal result type follows Spark's Pmod.resultDecimalType, which applies+# the Remainder rule to the declared argument types:+# scale = max(s1, s2)+# precision = min(p1 - s1, p2 - s2) + scale+query T+SELECT arrow_typeof(pmod(2.5::decimal(3,1), 1.2::decimal(2,1)));+----+Decimal128(2, 1)++# The divisor bounds the result, so the narrower argument decides the precision+query T+SELECT arrow_typeof(pmod(10.0::decimal(5,2), 3.0::decimal(4,1)));+----+Decimal128(5, 2)++# Differing scales: the wider scale wins+query T+SELECT arrow_typeof(pmod(1.234::decimal(6,3), 2.1::decimal(4,1)));+----+Decimal128(6, 3)++# The dividend does not fit the result type, so it must not be narrowed before+# the remainder is taken+query T+SELECT arrow_typeof(pmod(99.9::decimal(3,1), 2.5::decimal(2,1)));+----+Decimal128(2, 1)++query R+SELECT pmod(99.9::decimal(3,1), 2.5::decimal(2,1));+----+2.4++# A remainder is bounded by the divisor, not by the result type, so a divisor+# wider than the dividend can produce a value the result type cannot hold. Spark+# wraps decimal arithmetic in CheckOverflow(nullOnOverflow = !ansiEnabled), so+# this is NULL in legacy mode; the ANSI error is asserted in the ANSI block above.+query T+SELECT arrow_typeof(pmod(-0.1::decimal(3,1), 9999.9::decimal(5,1)));+----+Decimal128(3, 1)++query R+SELECT pmod(-0.1::decimal(3,1), 9999.9::decimal(5,1));+----+NULL+# PMOD tests with different integer typesquery ISELECT pmod(10::int8, 3::int8) as pmod_int8_1;