Apache DataFusion SQL Query Engine
fix(spark): return NaN and -Infinity from `pow` where Spark does (#26382)
Alejandro Borrego3d7e19156665
diff --git a/datafusion/spark/src/function/math/pow.rs b/datafusion/spark/src/function/math/pow.rsindex 06f789ec25..bd7eb653d2 100644--- a/datafusion/spark/src/function/math/pow.rs+++ b/datafusion/spark/src/function/math/pow.rs@@ -5,7 +5,8 @@//! Spark-compatible `pow` / `power` function.//!//! Unlike the default DataFusion (PostgreSQL) implementation, Spark returns-//! `Infinity` for `pow(0, <negative>)` rather than raising an error.+//! an infinity for `pow(0, <negative>)` rather than raising an error, and+//! follows `java.lang.StrictMath.pow` for the special cases.use std::sync::Arc;@@ -21,10 +22,13 @@ use datafusion_functions::math::power::PowerFunc;/// Spark-compatible implementation of `pow` / `power`.///-/// Behavioural difference from the DataFusion default:-/// - `pow(0, <negative>)` → `Infinity` (IEEE 754 / Spark semantics)-/// The default raises `"zero raised to a negative power is undefined"` to-/// match PostgreSQL.+/// Behavioural differences from the DataFusion default:+/// - `pow(0, <negative>)` gives an infinity instead of the default's error+/// `"zero raised to a negative power is undefined"` (PostgreSQL): `+Infinity`+/// for `0.0`, and for `-0.0` unless the exponent is a negative odd integer,+/// which gives `-Infinity`.+/// - `pow(1, NaN)` and `pow(±1, ±Infinity)` give `NaN` (`StrictMath.pow`)+/// instead of `1`. Any other base with a `NaN` exponent already gives `NaN`.#[derive(Debug, PartialEq, Eq, Hash)]pub struct SparkPow {inner: PowerFunc,@@ -87,13 +91,7 @@ impl ScalarUDFImpl for SparkPow {] = args.args.as_slice(){// base and exp are &Option<f64>; Option<f64> is Copy.- let result = (*base).zip(*exp).map(|(base, exp)| {- if base == 0.0 && exp < 0.0 {- f64::INFINITY- } else {- base.powf(exp)- }- });+ let result = (*base).zip(*exp).map(|(base, exp)| spark_pow(base, exp));return Ok(ColumnarValue::Scalar(ScalarValue::Float64(result)));}@@ -112,20 +110,11 @@ impl ScalarUDFImpl for SparkPow {.downcast_ref::<Float64Array>().expect("exponent must be Float64Array");- // Spark: 0^negative = +Infinity (covers both 0.0 and -0.0)- // IEEE 754: 0.0^-1.0 = +Infinity, -0.0^-1.0 = -Infinity- // Thus we need an explicit guard for base == 0.0 to ensure +Infinity.let result: Float64Array = base_f64.iter().zip(exp_f64.iter()).map(|(base, exp)| match (base, exp) {- (Some(base), Some(exp)) => {- if base == 0.0 && exp < 0.0 {- Some(f64::INFINITY)- } else {- Some(base.powf(exp))- }- }+ (Some(base), Some(exp)) => Some(spark_pow(base, exp)),_ => None,}).collect();@@ -137,3 +126,17 @@ impl ScalarUDFImpl for SparkPow {self.inner.documentation()}}++/// `pow` with the special cases of Java's `StrictMath.pow`, which Spark uses.+///+/// `f64::powf` follows C99, which agrees with Java except that C99 returns `1`+/// for `pow(1, NaN)` and `pow(±1, ±Infinity)`, where Java returns `NaN`.+/// Both give `+Infinity` for `pow(0.0, <negative>)` and `-Infinity` for+/// `pow(-0.0, <negative odd integer>)`.+fn spark_pow(base: f64, exp: f64) -> f64 {+ if exp.is_nan() || (base.abs() == 1.0 && exp.is_infinite()) {+ f64::NAN+ } else {+ base.powf(exp)+ }+}diff --git a/datafusion/sqllogictest/test_files/spark/math/pow.slt b/datafusion/sqllogictest/test_files/spark/math/pow.sltindex dd7fc13dce..b5c0c5c291 100644--- a/datafusion/sqllogictest/test_files/spark/math/pow.slt+++ b/datafusion/sqllogictest/test_files/spark/math/pow.slt@@ -57,10 +57,11 @@ SELECT pow(CAST('NaN' AS DOUBLE), 1.0);----NaN+## Spark follows java.lang.StrictMath.pow: a NaN exponent gives NaN, also for a base of 1query RSELECT pow(1.0, CAST('NaN' AS DOUBLE));-----1+NaNquery RSELECT pow(CAST('NaN' AS DOUBLE), 0.0);@@ -86,8 +87,18 @@ Infinityquery RSELECT pow(CAST('-0.0' AS DOUBLE), -1.0);----+-Infinity++query R+SELECT pow(CAST('-0.0' AS DOUBLE), -2.0);+----Infinity+query R+SELECT pow(CAST('-0.0' AS DOUBLE), -3.0);+----+-Infinity+# -inf, +infquery RSELECT pow(CAST('Infinity' AS DOUBLE), 1.0);@@ -129,12 +140,36 @@ SELECT pow(0.5, CAST('-Infinity' AS DOUBLE));----Infinity+## A base of 1 or -1 with an infinite exponent gives NaN in Spark+query R+SELECT pow(1.0, CAST('Infinity' AS DOUBLE));+----+NaN++query R+SELECT pow(-1.0, CAST('Infinity' AS DOUBLE));+----+NaN++query R+SELECT pow(1.0, CAST('-Infinity' AS DOUBLE));+----+NaN++query R+SELECT pow(-1.0, CAST('-Infinity' AS DOUBLE));+----+NaN+# Test Array x Arraystatement okCREATE TABLE t1(a DOUBLE, b DOUBLE) AS VALUES(0.0, -1.0),(2.0, 3.0),-(CAST(NULL AS DOUBLE), 1.0);+(CAST(NULL AS DOUBLE), 1.0),+(CAST('-0.0' AS DOUBLE), -3.0),+(1.0, CAST('NaN' AS DOUBLE)),+(-1.0, CAST('Infinity' AS DOUBLE));query RSELECT pow(a, b) FROM t1;@@ -142,6 +177,20 @@ SELECT pow(a, b) FROM t1;Infinity8NULL+-Infinity+NaN+NaN++## Scalar x Array and Array x Scalar, including a NULL base+query RR+SELECT pow(1.0, b), pow(a, CAST('Infinity' AS DOUBLE)) FROM t1;+----+1 0+1 Infinity+1 NULL+1 0+NaN NaN+NaN NaNstatement okDROP TABLE t1;