Skip to content

Apache DataFusion SQL Query Engine

fix(spark): return NaN and -Infinity from `pow` where Spark does (#26382)

Alejandro Borrego3d7e19156665
diff --git a/datafusion/spark/src/function/math/pow.rs b/datafusion/spark/src/function/math/pow.rs
index 06f789ec25..bd7eb653d2 100644
--- a/datafusion/spark/src/function/math/pow.rs
+++ b/datafusion/spark/src/function/math/pow.rs
@@ -5,7 +5,8 @@
//! Spark-compatible `pow` / `power` function.
//!
//! Unlike the default DataFusion (PostgreSQL) implementation, Spark returns
-//! `Infinity` for `pow(0, <negative>)` rather than raising an error.
+//! an infinity for `pow(0, <negative>)` rather than raising an error, and
+//! follows `java.lang.StrictMath.pow` for the special cases.
use std::sync::Arc;
@@ -21,10 +22,13 @@ use datafusion_functions::math::power::PowerFunc;
/// Spark-compatible implementation of `pow` / `power`.
///
-/// Behavioural difference from the DataFusion default:
-/// - `pow(0, <negative>)` → `Infinity` (IEEE 754 / Spark semantics)
-/// The default raises `"zero raised to a negative power is undefined"` to
-/// match PostgreSQL.
+/// Behavioural differences from the DataFusion default:
+/// - `pow(0, <negative>)` gives an infinity instead of the default's error
+/// `"zero raised to a negative power is undefined"` (PostgreSQL): `+Infinity`
+/// for `0.0`, and for `-0.0` unless the exponent is a negative odd integer,
+/// which gives `-Infinity`.
+/// - `pow(1, NaN)` and `pow(±1, ±Infinity)` give `NaN` (`StrictMath.pow`)
+/// instead of `1`. Any other base with a `NaN` exponent already gives `NaN`.
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkPow {
inner: PowerFunc,
@@ -87,13 +91,7 @@ impl ScalarUDFImpl for SparkPow {
] = args.args.as_slice()
{
// base and exp are &Option<f64>; Option<f64> is Copy.
- let result = (*base).zip(*exp).map(|(base, exp)| {
- if base == 0.0 && exp < 0.0 {
- f64::INFINITY
- } else {
- base.powf(exp)
- }
- });
+ let result = (*base).zip(*exp).map(|(base, exp)| spark_pow(base, exp));
return Ok(ColumnarValue::Scalar(ScalarValue::Float64(result)));
}
@@ -112,20 +110,11 @@ impl ScalarUDFImpl for SparkPow {
.downcast_ref::<Float64Array>()
.expect("exponent must be Float64Array");
- // Spark: 0^negative = +Infinity (covers both 0.0 and -0.0)
- // IEEE 754: 0.0^-1.0 = +Infinity, -0.0^-1.0 = -Infinity
- // Thus we need an explicit guard for base == 0.0 to ensure +Infinity.
let result: Float64Array = base_f64
.iter()
.zip(exp_f64.iter())
.map(|(base, exp)| match (base, exp) {
- (Some(base), Some(exp)) => {
- if base == 0.0 && exp < 0.0 {
- Some(f64::INFINITY)
- } else {
- Some(base.powf(exp))
- }
- }
+ (Some(base), Some(exp)) => Some(spark_pow(base, exp)),
_ => None,
})
.collect();
@@ -137,3 +126,17 @@ impl ScalarUDFImpl for SparkPow {
self.inner.documentation()
}
}
+
+/// `pow` with the special cases of Java's `StrictMath.pow`, which Spark uses.
+///
+/// `f64::powf` follows C99, which agrees with Java except that C99 returns `1`
+/// for `pow(1, NaN)` and `pow(±1, ±Infinity)`, where Java returns `NaN`.
+/// Both give `+Infinity` for `pow(0.0, <negative>)` and `-Infinity` for
+/// `pow(-0.0, <negative odd integer>)`.
+fn spark_pow(base: f64, exp: f64) -> f64 {
+ if exp.is_nan() || (base.abs() == 1.0 && exp.is_infinite()) {
+ f64::NAN
+ } else {
+ base.powf(exp)
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/spark/math/pow.slt b/datafusion/sqllogictest/test_files/spark/math/pow.slt
index dd7fc13dce..b5c0c5c291 100644
--- a/datafusion/sqllogictest/test_files/spark/math/pow.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/pow.slt
@@ -57,10 +57,11 @@ SELECT pow(CAST('NaN' AS DOUBLE), 1.0);
----
NaN
+## Spark follows java.lang.StrictMath.pow: a NaN exponent gives NaN, also for a base of 1
query R
SELECT pow(1.0, CAST('NaN' AS DOUBLE));
----
-1
+NaN
query R
SELECT pow(CAST('NaN' AS DOUBLE), 0.0);
@@ -86,8 +87,18 @@ Infinity
query R
SELECT pow(CAST('-0.0' AS DOUBLE), -1.0);
----
+-Infinity
+
+query R
+SELECT pow(CAST('-0.0' AS DOUBLE), -2.0);
+----
Infinity
+query R
+SELECT pow(CAST('-0.0' AS DOUBLE), -3.0);
+----
+-Infinity
+
# -inf, +inf
query R
SELECT pow(CAST('Infinity' AS DOUBLE), 1.0);
@@ -129,12 +140,36 @@ SELECT pow(0.5, CAST('-Infinity' AS DOUBLE));
----
Infinity
+## A base of 1 or -1 with an infinite exponent gives NaN in Spark
+query R
+SELECT pow(1.0, CAST('Infinity' AS DOUBLE));
+----
+NaN
+
+query R
+SELECT pow(-1.0, CAST('Infinity' AS DOUBLE));
+----
+NaN
+
+query R
+SELECT pow(1.0, CAST('-Infinity' AS DOUBLE));
+----
+NaN
+
+query R
+SELECT pow(-1.0, CAST('-Infinity' AS DOUBLE));
+----
+NaN
+
# Test Array x Array
statement ok
CREATE TABLE t1(a DOUBLE, b DOUBLE) AS VALUES
(0.0, -1.0),
(2.0, 3.0),
-(CAST(NULL AS DOUBLE), 1.0);
+(CAST(NULL AS DOUBLE), 1.0),
+(CAST('-0.0' AS DOUBLE), -3.0),
+(1.0, CAST('NaN' AS DOUBLE)),
+(-1.0, CAST('Infinity' AS DOUBLE));
query R
SELECT pow(a, b) FROM t1;
@@ -142,6 +177,20 @@ SELECT pow(a, b) FROM t1;
Infinity
8
NULL
+-Infinity
+NaN
+NaN
+
+## Scalar x Array and Array x Scalar, including a NULL base
+query RR
+SELECT pow(1.0, b), pow(a, CAST('Infinity' AS DOUBLE)) FROM t1;
+----
+1 0
+1 Infinity
+1 NULL
+1 0
+NaN NaN
+NaN NaN
statement ok
DROP TABLE t1;