Apache DataFusion SQL Query Engine
fix: widen `power(decimal, float)` to Float64, fix bugs (#22482)
Neil Conwaye20d59cec8c7
diff --git a/datafusion/core/tests/expr_api/simplification.rs b/datafusion/core/tests/expr_api/simplification.rsindex 245aba6684..6e1271ef19 100644--- a/datafusion/core/tests/expr_api/simplification.rs+++ b/datafusion/core/tests/expr_api/simplification.rs@@ -648,13 +648,19 @@ fn test_simplify_power() {let expected = col("c3_non_null");test_simplify(expr, expected)}- // Power(c3, Log(c3, c4)) ===> c4+ // Power(c3, Log(c3, c4)) ===> cast(c4 AS Int64)+ // The simplifier rewrites `power(b, log(b, x))` to `x`, but the+ // rewritten expression must keep the same type as the original+ // `power` call. `power`'s declared return type follows its base+ // argument (c3 = Int64), so the UInt32 c4 has to be cast to Int64+ // to preserve the output schema the optimizer already committed to.{let expr = power(col("c3_non_null"),log(col("c3_non_null"), col("c4_non_null")),);- let expected = col("c4_non_null");+ let expected =+ Expr::Cast(Cast::new(Box::new(col("c4_non_null")), DataType::Int64));test_simplify(expr, expected)}// Power(c3, c4) ===> Power(c3, c4)diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.tomlindex d6a6693d86..4eca16961f 100644--- a/datafusion/functions/Cargo.toml+++ b/datafusion/functions/Cargo.toml@@ -212,6 +212,11 @@ harness = falsename = "atan2"required-features = ["math_expressions"]+[[bench]]+harness = false+name = "power"+required-features = ["math_expressions"]+[[bench]]harness = falsename = "substr_index"diff --git a/datafusion/functions/benches/power.rs b/datafusion/functions/benches/power.rsnew file mode 100644index 0000000000..5336e42ebe--- /dev/null+++ b/datafusion/functions/benches/power.rs@@ -0,0 +1,140 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Microbenchmark for `power(decimal_array, int_*)`.+//!+//! Covers both array- and scalar-shaped integer exponents on a Decimal+//! base. Both shapes are dispatched to the native per-row decimal kernel;+//! the bench guards against any future change that routes either shape+//! through a Float64 round-trip, which is measurably slower than the+//! decimal kernel for the cases the kernel can handle.++extern crate criterion;++use arrow::array::{Decimal128Array, Int64Array};+use arrow::datatypes::{DataType, Field, FieldRef};+use criterion::{Criterion, criterion_group, criterion_main};+use datafusion_common::ScalarValue;+use datafusion_common::config::ConfigOptions;+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF};+use datafusion_functions::math::power;+use std::hint::black_box;+use std::sync::Arc;++fn make_decimal_array(size: usize, precision: u8, scale: i8) -> Decimal128Array {+ // Use a fixed unscaled value (250) so the bench is independent of `scale`.+ // The four-arm dispatch in `power` only cares about the Decimal variant+ // and the exponent's shape, not the numeric value.+ let arr = Decimal128Array::from(vec![250i128; size]);+ arr.with_precision_and_scale(precision, scale).unwrap()+}++fn make_int_array(size: usize, value: i64) -> Int64Array {+ Int64Array::from(vec![value; size])+}++fn run_power(+ power_fn: &ScalarUDF,+ args: &[ColumnarValue],+ arg_fields: &[FieldRef],+ return_field: &FieldRef,+ config_options: &Arc<ConfigOptions>,+ num_rows: usize,+) {+ black_box(+ power_fn+ .invoke_with_args(ScalarFunctionArgs {+ args: args.to_vec(),+ arg_fields: arg_fields.to_vec(),+ number_rows: num_rows,+ return_field: Arc::clone(return_field),+ config_options: Arc::clone(config_options),+ })+ .unwrap(),+ );+}++fn criterion_benchmark(c: &mut Criterion) {+ let power_fn = power();+ let config_options = Arc::new(ConfigOptions::default());+ let precision: u8 = 20;+ let scale: i8 = 2;+ let decimal_ty = DataType::Decimal128(precision, scale);++ // Exponents are bounded by what the native decimal kernel can handle+ // without overflowing the i128 intermediate; see+ // <https://github.com/apache/datafusion/issues/22480>+ let exponents = [2i64, 4, 8];++ for size in [1024usize, 8192] {+ let base_arr = Arc::new(make_decimal_array(size, precision, scale));+ let base_field: FieldRef = Field::new("base", decimal_ty.clone(), true).into();+ let exp_field: FieldRef = Field::new("exp", DataType::Int64, true).into();+ let return_field: FieldRef = Field::new("r", decimal_ty.clone(), true).into();+ let arg_fields = vec![base_field, exp_field];++ for &exp in &exponents {+ let exp_arr = Arc::new(make_int_array(size, exp));+ let array_args = vec![+ ColumnarValue::Array(base_arr.clone()),+ ColumnarValue::Array(exp_arr),+ ];+ c.bench_function(+ &format!(+ "power decimal({precision},{scale}) array x int array, exp={exp}, n={size}"+ ),+ |b| {+ b.iter(|| {+ run_power(+ &power_fn,+ &array_args,+ &arg_fields,+ &return_field,+ &config_options,+ size,+ )+ })+ },+ );++ let scalar_args = vec![+ ColumnarValue::Array(base_arr.clone()),+ ColumnarValue::Scalar(ScalarValue::Int64(Some(exp))),+ ];+ c.bench_function(+ &format!(+ "power decimal({precision},{scale}) array x int scalar, exp={exp}, n={size}"+ ),+ |b| {+ b.iter(|| {+ run_power(+ &power_fn,+ &scalar_args,+ &arg_fields,+ &return_field,+ &config_options,+ size,+ )+ })+ },+ );+ }+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/datafusion/functions/src/math/power.rs b/datafusion/functions/src/math/power.rsindex 3fe30a1ffa..fe8c179bff 100644--- a/datafusion/functions/src/math/power.rs+++ b/datafusion/functions/src/math/power.rs@@ -32,7 +32,7 @@ use datafusion_common::{Result, ScalarValue, internal_err};use datafusion_expr::expr::ScalarFunction;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};use datafusion_expr::{- Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,+ Cast, Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, lit,};use datafusion_macros::user_doc;@@ -92,8 +92,7 @@ impl PowerFunc {Self {signature: Signature::one_of(vec![- TypeSignature::Coercible(vec![decimal.clone(), integer]),- TypeSignature::Coercible(vec![decimal.clone(), float.clone()]),+ TypeSignature::Coercible(vec![decimal, integer]),TypeSignature::Coercible(vec![float; 2]),],Volatility::Immutable,@@ -249,8 +248,7 @@ where})}-/// Fallback implementation using f64 for negative or non-integer exponents.-/// This handles cases that cannot be computed using integer arithmetic.+/// Fallback for `pow_decimal_int` when the exponent is negative or non-integer.fn pow_decimal_float_fallback<T>(base: T, scale: i8, exp: f64) -> Result<T, ArrowError>whereT: ToPrimitive + NumCast + Copy,@@ -271,7 +269,7 @@ wheredecimal_from_i128(result_i128)}-/// Decimal256 specialized float exponent version.+/// Like `pow_decimal_float`, but specialized for Decimal256.fn pow_decimal256_float(base: i256, scale: i8, exp: f64) -> Result<i256, ArrowError> {if exp.is_finite() && exp.trunc() == exp && exp >= 0f64 && exp < u32::MAX as f64 {return pow_decimal256_int(base, scale, exp as i64);@@ -286,7 +284,7 @@ fn pow_decimal256_float(base: i256, scale: i8, exp: f64) -> Result<i256, ArrowErpow_decimal256_float_fallback(base, scale, exp)}-/// Decimal256 specialized integer exponent version.+/// Like `pow_decimal_int`, but specialized for Decimal256.fn pow_decimal256_int(base: i256, scale: i8, exp: i64) -> Result<i256, ArrowError> {if exp < 0 {return pow_decimal256_float(base, scale, exp as f64);@@ -346,7 +344,7 @@ fn pow_decimal256_int(base: i256, scale: i8, exp: i64) -> Result<i256, ArrowErro}}-/// Fallback implementation for Decimal256.+/// Like `pow_decimal_float_fallback`, but specialized for Decimal256.fn pow_decimal256_float_fallback(base: i256,scale: i8,@@ -369,37 +367,6 @@ fn pow_decimal256_float_fallback(Ok(i256::from_i128(result_i128))}-/// Fallback implementation for decimal power when exponent is an array.-/// Casts decimal to float64, computes power, and casts back to original decimal type.-/// This is used for performance when exponent varies per-row.-fn pow_decimal_with_float_fallback(- base: &ArrayRef,- exponent: &ColumnarValue,- num_rows: usize,-) -> Result<ColumnarValue> {- use arrow::compute::cast;-- let original_type = base.data_type().clone();- let base_f64 = cast(base.as_ref(), &DataType::Float64)?;-- let exp_f64 = match exponent {- ColumnarValue::Array(arr) => cast(arr.as_ref(), &DataType::Float64)?,- ColumnarValue::Scalar(scalar) => {- let scalar_f64 = scalar.cast_to(&DataType::Float64)?;- scalar_f64.to_array_of_size(num_rows)?- }- };-- let result_f64 = calculate_binary_math::<Float64Type, Float64Type, Float64Type, _>(- &base_f64,- &ColumnarValue::Array(exp_f64),- float64_power_checked,- )?;-- let result = cast(result_f64.as_ref(), &original_type)?;- Ok(ColumnarValue::Array(result))-}-impl ScalarUDFImpl for PowerFunc {fn name(&self) -> &str {"power"@@ -410,11 +377,17 @@ impl ScalarUDFImpl for PowerFunc {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- if arg_types[0].is_null() {- Ok(DataType::Float64)- } else {- Ok(arg_types[0].clone())+ // Return type as a function of (base, exponent). After signature+ // coercion, we have to handle the following cases:+ //+ // - NULL on either side -> Float64 (typed NULL)+ // - (Decimal, Int64) -> the base's Decimal type+ // - (Float64, Float64) -> Float64+ let [base, exponent] = take_function_args(self.name(), arg_types)?;+ if base.is_null() || exponent.is_null() {+ return Ok(DataType::Float64);}+ Ok(base.clone())}fn aliases(&self) -> &[String] {@@ -423,23 +396,18 @@ impl ScalarUDFImpl for PowerFunc {fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {let [base, exponent] = take_function_args(self.name(), &args.args)?;-- // For decimal types, only use native decimal- // operations when we have a scalar exponent. When the exponent is an array,- // fall back to float computation for better performance.- let use_float_fallback = matches!(- base.data_type(),- DataType::Decimal32(_, _)- | DataType::Decimal64(_, _)- | DataType::Decimal128(_, _)- | DataType::Decimal256(_, _)- ) && matches!(exponent, ColumnarValue::Array(_));-let base = base.to_array(args.number_rows)?;- // If decimal with array exponent, cast to float and compute- if use_float_fallback {- return pow_decimal_with_float_fallback(&base, exponent, args.number_rows);+ macro_rules! decimal_pow_arm {+ ($decimal_ty:ident, $pow_fn:ident, $precision:expr, $scale:expr) => {+ calculate_binary_decimal_math::<$decimal_ty, Int64Type, $decimal_ty, _>(+ &base,+ exponent,+ |b, e| $pow_fn(b, *$scale, e),+ *$precision,+ *$scale,+ )?+ };}let arr: ArrayRef = match (base.data_type(), exponent.data_type()) {@@ -451,106 +419,16 @@ impl ScalarUDFImpl for PowerFunc {)?}(DataType::Decimal32(precision, scale), DataType::Int64) => {- calculate_binary_decimal_math::<Decimal32Type, Int64Type, Decimal32Type, _>(- &base,- exponent,- |b, e| pow_decimal_int(b, *scale, e),- *precision,- *scale,- )?- }- (DataType::Decimal32(precision, scale), DataType::Float64) => {- calculate_binary_decimal_math::<- Decimal32Type,- Float64Type,- Decimal32Type,- _,- >(- &base,- exponent,- |b, e| pow_decimal_float(b, *scale, e),- *precision,- *scale,- )?+ decimal_pow_arm!(Decimal32Type, pow_decimal_int, precision, scale)}(DataType::Decimal64(precision, scale), DataType::Int64) => {- calculate_binary_decimal_math::<Decimal64Type, Int64Type, Decimal64Type, _>(- &base,- exponent,- |b, e| pow_decimal_int(b, *scale, e),- *precision,- *scale,- )?- }- (DataType::Decimal64(precision, scale), DataType::Float64) => {- calculate_binary_decimal_math::<- Decimal64Type,- Float64Type,- Decimal64Type,- _,- >(- &base,- exponent,- |b, e| pow_decimal_float(b, *scale, e),- *precision,- *scale,- )?+ decimal_pow_arm!(Decimal64Type, pow_decimal_int, precision, scale)}(DataType::Decimal128(precision, scale), DataType::Int64) => {- calculate_binary_decimal_math::<- Decimal128Type,- Int64Type,- Decimal128Type,- _,- >(- &base,- exponent,- |b, e| pow_decimal_int(b, *scale, e),- *precision,- *scale,- )?- }- (DataType::Decimal128(precision, scale), DataType::Float64) => {- calculate_binary_decimal_math::<- Decimal128Type,- Float64Type,- Decimal128Type,- _,- >(- &base,- exponent,- |b, e| pow_decimal_float(b, *scale, e),- *precision,- *scale,- )?+ decimal_pow_arm!(Decimal128Type, pow_decimal_int, precision, scale)}(DataType::Decimal256(precision, scale), DataType::Int64) => {- calculate_binary_decimal_math::<- Decimal256Type,- Int64Type,- Decimal256Type,- _,- >(- &base,- exponent,- |b, e| pow_decimal256_int(b, *scale, e),- *precision,- *scale,- )?- }- (DataType::Decimal256(precision, scale), DataType::Float64) => {- calculate_binary_decimal_math::<- Decimal256Type,- Float64Type,- Decimal256Type,- _,- >(- &base,- exponent,- |b, e| pow_decimal256_float(b, *scale, e),- *precision,- *scale,- )?+ decimal_pow_arm!(Decimal256Type, pow_decimal256_int, precision, scale)}(base_type, exp_type) => {return internal_err!(@@ -582,12 +460,13 @@ impl ScalarUDFImpl for PowerFunc {)));}+ let return_type = self.return_type(&[base_type, exponent_type.clone()])?;match exponent {Expr::Literal(value, _)if value == ScalarValue::new_zero(&exponent_type)? =>{Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(- &base_type,+ &return_type,)?)))}Expr::Literal(value, _) if value == ScalarValue::new_one(&exponent_type)? => {@@ -596,8 +475,17 @@ impl ScalarUDFImpl for PowerFunc {Expr::ScalarFunction(ScalarFunction { func, mut args })if is_log(&func) && args.len() == 2 && base == args[0] =>{+ // The inner `b` may have a different type than the power+ // call's `return_type` (e.g. `power(int64, log(int64,+ // uint32))` returns Int64 but `b` is UInt32). Wrap it+ // in a cast to preserve the optimizer's expected schema.let b = args.pop().unwrap(); // length checked above- Ok(ExprSimplifyResult::Simplified(b))+ let result = if info.get_data_type(&b)? != return_type {+ Expr::Cast(Cast::new(Box::new(b), return_type))+ } else {+ b+ };+ Ok(ExprSimplifyResult::Simplified(result))}_ => Ok(ExprSimplifyResult::Original(vec![base, exponent])),}diff --git a/datafusion/sqllogictest/test_files/decimal.slt b/datafusion/sqllogictest/test_files/decimal.sltindex 5faf801c84..d9eac84928 100644--- a/datafusion/sqllogictest/test_files/decimal.slt+++ b/datafusion/sqllogictest/test_files/decimal.slt@@ -1121,23 +1121,31 @@ SELECT power(2.5::decimal(38, 3), 4), arrow_typeof(power(2.5::decimal(38, 3), 4)query RTSELECT power(2.5, 4.0), arrow_typeof(power(2.5, 4.0));-----39 Decimal128(2, 1)+39.0625 Float64-# Non-integer exponent now works (fallback to f64)query RTSELECT power(2.5, 4.2), arrow_typeof(power(2.5, 4.2));-----46.9 Decimal128(2, 1)+46.9189232024 Float64-query error Compute error: Cannot use non-finite exp: NaN-SELECT power(2::decimal(38, 0), arrow_cast('NaN','Float64'))+query RT+SELECT power(2::decimal(38, 0), arrow_cast('NaN','Float64')),+ arrow_typeof(power(2::decimal(38, 0), arrow_cast('NaN','Float64')));+----+NaN Float64-query error Compute error: Cannot use non-finite exp: inf-SELECT power(2::decimal(38, 0), arrow_cast('INF','Float64'))+query RT+SELECT power(2::decimal(38, 0), arrow_cast('INF','Float64')),+ arrow_typeof(power(2::decimal(38, 0), arrow_cast('INF','Float64')));+----+Infinity Float64-# Floating above u32::max now works (fallback to f64, returns infinity which is an error)-query error Arrow error: Arithmetic overflow: Result of 2\^5000000000.1 is not finite-SELECT power(2::decimal(38, 0), 5000000000.1)+# Result overflows finite Float64 range+query RT+SELECT power(2::decimal(38, 0), 5000000000.1),+ arrow_typeof(power(2::decimal(38, 0), 5000000000.1));+----+Infinity Float64# Integer Above u32::max - still goes through integer path which failsquery error Arrow error: Arithmetic overflow: Unsupported exp valuediff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.sltindex 475434883d..e261bada87 100644--- a/datafusion/sqllogictest/test_files/math.slt+++ b/datafusion/sqllogictest/test_files/math.slt@@ -818,6 +818,8 @@ from values81NULL+# There is no variant of `power` that accepts (Decimal, Decimal); type coercion+# casts both arguments to `Float64`, so the result is `Float64`.query RT rowsortselectpower(base::decimal(38, 0), exponent::decimal(38, 0)),@@ -830,12 +832,12 @@ from values(2, 3),(3, 4) as t(base, exponent);-----0 Decimal128(38, 0)-1 Decimal128(38, 0)-4 Decimal128(38, 0)-625 Decimal128(38, 0)-8 Decimal128(38, 0)-81 Decimal128(38, 0)+0 Float64+1 Float64+4 Float64+625 Float64+8 Float64+81 Float64query RTselect