Skip to content

Apache DataFusion SQL Query Engine

fix: Avoid incorrectly rounding large integers in `nanvl` (#22575)

Neil Conway8bbc46004183
diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs
index 251e98bb72..b1f69032ef 100644
--- a/datafusion/functions/src/math/nanvl.rs
+++ b/datafusion/functions/src/math/nanvl.rs
@@ -20,11 +20,11 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, AsArray, Float16Array, Float32Array, Float64Array};
use arrow::datatypes::DataType::{Float16, Float32, Float64};
use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};
+use datafusion_common::types::{NativeType, logical_float64};
use datafusion_common::{Result, ScalarValue, exec_err, utils::take_function_args};
-use datafusion_expr::TypeSignature::Exact;
use datafusion_expr::{
- ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
- Volatility,
+ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_macros::user_doc;
@@ -63,12 +63,32 @@ impl Default for NanvlFunc {
impl NanvlFunc {
pub fn new() -> Self {
+ // Non-float numerics (integers, decimals) and NULL coerce to Float64,
+ // which represents as many inputs as possible before rounding.
+ let non_float = Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_float64()),
+ vec![TypeSignatureClass::Integer, TypeSignatureClass::Decimal],
+ NativeType::Float64,
+ );
+ // Any numeric (including floats) coerces to Float64.
+ let to_float64 = Coercion::new_implicit(
+ TypeSignatureClass::Native(logical_float64()),
+ vec![TypeSignatureClass::Numeric],
+ NativeType::Float64,
+ );
Self {
signature: Signature::one_of(
vec![
- Exact(vec![Float16, Float16]),
- Exact(vec![Float32, Float32]),
- Exact(vec![Float64, Float64]),
+ // If either argument is a non-float numeric (or NULL), both
+ // are computed in Float64. Two arms cover either argument
+ // order.
+ TypeSignature::Coercible(vec![non_float.clone(), to_float64.clone()]),
+ TypeSignature::Coercible(vec![to_float64, non_float]),
+ // Otherwise both arguments are floats; preserve their
+ // (widest common) precision rather than widening to Float64.
+ TypeSignature::Exact(vec![Float16, Float16]),
+ TypeSignature::Exact(vec![Float32, Float32]),
+ TypeSignature::Exact(vec![Float64, Float64]),
],
Volatility::Immutable,
),
@@ -86,9 +106,9 @@ impl ScalarUDFImpl for NanvlFunc {
}
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- match &arg_types[0] {
- Float16 => Ok(Float16),
- Float32 => Ok(Float32),
+ match (&arg_types[0], &arg_types[1]) {
+ (Float16, Float16) => Ok(Float16),
+ (Float32, Float32) => Ok(Float32),
_ => Ok(Float64),
}
}
@@ -97,16 +117,10 @@ impl ScalarUDFImpl for NanvlFunc {
let [x, y] = take_function_args(self.name(), args.args)?;
match (x, y) {
- (ColumnarValue::Scalar(ScalarValue::Float16(Some(v))), y) if v.is_nan() => {
- Ok(y)
- }
- (ColumnarValue::Scalar(ScalarValue::Float32(Some(v))), y) if v.is_nan() => {
- Ok(y)
- }
- (ColumnarValue::Scalar(ScalarValue::Float64(Some(v))), y) if v.is_nan() => {
- Ok(y)
- }
+ // Scalar x: return y if x is NaN, otherwise x (which may be NULL).
+ (ColumnarValue::Scalar(ref x), y) if scalar_is_nan(x) => Ok(y),
(x @ ColumnarValue::Scalar(_), _) => Ok(x),
+ // At least one argument is an array: evaluate element-wise.
(x, y) => {
let args = ColumnarValue::values_to_arrays(&[x, y])?;
Ok(ColumnarValue::Array(nanvl(&args)?))
@@ -119,6 +133,15 @@ impl ScalarUDFImpl for NanvlFunc {
}
}
+fn scalar_is_nan(scalar: &ScalarValue) -> bool {
+ match scalar {
+ ScalarValue::Float16(Some(v)) => v.is_nan(),
+ ScalarValue::Float32(Some(v)) => v.is_nan(),
+ ScalarValue::Float64(Some(v)) => v.is_nan(),
+ _ => false,
+ }
+}
+
/// Nanvl SQL function
///
/// - x is NaN -> output is y (which may itself be NULL)
diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt
index 9dbf8f16d8..64874eb316 100644
--- a/datafusion/sqllogictest/test_files/scalar.slt
+++ b/datafusion/sqllogictest/test_files/scalar.slt
@@ -796,6 +796,31 @@ select nanvl(null, null);
----
NULL
+# nanvl evaluates in the common (widest) float type of its arguments. Mixing
+# narrower floats widens losslessly (Float16 + Float32 -> Float32), while
+# integers, decimals, and NULL are coerced to Float64.
+query TTTTTTTT
+select
+ arrow_typeof(nanvl(arrow_cast(1.0, 'Float16'), arrow_cast(2.0, 'Float16'))),
+ arrow_typeof(nanvl(arrow_cast(1.0, 'Float32'), arrow_cast(2.0, 'Float32'))),
+ arrow_typeof(nanvl(arrow_cast(1.0, 'Float16'), arrow_cast(2.0, 'Float32'))),
+ arrow_typeof(nanvl(arrow_cast(1.0, 'Float16'), arrow_cast(2.0, 'Float64'))),
+ arrow_typeof(nanvl(arrow_cast(1.0, 'Float32'), arrow_cast(2.0, 'Float64'))),
+ arrow_typeof(nanvl(1, 2)),
+ arrow_typeof(nanvl(1, arrow_cast(2.0, 'Float32'))),
+ arrow_typeof(nanvl(null, null));
+----
+Float16 Float32 Float32 Float64 Float64 Float64 Float64 Float64
+
+# nanvl with an integer argument is computed in double precision, even when the
+# other argument is Float32.
+query BB
+select
+ nanvl(16777217, 1) = nanvl(arrow_cast(16777217, 'Float64'), 1.0),
+ nanvl(16777217, arrow_cast(1.0, 'Float32')) = nanvl(arrow_cast(16777217, 'Float64'), 1.0);
+----
+true true
+
# nanvl with columns (round is needed to normalize the outputs of different operating systems)
query RRR rowsort
select round(nanvl(asin(f + a), 2), 5), round(nanvl(asin(b + c), 3), 5), round(nanvl(asin(d + e), 4), 5) from small_floats;