Skip to content

Apache DataFusion SQL Query Engine

fix null handling for `nanvl` & implement fast path (#20205)

Kumar Ujjawal51e5c98cf52c
diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml
index 5af901a6bf..599b85116b 100644
--- a/datafusion/functions/Cargo.toml
+++ b/datafusion/functions/Cargo.toml
@@ -132,6 +132,11 @@ harness = false
name = "gcd"
required-features = ["math_expressions"]
+[[bench]]
+harness = false
+name = "nanvl"
+required-features = ["math_expressions"]
+
[[bench]]
harness = false
name = "uuid"
diff --git a/datafusion/functions/benches/nanvl.rs b/datafusion/functions/benches/nanvl.rs
new file mode 100644
index 0000000000..206eebd81e
--- /dev/null
+++ b/datafusion/functions/benches/nanvl.rs
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+extern crate criterion;
+
+use arrow::array::{ArrayRef, Float32Array, Float64Array};
+use arrow::datatypes::{DataType, Field};
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+use datafusion_functions::math::nanvl;
+use std::hint::black_box;
+use std::sync::Arc;
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let nanvl_fn = nanvl();
+ let config_options = Arc::new(ConfigOptions::default());
+
+ // Scalar benchmarks
+ c.bench_function("nanvl/scalar_f64", |b| {
+ let args = ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::NAN))),
+ ColumnarValue::Scalar(ScalarValue::Float64(Some(1.0))),
+ ],
+ arg_fields: vec![
+ Field::new("a", DataType::Float64, true).into(),
+ Field::new("b", DataType::Float64, true).into(),
+ ],
+ number_rows: 1,
+ return_field: Field::new("f", DataType::Float64, true).into(),
+ config_options: Arc::clone(&config_options),
+ };
+
+ b.iter(|| black_box(nanvl_fn.invoke_with_args(args.clone()).unwrap()))
+ });
+
+ c.bench_function("nanvl/scalar_f32", |b| {
+ let args = ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Scalar(ScalarValue::Float32(Some(f32::NAN))),
+ ColumnarValue::Scalar(ScalarValue::Float32(Some(1.0))),
+ ],
+ arg_fields: vec![
+ Field::new("a", DataType::Float32, true).into(),
+ Field::new("b", DataType::Float32, true).into(),
+ ],
+ number_rows: 1,
+ return_field: Field::new("f", DataType::Float32, true).into(),
+ config_options: Arc::clone(&config_options),
+ };
+
+ b.iter(|| black_box(nanvl_fn.invoke_with_args(args.clone()).unwrap()))
+ });
+
+ // Array benchmarks
+ for size in [1024, 4096, 8192] {
+ let a64: ArrayRef = Arc::new(Float64Array::from(vec![f64::NAN; size]));
+ let b64: ArrayRef = Arc::new(Float64Array::from(vec![1.0; size]));
+ c.bench_function(&format!("nanvl/array_f64/{size}"), |bench| {
+ let args = ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(Arc::clone(&a64)),
+ ColumnarValue::Array(Arc::clone(&b64)),
+ ],
+ arg_fields: vec![
+ Field::new("a", DataType::Float64, true).into(),
+ Field::new("b", DataType::Float64, true).into(),
+ ],
+ number_rows: size,
+ return_field: Field::new("f", DataType::Float64, true).into(),
+ config_options: Arc::clone(&config_options),
+ };
+ bench.iter(|| black_box(nanvl_fn.invoke_with_args(args.clone()).unwrap()))
+ });
+
+ let a32: ArrayRef = Arc::new(Float32Array::from(vec![f32::NAN; size]));
+ let b32: ArrayRef = Arc::new(Float32Array::from(vec![1.0; size]));
+ c.bench_function(&format!("nanvl/array_f32/{size}"), |bench| {
+ let args = ScalarFunctionArgs {
+ args: vec![
+ ColumnarValue::Array(Arc::clone(&a32)),
+ ColumnarValue::Array(Arc::clone(&b32)),
+ ],
+ arg_fields: vec![
+ Field::new("a", DataType::Float32, true).into(),
+ Field::new("b", DataType::Float32, true).into(),
+ ],
+ number_rows: size,
+ return_field: Field::new("f", DataType::Float32, true).into(),
+ config_options: Arc::clone(&config_options),
+ };
+ bench.iter(|| black_box(nanvl_fn.invoke_with_args(args.clone()).unwrap()))
+ });
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs
index 6daf476e25..2bdc3fbbc6 100644
--- a/datafusion/functions/src/math/nanvl.rs
+++ b/datafusion/functions/src/math/nanvl.rs
@@ -18,12 +18,10 @@
use std::any::Any;
use std::sync::Arc;
-use crate::utils::make_scalar_function;
-
use arrow::array::{ArrayRef, AsArray, Float16Array, Float32Array, Float64Array};
use arrow::datatypes::DataType::{Float16, Float32, Float64};
use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};
-use datafusion_common::{DataFusionError, Result, exec_err};
+use datafusion_common::{Result, ScalarValue, exec_err, utils::take_function_args};
use datafusion_expr::TypeSignature::Exact;
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
@@ -101,7 +99,24 @@ impl ScalarUDFImpl for NanvlFunc {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- make_scalar_function(nanvl, vec![])(&args.args)
+ let [x, y] = take_function_args(self.name(), args.args)?;
+
+ match (x, y) {
+ (ColumnarValue::Scalar(ScalarValue::Float16(Some(v))), y) if v.is_nan() => {
+ Ok(y)
+ }
+ (ColumnarValue::Scalar(ScalarValue::Float32(Some(v))), y) if v.is_nan() => {
+ Ok(y)
+ }
+ (ColumnarValue::Scalar(ScalarValue::Float64(Some(v))), y) if v.is_nan() => {
+ Ok(y)
+ }
+ (x @ ColumnarValue::Scalar(_), _) => Ok(x),
+ (x, y) => {
+ let args = ColumnarValue::values_to_arrays(&[x, y])?;
+ Ok(ColumnarValue::Array(nanvl(&args)?))
+ }
+ }
}
fn documentation(&self) -> Option<&Documentation> {
@@ -110,42 +125,49 @@ impl ScalarUDFImpl for NanvlFunc {
}
/// Nanvl SQL function
+///
+/// - x is NaN -> output is y (which may itself be NULL)
+/// - otherwise -> output is x (which may itself be NULL)
fn nanvl(args: &[ArrayRef]) -> Result<ArrayRef> {
match args[0].data_type() {
Float64 => {
- let compute_nanvl = |x: f64, y: f64| {
- if x.is_nan() { y } else { x }
- };
-
- let x = args[0].as_primitive() as &Float64Array;
- let y = args[1].as_primitive() as &Float64Array;
- arrow::compute::binary::<_, _, _, Float64Type>(x, y, compute_nanvl)
- .map(|res| Arc::new(res) as _)
- .map_err(DataFusionError::from)
+ let x = args[0].as_primitive::<Float64Type>();
+ let y = args[1].as_primitive::<Float64Type>();
+ let result: Float64Array = x
+ .iter()
+ .zip(y.iter())
+ .map(|(x_value, y_value)| match x_value {
+ Some(x_value) if x_value.is_nan() => y_value,
+ _ => x_value,
+ })
+ .collect();
+ Ok(Arc::new(result) as ArrayRef)
}
Float32 => {
- let compute_nanvl = |x: f32, y: f32| {
- if x.is_nan() { y } else { x }
- };
-
- let x = args[0].as_primitive() as &Float32Array;
- let y = args[1].as_primitive() as &Float32Array;
- arrow::compute::binary::<_, _, _, Float32Type>(x, y, compute_nanvl)
- .map(|res| Arc::new(res) as _)
- .map_err(DataFusionError::from)
+ let x = args[0].as_primitive::<Float32Type>();
+ let y = args[1].as_primitive::<Float32Type>();
+ let result: Float32Array = x
+ .iter()
+ .zip(y.iter())
+ .map(|(x_value, y_value)| match x_value {
+ Some(x_value) if x_value.is_nan() => y_value,
+ _ => x_value,
+ })
+ .collect();
+ Ok(Arc::new(result) as ArrayRef)
}
Float16 => {
- let compute_nanvl =
- |x: <Float16Type as arrow::datatypes::ArrowPrimitiveType>::Native,
- y: <Float16Type as arrow::datatypes::ArrowPrimitiveType>::Native| {
- if x.is_nan() { y } else { x }
- };
-
- let x = args[0].as_primitive() as &Float16Array;
- let y = args[1].as_primitive() as &Float16Array;
- arrow::compute::binary::<_, _, _, Float16Type>(x, y, compute_nanvl)
- .map(|res| Arc::new(res) as _)
- .map_err(DataFusionError::from)
+ let x = args[0].as_primitive::<Float16Type>();
+ let y = args[1].as_primitive::<Float16Type>();
+ let result: Float16Array = x
+ .iter()
+ .zip(y.iter())
+ .map(|(x_value, y_value)| match x_value {
+ Some(x_value) if x_value.is_nan() => y_value,
+ _ => x_value,
+ })
+ .collect();
+ Ok(Arc::new(result) as ArrayRef)
}
other => exec_err!("Unsupported data type {other:?} for function nanvl"),
}
@@ -163,8 +185,8 @@ mod test {
#[test]
fn test_nanvl_f64() {
let args: Vec<ArrayRef> = vec![
- Arc::new(Float64Array::from(vec![1.0, f64::NAN, 3.0, f64::NAN])), // y
- Arc::new(Float64Array::from(vec![5.0, 6.0, f64::NAN, f64::NAN])), // x
+ Arc::new(Float64Array::from(vec![1.0, f64::NAN, 3.0, f64::NAN])), // x
+ Arc::new(Float64Array::from(vec![5.0, 6.0, f64::NAN, f64::NAN])), // y
];
let result = nanvl(&args).expect("failed to initialize function nanvl");
@@ -181,8 +203,8 @@ mod test {
#[test]
fn test_nanvl_f32() {
let args: Vec<ArrayRef> = vec![
- Arc::new(Float32Array::from(vec![1.0, f32::NAN, 3.0, f32::NAN])), // y
- Arc::new(Float32Array::from(vec![5.0, 6.0, f32::NAN, f32::NAN])), // x
+ Arc::new(Float32Array::from(vec![1.0, f32::NAN, 3.0, f32::NAN])), // x
+ Arc::new(Float32Array::from(vec![5.0, 6.0, f32::NAN, f32::NAN])), // y
];
let result = nanvl(&args).expect("failed to initialize function nanvl");
diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt
index 4e078d1e69..57769941e2 100644
--- a/datafusion/sqllogictest/test_files/expr.slt
+++ b/datafusion/sqllogictest/test_files/expr.slt
@@ -60,7 +60,7 @@ SELECT
isnan(NULL),
iszero(NULL)
----
-NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 NULL NULL NULL
# test_array_cast_invalid_timezone_will_panic
statement error Parser error: Invalid timezone "Foo": failed to parse timezone
diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt
index f5d6619d52..7a4a81b5fa 100644
--- a/datafusion/sqllogictest/test_files/scalar.slt
+++ b/datafusion/sqllogictest/test_files/scalar.slt
@@ -765,11 +765,11 @@ select nanvl(null, 64);
----
NULL
-# nanvl scalar nulls #1
+# nanvl scalar nulls #1 - x is not NaN, so return x even if y is NULL
query R rowsort
select nanvl(2, null);
----
-NULL
+2
# nanvl scalar nulls #2
query R rowsort