Skip to content

Apache DataFusion SQL Query Engine

Add support for nested types to nullif. (#21764)

Anastasios Bakogiannis22bb4e6b752c
diff --git a/Cargo.lock b/Cargo.lock
index 033a430c57..dbddaf300d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2185,6 +2185,7 @@ dependencies = [
"datafusion-expr",
"datafusion-expr-common",
"datafusion-macros",
+ "datafusion-physical-expr-common",
"env_logger",
"hex",
"itertools 0.14.0",
diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml
index 410eb3c968..2d69728bda 100644
--- a/datafusion/functions/Cargo.toml
+++ b/datafusion/functions/Cargo.toml
@@ -78,6 +78,7 @@ datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-expr-common = { workspace = true }
datafusion-macros = { workspace = true }
+datafusion-physical-expr-common = { workspace = true }
hex = { workspace = true, optional = true }
itertools = { workspace = true }
log = { workspace = true }
diff --git a/datafusion/functions/src/core/nullif.rs b/datafusion/functions/src/core/nullif.rs
index 89c4d4eb0f..f58ae857d4 100644
--- a/datafusion/functions/src/core/nullif.rs
+++ b/datafusion/functions/src/core/nullif.rs
@@ -18,11 +18,11 @@
use arrow::datatypes::DataType;
use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs};
-use arrow::compute::kernels::cmp::eq;
use arrow::compute::kernels::nullif::nullif;
use datafusion_common::{Result, ScalarValue, utils::take_function_args};
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
use datafusion_macros::user_doc;
+use datafusion_physical_expr_common::datum::compare_with_eq;
#[user_doc(
doc_section(label = "Conditional Functions"),
@@ -111,25 +111,29 @@ impl ScalarUDFImpl for NullIfFunc {
/// 1 - if the left is equal to this expr2, then the result is NULL, otherwise left value is passed.
fn nullif_func(args: &[ColumnarValue]) -> Result<ColumnarValue> {
let [lhs, rhs] = take_function_args("nullif", args)?;
+ let is_nested = lhs.data_type().is_nested();
match (lhs, rhs) {
(ColumnarValue::Array(lhs), ColumnarValue::Scalar(rhs)) => {
let rhs = rhs.to_scalar()?;
- let array = nullif(lhs, &eq(&lhs, &rhs)?)?;
+ let eq_array = compare_with_eq(lhs, &rhs, is_nested)?;
+ let array = nullif(lhs, &eq_array)?;
Ok(ColumnarValue::Array(array))
}
(ColumnarValue::Array(lhs), ColumnarValue::Array(rhs)) => {
- let array = nullif(lhs, &eq(&lhs, &rhs)?)?;
+ let eq_array = compare_with_eq(lhs, rhs, is_nested)?;
+ let array = nullif(lhs, &eq_array)?;
Ok(ColumnarValue::Array(array))
}
(ColumnarValue::Scalar(lhs), ColumnarValue::Array(rhs)) => {
let lhs_s = lhs.to_scalar()?;
let lhs_a = lhs.to_array_of_size(rhs.len())?;
+ let eq_array = compare_with_eq(&lhs_s, rhs, is_nested)?;
let array = nullif(
// nullif in arrow-select does not support Datum, so we need to convert to array
lhs_a.as_ref(),
- &eq(&lhs_s, &rhs)?,
+ &eq_array,
)?;
Ok(ColumnarValue::Array(array))
}
@@ -148,7 +152,12 @@ fn nullif_func(args: &[ColumnarValue]) -> Result<ColumnarValue> {
mod tests {
use std::sync::Arc;
- use arrow::array::*;
+ use arrow::{
+ array::*,
+ buffer::NullBuffer,
+ datatypes::{Field, Fields, Int64Type},
+ };
+ use datafusion_common::DataFusionError;
use super::*;
@@ -251,6 +260,104 @@ mod tests {
Ok(())
}
+ #[test]
+ fn nullif_struct() -> Result<()> {
+ let fields = Fields::from(vec![
+ Field::new("a", DataType::Int64, true),
+ Field::new("b", DataType::Utf8, true),
+ ]);
+
+ let lhs_a = Arc::new(Int64Array::from(vec![Some(1), Some(2), None]));
+ let lhs_b = Arc::new(StringArray::from(vec![Some("1"), Some("2"), None]));
+ let lhs_nulls = Some(NullBuffer::from(vec![true, true, false]));
+ let lhs = ColumnarValue::Array(Arc::new(StructArray::new(
+ fields.clone(),
+ vec![lhs_a, lhs_b],
+ lhs_nulls,
+ )));
+
+ let rhs_a = Arc::new(Int64Array::from(vec![Some(1), Some(9), None]));
+ let rhs_b = Arc::new(StringArray::from(vec![Some("1"), Some("2"), None]));
+ let rhs_nulls = Some(NullBuffer::from(vec![true, true, false]));
+ let rhs = ColumnarValue::Array(Arc::new(StructArray::new(
+ fields.clone(),
+ vec![rhs_a, rhs_b],
+ rhs_nulls,
+ )));
+
+ let result = nullif_func(&[lhs, rhs])?;
+ let result = result.into_array(0).expect("Failed to convert to array");
+
+ let expected_arrays = vec![
+ Arc::new(Int64Array::from(vec![None, Some(2), None])) as ArrayRef,
+ Arc::new(StringArray::from(vec![None, Some("2"), None])) as ArrayRef,
+ ];
+ let expected_nulls = NullBuffer::from(vec![false, true, false]);
+
+ let expected = Arc::new(StructArray::try_new(
+ fields,
+ expected_arrays,
+ Some(expected_nulls),
+ )?) as ArrayRef;
+
+ assert_eq!(expected.as_ref(), result.as_ref());
+
+ Ok(())
+ }
+
+ #[test]
+ fn nullif_list() -> Result<()> {
+ let lhs = Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(3)]),
+ Some(vec![]),
+ Some(vec![Some(5), Some(6), Some(7)]),
+ None,
+ ]));
+ let lhs = ColumnarValue::Array(lhs);
+
+ let rhs = Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ ]));
+ let rhs = ColumnarValue::Scalar(ScalarValue::List(rhs));
+
+ let result = nullif_func(&[lhs, rhs])?;
+ let result = result.into_array(0).expect("Failed to convert to array");
+
+ let expected = Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
+ None,
+ Some(vec![Some(3)]),
+ Some(vec![]),
+ Some(vec![Some(5), Some(6), Some(7)]),
+ None,
+ ])) as ArrayRef;
+
+ assert_eq!(expected.as_ref(), result.as_ref());
+
+ Ok(())
+ }
+
+ #[test]
+ fn nullif_compare_nested_to_unnested() -> Result<()> {
+ let lhs = Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
+ Some(vec![Some(1), Some(2)]),
+ Some(vec![Some(3)]),
+ Some(vec![]),
+ Some(vec![Some(5), Some(6), Some(7)]),
+ None,
+ ]));
+ let lhs = ColumnarValue::Array(lhs);
+
+ let rhs = Arc::new(Int64Array::from(vec![Some(1), Some(3), None, None, None]));
+ let rhs = ColumnarValue::Array(rhs);
+
+ let result = nullif_func(&[lhs, rhs]);
+
+ assert!(matches!(result, Err(DataFusionError::ArrowError(_, _))));
+
+ Ok(())
+ }
+
#[test]
fn nullif_literal_first() -> Result<()> {
let a = Int32Array::from(vec![Some(1), Some(2), None, None, Some(3), Some(4)]);