Skip to content

Apache DataFusion SQL Query Engine

fix: do not evaluate a volatile coalesce or nvl argument twice (#25561)

DeviousCardid75fd7b86042
diff --git a/datafusion/functions/src/core/coalesce.rs b/datafusion/functions/src/core/coalesce.rs
index 831f26fcb5..d554601f5a 100644
--- a/datafusion/functions/src/core/coalesce.rs
+++ b/datafusion/functions/src/core/coalesce.rs
@@ -2,10 +2,14 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+use arrow::array::{BooleanArray, new_null_array};
+use arrow::compute::kernels::zip::zip;
+use arrow::compute::{and, is_not_null, is_null};
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::{Result, exec_err, internal_err, plan_err};
use datafusion_expr::binary::try_type_union_resolution;
use datafusion_expr::conditional_expressions::CaseBuilder;
+use datafusion_expr::expr::ScalarFunction;
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
use datafusion_expr::{
ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,
@@ -90,22 +94,88 @@ impl ScalarUDFImpl for CoalesceFunc {
));
}
- let n = args.len();
- let (init, last_elem) = args.split_at(n - 1);
- let whens = init
- .iter()
- .map(|x| x.clone().is_not_null())
- .collect::<Vec<_>>();
- let cases = init.to_vec();
+ // `coalesce(a, b)` is rewritten to the (lazy, fast)
+ // `CASE WHEN a IS NOT NULL THEN a ELSE b END`, which names every argument
+ // but the last one twice. A volatile argument named twice is evaluated
+ // twice, so the rewrite stops at the first volatile non-final argument
+ // and the rest is left to the `invoke_with_args` kernel, which evaluates
+ // each argument once. See https://github.com/apache/datafusion/issues/25477
+ let (init, last) = args.split_at(args.len() - 1);
+ let (prefix, rest) = match init.iter().position(|arg| arg.is_volatile()) {
+ None => (init, last[0].clone()),
+ Some(0) => return Ok(ExprSimplifyResult::Original(args)),
+ Some(i) => (
+ &init[..i],
+ Expr::ScalarFunction(ScalarFunction::new_udf(
+ super::coalesce(),
+ args[i..].to_vec(),
+ )),
+ ),
+ };
+
+ let whens = prefix.iter().map(|x| x.clone().is_not_null()).collect();
Ok(ExprSimplifyResult::Simplified(
- CaseBuilder::new(None, whens, cases, Some(Box::new(last_elem[0].clone())))
- .end()?,
+ CaseBuilder::new(None, whens, prefix.to_vec(), Some(Box::new(rest))).end()?,
))
}
/// coalesce evaluates to the first value which is not NULL
- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- internal_err!("coalesce should have been simplified to case")
+ ///
+ /// Only reached when [`Self::simplify`] keeps the call because an argument
+ /// is volatile; evaluates each argument exactly once.
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let args = args.args;
+ // do not accept 0 arguments.
+ if args.is_empty() {
+ return exec_err!(
+ "coalesce was called with {} arguments. It requires at least 1.",
+ args.len()
+ );
+ }
+
+ let return_type = args[0].data_type();
+ let mut return_array = args.iter().filter_map(|x| match x {
+ ColumnarValue::Array(array) => Some(array.len()),
+ _ => None,
+ });
+
+ if let Some(size) = return_array.next() {
+ // start with nulls as default output
+ let mut current_value = new_null_array(&return_type, size);
+ let mut remainder = BooleanArray::from(vec![true; size]);
+
+ for arg in args {
+ match arg {
+ ColumnarValue::Array(ref array) => {
+ let to_apply = and(&remainder, &is_not_null(array.as_ref())?)?;
+ current_value = zip(&to_apply, array, &current_value)?;
+ remainder = and(&remainder, &is_null(array)?)?;
+ }
+ ColumnarValue::Scalar(value) => {
+ if value.is_null() {
+ continue;
+ } else {
+ let last_value = value.to_scalar()?;
+ current_value = zip(&remainder, &last_value, &current_value)?;
+ break;
+ }
+ }
+ }
+ if remainder.true_count() == 0 {
+ break;
+ }
+ }
+ Ok(ColumnarValue::Array(current_value))
+ } else {
+ let result = args
+ .iter()
+ .find_map(|x| match x {
+ ColumnarValue::Scalar(s) if !s.is_null() => Some(x.clone()),
+ _ => None,
+ })
+ .unwrap_or_else(|| args[0].clone());
+ Ok(result)
+ }
}
fn conditional_arguments<'a>(
diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.slt
index 40c158648b..8fd1c5518d 100644
--- a/datafusion/sqllogictest/test_files/coalesce.slt
+++ b/datafusion/sqllogictest/test_files/coalesce.slt
@@ -445,3 +445,90 @@ world
statement ok
drop table ree_t;
+
+##########
+# Volatile arguments must be evaluated exactly once
+# https://github.com/apache/datafusion/issues/25477
+##########
+
+# The CASE rewrite would name a volatile non-final argument twice (two draws),
+# so a volatile first argument keeps the `coalesce` call.
+query TT
+explain select coalesce(nullif(floor(random() * 2), 0), -1) as c;
+----
+logical_plan
+01)Projection: coalesce(nullif(floor(random() * Float64(2)), Float64(0)), Float64(-1)) AS c
+02)--EmptyRelation: rows=1
+physical_plan
+01)ProjectionExec: expr=[coalesce(nullif(floor(random() * 2), 0), -1) as c]
+02)--PlaceholderRowExec
+
+# With two draws, about 25% of rows were NULL (and Arrow rejected them because
+# the `-1` fallback makes the result non-nullable). With one draw, every row is
+# either 1 or -1.
+statement ok
+create table volatile_t as select * from generate_series(1, 100000) as t(i);
+
+query III
+select
+ count(coalesce(nullif(floor(random() * 2), 0), -1)),
+ count(nvl(nullif(floor(random() * 2), 0), -1)),
+ count(*) filter (where coalesce(nullif(floor(random() * 2), 0), -1) not in (1, -1))
+from volatile_t;
+----
+100000 100000 0
+
+statement ok
+drop table volatile_t;
+
+# `case when random() >= 0 then x end` is volatile but always equals `x`, which
+# makes the results of the volatile code path deterministic.
+statement ok
+create table volatile_args(a int, b int, c int, s varchar, x int, y int) as values
+ (1, 9, 8, null, 0, 1),
+ (null, 2, 8, 'b', 0, 1),
+ (null, null, 3, null, 0, 1),
+ (null, null, null, null, 0, 1);
+
+query IIIIT
+select
+ coalesce(case when random() >= 0 then a end, b, c),
+ coalesce(case when random() >= 0 then a end, -1),
+ coalesce(case when random() >= 0 then a end, null::int),
+ coalesce(a, case when random() >= 0 then b end, c),
+ coalesce(case when random() >= 0 then s end, 'none')
+from volatile_args;
+----
+1 1 1 1 none
+2 -1 NULL 2 b
+3 -1 NULL 3 none
+NULL -1 NULL NULL none
+
+query TI
+select
+ coalesce(case when random() >= 0 then null::varchar end, 'datafusion', 'unused'),
+ coalesce(case when random() >= 0 then null::int end, null::int);
+----
+datafusion NULL
+
+# Arguments before the first volatile one keep the CASE rewrite; only the rest
+# is left to `coalesce`.
+query TT
+explain select coalesce(a, case when random() >= 0 then b end, c) from volatile_args;
+----
+logical_plan
+01)Projection: CASE WHEN volatile_args.a IS NOT NULL THEN volatile_args.a ELSE coalesce(CASE WHEN random() >= Float64(0) THEN volatile_args.b END, volatile_args.c) END AS coalesce(volatile_args.a,CASE WHEN random() >= Int64(0) THEN volatile_args.b END,volatile_args.c)
+02)--TableScan: volatile_args projection=[a, b, c]
+physical_plan
+01)ProjectionExec: expr=[CASE WHEN a@0 IS NOT NULL THEN a@0 ELSE coalesce(CASE WHEN random() >= 0 THEN b@1 END, c@2) END as coalesce(volatile_args.a,CASE WHEN random() >= Int64(0) THEN volatile_args.b END,volatile_args.c)]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# A volatile last argument is named once, so it keeps the lazy CASE rewrite:
+# `y / x` must not divide by zero when `a` is not null.
+query R
+select coalesce(a, y / x + random()) from volatile_args where a is not null;
+----
+1
+
+statement ok
+drop table volatile_args;