Apache DataFusion SQL Query Engine
fix: do not evaluate a volatile coalesce or nvl argument twice (#25561)
DeviousCardid75fd7b86042
diff --git a/datafusion/functions/src/core/coalesce.rs b/datafusion/functions/src/core/coalesce.rsindex 831f26fcb5..d554601f5a 100644--- a/datafusion/functions/src/core/coalesce.rs+++ b/datafusion/functions/src/core/coalesce.rs@@ -2,10 +2,14 @@// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements+use arrow::array::{BooleanArray, new_null_array};+use arrow::compute::kernels::zip::zip;+use arrow::compute::{and, is_not_null, is_null};use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::{Result, exec_err, internal_err, plan_err};use datafusion_expr::binary::try_type_union_resolution;use datafusion_expr::conditional_expressions::CaseBuilder;+use datafusion_expr::expr::ScalarFunction;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};use datafusion_expr::{ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,@@ -90,22 +94,88 @@ impl ScalarUDFImpl for CoalesceFunc {));}- let n = args.len();- let (init, last_elem) = args.split_at(n - 1);- let whens = init- .iter()- .map(|x| x.clone().is_not_null())- .collect::<Vec<_>>();- let cases = init.to_vec();+ // `coalesce(a, b)` is rewritten to the (lazy, fast)+ // `CASE WHEN a IS NOT NULL THEN a ELSE b END`, which names every argument+ // but the last one twice. A volatile argument named twice is evaluated+ // twice, so the rewrite stops at the first volatile non-final argument+ // and the rest is left to the `invoke_with_args` kernel, which evaluates+ // each argument once. See https://github.com/apache/datafusion/issues/25477+ let (init, last) = args.split_at(args.len() - 1);+ let (prefix, rest) = match init.iter().position(|arg| arg.is_volatile()) {+ None => (init, last[0].clone()),+ Some(0) => return Ok(ExprSimplifyResult::Original(args)),+ Some(i) => (+ &init[..i],+ Expr::ScalarFunction(ScalarFunction::new_udf(+ super::coalesce(),+ args[i..].to_vec(),+ )),+ ),+ };++ let whens = prefix.iter().map(|x| x.clone().is_not_null()).collect();Ok(ExprSimplifyResult::Simplified(- CaseBuilder::new(None, whens, cases, Some(Box::new(last_elem[0].clone())))- .end()?,+ CaseBuilder::new(None, whens, prefix.to_vec(), Some(Box::new(rest))).end()?,))}/// coalesce evaluates to the first value which is not NULL- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {- internal_err!("coalesce should have been simplified to case")+ ///+ /// Only reached when [`Self::simplify`] keeps the call because an argument+ /// is volatile; evaluates each argument exactly once.+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let args = args.args;+ // do not accept 0 arguments.+ if args.is_empty() {+ return exec_err!(+ "coalesce was called with {} arguments. It requires at least 1.",+ args.len()+ );+ }++ let return_type = args[0].data_type();+ let mut return_array = args.iter().filter_map(|x| match x {+ ColumnarValue::Array(array) => Some(array.len()),+ _ => None,+ });++ if let Some(size) = return_array.next() {+ // start with nulls as default output+ let mut current_value = new_null_array(&return_type, size);+ let mut remainder = BooleanArray::from(vec![true; size]);++ for arg in args {+ match arg {+ ColumnarValue::Array(ref array) => {+ let to_apply = and(&remainder, &is_not_null(array.as_ref())?)?;+ current_value = zip(&to_apply, array, ¤t_value)?;+ remainder = and(&remainder, &is_null(array)?)?;+ }+ ColumnarValue::Scalar(value) => {+ if value.is_null() {+ continue;+ } else {+ let last_value = value.to_scalar()?;+ current_value = zip(&remainder, &last_value, ¤t_value)?;+ break;+ }+ }+ }+ if remainder.true_count() == 0 {+ break;+ }+ }+ Ok(ColumnarValue::Array(current_value))+ } else {+ let result = args+ .iter()+ .find_map(|x| match x {+ ColumnarValue::Scalar(s) if !s.is_null() => Some(x.clone()),+ _ => None,+ })+ .unwrap_or_else(|| args[0].clone());+ Ok(result)+ }}fn conditional_arguments<'a>(diff --git a/datafusion/sqllogictest/test_files/coalesce.slt b/datafusion/sqllogictest/test_files/coalesce.sltindex 40c158648b..8fd1c5518d 100644--- a/datafusion/sqllogictest/test_files/coalesce.slt+++ b/datafusion/sqllogictest/test_files/coalesce.slt@@ -445,3 +445,90 @@ worldstatement okdrop table ree_t;++##########+# Volatile arguments must be evaluated exactly once+# https://github.com/apache/datafusion/issues/25477+##########++# The CASE rewrite would name a volatile non-final argument twice (two draws),+# so a volatile first argument keeps the `coalesce` call.+query TT+explain select coalesce(nullif(floor(random() * 2), 0), -1) as c;+----+logical_plan+01)Projection: coalesce(nullif(floor(random() * Float64(2)), Float64(0)), Float64(-1)) AS c+02)--EmptyRelation: rows=1+physical_plan+01)ProjectionExec: expr=[coalesce(nullif(floor(random() * 2), 0), -1) as c]+02)--PlaceholderRowExec++# With two draws, about 25% of rows were NULL (and Arrow rejected them because+# the `-1` fallback makes the result non-nullable). With one draw, every row is+# either 1 or -1.+statement ok+create table volatile_t as select * from generate_series(1, 100000) as t(i);++query III+select+ count(coalesce(nullif(floor(random() * 2), 0), -1)),+ count(nvl(nullif(floor(random() * 2), 0), -1)),+ count(*) filter (where coalesce(nullif(floor(random() * 2), 0), -1) not in (1, -1))+from volatile_t;+----+100000 100000 0++statement ok+drop table volatile_t;++# `case when random() >= 0 then x end` is volatile but always equals `x`, which+# makes the results of the volatile code path deterministic.+statement ok+create table volatile_args(a int, b int, c int, s varchar, x int, y int) as values+ (1, 9, 8, null, 0, 1),+ (null, 2, 8, 'b', 0, 1),+ (null, null, 3, null, 0, 1),+ (null, null, null, null, 0, 1);++query IIIIT+select+ coalesce(case when random() >= 0 then a end, b, c),+ coalesce(case when random() >= 0 then a end, -1),+ coalesce(case when random() >= 0 then a end, null::int),+ coalesce(a, case when random() >= 0 then b end, c),+ coalesce(case when random() >= 0 then s end, 'none')+from volatile_args;+----+1 1 1 1 none+2 -1 NULL 2 b+3 -1 NULL 3 none+NULL -1 NULL NULL none++query TI+select+ coalesce(case when random() >= 0 then null::varchar end, 'datafusion', 'unused'),+ coalesce(case when random() >= 0 then null::int end, null::int);+----+datafusion NULL++# Arguments before the first volatile one keep the CASE rewrite; only the rest+# is left to `coalesce`.+query TT+explain select coalesce(a, case when random() >= 0 then b end, c) from volatile_args;+----+logical_plan+01)Projection: CASE WHEN volatile_args.a IS NOT NULL THEN volatile_args.a ELSE coalesce(CASE WHEN random() >= Float64(0) THEN volatile_args.b END, volatile_args.c) END AS coalesce(volatile_args.a,CASE WHEN random() >= Int64(0) THEN volatile_args.b END,volatile_args.c)+02)--TableScan: volatile_args projection=[a, b, c]+physical_plan+01)ProjectionExec: expr=[CASE WHEN a@0 IS NOT NULL THEN a@0 ELSE coalesce(CASE WHEN random() >= 0 THEN b@1 END, c@2) END as coalesce(volatile_args.a,CASE WHEN random() >= Int64(0) THEN volatile_args.b END,volatile_args.c)]+02)--DataSourceExec: partitions=1, partition_sizes=[1]++# A volatile last argument is named once, so it keeps the lazy CASE rewrite:+# `y / x` must not divide by zero when `a` is not null.+query R+select coalesce(a, y / x + random()) from volatile_args where a is not null;+----+1++statement ok+drop table volatile_args;