Apache DataFusion SQL Query Engine
fix: do not duplicate volatile expressions when extracting leaf expressions (#24720)
Jaideep Pyne27e81e9a4531
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rsindex 107758cf6b..dfaf93bc1e 100644--- a/datafusion/optimizer/src/extract_leaf_expressions.rs+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs@@ -57,6 +57,58 @@ fn has_extractable_expr(exprs: &[Expr]) -> bool {})}+/// Returns the flat names of `plan`'s output columns whose defining expression+/// is volatile (e.g. `random()`).+///+/// Only a [`LogicalPlan::Projection`] can define such a column: anywhere else+/// the value has already been materialized by the projection that produced it,+/// so referencing the column again does not re-evaluate anything.+fn volatile_output_columns(plan: &LogicalPlan) -> BTreeSet<String> {+ let LogicalPlan::Projection(projection) = plan else {+ return BTreeSet::new();+ };+ projection+ .schema+ .iter()+ .zip(projection.expr.iter())+ .filter(|(_, expr)| expr.is_volatile())+ .map(|((qualifier, field), _)| Column::from((qualifier, field)).flat_name())+ .collect()+}++/// Returns `true` if building an extraction projection for `exprs` on top of+/// `input` would duplicate a volatile computation.+///+/// When `input` is already a projection, [`build_extraction_projection_impl`]+/// *merges* into it: every column reference in an extracted expression is+/// replaced by that column's defining expression (see+/// [`build_projection_replace_map`]). Inlining a volatile definition makes the+/// merged projection evaluate it a second, independent time, so the extracted+/// value no longer matches the column it was derived from:+///+/// ```text+/// Projection: s, get_field(s, 'a') AS field+/// Projection: named_struct('a', random()) AS s+/// ```+///+/// would merge into a single projection computing `random()` twice, and+/// `field` would then differ from `s['a']` on every row. Callers skip the+/// extraction instead.+fn would_duplicate_volatile<'a>(+ exprs: impl IntoIterator<Item = &'a Expr>,+ input: &LogicalPlan,+) -> bool {+ let volatile = volatile_output_columns(input);+ if volatile.is_empty() {+ return false;+ }+ exprs.into_iter().any(|expr| {+ expr.column_refs()+ .iter()+ .any(|col| volatile.contains(&col.flat_name()))+ })+}+/// Extracts `MoveTowardsLeafNodes` sub-expressions from non-projection nodes/// into **extraction projections** (pass 1 of 2).///@@ -195,7 +247,18 @@ fn extract_from_plan(}// Fast pre-check: skip all allocations if no extractable expressions exist- if !has_extractable_expr(&plan.expressions()) {+ let node_exprs = plan.expressions();+ if !has_extractable_expr(&node_exprs) {+ return Ok(Transformed::no(plan));+ }++ // The extraction projection is merged into an input that is already a+ // projection, which inlines the referenced columns' definitions. Skip the+ // extraction when that would duplicate a volatile computation.+ if inputs+ .iter()+ .any(|input| would_duplicate_volatile(node_exprs.iter(), input))+ {return Ok(Transformed::no(plan));}@@ -917,6 +980,16 @@ fn split_and_push_projection(return Ok(None);}+ // Pushing into an input that is already a projection merges into it and+ // inlines the referenced columns' definitions. Leave the projection alone+ // when that would duplicate a volatile computation.+ if would_duplicate_volatile(+ extraction_pairs.iter().map(|(expr, _)| expr),+ input.as_ref(),+ ) {+ return Ok(None);+ }+// ── Phase 2: Push down ──────────────────────────────────────────────let proj_input = Arc::clone(&proj.input);let pushed = push_extraction_pairs(@@ -1215,6 +1288,15 @@ fn try_push_into_inputs(if per_input[idx].pairs.is_empty() {new_inputs.push(input.clone());} else {+ // Merging into an input projection inlines the referenced columns'+ // definitions; bail out when that would duplicate a volatile+ // computation.+ if would_duplicate_volatile(+ per_input[idx].pairs.iter().map(|(expr, _)| expr),+ input,+ ) {+ return Ok(None);+ }let input_arc = Arc::new(input.clone());let target_schema = Arc::clone(input.schema());let proj = build_extraction_projection_impl(@@ -1270,7 +1352,7 @@ mod tests {use crate::{Optimizer, OptimizerContext};use datafusion_expr::expr::ScalarFunction;use datafusion_expr::{- ScalarUDF, col, lit, logical_plan::builder::LogicalPlanBuilder,+ ScalarUDF, Volatility, col, lit, logical_plan::builder::LogicalPlanBuilder,};fn leaf_udf(expr: Expr, name: &str) -> Expr {@@ -1283,6 +1365,19 @@ mod tests {))}+ /// A stand-in for `random()`: a volatile expression that must be evaluated+ /// exactly once per row.+ fn volatile_udf(expr: Expr) -> Expr {+ Expr::ScalarFunction(ScalarFunction::new_udf(+ Arc::new(ScalarUDF::new_from_impl(+ PlacementTestUDF::new()+ .with_placement(ExpressionPlacement::KeepInPlace)+ .with_volatility(Volatility::Volatile),+ )),+ vec![expr],+ ))+ }+// =========================================================================// Combined optimization stage formatter// =========================================================================@@ -1960,6 +2055,63 @@ mod tests {"#)}+ /// Merging an extraction into an input projection inlines the definition of+ /// every column the extraction references. When that definition is volatile+ /// the inlined copy is an independent evaluation, so the extraction must be+ /// skipped and the plan left alone.+ #[test]+ fn test_no_merge_into_volatile_projection() -> Result<()> {+ let table_scan = test_table_scan()?;+ let plan = LogicalPlanBuilder::from(table_scan)+ .project(vec![volatile_udf(col("a")).alias("v")])?+ .project(vec![col("v"), leaf_udf(col("v"), "x")])?+ .build()?;++ assert_stages!(plan, @r#"+ ## Original Plan+ Projection: v, leaf_udf(v, Utf8("x"))+ Projection: keep_in_place_udf(test.a) AS v+ TableScan: test projection=[a]++ ## After Extraction+ (same as original)++ ## After Pushdown+ (same as after extraction)++ ## Optimized+ (same as after pushdown)+ "#)+ }++ /// Same guard for pass 1: extracting out of a `Filter` whose input+ /// projection defines the referenced column volatilely would make the+ /// predicate test a second, independent evaluation.+ #[test]+ fn test_no_extraction_from_filter_over_volatile_projection() -> Result<()> {+ let table_scan = test_table_scan()?;+ let plan = LogicalPlanBuilder::from(table_scan)+ .project(vec![volatile_udf(col("a")).alias("v")])?+ .filter(leaf_udf(col("v"), "x").eq(lit(1u32)))?+ .build()?;++ assert_stages!(plan, @r#"+ ## Original Plan+ Filter: leaf_udf(v, Utf8("x")) = UInt32(1)+ Projection: keep_in_place_udf(test.a) AS v+ TableScan: test projection=[a]++ ## After Extraction+ (same as original)++ ## After Pushdown+ (same as after extraction)++ ## Optimized+ (same as after pushdown)+ "#)+ }+/// Projections with aliased columns (nothing to extract) return unchanged.#[test]fn test_projection_early_return_no_extraction() -> Result<()> {diff --git a/datafusion/optimizer/src/test/udfs.rs b/datafusion/optimizer/src/test/udfs.rsindex ba71b6a04a..72b1a63f67 100644--- a/datafusion/optimizer/src/test/udfs.rs+++ b/datafusion/optimizer/src/test/udfs.rs@@ -19,7 +19,7 @@ use arrow::datatypes::DataType;use datafusion_common::Result;use datafusion_expr::{ColumnarValue, Expr, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,- ScalarUDFImpl, Signature, TypeSignature,+ ScalarUDFImpl, Signature, TypeSignature, Volatility,};/// A configurable test UDF for optimizer tests.@@ -44,7 +44,7 @@ impl PlacementTestUDF {// The actual types don't matter since this UDF is not intended for execution.signature: Signature::new(TypeSignature::OneOf(vec![TypeSignature::Any(1), TypeSignature::Any(2)]),- datafusion_expr::Volatility::Immutable,+ Volatility::Immutable,),placement: ExpressionPlacement::MoveTowardsLeafNodes,id: 0,@@ -64,6 +64,13 @@ impl PlacementTestUDF {self.id = id;self}++ /// Set the volatility of the UDF, so that rules which must not duplicate a+ /// volatile computation (e.g. `random()`) can be exercised.+ pub fn with_volatility(mut self, volatility: Volatility) -> Self {+ self.signature.volatility = volatility;+ self+ }}impl ScalarUDFImpl for PlacementTestUDF {diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex 3c7b6f4cb1..1f9176f713 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -2144,6 +2144,83 @@ truetruetrue+#####################+# Section: leaf expression extraction does not duplicate volatile expressions+#+# Regression test for #24678. `extract_leaf_expressions` / `push_down_leaf_projections`+# build an extraction projection by *merging* into the input projection, which+# inlines the definition of every column the extracted expression references.+# When that definition is volatile the inlined copy is an independent+# evaluation, so `random()` is drawn twice and the extracted value no longer+# matches the struct it was derived from.+#+# Before the fix the plan below read+# Projection: named_struct(Utf8("a"), random()) AS s, random() AS field+# and `field` differed from `s['a']` on every row. This is the same invariant+# `FileScanConfig::try_swapping_with_projection` already enforces for the+# physical pushdown path (#23220).+#####################++statement ok+set datafusion.explain.logical_plan_only = true;++# `random()` must appear once; `field` reads the struct materialized below it.+query TT+EXPLAIN SELECT s, s['a'] AS field+FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 3));+----+logical_plan+01)Projection: s, get_field(s, Utf8("a")) AS field+02)--Projection: named_struct(Utf8("a"), random()) AS s+03)----TableScan: generate_series() projection=[]++# Same for an extraction out of a Filter: the predicate must test the struct+# that is returned, not a second draw.+query TT+EXPLAIN SELECT s+FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 3))+WHERE s['a'] > 0.5;+----+logical_plan+01)Filter: get_field(s, Utf8("a")) > Float64(0.5)+02)--Projection: named_struct(Utf8("a"), random()) AS s+03)----TableScan: generate_series() projection=[]++statement ok+set datafusion.explain.logical_plan_only = false;++# Every row's `field` is the `a` field of that row's own struct.+query B+SELECT bool_and(field = s['a'])+FROM (+ SELECT s, s['a'] AS field+ FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000))+);+----+true++# Same invariant when the extraction is routed into one side of a join.+query B+SELECT bool_and(field = s['a'])+FROM (+ SELECT l.s, l.s['a'] AS field+ FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000)) AS l+ INNER JOIN (SELECT 1 AS k) AS r ON true+);+----+true++# Every surviving row satisfies the predicate it was filtered on.+query B+SELECT bool_and(s['a'] > 0.5)+FROM (+ SELECT s+ FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000))+ WHERE s['a'] > 0.5+);+----+true+###################### Section: expensive expressions are not re-inlined by projection pushdown#