Skip to content

Apache DataFusion SQL Query Engine

fix: do not duplicate volatile expressions when extracting leaf expressions (#24720)

Jaideep Pyne27e81e9a4531
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rs
index 107758cf6b..dfaf93bc1e 100644
--- a/datafusion/optimizer/src/extract_leaf_expressions.rs
+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs
@@ -57,6 +57,58 @@ fn has_extractable_expr(exprs: &[Expr]) -> bool {
})
}
+/// Returns the flat names of `plan`'s output columns whose defining expression
+/// is volatile (e.g. `random()`).
+///
+/// Only a [`LogicalPlan::Projection`] can define such a column: anywhere else
+/// the value has already been materialized by the projection that produced it,
+/// so referencing the column again does not re-evaluate anything.
+fn volatile_output_columns(plan: &LogicalPlan) -> BTreeSet<String> {
+ let LogicalPlan::Projection(projection) = plan else {
+ return BTreeSet::new();
+ };
+ projection
+ .schema
+ .iter()
+ .zip(projection.expr.iter())
+ .filter(|(_, expr)| expr.is_volatile())
+ .map(|((qualifier, field), _)| Column::from((qualifier, field)).flat_name())
+ .collect()
+}
+
+/// Returns `true` if building an extraction projection for `exprs` on top of
+/// `input` would duplicate a volatile computation.
+///
+/// When `input` is already a projection, [`build_extraction_projection_impl`]
+/// *merges* into it: every column reference in an extracted expression is
+/// replaced by that column's defining expression (see
+/// [`build_projection_replace_map`]). Inlining a volatile definition makes the
+/// merged projection evaluate it a second, independent time, so the extracted
+/// value no longer matches the column it was derived from:
+///
+/// ```text
+/// Projection: s, get_field(s, 'a') AS field
+/// Projection: named_struct('a', random()) AS s
+/// ```
+///
+/// would merge into a single projection computing `random()` twice, and
+/// `field` would then differ from `s['a']` on every row. Callers skip the
+/// extraction instead.
+fn would_duplicate_volatile<'a>(
+ exprs: impl IntoIterator<Item = &'a Expr>,
+ input: &LogicalPlan,
+) -> bool {
+ let volatile = volatile_output_columns(input);
+ if volatile.is_empty() {
+ return false;
+ }
+ exprs.into_iter().any(|expr| {
+ expr.column_refs()
+ .iter()
+ .any(|col| volatile.contains(&col.flat_name()))
+ })
+}
+
/// Extracts `MoveTowardsLeafNodes` sub-expressions from non-projection nodes
/// into **extraction projections** (pass 1 of 2).
///
@@ -195,7 +247,18 @@ fn extract_from_plan(
}
// Fast pre-check: skip all allocations if no extractable expressions exist
- if !has_extractable_expr(&plan.expressions()) {
+ let node_exprs = plan.expressions();
+ if !has_extractable_expr(&node_exprs) {
+ return Ok(Transformed::no(plan));
+ }
+
+ // The extraction projection is merged into an input that is already a
+ // projection, which inlines the referenced columns' definitions. Skip the
+ // extraction when that would duplicate a volatile computation.
+ if inputs
+ .iter()
+ .any(|input| would_duplicate_volatile(node_exprs.iter(), input))
+ {
return Ok(Transformed::no(plan));
}
@@ -917,6 +980,16 @@ fn split_and_push_projection(
return Ok(None);
}
+ // Pushing into an input that is already a projection merges into it and
+ // inlines the referenced columns' definitions. Leave the projection alone
+ // when that would duplicate a volatile computation.
+ if would_duplicate_volatile(
+ extraction_pairs.iter().map(|(expr, _)| expr),
+ input.as_ref(),
+ ) {
+ return Ok(None);
+ }
+
// ── Phase 2: Push down ──────────────────────────────────────────────
let proj_input = Arc::clone(&proj.input);
let pushed = push_extraction_pairs(
@@ -1215,6 +1288,15 @@ fn try_push_into_inputs(
if per_input[idx].pairs.is_empty() {
new_inputs.push(input.clone());
} else {
+ // Merging into an input projection inlines the referenced columns'
+ // definitions; bail out when that would duplicate a volatile
+ // computation.
+ if would_duplicate_volatile(
+ per_input[idx].pairs.iter().map(|(expr, _)| expr),
+ input,
+ ) {
+ return Ok(None);
+ }
let input_arc = Arc::new(input.clone());
let target_schema = Arc::clone(input.schema());
let proj = build_extraction_projection_impl(
@@ -1270,7 +1352,7 @@ mod tests {
use crate::{Optimizer, OptimizerContext};
use datafusion_expr::expr::ScalarFunction;
use datafusion_expr::{
- ScalarUDF, col, lit, logical_plan::builder::LogicalPlanBuilder,
+ ScalarUDF, Volatility, col, lit, logical_plan::builder::LogicalPlanBuilder,
};
fn leaf_udf(expr: Expr, name: &str) -> Expr {
@@ -1283,6 +1365,19 @@ mod tests {
))
}
+ /// A stand-in for `random()`: a volatile expression that must be evaluated
+ /// exactly once per row.
+ fn volatile_udf(expr: Expr) -> Expr {
+ Expr::ScalarFunction(ScalarFunction::new_udf(
+ Arc::new(ScalarUDF::new_from_impl(
+ PlacementTestUDF::new()
+ .with_placement(ExpressionPlacement::KeepInPlace)
+ .with_volatility(Volatility::Volatile),
+ )),
+ vec![expr],
+ ))
+ }
+
// =========================================================================
// Combined optimization stage formatter
// =========================================================================
@@ -1960,6 +2055,63 @@ mod tests {
"#)
}
+ /// Merging an extraction into an input projection inlines the definition of
+ /// every column the extraction references. When that definition is volatile
+ /// the inlined copy is an independent evaluation, so the extraction must be
+ /// skipped and the plan left alone.
+ #[test]
+ fn test_no_merge_into_volatile_projection() -> Result<()> {
+ let table_scan = test_table_scan()?;
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .project(vec![volatile_udf(col("a")).alias("v")])?
+ .project(vec![col("v"), leaf_udf(col("v"), "x")])?
+ .build()?;
+
+ assert_stages!(plan, @r#"
+ ## Original Plan
+ Projection: v, leaf_udf(v, Utf8("x"))
+ Projection: keep_in_place_udf(test.a) AS v
+ TableScan: test projection=[a]
+
+ ## After Extraction
+ (same as original)
+
+ ## After Pushdown
+ (same as after extraction)
+
+ ## Optimized
+ (same as after pushdown)
+ "#)
+ }
+
+ /// Same guard for pass 1: extracting out of a `Filter` whose input
+ /// projection defines the referenced column volatilely would make the
+ /// predicate test a second, independent evaluation.
+ #[test]
+ fn test_no_extraction_from_filter_over_volatile_projection() -> Result<()> {
+ let table_scan = test_table_scan()?;
+ let plan = LogicalPlanBuilder::from(table_scan)
+ .project(vec![volatile_udf(col("a")).alias("v")])?
+ .filter(leaf_udf(col("v"), "x").eq(lit(1u32)))?
+ .build()?;
+
+ assert_stages!(plan, @r#"
+ ## Original Plan
+ Filter: leaf_udf(v, Utf8("x")) = UInt32(1)
+ Projection: keep_in_place_udf(test.a) AS v
+ TableScan: test projection=[a]
+
+ ## After Extraction
+ (same as original)
+
+ ## After Pushdown
+ (same as after extraction)
+
+ ## Optimized
+ (same as after pushdown)
+ "#)
+ }
+
/// Projections with aliased columns (nothing to extract) return unchanged.
#[test]
fn test_projection_early_return_no_extraction() -> Result<()> {
diff --git a/datafusion/optimizer/src/test/udfs.rs b/datafusion/optimizer/src/test/udfs.rs
index ba71b6a04a..72b1a63f67 100644
--- a/datafusion/optimizer/src/test/udfs.rs
+++ b/datafusion/optimizer/src/test/udfs.rs
@@ -19,7 +19,7 @@ use arrow::datatypes::DataType;
use datafusion_common::Result;
use datafusion_expr::{
ColumnarValue, Expr, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,
- ScalarUDFImpl, Signature, TypeSignature,
+ ScalarUDFImpl, Signature, TypeSignature, Volatility,
};
/// A configurable test UDF for optimizer tests.
@@ -44,7 +44,7 @@ impl PlacementTestUDF {
// The actual types don't matter since this UDF is not intended for execution.
signature: Signature::new(
TypeSignature::OneOf(vec![TypeSignature::Any(1), TypeSignature::Any(2)]),
- datafusion_expr::Volatility::Immutable,
+ Volatility::Immutable,
),
placement: ExpressionPlacement::MoveTowardsLeafNodes,
id: 0,
@@ -64,6 +64,13 @@ impl PlacementTestUDF {
self.id = id;
self
}
+
+ /// Set the volatility of the UDF, so that rules which must not duplicate a
+ /// volatile computation (e.g. `random()`) can be exercised.
+ pub fn with_volatility(mut self, volatility: Volatility) -> Self {
+ self.signature.volatility = volatility;
+ self
+ }
}
impl ScalarUDFImpl for PlacementTestUDF {
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index 3c7b6f4cb1..1f9176f713 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -2144,6 +2144,83 @@ true
true
true
+#####################
+# Section: leaf expression extraction does not duplicate volatile expressions
+#
+# Regression test for #24678. `extract_leaf_expressions` / `push_down_leaf_projections`
+# build an extraction projection by *merging* into the input projection, which
+# inlines the definition of every column the extracted expression references.
+# When that definition is volatile the inlined copy is an independent
+# evaluation, so `random()` is drawn twice and the extracted value no longer
+# matches the struct it was derived from.
+#
+# Before the fix the plan below read
+# Projection: named_struct(Utf8("a"), random()) AS s, random() AS field
+# and `field` differed from `s['a']` on every row. This is the same invariant
+# `FileScanConfig::try_swapping_with_projection` already enforces for the
+# physical pushdown path (#23220).
+#####################
+
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+# `random()` must appear once; `field` reads the struct materialized below it.
+query TT
+EXPLAIN SELECT s, s['a'] AS field
+FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 3));
+----
+logical_plan
+01)Projection: s, get_field(s, Utf8("a")) AS field
+02)--Projection: named_struct(Utf8("a"), random()) AS s
+03)----TableScan: generate_series() projection=[]
+
+# Same for an extraction out of a Filter: the predicate must test the struct
+# that is returned, not a second draw.
+query TT
+EXPLAIN SELECT s
+FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 3))
+WHERE s['a'] > 0.5;
+----
+logical_plan
+01)Filter: get_field(s, Utf8("a")) > Float64(0.5)
+02)--Projection: named_struct(Utf8("a"), random()) AS s
+03)----TableScan: generate_series() projection=[]
+
+statement ok
+set datafusion.explain.logical_plan_only = false;
+
+# Every row's `field` is the `a` field of that row's own struct.
+query B
+SELECT bool_and(field = s['a'])
+FROM (
+ SELECT s, s['a'] AS field
+ FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000))
+);
+----
+true
+
+# Same invariant when the extraction is routed into one side of a join.
+query B
+SELECT bool_and(field = s['a'])
+FROM (
+ SELECT l.s, l.s['a'] AS field
+ FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000)) AS l
+ INNER JOIN (SELECT 1 AS k) AS r ON true
+);
+----
+true
+
+# Every surviving row satisfies the predicate it was filtered on.
+query B
+SELECT bool_and(s['a'] > 0.5)
+FROM (
+ SELECT s
+ FROM (SELECT named_struct('a', random()) AS s FROM generate_series(1, 1000))
+ WHERE s['a'] > 0.5
+);
+----
+true
+
#####################
# Section: expensive expressions are not re-inlined by projection pushdown
#