Apache DataFusion SQL Query Engine
chore: Cleanup and refactor `build_join` in `ScalarSubqueryToJoin` (#22316)
Neil Conway05ea11e7b51c
diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rsindex 76d22c7fb3..fee430047a 100644--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs@@ -34,7 +34,7 @@ use datafusion_common::{Column, Result, ScalarValue, assert_or_internal_err, plause datafusion_expr::expr_rewriter::create_col_from_scalar_expr;use datafusion_expr::logical_plan::{JoinType, Subquery};use datafusion_expr::utils::conjunction;-use datafusion_expr::{EmptyRelation, Expr, LogicalPlan, LogicalPlanBuilder, expr};+use datafusion_expr::{Expr, LogicalPlan, LogicalPlanBuilder, lit, not, when};/// Optimizer rule that rewrites correlated scalar subquery filters to joins and/// places an additional projection on top of the filter, to preserve the@@ -107,18 +107,17 @@ impl OptimizerRule for ScalarSubqueryToJoin {// iterate through all subqueries in predicate, turning each into a left joinlet mut cur_input = filter.input.as_ref().clone();for (subquery, alias) in subqueries {- if let Some((optimized_subquery, expr_check_map)) =+ if let Some((optimized_subquery, compensation_exprs)) =build_join(&subquery, &cur_input, &alias)?{- if !expr_check_map.is_empty() {+ if !compensation_exprs.is_empty() {rewrite_expr = rewrite_expr.transform_up(|expr| {- // replace column references with entry in map, if it exists- if let Some(map_expr) = expr+ if let Some(compensation_expr) = expr.try_as_col()- .and_then(|col| expr_check_map.get(col))+ .and_then(|col| compensation_exprs.get(col)){- Ok(Transformed::yes(map_expr.clone()))+ Ok(Transformed::yes(compensation_expr.clone()))} else {Ok(Transformed::no(expr))}@@ -172,22 +171,21 @@ impl OptimizerRule for ScalarSubqueryToJoin {// iterate through all subqueries in predicate, turning each into a left joinlet mut cur_input = projection.input.as_ref().clone();for (subquery, alias) in all_subqueries {- if let Some((optimized_subquery, expr_check_map)) =+ if let Some((optimized_subquery, compensation_exprs)) =build_join(&subquery, &cur_input, &alias)?{cur_input = optimized_subquery;- if !expr_check_map.is_empty()+ if !compensation_exprs.is_empty()&& let Some(&idx) = alias_to_index.get(&alias){let new_expr = rewrite_exprs[idx].clone().transform_up(|expr| {- // replace column references with entry in map, if it exists- if let Some(map_expr) = expr+ if let Some(compensation_expr) = expr.try_as_col()- .and_then(|col| expr_check_map.get(col))+ .and_then(|col| compensation_exprs.get(col)){- Ok(Transformed::yes(map_expr.clone()))+ Ok(Transformed::yes(compensation_expr.clone()))} else {Ok(Transformed::no(expr))}@@ -285,90 +283,95 @@ impl TreeNodeRewriter for ExtractScalarSubQuery<'_> {////// ```text/// select c.id from customers c-/// left join (select c_id, avg(total) as val from orders group by c_id) o on o.c_id = c.c_id-/// where c.balance > o.val-/// ```-///-/// Or a query like:-///-/// ```text-/// select id from customers where balance >-/// (select avg(total) from orders)-/// ```-///-/// and optimizes it into:-///-/// ```text-/// select c.id from customers c-/// left join (select avg(total) as val from orders) a-/// where c.balance > a.val+/// left join (select c_id, avg(total) from orders group by c_id) o+/// on o.c_id = c.id+/// where c.balance > o."avg(total)"/// ```////// # Arguments///-/// * `query_info` - The subquery portion of the `where` (select avg(total) from orders)-/// * `filter_input` - The non-subquery portion (from customers)-/// * `outer_others` - Any additional parts to the `where` expression (and c.x = y)-/// * `subquery_alias` - Subquery aliases+/// * `subquery` - The correlated scalar subquery to decorrelate.+/// * `outer_input` - The outer plan that the decorrelated subquery is+/// left-joined onto — the input of the `Filter` or `Projection` node+/// that contained the subquery.+/// * `subquery_alias` - The unique alias assigned to the decorrelated+/// subquery; used both to qualify the join condition and to produce+/// column references for the caller to substitute.+///+/// Returns `Ok(None)` if the subquery cannot be decorrelated. On success,+/// returns the rewritten outer plan and a map from each count-bug-affected+/// column to its `CASE WHEN __always_true IS NULL THEN ... END` compensation+/// expression, which the caller must substitute into any expression that+/// references those columns.fn build_join(subquery: &Subquery,- filter_input: &LogicalPlan,+ outer_input: &LogicalPlan,subquery_alias: &str,) -> Result<Option<(LogicalPlan, HashMap<Column, Expr>)>> {+ assert_or_internal_err!(+ !subquery.outer_ref_columns.is_empty(),+ "build_join should only be called for correlated subqueries"+ );let subquery_plan = subquery.subquery.as_ref();let mut pull_up = PullUpCorrelatedExpr::new().with_need_handle_count_bug(true);- let new_plan = subquery_plan.clone().rewrite(&mut pull_up).data()?;+ let decorrelated_subquery = subquery_plan.clone().rewrite(&mut pull_up).data()?;if !pull_up.can_pull_up {return Ok(None);}- let collected_count_expr_map =- pull_up.collected_count_expr_map.get(&new_plan).cloned();- let sub_query_alias = LogicalPlanBuilder::from(new_plan)+ let collected_count_expr_map = pull_up+ .collected_count_expr_map+ .get(&decorrelated_subquery)+ .cloned();+ let aliased_subquery = LogicalPlanBuilder::from(decorrelated_subquery).alias(subquery_alias.to_string())?.build()?;- let mut all_correlated_cols = BTreeSet::new();- pull_up+ let all_correlated_cols: BTreeSet<Column> = pull_up.correlated_subquery_cols_map.values()- .for_each(|cols| all_correlated_cols.extend(cols.clone()));+ .flatten()+ .cloned()+ .collect();- // alias the join filter+ // Correlated columns now live in the decorrelated subquery's output,+ // so re-qualify them with the subquery alias.let join_filter_opt =conjunction(pull_up.join_filters).map_or(Ok(None), |filter| {replace_qualified_name(filter, &all_correlated_cols, subquery_alias).map(Some)})?;- // join our sub query into the main plan- let new_plan = if join_filter_opt.is_none() {- match filter_input {- LogicalPlan::EmptyRelation(EmptyRelation {- produce_one_row: true,- schema: _,- }) => sub_query_alias,- _ => {- // if not correlated, group down to 1 row and left join on that (preserving row count)- LogicalPlanBuilder::from(filter_input.clone())- .join_on(- sub_query_alias,- JoinType::Left,- vec![Expr::Literal(ScalarValue::Boolean(Some(true)), None)],- )?- .build()?- }- }- } else {- // left join if correlated, grouping by the join keys so we don't change row count- LogicalPlanBuilder::from(filter_input.clone())- .join_on(sub_query_alias, JoinType::Left, join_filter_opt)?- .build()?- };- let mut computation_project_expr = HashMap::new();+ // When pull-up did not extract any usable join keys (a correlated subquery+ // whose predicate references only outer columns), fall back to `ON true`:+ // the decorrelated subquery still yields at most one row per outer row+ // because its aggregate is grouped by the (empty) set of correlated inner+ // columns.+ let join_filter = join_filter_opt.or_else(|| Some(lit(true)));++ let new_plan = LogicalPlanBuilder::from(outer_input.clone())+ .join_on(aliased_subquery, JoinType::Left, join_filter)?+ .build()?;++ // Add count-bug compensation for each of the subquery's projected+ // expressions that yield non-NULL values on empty input. We wrap each+ // such expression in a CASE that substitutes the empty-input value+ // when the LEFT JOIN produced synthetic right-side NULLs (no inner+ // row matched), and uses the actual right-side value (which may+ // itself be NULL) otherwise.+ let mut compensation_exprs = HashMap::new();if let Some(expr_map) = collected_count_expr_map {+ let mut expr_rewrite = TypeCoercionRewriter {+ schema: new_plan.schema(),+ };+ let having_arm = pull_up+ .pull_up_having_expr+ .as_ref()+ .map(|f| (not(f.clone()), lit(ScalarValue::Null)));for (name, result) in expr_map {if evaluates_to_null(result.clone(), result.column_refs())? {- // If expr always returns null when column is null, skip processing+ // Aggregates whose empty-input value is NULL (max/min/sum/…)+ // need no compensation: the LEFT JOIN already produces NULL+ // for unmatched outer rows.continue;}@@ -376,42 +379,21 @@ fn build_join(Column::new(Some(subquery_alias), UN_MATCHED_ROW_INDICATOR);// Qualify with the subquery alias to avoid ambiguity when the// outer table has a column with the same name as the aggregate.- let value_col = Column::new(Some(subquery_alias), name.clone());-- let computer_expr = if let Some(filter) = &pull_up.pull_up_having_expr {- Expr::Case(expr::Case {- expr: None,- when_then_expr: vec![- (- Box::new(Expr::IsNull(Box::new(Expr::Column(indicator_col)))),- Box::new(result),- ),- (- Box::new(Expr::Not(Box::new(filter.clone()))),- Box::new(Expr::Literal(ScalarValue::Null, None)),- ),- ],- else_expr: Some(Box::new(Expr::Column(value_col.clone()))),- })- } else {- Expr::Case(expr::Case {- expr: None,- when_then_expr: vec![(- Box::new(Expr::IsNull(Box::new(Expr::Column(indicator_col)))),- Box::new(result),- )],- else_expr: Some(Box::new(Expr::Column(value_col.clone()))),- })- };- let mut expr_rewrite = TypeCoercionRewriter {- schema: new_plan.schema(),- };- computation_project_expr- .insert(value_col, computer_expr.rewrite(&mut expr_rewrite).data()?);+ let value_col = Column::new(Some(subquery_alias), name);++ let mut builder = when(Expr::Column(indicator_col).is_null(), result);+ if let Some((when_expr, then_expr)) = &having_arm {+ builder = builder.when(when_expr.clone(), then_expr.clone());+ }+ let compensation_expr = builder.otherwise(Expr::Column(value_col.clone()))?;+ compensation_exprs.insert(+ value_col,+ compensation_expr.rewrite(&mut expr_rewrite).data()?,+ );}}- Ok(Some((new_plan, computation_project_expr)))+ Ok(Some((new_plan, compensation_exprs)))}#[cfg(test)]@@ -425,7 +407,7 @@ mod tests {use datafusion_expr::test::function_stub::sum;use crate::assert_optimized_plan_eq_display_indent_snapshot;- use datafusion_expr::{Between, col, lit, out_ref_col, scalar_subquery};+ use datafusion_expr::{Between, col, expr, out_ref_col, scalar_subquery};use datafusion_functions_aggregate::min_max::{max, min};macro_rules! assert_optimized_plan_equal {