Apache DataFusion SQL Query Engine
perf: avoid quadratic planning for SELECTs with many aggregates (#25739)
Stefan Wangc222b69bd003
diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rsindex 48770541ca..5e5a63c0e9 100644--- a/datafusion/core/benches/sql_planner.rs+++ b/datafusion/core/benches/sql_planner.rs@@ -829,6 +829,14 @@ fn criterion_benchmark(c: &mut Criterion) {});}+ {+ let aggs: Vec<String> = (0..1000).map(|i| format!("SUM(d{i})")).collect();+ let query = format!("SELECT {} FROM t1000", aggs.join(", "));+ c.bench_function("logical_wide_aggregate_1000_exprs", |b| {+ b.iter(|| logical_plan(&ctx, &rt, &query))+ });+ }+// Many CASE WHEN expressions (complex expressions){let cases: Vec<String> = (0..50)diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex 964c6de396..64fc9b81ee 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -38,7 +38,7 @@ use crate::logical_plan::{};use crate::select_expr::SelectExpr;use crate::utils::{- can_hash, check_all_columns_from_schema, columnize_expr, compare_sort_expr,+ Columnizer, can_hash, check_all_columns_from_schema, compare_sort_expr,expand_qualified_wildcard, expand_wildcard, expr_to_columns,find_valid_equijoin_key_pair, group_window_expr_by_sort_keys,split_conjunction_owned,@@ -2082,6 +2082,7 @@ fn project_with_validation(let mut projected_expr = vec![];let mut has_wildcard = false;let mut normalizer = ColumnNormalizer::new(&plan);+ let columnizer = Columnizer::new(&plan);for (e, validate) in expr {let e = e.into();match e {@@ -2100,7 +2101,7 @@ fn project_with_validation(for e in expanded {if validate {projected_expr- .push(columnize_expr(normalizer.normalize(e)?, &plan)?)+ .push(columnizer.columnize(normalizer.normalize(e)?)?)} else {projected_expr.push(e)}@@ -2122,7 +2123,7 @@ fn project_with_validation(for e in expanded {if validate {projected_expr- .push(columnize_expr(normalizer.normalize(e)?, &plan)?)+ .push(columnizer.columnize(normalizer.normalize(e)?)?)} else {projected_expr.push(e)}@@ -2130,7 +2131,7 @@ fn project_with_validation(}SelectExpr::Expression(e) => {if validate {- projected_expr.push(columnize_expr(normalizer.normalize(e)?, &plan)?)+ projected_expr.push(columnizer.columnize(normalizer.normalize(e)?)?)} else {projected_expr.push(e)}@@ -2157,6 +2158,8 @@ fn project_with_validation(validate_unique_names("Projections", projected_expr.iter())?;+ // `columnizer` borrows `plan`, which moves into the projection below.+ drop(columnizer);Projection::try_new(projected_expr, Arc::new(plan)).map(LogicalPlan::Projection)}diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rsindex 07af8b1f56..238025b00a 100644--- a/datafusion/expr/src/utils.rs+++ b/datafusion/expr/src/utils.rs@@ -963,20 +963,38 @@ pub fn exprlist_to_fields<'a>(/// .project(vec![col("c1"), col("SUM(c2)")?/// ```pub fn columnize_expr(e: Expr, input: &LogicalPlan) -> Result<Expr> {- let output_exprs = match input.columnized_output_exprs() {- Ok(exprs) if !exprs.is_empty() => exprs,- _ => return Ok(e),- };- let exprs_map: HashMap<&Expr, Column> = output_exprs.into_iter().collect();- e.transform_down(|node: Expr| match exprs_map.get(&node) {- Some(column) => Ok(Transformed::new(- Expr::Column(column.clone()),- true,- TreeNodeRecursion::Jump,- )),- None => Ok(Transformed::no(node)),- })- .data()+ Columnizer::new(input).columnize(e)+}++/// Applies [`columnize_expr`] to many expressions over the same input plan,+/// building the lookup of the input's output expressions only once.+pub(crate) struct Columnizer<'a> {+ output_exprs: HashMap<&'a Expr, Column>,+}++impl<'a> Columnizer<'a> {+ pub(crate) fn new(input: &'a LogicalPlan) -> Self {+ let output_exprs = input+ .columnized_output_exprs()+ .map(|exprs| exprs.into_iter().collect())+ .unwrap_or_default();+ Self { output_exprs }+ }++ pub(crate) fn columnize(&self, e: Expr) -> Result<Expr> {+ if self.output_exprs.is_empty() {+ return Ok(e);+ }+ e.transform_down(|node: Expr| match self.output_exprs.get(&node) {+ Some(column) => Ok(Transformed::new(+ Expr::Column(column.clone()),+ true,+ TreeNodeRecursion::Jump,+ )),+ None => Ok(Transformed::no(node)),+ })+ .data()+ }}/// Collect all deeply nested `Expr::Column`'s. They are returned in order of