Skip to content

Apache DataFusion SQL Query Engine

perf: avoid quadratic planning for SELECTs with many aggregates (#25739)

Stefan Wangc222b69bd003
diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rs
index 48770541ca..5e5a63c0e9 100644
--- a/datafusion/core/benches/sql_planner.rs
+++ b/datafusion/core/benches/sql_planner.rs
@@ -829,6 +829,14 @@ fn criterion_benchmark(c: &mut Criterion) {
});
}
+ {
+ let aggs: Vec<String> = (0..1000).map(|i| format!("SUM(d{i})")).collect();
+ let query = format!("SELECT {} FROM t1000", aggs.join(", "));
+ c.bench_function("logical_wide_aggregate_1000_exprs", |b| {
+ b.iter(|| logical_plan(&ctx, &rt, &query))
+ });
+ }
+
// Many CASE WHEN expressions (complex expressions)
{
let cases: Vec<String> = (0..50)
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index 964c6de396..64fc9b81ee 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -38,7 +38,7 @@ use crate::logical_plan::{
};
use crate::select_expr::SelectExpr;
use crate::utils::{
- can_hash, check_all_columns_from_schema, columnize_expr, compare_sort_expr,
+ Columnizer, can_hash, check_all_columns_from_schema, compare_sort_expr,
expand_qualified_wildcard, expand_wildcard, expr_to_columns,
find_valid_equijoin_key_pair, group_window_expr_by_sort_keys,
split_conjunction_owned,
@@ -2082,6 +2082,7 @@ fn project_with_validation(
let mut projected_expr = vec![];
let mut has_wildcard = false;
let mut normalizer = ColumnNormalizer::new(&plan);
+ let columnizer = Columnizer::new(&plan);
for (e, validate) in expr {
let e = e.into();
match e {
@@ -2100,7 +2101,7 @@ fn project_with_validation(
for e in expanded {
if validate {
projected_expr
- .push(columnize_expr(normalizer.normalize(e)?, &plan)?)
+ .push(columnizer.columnize(normalizer.normalize(e)?)?)
} else {
projected_expr.push(e)
}
@@ -2122,7 +2123,7 @@ fn project_with_validation(
for e in expanded {
if validate {
projected_expr
- .push(columnize_expr(normalizer.normalize(e)?, &plan)?)
+ .push(columnizer.columnize(normalizer.normalize(e)?)?)
} else {
projected_expr.push(e)
}
@@ -2130,7 +2131,7 @@ fn project_with_validation(
}
SelectExpr::Expression(e) => {
if validate {
- projected_expr.push(columnize_expr(normalizer.normalize(e)?, &plan)?)
+ projected_expr.push(columnizer.columnize(normalizer.normalize(e)?)?)
} else {
projected_expr.push(e)
}
@@ -2157,6 +2158,8 @@ fn project_with_validation(
validate_unique_names("Projections", projected_expr.iter())?;
+ // `columnizer` borrows `plan`, which moves into the projection below.
+ drop(columnizer);
Projection::try_new(projected_expr, Arc::new(plan)).map(LogicalPlan::Projection)
}
diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs
index 07af8b1f56..238025b00a 100644
--- a/datafusion/expr/src/utils.rs
+++ b/datafusion/expr/src/utils.rs
@@ -963,20 +963,38 @@ pub fn exprlist_to_fields<'a>(
/// .project(vec![col("c1"), col("SUM(c2)")?
/// ```
pub fn columnize_expr(e: Expr, input: &LogicalPlan) -> Result<Expr> {
- let output_exprs = match input.columnized_output_exprs() {
- Ok(exprs) if !exprs.is_empty() => exprs,
- _ => return Ok(e),
- };
- let exprs_map: HashMap<&Expr, Column> = output_exprs.into_iter().collect();
- e.transform_down(|node: Expr| match exprs_map.get(&node) {
- Some(column) => Ok(Transformed::new(
- Expr::Column(column.clone()),
- true,
- TreeNodeRecursion::Jump,
- )),
- None => Ok(Transformed::no(node)),
- })
- .data()
+ Columnizer::new(input).columnize(e)
+}
+
+/// Applies [`columnize_expr`] to many expressions over the same input plan,
+/// building the lookup of the input's output expressions only once.
+pub(crate) struct Columnizer<'a> {
+ output_exprs: HashMap<&'a Expr, Column>,
+}
+
+impl<'a> Columnizer<'a> {
+ pub(crate) fn new(input: &'a LogicalPlan) -> Self {
+ let output_exprs = input
+ .columnized_output_exprs()
+ .map(|exprs| exprs.into_iter().collect())
+ .unwrap_or_default();
+ Self { output_exprs }
+ }
+
+ pub(crate) fn columnize(&self, e: Expr) -> Result<Expr> {
+ if self.output_exprs.is_empty() {
+ return Ok(e);
+ }
+ e.transform_down(|node: Expr| match self.output_exprs.get(&node) {
+ Some(column) => Ok(Transformed::new(
+ Expr::Column(column.clone()),
+ true,
+ TreeNodeRecursion::Jump,
+ )),
+ None => Ok(Transformed::no(node)),
+ })
+ .data()
+ }
}
/// Collect all deeply nested `Expr::Column`'s. They are returned in order of