Skip to content

Apache DataFusion SQL Query Engine

perf: Use agg `DistinctHandling` in join optimization (#25385)

Neil Conway42f3888ca2fc
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rs
index c013b70c04..b8bb32f0ed 100644
--- a/datafusion/core/tests/sql/joins.rs
+++ b/datafusion/core/tests/sql/joins.rs
@@ -27,6 +27,71 @@ use datafusion_sql::unparser::plan_to_sql;
use super::*;
+#[tokio::test]
+async fn volatile_join_filter_preserves_evaluations_below_min() -> Result<()> {
+ use arrow::array::record_batch;
+ use datafusion::logical_expr::{ColumnarValue, Volatility, create_udf};
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ // Request sort-merge joins; these small inputs remain in a single partition.
+ let mut config = SessionConfig::new().with_target_partitions(2);
+ config.options_mut().optimizer.prefer_hash_join = false;
+ let ctx = SessionContext::new_with_config(config);
+
+ // Return true, false, true, ... across successive input rows, including
+ // across batches. Both arguments keep the predicate at the join.
+ let evaluations = AtomicUsize::new(0);
+ ctx.register_udf(create_udf(
+ "alternating",
+ vec![DataType::Int32, DataType::Int32],
+ DataType::Boolean,
+ Volatility::Volatile,
+ Arc::new(move |args| {
+ let args = ColumnarValue::values_to_arrays(args)?;
+ let len = args[0].len();
+ let first = evaluations.fetch_add(len, Ordering::Relaxed);
+ let values = BooleanArray::from_iter(
+ (first..first + len).map(|i| Some(i.is_multiple_of(2))),
+ );
+ Ok(ColumnarValue::Array(Arc::new(values)))
+ }),
+ ));
+
+ ctx.register_batch(
+ "l",
+ record_batch!(("id", Int32, vec![1, 2]), ("x", Int32, vec![20, 10]))?,
+ )?;
+ ctx.register_batch(
+ "r",
+ record_batch!(("id", Int32, vec![1, 1, 2]), ("y", Int32, vec![0, 0, 0]))?,
+ )?;
+
+ // The inner join evaluates both pairs for id=1, so id=2 receives the
+ // third (true) result. A semi join would stop after the first match for
+ // id=1, give id=2 the second (false) result, and incorrectly return 20.
+ let df = ctx
+ .sql(
+ "SELECT MIN(l.x) AS minimum FROM l JOIN r \
+ ON l.id = r.id AND alternating(l.x, r.y)",
+ )
+ .await?;
+ let plan = df.create_physical_plan().await?;
+ let formatted = displayable(plan.as_ref()).indent(true).to_string();
+ assert_contains!(formatted, "SortMergeJoinExec: join_type=Inner");
+ let batches = collect(plan, ctx.task_ctx()).await?;
+ assert_batches_eq!(
+ [
+ "+---------+",
+ "| minimum |",
+ "+---------+",
+ "| 10 |",
+ "+---------+",
+ ],
+ &batches
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn join_change_in_planner() -> Result<()> {
let config = SessionConfig::new().with_target_partitions(8);
diff --git a/datafusion/optimizer/src/eliminate_aggregate_distinct.rs b/datafusion/optimizer/src/eliminate_aggregate_distinct.rs
index 66bfea45f0..a0c0fde4fd 100644
--- a/datafusion/optimizer/src/eliminate_aggregate_distinct.rs
+++ b/datafusion/optimizer/src/eliminate_aggregate_distinct.rs
@@ -165,18 +165,13 @@ mod tests {
use super::*;
use crate::OptimizerContext;
use crate::assert_optimized_plan_eq_snapshot;
+ use crate::test::udfs::DistinctHandlingTestUDAF;
use crate::test::*;
use crate::single_distinct_to_groupby::SingleDistinctToGroupBy;
- use arrow::datatypes::DataType;
- use datafusion_expr::function::AccumulatorArgs;
- use datafusion_expr::{
- Accumulator, AggregateUDF, AggregateUDFImpl, ExprFunctionExt, LogicalPlanBuilder,
- Signature, Volatility, col, lit,
- };
+ use datafusion_expr::{AggregateUDF, ExprFunctionExt, LogicalPlanBuilder, col, lit};
use datafusion_functions_aggregate::expr_fn::{bit_xor, max, min, sum};
- use std::hash::{Hash, Hasher};
use std::sync::Arc;
macro_rules! assert_optimized_plan_equal {
@@ -196,66 +191,12 @@ mod tests {
}};
}
- /// A user defined aggregate that reports the [`DistinctHandling`] it was
- /// built with.
+ /// A user defined aggregate that reports the given [`DistinctHandling`].
///
/// The tests above read the tag off built-in functions, which only covers
- /// the variants those functions happen to carry. This one exercises the
- /// public API a third-party function uses: an
- /// [`AggregateUDFImpl::distinct_handling`] override.
- #[derive(Debug, Clone, PartialEq, Eq)]
- struct TaggedUdaf {
- name: &'static str,
- handling: DistinctHandling,
- signature: Signature,
- }
-
- impl TaggedUdaf {
- fn new(name: &'static str, handling: DistinctHandling) -> Self {
- Self {
- name,
- handling,
- signature: Signature::any(1, Volatility::Immutable),
- }
- }
- }
-
- /// Hashed by name, which identifies the function here. `DistinctHandling`
- /// is not `Hash`, and `AggregateUDFImpl` requires one through `DynHash`.
- impl Hash for TaggedUdaf {
- fn hash<H: Hasher>(&self, state: &mut H) {
- self.name.hash(state);
- self.signature.hash(state);
- }
- }
-
- impl AggregateUDFImpl for TaggedUdaf {
- fn name(&self) -> &str {
- self.name
- }
-
- fn signature(&self) -> &Signature {
- &self.signature
- }
-
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::UInt32)
- }
-
- fn accumulator(
- &self,
- _acc_args: AccumulatorArgs,
- ) -> Result<Box<dyn Accumulator>> {
- unimplemented!("the rule only rewrites the logical plan")
- }
-
- fn distinct_handling(&self) -> DistinctHandling {
- self.handling
- }
- }
-
+ /// the variants those functions happen to carry.
fn tagged(name: &'static str, handling: DistinctHandling) -> AggregateUDF {
- AggregateUDF::from(TaggedUdaf::new(name, handling))
+ AggregateUDF::from(DistinctHandlingTestUDAF::new(name, handling))
}
/// `min(DISTINCT b)` loses the flag but keeps its column name.
diff --git a/datafusion/optimizer/src/eliminate_join.rs b/datafusion/optimizer/src/eliminate_join.rs
index 56aa888706..25151fd484 100644
--- a/datafusion/optimizer/src/eliminate_join.rs
+++ b/datafusion/optimizer/src/eliminate_join.rs
@@ -28,9 +28,10 @@
//!
//! 1. None of R's columns are referenced above the join.
//! 2. R does not observably multiply L's rows. This holds when either the
-//! join's ancestors are duplicate-insensitive (e.g., DISTINCT) or we can use
-//! functional dependencies to prove that each L row matches at most one R
-//! row (R is provably unique on the join keys).
+//! join's ancestors are duplicate-insensitive (e.g., DISTINCT) and its
+//! conditions are repeatable, or we can use functional dependencies to
+//! prove that each L row matches at most one R row (R is provably unique
+//! on the join keys).
//!
//! * A left outer join `L ⟕ R` can be removed entirely, i.e. replaced by `L`,
//! under the same two conditions. Unlike an inner join, a left join
@@ -38,11 +39,11 @@
//! columns are unused and R cannot multiply L's rows the join has no
//! observable effect at all. Such joins commonly appear in generated SQL
//! and in queries over views that join in lookup tables the query does not
-//! read. A join filter does not prevent this rewrite: for a left join it
-//! only decides whether a left row is matched or null-padded, and either
-//! way the row is emitted. Symmetrically, a right outer join `L ⟖ R` can be
-//! replaced by `R` when L's columns are unused and L cannot multiply R's
-//! rows.
+//! read. A repeatable join filter does not prevent this rewrite: for a left
+//! join it only decides whether a left row is matched or null-padded, and
+//! either way the row is emitted. Symmetrically, a right outer join `L ⟖ R`
+//! can be replaced by `R` when L's columns are unused and L cannot multiply
+//! R's rows.
//!
//! # Overview
//!
@@ -56,11 +57,13 @@
//! set across its two inputs.
//! * `duplicate_insensitive` — whether emitting each row once instead of many
//! times will not change the output. A duplicate-collapsing node (e.g.,
-//! DISTINCT, GROUP BY with no aggregate functions, or the existence side of a
-//! semi/anti/mark join) sets it `true` for its subtree, and it propagates
-//! downward until a node that makes the row count observable again (a `LIMIT`,
-//! a top-N sort, ...) clears it. It is therefore fixed by the nearest such
-//! node, not by the whole ancestor chain: a collapsing node shields its subtree,
+//! DISTINCT, an `Aggregate` plan node whose aggregate expressions all ignore
+//! duplicate input rows, or the existence side of a semi/anti/mark join) sets
+//! it `true` for its subtree, and it propagates downward until a node that
+//! makes the row count observable again (a `LIMIT`, a top-N sort, an
+//! expression that is not repeatable such as `random()` or a subquery, ...)
+//! clears it. It is therefore fixed by the nearest such node, not by the
+//! whole ancestor chain: a collapsing node shields its subtree,
//! so a duplicate-sensitive node further above does not matter.
//!
//! At each join, `rewritten_join_type` combines this context with the side's
@@ -69,7 +72,9 @@
//! node types just forward the context to their single child via
//! `rewrite_single_input`; nodes that alter column requirements or
//! duplicate-sensitivity (projection, aggregate, sort, ...) adjust it first.
-use crate::utils::for_each_referenced_index;
+use crate::utils::{
+ for_each_referenced_index, is_duplicate_insensitive_aggregate, is_repeatable,
+};
use crate::{OptimizerConfig, OptimizerRule};
use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::{
@@ -219,11 +224,18 @@ fn rewrite_node(
}) => {
// Narrows `live` to the columns the projection's expressions reference.
let child_live = LiveColumns::try_new(&expr, input.schema())?;
- rewrite_single_input(input, child_live, duplicate_insensitive, |input| {
- Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
- expr, input, schema,
- )?))
- })
+ let child_duplicate_insensitive =
+ duplicate_insensitive && expr.iter().all(is_repeatable);
+ rewrite_single_input(
+ input,
+ child_live,
+ child_duplicate_insensitive,
+ |input| {
+ Ok(LogicalPlan::Projection(Projection::try_new_with_schema(
+ expr, input, schema,
+ )?))
+ },
+ )
}
LogicalPlan::Filter(Filter {
predicate, input, ..
@@ -231,9 +243,14 @@ fn rewrite_node(
// Adds the predicate's columns to `live` (a side used only by the filter stays live).
let mut child_live = live;
child_live.extend_from([&predicate], input.schema())?;
- rewrite_single_input(input, child_live, duplicate_insensitive, |input| {
- Ok(LogicalPlan::Filter(Filter::new(predicate, input)))
- })
+ let child_duplicate_insensitive =
+ duplicate_insensitive && is_repeatable(&predicate);
+ rewrite_single_input(
+ input,
+ child_live,
+ child_duplicate_insensitive,
+ |input| Ok(LogicalPlan::Filter(Filter::new(predicate, input))),
+ )
}
LogicalPlan::Aggregate(Aggregate {
input,
@@ -248,12 +265,15 @@ fn rewrite_node(
input.schema(),
)?;
- // A grouping aggregate with no aggregate functions (`GROUP BY` with
- // an empty `aggr_expr`) only observes which group-key values exist,
- // not how many rows produced them, so its input is duplicate-
- // insensitive.
+ // The input can ignore repeated rows when grouping expressions are
+ // repeatable and every aggregate ignores duplicates, either by
+ // nature (`min`) or because it deduplicates its own input
+ // (`count(DISTINCT x)`). This covers grouping-only and global
+ // aggregates. One sensitive aggregate makes input multiplicity
+ // observable, even beneath an insensitive ancestor.
let child_duplicate_insensitive =
- !group_expr.is_empty() && aggr_expr.is_empty();
+ aggr_expr.iter().all(is_duplicate_insensitive_aggregate)
+ && group_expr.iter().all(is_repeatable);
rewrite_single_input(
input,
@@ -286,22 +306,28 @@ fn rewrite_node(
// `GROUP BY` on the columns it reads, so its input is duplicate-
// insensitive; the live columns are exactly those of the
// ON/SELECT/ORDER BY expressions.
- let mut child_live =
- LiveColumns::try_new(on_expr.iter().chain(&select_expr), input.schema())?;
- if let Some(sort_expr) = &sort_expr {
- child_live
- .extend_from(sort_expr.iter().map(|s| &s.expr), input.schema())?;
- }
-
- rewrite_single_input(input, child_live, true, |input| {
- Ok(LogicalPlan::Distinct(Distinct::On(DistinctOn {
- on_expr,
- select_expr,
- sort_expr,
- input,
- schema,
- })))
- })
+ let exprs = || {
+ on_expr
+ .iter()
+ .chain(&select_expr)
+ .chain(sort_expr.iter().flatten().map(|sort| &sort.expr))
+ };
+ let child_live = LiveColumns::try_new(exprs(), input.schema())?;
+ let child_duplicate_insensitive = exprs().all(is_repeatable);
+ rewrite_single_input(
+ input,
+ child_live,
+ child_duplicate_insensitive,
+ |input| {
+ Ok(LogicalPlan::Distinct(Distinct::On(DistinctOn {
+ on_expr,
+ select_expr,
+ sort_expr,
+ input,
+ schema,
+ })))
+ },
+ )
}
LogicalPlan::Sort(Sort { expr, input, fetch }) => {
// Adds the sort-key columns to `live`.
@@ -310,7 +336,9 @@ fn rewrite_node(
// A `fetch` (top-N) makes the row count observable, so duplicate-
// insensitivity does not survive past it.
- let child_duplicate_insensitive = duplicate_insensitive && fetch.is_none();
+ let child_duplicate_insensitive = duplicate_insensitive
+ && fetch.is_none()
+ && expr.iter().all(|sort| is_repeatable(&sort.expr));
rewrite_single_input(
input,
child_live,
@@ -409,6 +437,17 @@ fn rewrite_join(
let (visible_left, visible_right) = split_join_output_columns(&join, live);
+ // A semi join may stop evaluating conditions after finding a match.
+ // If the conditions are not repeatable, skipping evaluations can change
+ // later matches. Removing duplicate input rows can have the same effect.
+ // Require repeatable conditions for this join and rewrites of its inputs.
+ let repeatable = join
+ .on
+ .iter()
+ .all(|(left, right)| is_repeatable(left) && is_repeatable(right))
+ && join.filter.iter().all(is_repeatable);
+ let duplicate_insensitive = duplicate_insensitive && repeatable;
+
let rewritten_join_type = match rewritten_join_type(
&join,
&visible_left,
@@ -446,8 +485,11 @@ fn rewrite_join(
add_join_condition_columns(&join, &mut left_live, &mut right_live)?;
- let (left_dup_insensitive, right_dup_insensitive) =
- child_duplicate_insensitivity(rewritten_join_type, duplicate_insensitive);
+ let (left_dup_insensitive, right_dup_insensitive) = child_duplicate_insensitivity(
+ rewritten_join_type,
+ duplicate_insensitive,
+ repeatable,
+ );
let left = rewrite_subtree(
Arc::unwrap_or_clone(join.left),
@@ -497,18 +539,21 @@ fn rewrite_join(
/// Returns which join inputs can safely ignore duplicate rows from their own
/// descendants. For semi/anti/mark joins, duplicates from the existence side do
-/// not change the result even when the parent itself is duplicate-sensitive.
+/// not change the result even when the parent itself is duplicate-sensitive,
+/// provided the join conditions are `repeatable`: removing duplicate rows
+/// changes how often the conditions are evaluated.
fn child_duplicate_insensitivity(
join_type: JoinType,
duplicate_insensitive: bool,
+ repeatable: bool,
) -> (bool, bool) {
match join_type {
JoinType::Inner => (duplicate_insensitive, duplicate_insensitive),
JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {
- (duplicate_insensitive, true)
+ (duplicate_insensitive, repeatable)
}
JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {
- (true, duplicate_insensitive)
+ (repeatable, duplicate_insensitive)
}
JoinType::Left | JoinType::Right | JoinType::Full => (false, false),
}
@@ -546,9 +591,10 @@ fn rewritten_join_type(
));
// A LEFT JOIN preserves every left row, so with a redundant right side the
- // join has no observable effect and can be replaced by its left input. A
- // join filter cannot prevent this: it only decides whether a left row is
- // matched or null-padded, and either way the row is emitted.
+ // join has no observable effect and can be replaced by its left input.
+ // A filter only decides whether a row is matched or null-padded. When
+ // relying on duplicate-insensitivity, the caller has already required
+ // repeatable join conditions.
if join.join_type == JoinType::Left && can_remove_right {
return JoinRewrite::ReplaceWithLeft;
}
@@ -641,21 +687,27 @@ fn side_unique_on_join<'a>(
#[cfg(test)]
mod tests {
use crate::OptimizerContext;
+ use crate::OptimizerRule;
use crate::assert_optimized_plan_eq_snapshot;
use crate::eliminate_join::EliminateJoin;
+ use crate::test::udfs::{DistinctHandlingTestUDAF, PlacementTestUDF};
use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::tree_node::Transformed;
use datafusion_common::{
Constraint, Constraints, NullEquality, Result, ScalarValue, SplitPoint,
};
use datafusion_expr::JoinType::Inner;
use datafusion_expr::{
- Expr, JoinType, Partitioning, RangePartitioning, col, exists, lit,
+ AggregateUDF, DistinctHandling, Expr, ExprFunctionExt, JoinType, LogicalPlan,
+ Partitioning, RangePartitioning, ScalarUDF, Volatility, col, exists, lit,
logical_plan::builder::{
LogicalPlanBuilder, table_scan, table_source_with_constraints,
},
- out_ref_col,
+ out_ref_col, scalar_subquery,
+ };
+ use datafusion_functions_aggregate::expr_fn::{
+ corr, count, count_distinct, max, min, regr_count, stddev,
};
- use datafusion_functions_aggregate::expr_fn::count;
use std::sync::Arc;
macro_rules! assert_optimized_plan_equal {
@@ -674,6 +726,20 @@ mod tests {
}};
}
+ fn rewrite(plan: LogicalPlan) -> Result<Transformed<LogicalPlan>> {
+ EliminateJoin::new().rewrite(plan, &OptimizerContext::new())
+ }
+
+ fn assert_not_rewritten(plan: LogicalPlan) -> Result<()> {
+ assert!(!rewrite(plan)?.transformed);
+ Ok(())
+ }
+
+ /// A scalar UDF with the given volatility.
+ fn test_udf(volatility: Volatility) -> ScalarUDF {
+ ScalarUDF::from(PlacementTestUDF::new().with_volatility(volatility))
+ }
+
#[test]
fn join_on_false() -> Result<()> {
let plan = LogicalPlanBuilder::empty(false)
@@ -752,9 +818,8 @@ mod tests {
}
#[test]
- fn aggregate_with_aggregates_is_not_duplicate_insensitive() -> Result<()> {
- // A `GROUP BY` *with* aggregate functions observes how many rows fall in
- // each group, so its input is not duplicate-insensitive. With a non-unique
+ fn count_is_not_duplicate_insensitive() -> Result<()> {
+ // COUNT observes how many rows fall in each group. With a non-unique
// right side the join must stay an inner join: collapsing it to a semi
// join would drop matching duplicates and undercount `count(l.id)`.
let plan = left_join_right()?
@@ -769,6 +834,383 @@ mod tests {
")
}
+ #[test]
+ fn insensitive_aggregates_enable_semi_joins() -> Result<()> {
+ for column in ["l.x", "r.x"] {
+ let aggr_expr = vec![
+ min(col(column)).alias("minimum"),
+ max(col(column)).distinct().build()?,
+ ];
+ // Both global and grouped aggregates ignore duplicate input rows.
+ for group_expr in [vec![], vec![col(column)]] {
+ let plan = left_join_right()?
+ .aggregate(group_expr, aggr_expr.clone())?
+ .build()?;
+ let result = rewrite(plan)?;
+ assert!(result.transformed);
+ let LogicalPlan::Aggregate(aggregate) = result.data else {
+ panic!("expected aggregate");
+ };
+ assert_eq!(aggregate.aggr_expr, aggr_expr);
+ let LogicalPlan::Join(join) = aggregate.input.as_ref() else {
+ panic!("expected join");
+ };
+ assert_eq!(
+ join.join_type,
+ if column == "l.x" {
+ JoinType::LeftSemi
+ } else {
+ JoinType::RightSemi
+ }
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn global_min_removes_unused_outer_join() -> Result<()> {
+ for (join_type, column, table) in
+ [(JoinType::Left, "l.x", "l"), (JoinType::Right, "r.x", "r")]
+ {
+ let left = scan("l", &test_schema(), Constraints::default())?;
+ let right = scan("r", &test_schema(), Constraints::default())?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join(right, join_type, (vec!["l.id"], vec!["r.id"]), None)?
+ .aggregate(Vec::<Expr>::new(), vec![min(col(column))])?
+ .build()?;
+ let optimized = rewrite(plan)?.data;
+ let expected = LogicalPlanBuilder::from(scan(
+ table,
+ &test_schema(),
+ Constraints::default(),
+ )?)
+ .aggregate(Vec::<Expr>::new(), vec![min(col(column))])?
+ .build()?;
+ assert_eq!(optimized, expected);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_sensitive_aggregates_enable_semi_joins() -> Result<()> {
+ // A `Sensitive` function called with DISTINCT deduplicates its own
+ // input, so it cannot observe rows repeated by the join.
+ for aggr_expr in [
+ vec![count_distinct(col("l.x"))],
+ vec![count_distinct(col("l.x")), count_distinct(col("l.y"))],
+ vec![
+ min(col("l.x")),
+ count(col("l.x"))
+ .distinct()
+ .filter(col("l.y").gt(lit(0)))
+ .build()?,
+ ],
+ ] {
+ let plan = left_join_right()?
+ .aggregate(vec![col("l.id")], aggr_expr)?
+ .build()?;
+ let result = rewrite(plan)?;
+ assert!(result.transformed);
+ let LogicalPlan::Aggregate(aggregate) = result.data else {
+ panic!("expected aggregate");
+ };
+ let LogicalPlan::Join(join) = aggregate.input.as_ref() else {
+ panic!("expected join");
+ };
+ assert_eq!(join.join_type, JoinType::LeftSemi);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn duplicate_sensitive_aggregates_block_rewrite() -> Result<()> {
+ // One aggregate that observes repeated rows keeps the join, even
+ // beside aggregates that do not. DISTINCT does not qualify an
+ // `Unsupported` function: its accumulator does not deduplicate, and
+ // may silently compute the non-distinct answer.
+ for sensitive in [
+ count(col("l.x")),
+ stddev(col("l.x")).distinct().build()?,
+ corr(col("l.x"), col("l.y")).distinct().build()?,
+ regr_count(col("l.x"), col("l.y")).distinct().build()?,
+ ] {
+ let plan = left_join_right()?
+ .aggregate(
+ Vec::<Expr>::new(),
+ vec![min(col("l.x")), count_distinct(col("l.x")), sensitive],
+ )?
+ .build()?;
+ assert_not_rewritten(plan)?;
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn sensitive_aggregate_blocks_insensitive_ancestor() -> Result<()> {
+ let plan = left_join_right()?
+ .aggregate(vec![col("l.x")], vec![count(col("l.id")).alias("n")])?
+ .aggregate(Vec::<Expr>::new(), vec![min(col("n"))])?
+ .build()?;
+ assert_not_rewritten(plan)?;
+ Ok(())
+ }
+
+ #[test]
+ fn subquery_aggregate_argument_blocks_rewrite() -> Result<()> {
+ // Expr's usual volatility check does not descend into a subquery plan.
+ let volatile = test_udf(Volatility::Volatile).call(vec![lit(1)]);
+ let subquery = LogicalPlanBuilder::empty(true)
+ .project(vec![volatile])?
+ .build()?;
+ let plan = left_join_right()?
+ .aggregate(
+ vec![col("l.x")],
+ vec![min(scalar_subquery(Arc::new(subquery)))],
+ )?
+ .build()?;
+ assert_not_rewritten(plan)?;
+ Ok(())
+ }
+
+ #[test]
+ fn aggregate_filter_and_ordering_keep_columns_live() -> Result<()> {
+ for aggr in [
+ min(col("l.x")).filter(col("r.y").gt(lit(0))).build()?,
+ min(col("l.x"))
+ .order_by(vec![col("r.y").sort(true, false)])
+ .build()?,
+ ] {
+ let plan = left_join_right()?
+ .aggregate(Vec::<Expr>::new(), vec![aggr])?
+ .build()?;
+ assert_not_rewritten(plan)?;
+ }
+
+ let plan = left_join_right()?
+ .aggregate(
+ Vec::<Expr>::new(),
+ vec![min(col("l.x")).filter(col("l.y").gt(lit(0))).build()?],
+ )?
+ .build()?;
+ assert_optimized_plan_equal!(plan, @r"
+ Aggregate: groupBy=[[]], aggr=[[min(l.x) FILTER (WHERE l.y > Int32(0))]]
+ LeftSemi Join: l.id = r.id
+ TableScan: l
+ TableScan: r
+ ")
+ }
+
+ #[test]
+ fn volatile_aggregate_expressions_block_rewrite() -> Result<()> {
+ // `Stable` is the control: the same plan with a repeatable expression
+ // is rewritten.
+ for volatility in [Volatility::Stable, Volatility::Volatile] {
+ let expr = test_udf(volatility).call(vec![col("l.x")]);
+ for (group_expr, aggr) in [
+ (vec![], min(expr.clone())),
+ (vec![expr.clone()], min(col("l.x"))),
+ (
+ vec![],
+ min(col("l.x"))
+ .filter(expr.clone().gt(lit(0_u32)))
+ .build()?,
+ ),
+ (
+ vec![],
+ min(col("l.x"))
+ .order_by(vec![expr.clone().sort(true, false)])
+ .build()?,
+ ),
+ ] {
+ let plan = left_join_right()?
+ .aggregate(group_expr, vec![aggr])?
+ .build()?;
+ let result = rewrite(plan.clone())?;
+ assert_eq!(
+ result.transformed,
+ volatility != Volatility::Volatile,
+ "{volatility:?}: {}",
+ plan.display_indent(),
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn volatile_intervening_expressions_block_rewrite() -> Result<()> {
+ // `Stable` is the control: the same plan with a repeatable expression
+ // is rewritten.
+ for volatility in [Volatility::Stable, Volatility::Volatile] {
+ let expr = test_udf(volatility).call(vec![col("l.x")]);
+ for input in [
+ left_join_right()?.project(vec![col("l.x"), expr.clone().alias("v")])?,
+ left_join_right()?.filter(expr.clone().gt(lit(0_u32)))?,
+ left_join_right()?.sort(vec![expr.clone().sort(true, false)])?,
+ ] {
+ let plan = input
+ .aggregate(Vec::<Expr>::new(), vec![min(col("l.x"))])?
+ .build()?;
+ let result = rewrite(plan.clone())?;
+ assert_eq!(
+ result.transformed,
+ volatility != Volatility::Volatile,
+ "{volatility:?}: {}",
+ plan.display_indent(),
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn join_conditions_must_be_repeatable() -> Result<()> {
+ for volatility in [Volatility::Stable, Volatility::Volatile] {
+ let udf = test_udf(volatility);
+ for (left_key, right_key, filter) in [
+ (udf.call(vec![col("l.id")]), col("r.id"), None),
+ (col("l.id"), udf.call(vec![col("r.id")]), None),
+ (
+ col("l.id"),
+ col("r.id"),
+ Some(udf.call(vec![col("l.x")]).gt(lit(0_u32))),
+ ),
+ ] {
+ let plan = LogicalPlanBuilder::from(scan(
+ "l",
+ &test_schema(),
+ Constraints::default(),
+ )?)
+ .join_with_expr_keys(
+ scan("r", &test_schema(), Constraints::default())?,
+ Inner,
+ (vec![left_key], vec![right_key]),
+ filter,
+ )?
+ .aggregate(Vec::<Expr>::new(), vec![min(col("l.x"))])?
+ .build()?;
+ let result = rewrite(plan.clone())?;
+ assert_eq!(
+ result.transformed,
+ volatility != Volatility::Volatile,
+ "{volatility:?}: {}",
+ plan.display_indent(),
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn existence_side_rewrites_require_repeatable_join_conditions() -> Result<()> {
+ // There is no duplicate-insensitive ancestor. Only the semi join's
+ // existence side can make the nested inner join eligible for rewriting.
+ for join_type in [JoinType::LeftSemi, JoinType::RightSemi] {
+ for volatility in [Volatility::Stable, Volatility::Volatile] {
+ let inner = left_join_right()?.build()?;
+ let other = scan("s", &test_schema(), Constraints::default())?;
+ let (left, right, keys) = if join_type == JoinType::LeftSemi {
+ (other, inner, (vec!["s.id"], vec!["l.id"]))
+ } else {
+ (inner, other, (vec!["l.id"], vec!["s.id"]))
+ };
+ let predicate =
+ test_udf(volatility).call(vec![col("l.x")]).gt(lit(0_u32));
+ let plan = LogicalPlanBuilder::from(left)
+ .join(right, join_type, keys, Some(predicate))?
+ .build()?;
+ let result = rewrite(plan)?;
+ let repeatable = volatility != Volatility::Volatile;
+ assert_eq!(
+ result.transformed, repeatable,
+ "{join_type:?}, {volatility:?}"
+ );
+ let LogicalPlan::Join(join) = result.data else {
+ panic!("expected semi join");
+ };
+ assert_eq!(join.join_type, join_type);
+ let existence_side = if join_type == JoinType::LeftSemi {
+ join.right
+ } else {
+ join.left
+ };
+ let LogicalPlan::Join(nested) = existence_side.as_ref() else {
+ panic!("expected nested join");
+ };
+ assert_eq!(
+ nested.join_type,
+ if repeatable {
+ JoinType::LeftSemi
+ } else {
+ Inner
+ },
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_only_qualifies_sensitive_udaf() -> Result<()> {
+ for (distinct_handling, distinct, expect_rewrite) in [
+ (DistinctHandling::Insensitive, false, true),
+ (DistinctHandling::Insensitive, true, true),
+ (DistinctHandling::Sensitive, false, false),
+ // The accumulator deduplicates its own input.
+ (DistinctHandling::Sensitive, true, true),
+ (DistinctHandling::Unsupported, false, false),
+ // The accumulator does not implement DISTINCT, so the input's
+ // repeated rows stay observable.
+ (DistinctHandling::Unsupported, true, false),
+ ] {
+ let udf = AggregateUDF::from(DistinctHandlingTestUDAF::new(
+ "custom_udaf",
+ distinct_handling,
+ ));
+ let mut aggr = udf.call(vec![col("l.x")]);
+ if distinct {
+ aggr = aggr.distinct().build()?;
+ }
+ let plan = left_join_right()?
+ .aggregate(Vec::<Expr>::new(), vec![aggr])?
+ .build()?;
+ let result = rewrite(plan)?;
+ assert_eq!(
+ result.transformed, expect_rewrite,
+ "{distinct_handling:?}, distinct={distinct}"
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn aliased_udaf_uses_declared_handling() -> Result<()> {
+ for volatility in [
+ Volatility::Immutable,
+ Volatility::Stable,
+ Volatility::Volatile,
+ ] {
+ let udf = AggregateUDF::from(
+ DistinctHandlingTestUDAF::new(
+ "custom_udaf",
+ DistinctHandling::Insensitive,
+ )
+ .with_volatility(volatility),
+ )
+ .with_aliases(["custom_alias"]);
+ let plan = left_join_right()?
+ .aggregate(
+ Vec::<Expr>::new(),
+ vec![udf.call(vec![col("l.x")]).alias("result")],
+ )?
+ .build()?;
+ let result = rewrite(plan)?;
+ assert_eq!(result.transformed, volatility != Volatility::Volatile);
+ }
+ Ok(())
+ }
+
#[test]
fn duplicate_insensitive_context_propagates_through_join_tree() -> Result<()> {
let left = scan("l", &test_schema(), Constraints::default())?;
@@ -867,18 +1309,16 @@ mod tests {
#[test]
fn correlated_subquery_outer_ref_prevents_rewrite() -> Result<()> {
- // The aggregate makes the parent duplicate-insensitive, so absent any
- // other use of the right side the join would collapse to a semi join.
- // But the `EXISTS` subquery correlates on `r.y`, so the right side is
- // still needed and the join must stay an inner join. Otherwise the
- // semi join would drop `r`, orphaning the correlated `r.y` reference.
+ // The right side is unique, so the subquery's repeatability barrier
+ // alone cannot prevent a semi-join rewrite. Tracking the correlated
+ // `r.y` reference must keep the right side live and the join inner.
let subquery =
LogicalPlanBuilder::from(scan("s", &test_schema(), Constraints::default())?)
.filter(col("s.id").eq(out_ref_col(DataType::Int32, "r.y")))?
.project(vec![lit(1)])?
.build()?;
- let plan = left_join_right()?
+ let plan = left_join_right_with_constraints(primary_key_on_id())?
.filter(exists(Arc::new(subquery)))?
.aggregate(vec![col("l.x")], Vec::<Expr>::new())?
.build()?;
@@ -1176,6 +1616,34 @@ mod tests {
")
}
+ #[test]
+ fn distinct_on_expressions_must_be_repeatable() -> Result<()> {
+ for volatility in [Volatility::Stable, Volatility::Volatile] {
+ let expr = test_udf(volatility).call(vec![col("l.x")]);
+ for (on_expr, select_expr, sort_expr) in [
+ (vec![expr.clone()], vec![col("l.x")], None),
+ (vec![col("l.x")], vec![expr.clone()], None),
+ (
+ vec![col("l.x")],
+ vec![col("l.x")],
+ Some(vec![col("l.x").sort(true, false), expr.sort(true, false)]),
+ ),
+ ] {
+ let plan = left_join_right()?
+ .distinct_on(on_expr, select_expr, sort_expr)?
+ .build()?;
+ let result = rewrite(plan.clone())?;
+ assert_eq!(
+ result.transformed,
+ volatility != Volatility::Volatile,
+ "{volatility:?}: {}",
+ plan.display_indent(),
+ );
+ }
+ }
+ Ok(())
+ }
+
#[test]
fn existing_semi_join_passes_through_unchanged() -> Result<()> {
// A join that is already a semi join is threaded through unchanged: the rule
@@ -1237,7 +1705,7 @@ mod tests {
name: &str,
schema: &Schema,
constraints: Constraints,
- ) -> Result<datafusion_expr::logical_plan::LogicalPlan> {
+ ) -> Result<LogicalPlan> {
if constraints.is_empty() {
table_scan(Some(name), schema, None)?.build()
} else {
diff --git a/datafusion/optimizer/src/test/udfs.rs b/datafusion/optimizer/src/test/udfs.rs
index e139039957..c1f71a1fff 100644
--- a/datafusion/optimizer/src/test/udfs.rs
+++ b/datafusion/optimizer/src/test/udfs.rs
@@ -17,10 +17,13 @@
use arrow::datatypes::DataType;
use datafusion_common::Result;
+use datafusion_expr::function::AccumulatorArgs;
use datafusion_expr::{
- ColumnarValue, Expr, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,
- ScalarUDFImpl, Signature, TypeSignature, Volatility, lit,
+ Accumulator, AggregateUDFImpl, ColumnarValue, DistinctHandling, Expr,
+ ExpressionPlacement, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
+ TypeSignature, Volatility, lit,
};
+use std::hash::{Hash, Hasher};
/// A configurable test UDF for optimizer tests.
/// Defaults to `MoveTowardsLeafNodes` placement. Use `with_placement()` to override.
@@ -166,3 +169,58 @@ pub fn get_field_like(base: Expr, key: &str) -> Expr {
let udf = ScalarUDF::new_from_impl(GetFieldLikeUDF::new());
udf.call(vec![base, lit(key)])
}
+
+/// A test aggregate that reports the [`DistinctHandling`] it was built with.
+///
+/// Built-in functions only cover the variants they happen to carry. This one
+/// exercises the public API a third-party function uses: an
+/// [`AggregateUDFImpl::distinct_handling`] override.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DistinctHandlingTestUDAF {
+ name: &'static str,
+ handling: DistinctHandling,
+ signature: Signature,
+}
+
+impl DistinctHandlingTestUDAF {
+ pub fn new(name: &'static str, handling: DistinctHandling) -> Self {
+ Self {
+ name,
+ handling,
+ signature: Signature::any(1, Volatility::Immutable),
+ }
+ }
+
+ /// Set the volatility of the aggregate function itself.
+ pub fn with_volatility(mut self, volatility: Volatility) -> Self {
+ self.signature.volatility = volatility;
+ self
+ }
+}
+
+/// Hashed by name and signature. `DistinctHandling` is not `Hash`, and
+/// `AggregateUDFImpl` requires one through `DynHash`.
+impl Hash for DistinctHandlingTestUDAF {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.name.hash(state);
+ self.signature.hash(state);
+ }
+}
+
+impl AggregateUDFImpl for DistinctHandlingTestUDAF {
+ fn name(&self) -> &str {
+ self.name
+ }
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::UInt32)
+ }
+ fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
+ panic!("DistinctHandlingTestUDAF: not intended for execution")
+ }
+ fn distinct_handling(&self) -> DistinctHandling {
+ self.handling
+ }
+}
diff --git a/datafusion/optimizer/src/unions_to_filter.rs b/datafusion/optimizer/src/unions_to_filter.rs
index 2eda0daba1..7bc39a084f 100644
--- a/datafusion/optimizer/src/unions_to_filter.rs
+++ b/datafusion/optimizer/src/unions_to_filter.rs
@@ -18,6 +18,7 @@
//! Rewrites `UNION DISTINCT` branches that differ only by filter predicates
//! into a single filtered branch plus `DISTINCT`.
+use crate::utils::is_repeatable;
use crate::{OptimizerConfig, OptimizerRule};
use datafusion_common::Result;
use datafusion_common::tree_node::{
@@ -180,7 +181,7 @@ fn extract_branch(plan: LogicalPlan) -> Result<Option<UnionBranch>> {
LogicalPlan::Filter(Filter {
predicate, input, ..
}) => {
- if !is_mergeable_predicate(&predicate) {
+ if !is_repeatable(&predicate) {
debug!(
"unions_to_filter skipped: branch predicate contains volatility or a subquery"
);
@@ -334,10 +335,6 @@ fn align_plan_to_schema(
)?))
}
-fn is_mergeable_predicate(expr: &Expr) -> bool {
- !expr.is_volatile() && !expr_contains_subquery(expr)
-}
-
/// Check every expression in the retained source, including its descendants.
/// Merging branches also merges their source evaluations, so the same
/// restrictions as for projection wrappers apply throughout the source.
@@ -345,7 +342,7 @@ fn source_is_safe(source: &LogicalPlan) -> Result<bool> {
let mut safe = true;
source.apply(|node| {
node.apply_expressions(|expr| {
- if is_mergeable_predicate(expr) {
+ if is_repeatable(expr) {
Ok(TreeNodeRecursion::Continue)
} else {
safe = false;
@@ -370,30 +367,17 @@ fn collect_table_sources(source: &LogicalPlan) -> Result<Vec<Arc<dyn TableSource
/// Returns `true` when every projection expression in `wrappers` is both
/// non-volatile and subquery-free.
///
-/// Volatile expressions (e.g. `random()`, `now()`) or correlated subqueries
+/// Volatile expressions (e.g. `random()`) or correlated subqueries
/// in the SELECT list cannot be safely merged: the original plan evaluates
/// them once per branch execution, while the rewritten plan evaluates them
/// once per combined row, which can change the set of output rows.
fn wrapper_projections_are_safe(wrappers: &[Wrapper]) -> bool {
wrappers.iter().all(|w| match w {
- Wrapper::Projection { expr, .. } => expr
- .iter()
- .all(|e| !e.is_volatile() && !expr_contains_subquery(e)),
+ Wrapper::Projection { expr, .. } => expr.iter().all(is_repeatable),
Wrapper::SubqueryAlias { .. } => true,
})
}
-fn expr_contains_subquery(expr: &Expr) -> bool {
- expr.exists(|e| match e {
- Expr::ScalarSubquery(_)
- | Expr::Exists(_)
- | Expr::InSubquery(_)
- | Expr::SetComparison(_) => Ok(true),
- _ => Ok(false),
- })
- .expect("boolean expression walk is infallible")
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -401,11 +385,10 @@ mod tests {
use crate::assert_optimized_plan_eq_snapshot;
use crate::test::test_table_scan_with_name;
use arrow::datatypes::DataType;
- use datafusion_common::{Result, Spans};
- use datafusion_expr::expr::{SetComparison, SetQuantifier};
+ use datafusion_common::Result;
use datafusion_expr::{
- ColumnarValue, Expr, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
- Signature, Subquery, Volatility, col, lit,
+ ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
+ Volatility, col, lit,
};
macro_rules! assert_optimized_plan_equal {
@@ -455,27 +438,6 @@ mod tests {
ScalarUDF::new_from_impl(VolatileTestUdf).call(vec![])
}
- #[test]
- fn set_comparison_is_detected_as_subquery() {
- let subquery = LogicalPlanBuilder::from(test_table_scan_with_name("t2").unwrap())
- .project(vec![col("b")])
- .unwrap()
- .build()
- .unwrap();
- let expr = Expr::SetComparison(SetComparison::new(
- Box::new(col("a")),
- Subquery {
- subquery: Arc::new(subquery),
- outer_ref_columns: vec![],
- spans: Spans::new(),
- },
- Operator::Gt,
- SetQuantifier::Any,
- ));
-
- assert!(expr_contains_subquery(&expr));
- }
-
fn assert_not_rewritten(plan: LogicalPlan) {
let mut options = datafusion_common::config::ConfigOptions::default();
options.optimizer.enable_unions_to_filter = true;
diff --git a/datafusion/optimizer/src/utils.rs b/datafusion/optimizer/src/utils.rs
index d6a5480861..716efc8197 100644
--- a/datafusion/optimizer/src/utils.rs
+++ b/datafusion/optimizer/src/utils.rs
@@ -30,7 +30,9 @@ use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::expr::{Exists, InSubquery, SetComparison};
use datafusion_expr::expr_rewriter::replace_col;
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
-use datafusion_expr::{ColumnarValue, Expr, WriteOp, logical_plan::LogicalPlan};
+use datafusion_expr::{
+ ColumnarValue, DistinctHandling, Expr, Volatility, WriteOp, logical_plan::LogicalPlan,
+};
use datafusion_physical_expr::create_physical_expr;
use log::{debug, trace};
use std::sync::Arc;
@@ -39,6 +41,51 @@ use std::sync::Arc;
/// as it was initially placed here and then moved elsewhere.
pub use datafusion_expr::expr_rewriter::NamePreserver;
+/// Whether an expression is free of volatile scalar functions and subqueries.
+/// Subqueries are conservative barriers because their plans may contain
+/// volatile expressions that [`Expr::is_volatile`] does not visit.
+pub(crate) fn is_repeatable(expr: &Expr) -> bool {
+ !expr
+ .exists(|expr| {
+ Ok(expr.is_volatile_node()
+ || matches!(
+ expr,
+ Expr::Exists(_)
+ | Expr::InSubquery(_)
+ | Expr::SetComparison(_)
+ | Expr::ScalarSubquery(_)
+ ))
+ })
+ .expect("expression traversal is infallible")
+}
+
+/// Whether an aggregate can safely ignore repeated input rows. This holds when
+/// the function declares [`DistinctHandling::Insensitive`], or when it is
+/// called with `DISTINCT` and declares [`DistinctHandling::Sensitive`], so that
+/// its accumulator removes the repeated rows itself. Arguments, FILTER, and
+/// ORDER BY must also be repeatable: MIN(random()) still observes repetitions.
+pub(crate) fn is_duplicate_insensitive_aggregate(mut expr: &Expr) -> bool {
+ while let Expr::Alias(alias) = expr {
+ expr = &alias.expr;
+ }
+ let Expr::AggregateFunction(aggregate) = expr else {
+ return false;
+ };
+ let ignores_duplicates = match aggregate.func.distinct_handling() {
+ DistinctHandling::Insensitive => true,
+ DistinctHandling::Sensitive => aggregate.params.distinct,
+ // The accumulator does not implement `DISTINCT` and may silently
+ // compute the non-distinct answer, so the flag proves nothing.
+ // Variants added in the future are treated the same way.
+ _ => false,
+ };
+ // Expr::is_volatile checks scalar functions only; check the aggregate
+ // function's own volatility separately.
+ ignores_duplicates
+ && aggregate.func.signature().volatility != Volatility::Volatile
+ && is_repeatable(expr)
+}
+
/// Return the expression schema for a MERGE DML node.
pub(crate) fn merge_into_schema(plan: &LogicalPlan) -> Result<Option<DFSchema>> {
let LogicalPlan::Dml(dml) = plan else {
@@ -262,7 +309,34 @@ fn coerce(expr: Expr, schema: &DFSchema) -> Result<Expr> {
#[cfg(test)]
mod tests {
use super::*;
- use datafusion_expr::{Operator, binary_expr, case, col, in_list, is_null, lit};
+ use crate::test::test_table_scan_with_name;
+ use datafusion_common::Spans;
+ use datafusion_expr::expr::SetQuantifier;
+ use datafusion_expr::logical_plan::builder::LogicalPlanBuilder;
+ use datafusion_expr::{
+ Operator, Subquery, binary_expr, case, col, in_list, is_null, lit,
+ };
+
+ #[test]
+ fn set_comparison_is_not_repeatable() {
+ let subquery = LogicalPlanBuilder::from(test_table_scan_with_name("t2").unwrap())
+ .project(vec![col("b")])
+ .unwrap()
+ .build()
+ .unwrap();
+ let expr = Expr::SetComparison(SetComparison::new(
+ Box::new(col("a")),
+ Subquery {
+ subquery: Arc::new(subquery),
+ outer_ref_columns: vec![],
+ spans: Spans::new(),
+ },
+ Operator::Gt,
+ SetQuantifier::Any,
+ ));
+
+ assert!(!is_repeatable(&expr));
+ }
#[test]
fn expr_is_restrict_null_predicate() -> Result<()> {
diff --git a/datafusion/sqllogictest/test_files/eliminate_join_distinct.slt b/datafusion/sqllogictest/test_files/eliminate_join_distinct.slt
new file mode 100644
index 0000000000..006cad921f
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/eliminate_join_distinct.slt
@@ -0,0 +1,222 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Duplicate-insensitive aggregates may ignore join fanout, including NULLs.
+statement ok
+CREATE TABLE l(id INT, g INT, x INT);
+
+statement ok
+CREATE TABLE r(id INT, y INT);
+
+statement ok
+INSERT INTO l VALUES (1, 0, 10), (1, 0, 20), (2, 0, NULL), (3, 1, 30), (4, 2, NULL), (NULL, 3, 99);
+
+statement ok
+INSERT INTO r VALUES (1, 100), (1, 200), (2, 300), (4, 400), (4, 500), (NULL, 600);
+
+statement ok
+SET datafusion.explain.logical_plan_only = true;
+
+query TT
+EXPLAIN SELECT l.g, MIN(l.x), MAX(l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+logical_plan
+01)Aggregate: groupBy=[[l.g]], aggr=[[min(l.x), max(l.x)]]
+02)--Projection: l.g, l.x
+03)----LeftSemi Join: l.id = r.id
+04)------TableScan: l projection=[id, g, x]
+05)------TableScan: r projection=[id]
+
+query III rowsort
+SELECT l.g, MIN(l.x), MAX(l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+0 10 20
+2 NULL NULL
+
+query II rowsort
+SELECT l.g, APPROX_DISTINCT(l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+0 2
+2 0
+
+query IBBII rowsort
+SELECT l.g, BOOL_AND(l.x > 15), BOOL_OR(l.x > 15), BIT_AND(l.x), BIT_OR(l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+0 false true 0 30
+2 NULL NULL NULL NULL
+
+# An insensitive aggregate with a sensitive companion must retain join fanout.
+query IIII rowsort
+SELECT l.g, MIN(l.x), COUNT(*), SUM(l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+0 10 5 60
+2 NULL 2 NULL
+
+# The right side remains live through an aggregate FILTER.
+# The filter removes all non-NULL values, so ignoring it would change the result.
+query TT
+EXPLAIN SELECT MIN(l.x) FILTER (WHERE r.y > 250) FROM l JOIN r ON l.id = r.id;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[min(l.x) FILTER (WHERE r.y > Int32(250)) AS min(l.x) FILTER (WHERE r.y > Int64(250))]]
+02)--Projection: l.x, r.y
+03)----Inner Join: l.id = r.id
+04)------TableScan: l projection=[id, x]
+05)------TableScan: r projection=[id, y]
+
+query I
+SELECT MIN(l.x) FILTER (WHERE r.y > 250) FROM l JOIN r ON l.id = r.id;
+----
+NULL
+
+# The unused non-preserved side of an outer join disappears entirely.
+query TT
+EXPLAIN SELECT MIN(l.x) FROM l LEFT JOIN r ON l.id = r.id;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[min(l.x)]]
+02)--TableScan: l projection=[x]
+
+query III rowsort
+SELECT l.g, MIN(l.x), MAX(l.x) FROM l LEFT JOIN r ON l.id = r.id GROUP BY l.g;
+----
+0 10 20
+1 30 30
+2 NULL NULL
+3 99 99
+
+# The surviving join side can also be the right side.
+query TT
+EXPLAIN SELECT MAX(r.y) FROM l JOIN r ON l.id = r.id;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[max(r.y)]]
+02)--Projection: r.y
+03)----RightSemi Join: l.id = r.id
+04)------TableScan: l projection=[id]
+05)------TableScan: r projection=[id, y]
+
+query I
+SELECT MAX(r.y) FROM l JOIN r ON l.id = r.id;
+----
+500
+
+# Empty join inputs retain global-aggregate semantics.
+statement ok
+CREATE TABLE empty_r(id INT);
+
+query II
+SELECT MIN(l.x), APPROX_DISTINCT(l.x) FROM l JOIN empty_r ON l.id = empty_r.id;
+----
+NULL 0
+
+query I
+SELECT MIN(l.x) FROM l LEFT JOIN empty_r ON l.id = empty_r.id;
+----
+10
+
+query I
+SELECT MIN(empty_r.id) FROM empty_r LEFT JOIN l ON l.id = empty_r.id;
+----
+NULL
+
+# Aggregates that deduplicate their own input ignore join fanout as well.
+query TT
+EXPLAIN SELECT l.g, COUNT(DISTINCT l.x), SUM(DISTINCT l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+logical_plan
+01)Aggregate: groupBy=[[l.g]], aggr=[[count(DISTINCT l.x), sum(DISTINCT CAST(l.x AS Int64))]]
+02)--Projection: l.g, l.x
+03)----LeftSemi Join: l.id = r.id
+04)------TableScan: l projection=[id, g, x]
+05)------TableScan: r projection=[id]
+
+query III rowsort
+SELECT l.g, COUNT(DISTINCT l.x), SUM(DISTINCT l.x) FROM l JOIN r ON l.id = r.id GROUP BY l.g;
+----
+0 2 30
+2 0 NULL
+
+query TT
+EXPLAIN SELECT COUNT(DISTINCT l.x) FILTER (WHERE l.g = 0), MIN(l.x) FROM l JOIN r ON l.id = r.id;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[count(DISTINCT l.x) FILTER (WHERE l.g = Int32(0)) AS count(DISTINCT l.x) FILTER (WHERE l.g = Int64(0)), min(l.x)]]
+02)--Projection: l.g, l.x
+03)----LeftSemi Join: l.id = r.id
+04)------TableScan: l projection=[id, g, x]
+05)------TableScan: r projection=[id]
+
+query II
+SELECT COUNT(DISTINCT l.x) FILTER (WHERE l.g = 0), MIN(l.x) FROM l JOIN r ON l.id = r.id;
+----
+2 10
+
+query ?
+SELECT ARRAY_AGG(DISTINCT l.x ORDER BY l.x) FROM l JOIN r ON l.id = r.id;
+----
+[10, 20, NULL]
+
+# A DISTINCT aggregate with a non-DISTINCT sensitive companion must retain join fanout.
+query TT
+EXPLAIN SELECT COUNT(DISTINCT l.x), COUNT(l.g) FROM l JOIN r ON l.id = r.id;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[count(DISTINCT l.x), count(l.g)]]
+02)--Projection: l.g, l.x
+03)----Inner Join: l.id = r.id
+04)------TableScan: l projection=[id, g, x]
+05)------TableScan: r projection=[id]
+
+query II
+SELECT COUNT(DISTINCT l.x), COUNT(l.g) FROM l JOIN r ON l.id = r.id;
+----
+2 7
+
+# REGR_COUNT declares `DistinctHandling::Unsupported`: its accumulator ignores
+# `is_distinct` and counts every joined row, so the results below are the
+# non-distinct counts (4 and 6; the distinct counts would be 2 and 4). DISTINCT
+# therefore does not hide the join fanout: the inner join must stay an inner
+# join, and the left join must be kept.
+query TT
+EXPLAIN SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l JOIN r ON l.id = r.id;
+----
+logical_plan
+01)Aggregate: groupBy=[[]], aggr=[[regr_count(DISTINCT CAST(l.x AS Float64), CAST(l.g AS Float64))]]
+02)--Projection: l.g, l.x
+03)----Inner Join: l.id = r.id
+04)------TableScan: l projection=[id, g, x]
+05)------TableScan: r projection=[id]
+
+query I
+SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l JOIN r ON l.id = r.id;
+----
+4
+
+query I
+SELECT REGR_COUNT(DISTINCT l.x, l.g) FROM l LEFT JOIN r ON l.id = r.id;
+----
+6
+
+# A limit below the aggregate still observes repeated join rows.
+query I
+SELECT MAX(x) FROM (SELECT l.x FROM l JOIN r ON l.id = r.id ORDER BY l.x NULLS LAST LIMIT 2);
+----
+10
+
+statement ok
+RESET datafusion.explain.logical_plan_only;
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index d2453af739..b594c68ce1 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -1364,8 +1364,9 @@ logical_plan
05)--------TableScan: join_t1 projection=[t1_id, t1_int]
06)--------TableScan: join_t2 projection=[t2_id]
-# A similar query with two DISTINCT aggregates is currently not rewritten
-# TODO: https://github.com/apache/datafusion/issues/22644
+# A similar query with two DISTINCT aggregates is also rewritten: each
+# `count(DISTINCT ...)` removes its own duplicates, so the join's duplicates
+# are not observable.
query TT
EXPLAIN
select join_t1.t1_id, count(distinct join_t1.t1_int), count(distinct join_t1.t1_name)
@@ -1375,10 +1376,9 @@ group by join_t1.t1_id
----
logical_plan
01)Aggregate: groupBy=[[join_t1.t1_id]], aggr=[[count(DISTINCT join_t1.t1_int), count(DISTINCT join_t1.t1_name)]]
-02)--Projection: join_t1.t1_id, join_t1.t1_name, join_t1.t1_int
-03)----Inner Join: join_t1.t1_id = join_t2.t2_id
-04)------TableScan: join_t1 projection=[t1_id, t1_name, t1_int]
-05)------TableScan: join_t2 projection=[t2_id]
+02)--LeftSemi Join: join_t1.t1_id = join_t2.t2_id
+03)----TableScan: join_t1 projection=[t1_id, t1_name, t1_int]
+04)----TableScan: join_t2 projection=[t2_id]
statement ok
set datafusion.explain.logical_plan_only = false;