Apache DataFusion SQL Query Engine
fix: do not remove DISTINCT when a unique key was downgraded by a join (#23548)
Simon Vandel Sillesen95de3853415d
diff --git a/datafusion/common/src/functional_dependencies.rs b/datafusion/common/src/functional_dependencies.rsindex 24ca33c0c2..8b15c49c56 100644--- a/datafusion/common/src/functional_dependencies.rs+++ b/datafusion/common/src/functional_dependencies.rs@@ -151,8 +151,10 @@ pub struct FunctionalDependence {/// Describes functional dependency mode.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum Dependency {- Single, // A determinant key may occur only once.- Multi, // A determinant key may occur multiple times (in multiple rows).+ /// A determinant key may occur only once.+ Single,+ /// A determinant key may occur multiple times (in multiple rows).+ Multi,}impl FunctionalDependence {diff --git a/datafusion/optimizer/src/replace_distinct_aggregate.rs b/datafusion/optimizer/src/replace_distinct_aggregate.rsindex 06df61e766..cc26163790 100644--- a/datafusion/optimizer/src/replace_distinct_aggregate.rs+++ b/datafusion/optimizer/src/replace_distinct_aggregate.rs@@ -22,7 +22,7 @@ use crate::{OptimizerConfig, OptimizerRule};use std::sync::Arc;use datafusion_common::tree_node::Transformed;-use datafusion_common::{Column, Result};+use datafusion_common::{Column, Dependency, Result};use datafusion_expr::expr_rewriter::normalize_cols;use datafusion_expr::utils::expand_wildcard;use datafusion_expr::{Aggregate, Distinct, DistinctOn, Expr, LogicalPlan};@@ -101,9 +101,14 @@ impl OptimizerRule for ReplaceDistinctWithAggregate {let field_count = input.schema().fields().len();for dep in input.schema().functional_dependencies().iter() {- // If distinct is exactly the same with a previous GROUP BY, we can- // simply remove it:- if dep.source_indices.len() >= field_count+ // If the input is already unique on all of its columns (e.g.+ // it is a GROUP BY over exactly these columns), the DISTINCT+ // is a no-op and we can simply remove it. The dependency mode+ // must be `Single`: a `Multi` dependence (e.g. a former key+ // downgraded by a join) means equal rows may occur multiple+ // times, so the DISTINCT still has work to do.+ if dep.mode == Dependency::Single+ && dep.source_indices.len() >= field_count&& dep.source_indices[..field_count].iter().enumerate()diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.sltindex 7b0d8a00d5..de493d6e4a 100644--- a/datafusion/sqllogictest/test_files/group_by.slt+++ b/datafusion/sqllogictest/test_files/group_by.slt@@ -5641,3 +5641,56 @@ set datafusion.execution.target_partitions = 4;statement count 0drop table t;++# DISTINCT must not be removed when a unique key is downgraded to a+# non-unique functional dependency by a join: `u.id` is a primary key, but+# after the LEFT JOIN each `u` row can occur once per matching order.+statement ok+CREATE TABLE users_with_pk (id INT, name VARCHAR, primary key(id)) AS VALUES+ (1, 'alice'),+ (2, 'bob');++statement ok+CREATE TABLE user_orders (user_id INT, amount INT) AS VALUES+ (1, 10),+ (1, 20),+ (2, 30);++query I+SELECT DISTINCT u.id+ FROM users_with_pk u+ LEFT JOIN user_orders o ON u.id = o.user_id+ ORDER BY u.id;+----+1+2++# The DISTINCT must be planned as an Aggregate; it cannot be removed based+# on the (join-downgraded) primary key of `users_with_pk`.+query TT+EXPLAIN SELECT DISTINCT u.id+ FROM users_with_pk u+ LEFT JOIN user_orders o ON u.id = o.user_id;+----+logical_plan+01)Aggregate: groupBy=[[u.id]], aggr=[[]]+02)--Projection: u.id+03)----Left Join: u.id = o.user_id+04)------SubqueryAlias: u+05)--------TableScan: users_with_pk projection=[id]+06)------SubqueryAlias: o+07)--------TableScan: user_orders projection=[user_id]+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[]+02)--RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=4+03)----AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[]+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+05)--------HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@0, user_id@0)], projection=[id@0]+06)----------DataSourceExec: partitions=1, partition_sizes=[1]+07)----------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop table users_with_pk;++statement ok+drop table user_orders;