Skip to content

Apache DataFusion SQL Query Engine

fix: do not use the top-k aggregate stream for a DISTINCT limit with no ordering direction (#25349)

Adrian Garcia Badaracco18cbb877ba9a
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 5b0d2c2d2e..49a726f9be 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -1219,9 +1219,26 @@ impl AggregateExec {
)?));
}
- // grouping by an expression that has a sort/limit upstream
+ // Grouping by an expression that has a sort/limit upstream.
+ //
+ // `GroupedTopKAggregateStream` keeps a priority queue, so it only works
+ // when an ordering direction is available: either this is a MIN/MAX
+ // aggregate, whose direction is implied by the accumulator, or the
+ // optimizer pushed a direction down next to the limit.
+ //
+ // A direction-less limit is a soft hint pushed by
+ // `LimitedDistinctAggregation`, which only pushes it while
+ // `is_unordered_unfiltered_group_by_distinct()` holds. That predicate is
+ // not stable: a later rule such as `EnsureRequirements` can insert the
+ // sort a window function requires below this aggregate, which gives the
+ // rebuilt aggregate an output ordering and makes the predicate false
+ // without ever supplying a direction. Falling back to the regular
+ // grouped streams is correct in that case, because they treat the limit
+ // as a soft limit and the `LIMIT` above this aggregate still truncates
+ // the result.
if let Some(config) = self.limit_options
&& !self.is_unordered_unfiltered_group_by_distinct()
+ && (config.descending.is_some() || self.get_minmax_desc().is_some())
{
return Ok(StreamType::GroupedPriorityQueue(
GroupedTopKAggregateStream::new(self, context, partition, config.limit)?,
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 3eabd5c4ff..a4a54cdee2 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -7522,6 +7522,35 @@ set datafusion.optimizer.enable_distinct_aggregation_soft_limit = true;
statement ok
set datafusion.optimizer.repartition_aggregations = true;
+# A window function in the projection makes `EnsureRequirements` add a sort below
+# the distinct aggregation, so the aggregation ends up with an output ordering after
+# `LimitedDistinctAggregation` pushed a direction-less limit into it. The query
+# must still execute and return the distinct rows.
+statement ok
+CREATE TABLE distinct_limit_window_table(env VARCHAR, region VARCHAR, deployment VARCHAR) AS VALUES
+ ('prod', 'a', 'x'),
+ ('prod', 'a', 'y'),
+ ('staging', 'b', 'x');
+
+query TTI rowsort
+SELECT DISTINCT env, region, count(*) OVER (PARTITION BY env, deployment) AS n
+FROM distinct_limit_window_table
+LIMIT 30;
+----
+prod a 1
+staging b 1
+
+query TI rowsort
+SELECT DISTINCT env, count(*) OVER (PARTITION BY env) AS n
+FROM distinct_limit_window_table
+LIMIT 30;
+----
+prod 2
+staging 1
+
+statement ok
+DROP TABLE distinct_limit_window_table;
+
#
# regr_*() tests
#