Skip to content

Apache DataFusion SQL Query Engine

feat: support `SELECT DISTINCT id FROM t ORDER BY id LIMIT n` query use GroupedTopKAggregateStream (#19653)

Huaijin0aab6a3d2f67
diff --git a/datafusion/core/benches/topk_aggregate.rs b/datafusion/core/benches/topk_aggregate.rs
index 7979efdec6..f71cf1087b 100644
--- a/datafusion/core/benches/topk_aggregate.rs
+++ b/datafusion/core/benches/topk_aggregate.rs
@@ -17,6 +17,9 @@
mod data_utils;
+use arrow::array::Int64Builder;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow::record_batch::RecordBatch;
use arrow::util::pretty::pretty_format_batches;
use criterion::{Criterion, criterion_group, criterion_main};
use data_utils::make_data;
@@ -24,12 +27,53 @@ use datafusion::physical_plan::{collect, displayable};
use datafusion::prelude::SessionContext;
use datafusion::{datasource::MemTable, error::Result};
use datafusion_execution::config::SessionConfig;
+use rand::SeedableRng;
+use rand::seq::SliceRandom;
use std::hint::black_box;
use std::sync::Arc;
use tokio::runtime::Runtime;
const LIMIT: usize = 10;
+/// Create deterministic data for DISTINCT benchmarks with predictable trace_ids
+/// This ensures consistent results across benchmark runs
+fn make_distinct_data(
+ partition_cnt: i32,
+ sample_cnt: i32,
+) -> Result<(Arc<Schema>, Vec<Vec<RecordBatch>>)> {
+ let mut rng = rand::rngs::SmallRng::from_seed([42; 32]);
+ let total_samples = partition_cnt as usize * sample_cnt as usize;
+ let mut ids = Vec::new();
+ for i in 0..total_samples {
+ ids.push(i as i64);
+ }
+ ids.shuffle(&mut rng);
+
+ let mut global_idx = 0;
+ let schema = test_distinct_schema();
+ let mut partitions = vec![];
+ for _ in 0..partition_cnt {
+ let mut id_builder = Int64Builder::new();
+
+ for _ in 0..sample_cnt {
+ let id = ids[global_idx];
+ id_builder.append_value(id);
+ global_idx += 1;
+ }
+
+ let id_col = Arc::new(id_builder.finish());
+ let batch = RecordBatch::try_new(schema.clone(), vec![id_col])?;
+ partitions.push(vec![batch]);
+ }
+
+ Ok((schema, partitions))
+}
+
+/// Returns a Schema for distinct benchmarks with i64 trace_id
+fn test_distinct_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]))
+}
+
async fn create_context(
partition_cnt: i32,
sample_cnt: i32,
@@ -50,6 +94,25 @@ async fn create_context(
Ok(ctx)
}
+async fn create_context_distinct(
+ partition_cnt: i32,
+ sample_cnt: i32,
+ use_topk: bool,
+) -> Result<SessionContext> {
+ // Use deterministic data generation for DISTINCT queries to ensure consistent results
+ let (schema, parts) = make_distinct_data(partition_cnt, sample_cnt).unwrap();
+ let mem_table = Arc::new(MemTable::try_new(schema, parts).unwrap());
+
+ // Create the DataFrame
+ let mut cfg = SessionConfig::new();
+ let opts = cfg.options_mut();
+ opts.optimizer.enable_topk_aggregation = use_topk;
+ let ctx = SessionContext::new_with_config(cfg);
+ let _ = ctx.register_table("traces", mem_table)?;
+
+ Ok(ctx)
+}
+
fn run(rt: &Runtime, ctx: SessionContext, limit: usize, use_topk: bool, asc: bool) {
black_box(rt.block_on(async { aggregate(ctx, limit, use_topk, asc).await })).unwrap();
}
@@ -59,6 +122,17 @@ fn run_string(rt: &Runtime, ctx: SessionContext, limit: usize, use_topk: bool) {
.unwrap();
}
+fn run_distinct(
+ rt: &Runtime,
+ ctx: SessionContext,
+ limit: usize,
+ use_topk: bool,
+ asc: bool,
+) {
+ black_box(rt.block_on(async { aggregate_distinct(ctx, limit, use_topk, asc).await }))
+ .unwrap();
+}
+
async fn aggregate(
ctx: SessionContext,
limit: usize,
@@ -133,6 +207,84 @@ async fn aggregate_string(
Ok(())
}
+async fn aggregate_distinct(
+ ctx: SessionContext,
+ limit: usize,
+ use_topk: bool,
+ asc: bool,
+) -> Result<()> {
+ let order_direction = if asc { "asc" } else { "desc" };
+ let sql = format!(
+ "select id from traces group by id order by id {order_direction} limit {limit};"
+ );
+ let df = ctx.sql(sql.as_str()).await?;
+ let plan = df.create_physical_plan().await?;
+ let actual_phys_plan = displayable(plan.as_ref()).indent(true).to_string();
+ assert_eq!(
+ actual_phys_plan.contains(&format!("lim=[{limit}]")),
+ use_topk
+ );
+ let batches = collect(plan, ctx.task_ctx()).await?;
+ assert_eq!(batches.len(), 1);
+ let batch = batches.first().unwrap();
+ assert_eq!(batch.num_rows(), LIMIT);
+
+ let actual = format!("{}", pretty_format_batches(&batches)?).to_lowercase();
+
+ let expected_asc = r#"
++----+
+| id |
++----+
+| 0 |
+| 1 |
+| 2 |
+| 3 |
+| 4 |
+| 5 |
+| 6 |
+| 7 |
+| 8 |
+| 9 |
++----+
+"#
+ .trim();
+
+ let expected_desc = r#"
++---------+
+| id |
++---------+
+| 9999999 |
+| 9999998 |
+| 9999997 |
+| 9999996 |
+| 9999995 |
+| 9999994 |
+| 9999993 |
+| 9999992 |
+| 9999991 |
+| 9999990 |
++---------+
+"#
+ .trim();
+
+ // Verify exact results match expected values
+ if asc {
+ assert_eq!(
+ actual.trim(),
+ expected_asc,
+ "Ascending DISTINCT results do not match expected values"
+ );
+ } else {
+ assert_eq!(
+ actual.trim(),
+ expected_desc,
+ "Descending DISTINCT results do not match expected values"
+ );
+ }
+
+ Ok(())
+}
+
fn criterion_benchmark(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let limit = LIMIT;
@@ -253,6 +405,37 @@ fn criterion_benchmark(c: &mut Criterion) {
.as_str(),
|b| b.iter(|| run_string(&rt, ctx.clone(), limit, true)),
);
+
+ // DISTINCT benchmarks
+ let ctx = rt.block_on(async {
+ create_context_distinct(partitions, samples, false)
+ .await
+ .unwrap()
+ });
+ c.bench_function(
+ format!("distinct {} rows desc [no TopK]", partitions * samples).as_str(),
+ |b| b.iter(|| run_distinct(&rt, ctx.clone(), limit, false, false)),
+ );
+
+ c.bench_function(
+ format!("distinct {} rows asc [no TopK]", partitions * samples).as_str(),
+ |b| b.iter(|| run_distinct(&rt, ctx.clone(), limit, false, true)),
+ );
+
+ let ctx_topk = rt.block_on(async {
+ create_context_distinct(partitions, samples, true)
+ .await
+ .unwrap()
+ });
+ c.bench_function(
+ format!("distinct {} rows desc [TopK]", partitions * samples).as_str(),
+ |b| b.iter(|| run_distinct(&rt, ctx_topk.clone(), limit, true, false)),
+ );
+
+ c.bench_function(
+ format!("distinct {} rows asc [TopK]", partitions * samples).as_str(),
+ |b| b.iter(|| run_distinct(&rt, ctx_topk.clone(), limit, true, true)),
+ );
}
criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/core/tests/execution/coop.rs b/datafusion/core/tests/execution/coop.rs
index b7c06e7804..528703ba78 100644
--- a/datafusion/core/tests/execution/coop.rs
+++ b/datafusion/core/tests/execution/coop.rs
@@ -24,7 +24,7 @@ use datafusion::physical_expr::aggregate::AggregateExprBuilder;
use datafusion::physical_plan;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::physical_plan::aggregates::{
- AggregateExec, AggregateMode, PhysicalGroupBy,
+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
};
use datafusion::physical_plan::execution_plan::Boundedness;
use datafusion::prelude::SessionContext;
@@ -233,6 +233,7 @@ async fn agg_grouped_topk_yields(
#[values(false, true)] pretend_infinite: bool,
) -> Result<(), Box<dyn Error>> {
// build session
+
let session_ctx = SessionContext::new();
// set up a top-k aggregation
@@ -260,7 +261,7 @@ async fn agg_grouped_topk_yields(
inf.clone(),
inf.schema(),
)?
- .with_limit(Some(100)),
+ .with_limit_options(Some(LimitOptions::new(100))),
);
query_yields(aggr, session_ctx.task_ctx()).await
diff --git a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs
index 2fdfece2a8..9e63c341c9 100644
--- a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs
+++ b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs
@@ -37,7 +37,7 @@ use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::aggregates::{
- AggregateExec, AggregateMode, PhysicalGroupBy,
+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
};
use datafusion_physical_plan::displayable;
use datafusion_physical_plan::repartition::RepartitionExec;
@@ -260,7 +260,7 @@ fn aggregations_with_limit_combined() -> datafusion_common::Result<()> {
schema,
)
.unwrap()
- .with_limit(Some(5)),
+ .with_limit_options(Some(LimitOptions::new(5))),
);
let plan: Arc<dyn ExecutionPlan> = final_agg;
// should combine the Partial/Final AggregateExecs to a Single AggregateExec
diff --git a/datafusion/physical-optimizer/src/combine_partial_final_agg.rs b/datafusion/physical-optimizer/src/combine_partial_final_agg.rs
index 782e0754b7..6d8e7995c1 100644
--- a/datafusion/physical-optimizer/src/combine_partial_final_agg.rs
+++ b/datafusion/physical-optimizer/src/combine_partial_final_agg.rs
@@ -98,7 +98,9 @@ impl PhysicalOptimizerRule for CombinePartialFinalAggregate {
Arc::clone(input_agg_exec.input()),
input_agg_exec.input_schema(),
)
- .map(|combined_agg| combined_agg.with_limit(agg_exec.limit()))
+ .map(|combined_agg| {
+ combined_agg.with_limit_options(agg_exec.limit_options())
+ })
.ok()
.map(Arc::new)
} else {
diff --git a/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs b/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs
index 671d247cf3..fe9636f676 100644
--- a/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs
+++ b/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs
@@ -20,7 +20,7 @@
use std::sync::Arc;
-use datafusion_physical_plan::aggregates::AggregateExec;
+use datafusion_physical_plan::aggregates::{AggregateExec, LimitOptions};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
@@ -63,7 +63,7 @@ impl LimitedDistinctAggregation {
aggr.input_schema(),
)
.expect("Unable to copy Aggregate!")
- .with_limit(Some(limit));
+ .with_limit_options(Some(LimitOptions::new(limit)));
Some(Arc::new(new_aggr))
}
diff --git a/datafusion/physical-optimizer/src/topk_aggregation.rs b/datafusion/physical-optimizer/src/topk_aggregation.rs
index 7b2983ee71..cec6bd70a2 100644
--- a/datafusion/physical-optimizer/src/topk_aggregation.rs
+++ b/datafusion/physical-optimizer/src/topk_aggregation.rs
@@ -25,6 +25,7 @@ use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_plan::ExecutionPlan;
+use datafusion_physical_plan::aggregates::LimitOptions;
use datafusion_physical_plan::aggregates::{AggregateExec, topk_types_supported};
use datafusion_physical_plan::execution_plan::CardinalityEffect;
use datafusion_physical_plan::projection::ProjectionExec;
@@ -47,28 +48,47 @@ impl TopKAggregation {
order_desc: bool,
limit: usize,
) -> Option<Arc<dyn ExecutionPlan>> {
- // ensure the sort direction matches aggregate function
- let (field, desc) = aggr.get_minmax_desc()?;
- if desc != order_desc {
- return None;
- }
- let group_key = aggr.group_expr().expr().iter().exactly_one().ok()?;
- let kt = group_key.0.data_type(&aggr.input().schema()).ok()?;
- let vt = field.data_type();
- if !topk_types_supported(&kt, vt) {
+ // Current only support single group key
+ let (group_key, group_key_alias) =
+ aggr.group_expr().expr().iter().exactly_one().ok()?;
+ let kt = group_key.data_type(&aggr.input().schema()).ok()?;
+ let vt = if let Some((field, _)) = aggr.get_minmax_desc() {
+ field.data_type().clone()
+ } else {
+ kt.clone()
+ };
+ if !topk_types_supported(&kt, &vt) {
return None;
}
if aggr.filter_expr().iter().any(|e| e.is_some()) {
return None;
}
- // ensure the sort is on the same field as the aggregate output
- if order_by != field.name() {
+ // Check if this is ordering by an aggregate function (MIN/MAX)
+ if let Some((field, desc)) = aggr.get_minmax_desc() {
+ // ensure the sort direction matches aggregate function
+ if desc != order_desc {
+ return None;
+ }
+ // ensure the sort is on the same field as the aggregate output
+ if order_by != field.name() {
+ return None;
+ }
+ } else if aggr.aggr_expr().is_empty() {
+ // This is a GROUP BY without aggregates, check if ordering is on the group key itself
+ if order_by != group_key_alias {
+ return None;
+ }
+ } else {
+ // Has aggregates but not MIN/MAX, or doesn't DISTINCT
return None;
}
// We found what we want: clone, copy the limit down, and return modified node
- let new_aggr = aggr.with_new_limit(Some(limit));
+ let new_aggr = AggregateExec::with_new_limit_options(
+ aggr,
+ Some(LimitOptions::new_with_order(limit, order_desc)),
+ );
Some(Arc::new(new_aggr))
}
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 4dd9482ac4..d645f5c55d 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -502,6 +502,42 @@ enum DynamicFilterAggregateType {
Max,
}
+/// Configuration for limit-based optimizations in aggregation
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct LimitOptions {
+ /// The maximum number of rows to return
+ pub limit: usize,
+ /// Optional ordering direction (true = descending, false = ascending)
+ /// This is used for TopK aggregation to maintain a priority queue with the correct ordering
+ pub descending: Option<bool>,
+}
+
+impl LimitOptions {
+ /// Create a new LimitOptions with a limit and no specific ordering
+ pub fn new(limit: usize) -> Self {
+ Self {
+ limit,
+ descending: None,
+ }
+ }
+
+ /// Create a new LimitOptions with a limit and ordering direction
+ pub fn new_with_order(limit: usize, descending: bool) -> Self {
+ Self {
+ limit,
+ descending: Some(descending),
+ }
+ }
+
+ pub fn limit(&self) -> usize {
+ self.limit
+ }
+
+ pub fn descending(&self) -> Option<bool> {
+ self.descending
+ }
+}
+
/// Hash aggregate execution plan
#[derive(Debug, Clone)]
pub struct AggregateExec {
@@ -513,8 +549,8 @@ pub struct AggregateExec {
aggr_expr: Vec<Arc<AggregateFunctionExpr>>,
/// FILTER (WHERE clause) expression for each aggregate expression
filter_expr: Vec<Option<Arc<dyn PhysicalExpr>>>,
- /// Set if the output of this aggregation is truncated by a upstream sort/limit clause
- limit: Option<usize>,
+ /// Configuration for limit-based optimizations
+ limit_options: Option<LimitOptions>,
/// Input plan, could be a partial aggregate or the input to the aggregate
pub input: Arc<dyn ExecutionPlan>,
/// Schema after the aggregate is applied
@@ -558,7 +594,7 @@ impl AggregateExec {
mode: self.mode,
group_by: self.group_by.clone(),
filter_expr: self.filter_expr.clone(),
- limit: self.limit,
+ limit_options: self.limit_options,
input: Arc::clone(&self.input),
schema: Arc::clone(&self.schema),
input_schema: Arc::clone(&self.input_schema),
@@ -567,9 +603,9 @@ impl AggregateExec {
}
/// Clone this exec, overriding only the limit hint.
- pub fn with_new_limit(&self, limit: Option<usize>) -> Self {
+ pub fn with_new_limit_options(&self, limit_options: Option<LimitOptions>) -> Self {
Self {
- limit,
+ limit_options,
// clone the rest of the fields
required_input_ordering: self.required_input_ordering.clone(),
metrics: ExecutionPlanMetricsSet::new(),
@@ -709,7 +745,7 @@ impl AggregateExec {
input_schema,
metrics: ExecutionPlanMetricsSet::new(),
required_input_ordering,
- limit: None,
+ limit_options: None,
input_order_mode,
cache,
dynamic_filter: None,
@@ -725,11 +761,17 @@ impl AggregateExec {
&self.mode
}
- /// Set the `limit` of this AggExec
- pub fn with_limit(mut self, limit: Option<usize>) -> Self {
- self.limit = limit;
+ /// Set the limit options for this AggExec
+ pub fn with_limit_options(mut self, limit_options: Option<LimitOptions>) -> Self {
+ self.limit_options = limit_options;
self
}
+
+ /// Get the limit options (if set)
+ pub fn limit_options(&self) -> Option<LimitOptions> {
+ self.limit_options
+ }
+
/// Grouping expressions
pub fn group_expr(&self) -> &PhysicalGroupBy {
&self.group_by
@@ -760,11 +802,6 @@ impl AggregateExec {
Arc::clone(&self.input_schema)
}
- /// number of rows soft limit of the AggregateExec
- pub fn limit(&self) -> Option<usize> {
- self.limit
- }
-
fn execute_typed(
&self,
partition: usize,
@@ -777,11 +814,11 @@ impl AggregateExec {
}
// grouping by an expression that has a sort/limit upstream
- if let Some(limit) = self.limit
+ if let Some(config) = self.limit_options
&& !self.is_unordered_unfiltered_group_by_distinct()
{
return Ok(StreamType::GroupedPriorityQueue(
- GroupedTopKAggregateStream::new(self, context, partition, limit)?,
+ GroupedTopKAggregateStream::new(self, context, partition, config.limit)?,
));
}
@@ -802,6 +839,13 @@ impl AggregateExec {
/// This method qualifies the use of the LimitedDistinctAggregation rewrite rule
/// on an AggregateExec.
pub fn is_unordered_unfiltered_group_by_distinct(&self) -> bool {
+ if self
+ .limit_options()
+ .and_then(|config| config.descending)
+ .is_some()
+ {
+ return false;
+ }
// ensure there is a group by
if self.group_expr().is_empty() && !self.group_expr().has_grouping_set() {
return false;
@@ -1119,8 +1163,8 @@ impl DisplayAs for AggregateExec {
.map(|agg| agg.name().to_string())
.collect();
write!(f, ", aggr=[{}]", a.join(", "))?;
- if let Some(limit) = self.limit {
- write!(f, ", lim=[{limit}]")?;
+ if let Some(config) = self.limit_options {
+ write!(f, ", lim=[{}]", config.limit)?;
}
if self.input_order_mode != InputOrderMode::Linear {
@@ -1179,6 +1223,9 @@ impl DisplayAs for AggregateExec {
if !a.is_empty() {
writeln!(f, "aggr={}", a.join(", "))?;
}
+ if let Some(config) = self.limit_options {
+ writeln!(f, "limit={}", config.limit)?;
+ }
}
}
Ok(())
@@ -1247,7 +1294,7 @@ impl ExecutionPlan for AggregateExec {
Arc::clone(&self.input_schema),
Arc::clone(&self.schema),
)?;
- me.limit = self.limit;
+ me.limit_options = self.limit_options;
me.dynamic_filter = self.dynamic_filter.clone();
Ok(Arc::new(me))
diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rs
index 1ae7202711..49ce125e73 100644
--- a/datafusion/physical-plan/src/aggregates/row_hash.rs
+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs
@@ -679,7 +679,7 @@ impl GroupedHashAggregateStream {
group_ordering,
input_done: false,
spill_state,
- group_values_soft_limit: agg.limit,
+ group_values_soft_limit: agg.limit_options().map(|config| config.limit()),
skip_aggregation_probe,
reduction_factor,
})
diff --git a/datafusion/physical-plan/src/aggregates/topk_stream.rs b/datafusion/physical-plan/src/aggregates/topk_stream.rs
index a43b5cff12..72c5d0c867 100644
--- a/datafusion/physical-plan/src/aggregates/topk_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/topk_stream.rs
@@ -19,6 +19,7 @@
use crate::aggregates::group_values::GroupByMetrics;
use crate::aggregates::topk::priority_map::PriorityMap;
+#[cfg(debug_assertions)]
use crate::aggregates::topk_types_supported;
use crate::aggregates::{
AggregateExec, PhysicalGroupBy, aggregate_expressions, evaluate_group_by,
@@ -33,6 +34,7 @@ use datafusion_common::Result;
use datafusion_common::internal_datafusion_err;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::PhysicalExpr;
+use datafusion_physical_expr_common::metrics::RecordOutput;
use futures::stream::{Stream, StreamExt};
use log::{Level, trace};
use std::pin::Pin;
@@ -66,13 +68,27 @@ impl GroupedTopKAggregateStream {
let group_by_metrics = GroupByMetrics::new(&aggr.metrics, partition);
let aggregate_arguments =
aggregate_expressions(&aggr.aggr_expr, &aggr.mode, group_by.expr.len())?;
- let (val_field, desc) = aggr
- .get_minmax_desc()
- .ok_or_else(|| internal_datafusion_err!("Min/max required"))?;
let (expr, _) = &aggr.group_expr().expr()[0];
let kt = expr.data_type(&aggr.input().schema())?;
- let vt = val_field.data_type().clone();
+
+ // Check if this is a MIN/MAX aggregate or a DISTINCT-like operation
+ let (vt, desc) = if let Some((val_field, desc)) = aggr.get_minmax_desc() {
+ // MIN/MAX case: use the aggregate output type
+ (val_field.data_type().clone(), desc)
+ } else {
+ // DISTINCT case: use the group key type and get ordering from limit_order_descending
+ // The ordering direction is set by the optimizer when it pushes down the limit
+ let desc = aggr
+ .limit_options()
+ .and_then(|config| config.descending)
+ .ok_or_else(|| {
+ internal_datafusion_err!(
+ "Ordering direction required for DISTINCT with limit"
+ )
+ })?;
+ (kt.clone(), desc)
+ };
// Type validation is performed by the optimizer and can_use_topk() check.
// This debug assertion documents the contract without runtime overhead in release builds.
@@ -168,18 +184,21 @@ impl Stream for GroupedTopKAggregateStream {
"Exactly 1 group value required"
);
let group_by_values = Arc::clone(&group_by_values[0][0]);
- let input_values = {
- let _timer = (!self.aggregate_arguments.is_empty()).then(|| {
- self.group_by_metrics.aggregate_arguments_time.timer()
- });
- evaluate_many(
+ let input_values = if self.aggregate_arguments.is_empty() {
+ // DISTINCT case: use group key as both key and value
+ Arc::clone(&group_by_values)
+ } else {
+ // MIN/MAX case: evaluate aggregate expressions
+ let _timer =
+ self.group_by_metrics.aggregate_arguments_time.timer();
+ let input_values = evaluate_many(
&self.aggregate_arguments,
batches.first().unwrap(),
- )?
+ )?;
+ assert_eq!(input_values.len(), 1, "Exactly 1 input required");
+ assert_eq!(input_values[0].len(), 1, "Exactly 1 input required");
+ Arc::clone(&input_values[0][0])
};
- assert_eq!(input_values.len(), 1, "Exactly 1 input required");
- assert_eq!(input_values[0].len(), 1, "Exactly 1 input required");
- let input_values = Arc::clone(&input_values[0][0]);
// iterate over each column of group_by values
(*self).intern(&group_by_values, &input_values)?;
@@ -192,9 +211,15 @@ impl Stream for GroupedTopKAggregateStream {
}
let batch = {
let _timer = emitting_time.timer();
- let cols = self.priority_map.emit()?;
+ let mut cols = self.priority_map.emit()?;
+ // For DISTINCT case (no aggregate expressions), only use the group key column
+ // since the schema only has one field and key/value are the same
+ if self.aggregate_arguments.is_empty() {
+ cols.truncate(1);
+ }
RecordBatch::try_new(Arc::clone(&self.schema), cols)?
};
+ let batch = batch.record_output(&self.baseline_metrics);
trace!(
"partition {} emit batch with {} rows",
self.partition,
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index 5f590560c4..b035a4673b 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -1220,6 +1220,8 @@ message MaybePhysicalSortExprs {
message AggLimit {
// wrap into a message to make it optional
uint64 limit = 1;
+ // Optional ordering direction for TopK aggregation (true = descending, false = ascending)
+ optional bool descending = 2;
}
message AggregateExecNode {
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index f6d364f269..d6af3a6270 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -9,12 +9,18 @@ impl serde::Serialize for AggLimit {
if self.limit != 0 {
len += 1;
}
+ if self.descending.is_some() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.AggLimit", len)?;
if self.limit != 0 {
#[allow(clippy::needless_borrow)]
#[allow(clippy::needless_borrows_for_generic_args)]
struct_ser.serialize_field("limit", ToString::to_string(&self.limit).as_str())?;
}
+ if let Some(v) = self.descending.as_ref() {
+ struct_ser.serialize_field("descending", v)?;
+ }
struct_ser.end()
}
}
@@ -26,11 +32,13 @@ impl<'de> serde::Deserialize<'de> for AggLimit {
{
const FIELDS: &[&str] = &[
"limit",
+ "descending",
];
#[allow(clippy::enum_variant_names)]
enum GeneratedField {
Limit,
+ Descending,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -53,6 +61,7 @@ impl<'de> serde::Deserialize<'de> for AggLimit {
{
match value {
"limit" => Ok(GeneratedField::Limit),
+ "descending" => Ok(GeneratedField::Descending),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -73,6 +82,7 @@ impl<'de> serde::Deserialize<'de> for AggLimit {
V: serde::de::MapAccess<'de>,
{
let mut limit__ = None;
+ let mut descending__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Limit => {
@@ -83,10 +93,17 @@ impl<'de> serde::Deserialize<'de> for AggLimit {
Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)
;
}
+ GeneratedField::Descending => {
+ if descending__.is_some() {
+ return Err(serde::de::Error::duplicate_field("descending"));
+ }
+ descending__ = map_.next_value()?;
+ }
}
}
Ok(AggLimit {
limit: limit__.unwrap_or_default(),
+ descending: descending__,
})
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index c1afd73ec3..c6110865e2 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1832,6 +1832,9 @@ pub struct AggLimit {
/// wrap into a message to make it optional
#[prost(uint64, tag = "1")]
pub limit: u64,
+ /// Optional ordering direction for TopK aggregation (true = descending, false = ascending)
+ #[prost(bool, optional, tag = "2")]
+ pub descending: ::core::option::Option<bool>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct AggregateExecNode {
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 45868df4ce..39472c7794 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -71,8 +71,8 @@ use datafusion_functions_table::generate_series::{
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};
-use datafusion_physical_plan::aggregates::AggregateMode;
use datafusion_physical_plan::aggregates::{AggregateExec, PhysicalGroupBy};
+use datafusion_physical_plan::aggregates::{AggregateMode, LimitOptions};
use datafusion_physical_plan::analyze::AnalyzeExec;
#[expect(deprecated)]
use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;
@@ -1105,11 +1105,6 @@ impl protobuf::PhysicalPlanNode {
})
.collect::<Result<Vec<_>, _>>()?;
- let limit = hash_agg
- .limit
- .as_ref()
- .map(|lit_value| lit_value.limit as usize);
-
let agg = AggregateExec::try_new(
agg_mode,
PhysicalGroupBy::new(group_expr, null_expr, groups, has_grouping_set),
@@ -1119,7 +1114,16 @@ impl protobuf::PhysicalPlanNode {
physical_schema,
)?;
- let agg = agg.with_limit(limit);
+ let agg = if let Some(limit_proto) = &hash_agg.limit {
+ let limit = limit_proto.limit as usize;
+ let limit_options = match limit_proto.descending {
+ Some(descending) => LimitOptions::new_with_order(limit, descending),
+ None => LimitOptions::new(limit),
+ };
+ agg.with_limit_options(Some(limit_options))
+ } else {
+ agg
+ };
Ok(Arc::new(agg))
}
@@ -2529,8 +2533,9 @@ impl protobuf::PhysicalPlanNode {
.map(|expr| serialize_physical_expr(&expr.0, extension_codec))
.collect::<Result<Vec<_>>>()?;
- let limit = exec.limit().map(|value| protobuf::AggLimit {
- limit: value as u64,
+ let limit = exec.limit_options().map(|config| protobuf::AggLimit {
+ limit: config.limit() as u64,
+ descending: config.descending(),
});
Ok(protobuf::PhysicalPlanNode {
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index 31878e2e34..728d15d6c6 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -70,7 +70,7 @@ use datafusion::physical_expr::{
LexOrdering, PhysicalSortRequirement, ScalarFunctionExpr,
};
use datafusion::physical_plan::aggregates::{
- AggregateExec, AggregateMode, PhysicalGroupBy,
+ AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,
};
use datafusion::physical_plan::analyze::AnalyzeExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
@@ -616,7 +616,7 @@ fn roundtrip_aggregate_with_limit() -> Result<()> {
Arc::new(EmptyExec::new(schema.clone())),
schema,
)?;
- let agg = agg.with_limit(Some(12));
+ let agg = agg.with_limit_options(Some(LimitOptions::new_with_order(12, false)));
roundtrip_test(Arc::new(agg))
}
diff --git a/datafusion/sqllogictest/test_files/aggregates_topk.slt b/datafusion/sqllogictest/test_files/aggregates_topk.slt
index 05f3e02bbc..19ead8965e 100644
--- a/datafusion/sqllogictest/test_files/aggregates_topk.slt
+++ b/datafusion/sqllogictest/test_files/aggregates_topk.slt
@@ -344,5 +344,123 @@ physical_plan
06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+## Test GROUP BY with ORDER BY on the same column (no aggregate functions)
+statement ok
+CREATE TABLE ids(id int, value int) AS VALUES
+(1, 10),
+(2, 20),
+(3, 30),
+(4, 40),
+(1, 50),
+(2, 60),
+(5, 70);
+
+query TT
+explain select id from ids group by id order by id desc limit 3;
+----
+logical_plan
+01)Sort: ids.id DESC NULLS FIRST, fetch=3
+02)--Aggregate: groupBy=[[ids.id]], aggr=[[]]
+03)----TableScan: ids projection=[id]
+physical_plan
+01)SortPreservingMergeExec: [id@0 DESC], fetch=3
+02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]
+03)----AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[], lim=[3]
+04)------RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=1
+05)--------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[], lim=[3]
+06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+select id from ids group by id order by id desc limit 3;
+----
+5
+4
+3
+
+query TT
+explain select id from ids group by id order by id asc limit 2;
+----
+logical_plan
+01)Sort: ids.id ASC NULLS LAST, fetch=2
+02)--Aggregate: groupBy=[[ids.id]], aggr=[[]]
+03)----TableScan: ids projection=[id]
+physical_plan
+01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=2
+02)--SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[], lim=[2]
+04)------RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=1
+05)--------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[], lim=[2]
+06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+select id from ids group by id order by id asc limit 2;
+----
+1
+2
+
+# Test with larger limit than distinct values
+query I
+select id from ids group by id order by id desc limit 100;
+----
+5
+4
+3
+2
+1
+
+# Test with bigint group by
+statement ok
+CREATE TABLE values_table (value INT, category BIGINT) AS VALUES
+(10, 100),
+(20, 200),
+(30, 300),
+(40, 400),
+(50, 500),
+(20, 200),
+(10, 100),
+(40, 400);
+
+query TT
+explain select category from values_table group by category order by category desc limit 3;
+----
+logical_plan
+01)Sort: values_table.category DESC NULLS FIRST, fetch=3
+02)--Aggregate: groupBy=[[values_table.category]], aggr=[[]]
+03)----TableScan: values_table projection=[category]
+physical_plan
+01)SortPreservingMergeExec: [category@0 DESC], fetch=3
+02)--SortExec: TopK(fetch=3), expr=[category@0 DESC], preserve_partitioning=[true]
+03)----AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[], lim=[3]
+04)------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=1
+05)--------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[], lim=[3]
+06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+select category from values_table group by category order by category desc limit 3;
+----
+500
+400
+300
+
+# Test with integer group by
+query I
+select value from values_table group by value order by value asc limit 3;
+----
+10
+20
+30
+
+# Test DISTINCT semantics are preserved
+query I
+select count(*) from (select category from values_table group by category order by category desc limit 3);
+----
+3
+
+statement ok
+drop table values_table;
+
+statement ok
+drop table ids;
+
statement ok
drop table traces;
diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt
index cd1ed2bc0c..db4ec83f10 100644
--- a/datafusion/sqllogictest/test_files/group_by.slt
+++ b/datafusion/sqllogictest/test_files/group_by.slt
@@ -4329,9 +4329,9 @@ physical_plan
01)SortPreservingMergeExec: [months@0 DESC], fetch=5
02)--SortExec: TopK(fetch=5), expr=[months@0 DESC], preserve_partitioning=[true]
03)----ProjectionExec: expr=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 as months]
-04)------AggregateExec: mode=FinalPartitioned, gby=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 as date_part(Utf8("MONTH"),csv_with_timestamps.ts)], aggr=[]
+04)------AggregateExec: mode=FinalPartitioned, gby=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 as date_part(Utf8("MONTH"),csv_with_timestamps.ts)], aggr=[], lim=[5]
05)--------RepartitionExec: partitioning=Hash([date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0], 8), input_partitions=8
-06)----------AggregateExec: mode=Partial, gby=[date_part(MONTH, ts@0) as date_part(Utf8("MONTH"),csv_with_timestamps.ts)], aggr=[]
+06)----------AggregateExec: mode=Partial, gby=[date_part(MONTH, ts@0) as date_part(Utf8("MONTH"),csv_with_timestamps.ts)], aggr=[], lim=[5]
07)------------RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1, maintains_sort_order=true
08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/timestamps.csv]]}, projection=[ts], output_ordering=[ts@0 DESC], file_type=csv, has_header=false