Apache DataFusion SQL Query Engine
Fix topk emitted batches metric (#24481)
Peter L54557c33d81a
diff --git a/datafusion/core/tests/physical_optimizer/window_topn.rs b/datafusion/core/tests/physical_optimizer/window_topn.rsindex a31e7a45e5..85801e84c9 100644--- a/datafusion/core/tests/physical_optimizer/window_topn.rs+++ b/datafusion/core/tests/physical_optimizer/window_topn.rs@@ -20,6 +20,7 @@use std::sync::Arc;use arrow::datatypes::{DataType, Field, Schema};+use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::Result;use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;@@ -32,10 +33,13 @@ use datafusion_physical_expr::window::StandardWindowExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::window_topn::WindowTopN;+use datafusion_physical_plan::collect;use datafusion_physical_plan::displayable;use datafusion_physical_plan::filter::FilterExec;+use datafusion_physical_plan::metrics::MetricValue;use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use datafusion_physical_plan::projection::ProjectionExec;+use datafusion_physical_plan::sorts::partitioned_topk::PartitionedTopKExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_udwf_window_expr};use datafusion_physical_plan::{ExecutionPlan, InputOrderMode};@@ -658,3 +662,68 @@ fn predicate_lt_1_no_change() -> Result<()> {}Ok(())}++// ----------------------------------------------------------------------+// Execution: metrics exposure+// ----------------------------------------------------------------------++/// Recursively finds the `PartitionedTopKExec` node in `plan`.+fn find_partitioned_topk(+ plan: &Arc<dyn ExecutionPlan>,+) -> Option<Arc<dyn ExecutionPlan>> {+ if plan.is::<PartitionedTopKExec>() {+ return Some(Arc::clone(plan));+ }+ plan.children().into_iter().find_map(find_partitioned_topk)+}++/// Regression test for #24470: `PartitionedTopKExec` builds an+/// `ExecutionPlanMetricsSet` and hands it to the top-K state, but used to omit+/// `ExecutionPlan::metrics()`, so nothing it recorded was ever observable (and+/// `EXPLAIN ANALYZE` showed no metrics for the operator).+///+/// The `output_batches` assertion also covers the emit-side counting fixed for+/// #24468: the per-partition heaps are coalesced into `batch_size` batches, so+/// the metric must count the coalesced output, not the per-partition inputs.+#[tokio::test]+async fn partitioned_topk_exec_exposes_metrics() -> Result<()> {+ let mut config = SessionConfig::new()+ .with_batch_size(10)+ .with_target_partitions(1);+ config.options_mut().optimizer.enable_window_topn = true;+ let ctx = SessionContext::new_with_config(config);++ // 10 partition keys x 5 rows each; top-5 per key keeps all 50 rows, which+ // at batch_size 10 must be emitted as 5 batches.+ ctx.sql(+ "CREATE TABLE t AS+ SELECT v % 10 AS pk, v AS val FROM (SELECT unnest(range(0, 50)) AS v)",+ )+ .await?+ .collect()+ .await?;++ let df = ctx+ .sql(+ "SELECT pk, val FROM (+ SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) AS rn FROM t+ ) WHERE rn <= 5",+ )+ .await?;+ let plan = df.create_physical_plan().await?;+ let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?;+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 50);++ let topk = find_partitioned_topk(&plan).expect("plan has a PartitionedTopKExec");+ let metrics = topk+ .metrics()+ .expect("PartitionedTopKExec should expose metrics");+ assert_eq!(metrics.output_rows(), Some(50));+ let output_batches = metrics+ .sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))+ .expect("output_batches metric")+ .as_usize();+ assert_eq!(output_batches, 5);++ Ok(())+}diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rsindex cd8bd340d2..f87a99944f 100644--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs@@ -46,7 +46,7 @@ use futures::StreamExt;use futures::TryStreamExt;use crate::execution_plan::{Boundedness, EmissionType};-use crate::metrics::ExecutionPlanMetricsSet;+use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};use crate::topk::{PartitionedTopK, PartitionedTopKDenseRank, PartitionedTopKRank, build_sort_fields,};@@ -463,6 +463,10 @@ impl ExecutionPlan for PartitionedTopKExec {stream,)))}++ fn metrics(&self) -> Option<MetricsSet> {+ Some(self.metrics_set.clone_inner())+ }}/// Read all input, feed each batch into a per-partition top-K statediff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex 030614eed0..940033857d 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -810,14 +810,15 @@ impl TopK {// break into record batches as neededlet mut batches = vec![];if let Some(mut batch) = heap.emit()? {- (&batch).record_output(&metrics.baseline);-loop {if batch.num_rows() <= batch_size {+ (&batch).record_output(&metrics.baseline);batches.push(Ok(batch));break;} else {- batches.push(Ok(batch.slice(0, batch_size)));+ let head = batch.slice(0, batch_size);+ (&head).record_output(&metrics.baseline);+ batches.push(Ok(head));let remaining_length = batch.num_rows() - batch_size;batch = batch.slice(batch_size, remaining_length);}@@ -1430,7 +1431,6 @@ impl PartitionedTopK {for pk in sorted_pks {let mut heap = heaps.remove(&pk).expect("key from heaps.keys()");if let Some(batch) = heap.emit()? {- (&batch).record_output(&metrics.baseline);coalescer.push_batch(batch)?;}}@@ -1438,6 +1438,7 @@ impl PartitionedTopK {let mut out: Vec<Result<RecordBatch>> = Vec::new();while let Some(b) = coalescer.next_completed_batch() {+ (&b).record_output(&metrics.baseline);out.push(Ok(b));}@@ -1775,13 +1776,11 @@ impl PartitionedTopKRank {let RankPartitionState { mut heap, ties } =states.remove(&pk).expect("key from states.keys()");if let Some(batch) = heap.emit()? {- (&batch).record_output(&metrics.baseline);coalescer.push_batch(batch)?;}for tie in ties {let indices = UInt32Array::from(tie.row_indices);let tie_batch = take_record_batch(&tie.batch, &indices)?;- (&tie_batch).record_output(&metrics.baseline);coalescer.push_batch(tie_batch)?;}}@@ -1789,6 +1788,7 @@ impl PartitionedTopKRank {let mut out: Vec<Result<RecordBatch>> = Vec::new();while let Some(b) = coalescer.next_completed_batch() {+ (&b).record_output(&metrics.baseline);out.push(Ok(b));}@@ -2206,7 +2206,6 @@ impl PartitionedTopKDenseRank {.batch;let indices = UInt32Array::from(entry.row_indices);let sub = take_record_batch(batch, &indices)?;- (&sub).record_output(&metrics.baseline);coalescer.push_batch(sub)?;}}@@ -2215,6 +2214,7 @@ impl PartitionedTopKDenseRank {let mut out: Vec<Result<RecordBatch>> = Vec::new();while let Some(b) = coalescer.next_completed_batch() {+ (&b).record_output(&metrics.baseline);out.push(Ok(b));}@@ -2258,6 +2258,7 @@ impl PartitionedTopKDenseRank {#[cfg(test)]mod tests {use super::*;+ use crate::metrics::MetricValue;use arrow::array::{BooleanArray, Float64Array, Int32Array, StringArray};use arrow::datatypes::{DataType, Field, Schema};use arrow_schema::SortOptions;@@ -2361,6 +2362,54 @@ mod tests {)}+ /// Reads the `output_batches` and `output_rows` metrics recorded by an emit.+ fn output_batches_and_rows(metrics: &ExecutionPlanMetricsSet) -> (usize, usize) {+ let metrics = metrics.clone_inner();+ let batches = metrics+ .sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))+ .expect("output_batches metric")+ .as_usize();+ (batches, metrics.output_rows().expect("output_rows metric"))+ }++ /// Regression test for #24468: `emit` splits the heap's single batch into+ /// `batch_size` chunks, so `output_batches` must count each emitted chunk+ /// rather than the one pre-split batch.+ #[tokio::test]+ async fn test_topk_output_batches_metric_counts_emitted_batches() -> Result<()> {+ let schema = make_ab_schema();+ let metrics = ExecutionPlanMetricsSet::new();+ let sort_expr = PhysicalSortExpr {+ expr: col("a", schema.as_ref())?,+ options: SortOptions::default(),+ };+ // k = 5 with batch_size = 2 => emitted batches of [2, 2, 1]+ let mut topk = TopK::try_new(+ 0,+ Arc::clone(&schema),+ vec![],+ LexOrdering::from([sort_expr]),+ 5,+ 2,+ Arc::new(RuntimeEnv::default()),+ &metrics,+ make_topk_filter(),+ )?;++ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(5), Some(4), Some(3), Some(2), Some(1), Some(0)],+ &[0.0, 1.0, 2.0, 3.0, 4.0, 5.0],+ )?)?;++ let results: Vec<_> = topk.emit()?.try_collect().await?;+ let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();+ assert_eq!(row_counts, vec![2, 2, 1]);+ assert_eq!(output_batches_and_rows(&metrics), (3, 5));++ Ok(())+ }+fn make_ab_batch(schema: SchemaRef,a: &[Option<i32>],@@ -2980,6 +3029,56 @@ mod tests {)?)}+ /// Companion to [`test_topk_output_batches_metric_counts_emitted_batches`] for+ /// the partitioned emit path: rows from several per-partition heaps are+ /// coalesced into `batch_size` batches, so the metric must count the+ /// coalesced output rather than the per-partition inputs.+ #[tokio::test]+ async fn test_partitioned_topk_output_batches_metric_counts_emitted_batches()+ -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("pk", DataType::Int32, false),+ Field::new("val", DataType::Int32, false),+ ]));+ let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;+ let partition_sort_fields = build_sort_fields(+ &[PhysicalSortExpr {+ expr: Arc::clone(&pk_expr),+ options: SortOptions::default(),+ }],+ &schema,+ )?;+ let metrics = ExecutionPlanMetricsSet::new();+ // 3 per-partition heaps of 2 rows each, coalesced into 2 batches of 3+ let mut state = PartitionedTopK::try_new(+ 0,+ Arc::clone(&schema),+ vec![pk_expr],+ partition_sort_fields,+ LexOrdering::from([PhysicalSortExpr {+ expr: col("val", schema.as_ref())?,+ options: SortOptions::default(),+ }]),+ 2,+ 3, // batch_size+ &Arc::new(RuntimeEnv::default()),+ &metrics,+ )?;++ state.insert_batch(&pk_val_batch(+ &schema,+ vec![1, 1, 2, 2, 3, 3],+ vec![10, 5, 20, 15, 30, 25],+ )?)?;++ let results: Vec<_> = state.emit()?.try_collect().await?;+ let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();+ assert_eq!(row_counts, vec![3, 3]);+ assert_eq!(output_batches_and_rows(&metrics), (2, 6));++ Ok(())+ }+/// Multiple distinct partition keys interleaved within a single/// input batch — grouping rows by partition key, per-partition heap eviction,/// and partition-key-ordered emit must all behave correctly.