Skip to content

Apache DataFusion SQL Query Engine

Fix topk emitted batches metric (#24481)

Peter L54557c33d81a
diff --git a/datafusion/core/tests/physical_optimizer/window_topn.rs b/datafusion/core/tests/physical_optimizer/window_topn.rs
index a31e7a45e5..85801e84c9 100644
--- a/datafusion/core/tests/physical_optimizer/window_topn.rs
+++ b/datafusion/core/tests/physical_optimizer/window_topn.rs
@@ -20,6 +20,7 @@
use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Schema};
+use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::Result;
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
@@ -32,10 +33,13 @@ use datafusion_physical_expr::window::StandardWindowExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::window_topn::WindowTopN;
+use datafusion_physical_plan::collect;
use datafusion_physical_plan::displayable;
use datafusion_physical_plan::filter::FilterExec;
+use datafusion_physical_plan::metrics::MetricValue;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::ProjectionExec;
+use datafusion_physical_plan::sorts::partitioned_topk::PartitionedTopKExec;
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_udwf_window_expr};
use datafusion_physical_plan::{ExecutionPlan, InputOrderMode};
@@ -658,3 +662,68 @@ fn predicate_lt_1_no_change() -> Result<()> {
}
Ok(())
}
+
+// ----------------------------------------------------------------------
+// Execution: metrics exposure
+// ----------------------------------------------------------------------
+
+/// Recursively finds the `PartitionedTopKExec` node in `plan`.
+fn find_partitioned_topk(
+ plan: &Arc<dyn ExecutionPlan>,
+) -> Option<Arc<dyn ExecutionPlan>> {
+ if plan.is::<PartitionedTopKExec>() {
+ return Some(Arc::clone(plan));
+ }
+ plan.children().into_iter().find_map(find_partitioned_topk)
+}
+
+/// Regression test for #24470: `PartitionedTopKExec` builds an
+/// `ExecutionPlanMetricsSet` and hands it to the top-K state, but used to omit
+/// `ExecutionPlan::metrics()`, so nothing it recorded was ever observable (and
+/// `EXPLAIN ANALYZE` showed no metrics for the operator).
+///
+/// The `output_batches` assertion also covers the emit-side counting fixed for
+/// #24468: the per-partition heaps are coalesced into `batch_size` batches, so
+/// the metric must count the coalesced output, not the per-partition inputs.
+#[tokio::test]
+async fn partitioned_topk_exec_exposes_metrics() -> Result<()> {
+ let mut config = SessionConfig::new()
+ .with_batch_size(10)
+ .with_target_partitions(1);
+ config.options_mut().optimizer.enable_window_topn = true;
+ let ctx = SessionContext::new_with_config(config);
+
+ // 10 partition keys x 5 rows each; top-5 per key keeps all 50 rows, which
+ // at batch_size 10 must be emitted as 5 batches.
+ ctx.sql(
+ "CREATE TABLE t AS
+ SELECT v % 10 AS pk, v AS val FROM (SELECT unnest(range(0, 50)) AS v)",
+ )
+ .await?
+ .collect()
+ .await?;
+
+ let df = ctx
+ .sql(
+ "SELECT pk, val FROM (
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) AS rn FROM t
+ ) WHERE rn <= 5",
+ )
+ .await?;
+ let plan = df.create_physical_plan().await?;
+ let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?;
+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 50);
+
+ let topk = find_partitioned_topk(&plan).expect("plan has a PartitionedTopKExec");
+ let metrics = topk
+ .metrics()
+ .expect("PartitionedTopKExec should expose metrics");
+ assert_eq!(metrics.output_rows(), Some(50));
+ let output_batches = metrics
+ .sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))
+ .expect("output_batches metric")
+ .as_usize();
+ assert_eq!(output_batches, 5);
+
+ Ok(())
+}
diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rs
index cd8bd340d2..f87a99944f 100644
--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs
+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs
@@ -46,7 +46,7 @@ use futures::StreamExt;
use futures::TryStreamExt;
use crate::execution_plan::{Boundedness, EmissionType};
-use crate::metrics::ExecutionPlanMetricsSet;
+use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};
use crate::topk::{
PartitionedTopK, PartitionedTopKDenseRank, PartitionedTopKRank, build_sort_fields,
};
@@ -463,6 +463,10 @@ impl ExecutionPlan for PartitionedTopKExec {
stream,
)))
}
+
+ fn metrics(&self) -> Option<MetricsSet> {
+ Some(self.metrics_set.clone_inner())
+ }
}
/// Read all input, feed each batch into a per-partition top-K state
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs
index 030614eed0..940033857d 100644
--- a/datafusion/physical-plan/src/topk/mod.rs
+++ b/datafusion/physical-plan/src/topk/mod.rs
@@ -810,14 +810,15 @@ impl TopK {
// break into record batches as needed
let mut batches = vec![];
if let Some(mut batch) = heap.emit()? {
- (&batch).record_output(&metrics.baseline);
-
loop {
if batch.num_rows() <= batch_size {
+ (&batch).record_output(&metrics.baseline);
batches.push(Ok(batch));
break;
} else {
- batches.push(Ok(batch.slice(0, batch_size)));
+ let head = batch.slice(0, batch_size);
+ (&head).record_output(&metrics.baseline);
+ batches.push(Ok(head));
let remaining_length = batch.num_rows() - batch_size;
batch = batch.slice(batch_size, remaining_length);
}
@@ -1430,7 +1431,6 @@ impl PartitionedTopK {
for pk in sorted_pks {
let mut heap = heaps.remove(&pk).expect("key from heaps.keys()");
if let Some(batch) = heap.emit()? {
- (&batch).record_output(&metrics.baseline);
coalescer.push_batch(batch)?;
}
}
@@ -1438,6 +1438,7 @@ impl PartitionedTopK {
let mut out: Vec<Result<RecordBatch>> = Vec::new();
while let Some(b) = coalescer.next_completed_batch() {
+ (&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -1775,13 +1776,11 @@ impl PartitionedTopKRank {
let RankPartitionState { mut heap, ties } =
states.remove(&pk).expect("key from states.keys()");
if let Some(batch) = heap.emit()? {
- (&batch).record_output(&metrics.baseline);
coalescer.push_batch(batch)?;
}
for tie in ties {
let indices = UInt32Array::from(tie.row_indices);
let tie_batch = take_record_batch(&tie.batch, &indices)?;
- (&tie_batch).record_output(&metrics.baseline);
coalescer.push_batch(tie_batch)?;
}
}
@@ -1789,6 +1788,7 @@ impl PartitionedTopKRank {
let mut out: Vec<Result<RecordBatch>> = Vec::new();
while let Some(b) = coalescer.next_completed_batch() {
+ (&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -2206,7 +2206,6 @@ impl PartitionedTopKDenseRank {
.batch;
let indices = UInt32Array::from(entry.row_indices);
let sub = take_record_batch(batch, &indices)?;
- (&sub).record_output(&metrics.baseline);
coalescer.push_batch(sub)?;
}
}
@@ -2215,6 +2214,7 @@ impl PartitionedTopKDenseRank {
let mut out: Vec<Result<RecordBatch>> = Vec::new();
while let Some(b) = coalescer.next_completed_batch() {
+ (&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -2258,6 +2258,7 @@ impl PartitionedTopKDenseRank {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::metrics::MetricValue;
use arrow::array::{BooleanArray, Float64Array, Int32Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow_schema::SortOptions;
@@ -2361,6 +2362,54 @@ mod tests {
)
}
+ /// Reads the `output_batches` and `output_rows` metrics recorded by an emit.
+ fn output_batches_and_rows(metrics: &ExecutionPlanMetricsSet) -> (usize, usize) {
+ let metrics = metrics.clone_inner();
+ let batches = metrics
+ .sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))
+ .expect("output_batches metric")
+ .as_usize();
+ (batches, metrics.output_rows().expect("output_rows metric"))
+ }
+
+ /// Regression test for #24468: `emit` splits the heap's single batch into
+ /// `batch_size` chunks, so `output_batches` must count each emitted chunk
+ /// rather than the one pre-split batch.
+ #[tokio::test]
+ async fn test_topk_output_batches_metric_counts_emitted_batches() -> Result<()> {
+ let schema = make_ab_schema();
+ let metrics = ExecutionPlanMetricsSet::new();
+ let sort_expr = PhysicalSortExpr {
+ expr: col("a", schema.as_ref())?,
+ options: SortOptions::default(),
+ };
+ // k = 5 with batch_size = 2 => emitted batches of [2, 2, 1]
+ let mut topk = TopK::try_new(
+ 0,
+ Arc::clone(&schema),
+ vec![],
+ LexOrdering::from([sort_expr]),
+ 5,
+ 2,
+ Arc::new(RuntimeEnv::default()),
+ &metrics,
+ make_topk_filter(),
+ )?;
+
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(5), Some(4), Some(3), Some(2), Some(1), Some(0)],
+ &[0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
+ )?)?;
+
+ let results: Vec<_> = topk.emit()?.try_collect().await?;
+ let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();
+ assert_eq!(row_counts, vec![2, 2, 1]);
+ assert_eq!(output_batches_and_rows(&metrics), (3, 5));
+
+ Ok(())
+ }
+
fn make_ab_batch(
schema: SchemaRef,
a: &[Option<i32>],
@@ -2980,6 +3029,56 @@ mod tests {
)?)
}
+ /// Companion to [`test_topk_output_batches_metric_counts_emitted_batches`] for
+ /// the partitioned emit path: rows from several per-partition heaps are
+ /// coalesced into `batch_size` batches, so the metric must count the
+ /// coalesced output rather than the per-partition inputs.
+ #[tokio::test]
+ async fn test_partitioned_topk_output_batches_metric_counts_emitted_batches()
+ -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("pk", DataType::Int32, false),
+ Field::new("val", DataType::Int32, false),
+ ]));
+ let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
+ let partition_sort_fields = build_sort_fields(
+ &[PhysicalSortExpr {
+ expr: Arc::clone(&pk_expr),
+ options: SortOptions::default(),
+ }],
+ &schema,
+ )?;
+ let metrics = ExecutionPlanMetricsSet::new();
+ // 3 per-partition heaps of 2 rows each, coalesced into 2 batches of 3
+ let mut state = PartitionedTopK::try_new(
+ 0,
+ Arc::clone(&schema),
+ vec![pk_expr],
+ partition_sort_fields,
+ LexOrdering::from([PhysicalSortExpr {
+ expr: col("val", schema.as_ref())?,
+ options: SortOptions::default(),
+ }]),
+ 2,
+ 3, // batch_size
+ &Arc::new(RuntimeEnv::default()),
+ &metrics,
+ )?;
+
+ state.insert_batch(&pk_val_batch(
+ &schema,
+ vec![1, 1, 2, 2, 3, 3],
+ vec![10, 5, 20, 15, 30, 25],
+ )?)?;
+
+ let results: Vec<_> = state.emit()?.try_collect().await?;
+ let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();
+ assert_eq!(row_counts, vec![3, 3]);
+ assert_eq!(output_batches_and_rows(&metrics), (2, 6));
+
+ Ok(())
+ }
+
/// Multiple distinct partition keys interleaved within a single
/// input batch — grouping rows by partition key, per-partition heap eviction,
/// and partition-key-ordered emit must all behave correctly.