Apache DataFusion SQL Query Engine
fix(topk): call attempt_early_completion when filter rejects entire batch (#22852)
ajegou6520315d4185
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex 9da606dc90..8a8bfd204e 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -255,7 +255,9 @@ impl TopK {let array = filtered.into_array(num_rows)?;let mut filter = array.as_boolean().clone();if !filter.has_true() {- // nothing to filter, so no need to update+ // The heap is unchanged, but a fully rejected batch can still prove+ // that the shared sort prefix has passed the heap boundary.+ self.attempt_early_completion(&batch)?;return Ok(());}// only update the keys / rows if the filter does not match all rows@@ -1099,20 +1101,15 @@ mod tests {assert_eq!(record_batch_store.batches_size, 0);}- /// This test validates that the `try_finish` method marks the TopK operator as finished- /// when the prefix (on column "a") of the last row in the current batch is strictly greater- /// than the max top‑k row.- /// The full sort expression is defined on both columns ("a", "b"), but the input ordering is only on "a".- #[tokio::test]- async fn test_try_finish_marks_finished_with_prefix() -> Result<()> {- // Create a schema with two columns.+ /// Builds an `(a Int32, b Float64)` schema and a `TopK` with full sort+ /// `(a ASC, b ASC)`, input prefix `[a]`, `k = 3`, `batch_size = 2`. Used by+ /// the prefix-completion tests below to keep their per-scenario logic in focus.+ fn build_ab_prefix_topk() -> Result<(Arc<Schema>, TopK)> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false),Field::new("b", DataType::Float64, false),]));- // Create sort expressions.- // Full sort: first by "a", then by "b".let sort_expr_a = PhysicalSortExpr {expr: col("a", schema.as_ref())?,options: SortOptions::default(),@@ -1122,28 +1119,33 @@ mod tests {options: SortOptions::default(),};- // Input ordering uses only column "a" (a prefix of the full sort).+ // Input ordering uses only column "a" (a prefix of the full sort on (a, b)).let prefix = vec![sort_expr_a.clone()];let full_expr = LexOrdering::from([sort_expr_a, sort_expr_b]);- // Create a dummy runtime environment and metrics.- let runtime = Arc::new(RuntimeEnv::default());- let metrics = ExecutionPlanMetricsSet::new();-- // Create a TopK instance with k = 3 and batch_size = 2.- let mut topk = TopK::try_new(+ let topk = TopK::try_new(0,Arc::clone(&schema),prefix,full_expr,3,2,- runtime,- &metrics,+ Arc::new(RuntimeEnv::default()),+ &ExecutionPlanMetricsSet::new(),Arc::new(RwLock::new(TopKDynamicFilters::new(Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)),)))),)?;+ Ok((schema, topk))+ }++ /// This test validates that the `try_finish` method marks the TopK operator as finished+ /// when the prefix (on column "a") of the last row in the current batch is strictly greater+ /// than the max top‑k row.+ /// The full sort expression is defined on both columns ("a", "b"), but the input ordering is only on "a".+ #[tokio::test]+ async fn test_try_finish_marks_finished_with_prefix() -> Result<()> {+ let (schema, mut topk) = build_ab_prefix_topk()?;// Create the first batch with two columns:// Column "a": [1, 1, 2], Column "b": [20.0, 15.0, 30.0].@@ -1196,6 +1198,67 @@ mod tests {Ok(())}+ /// Regression test for #22849: a batch whose rows are entirely rejected by the+ /// heap's dynamic filter must still trigger `attempt_early_completion` when its+ /// last row's prefix is worse than the heap's worst.+ ///+ /// Before the fix, the `!filter.has_true()` short-circuit returned without calling+ /// `attempt_early_completion`. Because the heap's filter is itself derived from the+ /// heap's worst row, a batch from a strictly-worse prefix is exactly the case the+ /// filter rejects entirely — i.e. the very signal the early-exit was designed to+ /// detect was being silently dropped.+ #[tokio::test]+ async fn test_try_finish_fires_when_filter_rejects_entire_batch() -> Result<()> {+ let (schema, mut topk) = build_ab_prefix_topk()?;++ // Batch 1 fills the heap with (1, 20.0), (1, 15.0), (2, 30.0).+ // heap.max becomes (a=2, b=30.0); update_filter tightens the heap filter to+ // a < 2 OR (a = 2 AND b < 30.0).+ let array_a1: ArrayRef =+ Arc::new(Int32Array::from(vec![Some(1), Some(1), Some(2)]));+ let array_b1: ArrayRef = Arc::new(Float64Array::from(vec![20.0, 15.0, 30.0]));+ let batch1 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a1, array_b1])?;+ topk.insert_batch(batch1)?;+ assert!(+ !topk.finished,+ "Expected 'finished' to be false after batch 1 \+ (last row prefix a=2 equals heap.max prefix a=2, not strictly greater)."+ );++ // Batch 2: every row has a=3, so the heap's filter (a < 2 OR (a = 2 AND b < 30))+ // rejects every row. Before the fix, `insert_batch` would short-circuit on+ // `!filter.has_true()` and return without checking the prefix; `finished`+ // would stay false even though no future batch could improve the heap.+ let array_a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(3)]));+ let array_b2: ArrayRef = Arc::new(Float64Array::from(vec![10.0, 20.0]));+ let batch2 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a2, array_b2])?;+ topk.insert_batch(batch2)?;+ assert!(+ topk.finished,+ "Expected 'finished' to be true after batch 2 \+ (filter rejected every row, but the batch's last row prefix a=3 \+ is strictly greater than heap.max prefix a=2)."+ );++ // The emitted top-k is unchanged from after batch 1 since none of batch 2's+ // rows could improve the heap.+ let results: Vec<_> = topk.emit()?.try_collect().await?;+ assert_batches_eq!(+ &[+ "+---+------+",+ "| a | b |",+ "+---+------+",+ "| 1 | 15.0 |",+ "| 1 | 20.0 |",+ "| 2 | 30.0 |",+ "+---+------+",+ ],+ &results+ );++ Ok(())+ }+/// This test verifies that the dynamic filter is marked as complete after TopK processing finishes.#[tokio::test]async fn test_topk_marks_filter_complete() -> Result<()> {