Apache DataFusion SQL Query Engine
Fix shared TopK early exit with shared prefix threshold (#22991)
Geoffrey Claudeba67bb467a30
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex ccc675c6ef..f48bb15a7b 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -905,19 +905,51 @@ impl SortExec {self.preserve_partitioning = preserve_partitioning;Arc::make_mut(&mut self.cache).partitioning =Self::output_partitioning_helper(&self.input, self.preserve_partitioning);+ if self.fetch.is_some() {+ self.rebuild_filter_for_current_partitioning();+ }self}- /// Add or reset `self.filter` to a new `TopKDynamicFilters`.+ fn topk_emitter_count(&self) -> usize {+ self.cache.output_partitioning().partition_count()+ }++ /// Build a new shared TopK dynamic filter wrapper for this `SortExec`.fn create_filter(&self) -> Arc<RwLock<TopKDynamicFilters>> {let children = self.expr.iter().map(|sort_expr| Arc::clone(&sort_expr.expr)).collect::<Vec<_>>();- Arc::new(RwLock::new(TopKDynamicFilters::new(Arc::new(- DynamicFilterPhysicalExpr::new(children, lit(true)),- ))))+ self.create_filter_with_expr(Arc::new(DynamicFilterPhysicalExpr::new(+ children,+ lit(true),+ )))+ }++ fn create_filter_with_expr(+ &self,+ expr: Arc<DynamicFilterPhysicalExpr>,+ ) -> Arc<RwLock<TopKDynamicFilters>> {+ Arc::new(RwLock::new(+ TopKDynamicFilters::new_with_topk_emitter_count(+ expr,+ self.topk_emitter_count(),+ ),+ ))+ }++ /// Rebuild the shared TopK filter wrapper for the current output partitioning.+ ///+ /// The dynamic filter expression is preserved, but wrapper state such as the+ /// shared threshold and remaining emitter count is reset for the new+ /// partitioning.+ fn rebuild_filter_for_current_partitioning(&mut self) {+ let filter_expr = self.filter.as_ref().map(|filter| filter.read().expr());+ if let Some(filter_expr) = filter_expr {+ self.filter = Some(self.create_filter_with_expr(filter_expr));+ }}fn cloned(&self) -> Self {@@ -952,14 +984,20 @@ impl SortExec {if fetch.is_some() && is_pipeline_friendly {cache = cache.with_boundedness(Boundedness::Bounded);}- let filter = fetch.is_some().then(|| {- // If we already have a filter, keep it. Otherwise, create a new one.- self.filter.clone().unwrap_or_else(|| self.create_filter())- });let mut new_sort = self.cloned();new_sort.fetch = fetch;new_sort.cache = cache.into();- new_sort.filter = filter;+ if fetch.is_some() {+ if new_sort.filter.is_some() {+ // Keep the dynamic filter expression, but reset wrapper state+ // such as the shared threshold and expected emitter count.+ new_sort.rebuild_filter_for_current_partitioning();+ } else {+ new_sort.filter = Some(new_sort.create_filter());+ }+ } else {+ new_sort.filter = None;+ }new_sort}@@ -998,7 +1036,7 @@ impl SortExec {for child in filter.children() {child.data_type(&input_schema)?;}- self.filter = Some(Arc::new(RwLock::new(TopKDynamicFilters::new(filter))));+ self.filter = Some(self.create_filter_with_expr(filter));Ok(self)}@@ -1173,6 +1211,9 @@ impl ExecutionPlan for SortExec {)?;new_sort.cache = Arc::new(cache);new_sort.common_sort_prefix = sort_prefix;+ if new_sort.fetch.is_some() {+ new_sort.rebuild_filter_for_current_partitioning();+ }}Ok(Arc::new(new_sort))@@ -1452,8 +1493,8 @@ mod tests {GreedyMemoryPool, MemoryConsumer, MemoryPool,};use datafusion_execution::runtime_env::RuntimeEnvBuilder;- use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_expr::expressions::{Column, Literal};+ use datafusion_physical_expr::{DynamicFilterTracking, EquivalenceProperties};use futures::{FutureExt, Stream, TryStreamExt};use insta::assert_snapshot;@@ -2766,6 +2807,127 @@ mod tests {Ok(())}+ async fn emit_sort_partition(+ sort: &Arc<SortExec>,+ partition: usize,+ task_ctx: Arc<TaskContext>,+ ) -> Result<()> {+ let _batches: Vec<RecordBatch> =+ sort.execute(partition, task_ctx)?.try_collect().await?;+ Ok(())+ }++ fn assert_filter_still_waiting(filter: &Arc<DynamicFilterPhysicalExpr>) {+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> =+ Arc::<DynamicFilterPhysicalExpr>::clone(filter);+ assert!(+ matches!(+ DynamicFilterTracking::classify(&dynamic_filter_expr),+ DynamicFilterTracking::Watching(_)+ ),+ "the shared filter should remain watchable until every partition emits"+ );+ }++ #[tokio::test]+ async fn test_preserved_topk_filter_waits_for_all_sort_partitions() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let partitions = vec![+ vec![RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],+ )?],+ vec![RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![6, 4, 5]))],+ )?],+ ];+ let input = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;+ let sort = SortExec::new(+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),+ input,+ )+ // `with_fetch` creates the TopK filter; preserving partitioning after+ // that must rebuild it with one emitter per output partition.+ .with_fetch(Some(2))+ .with_preserve_partitioning(true);++ let dynamic_filter = sort+ .dynamic_filter_expr()+ .expect("fetch sort should create a dynamic filter");+ let sort = Arc::new(sort);+ let task_ctx = Arc::new(TaskContext::default());++ emit_sort_partition(&sort, 0, Arc::clone(&task_ctx)).await?;+ assert_filter_still_waiting(&dynamic_filter);++ emit_sort_partition(&sort, 1, task_ctx).await?;+ tokio::time::timeout(+ std::time::Duration::from_secs(1),+ dynamic_filter.wait_complete(),+ )+ .await+ .expect("the final preserved SortExec partition should complete the filter");++ Ok(())+ }++ #[tokio::test]+ async fn test_with_fetch_rebuilds_existing_topk_filter() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let partitions = vec![+ vec![RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],+ )?],+ vec![RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![6, 4, 5]))],+ )?],+ ];+ let input = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("a", 0))],+ lit(true),+ ));+ let dynamic_filter_id = dynamic_filter+ .expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id");+ let sort = SortExec::new(+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),+ input,+ )+ .with_dynamic_filter_expr(dynamic_filter)?+ .with_preserve_partitioning(true)+ .with_fetch(Some(2));++ let dynamic_filter = sort+ .dynamic_filter_expr()+ .expect("fetch sort should keep the dynamic filter");+ assert_eq!(+ dynamic_filter+ .expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id"),+ dynamic_filter_id+ );++ let sort = Arc::new(sort);+ let task_ctx = Arc::new(TaskContext::default());++ emit_sort_partition(&sort, 0, Arc::clone(&task_ctx)).await?;+ assert_filter_still_waiting(&dynamic_filter);++ emit_sort_partition(&sort, 1, task_ctx).await?;+ tokio::time::timeout(+ std::time::Duration::from_secs(1),+ dynamic_filter.wait_complete(),+ )+ .await+ .expect("the final preserved SortExec partition should complete the filter");++ Ok(())+ }+#[test]fn test_with_dynamic_filter_rejects_invalid_columns() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex 8a8bfd204e..11cf54c904 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -24,6 +24,7 @@ use arrow::{};use datafusion_expr::{ColumnarValue, Operator};use std::mem::size_of;+use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc};use super::metrics::{@@ -49,7 +50,7 @@ use datafusion_physical_expr::{use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use parking_lot::RwLock;-/// Global TopK+/// TopK////// # Background///@@ -84,10 +85,13 @@ use parking_lot::RwLock;/// # Partial Sort Optimization////// This implementation additionally optimizes queries where the input is already-/// partially sorted by a common prefix of the requested ordering. Once the top K-/// heap is full, if subsequent rows are guaranteed to be strictly greater (in sort-/// order) on this prefix than the largest row currently stored, the operator-/// safely terminates early.+/// partially sorted by a common prefix of the requested ordering. If subsequent+/// rows are guaranteed to be strictly greater (in sort order) than a known TopK+/// boundary on this prefix, the operator safely terminates early.+///+/// For a local TopK, that boundary comes from the local heap once it has K rows.+/// For a partitioned `SortExec`, a shared dynamic-filter threshold can provide+/// the same prefix boundary before a lagging partition has filled its local heap.////// ## Example///@@ -135,29 +139,104 @@ pub struct TopK {/// For more background, please also see the [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]////// [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]: https://datafusion.apache.org/blog/2025/09/10/dynamic-filters-#[derive(Debug, Clone)]+#[derive(Debug)]pub struct TopKDynamicFilters {- /// The current *global* threshold for the dynamic filter.- /// This is shared across all partitions and is updated by any of them.- /// Stored as row bytes for efficient comparison.- threshold_row: Option<Vec<u8>>,+ /// The current threshold shared by all TopK emitters that use this dynamic+ /// filter. Any emitter may tighten it.+ ///+ /// The full sort-key row and common-prefix row are stored together so they+ /// always describe the same heap row.+ shared_threshold: Option<TopKThreshold>,/// The expression used to evaluate the dynamic filter/// Only updated when lock held for the duration of the updateexpr: Arc<DynamicFilterPhysicalExpr>,+ /// Number of local TopK emitters that have not called `emit` yet.+ ///+ /// A partition-preserving `SortExec` creates one local TopK per output+ /// partition. The shared dynamic filter is complete only after every local+ /// TopK has emitted.+ ///+ /// `emit` only needs a read guard on the shared filter wrapper, so+ /// concurrent emitters use this atomic counter instead of taking an+ /// exclusive lock just to mark their partition done.+ remaining_topk_emitters: AtomicUsize,+}++#[derive(Debug, Clone)]+struct TopKThreshold {+ /// The full sort-key row bytes for efficient comparison.+ full_sort_key_row: Vec<u8>,+ /// The same heap row encoded with the common-prefix converter, when the+ /// input ordering shares a prefix with the TopK ordering.+ ///+ /// This lets each partition stop from a shared TopK threshold even if its+ /// local heap has not filled yet.+ common_prefix_row: Option<Vec<u8>>,+}++impl TopKThreshold {+ fn new(full_sort_key_row: Vec<u8>, common_prefix_row: Option<Vec<u8>>) -> Self {+ Self {+ full_sort_key_row,+ common_prefix_row,+ }+ }++ fn full_sort_key_row(&self) -> &[u8] {+ self.full_sort_key_row.as_slice()+ }++ fn common_prefix_row(&self) -> Option<&[u8]> {+ self.common_prefix_row.as_deref()+ }++ fn is_more_selective_than(&self, current: &Self) -> bool {+ self.full_sort_key_row() < current.full_sort_key_row()+ }}impl TopKDynamicFilters {/// Create a new `TopKDynamicFilters` with the given expressionpub fn new(expr: Arc<DynamicFilterPhysicalExpr>) -> Self {+ Self::new_with_topk_emitter_count(expr, 1)+ }++ /// Create a new `TopKDynamicFilters` with the expected number of local+ /// TopK emitters that share it.+ pub fn new_with_topk_emitter_count(+ expr: Arc<DynamicFilterPhysicalExpr>,+ topk_emitter_count: usize,+ ) -> Self {+ debug_assert!(topk_emitter_count > 0);Self {- threshold_row: None,+ shared_threshold: None,expr,+ remaining_topk_emitters: AtomicUsize::new(topk_emitter_count),}}pub fn expr(&self) -> Arc<DynamicFilterPhysicalExpr> {Arc::clone(&self.expr)}++ fn mark_topk_emitted(&self) {+ let previous = self+ .remaining_topk_emitters+ .fetch_update(+ AtomicOrdering::AcqRel,+ AtomicOrdering::Acquire,+ |remaining| remaining.checked_sub(1),+ )+ .unwrap_or(0);+ debug_assert!(+ previous > 0,+ "TopK dynamic filter emitter completed more times than expected"+ );++ if previous == 1 {+ self.expr.mark_complete();+ }+ }}// Guesstimate for memory allocation: estimated number of bytes used per row in the RowConverter@@ -206,7 +285,7 @@ impl TopK {let scratch_rows =row_converter.empty_rows(batch_size, ESTIMATED_BYTES_PER_ROW * batch_size);- let prefix_row_converter = if common_sort_prefix.is_empty() {+ let common_prefix_row_converter = if common_sort_prefix.is_empty() {None} else {let input_sort_fields = build_sort_fields(&common_sort_prefix, &schema)?;@@ -222,7 +301,7 @@ impl TopK {row_converter,scratch_rows,heap: TopKHeap::new(k),- common_sort_prefix_converter: prefix_row_converter,+ common_sort_prefix_converter: common_prefix_row_converter,common_sort_prefix: Arc::from(common_sort_prefix),finished: false,filter,@@ -314,6 +393,10 @@ impl TopK {// update the filter representation of our TopK heapself.update_filter()?;+ } else {+ // The heap did not change, but this batch's prefix may still prove+ // that no later rows can enter the TopK.+ self.attempt_early_completion(&batch)?;}Ok(())@@ -357,18 +440,18 @@ impl TopK {return Ok(());};- let new_threshold_row = &max_row.row;+ let new_threshold_row = max_row.row();// Fast path: check if the current value in topk is better than what is// currently set in the filter with a read only locklet needs_update = self.filter.read()- .threshold_row+ .shared_threshold.as_ref()- .map(|current_row| {+ .map(|current_threshold| {// new < current means new threshold is more selective- new_threshold_row < current_row+ new_threshold_row < current_threshold.full_sort_key_row()}).unwrap_or(true); // No current threshold, so we need to set one@@ -385,33 +468,25 @@ impl TopK {// Build the filter expression OUTSIDE any synchronizationlet predicate = Self::build_filter_expression(&self.expr, &thresholds)?;- let new_threshold = new_threshold_row.to_vec();+ let new_threshold = TopKThreshold::new(+ new_threshold_row.to_vec(),+ self.encode_topk_common_prefix_row(max_row)?,+ );// update the threshold. Since there was a lock gap, we must check if it is still the best// may have changed while we were building the expression without the locklet mut filter = self.filter.write();- let old_threshold = filter.threshold_row.take();-- // Update filter if we successfully updated the threshold- // (or if there was no previous threshold and we're the first)- match old_threshold {- Some(old_threshold) => {- // new threshold is still better than the old one- if new_threshold.as_slice() < old_threshold.as_slice() {- filter.threshold_row = Some(new_threshold);- } else {- // some other thread updated the threshold to a better- // one while we were building so there is no need to- // update the filter- filter.threshold_row = Some(old_threshold);- return Ok(());- }- }- None => {- // No previous threshold, so we can set the new one- filter.threshold_row = Some(new_threshold);- }- };+ let still_needs_update = filter+ .shared_threshold+ .as_ref()+ .map(|current| new_threshold.is_more_selective_than(current))+ .unwrap_or(true);+ if !still_needs_update {+ // some other thread updated the threshold to a better one while we+ // were building so there is no need to update the filter+ return Ok(());+ }+ filter.shared_threshold = Some(new_threshold);// Update the filter expressionif let Some(pred) = predicate@@ -509,78 +584,109 @@ impl TopK {Ok(dynamic_predicate)}- /// If input ordering shares a common sort prefix with the TopK, and if the TopK's heap is full,+ /// If input ordering shares a common sort prefix with the TopK,/// check if the computation can be finished early.- /// This is the case if the last row of the current batch is strictly greater than the max row in the heap,- /// comparing only on the shared prefix columns.+ ///+ /// This is the case if the last row of the current batch is strictly+ /// greater than either the shared dynamic-filter threshold prefix or the max+ /// row in the local heap, comparing only on the shared prefix columns.fn attempt_early_completion(&mut self, batch: &RecordBatch) -> Result<()> {// Early exit if the batch is empty as there is no last row to extract from it.if batch.num_rows() == 0 {return Ok(());}- // prefix_row_converter is only `Some` if the input ordering has a common prefix with the TopK,+ // common_prefix_row_converter is only `Some` if the input ordering has a common prefix with the TopK,// so early exit if it is `None`.let Some(prefix_converter) = &self.common_sort_prefix_converter else {return Ok(());};- // Early exit if the heap is not full (`heap.max()` only returns `Some` if the heap is full).- let Some(max_topk_row) = self.heap.max() else {- return Ok(());- };-// Evaluate the prefix for the last row of the current batch.let last_row_idx = batch.num_rows() - 1;let mut batch_prefix_scratch =prefix_converter.empty_rows(1, ESTIMATED_BYTES_PER_ROW); // 1 row with capacity ESTIMATED_BYTES_PER_ROW- self.compute_common_sort_prefix(batch, last_row_idx, &mut batch_prefix_scratch)?;-- // Retrieve the max row from the heap.- let store_entry = self- .heap- .store- .get(max_topk_row.batch_id)- .ok_or(internal_datafusion_err!("Invalid batch id in topK heap"))?;- let max_batch = &store_entry.batch;- let mut heap_prefix_scratch =- prefix_converter.empty_rows(1, ESTIMATED_BYTES_PER_ROW); // 1 row with capacity ESTIMATED_BYTES_PER_ROW- self.compute_common_sort_prefix(- max_batch,- max_topk_row.index,- &mut heap_prefix_scratch,+ self.append_common_prefix_row(+ prefix_converter,+ batch,+ last_row_idx,+ &mut batch_prefix_scratch,)?;+ let batch_common_prefix_row = batch_prefix_scratch.row(0);+ let batch_common_prefix = batch_common_prefix_row.as_ref();++ let finished_by_shared_threshold = self+ .filter+ .read()+ .shared_threshold+ .as_ref()+ .and_then(TopKThreshold::common_prefix_row)+ .map(|common_prefix_row| batch_common_prefix > common_prefix_row)+ .unwrap_or(false);+ if finished_by_shared_threshold {+ self.finished = true;+ return Ok(());+ }++ // Early exit if the heap is not full (`heap.max()` only returns `Some` if the heap is full).+ let Some(max_topk_row) = self.heap.max() else {+ return Ok(());+ };++ // Encode the local heap max row's common-prefix projection.+ let Some(heap_common_prefix_row) =+ self.encode_topk_common_prefix_row(max_topk_row)?+ else {+ return Ok(());+ };// If the last row's prefix is strictly greater than the max prefix, mark as finished.- if batch_prefix_scratch.row(0).as_ref() > heap_prefix_scratch.row(0).as_ref() {+ if batch_common_prefix > heap_common_prefix_row.as_slice() {self.finished = true;}Ok(())}- // Helper function to compute the prefix for a given batch and row index, storing the result in scratch.- fn compute_common_sort_prefix(+ fn encode_topk_common_prefix_row(&self,+ topk_row: &TopKRow,+ ) -> Result<Option<Vec<u8>>> {+ let Some(prefix_converter) = &self.common_sort_prefix_converter else {+ return Ok(None);+ };++ let store_entry = self+ .heap+ .store+ .get(topk_row.batch_id)+ .ok_or(internal_datafusion_err!("Invalid batch id in topK heap"))?;+ let mut scratch = prefix_converter.empty_rows(1, ESTIMATED_BYTES_PER_ROW);+ self.append_common_prefix_row(+ prefix_converter,+ &store_entry.batch,+ topk_row.index,+ &mut scratch,+ )?;+ Ok(Some(scratch.row(0).as_ref().to_vec()))+ }++ fn append_common_prefix_row(+ &self,+ prefix_converter: &RowConverter,batch: &RecordBatch,- last_row_idx: usize,+ row_idx: usize,scratch: &mut Rows,) -> Result<()> {- let last_row: Vec<ArrayRef> = self+ let row = batch.slice(row_idx, 1);+ let prefix_columns: Vec<ArrayRef> = self.common_sort_prefix.iter()- .map(|expr| {- expr.expr- .evaluate(&batch.slice(last_row_idx, 1))?- .into_array(1)- })+ .map(|expr| expr.expr.evaluate(&row)?.into_array(1)).collect::<Result<_>>()?;- self.common_sort_prefix_converter- .as_ref()- .unwrap()- .append(scratch, &last_row)?;+ prefix_converter.append(scratch, &prefix_columns)?;Ok(())}@@ -602,8 +708,9 @@ impl TopK {} = self;let _timer = metrics.baseline.elapsed_compute().timer(); // time updated on drop- // Mark the dynamic filter as complete now that TopK processing is finished.- filter.read().expr().mark_complete();+ // Mark this local TopK as emitted. For shared filters, the final+ // local emitter marks the dynamic filter complete.+ filter.read().mark_topk_emitted();// break into record batches as neededlet mut batches = vec![];@@ -1066,7 +1173,7 @@ mod tests {use arrow::datatypes::{DataType, Field, Schema};use arrow_schema::SortOptions;use datafusion_common::assert_batches_eq;- use datafusion_physical_expr::expressions::col;+ use datafusion_physical_expr::{DynamicFilterTracking, expressions::col};use futures::TryStreamExt;/// This test ensures the size calculation is correct for RecordBatches with multiple columns.@@ -1101,86 +1208,110 @@ mod tests {assert_eq!(record_batch_store.batches_size, 0);}- /// Builds an `(a Int32, b Float64)` schema and a `TopK` with full sort- /// `(a ASC, b ASC)`, input prefix `[a]`, `k = 3`, `batch_size = 2`. Used by- /// the prefix-completion tests below to keep their per-scenario logic in focus.- fn build_ab_prefix_topk() -> Result<(Arc<Schema>, TopK)> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, false),+ fn make_ab_schema() -> SchemaRef {+ make_ab_schema_with_nullable_a(false)+ }++ fn make_ab_schema_with_nullable_a(a_nullable: bool) -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, a_nullable),Field::new("b", DataType::Float64, false),- ]));+ ]))+ }++ // Local TopK tests use one emitter; shared-filter cases pass the partition count explicitly.+ fn make_topk_filter() -> Arc<RwLock<TopKDynamicFilters>> {+ make_shared_topk_filter(1)+ }+ fn make_shared_topk_filter(+ topk_emitter_count: usize,+ ) -> Arc<RwLock<TopKDynamicFilters>> {+ Arc::new(RwLock::new(+ TopKDynamicFilters::new_with_topk_emitter_count(+ Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true))),+ topk_emitter_count,+ ),+ ))+ }++ /// Builds the `(a, b)` fixture used by prefix-completion tests:+ /// full sort `(a, b)`, input prefix `[a]`, `k = 3`, and batch size 2.+ fn make_ab_topk(+ schema: SchemaRef,+ filter: Arc<RwLock<TopKDynamicFilters>>,+ ) -> Result<TopK> {+ make_ab_topk_with_options(0, schema, filter, SortOptions::default())+ }++ fn make_ab_topk_with_options(+ partition_id: usize,+ schema: SchemaRef,+ filter: Arc<RwLock<TopKDynamicFilters>>,+ a_options: SortOptions,+ ) -> Result<TopK> {let sort_expr_a = PhysicalSortExpr {expr: col("a", schema.as_ref())?,- options: SortOptions::default(),+ options: a_options,};let sort_expr_b = PhysicalSortExpr {expr: col("b", schema.as_ref())?,options: SortOptions::default(),};- // Input ordering uses only column "a" (a prefix of the full sort on (a, b)).- let prefix = vec![sort_expr_a.clone()];- let full_expr = LexOrdering::from([sort_expr_a, sort_expr_b]);-- let topk = TopK::try_new(- 0,- Arc::clone(&schema),- prefix,- full_expr,+ TopK::try_new(+ partition_id,+ schema,+ vec![sort_expr_a.clone()],+ LexOrdering::from([sort_expr_a, sort_expr_b]),3,2,Arc::new(RuntimeEnv::default()),&ExecutionPlanMetricsSet::new(),- Arc::new(RwLock::new(TopKDynamicFilters::new(Arc::new(- DynamicFilterPhysicalExpr::new(vec![], lit(true)),- )))),- )?;- Ok((schema, topk))+ filter,+ )+ }++ fn make_ab_batch(+ schema: SchemaRef,+ a: &[Option<i32>],+ b: &[f64],+ ) -> Result<RecordBatch> {+ Ok(RecordBatch::try_new(+ schema,+ vec![+ Arc::new(Int32Array::from(a.to_vec())) as ArrayRef,+ Arc::new(Float64Array::from(b.to_vec())) as ArrayRef,+ ],+ )?)+ }++ type AbRow = (Option<i32>, f64);++ fn make_ab_rows_batch(schema: SchemaRef, rows: &[AbRow]) -> Result<RecordBatch> {+ let (a, b): (Vec<_>, Vec<_>) = rows.iter().copied().unzip();+ make_ab_batch(schema, &a, &b)}- /// This test validates that the `try_finish` method marks the TopK operator as finished- /// when the prefix (on column "a") of the last row in the current batch is strictly greater- /// than the max top‑k row.- /// The full sort expression is defined on both columns ("a", "b"), but the input ordering is only on "a".#[tokio::test]- async fn test_try_finish_marks_finished_with_prefix() -> Result<()> {- let (schema, mut topk) = build_ab_prefix_topk()?;-- // Create the first batch with two columns:- // Column "a": [1, 1, 2], Column "b": [20.0, 15.0, 30.0].- let array_a1: ArrayRef =- Arc::new(Int32Array::from(vec![Some(1), Some(1), Some(2)]));- let array_b1: ArrayRef = Arc::new(Float64Array::from(vec![20.0, 15.0, 30.0]));- let batch1 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a1, array_b1])?;-- // Insert the first batch.- // At this point the heap is not yet “finished” because the prefix of the last row of the batch- // is not strictly greater than the prefix of the max top‑k row (both being `2`).- topk.insert_batch(batch1)?;- assert!(- !topk.finished,- "Expected 'finished' to be false after the first batch."- );+ async fn test_early_completion_marks_finished_with_prefix() -> Result<()> {+ let schema = make_ab_schema();+ let mut topk = make_ab_topk(Arc::clone(&schema), make_topk_filter())?;- // Create the second batch with two columns:- // Column "a": [2, 3], Column "b": [10.0, 20.0].- let array_a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(2), Some(3)]));- let array_b2: ArrayRef = Arc::new(Float64Array::from(vec![10.0, 20.0]));- let batch2 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a2, array_b2])?;+ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(1), Some(1), Some(2)],+ &[20.0, 15.0, 30.0],+ )?)?;+ assert!(!topk.finished);- // Insert the second batch.- // The last row in this batch has a prefix value of `3`,- // which is strictly greater than the max top‑k row (with value `2`),- // so try_finish should mark the TopK as finished.- topk.insert_batch(batch2)?;- assert!(- topk.finished,- "Expected 'finished' to be true after the second batch."- );+ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(2), Some(3)],+ &[10.0, 20.0],+ )?)?;+ assert!(topk.finished);- // Verify the TopK correctly emits the top k rows from both batches- // (the value 10.0 for b is from the second batch).let results: Vec<_> = topk.emit()?.try_collect().await?;assert_batches_eq!(&[@@ -1201,47 +1332,26 @@ mod tests {/// Regression test for #22849: a batch whose rows are entirely rejected by the/// heap's dynamic filter must still trigger `attempt_early_completion` when its/// last row's prefix is worse than the heap's worst.- ///- /// Before the fix, the `!filter.has_true()` short-circuit returned without calling- /// `attempt_early_completion`. Because the heap's filter is itself derived from the- /// heap's worst row, a batch from a strictly-worse prefix is exactly the case the- /// filter rejects entirely — i.e. the very signal the early-exit was designed to- /// detect was being silently dropped.#[tokio::test]- async fn test_try_finish_fires_when_filter_rejects_entire_batch() -> Result<()> {- let (schema, mut topk) = build_ab_prefix_topk()?;-- // Batch 1 fills the heap with (1, 20.0), (1, 15.0), (2, 30.0).- // heap.max becomes (a=2, b=30.0); update_filter tightens the heap filter to- // a < 2 OR (a = 2 AND b < 30.0).- let array_a1: ArrayRef =- Arc::new(Int32Array::from(vec![Some(1), Some(1), Some(2)]));- let array_b1: ArrayRef = Arc::new(Float64Array::from(vec![20.0, 15.0, 30.0]));- let batch1 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a1, array_b1])?;- topk.insert_batch(batch1)?;- assert!(- !topk.finished,- "Expected 'finished' to be false after batch 1 \- (last row prefix a=2 equals heap.max prefix a=2, not strictly greater)."- );+ async fn test_early_completion_fires_when_filter_rejects_entire_batch() -> Result<()>+ {+ let schema = make_ab_schema();+ let mut topk = make_ab_topk(Arc::clone(&schema), make_topk_filter())?;- // Batch 2: every row has a=3, so the heap's filter (a < 2 OR (a = 2 AND b < 30))- // rejects every row. Before the fix, `insert_batch` would short-circuit on- // `!filter.has_true()` and return without checking the prefix; `finished`- // would stay false even though no future batch could improve the heap.- let array_a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(3)]));- let array_b2: ArrayRef = Arc::new(Float64Array::from(vec![10.0, 20.0]));- let batch2 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a2, array_b2])?;- topk.insert_batch(batch2)?;- assert!(- topk.finished,- "Expected 'finished' to be true after batch 2 \- (filter rejected every row, but the batch's last row prefix a=3 \- is strictly greater than heap.max prefix a=2)."- );+ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(1), Some(1), Some(2)],+ &[20.0, 15.0, 30.0],+ )?)?;+ assert!(!topk.finished);++ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(3), Some(3)],+ &[10.0, 20.0],+ )?)?;+ assert!(topk.finished);- // The emitted top-k is unchanged from after batch 1 since none of batch 2's- // rows could improve the heap.let results: Vec<_> = topk.emit()?.try_collect().await?;assert_batches_eq!(&[@@ -1259,50 +1369,266 @@ mod tests {Ok(())}- /// This test verifies that the dynamic filter is marked as complete after TopK processing finishes.#[tokio::test]- async fn test_topk_marks_filter_complete() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ async fn test_early_completion_fires_when_batch_makes_no_replacements() -> Result<()>+ {+ let schema = make_ab_schema();+ let filter = make_topk_filter();+ let mut topk = make_ab_topk(Arc::clone(&schema), Arc::clone(&filter))?;- let sort_expr = PhysicalSortExpr {- expr: col("a", schema.as_ref())?,- options: SortOptions::default(),- };+ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(1), Some(1), Some(2)],+ &[20.0, 15.0, 30.0],+ )?)?;+ assert!(!topk.finished);- let full_expr = LexOrdering::from([sort_expr.clone()]);- let prefix = vec![sort_expr];+ let replacements_before = topk.metrics.row_replacements.value();- // Create a dummy runtime environment and metrics- let runtime = Arc::new(RuntimeEnv::default());- let metrics = ExecutionPlanMetricsSet::new();+ // Keep the dynamic filter permissive so the second batch reaches+ // `find_new_topk_items`; all of its rows are worse than the heap max,+ // so this specifically exercises the `replacements == 0` path.+ filter.read().expr().update(lit(true))?;+ topk.insert_batch(make_ab_batch(+ Arc::clone(&schema),+ &[Some(3), Some(3)],+ &[10.0, 20.0],+ )?)?;+ assert_eq!(topk.metrics.row_replacements.value(), replacements_before);+ assert!(topk.finished);- // Create a dynamic filter that we'll check for completion- let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));- let dynamic_filter_clone = Arc::clone(&dynamic_filter);+ let results: Vec<_> = topk.emit()?.try_collect().await?;+ assert_batches_eq!(+ &[+ "+---+------+",+ "| a | b |",+ "+---+------+",+ "| 1 | 15.0 |",+ "| 1 | 20.0 |",+ "| 2 | 30.0 |",+ "+---+------+",+ ],+ &results+ );- // Create a TopK instance- let mut topk = TopK::try_new(+ Ok(())+ }++ struct SharedPrefixCase {+ name: &'static str,+ a_nullable: bool,+ a_options: SortOptions,+ threshold_source_rows: &'static [AbRow],+ lagging_partition_rows: &'static [AbRow],+ expected_finished: bool,+ }++ fn assert_shared_prefix_case(case: SharedPrefixCase) -> Result<()> {+ let schema = make_ab_schema_with_nullable_a(case.a_nullable);+ let filter = make_shared_topk_filter(2);++ let mut threshold_source = make_ab_topk_with_options(0,Arc::clone(&schema),- prefix,- full_expr,+ Arc::clone(&filter),+ case.a_options,+ )?;+ threshold_source.insert_batch(make_ab_rows_batch(+ Arc::clone(&schema),+ case.threshold_source_rows,+ )?)?;+ assert!(+ filter+ .read()+ .shared_threshold+ .as_ref()+ .and_then(TopKThreshold::common_prefix_row)+ .is_some(),+ "{}: threshold-source partition should establish the shared prefix threshold",+ case.name+ );++ let mut lagging_partition = make_ab_topk_with_options(+ 1,+ Arc::clone(&schema),+ Arc::clone(&filter),+ case.a_options,+ )?;+ lagging_partition+ .insert_batch(make_ab_rows_batch(schema, case.lagging_partition_rows)?)?;++ assert!(+ lagging_partition.heap.inner.is_empty(),+ "{}: lagging partition's local heap should remain empty",+ case.name+ );+ assert_eq!(+ lagging_partition.finished, case.expected_finished,+ "{}",+ case.name+ );++ Ok(())+ }++ #[test]+ fn test_shared_filter_can_finish_partition_before_local_heap_is_full() -> Result<()> {+ assert_shared_prefix_case(SharedPrefixCase {+ name: "shared threshold should finish lagging partition",+ a_nullable: false,+ a_options: SortOptions::default(),+ threshold_source_rows: &[(Some(1), 20.0), (Some(1), 15.0), (Some(2), 30.0)],+ lagging_partition_rows: &[(Some(3), 10.0), (Some(3), 20.0)],+ expected_finished: true,+ })+ }++ #[test]+ fn test_shared_prefix_threshold_boundary_cases() -> Result<()> {+ for case in [+ SharedPrefixCase {+ name: "equal prefix cannot prove completion",+ a_nullable: false,+ a_options: SortOptions::default(),+ threshold_source_rows: &[+ (Some(1), 20.0),+ (Some(1), 15.0),+ (Some(2), 30.0),+ ],+ lagging_partition_rows: &[(Some(2), 40.0), (Some(2), 50.0)],+ expected_finished: false,+ },+ SharedPrefixCase {+ name: "descending prefix uses sort-order row encoding",+ a_nullable: false,+ a_options: SortOptions {+ descending: true,+ nulls_first: true,+ },+ threshold_source_rows: &[+ (Some(10), 1.0),+ (Some(10), 2.0),+ (Some(9), 3.0),+ ],+ lagging_partition_rows: &[(Some(8), 1.0), (Some(8), 2.0)],+ expected_finished: true,+ },+ SharedPrefixCase {+ name: "NULLS LAST prefix uses sort-order row encoding",+ a_nullable: true,+ a_options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ threshold_source_rows: &[+ (Some(1), 20.0),+ (Some(1), 15.0),+ (Some(2), 30.0),+ ],+ lagging_partition_rows: &[(None, 10.0), (None, 20.0)],+ expected_finished: true,+ },+ ] {+ assert_shared_prefix_case(case)?;+ }+ Ok(())+ }++ fn make_single_column_topk(+ dynamic_filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<(SchemaRef, TopK)> {+ make_single_column_topk_with_filter(+ 0,+ Arc::new(RwLock::new(TopKDynamicFilters::new(dynamic_filter))),+ )+ }++ fn make_single_column_topk_with_filter(+ partition_id: usize,+ filter: Arc<RwLock<TopKDynamicFilters>>,+ ) -> Result<(SchemaRef, TopK)> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let sort_expr = PhysicalSortExpr {+ expr: col("a", schema.as_ref())?,+ options: SortOptions::default(),+ };++ let topk = TopK::try_new(+ partition_id,+ Arc::clone(&schema),+ vec![sort_expr.clone()],+ LexOrdering::from([sort_expr]),2,10,- runtime,- &metrics,- Arc::new(RwLock::new(TopKDynamicFilters::new(dynamic_filter))),+ Arc::new(RuntimeEnv::default()),+ &ExecutionPlanMetricsSet::new(),+ filter,)?;+ Ok((schema, topk))+ }++ #[tokio::test]+ async fn test_topk_marks_filter_complete() -> Result<()> {+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);+ let (schema, mut topk) = make_single_column_topk(dynamic_filter)?;+let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(1), Some(2)]));let batch = RecordBatch::try_new(Arc::clone(&schema), vec![array])?;topk.insert_batch(batch)?;- // Call emit to finish TopK processinglet _results: Vec<_> = topk.emit()?.try_collect().await?;- // After emit is called, the dynamic filter should be marked as complete- // wait_complete() should return immediately- dynamic_filter_clone.wait_complete().await;+ tokio::time::timeout(+ std::time::Duration::from_secs(1),+ dynamic_filter_clone.wait_complete(),+ )+ .await+ .expect("single-emitter TopK should mark the dynamic filter complete");++ Ok(())+ }++ #[tokio::test]+ async fn test_shared_topk_filter_completes_after_last_emitter() -> Result<()> {+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);+ let shared_filter = Arc::new(RwLock::new(+ TopKDynamicFilters::new_with_topk_emitter_count(dynamic_filter, 2),+ ));++ let (schema, mut topk_0) =+ make_single_column_topk_with_filter(0, Arc::clone(&shared_filter))?;+ let (_, mut topk_1) =+ make_single_column_topk_with_filter(1, Arc::clone(&shared_filter))?;++ let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(1), Some(2)]));+ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![array])?;+ topk_0.insert_batch(batch)?;+ let _results: Vec<_> = topk_0.emit()?.try_collect().await?;++ let dynamic_filter_expr: Arc<dyn PhysicalExpr> =+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter_clone);+ assert!(+ matches!(+ DynamicFilterTracking::classify(&dynamic_filter_expr),+ DynamicFilterTracking::Watching(_)+ ),+ "the shared filter should remain watchable until every TopK emits"+ );++ let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(6), Some(4), Some(5)]));+ let batch = RecordBatch::try_new(schema, vec![array])?;+ topk_1.insert_batch(batch)?;+ let _results: Vec<_> = topk_1.emit()?.try_collect().await?;++ tokio::time::timeout(+ std::time::Duration::from_secs(1),+ dynamic_filter_clone.wait_complete(),+ )+ .await+ .expect("the final shared TopK emitter should mark the dynamic filter complete");Ok(())}