Skip to content

Apache DataFusion SQL Query Engine

Fix shared TopK early exit with shared prefix threshold (#22991)

Geoffrey Claudeba67bb467a30
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index ccc675c6ef..f48bb15a7b 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -905,19 +905,51 @@ impl SortExec {
self.preserve_partitioning = preserve_partitioning;
Arc::make_mut(&mut self.cache).partitioning =
Self::output_partitioning_helper(&self.input, self.preserve_partitioning);
+ if self.fetch.is_some() {
+ self.rebuild_filter_for_current_partitioning();
+ }
self
}
- /// Add or reset `self.filter` to a new `TopKDynamicFilters`.
+ fn topk_emitter_count(&self) -> usize {
+ self.cache.output_partitioning().partition_count()
+ }
+
+ /// Build a new shared TopK dynamic filter wrapper for this `SortExec`.
fn create_filter(&self) -> Arc<RwLock<TopKDynamicFilters>> {
let children = self
.expr
.iter()
.map(|sort_expr| Arc::clone(&sort_expr.expr))
.collect::<Vec<_>>();
- Arc::new(RwLock::new(TopKDynamicFilters::new(Arc::new(
- DynamicFilterPhysicalExpr::new(children, lit(true)),
- ))))
+ self.create_filter_with_expr(Arc::new(DynamicFilterPhysicalExpr::new(
+ children,
+ lit(true),
+ )))
+ }
+
+ fn create_filter_with_expr(
+ &self,
+ expr: Arc<DynamicFilterPhysicalExpr>,
+ ) -> Arc<RwLock<TopKDynamicFilters>> {
+ Arc::new(RwLock::new(
+ TopKDynamicFilters::new_with_topk_emitter_count(
+ expr,
+ self.topk_emitter_count(),
+ ),
+ ))
+ }
+
+ /// Rebuild the shared TopK filter wrapper for the current output partitioning.
+ ///
+ /// The dynamic filter expression is preserved, but wrapper state such as the
+ /// shared threshold and remaining emitter count is reset for the new
+ /// partitioning.
+ fn rebuild_filter_for_current_partitioning(&mut self) {
+ let filter_expr = self.filter.as_ref().map(|filter| filter.read().expr());
+ if let Some(filter_expr) = filter_expr {
+ self.filter = Some(self.create_filter_with_expr(filter_expr));
+ }
}
fn cloned(&self) -> Self {
@@ -952,14 +984,20 @@ impl SortExec {
if fetch.is_some() && is_pipeline_friendly {
cache = cache.with_boundedness(Boundedness::Bounded);
}
- let filter = fetch.is_some().then(|| {
- // If we already have a filter, keep it. Otherwise, create a new one.
- self.filter.clone().unwrap_or_else(|| self.create_filter())
- });
let mut new_sort = self.cloned();
new_sort.fetch = fetch;
new_sort.cache = cache.into();
- new_sort.filter = filter;
+ if fetch.is_some() {
+ if new_sort.filter.is_some() {
+ // Keep the dynamic filter expression, but reset wrapper state
+ // such as the shared threshold and expected emitter count.
+ new_sort.rebuild_filter_for_current_partitioning();
+ } else {
+ new_sort.filter = Some(new_sort.create_filter());
+ }
+ } else {
+ new_sort.filter = None;
+ }
new_sort
}
@@ -998,7 +1036,7 @@ impl SortExec {
for child in filter.children() {
child.data_type(&input_schema)?;
}
- self.filter = Some(Arc::new(RwLock::new(TopKDynamicFilters::new(filter))));
+ self.filter = Some(self.create_filter_with_expr(filter));
Ok(self)
}
@@ -1173,6 +1211,9 @@ impl ExecutionPlan for SortExec {
)?;
new_sort.cache = Arc::new(cache);
new_sort.common_sort_prefix = sort_prefix;
+ if new_sort.fetch.is_some() {
+ new_sort.rebuild_filter_for_current_partitioning();
+ }
}
Ok(Arc::new(new_sort))
@@ -1452,8 +1493,8 @@ mod tests {
GreedyMemoryPool, MemoryConsumer, MemoryPool,
};
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
- use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_expr::expressions::{Column, Literal};
+ use datafusion_physical_expr::{DynamicFilterTracking, EquivalenceProperties};
use futures::{FutureExt, Stream, TryStreamExt};
use insta::assert_snapshot;
@@ -2766,6 +2807,127 @@ mod tests {
Ok(())
}
+ async fn emit_sort_partition(
+ sort: &Arc<SortExec>,
+ partition: usize,
+ task_ctx: Arc<TaskContext>,
+ ) -> Result<()> {
+ let _batches: Vec<RecordBatch> =
+ sort.execute(partition, task_ctx)?.try_collect().await?;
+ Ok(())
+ }
+
+ fn assert_filter_still_waiting(filter: &Arc<DynamicFilterPhysicalExpr>) {
+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> =
+ Arc::<DynamicFilterPhysicalExpr>::clone(filter);
+ assert!(
+ matches!(
+ DynamicFilterTracking::classify(&dynamic_filter_expr),
+ DynamicFilterTracking::Watching(_)
+ ),
+ "the shared filter should remain watchable until every partition emits"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_preserved_topk_filter_waits_for_all_sort_partitions() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let partitions = vec![
+ vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
+ )?],
+ vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![6, 4, 5]))],
+ )?],
+ ];
+ let input = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;
+ let sort = SortExec::new(
+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),
+ input,
+ )
+ // `with_fetch` creates the TopK filter; preserving partitioning after
+ // that must rebuild it with one emitter per output partition.
+ .with_fetch(Some(2))
+ .with_preserve_partitioning(true);
+
+ let dynamic_filter = sort
+ .dynamic_filter_expr()
+ .expect("fetch sort should create a dynamic filter");
+ let sort = Arc::new(sort);
+ let task_ctx = Arc::new(TaskContext::default());
+
+ emit_sort_partition(&sort, 0, Arc::clone(&task_ctx)).await?;
+ assert_filter_still_waiting(&dynamic_filter);
+
+ emit_sort_partition(&sort, 1, task_ctx).await?;
+ tokio::time::timeout(
+ std::time::Duration::from_secs(1),
+ dynamic_filter.wait_complete(),
+ )
+ .await
+ .expect("the final preserved SortExec partition should complete the filter");
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_with_fetch_rebuilds_existing_topk_filter() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let partitions = vec![
+ vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
+ )?],
+ vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![6, 4, 5]))],
+ )?],
+ ];
+ let input = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::new(Column::new("a", 0))],
+ lit(true),
+ ));
+ let dynamic_filter_id = dynamic_filter
+ .expression_id()
+ .expect("DynamicFilterPhysicalExpr always has an expression_id");
+ let sort = SortExec::new(
+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),
+ input,
+ )
+ .with_dynamic_filter_expr(dynamic_filter)?
+ .with_preserve_partitioning(true)
+ .with_fetch(Some(2));
+
+ let dynamic_filter = sort
+ .dynamic_filter_expr()
+ .expect("fetch sort should keep the dynamic filter");
+ assert_eq!(
+ dynamic_filter
+ .expression_id()
+ .expect("DynamicFilterPhysicalExpr always has an expression_id"),
+ dynamic_filter_id
+ );
+
+ let sort = Arc::new(sort);
+ let task_ctx = Arc::new(TaskContext::default());
+
+ emit_sort_partition(&sort, 0, Arc::clone(&task_ctx)).await?;
+ assert_filter_still_waiting(&dynamic_filter);
+
+ emit_sort_partition(&sort, 1, task_ctx).await?;
+ tokio::time::timeout(
+ std::time::Duration::from_secs(1),
+ dynamic_filter.wait_complete(),
+ )
+ .await
+ .expect("the final preserved SortExec partition should complete the filter");
+
+ Ok(())
+ }
+
#[test]
fn test_with_dynamic_filter_rejects_invalid_columns() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs
index 8a8bfd204e..11cf54c904 100644
--- a/datafusion/physical-plan/src/topk/mod.rs
+++ b/datafusion/physical-plan/src/topk/mod.rs
@@ -24,6 +24,7 @@ use arrow::{
};
use datafusion_expr::{ColumnarValue, Operator};
use std::mem::size_of;
+use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
use std::{cmp::Ordering, collections::BinaryHeap, sync::Arc};
use super::metrics::{
@@ -49,7 +50,7 @@ use datafusion_physical_expr::{
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use parking_lot::RwLock;
-/// Global TopK
+/// TopK
///
/// # Background
///
@@ -84,10 +85,13 @@ use parking_lot::RwLock;
/// # Partial Sort Optimization
///
/// This implementation additionally optimizes queries where the input is already
-/// partially sorted by a common prefix of the requested ordering. Once the top K
-/// heap is full, if subsequent rows are guaranteed to be strictly greater (in sort
-/// order) on this prefix than the largest row currently stored, the operator
-/// safely terminates early.
+/// partially sorted by a common prefix of the requested ordering. If subsequent
+/// rows are guaranteed to be strictly greater (in sort order) than a known TopK
+/// boundary on this prefix, the operator safely terminates early.
+///
+/// For a local TopK, that boundary comes from the local heap once it has K rows.
+/// For a partitioned `SortExec`, a shared dynamic-filter threshold can provide
+/// the same prefix boundary before a lagging partition has filled its local heap.
///
/// ## Example
///
@@ -135,29 +139,104 @@ pub struct TopK {
/// For more background, please also see the [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]
///
/// [Dynamic Filters: Passing Information Between Operators During Execution for 25x Faster Queries blog]: https://datafusion.apache.org/blog/2025/09/10/dynamic-filters
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub struct TopKDynamicFilters {
- /// The current *global* threshold for the dynamic filter.
- /// This is shared across all partitions and is updated by any of them.
- /// Stored as row bytes for efficient comparison.
- threshold_row: Option<Vec<u8>>,
+ /// The current threshold shared by all TopK emitters that use this dynamic
+ /// filter. Any emitter may tighten it.
+ ///
+ /// The full sort-key row and common-prefix row are stored together so they
+ /// always describe the same heap row.
+ shared_threshold: Option<TopKThreshold>,
/// The expression used to evaluate the dynamic filter
/// Only updated when lock held for the duration of the update
expr: Arc<DynamicFilterPhysicalExpr>,
+ /// Number of local TopK emitters that have not called `emit` yet.
+ ///
+ /// A partition-preserving `SortExec` creates one local TopK per output
+ /// partition. The shared dynamic filter is complete only after every local
+ /// TopK has emitted.
+ ///
+ /// `emit` only needs a read guard on the shared filter wrapper, so
+ /// concurrent emitters use this atomic counter instead of taking an
+ /// exclusive lock just to mark their partition done.
+ remaining_topk_emitters: AtomicUsize,
+}
+
+#[derive(Debug, Clone)]
+struct TopKThreshold {
+ /// The full sort-key row bytes for efficient comparison.
+ full_sort_key_row: Vec<u8>,
+ /// The same heap row encoded with the common-prefix converter, when the
+ /// input ordering shares a prefix with the TopK ordering.
+ ///
+ /// This lets each partition stop from a shared TopK threshold even if its
+ /// local heap has not filled yet.
+ common_prefix_row: Option<Vec<u8>>,
+}
+
+impl TopKThreshold {
+ fn new(full_sort_key_row: Vec<u8>, common_prefix_row: Option<Vec<u8>>) -> Self {
+ Self {
+ full_sort_key_row,
+ common_prefix_row,
+ }
+ }
+
+ fn full_sort_key_row(&self) -> &[u8] {
+ self.full_sort_key_row.as_slice()
+ }
+
+ fn common_prefix_row(&self) -> Option<&[u8]> {
+ self.common_prefix_row.as_deref()
+ }
+
+ fn is_more_selective_than(&self, current: &Self) -> bool {
+ self.full_sort_key_row() < current.full_sort_key_row()
+ }
}
impl TopKDynamicFilters {
/// Create a new `TopKDynamicFilters` with the given expression
pub fn new(expr: Arc<DynamicFilterPhysicalExpr>) -> Self {
+ Self::new_with_topk_emitter_count(expr, 1)
+ }
+
+ /// Create a new `TopKDynamicFilters` with the expected number of local
+ /// TopK emitters that share it.
+ pub fn new_with_topk_emitter_count(
+ expr: Arc<DynamicFilterPhysicalExpr>,
+ topk_emitter_count: usize,
+ ) -> Self {
+ debug_assert!(topk_emitter_count > 0);
Self {
- threshold_row: None,
+ shared_threshold: None,
expr,
+ remaining_topk_emitters: AtomicUsize::new(topk_emitter_count),
}
}
pub fn expr(&self) -> Arc<DynamicFilterPhysicalExpr> {
Arc::clone(&self.expr)
}
+
+ fn mark_topk_emitted(&self) {
+ let previous = self
+ .remaining_topk_emitters
+ .fetch_update(
+ AtomicOrdering::AcqRel,
+ AtomicOrdering::Acquire,
+ |remaining| remaining.checked_sub(1),
+ )
+ .unwrap_or(0);
+ debug_assert!(
+ previous > 0,
+ "TopK dynamic filter emitter completed more times than expected"
+ );
+
+ if previous == 1 {
+ self.expr.mark_complete();
+ }
+ }
}
// Guesstimate for memory allocation: estimated number of bytes used per row in the RowConverter
@@ -206,7 +285,7 @@ impl TopK {
let scratch_rows =
row_converter.empty_rows(batch_size, ESTIMATED_BYTES_PER_ROW * batch_size);
- let prefix_row_converter = if common_sort_prefix.is_empty() {
+ let common_prefix_row_converter = if common_sort_prefix.is_empty() {
None
} else {
let input_sort_fields = build_sort_fields(&common_sort_prefix, &schema)?;
@@ -222,7 +301,7 @@ impl TopK {
row_converter,
scratch_rows,
heap: TopKHeap::new(k),
- common_sort_prefix_converter: prefix_row_converter,
+ common_sort_prefix_converter: common_prefix_row_converter,
common_sort_prefix: Arc::from(common_sort_prefix),
finished: false,
filter,
@@ -314,6 +393,10 @@ impl TopK {
// update the filter representation of our TopK heap
self.update_filter()?;
+ } else {
+ // The heap did not change, but this batch's prefix may still prove
+ // that no later rows can enter the TopK.
+ self.attempt_early_completion(&batch)?;
}
Ok(())
@@ -357,18 +440,18 @@ impl TopK {
return Ok(());
};
- let new_threshold_row = &max_row.row;
+ let new_threshold_row = max_row.row();
// Fast path: check if the current value in topk is better than what is
// currently set in the filter with a read only lock
let needs_update = self
.filter
.read()
- .threshold_row
+ .shared_threshold
.as_ref()
- .map(|current_row| {
+ .map(|current_threshold| {
// new < current means new threshold is more selective
- new_threshold_row < current_row
+ new_threshold_row < current_threshold.full_sort_key_row()
})
.unwrap_or(true); // No current threshold, so we need to set one
@@ -385,33 +468,25 @@ impl TopK {
// Build the filter expression OUTSIDE any synchronization
let predicate = Self::build_filter_expression(&self.expr, &thresholds)?;
- let new_threshold = new_threshold_row.to_vec();
+ let new_threshold = TopKThreshold::new(
+ new_threshold_row.to_vec(),
+ self.encode_topk_common_prefix_row(max_row)?,
+ );
// update the threshold. Since there was a lock gap, we must check if it is still the best
// may have changed while we were building the expression without the lock
let mut filter = self.filter.write();
- let old_threshold = filter.threshold_row.take();
-
- // Update filter if we successfully updated the threshold
- // (or if there was no previous threshold and we're the first)
- match old_threshold {
- Some(old_threshold) => {
- // new threshold is still better than the old one
- if new_threshold.as_slice() < old_threshold.as_slice() {
- filter.threshold_row = Some(new_threshold);
- } else {
- // some other thread updated the threshold to a better
- // one while we were building so there is no need to
- // update the filter
- filter.threshold_row = Some(old_threshold);
- return Ok(());
- }
- }
- None => {
- // No previous threshold, so we can set the new one
- filter.threshold_row = Some(new_threshold);
- }
- };
+ let still_needs_update = filter
+ .shared_threshold
+ .as_ref()
+ .map(|current| new_threshold.is_more_selective_than(current))
+ .unwrap_or(true);
+ if !still_needs_update {
+ // some other thread updated the threshold to a better one while we
+ // were building so there is no need to update the filter
+ return Ok(());
+ }
+ filter.shared_threshold = Some(new_threshold);
// Update the filter expression
if let Some(pred) = predicate
@@ -509,78 +584,109 @@ impl TopK {
Ok(dynamic_predicate)
}
- /// If input ordering shares a common sort prefix with the TopK, and if the TopK's heap is full,
+ /// If input ordering shares a common sort prefix with the TopK,
/// check if the computation can be finished early.
- /// This is the case if the last row of the current batch is strictly greater than the max row in the heap,
- /// comparing only on the shared prefix columns.
+ ///
+ /// This is the case if the last row of the current batch is strictly
+ /// greater than either the shared dynamic-filter threshold prefix or the max
+ /// row in the local heap, comparing only on the shared prefix columns.
fn attempt_early_completion(&mut self, batch: &RecordBatch) -> Result<()> {
// Early exit if the batch is empty as there is no last row to extract from it.
if batch.num_rows() == 0 {
return Ok(());
}
- // prefix_row_converter is only `Some` if the input ordering has a common prefix with the TopK,
+ // common_prefix_row_converter is only `Some` if the input ordering has a common prefix with the TopK,
// so early exit if it is `None`.
let Some(prefix_converter) = &self.common_sort_prefix_converter else {
return Ok(());
};
- // Early exit if the heap is not full (`heap.max()` only returns `Some` if the heap is full).
- let Some(max_topk_row) = self.heap.max() else {
- return Ok(());
- };
-
// Evaluate the prefix for the last row of the current batch.
let last_row_idx = batch.num_rows() - 1;
let mut batch_prefix_scratch =
prefix_converter.empty_rows(1, ESTIMATED_BYTES_PER_ROW); // 1 row with capacity ESTIMATED_BYTES_PER_ROW
- self.compute_common_sort_prefix(batch, last_row_idx, &mut batch_prefix_scratch)?;
-
- // Retrieve the max row from the heap.
- let store_entry = self
- .heap
- .store
- .get(max_topk_row.batch_id)
- .ok_or(internal_datafusion_err!("Invalid batch id in topK heap"))?;
- let max_batch = &store_entry.batch;
- let mut heap_prefix_scratch =
- prefix_converter.empty_rows(1, ESTIMATED_BYTES_PER_ROW); // 1 row with capacity ESTIMATED_BYTES_PER_ROW
- self.compute_common_sort_prefix(
- max_batch,
- max_topk_row.index,
- &mut heap_prefix_scratch,
+ self.append_common_prefix_row(
+ prefix_converter,
+ batch,
+ last_row_idx,
+ &mut batch_prefix_scratch,
)?;
+ let batch_common_prefix_row = batch_prefix_scratch.row(0);
+ let batch_common_prefix = batch_common_prefix_row.as_ref();
+
+ let finished_by_shared_threshold = self
+ .filter
+ .read()
+ .shared_threshold
+ .as_ref()
+ .and_then(TopKThreshold::common_prefix_row)
+ .map(|common_prefix_row| batch_common_prefix > common_prefix_row)
+ .unwrap_or(false);
+ if finished_by_shared_threshold {
+ self.finished = true;
+ return Ok(());
+ }
+
+ // Early exit if the heap is not full (`heap.max()` only returns `Some` if the heap is full).
+ let Some(max_topk_row) = self.heap.max() else {
+ return Ok(());
+ };
+
+ // Encode the local heap max row's common-prefix projection.
+ let Some(heap_common_prefix_row) =
+ self.encode_topk_common_prefix_row(max_topk_row)?
+ else {
+ return Ok(());
+ };
// If the last row's prefix is strictly greater than the max prefix, mark as finished.
- if batch_prefix_scratch.row(0).as_ref() > heap_prefix_scratch.row(0).as_ref() {
+ if batch_common_prefix > heap_common_prefix_row.as_slice() {
self.finished = true;
}
Ok(())
}
- // Helper function to compute the prefix for a given batch and row index, storing the result in scratch.
- fn compute_common_sort_prefix(
+ fn encode_topk_common_prefix_row(
&self,
+ topk_row: &TopKRow,
+ ) -> Result<Option<Vec<u8>>> {
+ let Some(prefix_converter) = &self.common_sort_prefix_converter else {
+ return Ok(None);
+ };
+
+ let store_entry = self
+ .heap
+ .store
+ .get(topk_row.batch_id)
+ .ok_or(internal_datafusion_err!("Invalid batch id in topK heap"))?;
+ let mut scratch = prefix_converter.empty_rows(1, ESTIMATED_BYTES_PER_ROW);
+ self.append_common_prefix_row(
+ prefix_converter,
+ &store_entry.batch,
+ topk_row.index,
+ &mut scratch,
+ )?;
+ Ok(Some(scratch.row(0).as_ref().to_vec()))
+ }
+
+ fn append_common_prefix_row(
+ &self,
+ prefix_converter: &RowConverter,
batch: &RecordBatch,
- last_row_idx: usize,
+ row_idx: usize,
scratch: &mut Rows,
) -> Result<()> {
- let last_row: Vec<ArrayRef> = self
+ let row = batch.slice(row_idx, 1);
+ let prefix_columns: Vec<ArrayRef> = self
.common_sort_prefix
.iter()
- .map(|expr| {
- expr.expr
- .evaluate(&batch.slice(last_row_idx, 1))?
- .into_array(1)
- })
+ .map(|expr| expr.expr.evaluate(&row)?.into_array(1))
.collect::<Result<_>>()?;
- self.common_sort_prefix_converter
- .as_ref()
- .unwrap()
- .append(scratch, &last_row)?;
+ prefix_converter.append(scratch, &prefix_columns)?;
Ok(())
}
@@ -602,8 +708,9 @@ impl TopK {
} = self;
let _timer = metrics.baseline.elapsed_compute().timer(); // time updated on drop
- // Mark the dynamic filter as complete now that TopK processing is finished.
- filter.read().expr().mark_complete();
+ // Mark this local TopK as emitted. For shared filters, the final
+ // local emitter marks the dynamic filter complete.
+ filter.read().mark_topk_emitted();
// break into record batches as needed
let mut batches = vec![];
@@ -1066,7 +1173,7 @@ mod tests {
use arrow::datatypes::{DataType, Field, Schema};
use arrow_schema::SortOptions;
use datafusion_common::assert_batches_eq;
- use datafusion_physical_expr::expressions::col;
+ use datafusion_physical_expr::{DynamicFilterTracking, expressions::col};
use futures::TryStreamExt;
/// This test ensures the size calculation is correct for RecordBatches with multiple columns.
@@ -1101,86 +1208,110 @@ mod tests {
assert_eq!(record_batch_store.batches_size, 0);
}
- /// Builds an `(a Int32, b Float64)` schema and a `TopK` with full sort
- /// `(a ASC, b ASC)`, input prefix `[a]`, `k = 3`, `batch_size = 2`. Used by
- /// the prefix-completion tests below to keep their per-scenario logic in focus.
- fn build_ab_prefix_topk() -> Result<(Arc<Schema>, TopK)> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("a", DataType::Int32, false),
+ fn make_ab_schema() -> SchemaRef {
+ make_ab_schema_with_nullable_a(false)
+ }
+
+ fn make_ab_schema_with_nullable_a(a_nullable: bool) -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, a_nullable),
Field::new("b", DataType::Float64, false),
- ]));
+ ]))
+ }
+
+ // Local TopK tests use one emitter; shared-filter cases pass the partition count explicitly.
+ fn make_topk_filter() -> Arc<RwLock<TopKDynamicFilters>> {
+ make_shared_topk_filter(1)
+ }
+ fn make_shared_topk_filter(
+ topk_emitter_count: usize,
+ ) -> Arc<RwLock<TopKDynamicFilters>> {
+ Arc::new(RwLock::new(
+ TopKDynamicFilters::new_with_topk_emitter_count(
+ Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true))),
+ topk_emitter_count,
+ ),
+ ))
+ }
+
+ /// Builds the `(a, b)` fixture used by prefix-completion tests:
+ /// full sort `(a, b)`, input prefix `[a]`, `k = 3`, and batch size 2.
+ fn make_ab_topk(
+ schema: SchemaRef,
+ filter: Arc<RwLock<TopKDynamicFilters>>,
+ ) -> Result<TopK> {
+ make_ab_topk_with_options(0, schema, filter, SortOptions::default())
+ }
+
+ fn make_ab_topk_with_options(
+ partition_id: usize,
+ schema: SchemaRef,
+ filter: Arc<RwLock<TopKDynamicFilters>>,
+ a_options: SortOptions,
+ ) -> Result<TopK> {
let sort_expr_a = PhysicalSortExpr {
expr: col("a", schema.as_ref())?,
- options: SortOptions::default(),
+ options: a_options,
};
let sort_expr_b = PhysicalSortExpr {
expr: col("b", schema.as_ref())?,
options: SortOptions::default(),
};
- // Input ordering uses only column "a" (a prefix of the full sort on (a, b)).
- let prefix = vec![sort_expr_a.clone()];
- let full_expr = LexOrdering::from([sort_expr_a, sort_expr_b]);
-
- let topk = TopK::try_new(
- 0,
- Arc::clone(&schema),
- prefix,
- full_expr,
+ TopK::try_new(
+ partition_id,
+ schema,
+ vec![sort_expr_a.clone()],
+ LexOrdering::from([sort_expr_a, sort_expr_b]),
3,
2,
Arc::new(RuntimeEnv::default()),
&ExecutionPlanMetricsSet::new(),
- Arc::new(RwLock::new(TopKDynamicFilters::new(Arc::new(
- DynamicFilterPhysicalExpr::new(vec![], lit(true)),
- )))),
- )?;
- Ok((schema, topk))
+ filter,
+ )
+ }
+
+ fn make_ab_batch(
+ schema: SchemaRef,
+ a: &[Option<i32>],
+ b: &[f64],
+ ) -> Result<RecordBatch> {
+ Ok(RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(Int32Array::from(a.to_vec())) as ArrayRef,
+ Arc::new(Float64Array::from(b.to_vec())) as ArrayRef,
+ ],
+ )?)
+ }
+
+ type AbRow = (Option<i32>, f64);
+
+ fn make_ab_rows_batch(schema: SchemaRef, rows: &[AbRow]) -> Result<RecordBatch> {
+ let (a, b): (Vec<_>, Vec<_>) = rows.iter().copied().unzip();
+ make_ab_batch(schema, &a, &b)
}
- /// This test validates that the `try_finish` method marks the TopK operator as finished
- /// when the prefix (on column "a") of the last row in the current batch is strictly greater
- /// than the max top‑k row.
- /// The full sort expression is defined on both columns ("a", "b"), but the input ordering is only on "a".
#[tokio::test]
- async fn test_try_finish_marks_finished_with_prefix() -> Result<()> {
- let (schema, mut topk) = build_ab_prefix_topk()?;
-
- // Create the first batch with two columns:
- // Column "a": [1, 1, 2], Column "b": [20.0, 15.0, 30.0].
- let array_a1: ArrayRef =
- Arc::new(Int32Array::from(vec![Some(1), Some(1), Some(2)]));
- let array_b1: ArrayRef = Arc::new(Float64Array::from(vec![20.0, 15.0, 30.0]));
- let batch1 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a1, array_b1])?;
-
- // Insert the first batch.
- // At this point the heap is not yet “finished” because the prefix of the last row of the batch
- // is not strictly greater than the prefix of the max top‑k row (both being `2`).
- topk.insert_batch(batch1)?;
- assert!(
- !topk.finished,
- "Expected 'finished' to be false after the first batch."
- );
+ async fn test_early_completion_marks_finished_with_prefix() -> Result<()> {
+ let schema = make_ab_schema();
+ let mut topk = make_ab_topk(Arc::clone(&schema), make_topk_filter())?;
- // Create the second batch with two columns:
- // Column "a": [2, 3], Column "b": [10.0, 20.0].
- let array_a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(2), Some(3)]));
- let array_b2: ArrayRef = Arc::new(Float64Array::from(vec![10.0, 20.0]));
- let batch2 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a2, array_b2])?;
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(1), Some(1), Some(2)],
+ &[20.0, 15.0, 30.0],
+ )?)?;
+ assert!(!topk.finished);
- // Insert the second batch.
- // The last row in this batch has a prefix value of `3`,
- // which is strictly greater than the max top‑k row (with value `2`),
- // so try_finish should mark the TopK as finished.
- topk.insert_batch(batch2)?;
- assert!(
- topk.finished,
- "Expected 'finished' to be true after the second batch."
- );
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(2), Some(3)],
+ &[10.0, 20.0],
+ )?)?;
+ assert!(topk.finished);
- // Verify the TopK correctly emits the top k rows from both batches
- // (the value 10.0 for b is from the second batch).
let results: Vec<_> = topk.emit()?.try_collect().await?;
assert_batches_eq!(
&[
@@ -1201,47 +1332,26 @@ mod tests {
/// Regression test for #22849: a batch whose rows are entirely rejected by the
/// heap's dynamic filter must still trigger `attempt_early_completion` when its
/// last row's prefix is worse than the heap's worst.
- ///
- /// Before the fix, the `!filter.has_true()` short-circuit returned without calling
- /// `attempt_early_completion`. Because the heap's filter is itself derived from the
- /// heap's worst row, a batch from a strictly-worse prefix is exactly the case the
- /// filter rejects entirely — i.e. the very signal the early-exit was designed to
- /// detect was being silently dropped.
#[tokio::test]
- async fn test_try_finish_fires_when_filter_rejects_entire_batch() -> Result<()> {
- let (schema, mut topk) = build_ab_prefix_topk()?;
-
- // Batch 1 fills the heap with (1, 20.0), (1, 15.0), (2, 30.0).
- // heap.max becomes (a=2, b=30.0); update_filter tightens the heap filter to
- // a < 2 OR (a = 2 AND b < 30.0).
- let array_a1: ArrayRef =
- Arc::new(Int32Array::from(vec![Some(1), Some(1), Some(2)]));
- let array_b1: ArrayRef = Arc::new(Float64Array::from(vec![20.0, 15.0, 30.0]));
- let batch1 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a1, array_b1])?;
- topk.insert_batch(batch1)?;
- assert!(
- !topk.finished,
- "Expected 'finished' to be false after batch 1 \
- (last row prefix a=2 equals heap.max prefix a=2, not strictly greater)."
- );
+ async fn test_early_completion_fires_when_filter_rejects_entire_batch() -> Result<()>
+ {
+ let schema = make_ab_schema();
+ let mut topk = make_ab_topk(Arc::clone(&schema), make_topk_filter())?;
- // Batch 2: every row has a=3, so the heap's filter (a < 2 OR (a = 2 AND b < 30))
- // rejects every row. Before the fix, `insert_batch` would short-circuit on
- // `!filter.has_true()` and return without checking the prefix; `finished`
- // would stay false even though no future batch could improve the heap.
- let array_a2: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(3)]));
- let array_b2: ArrayRef = Arc::new(Float64Array::from(vec![10.0, 20.0]));
- let batch2 = RecordBatch::try_new(Arc::clone(&schema), vec![array_a2, array_b2])?;
- topk.insert_batch(batch2)?;
- assert!(
- topk.finished,
- "Expected 'finished' to be true after batch 2 \
- (filter rejected every row, but the batch's last row prefix a=3 \
- is strictly greater than heap.max prefix a=2)."
- );
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(1), Some(1), Some(2)],
+ &[20.0, 15.0, 30.0],
+ )?)?;
+ assert!(!topk.finished);
+
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(3), Some(3)],
+ &[10.0, 20.0],
+ )?)?;
+ assert!(topk.finished);
- // The emitted top-k is unchanged from after batch 1 since none of batch 2's
- // rows could improve the heap.
let results: Vec<_> = topk.emit()?.try_collect().await?;
assert_batches_eq!(
&[
@@ -1259,50 +1369,266 @@ mod tests {
Ok(())
}
- /// This test verifies that the dynamic filter is marked as complete after TopK processing finishes.
#[tokio::test]
- async fn test_topk_marks_filter_complete() -> Result<()> {
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ async fn test_early_completion_fires_when_batch_makes_no_replacements() -> Result<()>
+ {
+ let schema = make_ab_schema();
+ let filter = make_topk_filter();
+ let mut topk = make_ab_topk(Arc::clone(&schema), Arc::clone(&filter))?;
- let sort_expr = PhysicalSortExpr {
- expr: col("a", schema.as_ref())?,
- options: SortOptions::default(),
- };
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(1), Some(1), Some(2)],
+ &[20.0, 15.0, 30.0],
+ )?)?;
+ assert!(!topk.finished);
- let full_expr = LexOrdering::from([sort_expr.clone()]);
- let prefix = vec![sort_expr];
+ let replacements_before = topk.metrics.row_replacements.value();
- // Create a dummy runtime environment and metrics
- let runtime = Arc::new(RuntimeEnv::default());
- let metrics = ExecutionPlanMetricsSet::new();
+ // Keep the dynamic filter permissive so the second batch reaches
+ // `find_new_topk_items`; all of its rows are worse than the heap max,
+ // so this specifically exercises the `replacements == 0` path.
+ filter.read().expr().update(lit(true))?;
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(3), Some(3)],
+ &[10.0, 20.0],
+ )?)?;
+ assert_eq!(topk.metrics.row_replacements.value(), replacements_before);
+ assert!(topk.finished);
- // Create a dynamic filter that we'll check for completion
- let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));
- let dynamic_filter_clone = Arc::clone(&dynamic_filter);
+ let results: Vec<_> = topk.emit()?.try_collect().await?;
+ assert_batches_eq!(
+ &[
+ "+---+------+",
+ "| a | b |",
+ "+---+------+",
+ "| 1 | 15.0 |",
+ "| 1 | 20.0 |",
+ "| 2 | 30.0 |",
+ "+---+------+",
+ ],
+ &results
+ );
- // Create a TopK instance
- let mut topk = TopK::try_new(
+ Ok(())
+ }
+
+ struct SharedPrefixCase {
+ name: &'static str,
+ a_nullable: bool,
+ a_options: SortOptions,
+ threshold_source_rows: &'static [AbRow],
+ lagging_partition_rows: &'static [AbRow],
+ expected_finished: bool,
+ }
+
+ fn assert_shared_prefix_case(case: SharedPrefixCase) -> Result<()> {
+ let schema = make_ab_schema_with_nullable_a(case.a_nullable);
+ let filter = make_shared_topk_filter(2);
+
+ let mut threshold_source = make_ab_topk_with_options(
0,
Arc::clone(&schema),
- prefix,
- full_expr,
+ Arc::clone(&filter),
+ case.a_options,
+ )?;
+ threshold_source.insert_batch(make_ab_rows_batch(
+ Arc::clone(&schema),
+ case.threshold_source_rows,
+ )?)?;
+ assert!(
+ filter
+ .read()
+ .shared_threshold
+ .as_ref()
+ .and_then(TopKThreshold::common_prefix_row)
+ .is_some(),
+ "{}: threshold-source partition should establish the shared prefix threshold",
+ case.name
+ );
+
+ let mut lagging_partition = make_ab_topk_with_options(
+ 1,
+ Arc::clone(&schema),
+ Arc::clone(&filter),
+ case.a_options,
+ )?;
+ lagging_partition
+ .insert_batch(make_ab_rows_batch(schema, case.lagging_partition_rows)?)?;
+
+ assert!(
+ lagging_partition.heap.inner.is_empty(),
+ "{}: lagging partition's local heap should remain empty",
+ case.name
+ );
+ assert_eq!(
+ lagging_partition.finished, case.expected_finished,
+ "{}",
+ case.name
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_shared_filter_can_finish_partition_before_local_heap_is_full() -> Result<()> {
+ assert_shared_prefix_case(SharedPrefixCase {
+ name: "shared threshold should finish lagging partition",
+ a_nullable: false,
+ a_options: SortOptions::default(),
+ threshold_source_rows: &[(Some(1), 20.0), (Some(1), 15.0), (Some(2), 30.0)],
+ lagging_partition_rows: &[(Some(3), 10.0), (Some(3), 20.0)],
+ expected_finished: true,
+ })
+ }
+
+ #[test]
+ fn test_shared_prefix_threshold_boundary_cases() -> Result<()> {
+ for case in [
+ SharedPrefixCase {
+ name: "equal prefix cannot prove completion",
+ a_nullable: false,
+ a_options: SortOptions::default(),
+ threshold_source_rows: &[
+ (Some(1), 20.0),
+ (Some(1), 15.0),
+ (Some(2), 30.0),
+ ],
+ lagging_partition_rows: &[(Some(2), 40.0), (Some(2), 50.0)],
+ expected_finished: false,
+ },
+ SharedPrefixCase {
+ name: "descending prefix uses sort-order row encoding",
+ a_nullable: false,
+ a_options: SortOptions {
+ descending: true,
+ nulls_first: true,
+ },
+ threshold_source_rows: &[
+ (Some(10), 1.0),
+ (Some(10), 2.0),
+ (Some(9), 3.0),
+ ],
+ lagging_partition_rows: &[(Some(8), 1.0), (Some(8), 2.0)],
+ expected_finished: true,
+ },
+ SharedPrefixCase {
+ name: "NULLS LAST prefix uses sort-order row encoding",
+ a_nullable: true,
+ a_options: SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ threshold_source_rows: &[
+ (Some(1), 20.0),
+ (Some(1), 15.0),
+ (Some(2), 30.0),
+ ],
+ lagging_partition_rows: &[(None, 10.0), (None, 20.0)],
+ expected_finished: true,
+ },
+ ] {
+ assert_shared_prefix_case(case)?;
+ }
+ Ok(())
+ }
+
+ fn make_single_column_topk(
+ dynamic_filter: Arc<DynamicFilterPhysicalExpr>,
+ ) -> Result<(SchemaRef, TopK)> {
+ make_single_column_topk_with_filter(
+ 0,
+ Arc::new(RwLock::new(TopKDynamicFilters::new(dynamic_filter))),
+ )
+ }
+
+ fn make_single_column_topk_with_filter(
+ partition_id: usize,
+ filter: Arc<RwLock<TopKDynamicFilters>>,
+ ) -> Result<(SchemaRef, TopK)> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let sort_expr = PhysicalSortExpr {
+ expr: col("a", schema.as_ref())?,
+ options: SortOptions::default(),
+ };
+
+ let topk = TopK::try_new(
+ partition_id,
+ Arc::clone(&schema),
+ vec![sort_expr.clone()],
+ LexOrdering::from([sort_expr]),
2,
10,
- runtime,
- &metrics,
- Arc::new(RwLock::new(TopKDynamicFilters::new(dynamic_filter))),
+ Arc::new(RuntimeEnv::default()),
+ &ExecutionPlanMetricsSet::new(),
+ filter,
)?;
+ Ok((schema, topk))
+ }
+
+ #[tokio::test]
+ async fn test_topk_marks_filter_complete() -> Result<()> {
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));
+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);
+ let (schema, mut topk) = make_single_column_topk(dynamic_filter)?;
+
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(1), Some(2)]));
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![array])?;
topk.insert_batch(batch)?;
- // Call emit to finish TopK processing
let _results: Vec<_> = topk.emit()?.try_collect().await?;
- // After emit is called, the dynamic filter should be marked as complete
- // wait_complete() should return immediately
- dynamic_filter_clone.wait_complete().await;
+ tokio::time::timeout(
+ std::time::Duration::from_secs(1),
+ dynamic_filter_clone.wait_complete(),
+ )
+ .await
+ .expect("single-emitter TopK should mark the dynamic filter complete");
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_shared_topk_filter_completes_after_last_emitter() -> Result<()> {
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));
+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);
+ let shared_filter = Arc::new(RwLock::new(
+ TopKDynamicFilters::new_with_topk_emitter_count(dynamic_filter, 2),
+ ));
+
+ let (schema, mut topk_0) =
+ make_single_column_topk_with_filter(0, Arc::clone(&shared_filter))?;
+ let (_, mut topk_1) =
+ make_single_column_topk_with_filter(1, Arc::clone(&shared_filter))?;
+
+ let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(1), Some(2)]));
+ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![array])?;
+ topk_0.insert_batch(batch)?;
+ let _results: Vec<_> = topk_0.emit()?.try_collect().await?;
+
+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> =
+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter_clone);
+ assert!(
+ matches!(
+ DynamicFilterTracking::classify(&dynamic_filter_expr),
+ DynamicFilterTracking::Watching(_)
+ ),
+ "the shared filter should remain watchable until every TopK emits"
+ );
+
+ let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(6), Some(4), Some(5)]));
+ let batch = RecordBatch::try_new(schema, vec![array])?;
+ topk_1.insert_batch(batch)?;
+ let _results: Vec<_> = topk_1.emit()?.try_collect().await?;
+
+ tokio::time::timeout(
+ std::time::Duration::from_secs(1),
+ dynamic_filter_clone.wait_complete(),
+ )
+ .await
+ .expect("the final shared TopK emitter should mark the dynamic filter complete");
Ok(())
}