Skip to content

Apache DataFusion SQL Query Engine

fix: give each SortExec reset_state copy its own TopK dynamic filter (revert #26670) (#27142)

RAGHVENDRA KUMAR YADAV25beec8ebe6e
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 53c2929d4a..5c91c3a834 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -1755,8 +1755,8 @@ impl ExecutionPlan for SortExec {
// we need to reset.
if let Some(filter) = &new_sort.filter {
filter.read().expr().reset();
+ new_sort.filter = Some(new_sort.create_filter());
}
- new_sort.rebuild_filter_for_current_partitioning();
new_sort.metrics_set = ExecutionPlanMetricsSet::new();
new_sort.sort_concurrency = Arc::new(OnceLock::new());
@@ -4764,4 +4764,87 @@ mod tests {
assert_eq!(num_rows(&topk(false), None)?, Precision::Exact(10));
Ok(())
}
+
+ /// Regression test for <https://github.com/apache/datafusion/issues/27130>
+ ///
+ /// Plans derived from the same TopK plan via `reset_state` must not share
+ /// dynamic filter state. This models a cached `LIMIT 1` plan that is reset
+ /// and executed by two queries at the same time: both copies are created
+ /// before either executes, and the first one to finish must not cause the
+ /// other to return an empty result.
+ #[tokio::test]
+ async fn test_topk_reset_state_copies_are_independent() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
+ )?;
+ let input =
+ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let sort: Arc<dyn ExecutionPlan> = Arc::new(
+ SortExec::new(
+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),
+ input,
+ )
+ .with_fetch(Some(1)),
+ );
+
+ // Both executions are prepared before either runs
+ let first = Arc::clone(&sort).reset_state()?;
+ let second = Arc::clone(&sort).reset_state()?;
+
+ let task_ctx = Arc::new(TaskContext::default());
+ let first_result = collect(first, Arc::clone(&task_ctx)).await?;
+ let second_result = collect(second, task_ctx).await?;
+
+ assert_snapshot!(batches_to_string(&first_result), @r"
+ +---+
+ | a |
+ +---+
+ | 1 |
+ +---+
+ ");
+ assert_snapshot!(batches_to_string(&second_result), @r"
+ +---+
+ | a |
+ +---+
+ | 1 |
+ +---+
+ ");
+
+ Ok(())
+ }
+
+ /// A scan below a TopK keeps the dynamic filter it was given at planning
+ /// time. After the plan has run once, `reset_state` must clear that
+ /// filter's bounds, or running the reset plan again can return too few rows.
+ #[tokio::test]
+ async fn test_topk_reset_state_clears_pushed_down_filter() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
+ )?;
+ let input =
+ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let sort: Arc<dyn ExecutionPlan> = Arc::new(
+ SortExec::new(
+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),
+ input,
+ )
+ .with_fetch(Some(1)),
+ );
+ // The filter a scan would have received during filter pushdown
+ let pushed_down = sort.dynamic_expressions_produced();
+ assert_eq!(pushed_down.len(), 1);
+ let pushed_down = Arc::clone(&pushed_down[0]);
+
+ collect(Arc::clone(&sort), Arc::new(TaskContext::default())).await?;
+ assert_snapshot!(pushed_down.to_string(), @"DynamicFilter [ a@0 IS NULL OR a@0 < 1 ]");
+
+ let _reset = Arc::clone(&sort).reset_state()?;
+ assert_snapshot!(pushed_down.to_string(), @"DynamicFilter [ true ]");
+
+ Ok(())
+ }
}
diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.slt
index 4da264bcd0..edba7eedc1 100644
--- a/datafusion/sqllogictest/test_files/cte.slt
+++ b/datafusion/sqllogictest/test_files/cte.slt
@@ -1993,86 +1993,3 @@ SET datafusion.execution.target_partitions = 4;
statement ok
RESET datafusion.execution.parquet.pushdown_filters;
-
-# TopK must reset its dynamic filter in place so the Parquet scan keeps receiving
-# updated bounds after each recursive iteration (#26642). The minimum is in the
-# first row group, so each iteration should scan only 10 rows instead of 100.
-statement ok
-SET datafusion.execution.target_partitions = 1;
-
-statement ok
-SET datafusion.execution.batch_size = 10;
-
-statement ok
-SET datafusion.execution.parquet.pushdown_filters = true;
-
-statement ok
-COPY (
- SELECT CASE WHEN value = 1 THEN 1 ELSE 102 - value END AS k
- FROM generate_series(1, 100)
-) TO 'test_files/scratch/cte/topk_dynamic_filter.parquet'
-OPTIONS ('format.max_row_group_size' '10');
-
-statement ok
-CREATE EXTERNAL TABLE topk_dynamic_filter STORED AS PARQUET
-LOCATION 'test_files/scratch/cte/topk_dynamic_filter.parquet';
-
-query I
-WITH RECURSIVE w(n) AS (
- SELECT 1
- UNION ALL
- SELECT w.n + s.k
- FROM w CROSS JOIN (
- SELECT k FROM topk_dynamic_filter ORDER BY k LIMIT 1
- ) s
- WHERE w.n < 4
-)
-SELECT * FROM w ORDER BY n;
-----
-1
-2
-3
-4
-
-statement ok
-SET datafusion.explain.analyze_categories = 'rows';
-
-query TT
-EXPLAIN ANALYZE
-WITH RECURSIVE w(n) AS (
- SELECT 1
- UNION ALL
- SELECT w.n + s.k
- FROM w CROSS JOIN (
- SELECT k FROM topk_dynamic_filter ORDER BY k LIMIT 1
- ) s
- WHERE w.n < 4
-)
-SELECT * FROM w ORDER BY n;
-----
-Plan with Metrics
-01)SortExec: expr=[n@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=4, output_batches=1, spill_count=0, spilled_rows=0]
-02)--RecursiveQueryExec: name=w, is_distinct=false, metrics=[output_rows=4, output_batches=4]
-03)----ProjectionExec: expr=[CAST(1 AS Int64;N) as n], metrics=[output_rows=1, output_batches=1]
-04)------PlaceholderRowExec, metrics=[]
-05)----ProjectionExec: expr=[n@0 + k@1 as n], metrics=[]
-06)------CrossJoinExec, metrics=[]
-07)--------FilterExec: n@0 < 4, metrics=[]
-08)----------WorkTableExec: name=w, metrics=[]
-09)--------SortExec: TopK(fetch=1), expr=[k@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[]
-10)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/cte/topk_dynamic_filter.parquet]]}, projection=[k], file_type=parquet, predicate=DynamicFilter [ true ], sort_order_for_reorder=[k@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, metrics=[output_rows=30, output_batches=3, files_ranges_pruned_statistics=3 total → 3 matched, row_groups_pruned_statistics=30 total → 30 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=3, files_processed=3, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=30, pushdown_rows_pruned=0, predicate_cache_inner_records=30, predicate_cache_records=30, scan_efficiency_ratio=9.96% (279/2.80 K)]
-
-statement ok
-DROP TABLE topk_dynamic_filter;
-
-statement ok
-RESET datafusion.explain.analyze_categories;
-
-statement ok
-SET datafusion.execution.target_partitions = 4;
-
-statement ok
-RESET datafusion.execution.batch_size;
-
-statement ok
-RESET datafusion.execution.parquet.pushdown_filters;