Apache DataFusion SQL Query Engine
fix: give each SortExec reset_state copy its own TopK dynamic filter (revert #26670) (#27142)
RAGHVENDRA KUMAR YADAV25beec8ebe6e
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 53c2929d4a..5c91c3a834 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -1755,8 +1755,8 @@ impl ExecutionPlan for SortExec {// we need to reset.if let Some(filter) = &new_sort.filter {filter.read().expr().reset();+ new_sort.filter = Some(new_sort.create_filter());}- new_sort.rebuild_filter_for_current_partitioning();new_sort.metrics_set = ExecutionPlanMetricsSet::new();new_sort.sort_concurrency = Arc::new(OnceLock::new());@@ -4764,4 +4764,87 @@ mod tests {assert_eq!(num_rows(&topk(false), None)?, Precision::Exact(10));Ok(())}++ /// Regression test for <https://github.com/apache/datafusion/issues/27130>+ ///+ /// Plans derived from the same TopK plan via `reset_state` must not share+ /// dynamic filter state. This models a cached `LIMIT 1` plan that is reset+ /// and executed by two queries at the same time: both copies are created+ /// before either executes, and the first one to finish must not cause the+ /// other to return an empty result.+ #[tokio::test]+ async fn test_topk_reset_state_copies_are_independent() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],+ )?;+ let input =+ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let sort: Arc<dyn ExecutionPlan> = Arc::new(+ SortExec::new(+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),+ input,+ )+ .with_fetch(Some(1)),+ );++ // Both executions are prepared before either runs+ let first = Arc::clone(&sort).reset_state()?;+ let second = Arc::clone(&sort).reset_state()?;++ let task_ctx = Arc::new(TaskContext::default());+ let first_result = collect(first, Arc::clone(&task_ctx)).await?;+ let second_result = collect(second, task_ctx).await?;++ assert_snapshot!(batches_to_string(&first_result), @r"+ +---++ | a |+ +---++ | 1 |+ +---++ ");+ assert_snapshot!(batches_to_string(&second_result), @r"+ +---++ | a |+ +---++ | 1 |+ +---++ ");++ Ok(())+ }++ /// A scan below a TopK keeps the dynamic filter it was given at planning+ /// time. After the plan has run once, `reset_state` must clear that+ /// filter's bounds, or running the reset plan again can return too few rows.+ #[tokio::test]+ async fn test_topk_reset_state_clears_pushed_down_filter() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],+ )?;+ let input =+ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;+ let sort: Arc<dyn ExecutionPlan> = Arc::new(+ SortExec::new(+ [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),+ input,+ )+ .with_fetch(Some(1)),+ );+ // The filter a scan would have received during filter pushdown+ let pushed_down = sort.dynamic_expressions_produced();+ assert_eq!(pushed_down.len(), 1);+ let pushed_down = Arc::clone(&pushed_down[0]);++ collect(Arc::clone(&sort), Arc::new(TaskContext::default())).await?;+ assert_snapshot!(pushed_down.to_string(), @"DynamicFilter [ a@0 IS NULL OR a@0 < 1 ]");++ let _reset = Arc::clone(&sort).reset_state()?;+ assert_snapshot!(pushed_down.to_string(), @"DynamicFilter [ true ]");++ Ok(())+ }}diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex 4da264bcd0..edba7eedc1 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -1993,86 +1993,3 @@ SET datafusion.execution.target_partitions = 4;statement okRESET datafusion.execution.parquet.pushdown_filters;--# TopK must reset its dynamic filter in place so the Parquet scan keeps receiving-# updated bounds after each recursive iteration (#26642). The minimum is in the-# first row group, so each iteration should scan only 10 rows instead of 100.-statement ok-SET datafusion.execution.target_partitions = 1;--statement ok-SET datafusion.execution.batch_size = 10;--statement ok-SET datafusion.execution.parquet.pushdown_filters = true;--statement ok-COPY (- SELECT CASE WHEN value = 1 THEN 1 ELSE 102 - value END AS k- FROM generate_series(1, 100)-) TO 'test_files/scratch/cte/topk_dynamic_filter.parquet'-OPTIONS ('format.max_row_group_size' '10');--statement ok-CREATE EXTERNAL TABLE topk_dynamic_filter STORED AS PARQUET-LOCATION 'test_files/scratch/cte/topk_dynamic_filter.parquet';--query I-WITH RECURSIVE w(n) AS (- SELECT 1- UNION ALL- SELECT w.n + s.k- FROM w CROSS JOIN (- SELECT k FROM topk_dynamic_filter ORDER BY k LIMIT 1- ) s- WHERE w.n < 4-)-SELECT * FROM w ORDER BY n;------1-2-3-4--statement ok-SET datafusion.explain.analyze_categories = 'rows';--query TT-EXPLAIN ANALYZE-WITH RECURSIVE w(n) AS (- SELECT 1- UNION ALL- SELECT w.n + s.k- FROM w CROSS JOIN (- SELECT k FROM topk_dynamic_filter ORDER BY k LIMIT 1- ) s- WHERE w.n < 4-)-SELECT * FROM w ORDER BY n;------Plan with Metrics-01)SortExec: expr=[n@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[output_rows=4, output_batches=1, spill_count=0, spilled_rows=0]-02)--RecursiveQueryExec: name=w, is_distinct=false, metrics=[output_rows=4, output_batches=4]-03)----ProjectionExec: expr=[CAST(1 AS Int64;N) as n], metrics=[output_rows=1, output_batches=1]-04)------PlaceholderRowExec, metrics=[]-05)----ProjectionExec: expr=[n@0 + k@1 as n], metrics=[]-06)------CrossJoinExec, metrics=[]-07)--------FilterExec: n@0 < 4, metrics=[]-08)----------WorkTableExec: name=w, metrics=[]-09)--------SortExec: TopK(fetch=1), expr=[k@0 ASC NULLS LAST], preserve_partitioning=[false], metrics=[]-10)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/cte/topk_dynamic_filter.parquet]]}, projection=[k], file_type=parquet, predicate=DynamicFilter [ true ], sort_order_for_reorder=[k@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, metrics=[output_rows=30, output_batches=3, files_ranges_pruned_statistics=3 total → 3 matched, row_groups_pruned_statistics=30 total → 30 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=3, files_processed=3, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=30, pushdown_rows_pruned=0, predicate_cache_inner_records=30, predicate_cache_records=30, scan_efficiency_ratio=9.96% (279/2.80 K)]--statement ok-DROP TABLE topk_dynamic_filter;--statement ok-RESET datafusion.explain.analyze_categories;--statement ok-SET datafusion.execution.target_partitions = 4;--statement ok-RESET datafusion.execution.batch_size;--statement ok-RESET datafusion.execution.parquet.pushdown_filters;