Apache DataFusion SQL Query Engine
Add field to DynamicPhysicalExpr to indicate when the filter is complete or updated (#18799)
Lía Adriana7fa2a694bbc1
diff --git a/Cargo.lock b/Cargo.lockindex 1afa1e3491..e0f811f16f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2494,6 +2494,7 @@ dependencies = ["petgraph 0.8.3","rand 0.9.2","rstest",+ "tokio",][[package]]diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.tomlindex 953a46929c..69670941ad 100644--- a/datafusion/physical-expr/Cargo.toml+++ b/datafusion/physical-expr/Cargo.toml@@ -55,6 +55,7 @@ itertools = { workspace = true, features = ["use_std"] }parking_lot = { workspace = true }paste = "^1.0"petgraph = "0.8.3"+tokio = { workspace = true }[dev-dependencies]arrow = { workspace = true, features = ["test_utils"] }@@ -79,3 +80,6 @@ name = "is_null"[[bench]]harness = falsename = "binary_op"++[package.metadata.cargo-machete]+ignored = ["half"]diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rsindex 964a193db8..43a242472b 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs@@ -17,6 +17,7 @@use parking_lot::RwLock;use std::{any::Any, fmt::Display, hash::Hash, sync::Arc};+use tokio::sync::watch;use crate::PhysicalExpr;use arrow::datatypes::{DataType, Schema};@@ -27,6 +28,24 @@ use datafusion_common::{use datafusion_expr::ColumnarValue;use datafusion_physical_expr_common::physical_expr::{DynEq, DynHash};+/// State of a dynamic filter, tracking both updates and completion.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum FilterState {+ /// Filter is in progress and may receive more updates.+ InProgress { generation: u64 },+ /// Filter is complete and will not receive further updates.+ Complete { generation: u64 },+}++impl FilterState {+ fn generation(&self) -> u64 {+ match self {+ FilterState::InProgress { generation }+ | FilterState::Complete { generation } => *generation,+ }+ }+}+/// A dynamic [`PhysicalExpr`] that can be updated by anyone with a reference to it.////// Any `ExecutionPlan` that uses this expression and holds a reference to it internally should probably also@@ -44,6 +63,8 @@ pub struct DynamicFilterPhysicalExpr {remapped_children: Option<Vec<Arc<dyn PhysicalExpr>>>,/// The source of dynamic filters.inner: Arc<RwLock<Inner>>,+ /// Broadcasts filter state (updates and completion) to all waiters.+ state_watch: watch::Sender<FilterState>,/// For testing purposes track the data type and nullability to make sure they don't change./// If they do, there's a bug in the implementation./// But this can have overhead in production, so it's only included in our tests.@@ -57,6 +78,10 @@ struct Inner {/// This is used for [`PhysicalExpr::snapshot_generation`] to have a cheap check for changes.generation: u64,expr: Arc<dyn PhysicalExpr>,+ /// Flag for quick synchronous check if filter is complete.+ /// This is redundant with the watch channel state, but allows us to return immediately+ /// from `wait_complete()` without subscribing if already complete.+ is_complete: bool,}impl Inner {@@ -66,6 +91,7 @@ impl Inner {// This is not currently used anywhere but it seems useful to have this simple distinction.generation: 1,expr,+ is_complete: false,}}@@ -134,10 +160,12 @@ impl DynamicFilterPhysicalExpr {children: Vec<Arc<dyn PhysicalExpr>>,inner: Arc<dyn PhysicalExpr>,) -> Self {+ let (state_watch, _) = watch::channel(FilterState::InProgress { generation: 1 });Self {children,remapped_children: None, // Initially no remapped childreninner: Arc::new(RwLock::new(Inner::new(inner))),+ state_watch,data_type: Arc::new(RwLock::new(None)),nullable: Arc::new(RwLock::new(None)),}@@ -185,7 +213,7 @@ impl DynamicFilterPhysicalExpr {Self::remap_children(&self.children, self.remapped_children.as_ref(), expr)}- /// Update the current expression.+ /// Update the current expression and notify all waiters./// Any children of this expression must be a subset of the original children/// passed to the constructor./// This should be called e.g.:@@ -204,13 +232,68 @@ impl DynamicFilterPhysicalExpr {// Load the current inner, increment generation, and store the new onelet mut current = self.inner.write();+ let new_generation = current.generation + 1;*current = Inner {- generation: current.generation + 1,+ generation: new_generation,expr: new_expr,+ is_complete: current.is_complete,};+ drop(current); // Release the lock before broadcasting++ // Broadcast the new state to all waiters+ let _ = self.state_watch.send(FilterState::InProgress {+ generation: new_generation,+ });Ok(())}+ /// Mark this dynamic filter as complete and broadcast to all waiters.+ ///+ /// This signals that all expected updates have been received.+ /// Waiters using [`Self::wait_complete`] will be notified.+ pub fn mark_complete(&self) {+ let mut current = self.inner.write();+ let current_generation = current.generation;+ current.is_complete = true;+ drop(current);++ // Broadcast completion to all waiters+ let _ = self.state_watch.send(FilterState::Complete {+ generation: current_generation,+ });+ }++ /// Wait asynchronously for any update to this filter.+ ///+ /// This method will return when [`Self::update`] is called and the generation increases.+ /// It does not guarantee that the filter is complete.+ pub async fn wait_update(&self) {+ let mut rx = self.state_watch.subscribe();+ // Get the current generation+ let current_gen = rx.borrow_and_update().generation();++ // Wait until generation increases+ let _ = rx.wait_for(|state| state.generation() > current_gen).await;+ }++ /// Wait asynchronously until this dynamic filter is marked as complete.+ ///+ /// This method returns immediately if the filter is already complete.+ /// Otherwise, it waits until [`Self::mark_complete`] is called.+ ///+ /// Unlike [`Self::wait_update`], this method guarantees that when it returns,+ /// the filter is fully complete with no more updates expected.+ pub async fn wait_complete(&self) {+ if self.inner.read().is_complete {+ return;+ }++ let mut rx = self.state_watch.subscribe();+ let _ = rx+ .wait_for(|state| matches!(state, FilterState::Complete { .. }))+ .await;+ }+fn render(&self,f: &mut std::fmt::Formatter<'_>,@@ -253,6 +336,7 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {children: self.children.clone(),remapped_children: Some(children),inner: Arc::clone(&self.inner),+ state_watch: self.state_watch.clone(),data_type: Arc::clone(&self.data_type),nullable: Arc::clone(&self.nullable),}))@@ -509,4 +593,18 @@ mod test {"Expected err when evaluate is called after changing the expression.");}++ #[tokio::test]+ async fn test_wait_complete_already_complete() {+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![],+ lit(42) as Arc<dyn PhysicalExpr>,+ ));++ // Mark as complete immediately+ dynamic_filter.mark_complete();++ // wait_complete should return immediately+ dynamic_filter.wait_complete().await;+ }}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 109ea479e4..680e4368ae 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -4486,4 +4486,103 @@ mod tests {fn columns(schema: &Schema) -> Vec<String> {schema.fields().iter().map(|f| f.name().clone()).collect()}++ /// This test verifies that the dynamic filter is marked as complete after HashJoinExec finishes building the hash table.+ #[tokio::test]+ async fn test_hash_join_marks_filter_complete() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());+ let left = build_table(+ ("a1", &vec![1, 2, 3]),+ ("b1", &vec![4, 5, 6]),+ ("c1", &vec![7, 8, 9]),+ );+ let right = build_table(+ ("a2", &vec![10, 20, 30]),+ ("b1", &vec![4, 5, 6]),+ ("c2", &vec![70, 80, 90]),+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,+ )];++ // Create a dynamic filter manually+ let dynamic_filter = HashJoinExec::create_dynamic_filter(&on);+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);++ // Create HashJoinExec with the dynamic filter+ let mut join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ )?;+ join.dynamic_filter = Some(HashJoinExecDynamicFilter {+ filter: dynamic_filter,+ build_accumulator: OnceLock::new(),+ });++ // Execute the join+ let stream = join.execute(0, task_ctx)?;+ let _batches = common::collect(stream).await?;++ // After the join completes, the dynamic filter should be marked as complete+ // wait_complete() should return immediately+ dynamic_filter_clone.wait_complete().await;++ Ok(())+ }++ /// This test verifies that the dynamic filter is marked as complete even when the build side is empty.+ #[tokio::test]+ async fn test_hash_join_marks_filter_complete_empty_build_side() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());+ // Empty left side (build side)+ let left = build_table(("a1", &vec![]), ("b1", &vec![]), ("c1", &vec![]));+ let right = build_table(+ ("a2", &vec![10, 20, 30]),+ ("b1", &vec![4, 5, 6]),+ ("c2", &vec![70, 80, 90]),+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,+ )];++ // Create a dynamic filter manually+ let dynamic_filter = HashJoinExec::create_dynamic_filter(&on);+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);++ // Create HashJoinExec with the dynamic filter+ let mut join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ )?;+ join.dynamic_filter = Some(HashJoinExecDynamicFilter {+ filter: dynamic_filter,+ build_accumulator: OnceLock::new(),+ });++ // Execute the join+ let stream = join.execute(0, task_ctx)?;+ let _batches = common::collect(stream).await?;++ // Even with empty build side, the dynamic filter should be marked as complete+ // wait_complete() should return immediately+ dynamic_filter_clone.wait_complete().await;++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex 7cf192bdf6..cb727f40a2 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -410,6 +410,7 @@ impl SharedBuildAccumulator {self.dynamic_filter.update(case_expr)?;}}+ self.dynamic_filter.mark_complete();}Ok(())diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex 0b5ab784df..96356e50d7 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -591,10 +591,13 @@ impl TopK {common_sort_prefix_converter: _,common_sort_prefix: _,finished: _,- filter: _,+ filter,} = self;let _timer = metrics.baseline.elapsed_compute().timer(); // time updated on drop+ // Mark the dynamic filter as complete now that TopK processing is finished.+ filter.read().expr().mark_complete();+// break into record batches as neededlet mut batches = vec![];if let Some(mut batch) = heap.emit()? {@@ -1198,4 +1201,52 @@ mod tests {Ok(())}++ /// This test verifies that the dynamic filter is marked as complete after TopK processing finishes.+ #[tokio::test]+ async fn test_topk_marks_filter_complete() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));++ let sort_expr = PhysicalSortExpr {+ expr: col("a", schema.as_ref())?,+ options: SortOptions::default(),+ };++ let full_expr = LexOrdering::from([sort_expr.clone()]);+ let prefix = vec![sort_expr];++ // Create a dummy runtime environment and metrics+ let runtime = Arc::new(RuntimeEnv::default());+ let metrics = ExecutionPlanMetricsSet::new();++ // Create a dynamic filter that we'll check for completion+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));+ let dynamic_filter_clone = Arc::clone(&dynamic_filter);++ // Create a TopK instance+ let mut topk = TopK::try_new(+ 0,+ Arc::clone(&schema),+ prefix,+ full_expr,+ 2,+ 10,+ runtime,+ &metrics,+ Arc::new(RwLock::new(TopKDynamicFilters::new(dynamic_filter))),+ )?;++ let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), Some(1), Some(2)]));+ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![array])?;+ topk.insert_batch(batch)?;++ // Call emit to finish TopK processing+ let _results: Vec<_> = topk.emit()?.try_collect().await?;++ // After emit is called, the dynamic filter should be marked as complete+ // wait_complete() should return immediately+ dynamic_filter_clone.wait_complete().await;++ Ok(())+ }}