Apache DataFusion SQL Query Engine
feat(physical-expr): DynamicFilterTracker for cheap dynamic-filter change detection (#22460)
Adrian Garcia Badaracco4c0f94476f53
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex f138a26bf4..5b40a947d9 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -52,9 +52,7 @@ use datafusion_common::{ColumnStatistics, Result, ScalarValue, Statistics, exec_use datafusion_datasource::{PartitionedFile, TableSchema};use datafusion_physical_expr::simplifier::PhysicalExprSimplifier;use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;-use datafusion_physical_expr_common::physical_expr::{- PhysicalExpr, is_dynamic_physical_expr,-};+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::metrics::{BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricCategory,@@ -618,18 +616,19 @@ impl ParquetMorselizer {.with_category(MetricCategory::Rows).global_counter("num_predicate_creation_errors");- // Apply literal replacements to projection and predicate- let file_pruner = predicate- .as_ref()- .filter(|p| is_dynamic_physical_expr(p) || partitioned_file.has_statistics())- .and_then(|p| {- FilePruner::try_new(- Arc::clone(p),- &logical_file_schema,- &partitioned_file,- predicate_creation_errors.clone(),- )- });+ // `FilePruner::try_new` decides whether a pruner is worthwhile (it needs+ // a statistics struct, and either real column statistics or a dynamic+ // filter that can prune via partition-value folding) and returns `None`+ // otherwise. For a static predicate the pruner's tracker reports no+ // changes, so it runs once and adds no ongoing cost.+ let file_pruner = predicate.as_ref().and_then(|p| {+ FilePruner::try_new(+ Arc::clone(p),+ &logical_file_schema,+ &partitioned_file,+ predicate_creation_errors.clone(),+ )+ });Ok(PreparedParquetOpen {partition_index: self.partition_index,@@ -677,30 +676,21 @@ impl PreparedParquetOpen {/// Returns `None` if the file can be skipped completely.fn prune_file(mut self) -> Result<Option<Self>> {// Prune this file using the file level statistics and partition values.- // Since dynamic filters may have been updated since planning it is possible that we are able- // to prune files now that we couldn't prune at planning time.- // It is assumed that there is no point in doing pruning here if the predicate is not dynamic,- // as it would have been done at planning time.- // We'll also check this after every record batch we read,- // and if at some point we are able to prove we can prune the file using just the file level statistics- // we can end the stream early.- //- // Make a FilePruner only if there is either- // 1. a dynamic expr in the predicate- // 2. the file has file-level statistics.- //- // File-level statistics may prune the file without loading- // any row groups or metadata.+ // Since dynamic filters may have been updated since planning it is+ // possible that we are able to prune files now that we couldn't prune at+ // planning time. The `FilePruner` (built when the predicate is dynamic or+ // the file carries statistics) also watches any still-active dynamic+ // filter, so the+ // `EarlyStoppingStream` wrapping the scan can re-check after each batch+ // and end the stream early once a tightened filter proves the file can+ // be skipped.//- // Dynamic filters may prune the file after initial- // planning, as the dynamic filter is updated during- // execution.- //- // The case where there is a dynamic filter but no- // statistics corresponds to a dynamic filter that- // references partition columns. While rare, this is possible- // e.g. `select * from table order by partition_col limit- // 10` could hit this condition.+ // File-level statistics may prune the file without loading any row+ // groups or metadata. Partition column predicates are already folded to+ // literals (see `replace_columns_with_literals` above), so a dynamic+ // filter that references only partition columns can prune here too even+ // when the file has no column statistics, e.g.+ // `select * from t order by partition_col limit 10`.if let Some(file_pruner) = &mut self.file_pruner&& file_pruner.should_prune()?{@@ -1247,16 +1237,21 @@ impl RowGroupsPrunedParquetOpen {}.into_stream();- // Wrap the stream so a dynamic filter can stop the file scan early.- if let Some(file_pruner) = prepared.file_pruner {- Ok(EarlyStoppingStream::new(- stream,- file_pruner,- files_ranges_pruned_statistics,- )- .boxed())- } else {- Ok(stream)+ // Wrap the stream so a dynamic filter can stop the file scan early, but+ // only when the pruner is still watching a filter that can change+ // mid-scan. For a static (or already-complete) predicate the up-front+ // `prune_file` check already captured everything that can be pruned, so+ // per-batch re-checking would only add overhead.+ match prepared.file_pruner {+ Some(file_pruner) if file_pruner.is_watching() => {+ Ok(EarlyStoppingStream::new(+ stream,+ file_pruner,+ files_ranges_pruned_statistics,+ )+ .boxed())+ }+ _ => Ok(stream),}}}diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 887ed73745..0e0efaf758 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -870,6 +870,12 @@ pub fn snapshot_generation(expr: &Arc<dyn PhysicalExpr>) -> u64 {/// Check if the given `PhysicalExpr` is dynamic./// Internally this calls [`snapshot_generation`] to check if the generation is non-zero,/// any dynamic `PhysicalExpr` should have a non-zero generation.+#[deprecated(+ since = "55.0.0",+ note = "Downcast to `DynamicFilterPhysicalExpr`, or use \+ `DynamicFilterTracking::classify(expr).contains_dynamic_filter()` from \+ `datafusion_physical_expr`"+)]pub fn is_dynamic_physical_expr(expr: &Arc<dyn PhysicalExpr>) -> bool {// If the generation is non-zero, then this `PhysicalExpr` is dynamic.snapshot_generation(expr) != 0diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rssimilarity index 91%rename from datafusion/physical-expr/src/expressions/dynamic_filters.rsrename to datafusion/physical-expr/src/expressions/dynamic_filters/mod.rsindex 5b9de88216..9fe3feb586 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs@@ -29,6 +29,9 @@ use datafusion_common::{use datafusion_expr::ColumnarValue;use datafusion_physical_expr_common::physical_expr::DynHash;+mod tracker;+pub use tracker::{DynamicFilterTracker, DynamicFilterTracking};+/// State of a dynamic filter, tracking both updates and completion.#[derive(Debug, Clone, Copy, PartialEq, Eq)]enum FilterState {@@ -326,6 +329,31 @@ impl DynamicFilterPhysicalExpr {.await;}+ /// Returns `true` if this filter has been marked complete via+ /// [`Self::mark_complete`] and will therefore never change again.+ pub(crate) fn is_complete(&self) -> bool {+ self.inner.read().is_complete+ }++ /// Subscribe to this filter's updates for cheap, synchronous change+ /// detection.+ ///+ /// The returned [`DynamicFilterSubscription`] lets a consumer poll whether+ /// the filter's expression has advanced since it last looked, without+ /// re-walking a predicate tree or re-deriving a generation on every check.+ /// This is the building block used by [`DynamicFilterTracker`] to watch+ /// every dynamic filter inside a (possibly composite) predicate.+ pub(crate) fn subscribe(&self) -> DynamicFilterSubscription {+ let mut receiver = self.state_watch.subscribe();+ // Mark the current state as already-seen so the first `observe()` only+ // reports updates that happen *after* subscription.+ let last_generation = receiver.borrow_and_update().generation();+ DynamicFilterSubscription {+ receiver,+ last_generation,+ }+ }+/// Check if this dynamic filter is being actively used by any consumers.////// Returns `true` if there are references beyond the producer (e.g., the HashJoinExec@@ -522,6 +550,77 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {}}+/// The result of polling a [`DynamicFilterSubscription`].+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) struct DynamicFilterChange {+ /// The filter's expression advanced since the previous observation.+ pub(crate) changed: bool,+ /// The filter has been marked complete; it will never change again and the+ /// subscription can be dropped.+ pub(crate) complete: bool,+}++/// A cheap, synchronous handle for observing updates to a single+/// [`DynamicFilterPhysicalExpr`].+///+/// Obtained via [`DynamicFilterPhysicalExpr::subscribe`]. Steady-state polling+/// via [`Self::observe`] is a single atomic load (the underlying+/// [`tokio::sync::watch`] version counter); the lock is only taken when the+/// filter has actually been updated.+#[derive(Debug)]+pub(crate) struct DynamicFilterSubscription {+ receiver: watch::Receiver<FilterState>,+ /// Last generation we reported as "seen". Used to distinguish a real+ /// expression update from a bare [`DynamicFilterPhysicalExpr::mark_complete`]+ /// (which re-broadcasts the current generation without changing the+ /// expression).+ last_generation: u64,+}++impl DynamicFilterSubscription {+ /// Observe the latest state of the filter.+ ///+ /// Reports whether the filter's expression advanced since the previous call+ /// and whether it has since been marked complete. Cheap when nothing has+ /// changed: a single atomic comparison with no lock acquisition.+ pub(crate) fn observe(&mut self) -> DynamicFilterChange {+ match self.receiver.has_changed() {+ Ok(true) => {+ let state = *self.receiver.borrow_and_update();+ let changed = state.generation() > self.last_generation;+ if changed {+ self.last_generation = state.generation();+ }+ DynamicFilterChange {+ changed,+ complete: matches!(state, FilterState::Complete { .. }),+ }+ }+ Ok(false) => DynamicFilterChange {+ changed: false,+ complete: false,+ },+ // The watch sender lives inside the predicate's+ // `DynamicFilterPhysicalExpr`, which the owner of this subscription+ // keeps alive, so observing a dropped sender signals a bug rather+ // than normal completion. Flag it loudly in debug builds; in release+ // degrade to "complete" (no further updates are possible) instead of+ // silently masking it.+ Err(_) => {+ debug_assert!(+ false,+ "DynamicFilterSubscription observed a dropped watch sender; \+ the owning predicate should keep it alive"+ );+ DynamicFilterChange {+ changed: false,+ complete: true,+ }+ }+ }+ }+}+/// An atomic counter used to generate monotonic u64 ids.struct ExpressionIdAtomicCounter {inner: AtomicU64,diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters/tracker.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/tracker.rsnew file mode 100644index 0000000000..fd4c18b07e--- /dev/null+++ b/datafusion/physical-expr/src/expressions/dynamic_filters/tracker.rs@@ -0,0 +1,331 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Tracking changes to the dynamic filters inside a predicate.+//!+//! Several operators (Parquet file/row-group pruning, remote execution, ...)+//! hold a predicate that *may* contain one or more+//! [`DynamicFilterPhysicalExpr`] nodes which are updated during execution+//! (e.g. a `TopK` tightening its threshold, or a `HashJoinExec` publishing the+//! build-side bounds). These consumers repeatedly ask two questions:+//!+//! 1. *"Does this predicate contain anything that can still change?"* — to+//! decide whether it is worth setting up runtime re-pruning at all.+//! 2. *"Has it changed since I last looked?"* — to decide whether to rebuild an+//! expensive derived artifact (e.g. a `PruningPredicate`).+//!+//! Historically each call site answered these by recursively folding+//! [`PhysicalExpr::snapshot_generation`] over the whole tree on *every* check+//! and diffing the resulting `u64`. [`DynamicFilterTracker`] replaces that with+//! a single up-front walk that subscribes to each still-incomplete dynamic+//! filter; subsequent checks only poll the (shrinking) set of subscriptions,+//! each of which is a cheap atomic load in the common "nothing changed" case.+//!+//! [`PhysicalExpr::snapshot_generation`]: crate::PhysicalExpr::snapshot_generation++use std::sync::Arc;++use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};++use crate::PhysicalExpr;++use super::{DynamicFilterPhysicalExpr, DynamicFilterSubscription};++/// Classification of a predicate according to the dynamic filters it contains.+///+/// Produced by [`DynamicFilterTracking::classify`] with a single tree walk so+/// callers can answer both "is it worth pruning at all?" and "do I need to keep+/// watching?" without traversing the predicate twice.+#[derive(Debug)]+pub enum DynamicFilterTracking {+ /// The predicate contains no [`DynamicFilterPhysicalExpr`] at all. It is+ /// fully static and will never change.+ Static,+ /// The predicate contains one or more dynamic filters, but all of them have+ /// already been marked complete. Their *current* values may differ from+ /// what was known at planning time (so a one-shot prune is still+ /// worthwhile), but they will not change again — there is nothing to watch.+ AllComplete,+ /// The predicate contains at least one dynamic filter that can still change.+ /// The embedded [`DynamicFilterTracker`] should be polled to detect updates.+ Watching(DynamicFilterTracker),+}++impl DynamicFilterTracking {+ /// Walk `predicate` once and classify its dynamic-filter content,+ /// subscribing to every filter that is not yet complete.+ pub fn classify(predicate: &Arc<dyn PhysicalExpr>) -> Self {+ let mut subscriptions = Vec::new();+ let mut found_any = false;+ predicate+ .apply(|expr| {+ if let Some(filter) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {+ found_any = true;+ // Already-complete filters can never change again, so there+ // is no point subscribing to them.+ if !filter.is_complete() {+ subscriptions.push(filter.subscribe());+ }+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .expect("traversal closure is infallible");++ if !found_any {+ DynamicFilterTracking::Static+ } else if subscriptions.is_empty() {+ DynamicFilterTracking::AllComplete+ } else {+ DynamicFilterTracking::Watching(DynamicFilterTracker { subscriptions })+ }+ }++ /// `true` if the predicate contains any dynamic filter (complete or not),+ /// i.e. its value may differ from what was known at planning time and is+ /// therefore worth re-evaluating at least once.+ pub fn contains_dynamic_filter(&self) -> bool {+ !matches!(self, DynamicFilterTracking::Static)+ }++ /// Mutable access to the underlying tracker when there is still something to+ /// watch.+ pub fn watcher(&mut self) -> Option<&mut DynamicFilterTracker> {+ match self {+ DynamicFilterTracking::Watching(tracker) => Some(tracker),+ _ => None,+ }+ }+}++/// Watches every still-incomplete [`DynamicFilterPhysicalExpr`] reachable from a+/// predicate and reports, cheaply, whether any of them has been updated since+/// the last check.+///+/// Obtain one from [`DynamicFilterTracking::classify`] via+/// [`DynamicFilterTracking::watcher`]; the `Watching` variant carries it only+/// when there is at least one dynamic filter that can still change.+#[derive(Debug)]+pub struct DynamicFilterTracker {+ /// Subscriptions to the not-yet-complete dynamic filters. Entries are+ /// dropped as their filters complete, so the set only shrinks.+ subscriptions: Vec<DynamicFilterSubscription>,+}++impl DynamicFilterTracker {+ /// Returns `true` if any watched filter's expression has advanced since the+ /// previous call.+ ///+ /// Filters that have completed are dropped from the watch set as they are+ /// observed; once every filter has completed this is a no-op that always+ /// returns `false`.+ pub fn changed(&mut self) -> bool {+ let mut changed = false;+ self.subscriptions.retain_mut(|subscription| {+ let change = subscription.observe();+ changed |= change.changed;+ // Keep the subscription only while the filter can still change.+ !change.complete+ });+ changed+ }+}++#[cfg(test)]+impl DynamicFilterTracker {+ /// Build a tracker directly, or `None` if `predicate` has no dynamic filter+ /// that can still change. Test-only; production builds a tracker via+ /// [`DynamicFilterTracking::classify`] + [`DynamicFilterTracking::watcher`].+ fn try_new(predicate: &Arc<dyn PhysicalExpr>) -> Option<Self> {+ match DynamicFilterTracking::classify(predicate) {+ DynamicFilterTracking::Watching(tracker) => Some(tracker),+ DynamicFilterTracking::Static | DynamicFilterTracking::AllComplete => None,+ }+ }++ /// `true` once every watched filter has completed and been dropped.+ fn is_exhausted(&self) -> bool {+ self.subscriptions.is_empty()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ use crate::expressions::{BinaryExpr, col, lit};+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_expr::Operator;++ /// `col > <dynamic>` where the dynamic filter starts as `lit(true)`.+ fn dynamic_predicate() -> (Arc<dyn PhysicalExpr>, Arc<DynamicFilterPhysicalExpr>) {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let column = col("a", &schema).unwrap();+ let filter = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::clone(&column)],+ lit(true),+ ));+ let predicate = Arc::new(BinaryExpr::new(+ column,+ Operator::Gt,+ Arc::clone(&filter) as Arc<dyn PhysicalExpr>,+ )) as Arc<dyn PhysicalExpr>;+ (predicate, filter)+ }++ #[test]+ fn static_predicate_is_not_watched() {+ let predicate = lit(true);+ assert!(matches!(+ DynamicFilterTracking::classify(&predicate),+ DynamicFilterTracking::Static+ ));+ assert!(DynamicFilterTracker::try_new(&predicate).is_none());+ }++ #[test]+ fn already_complete_filter_is_not_watched() {+ let (predicate, filter) = dynamic_predicate();+ filter.mark_complete();++ match DynamicFilterTracking::classify(&predicate) {+ DynamicFilterTracking::AllComplete => {}+ other => panic!("expected AllComplete, got {other:?}"),+ }+ // Still reported as dynamic (worth a one-shot prune)...+ assert!(DynamicFilterTracking::classify(&predicate).contains_dynamic_filter());+ // ...but there is nothing to watch.+ assert!(DynamicFilterTracker::try_new(&predicate).is_none());+ }++ #[test]+ fn detects_update_exactly_once() {+ let (predicate, filter) = dynamic_predicate();+ let mut tracker = DynamicFilterTracker::try_new(&predicate)+ .expect("predicate has an incomplete dynamic filter");++ // No update yet.+ assert!(!tracker.changed());++ filter.update(lit(false)).unwrap();+ // The update is reported once...+ assert!(tracker.changed());+ // ...and not repeatedly.+ assert!(!tracker.changed());+ }++ #[test]+ fn update_before_subscribe_is_not_reported() {+ let (predicate, filter) = dynamic_predicate();++ // An update that happens *before* the tracker subscribes must not be+ // reported on the first poll: `subscribe()` snapshots the current+ // generation via `borrow_and_update()`, so only post-subscription+ // updates count.+ filter.update(lit(false)).unwrap();++ let mut tracker = DynamicFilterTracker::try_new(&predicate)+ .expect("predicate has an incomplete dynamic filter");+ assert!(!tracker.changed());++ // A subsequent update is still reported.+ filter.update(lit(true)).unwrap();+ assert!(tracker.changed());+ }++ #[test]+ fn mark_complete_does_not_count_as_a_change() {+ let (predicate, filter) = dynamic_predicate();+ let mut tracker = DynamicFilterTracker::try_new(&predicate).unwrap();++ filter.update(lit(false)).unwrap();+ assert!(tracker.changed());++ // `mark_complete()` re-broadcasts the current generation without+ // changing the expression: it must not trigger a spurious rebuild.+ filter.mark_complete();+ assert!(!tracker.changed());+ // The filter has completed, so the tracker drains itself.+ assert!(tracker.is_exhausted());+ }++ #[test]+ fn coalesced_update_then_complete_is_one_change() {+ let (predicate, filter) = dynamic_predicate();+ let mut tracker = DynamicFilterTracker::try_new(&predicate).unwrap();++ // Update and complete before the tracker gets a chance to observe.+ // The watch channel only retains the latest value, so the tracker sees+ // `Complete` directly; it must still report the (final) change once.+ filter.update(lit(false)).unwrap();+ filter.mark_complete();++ assert!(tracker.changed());+ assert!(tracker.is_exhausted());+ assert!(!tracker.changed());+ }++ #[test]+ fn watches_multiple_filters_independently() {+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]);+ let col_a = col("a", &schema).unwrap();+ let col_b = col("b", &schema).unwrap();+ let filter_a = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::clone(&col_a)],+ lit(true),+ ));+ let filter_b = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::clone(&col_b)],+ lit(true),+ ));+ let predicate = Arc::new(BinaryExpr::new(+ Arc::new(BinaryExpr::new(+ col_a,+ Operator::Gt,+ Arc::clone(&filter_a) as Arc<dyn PhysicalExpr>,+ )),+ Operator::And,+ Arc::new(BinaryExpr::new(+ col_b,+ Operator::Lt,+ Arc::clone(&filter_b) as Arc<dyn PhysicalExpr>,+ )),+ )) as Arc<dyn PhysicalExpr>;++ let mut tracker = DynamicFilterTracker::try_new(&predicate).unwrap();+ assert!(!tracker.changed());++ filter_a.update(lit(false)).unwrap();+ assert!(tracker.changed());+ assert!(!tracker.changed());++ filter_b.update(lit(false)).unwrap();+ assert!(tracker.changed());+ assert!(!tracker.changed());++ // Completing one filter leaves the other still watched.+ filter_a.mark_complete();+ assert!(!tracker.changed());+ assert!(!tracker.is_exhausted());++ filter_b.mark_complete();+ assert!(!tracker.changed());+ assert!(tracker.is_exhausted());+ }+}diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rsindex 7cf874c448..05a04f88dc 100644--- a/datafusion/physical-expr/src/expressions/mod.rs+++ b/datafusion/physical-expr/src/expressions/mod.rs@@ -45,7 +45,10 @@ pub use case::{CaseExpr, case};pub use cast::{CastExpr, cast};pub use column::{Column, col, with_new_schema};pub use datafusion_expr::utils::format_state_name;-pub use dynamic_filters::{DynamicFilterPhysicalExpr, Inner as DynamicFilterInner};+pub use dynamic_filters::{+ DynamicFilterPhysicalExpr, DynamicFilterTracker, DynamicFilterTracking,+ Inner as DynamicFilterInner,+};pub use in_list::{InListExpr, in_list};pub use is_not_null::{IsNotNullExpr, is_not_null};pub use is_null::{IsNullExpr, is_null};diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rsindex 15598203b3..e67046987b 100644--- a/datafusion/physical-expr/src/lib.rs+++ b/datafusion/physical-expr/src/lib.rs@@ -61,6 +61,7 @@ pub use analysis::{AnalysisContext, ExprBoundaries, analyze};pub use equivalence::{AcrossPartitions, ConstExpr, EquivalenceProperties, calculate_union,};+pub use expressions::{DynamicFilterTracker, DynamicFilterTracking};pub use partitioning::{Distribution, Partitioning};pub use physical_expr::{add_offset_to_expr, add_offset_to_physical_sort_exprs, create_lex_ordering,diff --git a/datafusion/pruning/src/file_pruner.rs b/datafusion/pruning/src/file_pruner.rsindex f850e0c011..661832915c 100644--- a/datafusion/pruning/src/file_pruner.rs+++ b/datafusion/pruning/src/file_pruner.rs@@ -22,7 +22,8 @@ use std::sync::Arc;use arrow::datatypes::{FieldRef, SchemaRef};use datafusion_common::{Result, internal_datafusion_err, pruning::PrunableStatistics};use datafusion_datasource::PartitionedFile;-use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, snapshot_generation};+use datafusion_physical_expr::DynamicFilterTracking;+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_plan::metrics::Count;use log::debug;@@ -34,8 +35,14 @@ use crate::build_pruning_predicate;/// which substitutes partition column references with their literal values before/// the predicate reaches this pruner.pub struct FilePruner {- predicate_generation: Option<u64>,predicate: Arc<dyn PhysicalExpr>,+ /// Tracks the dynamic filters inside `predicate` so we only rebuild the+ /// pruning predicate when one of them has actually moved.+ tracking: DynamicFilterTracking,+ /// Whether [`Self::should_prune`] has built+evaluated the pruning predicate+ /// at least once. The first check always runs; subsequent checks only run+ /// when a watched dynamic filter changed.+ checked_once: bool,/// Schema used for pruning (the logical file schema).file_schema: SchemaRef,file_stats_pruning: PrunableStatistics,@@ -69,42 +76,72 @@ impl FilePruner {})}- /// Create a new file pruner if statistics are available.- /// Returns None if this file does not have statistics.+ /// Create a file pruner for this file, or `None` when pruning it cannot+ /// help.+ ///+ /// Returns `None` when the file has no statistics struct to evaluate a+ /// pruning predicate against, or when the predicate is purely static and the+ /// file has no usable column statistics — in that case planning already did+ /// everything such a pruner could. A predicate carrying a dynamic filter is+ /// always accepted (given a statistics struct), since it may prune via+ /// partition-value folding even without column statistics.pub fn try_new(predicate: Arc<dyn PhysicalExpr>,file_schema: &SchemaRef,partitioned_file: &PartitionedFile,predicate_creation_errors: Count,) -> Option<Self> {+ // A pruning predicate is evaluated against a statistics struct, so one+ // must exist (its columns may all be `Absent`).let file_stats = partitioned_file.statistics.as_ref()?;+ let tracking = DynamicFilterTracking::classify(&predicate);+ // Only build a pruner when it could prune something planning didn't+ // already: the file has real column statistics, or the predicate carries+ // a dynamic filter (whose value, or folded partition columns, can prune+ // even without column statistics). For a purely static predicate with no+ // usable stats there is nothing to gain.+ if !partitioned_file.has_statistics() && !tracking.contains_dynamic_filter() {+ return None;+ }let file_stats_pruning =PrunableStatistics::new(vec![file_stats.clone()], Arc::clone(file_schema));Some(Self {- predicate_generation: None,predicate,+ tracking,+ checked_once: false,file_schema: Arc::clone(file_schema),file_stats_pruning,predicate_creation_errors,})}+ /// Returns `true` if this pruner watches a dynamic filter that can still+ /// change, meaning [`Self::should_prune`] is worth re-checking as the scan+ /// progresses. When `false`, the predicate is effectively static for the+ /// remainder of the scan and the caller can avoid wrapping the stream in a+ /// per-batch re-pruning adapter.+ pub fn is_watching(&self) -> bool {+ matches!(self.tracking, DynamicFilterTracking::Watching(_))+ }+pub fn should_prune(&mut self) -> Result<bool> {- // Check if the predicate has changed since last invocation by tracking- // its "generation". Dynamic filter expressions can change their values- // during query execution, so we use generation tracking to detect when- // the predicate has been updated and needs to be rebuilt.+ // Building the pruning predicate is expensive (it involves expression+ // analysis), so we only do it on the first check and whenever a dynamic+ // filter inside the predicate has actually moved.//- // If the generation hasn't changed, we can skip rebuilding the pruning- // predicate, which is an expensive operation involving expression analysis.- let new_generation = snapshot_generation(&self.predicate);- if let Some(current_generation) = self.predicate_generation.as_mut() {- if *current_generation == new_generation {- return Ok(false);- }- *current_generation = new_generation;+ // Dynamic filter expressions can change their values during query+ // execution; `DynamicFilterTracking` watches the still-incomplete+ // filters and reports a change at most once per update. A purely static+ // predicate (or one whose dynamic filters have all completed) is checked+ // exactly once.+ let should_build = if self.checked_once {+ self.tracking.watcher().is_some_and(|w| w.changed())} else {- self.predicate_generation = Some(new_generation);+ self.checked_once = true;+ true+ };+ if !should_build {+ return Ok(false);}let pruning_predicate = build_pruning_predicate(Arc::clone(&self.predicate),diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdnew file mode 100644index 0000000000..e9a86332cf--- /dev/null+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -0,0 +1,72 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# Upgrade Guides++## DataFusion 55.0.0++**Note:** DataFusion `55.0.0` has not been released yet. The information provided+in this section pertains to features and changes that have already been merged+to the main branch and are awaiting release in this version.++### `is_dynamic_physical_expr` is deprecated++`datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr` is+deprecated. It was a thin wrapper over `snapshot_generation(expr) != 0` used to+ask "does this predicate contain a dynamic filter?".++Prefer asking the question directly against the concrete type. For a one-off+check, downcast to `DynamicFilterPhysicalExpr`:++```rust+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;+use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};++let mut is_dynamic = false;+predicate.apply(|e| {+ if e.downcast_ref::<DynamicFilterPhysicalExpr>().is_some() {+ is_dynamic = true;+ Ok(TreeNodeRecursion::Stop)+ } else {+ Ok(TreeNodeRecursion::Continue)+ }+})?;+```++If you also need to know whether the dynamic filters can still change (and to be+notified when they do), use the new `DynamicFilterTracking` /+`DynamicFilterTracker` API in `datafusion_physical_expr`:++```rust+use datafusion_physical_expr::DynamicFilterTracking;++let tracking = DynamicFilterTracking::classify(&predicate);+if tracking.contains_dynamic_filter() {+ // worth re-evaluating the predicate at runtime+}+```++### `FilePruner::try_new` no longer builds a pruner for static predicates without statistics++`datafusion_pruning::FilePruner::try_new` now returns `None` when the predicate+is purely static _and_ the file carries no usable column statistics, because+such a pruner can never prune anything beyond what planning already did.+Previously it returned `Some` whenever a statistics struct was present (the+"is this worth pruning?" decision lived in the Parquet opener). Files with column+statistics, and predicates that carry a dynamic filter, are unaffected.diff --git a/docs/source/library-user-guide/upgrading/index.rst b/docs/source/library-user-guide/upgrading/index.rstindex 1ed5eca2a5..51c7f14131 100644--- a/docs/source/library-user-guide/upgrading/index.rst+++ b/docs/source/library-user-guide/upgrading/index.rst@@ -21,6 +21,7 @@ Upgrade Guides.. toctree:::maxdepth: 1+ DataFusion 55.0.0 <55.0.0>DataFusion 54.0.0 <54.0.0>DataFusion 53.0.0 <53.0.0>DataFusion 52.0.0 <52.0.0>