Apache DataFusion SQL Query Engine
refactor: thread PhysicalOptimizerContext through join_selection stats helpers (#24716)
Alessandro Solimando19d31c509633
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex 5617f5bd89..d4561decc6 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -37,12 +37,16 @@ use datafusion_physical_expr::expressions::{BinaryExpr, Column, NegativeExpr};use datafusion_physical_expr::intervals::utils::check_support;use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+use datafusion_physical_optimizer::PhysicalOptimizerContext;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::join_selection::JoinSelection;use datafusion_physical_plan::displayable;use datafusion_physical_plan::joins::utils::ColumnIndex;use datafusion_physical_plan::joins::utils::JoinFilter;use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, PartitionMode};+use datafusion_physical_plan::operator_statistics::{+ ClosureStatisticsProvider, StatisticsRegistry, StatisticsResult,+};use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::{@@ -640,6 +644,72 @@ async fn test_null_aware_left_anti_with_filter_does_not_swap() -> Result<()> {Ok(())}+#[tokio::test]+async fn test_join_swap_uses_context_statistics_registry() -> Result<()> {+ let left: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(+ big_statistics(),+ Schema::new(vec![Field::new("left_col", DataType::Int32, false)]),+ ));+ let right: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(+ big_statistics(),+ Schema::new(vec![Field::new("right_col", DataType::Int32, false)]),+ ));+ let join = HashJoinExec::try_new(+ Arc::clone(&left),+ Arc::clone(&right),+ vec![(+ Arc::new(Column::new_with_schema("left_col", &left.schema())?),+ Arc::new(Column::new_with_schema("right_col", &right.schema())?),+ )],+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::Auto,+ NullEquality::NullEqualsNothing,+ false,+ )?;++ // Reports `right` as much smaller than its real stats: the swap below only+ // happens if `context`'s registry is actually threaded through.+ let mut registry = StatisticsRegistry::new();+ registry.register(Arc::new(ClosureStatisticsProvider::with_matches(+ |plan| plan.schema().field(0).name() == "right_col",+ |_plan, _child_stats| Ok(StatisticsResult::Computed(small_statistics().into())),+ )));++ struct ContextWithRegistry {+ config: ConfigOptions,+ registry: StatisticsRegistry,+ }++ impl PhysicalOptimizerContext for ContextWithRegistry {+ fn config_options(&self) -> &ConfigOptions {+ &self.config+ }++ fn statistics_registry(&self) -> Option<&StatisticsRegistry> {+ Some(&self.registry)+ }+ }++ let context = ContextWithRegistry {+ config: ConfigOptions::new(),+ registry,+ };++ let optimized_join =+ JoinSelection::new().optimize_with_context(Arc::new(join), &context)?;+ let swapped_join = optimized_join+ .downcast_ref::<HashJoinExec>()+ .expect("optimize_with_context should return a HashJoinExec");++ assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);+ assert_eq!(swapped_join.left().schema().field(0).name(), "right_col");+ assert_eq!(swapped_join.right().schema().field(0).name(), "left_col");++ Ok(())+}+#[tokio::test]async fn test_join_with_swap_mark() {let join_types = [JoinType::LeftMark, JoinType::RightMark];diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rsindex 6b5a6eb18f..f11e8612de 100644--- a/datafusion/physical-optimizer/src/join_selection.rs+++ b/datafusion/physical-optimizer/src/join_selection.rs@@ -39,7 +39,6 @@ use datafusion_physical_plan::joins::{CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode,StreamJoinPartitionMode, SymmetricHashJoinExec,};-use datafusion_physical_plan::operator_statistics::StatisticsRegistry;use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};use std::sync::Arc;@@ -61,9 +60,9 @@ impl JoinSelection {/// is available.fn get_stats(plan: &dyn ExecutionPlan,- registry: Option<&StatisticsRegistry>,+ context: &dyn PhysicalOptimizerContext,) -> Result<Arc<Statistics>> {- let ctx = match registry {+ let ctx = match context.statistics_registry() {Some(reg) => StatisticsContext::new_with_registry(reg.clone()),None => StatisticsContext::new(),};@@ -75,7 +74,8 @@ fn get_stats(/// Checks whether join inputs should be swapped using available statistics.////// It follows these steps:-/// 1. If a [`StatisticsRegistry`] is provided, use it for cross-operator estimates+/// 1. If a [`datafusion_physical_plan::operator_statistics::StatisticsRegistry`] is+/// provided, use it for cross-operator estimates/// (e.g., intermediate join outputs that would otherwise have `Absent` statistics)./// 2. Compare the in-memory sizes of both sides, and place the smaller side on/// the left (build) side.@@ -83,20 +83,19 @@ fn get_stats(/// 4. Do not reorder the join if neither statistic is available, or if/// `datafusion.optimizer.join_reordering` is disabled.///-/// Used configurations inside arg `config`-/// - `config.optimizer.join_reordering`: allows or forbids statistics-driven join swapping+/// Used configurations+/// - `optimizer.join_reordering`: allows or forbids statistics-driven join swappingpub(crate) fn should_swap_join_order(left: &dyn ExecutionPlan,right: &dyn ExecutionPlan,- config: &ConfigOptions,- registry: Option<&StatisticsRegistry>,+ context: &dyn PhysicalOptimizerContext,) -> Result<bool> {- if !config.optimizer.join_reordering {+ if !context.config_options().optimizer.join_reordering {return Ok(false);}- let left_stats = get_stats(left, registry)?;- let right_stats = get_stats(right, registry)?;+ let left_stats = get_stats(left, context)?;+ let right_stats = get_stats(right, context)?;// First compare total_byte_size, then fall back to num_rows if byte// sizes are unavailable.@@ -119,9 +118,9 @@ fn supports_collect_by_thresholds(plan: &dyn ExecutionPlan,threshold_byte_size: usize,threshold_num_rows: usize,- registry: Option<&StatisticsRegistry>,+ context: &dyn PhysicalOptimizerContext,) -> bool {- let Ok(stats) = get_stats(plan, registry) else {+ let Ok(stats) = get_stats(plan, context) else {return false;};@@ -152,19 +151,15 @@ impl PhysicalOptimizerRule for JoinSelection {plan: Arc<dyn ExecutionPlan>,context: &dyn PhysicalOptimizerContext,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.config_options();- let registry = context.statistics_registry();let subrules: Vec<Box<PipelineFixerSubrule>> = vec![Box::new(hash_join_convert_symmetric_subrule),Box::new(hash_join_swap_subrule),];let new_plan = plan- .transform_up(|p| apply_subrules(p, &subrules, config))+ .transform_up(|p| apply_subrules(p, &subrules, context.config_options())).data()?;new_plan- .transform_up(|plan| {- statistical_join_selection_subrule(plan, config, registry)- })+ .transform_up(|plan| statistical_join_selection_subrule(plan, context)).data()}@@ -192,40 +187,39 @@ fn can_swap_hash_join(hash_join: &HashJoinExec) -> bool {/// When the `ignore_threshold` is false, this function will also check left/// and right sizes in bytes or rows.///-/// Used configurations inside arg `config`-/// - `config.optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`-/// - `config.optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`-/// - `config.optimizer.join_reordering`: allows or forbids input swapping+/// Used configurations+/// - `optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`+/// - `optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`+/// - `optimizer.join_reordering`: allows or forbids input swappingpub(crate) fn try_collect_left(hash_join: &HashJoinExec,ignore_threshold: bool,- config: &ConfigOptions,- registry: Option<&StatisticsRegistry>,+ context: &dyn PhysicalOptimizerContext,) -> Result<Option<Arc<dyn ExecutionPlan>>> {let left = hash_join.left();let right = hash_join.right();- let optimizer_config = &config.optimizer;+ let optimizer_config = &context.config_options().optimizer;let left_can_collect = ignore_threshold|| supports_collect_by_thresholds(&**left,optimizer_config.hash_join_single_partition_threshold,optimizer_config.hash_join_single_partition_threshold_rows,- registry,+ context,);let right_can_collect = ignore_threshold|| supports_collect_by_thresholds(&**right,optimizer_config.hash_join_single_partition_threshold,optimizer_config.hash_join_single_partition_threshold_rows,- registry,+ context,);match (left_can_collect, right_can_collect) {(true, true) => {// For null-aware joins, we only swap `LeftAnti` joins where the left side is > right sideif can_swap_hash_join(hash_join)- && should_swap_join_order(&**left, &**right, config, registry)?+ && should_swap_join_order(&**left, &**right, context)?{Ok(Some(hash_join.swap_inputs(PartitionMode::CollectLeft)?))} else {@@ -260,12 +254,11 @@ pub(crate) fn try_collect_left(/// If swapping is optimal and supported, creates a swapped partitioned hash join; otherwise,/// creates a standard partitioned hash join.///-/// Used configurations inside arg `config`-/// - `config.optimizer.join_reordering`: allows or forbids statistics-driven join swapping+/// Used configurations+/// - `optimizer.join_reordering`: allows or forbids statistics-driven join swappingpub(crate) fn partitioned_hash_join(hash_join: &HashJoinExec,- config: &ConfigOptions,- registry: Option<&StatisticsRegistry>,+ context: &dyn PhysicalOptimizerContext,) -> Result<Arc<dyn ExecutionPlan>> {let left = hash_join.left();let right = hash_join.right();@@ -275,7 +268,7 @@ pub(crate) fn partitioned_hash_join(PartitionMode::Partitioned};if can_swap_hash_join(hash_join)- && should_swap_join_order(&**left, &**right, config, registry)?+ && should_swap_join_order(&**left, &**right, context)?{hash_join.swap_inputs(partition_mode)} else {@@ -297,33 +290,31 @@ pub(crate) fn partitioned_hash_join(/// optimize hash and cross joins in the plan according to available statistical/// information.///-/// Used configurations inside arg `config`-/// - `config.optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`-/// - `config.optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`-/// - `config.optimizer.join_reordering`: allows or forbids input swapping+/// Used configurations+/// - `optimizer.hash_join_single_partition_threshold`: byte threshold for `CollectLeft`+/// - `optimizer.hash_join_single_partition_threshold_rows`: row threshold for `CollectLeft`+/// - `optimizer.join_reordering`: allows or forbids input swappingfn statistical_join_selection_subrule(plan: Arc<dyn ExecutionPlan>,- config: &ConfigOptions,- registry: Option<&StatisticsRegistry>,+ context: &dyn PhysicalOptimizerContext,) -> Result<Transformed<Arc<dyn ExecutionPlan>>> {let transformed = if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() {match hash_join.partition_mode() {- PartitionMode::Auto => try_collect_left(hash_join, false, config, registry)?+ PartitionMode::Auto => try_collect_left(hash_join, false, context)?.map_or_else(- || partitioned_hash_join(hash_join, config, registry).map(Some),+ || partitioned_hash_join(hash_join, context).map(Some),|v| Ok(Some(v)),)?,- PartitionMode::CollectLeft => {- try_collect_left(hash_join, true, config, registry)?.map_or_else(- || partitioned_hash_join(hash_join, config, registry).map(Some),+ PartitionMode::CollectLeft => try_collect_left(hash_join, true, context)?+ .map_or_else(+ || partitioned_hash_join(hash_join, context).map(Some),|v| Ok(Some(v)),- )?- }+ )?,PartitionMode::Partitioned => {let left = hash_join.left();let right = hash_join.right();if can_swap_hash_join(hash_join)- && should_swap_join_order(&**left, &**right, config, registry)?+ && should_swap_join_order(&**left, &**right, context)?{// Null-aware RightAnti only supports CollectLeftlet partition_mode = if hash_join.null_aware {@@ -340,7 +331,7 @@ fn statistical_join_selection_subrule(} else if let Some(cross_join) = plan.downcast_ref::<CrossJoinExec>() {let left = cross_join.left();let right = cross_join.right();- if should_swap_join_order(&**left, &**right, config, registry)? {+ if should_swap_join_order(&**left, &**right, context)? {cross_join.swap_inputs().map(Some)?} else {None@@ -349,7 +340,7 @@ fn statistical_join_selection_subrule(let left = nl_join.left();let right = nl_join.right();if nl_join.join_type().supports_swap()- && should_swap_join_order(&**left, &**right, config, registry)?+ && should_swap_join_order(&**left, &**right, context)?{nl_join.swap_inputs().map(Some)?} else {