Apache DataFusion SQL Query Engine
Revert adding PhysicalOptimizerRule::optimize_plan (#19186)
Adrian Garcia Badaracco3fabee770516
diff --git a/datafusion/core/benches/push_down_filter.rs b/datafusion/core/benches/push_down_filter.rsindex bab103c171..139fb12c30 100644--- a/datafusion/core/benches/push_down_filter.rs+++ b/datafusion/core/benches/push_down_filter.rs@@ -20,10 +20,10 @@ use arrow::datatypes::{DataType, Field, Schema};use bytes::{BufMut, BytesMut};use criterion::{criterion_group, criterion_main, Criterion};use datafusion::config::ConfigOptions;-use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};+use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::ExecutionPlan;use object_store::memory::InMemory;use object_store::path::Path;@@ -106,13 +106,11 @@ fn bench_push_down_filter(c: &mut Criterion) {config.execution.parquet.pushdown_filters = true;let plan = BenchmarkPlan { plan, config };let optimizer = FilterPushdown::new();- let session_config = SessionConfig::from(plan.config.clone());- let optimizer_context = OptimizerContext::new(session_config);c.bench_function("push_down_filter", |b| {b.iter(|| {optimizer- .optimize_plan(Arc::clone(&plan.plan), &optimizer_context)+ .optimize(Arc::clone(&plan.plan), &plan.config).unwrap();});});diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex a942b5de41..5bce862dc9 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -92,7 +92,7 @@ use datafusion_physical_expr::expressions::Literal;use datafusion_physical_expr::{create_physical_sort_exprs, LexOrdering, PhysicalSortExpr,};-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::execution_plan::InvariantLevel;use datafusion_physical_plan::joins::PiecewiseMergeJoinExec;@@ -2291,14 +2291,11 @@ impl DefaultPhysicalPlanner {// to verify that the plan fulfills the base requirements.InvariantChecker(InvariantLevel::Always).check(&plan)?;- // Create optimizer context from session state- let optimizer_context = OptimizerContext::new(session_state.config().clone());-let mut new_plan = Arc::clone(&plan);for optimizer in optimizers {let before_schema = new_plan.schema();new_plan = optimizer- .optimize_plan(new_plan, &optimizer_context)+ .optimize(new_plan, session_state.config_options()).map_err(|e| {DataFusionError::Context(optimizer.name().to_string(), Box::new(e))})?;diff --git a/datafusion/core/tests/execution/coop.rs b/datafusion/core/tests/execution/coop.rsindex c85b1dcdb5..b6f406e967 100644--- a/datafusion/core/tests/execution/coop.rs+++ b/datafusion/core/tests/execution/coop.rs@@ -40,7 +40,7 @@ use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use datafusion_physical_optimizer::ensure_coop::EnsureCooperative;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion_physical_plan::coop::make_cooperative;use datafusion_physical_plan::filter::FilterExec;@@ -810,8 +810,8 @@ async fn query_yields(task_ctx: Arc<TaskContext>,) -> Result<(), Box<dyn Error>> {// Run plan through EnsureCooperative- let optimizer_context = OptimizerContext::new(task_ctx.session_config().clone());- let optimized = EnsureCooperative::new().optimize_plan(plan, &optimizer_context)?;+ let optimized =+ EnsureCooperative::new().optimize(plan, task_ctx.session_config().options())?;// Get the streamlet stream = physical_plan::execute_stream(optimized, task_ctx)?;diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rsindex cda8c2c448..942c1e29d4 100644--- a/datafusion/core/tests/parquet/file_statistics.rs+++ b/datafusion/core/tests/parquet/file_statistics.rs@@ -39,7 +39,7 @@ use datafusion_expr::{col, lit, Expr};use datafusion::datasource::physical_plan::FileScanConfig;use datafusion_common::config::ConfigOptions;use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::ExecutionPlan;use tempfile::tempdir;@@ -83,10 +83,8 @@ async fn check_stats_precision_with_filter_pushdown() {Arc::new(FilterExec::try_new(physical_filter, exec_with_filter).unwrap())as Arc<dyn ExecutionPlan>;- let session_config = SessionConfig::from(options.clone());- let optimizer_context = OptimizerContext::new(session_config);let optimized_exec = FilterPushdown::new()- .optimize_plan(filtered_exec, &optimizer_context)+ .optimize(filtered_exec, &options).unwrap();assert!(diff --git a/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs b/datafusion/core/tests/physical_optimizer/aggregate_statistics.rsindex 07c4a14283..a79d743cb2 100644--- a/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs@@ -25,13 +25,12 @@ use arrow::record_batch::RecordBatch;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::source::DataSourceExec;use datafusion_common::cast::as_int64_array;+use datafusion_common::config::ConfigOptions;use datafusion_common::Result;-use datafusion_execution::config::SessionConfig;use datafusion_execution::TaskContext;use datafusion_expr::Operator;use datafusion_physical_expr::expressions::{self, cast};use datafusion_physical_optimizer::aggregate_statistics::AggregateStatistics;-use datafusion_physical_optimizer::OptimizerContext;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::aggregates::AggregateExec;use datafusion_physical_plan::aggregates::AggregateMode;@@ -68,10 +67,8 @@ async fn assert_count_optim_success(let task_ctx = Arc::new(TaskContext::default());let plan: Arc<dyn ExecutionPlan> = Arc::new(plan);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());- let optimized = AggregateStatistics::new()- .optimize_plan(Arc::clone(&plan), &optimizer_context)?;+ let config = ConfigOptions::new();+ let optimized = AggregateStatistics::new().optimize(Arc::clone(&plan), &config)?;// A ProjectionExec is a sign that the count optimization was appliedassert!(optimized.as_any().is::<ProjectionExec>());@@ -267,10 +264,8 @@ async fn test_count_inexact_stat() -> Result<()> {Arc::clone(&schema),)?;- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());- let optimized = AggregateStatistics::new()- .optimize_plan(Arc::new(final_agg), &optimizer_context)?;+ let conf = ConfigOptions::new();+ let optimized = AggregateStatistics::new().optimize(Arc::new(final_agg), &conf)?;// check that the original ExecutionPlan was not replacedassert!(optimized.as_any().is::<AggregateExec>());@@ -313,10 +308,8 @@ async fn test_count_with_nulls_inexact_stat() -> Result<()> {Arc::clone(&schema),)?;- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());- let optimized = AggregateStatistics::new()- .optimize_plan(Arc::new(final_agg), &optimizer_context)?;+ let conf = ConfigOptions::new();+ let optimized = AggregateStatistics::new().optimize(Arc::new(final_agg), &conf)?;// check that the original ExecutionPlan was not replacedassert!(optimized.as_any().is::<AggregateExec>());diff --git a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rsindex b4b55a93a6..9c76f6ab6f 100644--- a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs+++ b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs@@ -26,7 +26,7 @@ use std::sync::Arc;use crate::physical_optimizer::test_utils::parquet_exec;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};-use datafusion_execution::config::SessionConfig;+use datafusion_common::config::ConfigOptions;use datafusion_functions_aggregate::count::count_udaf;use datafusion_functions_aggregate::sum::sum_udaf;use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};@@ -34,7 +34,7 @@ use datafusion_physical_expr::expressions::{col, lit};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};@@ -47,9 +47,8 @@ macro_rules! assert_optimized {($PLAN: expr, @ $EXPECTED_LINES: literal $(,)?) => {// run optimizerlet optimizer = CombinePartialFinalAggregate {};- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());- let optimized = optimizer.optimize_plan($PLAN, &optimizer_context)?;+ let config = ConfigOptions::new();+ let optimized = optimizer.optimize($PLAN, &config)?;// Now format correctlylet plan = displayable(optimized.as_ref()).indent(true).to_string();let actual_lines = plan.trim();diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex b1a0417703..a3d9a1e407 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -52,7 +52,7 @@ use datafusion_physical_expr_common::sort_expr::{use datafusion_physical_optimizer::enforce_distribution::*;use datafusion_physical_optimizer::enforce_sorting::EnforceSorting;use datafusion_physical_optimizer::output_requirements::OutputRequirements;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};@@ -489,9 +489,7 @@ impl TestConfig {) -> Result<Arc<dyn ExecutionPlan>> {// Add the ancillary output requirements operator at the start:let optimizer = OutputRequirements::new_add_mode();- let session_config = SessionConfig::from(self.config.clone());- let optimizer_context = OptimizerContext::new(session_config.clone());- let mut optimized = optimizer.optimize_plan(plan.clone(), &optimizer_context)?;+ let mut optimized = optimizer.optimize(plan.clone(), &self.config)?;// This file has 2 rules that use tree node, apply these rules to original plan consecutively// After these operations tree nodes should be in a consistent state.@@ -527,25 +525,21 @@ impl TestConfig {}for run in optimizers_to_run {- let session_config = SessionConfig::from(self.config.clone());- let optimizer_context = OptimizerContext::new(session_config.clone());optimized = match run {Run::Distribution => {let optimizer = EnforceDistribution::new();- optimizer.optimize_plan(optimized, &optimizer_context)?+ optimizer.optimize(optimized, &self.config)?}Run::Sorting => {let optimizer = EnforceSorting::new();- optimizer.optimize_plan(optimized, &optimizer_context)?+ optimizer.optimize(optimized, &self.config)?}};}// Remove the ancillary output requirements operator when done:let optimizer = OutputRequirements::new_remove_mode();- let session_config = SessionConfig::from(self.config.clone());- let optimizer_context = OptimizerContext::new(session_config.clone());- let optimized = optimizer.optimize_plan(optimized, &optimizer_context)?;+ let optimized = optimizer.optimize(optimized, &self.config)?;Ok(optimized)}@@ -3378,10 +3372,7 @@ SortRequiredExec: [a@0 ASC]config.execution.target_partitions = 10;config.optimizer.enable_round_robin_repartition = true;config.optimizer.prefer_existing_sort = false;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());- let dist_plan =- EnforceDistribution::new().optimize_plan(physical_plan, &optimizer_context)?;+ let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;// Since at the start of the rule ordering requirement is not satisfied// EnforceDistribution rule doesn't satisfy this requirement either.assert_plan!(dist_plan, @r"@@ -3417,10 +3408,7 @@ SortRequiredExec: [a@0 ASC]config.execution.target_partitions = 10;config.optimizer.enable_round_robin_repartition = true;config.optimizer.prefer_existing_sort = false;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());- let dist_plan =- EnforceDistribution::new().optimize_plan(physical_plan, &optimizer_context)?;+ let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;// Since at the start of the rule ordering requirement is satisfied// EnforceDistribution rule satisfy this requirement also.assert_plan!(dist_plan, @r"diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 881ba35ef6..3f7fa62d68 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -56,7 +56,7 @@ use datafusion_physical_optimizer::enforce_sorting::replace_with_order_preservinuse datafusion_physical_optimizer::enforce_sorting::sort_pushdown::{SortPushDown, assign_initial_requirements, pushdown_sorts};use datafusion_physical_optimizer::enforce_distribution::EnforceDistribution;use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion::prelude::*;use arrow::array::{Int32Array, RecordBatch};use arrow::datatypes::{Field};@@ -175,10 +175,8 @@ impl EnforceSortingTest {let input_plan_string = displayable(self.plan.as_ref()).indent(true).to_string();// Run the actual optimizer- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_physical_plan = EnforceSorting::new()- .optimize_plan(Arc::clone(&self.plan), &optimizer_context)+ .optimize(Arc::clone(&self.plan), &config).expect("enforce_sorting failed");// Get string representation of the plan@@ -2365,18 +2363,15 @@ async fn test_commutativity() -> Result<()> {"#);let config = ConfigOptions::new();- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let rules = vec![Arc::new(EnforceDistribution::new()) as Arc<dyn PhysicalOptimizerRule>,Arc::new(EnforceSorting::new()) as Arc<dyn PhysicalOptimizerRule>,];let mut first_plan = orig_plan.clone();for rule in rules {- first_plan = rule.optimize_plan(first_plan, &optimizer_context)?;+ first_plan = rule.optimize(first_plan, &config)?;}- let optimizer_context2 = OptimizerContext::new(session_config.clone());let rules = vec![Arc::new(EnforceSorting::new()) as Arc<dyn PhysicalOptimizerRule>,Arc::new(EnforceDistribution::new()) as Arc<dyn PhysicalOptimizerRule>,@@ -2384,7 +2379,7 @@ async fn test_commutativity() -> Result<()> {];let mut second_plan = orig_plan.clone();for rule in rules {- second_plan = rule.optimize_plan(second_plan, &optimizer_context2)?;+ second_plan = rule.optimize(second_plan, &config)?;}assert_eq!(get_plan_string(&first_plan), get_plan_string(&second_plan));diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rsindex cd8ea5528b..cfcdc02f36 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs@@ -51,7 +51,7 @@ use datafusion_physical_expr::{};use datafusion_physical_expr::{expressions::col, LexOrdering, PhysicalSortExpr};use datafusion_physical_optimizer::{- filter_pushdown::FilterPushdown, OptimizerContext, PhysicalOptimizerRule,+ filter_pushdown::FilterPushdown, PhysicalOptimizerRule,};use datafusion_physical_plan::{aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},@@ -249,10 +249,8 @@ async fn test_dynamic_filter_pushdown_through_hash_join_with_topk() {config.execution.parquet.pushdown_filters = true;// Apply the FilterPushdown optimizer rule- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(Arc::clone(&plan), &optimizer_context)+ .optimize(Arc::clone(&plan), &config).unwrap();// Test that filters are pushed down correctly to each side of the join@@ -728,10 +726,8 @@ async fn test_topk_dynamic_filter_pushdown() {// Actually apply the optimization to the plan and put some data through it to check that the filter is updated to reflect the TopK statelet mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();let config = SessionConfig::new().with_batch_size(2);let session_ctx = SessionContext::new_with_config(config);@@ -813,10 +809,8 @@ async fn test_topk_dynamic_filter_pushdown_multi_column_sort() {// Actually apply the optimization to the plan and put some data through it to check that the filter is updated to reflect the TopK statelet mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();let config = SessionConfig::new().with_batch_size(2);let session_ctx = SessionContext::new_with_config(config);@@ -1059,10 +1053,8 @@ async fn test_hashjoin_dynamic_filter_pushdown() {let mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();// Test for https://github.com/apache/datafusion/pull/17371: dynamic filter linking survives `with_new_children`@@ -1289,10 +1281,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {let mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();let config = SessionConfig::new().with_batch_size(10);let session_ctx = SessionContext::new_with_config(config);@@ -1488,10 +1478,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {let mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();let config = SessionConfig::new().with_batch_size(10);let session_ctx = SessionContext::new_with_config(config);@@ -1662,10 +1650,8 @@ async fn test_nested_hashjoin_dynamic_filter_pushdown() {let mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let session_config = SessionConfig::from(config);- let optimizer_context = OptimizerContext::new(session_config.clone());let plan = FilterPushdown::new_post_optimization()- .optimize_plan(outer_join, &optimizer_context)+ .optimize(outer_join, &config).unwrap();let config = SessionConfig::new().with_batch_size(10);let session_ctx = SessionContext::new_with_config(config);@@ -1957,11 +1943,8 @@ async fn run_aggregate_dyn_filter_case(case: AggregateDynFilterCase<'_>) {config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let session_config: SessionConfig = config.into();- let optimizer_context = OptimizerContext::new(session_config);-let optimized = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();let before = format_plan_for_test(&optimized);@@ -2323,11 +2306,8 @@ fn test_aggregate_dynamic_filter_not_created_for_single_mode() {config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let session_config: SessionConfig = config.into();- let optimizer_context = OptimizerContext::new(session_config);-let optimized = FilterPushdown::new_post_optimization()- .optimize_plan(plan, &optimizer_context)+ .optimize(plan, &config).unwrap();let formatted = format_plan_for_test(&optimized);@@ -2874,8 +2854,7 @@ async fn test_hashjoin_dynamic_filter_all_partitions_empty() {let mut config = SessionConfig::new();config.options_mut().execution.parquet.pushdown_filters = true;let optimizer = FilterPushdown::new_post_optimization();- let ctx = OptimizerContext::new(config.clone());- let plan = optimizer.optimize_plan(plan, &ctx).unwrap();+ let plan = optimizer.optimize(plan, config.options()).unwrap();insta::assert_snapshot!(format_plan_for_test(&plan),@@ -3012,8 +2991,7 @@ async fn test_hashjoin_dynamic_filter_with_nulls() {let mut config = SessionConfig::new();config.options_mut().execution.parquet.pushdown_filters = true;let optimizer = FilterPushdown::new_post_optimization();- let ctx = OptimizerContext::new(config.clone());- let plan = optimizer.optimize_plan(plan, &ctx).unwrap();+ let plan = optimizer.optimize(plan, config.options()).unwrap();insta::assert_snapshot!(format_plan_for_test(&plan),diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rsindex fc2be94b01..c32f7b2d0b 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs@@ -25,9 +25,8 @@ use datafusion_datasource::{file_stream::FileOpener, schema_adapter::DefaultSchemaAdapterFactory,schema_adapter::SchemaAdapterFactory, source::DataSourceExec, PartitionedFile,};-use datafusion_execution::config::SessionConfig;use datafusion_physical_expr_common::physical_expr::fmt_sql;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::filter::batch_filter;use datafusion_physical_plan::filter_pushdown::{FilterPushdownPhase, PushedDown};use datafusion_physical_plan::{@@ -375,9 +374,7 @@ impl OptimizationTest {let input = format_execution_plan(&input_plan);let input_schema = input_plan.schema();- let session_config = SessionConfig::from(parquet_pushdown_config);- let optimizer_context = OptimizerContext::new(session_config.clone());- let output_result = opt.optimize_plan(input_plan, &optimizer_context);+ let output_result = opt.optimize(input_plan, &parquet_pushdown_config);let output = output_result.and_then(|plan| {if opt.schema_check() && (plan.schema() != input_schema) {diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex 26b5e44853..f9d3a04546 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -29,7 +29,6 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::{stats::Precision, ColumnStatistics, JoinType, ScalarValue};use datafusion_common::{JoinSide, NullEquality};use datafusion_common::{Result, Statistics};-use datafusion_execution::config::SessionConfig;use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};use datafusion_expr::Operator;use datafusion_physical_expr::expressions::col;@@ -38,7 +37,6 @@ use datafusion_physical_expr::intervals::utils::check_support;use datafusion_physical_expr::PhysicalExprRef;use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};use datafusion_physical_optimizer::join_selection::JoinSelection;-use datafusion_physical_optimizer::OptimizerContext;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::displayable;use datafusion_physical_plan::joins::utils::ColumnIndex;@@ -228,10 +226,8 @@ async fn test_join_with_swap() {.unwrap(),);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(join, &optimizer_context)+ .optimize(join, &ConfigOptions::new()).unwrap();let swapping_projection = optimized_join@@ -292,10 +288,8 @@ async fn test_left_join_no_swap() {.unwrap(),);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(join, &optimizer_context)+ .optimize(join, &ConfigOptions::new()).unwrap();let swapped_join = optimized_join@@ -344,10 +338,8 @@ async fn test_join_with_swap_semi() {let original_schema = join.schema();- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(Arc::new(join), &optimizer_context)+ .optimize(Arc::new(join), &ConfigOptions::new()).unwrap();let swapped_join = optimized_join@@ -401,10 +393,8 @@ async fn test_join_with_swap_mark() {let original_schema = join.schema();- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(Arc::new(join), &optimizer_context)+ .optimize(Arc::new(join), &ConfigOptions::new()).unwrap();let swapped_join = optimized_join@@ -440,10 +430,8 @@ macro_rules! assert_optimized {($PLAN: expr, @$EXPECTED_LINES: literal $(,)?) => {let plan = Arc::new($PLAN);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized = JoinSelection::new()- .optimize_plan(plan.clone(), &optimizer_context)+ .optimize(plan.clone(), &ConfigOptions::new()).unwrap();let plan_string = displayable(optimized.as_ref()).indent(true).to_string();@@ -534,10 +522,8 @@ async fn test_join_no_swap() {.unwrap(),);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(join, &optimizer_context)+ .optimize(join, &ConfigOptions::new()).unwrap();let swapped_join = optimized_join@@ -585,10 +571,8 @@ async fn test_nl_join_with_swap(join_type: JoinType) {.unwrap(),);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(join, &optimizer_context)+ .optimize(join, &ConfigOptions::new()).unwrap();let swapping_projection = optimized_join@@ -665,10 +649,11 @@ async fn test_nl_join_with_swap_no_proj(join_type: JoinType) {.unwrap(),);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(Arc::<NestedLoopJoinExec>::clone(&join), &optimizer_context)+ .optimize(+ Arc::<NestedLoopJoinExec>::clone(&join),+ &ConfigOptions::new(),+ ).unwrap();let swapped_join = optimized_join@@ -925,10 +910,8 @@ fn check_join_partition_mode(.unwrap(),);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join = JoinSelection::new()- .optimize_plan(join, &optimizer_context)+ .optimize(join, &ConfigOptions::new()).unwrap();if !is_swapped {@@ -1572,10 +1555,8 @@ async fn test_join_with_maybe_swap_unbounded_case(t: TestCase) -> Result<()> {NullEquality::NullEqualsNothing,)?) as _;- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let optimized_join_plan =- JoinSelection::new().optimize_plan(Arc::clone(&join), &optimizer_context)?;+ JoinSelection::new().optimize(Arc::clone(&join), &ConfigOptions::new())?;// If swap did happenlet projection_added = optimized_join_plan.as_any().is::<ProjectionExec>();diff --git a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs b/datafusion/core/tests/physical_optimizer/limit_pushdown.rsindex ef333c3067..9d172db246 100644--- a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs@@ -24,14 +24,13 @@ use crate::physical_optimizer::test_utils::{use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;-use datafusion_execution::config::SessionConfig;use datafusion_expr::Operator;use datafusion_physical_expr::expressions::{col, lit, BinaryExpr};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;-use datafusion_physical_optimizer::OptimizerContext;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::filter::FilterExec;@@ -102,10 +101,8 @@ fn transforms_streaming_table_exec_into_fetching_version_when_skip_is_zero() ->];assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true, fetch=5"@@ -129,10 +126,8 @@ fn transforms_streaming_table_exec_into_fetching_version_and_keeps_the_global_li];assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["GlobalLimitExec: skip=2, fetch=5",@@ -167,10 +162,8 @@ fn transforms_coalesce_batches_exec_into_fetching_version_and_removes_local_limi];assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["CoalescePartitionsExec: fetch=5",@@ -201,10 +194,8 @@ fn pushes_global_limit_exec_through_projection_exec() -> Result<()> {];assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",@@ -235,10 +226,8 @@ fn pushes_global_limit_exec_through_projection_exec_and_transforms_coalesce_batcassert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",@@ -279,10 +268,8 @@ fn pushes_global_limit_into_multiple_fetch_plans() -> Result<()> {assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["SortPreservingMergeExec: [c1@0 ASC], fetch=5",@@ -317,10 +304,8 @@ fn keeps_pushed_local_limit_exec_when_there_are_multiple_input_partitions() -> R];assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["CoalescePartitionsExec: fetch=5",@@ -349,10 +334,8 @@ fn merges_local_limit_with_local_limit() -> Result<()> {assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(parent_local_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(parent_local_limit, &ConfigOptions::new())?;let expected = ["GlobalLimitExec: skip=0, fetch=10", " EmptyExec"];assert_eq!(get_plan_string(&after_optimize), expected);@@ -376,10 +359,8 @@ fn merges_global_limit_with_global_limit() -> Result<()> {assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(parent_global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(parent_global_limit, &ConfigOptions::new())?;let expected = ["GlobalLimitExec: skip=20, fetch=20", " EmptyExec"];assert_eq!(get_plan_string(&after_optimize), expected);@@ -403,10 +384,8 @@ fn merges_global_limit_with_local_limit() -> Result<()> {assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;let expected = ["GlobalLimitExec: skip=20, fetch=20", " EmptyExec"];assert_eq!(get_plan_string(&after_optimize), expected);@@ -430,10 +409,8 @@ fn merges_local_limit_with_global_limit() -> Result<()> {assert_eq!(initial, expected_initial);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- LimitPushdown::new().optimize_plan(local_limit, &optimizer_context)?;+ LimitPushdown::new().optimize(local_limit, &ConfigOptions::new())?;let expected = ["GlobalLimitExec: skip=20, fetch=20", " EmptyExec"];assert_eq!(get_plan_string(&after_optimize), expected);diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rsindex 1068e8ec03..80f4fbc305 100644--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs@@ -28,7 +28,6 @@ use datafusion_common::config::{ConfigOptions, CsvOptions};use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::TableSchema;-use datafusion_execution::config::SessionConfig;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::{@@ -45,7 +44,7 @@ use datafusion_physical_expr_common::sort_expr::{};use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;use datafusion_physical_optimizer::projection_pushdown::ProjectionPushdown;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};@@ -462,10 +461,8 @@ fn test_csv_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -502,10 +499,8 @@ fn test_memory_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -600,10 +595,8 @@ fn test_streaming_table_after_projection() -> Result<()> {Arc::new(streaming_table) as _,)?) as _;- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let result = after_optimize.as_any()@@ -702,10 +695,8 @@ fn test_projection_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(top_projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(top_projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -769,10 +760,8 @@ fn test_output_req_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -861,10 +850,8 @@ fn test_coalesce_partitions_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -920,10 +907,8 @@ fn test_filter_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1024,10 +1009,8 @@ fn test_join_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1154,10 +1137,8 @@ fn test_join_after_required_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1234,10 +1215,8 @@ fn test_nested_loop_join_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize_string =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize_string.as_ref()).indent(true).to_string();@@ -1333,10 +1312,8 @@ fn test_hash_join_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true).to_string();@@ -1363,10 +1340,8 @@ fn test_hash_join_after_projection() -> Result<()> {join.clone(),)?);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true).to_string();@@ -1418,10 +1393,8 @@ fn test_repartition_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1490,10 +1463,8 @@ fn test_sort_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1545,10 +1516,8 @@ fn test_sort_preserving_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1591,10 +1560,8 @@ fn test_union_after_projection() -> Result<()> {");- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1666,10 +1633,8 @@ fn test_partition_col_projection_pushdown() -> Result<()> {source,)?);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)@@ -1711,10 +1676,8 @@ fn test_partition_col_projection_pushdown_expr() -> Result<()> {source,)?);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());let after_optimize =- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;let after_optimize_string = displayable(after_optimize.as_ref()).indent(true)diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rsindex ccacf35ec8..f46147de1b 100644--- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs+++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs@@ -28,13 +28,13 @@ use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};use datafusion::prelude::{CsvReadOptions, SessionContext};+use datafusion_common::config::ConfigOptions;use datafusion_common::{JoinType, Result, ScalarValue};-use datafusion_execution::config::SessionConfig;use datafusion_physical_expr::expressions::{col, Literal};use datafusion_physical_expr::Partitioning;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::{displayable, ExecutionPlan};@@ -393,12 +393,9 @@ fn create_test_schema2() -> SchemaRef {/// Check if sanity checker should accept or reject plans.fn assert_sanity_check(plan: &Arc<dyn ExecutionPlan>, is_sane: bool) {let sanity_checker = SanityCheckPlan::new();- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());+ let opts = ConfigOptions::default();assert_eq!(- sanity_checker- .optimize_plan(plan.clone(), &optimizer_context)- .is_ok(),+ sanity_checker.optimize(plan.clone(), &opts).is_ok(),is_sane);}diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex e164e7617c..e410c495c8 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -29,12 +29,12 @@ use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::memory::MemorySourceConfig;use datafusion::datasource::physical_plan::ParquetSource;use datafusion::datasource::source::DataSourceExec;+use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;use datafusion_common::{ColumnStatistics, JoinType, NullEquality, Result, Statistics};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;-use datafusion_execution::config::SessionConfig;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::{WindowFrame, WindowFunctionDefinition};@@ -46,7 +46,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements, PhysicalSortExpr,};use datafusion_physical_optimizer::limited_distinct_aggregation::LimitedDistinctAggregation;-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};+use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};@@ -635,11 +635,10 @@ pub fn build_group_by(input_schema: &SchemaRef, columns: Vec<String>) -> Physica}pub fn get_optimized_plan(plan: &Arc<dyn ExecutionPlan>) -> Result<String> {- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config.clone());+ let config = ConfigOptions::new();- let optimized = LimitedDistinctAggregation::new()- .optimize_plan(Arc::clone(plan), &optimizer_context)?;+ let optimized =+ LimitedDistinctAggregation::new().optimize(Arc::clone(plan), &config)?;let optimized_result = displayable(optimized.as_ref()).indent(true).to_string();diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rsindex 2f78fb5532..5dcdab1552 100644--- a/datafusion/execution/src/config.rs+++ b/datafusion/execution/src/config.rs@@ -100,7 +100,7 @@ pub struct SessionConfig {/// references to the same options.options: Arc<ConfigOptions>,/// Opaque extensions.- extensions: Arc<Extensions>,+ extensions: AnyMap,}impl Default for SessionConfig {@@ -108,53 +108,14 @@ impl Default for SessionConfig {Self {options: Arc::new(ConfigOptions::new()),// Assume no extensions by default.- extensions: Arc::new(Extensions::default()),+ extensions: HashMap::with_capacity_and_hasher(+ 0,+ BuildHasherDefault::default(),+ ),}}}-/// A type map for storing extensions.-///-/// Extensions are indexed by their type `T`. If multiple values of the same type are provided, only the last one-/// will be kept.-///-/// Extensions are opaque objects that are unknown to DataFusion itself but can be downcast by optimizer rules,-/// execution plans, or other components that have access to the session config.-/// They provide a flexible way to attach extra data or behavior to the session config.-#[derive(Clone, Debug)]-pub struct Extensions {- inner: AnyMap,-}--impl Default for Extensions {- fn default() -> Self {- Self {- inner: HashMap::with_capacity_and_hasher(0, BuildHasherDefault::default()),- }- }-}--impl Extensions {- pub fn insert<T>(&mut self, value: Arc<T>)- where- T: Send + Sync + 'static,- {- let id = TypeId::of::<T>();- self.inner.insert(id, value);- }-- pub fn get<T>(&self) -> Option<Arc<T>>- where- T: Send + Sync + 'static,- {- let id = TypeId::of::<T>();- self.inner- .get(&id)- .cloned()- .map(|arc_any| Arc::downcast(arc_any).expect("TypeId unique"))- }-}-impl SessionConfig {/// Create an execution config with default settingpub fn new() -> Self {@@ -203,20 +164,6 @@ impl SessionConfig {Arc::make_mut(&mut self.options)}- /// Return a handle to the extensions.- ///- /// Can be used to read the current extensions.- pub fn extensions(&self) -> &Arc<Extensions> {- &self.extensions- }-- /// Return a mutable handle to the extensions.- ///- /// Can be used to set extensions.- pub fn extensions_mut(&mut self) -> &mut Extensions {- Arc::make_mut(&mut self.extensions)- }-/// Set a configuration optionpub fn set(self, key: &str, value: &ScalarValue) -> Self {self.set_str(key, &value.to_string())@@ -633,7 +580,9 @@ impl SessionConfig {whereT: Send + Sync + 'static,{- self.extensions_mut().insert::<T>(ext);+ let ext = ext as Arc<dyn Any + Send + Sync + 'static>;+ let id = TypeId::of::<T>();+ self.extensions.insert(id, ext);}/// Get extension, if any for the specified type `T` exists.@@ -643,7 +592,11 @@ impl SessionConfig {whereT: Send + Sync + 'static,{- self.extensions.get::<T>()+ let id = TypeId::of::<T>();+ self.extensions+ .get(&id)+ .cloned()+ .map(|ext| Arc::downcast(ext).expect("TypeId unique"))}}diff --git a/datafusion/physical-optimizer/src/aggregate_statistics.rs b/datafusion/physical-optimizer/src/aggregate_statistics.rsindex 19cc0b5070..4bd8e321f1 100644--- a/datafusion/physical-optimizer/src/aggregate_statistics.rs+++ b/datafusion/physical-optimizer/src/aggregate_statistics.rs@@ -16,6 +16,7 @@// under the License.//! Utilizing exact statistics from sources to avoid scanning data+use datafusion_common::config::ConfigOptions;use datafusion_common::scalar::ScalarValue;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::Result;@@ -26,7 +27,7 @@ use datafusion_physical_plan::udaf::{AggregateFunctionExpr, StatisticsArgs};use datafusion_physical_plan::{expressions, ExecutionPlan};use std::sync::Arc;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;/// Optimizer that uses available statistics for aggregate functions#[derive(Default, Debug)]@@ -43,10 +44,10 @@ impl PhysicalOptimizerRule for AggregateStatistics {#[cfg_attr(feature = "recursive_protection", recursive::recursive)]#[expect(clippy::allow_attributes)] // See https://github.com/apache/datafusion/issues/18881#issuecomment-3621545670#[allow(clippy::only_used_in_recursion)] // See https://github.com/rust-lang/rust-clippy/issues/14566- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {if let Some(partial_agg_exec) = take_optimizable(&*plan) {let partial_agg_exec = partial_agg_exec@@ -86,15 +87,13 @@ impl PhysicalOptimizerRule for AggregateStatistics {)?))} else {plan.map_children(|child| {- self.optimize_plan(child, context).map(Transformed::yes)+ self.optimize(child, config).map(Transformed::yes)}).data()}} else {- plan.map_children(|child| {- self.optimize_plan(child, context).map(Transformed::yes)- })- .data()+ plan.map_children(|child| self.optimize(child, config).map(Transformed::yes))+ .data()}}diff --git a/datafusion/physical-optimizer/src/coalesce_batches.rs b/datafusion/physical-optimizer/src/coalesce_batches.rsindex b0da29beea..ecfe193b34 100644--- a/datafusion/physical-optimizer/src/coalesce_batches.rs+++ b/datafusion/physical-optimizer/src/coalesce_batches.rs@@ -18,11 +18,12 @@//! CoalesceBatches optimizer that groups batches together rows//! in bigger batches to avoid overhead with small batches-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use std::sync::Arc;use datafusion_common::assert_eq_or_internal_err;+use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;use datafusion_physical_expr::Partitioning;use datafusion_physical_plan::{@@ -44,12 +45,11 @@ impl CoalesceBatches {}}impl PhysicalOptimizerRule for CoalesceBatches {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();if !config.execution.coalesce_batches {return Ok(plan);}diff --git a/datafusion/physical-optimizer/src/combine_partial_final_agg.rs b/datafusion/physical-optimizer/src/combine_partial_final_agg.rsindex 5aea611f4c..667250d646 100644--- a/datafusion/physical-optimizer/src/combine_partial_final_agg.rs+++ b/datafusion/physical-optimizer/src/combine_partial_final_agg.rs@@ -26,7 +26,8 @@ use datafusion_physical_plan::aggregates::{};use datafusion_physical_plan::ExecutionPlan;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_physical_expr::aggregate::AggregateFunctionExpr;use datafusion_physical_expr::{physical_exprs_equal, PhysicalExpr};@@ -46,10 +47,10 @@ impl CombinePartialFinalAggregate {}impl PhysicalOptimizerRule for CombinePartialFinalAggregate {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- _context: &OptimizerContext,+ _config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {plan.transform_down(|plan| {// Check if the plan is AggregateExecdiff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rsindex 571c07ff80..1f6ea61c09 100644--- a/datafusion/physical-optimizer/src/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/enforce_distribution.rs@@ -30,7 +30,6 @@ use crate::utils::{add_sort_above_with_check, is_coalesce_partitions, is_repartition,is_sort_preserving_merge,};-use crate::OptimizerContext;use arrow::compute::SortOptions;use datafusion_common::config::ConfigOptions;@@ -191,12 +190,11 @@ impl EnforceDistribution {}impl PhysicalOptimizerRule for EnforceDistribution {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();let top_down_join_key_reordering = config.optimizer.top_down_join_key_reordering;let adjusted = if top_down_join_key_reordering {diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rsindex 38bf1877a4..0c0b63c0b3 100644--- a/datafusion/physical-optimizer/src/ensure_coop.rs+++ b/datafusion/physical-optimizer/src/ensure_coop.rs@@ -23,8 +23,9 @@use std::fmt::{Debug, Formatter};use std::sync::Arc;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::Result;use datafusion_physical_plan::coop::CooperativeExec;@@ -61,10 +62,10 @@ impl PhysicalOptimizerRule for EnsureCooperative {"EnsureCooperative"}- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- _context: &OptimizerContext,+ _config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {plan.transform_up(|plan| {let is_leaf = plan.children().is_empty();@@ -95,17 +96,16 @@ impl PhysicalOptimizerRule for EnsureCooperative {#[cfg(test)]mod tests {use super::*;- use datafusion_execution::config::SessionConfig;+ use datafusion_common::config::ConfigOptions;use datafusion_physical_plan::{displayable, test::scan_partitioned};use insta::assert_snapshot;#[tokio::test]async fn test_cooperative_exec_for_custom_exec() {let test_custom_exec = scan_partitioned(1);- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config);+ let config = ConfigOptions::new();let optimized = EnsureCooperative::new()- .optimize_plan(test_custom_exec, &optimizer_context)+ .optimize(test_custom_exec, &config).unwrap();let display = displayable(optimized.as_ref()).indent(true).to_string();diff --git a/datafusion/physical-optimizer/src/filter_pushdown.rs b/datafusion/physical-optimizer/src/filter_pushdown.rsindex eacffa9198..22cb03fc3e 100644--- a/datafusion/physical-optimizer/src/filter_pushdown.rs+++ b/datafusion/physical-optimizer/src/filter_pushdown.rs@@ -33,7 +33,7 @@use std::sync::Arc;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion_common::{assert_eq_or_internal_err, config::ConfigOptions, Result};@@ -416,19 +416,6 @@ impl Default for FilterPushdown {}impl PhysicalOptimizerRule for FilterPushdown {- fn optimize_plan(- &self,- plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,- ) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();- Ok(- push_down_filters(&Arc::clone(&plan), vec![], config, self.phase)?- .updated_node- .unwrap_or(plan),- )- }-fn optimize(&self,plan: Arc<dyn ExecutionPlan>,diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rsindex 8f337bc8d9..1ffe6dbca3 100644--- a/datafusion/physical-optimizer/src/join_selection.rs+++ b/datafusion/physical-optimizer/src/join_selection.rs@@ -23,7 +23,7 @@//! pipeline-friendly ones. To achieve the second goal, it selects the proper//! `PartitionMode` and the build side using the available statistics for hash joins.-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};@@ -103,12 +103,11 @@ fn supports_collect_by_thresholds(}impl PhysicalOptimizerRule for JoinSelection {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();// First, we make pipeline-fixing modifications to joins so as to accommodate// unbounded inputs. Each pipeline-fixing subrule, which is a function// of type `PipelineFixerSubrule`, takes a single [`PipelineStatePropagator`]diff --git a/datafusion/physical-optimizer/src/lib.rs b/datafusion/physical-optimizer/src/lib.rsindex d1bf375bd0..cf8cf33cea 100644--- a/datafusion/physical-optimizer/src/lib.rs+++ b/datafusion/physical-optimizer/src/lib.rs@@ -47,4 +47,4 @@ pub mod topk_aggregation;pub mod update_aggr_exprs;pub mod utils;-pub use optimizer::{OptimizerContext, PhysicalOptimizerRule};+pub use optimizer::PhysicalOptimizerRule;diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rsindex 74f1af2ff5..4cb3abe30b 100644--- a/datafusion/physical-optimizer/src/limit_pushdown.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs@@ -21,8 +21,9 @@use std::fmt::Debug;use std::sync::Arc;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;+use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;use datafusion_common::tree_node::{Transformed, TreeNodeRecursion};use datafusion_common::utils::combine_limit;@@ -59,10 +60,10 @@ impl LimitPushdown {}impl PhysicalOptimizerRule for LimitPushdown {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- _context: &OptimizerContext,+ _config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {let global_state = GlobalRequirements {fetch: None,diff --git a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rsindex ee20f4d43b..1c671cd074 100644--- a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs@@ -15,7 +15,8 @@// specific language governing permissions and limitations// under the License.-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TreeNode};use datafusion_common::ScalarValue;use datafusion_expr::{LimitEffect, WindowFrameBound, WindowFrameUnits};@@ -70,12 +71,11 @@ impl TraverseState {}impl PhysicalOptimizerRule for LimitPushPastWindows {- fn optimize_plan(+ fn optimize(&self,original: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();if !config.optimizer.enable_window_limits {return Ok(original);}diff --git a/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs b/datafusion/physical-optimizer/src/limited_distinct_aggregation.rsindex 89828a87cb..3666ff3798 100644--- a/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs+++ b/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs@@ -24,10 +24,11 @@ use datafusion_physical_plan::aggregates::AggregateExec;use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::Result;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use itertools::Itertools;/// An optimizer rule that passes a `limit` hint into grouped aggregations which don't require all@@ -157,12 +158,11 @@ impl Default for LimitedDistinctAggregation {}impl PhysicalOptimizerRule for LimitedDistinctAggregation {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();if config.optimizer.enable_distinct_aggregation_soft_limit {plan.transform_down(|plan| {Ok(diff --git a/datafusion/physical-optimizer/src/optimizer.rs b/datafusion/physical-optimizer/src/optimizer.rsindex 4524947135..03c83bb5a0 100644--- a/datafusion/physical-optimizer/src/optimizer.rs+++ b/datafusion/physical-optimizer/src/optimizer.rs@@ -38,37 +38,10 @@ use crate::update_aggr_exprs::OptimizeAggregateOrder;use crate::limit_pushdown_past_window::LimitPushPastWindows;use datafusion_common::config::ConfigOptions;-use datafusion_common::{internal_err, Result};-use datafusion_execution::config::SessionConfig;+use datafusion_common::Result;use datafusion_physical_plan::ExecutionPlan;-/// Context for optimizing physical plans.-///-/// This context provides access to session configuration and optimizer extensions.-///-/// Similar to [`TaskContext`] which provides context during execution,-/// [`OptimizerContext`] provides context during optimization.-///-/// [`TaskContext`]: https://docs.rs/datafusion/latest/datafusion/execution/struct.TaskContext.html-#[derive(Debug, Clone)]-pub struct OptimizerContext {- /// Session configuration- session_config: SessionConfig,-}--impl OptimizerContext {- /// Create a new OptimizerContext- pub fn new(session_config: SessionConfig) -> Self {- Self { session_config }- }-- /// Return a reference to the session configuration- pub fn session_config(&self) -> &SessionConfig {- &self.session_config- }-}--/// `PhysicalOptimizerRule` transforms one [`ExecutionPlan`] into another which+/// `PhysicalOptimizerRule` transforms one ['ExecutionPlan'] into another which/// computes the same results, but in a potentially more efficient way.////// Use [`SessionState::add_physical_optimizer_rule`] to register additional@@ -76,52 +49,12 @@ impl OptimizerContext {////// [`SessionState::add_physical_optimizer_rule`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html#method.add_physical_optimizer_rulepub trait PhysicalOptimizerRule: Debug {- /// Rewrite `plan` to an optimized form with additional context- ///- /// This is the preferred method for implementing optimization rules as it- /// provides access to the full optimizer context including session configuration.- ///- /// The default implementation delegates to [`PhysicalOptimizerRule::optimize`] for- /// backwards compatibility with existing implementations.- ///- /// New implementations should override this method instead of `optimize()`.- ///- /// Once [`PhysicalOptimizerRule::optimize`] is deprecated and removed, this- /// default implementation will be removed and this method will become required.- /// This change is scheduled for DataFusion 54.0.0.- fn optimize_plan(- &self,- plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,- ) -> Result<Arc<dyn ExecutionPlan>> {- // Default implementation: delegate to the old method for backwards compatibility- #[expect(deprecated)]- self.optimize(plan, context.session_config().options())- }-/// Rewrite `plan` to an optimized form- ///- /// This method is kept for backwards compatibility. New implementations- /// should implement [`optimize_plan`](Self::optimize_plan) instead, which- /// provides access to additional context.- ///- /// The default implementation returns an error indicating that neither- /// `optimize` nor `optimize_plan` was properly implemented. At least one- /// of these methods must be overridden.- #[deprecated(- since = "52.0.0",- note = "use `PhysicalOptimizerRule::optimize_plan` instead"- )]fn optimize(&self,- _plan: Arc<dyn ExecutionPlan>,- _config: &ConfigOptions,- ) -> Result<Arc<dyn ExecutionPlan>> {- internal_err!(- "PhysicalOptimizerRule '{}' must implement either optimize() or optimize_plan()",- self.name()- )- }+ plan: Arc<dyn ExecutionPlan>,+ config: &ConfigOptions,+ ) -> Result<Arc<dyn ExecutionPlan>>;/// A human readable name for this optimizer rulefn name(&self) -> &str;diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rsindex c11b1d362a..9e5e980219 100644--- a/datafusion/physical-optimizer/src/output_requirements.rs+++ b/datafusion/physical-optimizer/src/output_requirements.rs@@ -24,8 +24,9 @@use std::sync::Arc;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{Result, Statistics};use datafusion_execution::TaskContext;@@ -302,10 +303,10 @@ impl ExecutionPlan for OutputRequirementExec {}impl PhysicalOptimizerRule for OutputRequirements {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- _context: &OptimizerContext,+ _config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {match self.mode {RuleMode::Add => require_top_ordering(plan),diff --git a/datafusion/physical-optimizer/src/projection_pushdown.rs b/datafusion/physical-optimizer/src/projection_pushdown.rsindex 89a595f61b..cb51c9167c 100644--- a/datafusion/physical-optimizer/src/projection_pushdown.rs+++ b/datafusion/physical-optimizer/src/projection_pushdown.rs@@ -20,12 +20,13 @@//! projections one by one if the operator below is amenable to this. If a//! projection reaches a source, it can even disappear from the plan entirely.-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use arrow::datatypes::{Fields, Schema, SchemaRef};use datafusion_common::alias::AliasGenerator;use std::collections::HashSet;use std::sync::Arc;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};@@ -56,10 +57,10 @@ impl ProjectionPushdown {}impl PhysicalOptimizerRule for ProjectionPushdown {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- _context: &OptimizerContext,+ _config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {let alias_generator = AliasGenerator::new();let plan = plan@@ -446,8 +447,6 @@ fn is_volatile_expression_tree(expr: &dyn PhysicalExpr) -> bool {mod test {use super::*;use arrow::datatypes::{DataType, Field, FieldRef, Schema};- use datafusion_common::config::ConfigOptions;- use datafusion_execution::config::SessionConfig;use datafusion_expr_common::operator::Operator;use datafusion_functions::math::random;use datafusion_physical_expr::expressions::{binary, lit};@@ -673,10 +672,7 @@ mod test {)?;let optimizer = ProjectionPushdown::new();- let session_config = SessionConfig::new();- let optimizer_context = OptimizerContext::new(session_config);- let optimized_plan =- optimizer.optimize_plan(Arc::new(join), &optimizer_context)?;+ let optimized_plan = optimizer.optimize(Arc::new(join), &Default::default())?;let displayable_plan = displayable(optimized_plan.as_ref()).indent(false);Ok(displayable_plan.to_string())diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rsindex 7e81ba5f15..23df6b4e55 100644--- a/datafusion/physical-optimizer/src/sanity_checker.rs+++ b/datafusion/physical-optimizer/src/sanity_checker.rs@@ -26,7 +26,7 @@ use std::sync::Arc;use datafusion_common::Result;use datafusion_physical_plan::ExecutionPlan;-use datafusion_common::config::OptimizerOptions;+use datafusion_common::config::{ConfigOptions, OptimizerOptions};use datafusion_common::plan_err;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported};@@ -34,7 +34,7 @@ use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion_physical_plan::joins::SymmetricHashJoinExec;use datafusion_physical_plan::{get_plan_string, ExecutionPlanProperties};-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use datafusion_physical_expr_common::sort_expr::format_physical_sort_requirement_list;use itertools::izip;@@ -54,12 +54,11 @@ impl SanityCheckPlan {}impl PhysicalOptimizerRule for SanityCheckPlan {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();plan.transform_up(|p| check_plan_sanity(p, &config.optimizer)).data()}diff --git a/datafusion/physical-optimizer/src/topk_aggregation.rs b/datafusion/physical-optimizer/src/topk_aggregation.rsindex a55b90e839..b7505f0df4 100644--- a/datafusion/physical-optimizer/src/topk_aggregation.rs+++ b/datafusion/physical-optimizer/src/topk_aggregation.rs@@ -19,8 +19,9 @@use std::sync::Arc;-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;use arrow::datatypes::DataType;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::Result;use datafusion_physical_expr::expressions::Column;@@ -144,12 +145,11 @@ impl Default for TopKAggregation {}impl PhysicalOptimizerRule for TopKAggregation {- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,+ config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let config = context.session_config().options();if config.optimizer.enable_topk_aggregation {plan.transform_down(|plan| {Ok(if let Some(plan) = TopKAggregation::transform_sort(&plan) {diff --git a/datafusion/physical-optimizer/src/update_aggr_exprs.rs b/datafusion/physical-optimizer/src/update_aggr_exprs.rsindex 604bc6c3e1..c22a63c0c1 100644--- a/datafusion/physical-optimizer/src/update_aggr_exprs.rs+++ b/datafusion/physical-optimizer/src/update_aggr_exprs.rs@@ -20,6 +20,7 @@use std::sync::Arc;+use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{plan_datafusion_err, Result};use datafusion_physical_expr::aggregate::AggregateFunctionExpr;@@ -28,7 +29,7 @@ use datafusion_physical_plan::aggregates::{concat_slices, AggregateExec};use datafusion_physical_plan::windows::get_ordered_partition_by_indices;use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};-use crate::{OptimizerContext, PhysicalOptimizerRule};+use crate::PhysicalOptimizerRule;/// This optimizer rule checks ordering requirements of aggregate expressions.///@@ -64,15 +65,15 @@ impl PhysicalOptimizerRule for OptimizeAggregateOrder {/// # Arguments////// * `plan` - The root of the execution plan to optimize.- /// * `context` - Optimizer context containing configuration options.+ /// * `_config` - Configuration options (currently unused).////// # Returns////// A `Result` containing the potentially optimized execution plan or an error.- fn optimize_plan(+ fn optimize(&self,plan: Arc<dyn ExecutionPlan>,- _context: &OptimizerContext,+ _config: &ConfigOptions,) -> Result<Arc<dyn ExecutionPlan>> {plan.transform_up(|plan| {if let Some(aggr_exec) = plan.as_any().downcast_ref::<AggregateExec>() {diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.mdindex caab9ad4e7..6462961ce6 100644--- a/docs/source/library-user-guide/upgrading.md+++ b/docs/source/library-user-guide/upgrading.md@@ -380,102 +380,6 @@ If you were using a custom `SchemaAdapterFactory` for schema adaptation (e.g., dSee the [default column values example](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_data_source/default_column_values.rs) for how to implement a custom `PhysicalExprAdapterFactory`.-### `PhysicalOptimizerRule::optimize` deprecated in favor of `optimize_plan`--The `PhysicalOptimizerRule` trait has been updated to provide optimizer rules with access to broader session context. A new method `optimize_plan` has been added that accepts an `OptimizerContext` parameter, and the existing `optimize` method has been deprecated.--**Who is affected:**--- Users who have implemented custom `PhysicalOptimizerRule` implementations--**Breaking changes:**--1. **New `optimize_plan` method**: This is the preferred method for implementing optimization rules. It provides access to the full `SessionConfig` through `OptimizerContext`, rather than just `ConfigOptions`.--2. **`optimize` method deprecated**: The old `optimize` method that takes `&ConfigOptions` is now deprecated and will be removed in DataFusion 58.0.0.--**Migration guide:**--If you have a custom `PhysicalOptimizerRule` implementation, update it to implement `optimize_plan` instead of `optimize`:--**Before:**--```rust-use datafusion::physical_optimizer::PhysicalOptimizerRule;-use datafusion_common::config::ConfigOptions;-use datafusion_common::Result;-use datafusion_physical_plan::ExecutionPlan;-use std::sync::Arc;--#[derive(Debug)]-struct MyOptimizerRule;--#[allow(deprecated)]-impl PhysicalOptimizerRule for MyOptimizerRule {- fn optimize(- &self,- plan: Arc<dyn ExecutionPlan>,- _config: &ConfigOptions,- ) -> Result<Arc<dyn ExecutionPlan>> {- // Use config.optimizer, config.execution, etc.- // ... optimization logic ...- Ok(plan)- }-- fn name(&self) -> &str {- "my_optimizer_rule"- }-- fn schema_check(&self) -> bool {- true- }-}-```--**After:**--```rust-use datafusion::physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};-use datafusion_common::Result;-use datafusion_physical_plan::ExecutionPlan;-use std::sync::Arc;--#[derive(Debug)]-struct MyOptimizerRule;--impl PhysicalOptimizerRule for MyOptimizerRule {- fn optimize_plan(- &self,- plan: Arc<dyn ExecutionPlan>,- context: &OptimizerContext,- ) -> Result<Arc<dyn ExecutionPlan>> {- // Access ConfigOptions through session_config- let _config = context.session_config().options();- // Or access extensions through session_config- let _extensions = context.session_config().extensions();- // ... optimization logic ...- Ok(plan)- }-- fn name(&self) -> &str {- "my_optimizer_rule"- }-- fn schema_check(&self) -> bool {- true- }-}-```--**What is `OptimizerContext`?**--`OptimizerContext` is a new struct that provides context during physical plan optimization, similar to how `TaskContext` provides context during execution. It wraps `SessionConfig`, giving optimizer rules access to:--- Configuration options via `context.session_config().options()`-- Session extensions via `context.session_config().extensions()`--This enables optimizer rules to access custom extensions registered with the session, which was not possible with the old `&ConfigOptions` parameter.-## DataFusion `51.0.0`### `arrow` / `parquet` updated to 57.0.0