Skip to content

Apache DataFusion SQL Query Engine

Revert adding PhysicalOptimizerRule::optimize_plan (#19186)

Adrian Garcia Badaracco3fabee770516
diff --git a/datafusion/core/benches/push_down_filter.rs b/datafusion/core/benches/push_down_filter.rs
index bab103c171..139fb12c30 100644
--- a/datafusion/core/benches/push_down_filter.rs
+++ b/datafusion/core/benches/push_down_filter.rs
@@ -20,10 +20,10 @@ use arrow::datatypes::{DataType, Field, Schema};
use bytes::{BufMut, BytesMut};
use criterion::{criterion_group, criterion_main, Criterion};
use datafusion::config::ConfigOptions;
-use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::ExecutionPlan;
use object_store::memory::InMemory;
use object_store::path::Path;
@@ -106,13 +106,11 @@ fn bench_push_down_filter(c: &mut Criterion) {
config.execution.parquet.pushdown_filters = true;
let plan = BenchmarkPlan { plan, config };
let optimizer = FilterPushdown::new();
- let session_config = SessionConfig::from(plan.config.clone());
- let optimizer_context = OptimizerContext::new(session_config);
c.bench_function("push_down_filter", |b| {
b.iter(|| {
optimizer
- .optimize_plan(Arc::clone(&plan.plan), &optimizer_context)
+ .optimize(Arc::clone(&plan.plan), &plan.config)
.unwrap();
});
});
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index a942b5de41..5bce862dc9 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -92,7 +92,7 @@ use datafusion_physical_expr::expressions::Literal;
use datafusion_physical_expr::{
create_physical_sort_exprs, LexOrdering, PhysicalSortExpr,
};
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::execution_plan::InvariantLevel;
use datafusion_physical_plan::joins::PiecewiseMergeJoinExec;
@@ -2291,14 +2291,11 @@ impl DefaultPhysicalPlanner {
// to verify that the plan fulfills the base requirements.
InvariantChecker(InvariantLevel::Always).check(&plan)?;
- // Create optimizer context from session state
- let optimizer_context = OptimizerContext::new(session_state.config().clone());
-
let mut new_plan = Arc::clone(&plan);
for optimizer in optimizers {
let before_schema = new_plan.schema();
new_plan = optimizer
- .optimize_plan(new_plan, &optimizer_context)
+ .optimize(new_plan, session_state.config_options())
.map_err(|e| {
DataFusionError::Context(optimizer.name().to_string(), Box::new(e))
})?;
diff --git a/datafusion/core/tests/execution/coop.rs b/datafusion/core/tests/execution/coop.rs
index c85b1dcdb5..b6f406e967 100644
--- a/datafusion/core/tests/execution/coop.rs
+++ b/datafusion/core/tests/execution/coop.rs
@@ -40,7 +40,7 @@ use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::ensure_coop::EnsureCooperative;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion_physical_plan::coop::make_cooperative;
use datafusion_physical_plan::filter::FilterExec;
@@ -810,8 +810,8 @@ async fn query_yields(
task_ctx: Arc<TaskContext>,
) -> Result<(), Box<dyn Error>> {
// Run plan through EnsureCooperative
- let optimizer_context = OptimizerContext::new(task_ctx.session_config().clone());
- let optimized = EnsureCooperative::new().optimize_plan(plan, &optimizer_context)?;
+ let optimized =
+ EnsureCooperative::new().optimize(plan, task_ctx.session_config().options())?;
// Get the stream
let stream = physical_plan::execute_stream(optimized, task_ctx)?;
diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rs
index cda8c2c448..942c1e29d4 100644
--- a/datafusion/core/tests/parquet/file_statistics.rs
+++ b/datafusion/core/tests/parquet/file_statistics.rs
@@ -39,7 +39,7 @@ use datafusion_expr::{col, lit, Expr};
use datafusion::datasource::physical_plan::FileScanConfig;
use datafusion_common::config::ConfigOptions;
use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::filter::FilterExec;
use datafusion_physical_plan::ExecutionPlan;
use tempfile::tempdir;
@@ -83,10 +83,8 @@ async fn check_stats_precision_with_filter_pushdown() {
Arc::new(FilterExec::try_new(physical_filter, exec_with_filter).unwrap())
as Arc<dyn ExecutionPlan>;
- let session_config = SessionConfig::from(options.clone());
- let optimizer_context = OptimizerContext::new(session_config);
let optimized_exec = FilterPushdown::new()
- .optimize_plan(filtered_exec, &optimizer_context)
+ .optimize(filtered_exec, &options)
.unwrap();
assert!(
diff --git a/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs b/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs
index 07c4a14283..a79d743cb2 100644
--- a/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs
+++ b/datafusion/core/tests/physical_optimizer/aggregate_statistics.rs
@@ -25,13 +25,12 @@ use arrow::record_batch::RecordBatch;
use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::datasource::source::DataSourceExec;
use datafusion_common::cast::as_int64_array;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::Result;
-use datafusion_execution::config::SessionConfig;
use datafusion_execution::TaskContext;
use datafusion_expr::Operator;
use datafusion_physical_expr::expressions::{self, cast};
use datafusion_physical_optimizer::aggregate_statistics::AggregateStatistics;
-use datafusion_physical_optimizer::OptimizerContext;
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::aggregates::AggregateExec;
use datafusion_physical_plan::aggregates::AggregateMode;
@@ -68,10 +67,8 @@ async fn assert_count_optim_success(
let task_ctx = Arc::new(TaskContext::default());
let plan: Arc<dyn ExecutionPlan> = Arc::new(plan);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let optimized = AggregateStatistics::new()
- .optimize_plan(Arc::clone(&plan), &optimizer_context)?;
+ let config = ConfigOptions::new();
+ let optimized = AggregateStatistics::new().optimize(Arc::clone(&plan), &config)?;
// A ProjectionExec is a sign that the count optimization was applied
assert!(optimized.as_any().is::<ProjectionExec>());
@@ -267,10 +264,8 @@ async fn test_count_inexact_stat() -> Result<()> {
Arc::clone(&schema),
)?;
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let optimized = AggregateStatistics::new()
- .optimize_plan(Arc::new(final_agg), &optimizer_context)?;
+ let conf = ConfigOptions::new();
+ let optimized = AggregateStatistics::new().optimize(Arc::new(final_agg), &conf)?;
// check that the original ExecutionPlan was not replaced
assert!(optimized.as_any().is::<AggregateExec>());
@@ -313,10 +308,8 @@ async fn test_count_with_nulls_inexact_stat() -> Result<()> {
Arc::clone(&schema),
)?;
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let optimized = AggregateStatistics::new()
- .optimize_plan(Arc::new(final_agg), &optimizer_context)?;
+ let conf = ConfigOptions::new();
+ let optimized = AggregateStatistics::new().optimize(Arc::new(final_agg), &conf)?;
// check that the original ExecutionPlan was not replaced
assert!(optimized.as_any().is::<AggregateExec>());
diff --git a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs
index b4b55a93a6..9c76f6ab6f 100644
--- a/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs
+++ b/datafusion/core/tests/physical_optimizer/combine_partial_final_agg.rs
@@ -26,7 +26,7 @@ use std::sync::Arc;
use crate::physical_optimizer::test_utils::parquet_exec;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion_execution::config::SessionConfig;
+use datafusion_common::config::ConfigOptions;
use datafusion_functions_aggregate::count::count_udaf;
use datafusion_functions_aggregate::sum::sum_udaf;
use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
@@ -34,7 +34,7 @@ use datafusion_physical_expr::expressions::{col, lit};
use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_optimizer::combine_partial_final_agg::CombinePartialFinalAggregate;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::aggregates::{
AggregateExec, AggregateMode, PhysicalGroupBy,
};
@@ -47,9 +47,8 @@ macro_rules! assert_optimized {
($PLAN: expr, @ $EXPECTED_LINES: literal $(,)?) => {
// run optimizer
let optimizer = CombinePartialFinalAggregate {};
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let optimized = optimizer.optimize_plan($PLAN, &optimizer_context)?;
+ let config = ConfigOptions::new();
+ let optimized = optimizer.optimize($PLAN, &config)?;
// Now format correctly
let plan = displayable(optimized.as_ref()).indent(true).to_string();
let actual_lines = plan.trim();
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index b1a0417703..a3d9a1e407 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -52,7 +52,7 @@ use datafusion_physical_expr_common::sort_expr::{
use datafusion_physical_optimizer::enforce_distribution::*;
use datafusion_physical_optimizer::enforce_sorting::EnforceSorting;
use datafusion_physical_optimizer::output_requirements::OutputRequirements;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::aggregates::{
AggregateExec, AggregateMode, PhysicalGroupBy,
};
@@ -489,9 +489,7 @@ impl TestConfig {
) -> Result<Arc<dyn ExecutionPlan>> {
// Add the ancillary output requirements operator at the start:
let optimizer = OutputRequirements::new_add_mode();
- let session_config = SessionConfig::from(self.config.clone());
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let mut optimized = optimizer.optimize_plan(plan.clone(), &optimizer_context)?;
+ let mut optimized = optimizer.optimize(plan.clone(), &self.config)?;
// This file has 2 rules that use tree node, apply these rules to original plan consecutively
// After these operations tree nodes should be in a consistent state.
@@ -527,25 +525,21 @@ impl TestConfig {
}
for run in optimizers_to_run {
- let session_config = SessionConfig::from(self.config.clone());
- let optimizer_context = OptimizerContext::new(session_config.clone());
optimized = match run {
Run::Distribution => {
let optimizer = EnforceDistribution::new();
- optimizer.optimize_plan(optimized, &optimizer_context)?
+ optimizer.optimize(optimized, &self.config)?
}
Run::Sorting => {
let optimizer = EnforceSorting::new();
- optimizer.optimize_plan(optimized, &optimizer_context)?
+ optimizer.optimize(optimized, &self.config)?
}
};
}
// Remove the ancillary output requirements operator when done:
let optimizer = OutputRequirements::new_remove_mode();
- let session_config = SessionConfig::from(self.config.clone());
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let optimized = optimizer.optimize_plan(optimized, &optimizer_context)?;
+ let optimized = optimizer.optimize(optimized, &self.config)?;
Ok(optimized)
}
@@ -3378,10 +3372,7 @@ SortRequiredExec: [a@0 ASC]
config.execution.target_partitions = 10;
config.optimizer.enable_round_robin_repartition = true;
config.optimizer.prefer_existing_sort = false;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let dist_plan =
- EnforceDistribution::new().optimize_plan(physical_plan, &optimizer_context)?;
+ let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;
// Since at the start of the rule ordering requirement is not satisfied
// EnforceDistribution rule doesn't satisfy this requirement either.
assert_plan!(dist_plan, @r"
@@ -3417,10 +3408,7 @@ SortRequiredExec: [a@0 ASC]
config.execution.target_partitions = 10;
config.optimizer.enable_round_robin_repartition = true;
config.optimizer.prefer_existing_sort = false;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let dist_plan =
- EnforceDistribution::new().optimize_plan(physical_plan, &optimizer_context)?;
+ let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;
// Since at the start of the rule ordering requirement is satisfied
// EnforceDistribution rule satisfy this requirement also.
assert_plan!(dist_plan, @r"
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index 881ba35ef6..3f7fa62d68 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -56,7 +56,7 @@ use datafusion_physical_optimizer::enforce_sorting::replace_with_order_preservin
use datafusion_physical_optimizer::enforce_sorting::sort_pushdown::{SortPushDown, assign_initial_requirements, pushdown_sorts};
use datafusion_physical_optimizer::enforce_distribution::EnforceDistribution;
use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion::prelude::*;
use arrow::array::{Int32Array, RecordBatch};
use arrow::datatypes::{Field};
@@ -175,10 +175,8 @@ impl EnforceSortingTest {
let input_plan_string = displayable(self.plan.as_ref()).indent(true).to_string();
// Run the actual optimizer
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_physical_plan = EnforceSorting::new()
- .optimize_plan(Arc::clone(&self.plan), &optimizer_context)
+ .optimize(Arc::clone(&self.plan), &config)
.expect("enforce_sorting failed");
// Get string representation of the plan
@@ -2365,18 +2363,15 @@ async fn test_commutativity() -> Result<()> {
"#);
let config = ConfigOptions::new();
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let rules = vec![
Arc::new(EnforceDistribution::new()) as Arc<dyn PhysicalOptimizerRule>,
Arc::new(EnforceSorting::new()) as Arc<dyn PhysicalOptimizerRule>,
];
let mut first_plan = orig_plan.clone();
for rule in rules {
- first_plan = rule.optimize_plan(first_plan, &optimizer_context)?;
+ first_plan = rule.optimize(first_plan, &config)?;
}
- let optimizer_context2 = OptimizerContext::new(session_config.clone());
let rules = vec![
Arc::new(EnforceSorting::new()) as Arc<dyn PhysicalOptimizerRule>,
Arc::new(EnforceDistribution::new()) as Arc<dyn PhysicalOptimizerRule>,
@@ -2384,7 +2379,7 @@ async fn test_commutativity() -> Result<()> {
];
let mut second_plan = orig_plan.clone();
for rule in rules {
- second_plan = rule.optimize_plan(second_plan, &optimizer_context2)?;
+ second_plan = rule.optimize(second_plan, &config)?;
}
assert_eq!(get_plan_string(&first_plan), get_plan_string(&second_plan));
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs
index cd8ea5528b..cfcdc02f36 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs
@@ -51,7 +51,7 @@ use datafusion_physical_expr::{
};
use datafusion_physical_expr::{expressions::col, LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::{
- filter_pushdown::FilterPushdown, OptimizerContext, PhysicalOptimizerRule,
+ filter_pushdown::FilterPushdown, PhysicalOptimizerRule,
};
use datafusion_physical_plan::{
aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},
@@ -249,10 +249,8 @@ async fn test_dynamic_filter_pushdown_through_hash_join_with_topk() {
config.execution.parquet.pushdown_filters = true;
// Apply the FilterPushdown optimizer rule
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(Arc::clone(&plan), &optimizer_context)
+ .optimize(Arc::clone(&plan), &config)
.unwrap();
// Test that filters are pushed down correctly to each side of the join
@@ -728,10 +726,8 @@ async fn test_topk_dynamic_filter_pushdown() {
// Actually apply the optimization to the plan and put some data through it to check that the filter is updated to reflect the TopK state
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
let config = SessionConfig::new().with_batch_size(2);
let session_ctx = SessionContext::new_with_config(config);
@@ -813,10 +809,8 @@ async fn test_topk_dynamic_filter_pushdown_multi_column_sort() {
// Actually apply the optimization to the plan and put some data through it to check that the filter is updated to reflect the TopK state
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
let config = SessionConfig::new().with_batch_size(2);
let session_ctx = SessionContext::new_with_config(config);
@@ -1059,10 +1053,8 @@ async fn test_hashjoin_dynamic_filter_pushdown() {
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
// Test for https://github.com/apache/datafusion/pull/17371: dynamic filter linking survives `with_new_children`
@@ -1289,10 +1281,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
let config = SessionConfig::new().with_batch_size(10);
let session_ctx = SessionContext::new_with_config(config);
@@ -1488,10 +1478,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
let config = SessionConfig::new().with_batch_size(10);
let session_ctx = SessionContext::new_with_config(config);
@@ -1662,10 +1650,8 @@ async fn test_nested_hashjoin_dynamic_filter_pushdown() {
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
- let session_config = SessionConfig::from(config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
let plan = FilterPushdown::new_post_optimization()
- .optimize_plan(outer_join, &optimizer_context)
+ .optimize(outer_join, &config)
.unwrap();
let config = SessionConfig::new().with_batch_size(10);
let session_ctx = SessionContext::new_with_config(config);
@@ -1957,11 +1943,8 @@ async fn run_aggregate_dyn_filter_case(case: AggregateDynFilterCase<'_>) {
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
- let session_config: SessionConfig = config.into();
- let optimizer_context = OptimizerContext::new(session_config);
-
let optimized = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
let before = format_plan_for_test(&optimized);
@@ -2323,11 +2306,8 @@ fn test_aggregate_dynamic_filter_not_created_for_single_mode() {
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
- let session_config: SessionConfig = config.into();
- let optimizer_context = OptimizerContext::new(session_config);
-
let optimized = FilterPushdown::new_post_optimization()
- .optimize_plan(plan, &optimizer_context)
+ .optimize(plan, &config)
.unwrap();
let formatted = format_plan_for_test(&optimized);
@@ -2874,8 +2854,7 @@ async fn test_hashjoin_dynamic_filter_all_partitions_empty() {
let mut config = SessionConfig::new();
config.options_mut().execution.parquet.pushdown_filters = true;
let optimizer = FilterPushdown::new_post_optimization();
- let ctx = OptimizerContext::new(config.clone());
- let plan = optimizer.optimize_plan(plan, &ctx).unwrap();
+ let plan = optimizer.optimize(plan, config.options()).unwrap();
insta::assert_snapshot!(
format_plan_for_test(&plan),
@@ -3012,8 +2991,7 @@ async fn test_hashjoin_dynamic_filter_with_nulls() {
let mut config = SessionConfig::new();
config.options_mut().execution.parquet.pushdown_filters = true;
let optimizer = FilterPushdown::new_post_optimization();
- let ctx = OptimizerContext::new(config.clone());
- let plan = optimizer.optimize_plan(plan, &ctx).unwrap();
+ let plan = optimizer.optimize(plan, config.options()).unwrap();
insta::assert_snapshot!(
format_plan_for_test(&plan),
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs
index fc2be94b01..c32f7b2d0b 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs
@@ -25,9 +25,8 @@ use datafusion_datasource::{
file_stream::FileOpener, schema_adapter::DefaultSchemaAdapterFactory,
schema_adapter::SchemaAdapterFactory, source::DataSourceExec, PartitionedFile,
};
-use datafusion_execution::config::SessionConfig;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::filter::batch_filter;
use datafusion_physical_plan::filter_pushdown::{FilterPushdownPhase, PushedDown};
use datafusion_physical_plan::{
@@ -375,9 +374,7 @@ impl OptimizationTest {
let input = format_execution_plan(&input_plan);
let input_schema = input_plan.schema();
- let session_config = SessionConfig::from(parquet_pushdown_config);
- let optimizer_context = OptimizerContext::new(session_config.clone());
- let output_result = opt.optimize_plan(input_plan, &optimizer_context);
+ let output_result = opt.optimize(input_plan, &parquet_pushdown_config);
let output = output_result
.and_then(|plan| {
if opt.schema_check() && (plan.schema() != input_schema) {
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs
index 26b5e44853..f9d3a04546 100644
--- a/datafusion/core/tests/physical_optimizer/join_selection.rs
+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs
@@ -29,7 +29,6 @@ use datafusion_common::config::ConfigOptions;
use datafusion_common::{stats::Precision, ColumnStatistics, JoinType, ScalarValue};
use datafusion_common::{JoinSide, NullEquality};
use datafusion_common::{Result, Statistics};
-use datafusion_execution::config::SessionConfig;
use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};
use datafusion_expr::Operator;
use datafusion_physical_expr::expressions::col;
@@ -38,7 +37,6 @@ use datafusion_physical_expr::intervals::utils::check_support;
use datafusion_physical_expr::PhysicalExprRef;
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
use datafusion_physical_optimizer::join_selection::JoinSelection;
-use datafusion_physical_optimizer::OptimizerContext;
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::displayable;
use datafusion_physical_plan::joins::utils::ColumnIndex;
@@ -228,10 +226,8 @@ async fn test_join_with_swap() {
.unwrap(),
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(join, &optimizer_context)
+ .optimize(join, &ConfigOptions::new())
.unwrap();
let swapping_projection = optimized_join
@@ -292,10 +288,8 @@ async fn test_left_join_no_swap() {
.unwrap(),
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(join, &optimizer_context)
+ .optimize(join, &ConfigOptions::new())
.unwrap();
let swapped_join = optimized_join
@@ -344,10 +338,8 @@ async fn test_join_with_swap_semi() {
let original_schema = join.schema();
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(Arc::new(join), &optimizer_context)
+ .optimize(Arc::new(join), &ConfigOptions::new())
.unwrap();
let swapped_join = optimized_join
@@ -401,10 +393,8 @@ async fn test_join_with_swap_mark() {
let original_schema = join.schema();
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(Arc::new(join), &optimizer_context)
+ .optimize(Arc::new(join), &ConfigOptions::new())
.unwrap();
let swapped_join = optimized_join
@@ -440,10 +430,8 @@ macro_rules! assert_optimized {
($PLAN: expr, @$EXPECTED_LINES: literal $(,)?) => {
let plan = Arc::new($PLAN);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized = JoinSelection::new()
- .optimize_plan(plan.clone(), &optimizer_context)
+ .optimize(plan.clone(), &ConfigOptions::new())
.unwrap();
let plan_string = displayable(optimized.as_ref()).indent(true).to_string();
@@ -534,10 +522,8 @@ async fn test_join_no_swap() {
.unwrap(),
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(join, &optimizer_context)
+ .optimize(join, &ConfigOptions::new())
.unwrap();
let swapped_join = optimized_join
@@ -585,10 +571,8 @@ async fn test_nl_join_with_swap(join_type: JoinType) {
.unwrap(),
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(join, &optimizer_context)
+ .optimize(join, &ConfigOptions::new())
.unwrap();
let swapping_projection = optimized_join
@@ -665,10 +649,11 @@ async fn test_nl_join_with_swap_no_proj(join_type: JoinType) {
.unwrap(),
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(Arc::<NestedLoopJoinExec>::clone(&join), &optimizer_context)
+ .optimize(
+ Arc::<NestedLoopJoinExec>::clone(&join),
+ &ConfigOptions::new(),
+ )
.unwrap();
let swapped_join = optimized_join
@@ -925,10 +910,8 @@ fn check_join_partition_mode(
.unwrap(),
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join = JoinSelection::new()
- .optimize_plan(join, &optimizer_context)
+ .optimize(join, &ConfigOptions::new())
.unwrap();
if !is_swapped {
@@ -1572,10 +1555,8 @@ async fn test_join_with_maybe_swap_unbounded_case(t: TestCase) -> Result<()> {
NullEquality::NullEqualsNothing,
)?) as _;
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let optimized_join_plan =
- JoinSelection::new().optimize_plan(Arc::clone(&join), &optimizer_context)?;
+ JoinSelection::new().optimize(Arc::clone(&join), &ConfigOptions::new())?;
// If swap did happen
let projection_added = optimized_join_plan.as_any().is::<ProjectionExec>();
diff --git a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs
index ef333c3067..9d172db246 100644
--- a/datafusion/core/tests/physical_optimizer/limit_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/limit_pushdown.rs
@@ -24,14 +24,13 @@ use crate::physical_optimizer::test_utils::{
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
-use datafusion_execution::config::SessionConfig;
use datafusion_expr::Operator;
use datafusion_physical_expr::expressions::{col, lit, BinaryExpr};
use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;
-use datafusion_physical_optimizer::OptimizerContext;
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::filter::FilterExec;
@@ -102,10 +101,8 @@ fn transforms_streaming_table_exec_into_fetching_version_when_skip_is_zero() ->
];
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"StreamingTableExec: partition_sizes=1, projection=[c1, c2, c3], infinite_source=true, fetch=5"
@@ -129,10 +126,8 @@ fn transforms_streaming_table_exec_into_fetching_version_and_keeps_the_global_li
];
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"GlobalLimitExec: skip=2, fetch=5",
@@ -167,10 +162,8 @@ fn transforms_coalesce_batches_exec_into_fetching_version_and_removes_local_limi
];
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"CoalescePartitionsExec: fetch=5",
@@ -201,10 +194,8 @@ fn pushes_global_limit_exec_through_projection_exec() -> Result<()> {
];
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",
@@ -235,10 +226,8 @@ fn pushes_global_limit_exec_through_projection_exec_and_transforms_coalesce_batc
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c3@2 as c3]",
@@ -279,10 +268,8 @@ fn pushes_global_limit_into_multiple_fetch_plans() -> Result<()> {
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"SortPreservingMergeExec: [c1@0 ASC], fetch=5",
@@ -317,10 +304,8 @@ fn keeps_pushed_local_limit_exec_when_there_are_multiple_input_partitions() -> R
];
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = [
"CoalescePartitionsExec: fetch=5",
@@ -349,10 +334,8 @@ fn merges_local_limit_with_local_limit() -> Result<()> {
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(parent_local_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(parent_local_limit, &ConfigOptions::new())?;
let expected = ["GlobalLimitExec: skip=0, fetch=10", " EmptyExec"];
assert_eq!(get_plan_string(&after_optimize), expected);
@@ -376,10 +359,8 @@ fn merges_global_limit_with_global_limit() -> Result<()> {
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(parent_global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(parent_global_limit, &ConfigOptions::new())?;
let expected = ["GlobalLimitExec: skip=20, fetch=20", " EmptyExec"];
assert_eq!(get_plan_string(&after_optimize), expected);
@@ -403,10 +384,8 @@ fn merges_global_limit_with_local_limit() -> Result<()> {
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(global_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(global_limit, &ConfigOptions::new())?;
let expected = ["GlobalLimitExec: skip=20, fetch=20", " EmptyExec"];
assert_eq!(get_plan_string(&after_optimize), expected);
@@ -430,10 +409,8 @@ fn merges_local_limit_with_global_limit() -> Result<()> {
assert_eq!(initial, expected_initial);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- LimitPushdown::new().optimize_plan(local_limit, &optimizer_context)?;
+ LimitPushdown::new().optimize(local_limit, &ConfigOptions::new())?;
let expected = ["GlobalLimitExec: skip=20, fetch=20", " EmptyExec"];
assert_eq!(get_plan_string(&after_optimize), expected);
diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
index 1068e8ec03..80f4fbc305 100644
--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
@@ -28,7 +28,6 @@ use datafusion_common::config::{ConfigOptions, CsvOptions};
use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue};
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
use datafusion_datasource::TableSchema;
-use datafusion_execution::config::SessionConfig;
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_expr::{
@@ -45,7 +44,7 @@ use datafusion_physical_expr_common::sort_expr::{
};
use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;
use datafusion_physical_optimizer::projection_pushdown::ProjectionPushdown;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion_physical_plan::filter::FilterExec;
use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};
@@ -462,10 +461,8 @@ fn test_csv_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -502,10 +499,8 @@ fn test_memory_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -600,10 +595,8 @@ fn test_streaming_table_after_projection() -> Result<()> {
Arc::new(streaming_table) as _,
)?) as _;
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let result = after_optimize
.as_any()
@@ -702,10 +695,8 @@ fn test_projection_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(top_projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(top_projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -769,10 +760,8 @@ fn test_output_req_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -861,10 +850,8 @@ fn test_coalesce_partitions_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -920,10 +907,8 @@ fn test_filter_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1024,10 +1009,8 @@ fn test_join_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1154,10 +1137,8 @@ fn test_join_after_required_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1234,10 +1215,8 @@ fn test_nested_loop_join_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize_string =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize_string.as_ref())
.indent(true)
.to_string();
@@ -1333,10 +1312,8 @@ fn test_hash_join_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
.to_string();
@@ -1363,10 +1340,8 @@ fn test_hash_join_after_projection() -> Result<()> {
join.clone(),
)?);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
.to_string();
@@ -1418,10 +1393,8 @@ fn test_repartition_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1490,10 +1463,8 @@ fn test_sort_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1545,10 +1516,8 @@ fn test_sort_preserving_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1591,10 +1560,8 @@ fn test_union_after_projection() -> Result<()> {
"
);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1666,10 +1633,8 @@ fn test_partition_col_projection_pushdown() -> Result<()> {
source,
)?);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
@@ -1711,10 +1676,8 @@ fn test_partition_col_projection_pushdown_expr() -> Result<()> {
source,
)?);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
let after_optimize =
- ProjectionPushdown::new().optimize_plan(projection, &optimizer_context)?;
+ ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?;
let after_optimize_string = displayable(after_optimize.as_ref())
.indent(true)
diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rs
index ccacf35ec8..f46147de1b 100644
--- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs
+++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs
@@ -28,13 +28,13 @@ use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};
use datafusion::prelude::{CsvReadOptions, SessionContext};
+use datafusion_common::config::ConfigOptions;
use datafusion_common::{JoinType, Result, ScalarValue};
-use datafusion_execution::config::SessionConfig;
use datafusion_physical_expr::expressions::{col, Literal};
use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::{displayable, ExecutionPlan};
@@ -393,12 +393,9 @@ fn create_test_schema2() -> SchemaRef {
/// Check if sanity checker should accept or reject plans.
fn assert_sanity_check(plan: &Arc<dyn ExecutionPlan>, is_sane: bool) {
let sanity_checker = SanityCheckPlan::new();
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
+ let opts = ConfigOptions::default();
assert_eq!(
- sanity_checker
- .optimize_plan(plan.clone(), &optimizer_context)
- .is_ok(),
+ sanity_checker.optimize(plan.clone(), &opts).is_ok(),
is_sane
);
}
diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs
index e164e7617c..e410c495c8 100644
--- a/datafusion/core/tests/physical_optimizer/test_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs
@@ -29,12 +29,12 @@ use datafusion::datasource::listing::PartitionedFile;
use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::datasource::physical_plan::ParquetSource;
use datafusion::datasource::source::DataSourceExec;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
use datafusion_common::{ColumnStatistics, JoinType, NullEquality, Result, Statistics};
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
-use datafusion_execution::config::SessionConfig;
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_expr::{WindowFrame, WindowFunctionDefinition};
@@ -46,7 +46,7 @@ use datafusion_physical_expr_common::sort_expr::{
LexOrdering, OrderingRequirements, PhysicalSortExpr,
};
use datafusion_physical_optimizer::limited_distinct_aggregation::LimitedDistinctAggregation;
-use datafusion_physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
+use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_plan::aggregates::{
AggregateExec, AggregateMode, PhysicalGroupBy,
};
@@ -635,11 +635,10 @@ pub fn build_group_by(input_schema: &SchemaRef, columns: Vec<String>) -> Physica
}
pub fn get_optimized_plan(plan: &Arc<dyn ExecutionPlan>) -> Result<String> {
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config.clone());
+ let config = ConfigOptions::new();
- let optimized = LimitedDistinctAggregation::new()
- .optimize_plan(Arc::clone(plan), &optimizer_context)?;
+ let optimized =
+ LimitedDistinctAggregation::new().optimize(Arc::clone(plan), &config)?;
let optimized_result = displayable(optimized.as_ref()).indent(true).to_string();
diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rs
index 2f78fb5532..5dcdab1552 100644
--- a/datafusion/execution/src/config.rs
+++ b/datafusion/execution/src/config.rs
@@ -100,7 +100,7 @@ pub struct SessionConfig {
/// references to the same options.
options: Arc<ConfigOptions>,
/// Opaque extensions.
- extensions: Arc<Extensions>,
+ extensions: AnyMap,
}
impl Default for SessionConfig {
@@ -108,53 +108,14 @@ impl Default for SessionConfig {
Self {
options: Arc::new(ConfigOptions::new()),
// Assume no extensions by default.
- extensions: Arc::new(Extensions::default()),
+ extensions: HashMap::with_capacity_and_hasher(
+ 0,
+ BuildHasherDefault::default(),
+ ),
}
}
}
-/// A type map for storing extensions.
-///
-/// Extensions are indexed by their type `T`. If multiple values of the same type are provided, only the last one
-/// will be kept.
-///
-/// Extensions are opaque objects that are unknown to DataFusion itself but can be downcast by optimizer rules,
-/// execution plans, or other components that have access to the session config.
-/// They provide a flexible way to attach extra data or behavior to the session config.
-#[derive(Clone, Debug)]
-pub struct Extensions {
- inner: AnyMap,
-}
-
-impl Default for Extensions {
- fn default() -> Self {
- Self {
- inner: HashMap::with_capacity_and_hasher(0, BuildHasherDefault::default()),
- }
- }
-}
-
-impl Extensions {
- pub fn insert<T>(&mut self, value: Arc<T>)
- where
- T: Send + Sync + 'static,
- {
- let id = TypeId::of::<T>();
- self.inner.insert(id, value);
- }
-
- pub fn get<T>(&self) -> Option<Arc<T>>
- where
- T: Send + Sync + 'static,
- {
- let id = TypeId::of::<T>();
- self.inner
- .get(&id)
- .cloned()
- .map(|arc_any| Arc::downcast(arc_any).expect("TypeId unique"))
- }
-}
-
impl SessionConfig {
/// Create an execution config with default setting
pub fn new() -> Self {
@@ -203,20 +164,6 @@ impl SessionConfig {
Arc::make_mut(&mut self.options)
}
- /// Return a handle to the extensions.
- ///
- /// Can be used to read the current extensions.
- pub fn extensions(&self) -> &Arc<Extensions> {
- &self.extensions
- }
-
- /// Return a mutable handle to the extensions.
- ///
- /// Can be used to set extensions.
- pub fn extensions_mut(&mut self) -> &mut Extensions {
- Arc::make_mut(&mut self.extensions)
- }
-
/// Set a configuration option
pub fn set(self, key: &str, value: &ScalarValue) -> Self {
self.set_str(key, &value.to_string())
@@ -633,7 +580,9 @@ impl SessionConfig {
where
T: Send + Sync + 'static,
{
- self.extensions_mut().insert::<T>(ext);
+ let ext = ext as Arc<dyn Any + Send + Sync + 'static>;
+ let id = TypeId::of::<T>();
+ self.extensions.insert(id, ext);
}
/// Get extension, if any for the specified type `T` exists.
@@ -643,7 +592,11 @@ impl SessionConfig {
where
T: Send + Sync + 'static,
{
- self.extensions.get::<T>()
+ let id = TypeId::of::<T>();
+ self.extensions
+ .get(&id)
+ .cloned()
+ .map(|ext| Arc::downcast(ext).expect("TypeId unique"))
}
}
diff --git a/datafusion/physical-optimizer/src/aggregate_statistics.rs b/datafusion/physical-optimizer/src/aggregate_statistics.rs
index 19cc0b5070..4bd8e321f1 100644
--- a/datafusion/physical-optimizer/src/aggregate_statistics.rs
+++ b/datafusion/physical-optimizer/src/aggregate_statistics.rs
@@ -16,6 +16,7 @@
// under the License.
//! Utilizing exact statistics from sources to avoid scanning data
+use datafusion_common::config::ConfigOptions;
use datafusion_common::scalar::ScalarValue;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::Result;
@@ -26,7 +27,7 @@ use datafusion_physical_plan::udaf::{AggregateFunctionExpr, StatisticsArgs};
use datafusion_physical_plan::{expressions, ExecutionPlan};
use std::sync::Arc;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
/// Optimizer that uses available statistics for aggregate functions
#[derive(Default, Debug)]
@@ -43,10 +44,10 @@ impl PhysicalOptimizerRule for AggregateStatistics {
#[cfg_attr(feature = "recursive_protection", recursive::recursive)]
#[expect(clippy::allow_attributes)] // See https://github.com/apache/datafusion/issues/18881#issuecomment-3621545670
#[allow(clippy::only_used_in_recursion)] // See https://github.com/rust-lang/rust-clippy/issues/14566
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
if let Some(partial_agg_exec) = take_optimizable(&*plan) {
let partial_agg_exec = partial_agg_exec
@@ -86,15 +87,13 @@ impl PhysicalOptimizerRule for AggregateStatistics {
)?))
} else {
plan.map_children(|child| {
- self.optimize_plan(child, context).map(Transformed::yes)
+ self.optimize(child, config).map(Transformed::yes)
})
.data()
}
} else {
- plan.map_children(|child| {
- self.optimize_plan(child, context).map(Transformed::yes)
- })
- .data()
+ plan.map_children(|child| self.optimize(child, config).map(Transformed::yes))
+ .data()
}
}
diff --git a/datafusion/physical-optimizer/src/coalesce_batches.rs b/datafusion/physical-optimizer/src/coalesce_batches.rs
index b0da29beea..ecfe193b34 100644
--- a/datafusion/physical-optimizer/src/coalesce_batches.rs
+++ b/datafusion/physical-optimizer/src/coalesce_batches.rs
@@ -18,11 +18,12 @@
//! CoalesceBatches optimizer that groups batches together rows
//! in bigger batches to avoid overhead with small batches
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use std::sync::Arc;
use datafusion_common::assert_eq_or_internal_err;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_physical_expr::Partitioning;
use datafusion_physical_plan::{
@@ -44,12 +45,11 @@ impl CoalesceBatches {
}
}
impl PhysicalOptimizerRule for CoalesceBatches {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
if !config.execution.coalesce_batches {
return Ok(plan);
}
diff --git a/datafusion/physical-optimizer/src/combine_partial_final_agg.rs b/datafusion/physical-optimizer/src/combine_partial_final_agg.rs
index 5aea611f4c..667250d646 100644
--- a/datafusion/physical-optimizer/src/combine_partial_final_agg.rs
+++ b/datafusion/physical-optimizer/src/combine_partial_final_agg.rs
@@ -26,7 +26,8 @@ use datafusion_physical_plan::aggregates::{
};
use datafusion_physical_plan::ExecutionPlan;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
use datafusion_physical_expr::{physical_exprs_equal, PhysicalExpr};
@@ -46,10 +47,10 @@ impl CombinePartialFinalAggregate {
}
impl PhysicalOptimizerRule for CombinePartialFinalAggregate {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- _context: &OptimizerContext,
+ _config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
plan.transform_down(|plan| {
// Check if the plan is AggregateExec
diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs
index 571c07ff80..1f6ea61c09 100644
--- a/datafusion/physical-optimizer/src/enforce_distribution.rs
+++ b/datafusion/physical-optimizer/src/enforce_distribution.rs
@@ -30,7 +30,6 @@ use crate::utils::{
add_sort_above_with_check, is_coalesce_partitions, is_repartition,
is_sort_preserving_merge,
};
-use crate::OptimizerContext;
use arrow::compute::SortOptions;
use datafusion_common::config::ConfigOptions;
@@ -191,12 +190,11 @@ impl EnforceDistribution {
}
impl PhysicalOptimizerRule for EnforceDistribution {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
let top_down_join_key_reordering = config.optimizer.top_down_join_key_reordering;
let adjusted = if top_down_join_key_reordering {
diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rs
index 38bf1877a4..0c0b63c0b3 100644
--- a/datafusion/physical-optimizer/src/ensure_coop.rs
+++ b/datafusion/physical-optimizer/src/ensure_coop.rs
@@ -23,8 +23,9 @@
use std::fmt::{Debug, Formatter};
use std::sync::Arc;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_common::Result;
use datafusion_physical_plan::coop::CooperativeExec;
@@ -61,10 +62,10 @@ impl PhysicalOptimizerRule for EnsureCooperative {
"EnsureCooperative"
}
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- _context: &OptimizerContext,
+ _config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
plan.transform_up(|plan| {
let is_leaf = plan.children().is_empty();
@@ -95,17 +96,16 @@ impl PhysicalOptimizerRule for EnsureCooperative {
#[cfg(test)]
mod tests {
use super::*;
- use datafusion_execution::config::SessionConfig;
+ use datafusion_common::config::ConfigOptions;
use datafusion_physical_plan::{displayable, test::scan_partitioned};
use insta::assert_snapshot;
#[tokio::test]
async fn test_cooperative_exec_for_custom_exec() {
let test_custom_exec = scan_partitioned(1);
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config);
+ let config = ConfigOptions::new();
let optimized = EnsureCooperative::new()
- .optimize_plan(test_custom_exec, &optimizer_context)
+ .optimize(test_custom_exec, &config)
.unwrap();
let display = displayable(optimized.as_ref()).indent(true).to_string();
diff --git a/datafusion/physical-optimizer/src/filter_pushdown.rs b/datafusion/physical-optimizer/src/filter_pushdown.rs
index eacffa9198..22cb03fc3e 100644
--- a/datafusion/physical-optimizer/src/filter_pushdown.rs
+++ b/datafusion/physical-optimizer/src/filter_pushdown.rs
@@ -33,7 +33,7 @@
use std::sync::Arc;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion_common::{assert_eq_or_internal_err, config::ConfigOptions, Result};
@@ -416,19 +416,6 @@ impl Default for FilterPushdown {
}
impl PhysicalOptimizerRule for FilterPushdown {
- fn optimize_plan(
- &self,
- plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
- Ok(
- push_down_filters(&Arc::clone(&plan), vec![], config, self.phase)?
- .updated_node
- .unwrap_or(plan),
- )
- }
-
fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rs
index 8f337bc8d9..1ffe6dbca3 100644
--- a/datafusion/physical-optimizer/src/join_selection.rs
+++ b/datafusion/physical-optimizer/src/join_selection.rs
@@ -23,7 +23,7 @@
//! pipeline-friendly ones. To achieve the second goal, it selects the proper
//! `PartitionMode` and the build side using the available statistics for hash joins.
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
@@ -103,12 +103,11 @@ fn supports_collect_by_thresholds(
}
impl PhysicalOptimizerRule for JoinSelection {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
// First, we make pipeline-fixing modifications to joins so as to accommodate
// unbounded inputs. Each pipeline-fixing subrule, which is a function
// of type `PipelineFixerSubrule`, takes a single [`PipelineStatePropagator`]
diff --git a/datafusion/physical-optimizer/src/lib.rs b/datafusion/physical-optimizer/src/lib.rs
index d1bf375bd0..cf8cf33cea 100644
--- a/datafusion/physical-optimizer/src/lib.rs
+++ b/datafusion/physical-optimizer/src/lib.rs
@@ -47,4 +47,4 @@ pub mod topk_aggregation;
pub mod update_aggr_exprs;
pub mod utils;
-pub use optimizer::{OptimizerContext, PhysicalOptimizerRule};
+pub use optimizer::PhysicalOptimizerRule;
diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rs
index 74f1af2ff5..4cb3abe30b 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs
@@ -21,8 +21,9 @@
use std::fmt::Debug;
use std::sync::Arc;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::error::Result;
use datafusion_common::tree_node::{Transformed, TreeNodeRecursion};
use datafusion_common::utils::combine_limit;
@@ -59,10 +60,10 @@ impl LimitPushdown {
}
impl PhysicalOptimizerRule for LimitPushdown {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- _context: &OptimizerContext,
+ _config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
let global_state = GlobalRequirements {
fetch: None,
diff --git a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
index ee20f4d43b..1c671cd074 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown_past_window.rs
@@ -15,7 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::ScalarValue;
use datafusion_expr::{LimitEffect, WindowFrameBound, WindowFrameUnits};
@@ -70,12 +71,11 @@ impl TraverseState {
}
impl PhysicalOptimizerRule for LimitPushPastWindows {
- fn optimize_plan(
+ fn optimize(
&self,
original: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
if !config.optimizer.enable_window_limits {
return Ok(original);
}
diff --git a/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs b/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs
index 89828a87cb..3666ff3798 100644
--- a/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs
+++ b/datafusion/physical-optimizer/src/limited_distinct_aggregation.rs
@@ -24,10 +24,11 @@ use datafusion_physical_plan::aggregates::AggregateExec;
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::Result;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use itertools::Itertools;
/// An optimizer rule that passes a `limit` hint into grouped aggregations which don't require all
@@ -157,12 +158,11 @@ impl Default for LimitedDistinctAggregation {
}
impl PhysicalOptimizerRule for LimitedDistinctAggregation {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
if config.optimizer.enable_distinct_aggregation_soft_limit {
plan.transform_down(|plan| {
Ok(
diff --git a/datafusion/physical-optimizer/src/optimizer.rs b/datafusion/physical-optimizer/src/optimizer.rs
index 4524947135..03c83bb5a0 100644
--- a/datafusion/physical-optimizer/src/optimizer.rs
+++ b/datafusion/physical-optimizer/src/optimizer.rs
@@ -38,37 +38,10 @@ use crate::update_aggr_exprs::OptimizeAggregateOrder;
use crate::limit_pushdown_past_window::LimitPushPastWindows;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{internal_err, Result};
-use datafusion_execution::config::SessionConfig;
+use datafusion_common::Result;
use datafusion_physical_plan::ExecutionPlan;
-/// Context for optimizing physical plans.
-///
-/// This context provides access to session configuration and optimizer extensions.
-///
-/// Similar to [`TaskContext`] which provides context during execution,
-/// [`OptimizerContext`] provides context during optimization.
-///
-/// [`TaskContext`]: https://docs.rs/datafusion/latest/datafusion/execution/struct.TaskContext.html
-#[derive(Debug, Clone)]
-pub struct OptimizerContext {
- /// Session configuration
- session_config: SessionConfig,
-}
-
-impl OptimizerContext {
- /// Create a new OptimizerContext
- pub fn new(session_config: SessionConfig) -> Self {
- Self { session_config }
- }
-
- /// Return a reference to the session configuration
- pub fn session_config(&self) -> &SessionConfig {
- &self.session_config
- }
-}
-
-/// `PhysicalOptimizerRule` transforms one [`ExecutionPlan`] into another which
+/// `PhysicalOptimizerRule` transforms one ['ExecutionPlan'] into another which
/// computes the same results, but in a potentially more efficient way.
///
/// Use [`SessionState::add_physical_optimizer_rule`] to register additional
@@ -76,52 +49,12 @@ impl OptimizerContext {
///
/// [`SessionState::add_physical_optimizer_rule`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html#method.add_physical_optimizer_rule
pub trait PhysicalOptimizerRule: Debug {
- /// Rewrite `plan` to an optimized form with additional context
- ///
- /// This is the preferred method for implementing optimization rules as it
- /// provides access to the full optimizer context including session configuration.
- ///
- /// The default implementation delegates to [`PhysicalOptimizerRule::optimize`] for
- /// backwards compatibility with existing implementations.
- ///
- /// New implementations should override this method instead of `optimize()`.
- ///
- /// Once [`PhysicalOptimizerRule::optimize`] is deprecated and removed, this
- /// default implementation will be removed and this method will become required.
- /// This change is scheduled for DataFusion 54.0.0.
- fn optimize_plan(
- &self,
- plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- // Default implementation: delegate to the old method for backwards compatibility
- #[expect(deprecated)]
- self.optimize(plan, context.session_config().options())
- }
-
/// Rewrite `plan` to an optimized form
- ///
- /// This method is kept for backwards compatibility. New implementations
- /// should implement [`optimize_plan`](Self::optimize_plan) instead, which
- /// provides access to additional context.
- ///
- /// The default implementation returns an error indicating that neither
- /// `optimize` nor `optimize_plan` was properly implemented. At least one
- /// of these methods must be overridden.
- #[deprecated(
- since = "52.0.0",
- note = "use `PhysicalOptimizerRule::optimize_plan` instead"
- )]
fn optimize(
&self,
- _plan: Arc<dyn ExecutionPlan>,
- _config: &ConfigOptions,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- internal_err!(
- "PhysicalOptimizerRule '{}' must implement either optimize() or optimize_plan()",
- self.name()
- )
- }
+ plan: Arc<dyn ExecutionPlan>,
+ config: &ConfigOptions,
+ ) -> Result<Arc<dyn ExecutionPlan>>;
/// A human readable name for this optimizer rule
fn name(&self) -> &str;
diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rs
index c11b1d362a..9e5e980219 100644
--- a/datafusion/physical-optimizer/src/output_requirements.rs
+++ b/datafusion/physical-optimizer/src/output_requirements.rs
@@ -24,8 +24,9 @@
use std::sync::Arc;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{Result, Statistics};
use datafusion_execution::TaskContext;
@@ -302,10 +303,10 @@ impl ExecutionPlan for OutputRequirementExec {
}
impl PhysicalOptimizerRule for OutputRequirements {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- _context: &OptimizerContext,
+ _config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
match self.mode {
RuleMode::Add => require_top_ordering(plan),
diff --git a/datafusion/physical-optimizer/src/projection_pushdown.rs b/datafusion/physical-optimizer/src/projection_pushdown.rs
index 89a595f61b..cb51c9167c 100644
--- a/datafusion/physical-optimizer/src/projection_pushdown.rs
+++ b/datafusion/physical-optimizer/src/projection_pushdown.rs
@@ -20,12 +20,13 @@
//! projections one by one if the operator below is amenable to this. If a
//! projection reaches a source, it can even disappear from the plan entirely.
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use arrow::datatypes::{Fields, Schema, SchemaRef};
use datafusion_common::alias::AliasGenerator;
use std::collections::HashSet;
use std::sync::Arc;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{
Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
};
@@ -56,10 +57,10 @@ impl ProjectionPushdown {
}
impl PhysicalOptimizerRule for ProjectionPushdown {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- _context: &OptimizerContext,
+ _config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
let alias_generator = AliasGenerator::new();
let plan = plan
@@ -446,8 +447,6 @@ fn is_volatile_expression_tree(expr: &dyn PhysicalExpr) -> bool {
mod test {
use super::*;
use arrow::datatypes::{DataType, Field, FieldRef, Schema};
- use datafusion_common::config::ConfigOptions;
- use datafusion_execution::config::SessionConfig;
use datafusion_expr_common::operator::Operator;
use datafusion_functions::math::random;
use datafusion_physical_expr::expressions::{binary, lit};
@@ -673,10 +672,7 @@ mod test {
)?;
let optimizer = ProjectionPushdown::new();
- let session_config = SessionConfig::new();
- let optimizer_context = OptimizerContext::new(session_config);
- let optimized_plan =
- optimizer.optimize_plan(Arc::new(join), &optimizer_context)?;
+ let optimized_plan = optimizer.optimize(Arc::new(join), &Default::default())?;
let displayable_plan = displayable(optimized_plan.as_ref()).indent(false);
Ok(displayable_plan.to_string())
diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rs
index 7e81ba5f15..23df6b4e55 100644
--- a/datafusion/physical-optimizer/src/sanity_checker.rs
+++ b/datafusion/physical-optimizer/src/sanity_checker.rs
@@ -26,7 +26,7 @@ use std::sync::Arc;
use datafusion_common::Result;
use datafusion_physical_plan::ExecutionPlan;
-use datafusion_common::config::OptimizerOptions;
+use datafusion_common::config::{ConfigOptions, OptimizerOptions};
use datafusion_common::plan_err;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported};
@@ -34,7 +34,7 @@ use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion_physical_plan::joins::SymmetricHashJoinExec;
use datafusion_physical_plan::{get_plan_string, ExecutionPlanProperties};
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use datafusion_physical_expr_common::sort_expr::format_physical_sort_requirement_list;
use itertools::izip;
@@ -54,12 +54,11 @@ impl SanityCheckPlan {
}
impl PhysicalOptimizerRule for SanityCheckPlan {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
plan.transform_up(|p| check_plan_sanity(p, &config.optimizer))
.data()
}
diff --git a/datafusion/physical-optimizer/src/topk_aggregation.rs b/datafusion/physical-optimizer/src/topk_aggregation.rs
index a55b90e839..b7505f0df4 100644
--- a/datafusion/physical-optimizer/src/topk_aggregation.rs
+++ b/datafusion/physical-optimizer/src/topk_aggregation.rs
@@ -19,8 +19,9 @@
use std::sync::Arc;
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
use arrow::datatypes::DataType;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::Result;
use datafusion_physical_expr::expressions::Column;
@@ -144,12 +145,11 @@ impl Default for TopKAggregation {
}
impl PhysicalOptimizerRule for TopKAggregation {
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
+ config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
- let config = context.session_config().options();
if config.optimizer.enable_topk_aggregation {
plan.transform_down(|plan| {
Ok(if let Some(plan) = TopKAggregation::transform_sort(&plan) {
diff --git a/datafusion/physical-optimizer/src/update_aggr_exprs.rs b/datafusion/physical-optimizer/src/update_aggr_exprs.rs
index 604bc6c3e1..c22a63c0c1 100644
--- a/datafusion/physical-optimizer/src/update_aggr_exprs.rs
+++ b/datafusion/physical-optimizer/src/update_aggr_exprs.rs
@@ -20,6 +20,7 @@
use std::sync::Arc;
+use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{plan_datafusion_err, Result};
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
@@ -28,7 +29,7 @@ use datafusion_physical_plan::aggregates::{concat_slices, AggregateExec};
use datafusion_physical_plan::windows::get_ordered_partition_by_indices;
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
-use crate::{OptimizerContext, PhysicalOptimizerRule};
+use crate::PhysicalOptimizerRule;
/// This optimizer rule checks ordering requirements of aggregate expressions.
///
@@ -64,15 +65,15 @@ impl PhysicalOptimizerRule for OptimizeAggregateOrder {
/// # Arguments
///
/// * `plan` - The root of the execution plan to optimize.
- /// * `context` - Optimizer context containing configuration options.
+ /// * `_config` - Configuration options (currently unused).
///
/// # Returns
///
/// A `Result` containing the potentially optimized execution plan or an error.
- fn optimize_plan(
+ fn optimize(
&self,
plan: Arc<dyn ExecutionPlan>,
- _context: &OptimizerContext,
+ _config: &ConfigOptions,
) -> Result<Arc<dyn ExecutionPlan>> {
plan.transform_up(|plan| {
if let Some(aggr_exec) = plan.as_any().downcast_ref::<AggregateExec>() {
diff --git a/docs/source/library-user-guide/upgrading.md b/docs/source/library-user-guide/upgrading.md
index caab9ad4e7..6462961ce6 100644
--- a/docs/source/library-user-guide/upgrading.md
+++ b/docs/source/library-user-guide/upgrading.md
@@ -380,102 +380,6 @@ If you were using a custom `SchemaAdapterFactory` for schema adaptation (e.g., d
See the [default column values example](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_data_source/default_column_values.rs) for how to implement a custom `PhysicalExprAdapterFactory`.
-### `PhysicalOptimizerRule::optimize` deprecated in favor of `optimize_plan`
-
-The `PhysicalOptimizerRule` trait has been updated to provide optimizer rules with access to broader session context. A new method `optimize_plan` has been added that accepts an `OptimizerContext` parameter, and the existing `optimize` method has been deprecated.
-
-**Who is affected:**
-
-- Users who have implemented custom `PhysicalOptimizerRule` implementations
-
-**Breaking changes:**
-
-1. **New `optimize_plan` method**: This is the preferred method for implementing optimization rules. It provides access to the full `SessionConfig` through `OptimizerContext`, rather than just `ConfigOptions`.
-
-2. **`optimize` method deprecated**: The old `optimize` method that takes `&ConfigOptions` is now deprecated and will be removed in DataFusion 58.0.0.
-
-**Migration guide:**
-
-If you have a custom `PhysicalOptimizerRule` implementation, update it to implement `optimize_plan` instead of `optimize`:
-
-**Before:**
-
-```rust
-use datafusion::physical_optimizer::PhysicalOptimizerRule;
-use datafusion_common::config::ConfigOptions;
-use datafusion_common::Result;
-use datafusion_physical_plan::ExecutionPlan;
-use std::sync::Arc;
-
-#[derive(Debug)]
-struct MyOptimizerRule;
-
-#[allow(deprecated)]
-impl PhysicalOptimizerRule for MyOptimizerRule {
- fn optimize(
- &self,
- plan: Arc<dyn ExecutionPlan>,
- _config: &ConfigOptions,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- // Use config.optimizer, config.execution, etc.
- // ... optimization logic ...
- Ok(plan)
- }
-
- fn name(&self) -> &str {
- "my_optimizer_rule"
- }
-
- fn schema_check(&self) -> bool {
- true
- }
-}
-```
-
-**After:**
-
-```rust
-use datafusion::physical_optimizer::{OptimizerContext, PhysicalOptimizerRule};
-use datafusion_common::Result;
-use datafusion_physical_plan::ExecutionPlan;
-use std::sync::Arc;
-
-#[derive(Debug)]
-struct MyOptimizerRule;
-
-impl PhysicalOptimizerRule for MyOptimizerRule {
- fn optimize_plan(
- &self,
- plan: Arc<dyn ExecutionPlan>,
- context: &OptimizerContext,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- // Access ConfigOptions through session_config
- let _config = context.session_config().options();
- // Or access extensions through session_config
- let _extensions = context.session_config().extensions();
- // ... optimization logic ...
- Ok(plan)
- }
-
- fn name(&self) -> &str {
- "my_optimizer_rule"
- }
-
- fn schema_check(&self) -> bool {
- true
- }
-}
-```
-
-**What is `OptimizerContext`?**
-
-`OptimizerContext` is a new struct that provides context during physical plan optimization, similar to how `TaskContext` provides context during execution. It wraps `SessionConfig`, giving optimizer rules access to:
-
-- Configuration options via `context.session_config().options()`
-- Session extensions via `context.session_config().extensions()`
-
-This enables optimizer rules to access custom extensions registered with the session, which was not possible with the old `&ConfigOptions` parameter.
-
## DataFusion `51.0.0`
### `arrow` / `parquet` updated to 57.0.0