Skip to content

Apache DataFusion SQL Query Engine

docs: Document that adding new optimizer rules are expensive (#20348)

Andrew Lamb89a85761717c
diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs
index 118ddef49b..0da4d6352a 100644
--- a/datafusion/optimizer/src/optimizer.rs
+++ b/datafusion/optimizer/src/optimizer.rs
@@ -236,6 +236,15 @@ impl Default for Optimizer {
impl Optimizer {
/// Create a new optimizer using the recommended list of rules
pub fn new() -> Self {
+ // NOTEs:
+ // - The order of rules in this list is important, as it determines the
+ // order in which they are applied.
+ // - Adding a new rule here is expensive as it will be applied to all
+ // queries, and will likely increase the optimization time. Please extend
+ // existing rules when possible, rather than adding a new rule.
+ // If you do add a new rule considering having aggressive no-op paths
+ // (e.g. if the plan doesn't contain any of the nodes you are looking for
+ // return `Transformed::no`; only works if you control the traversal).
let rules: Vec<Arc<dyn OptimizerRule + Sync + Send>> = vec![
Arc::new(RewriteSetComparison::new()),
Arc::new(OptimizeUnions::new()),
diff --git a/datafusion/physical-optimizer/src/optimizer.rs b/datafusion/physical-optimizer/src/optimizer.rs
index ff71c9ec64..49225db03a 100644
--- a/datafusion/physical-optimizer/src/optimizer.rs
+++ b/datafusion/physical-optimizer/src/optimizer.rs
@@ -82,6 +82,12 @@ impl Default for PhysicalOptimizer {
impl PhysicalOptimizer {
/// Create a new optimizer using the recommended list of rules
pub fn new() -> Self {
+ // NOTEs:
+ // - The order of rules in this list is important, as it determines the
+ // order in which they are applied.
+ // - Adding a new rule here is expensive as it will be applied to all
+ // queries, and will likely increase the optimization time. Please extend
+ // existing rules when possible, rather than adding a new rule.
let rules: Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>> = vec![
// If there is a output requirement of the query, make sure that
// this information is not lost across different rules during optimization.