Skip to content

Apache DataFusion SQL Query Engine

docs: Clarify `EquivalenceClass`, `ConstExpr`, and `OrderingEquivalenceClass` docs (#27238)

Andrew Lamb0a363f639eb8
diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs
index a94a38f47f..09dd79f12d 100644
--- a/datafusion/physical-expr/src/equivalence/class.rs
+++ b/datafusion/physical-expr/src/equivalence/class.rs
@@ -24,16 +24,19 @@ use datafusion_physical_expr_common::physical_expr::{
use indexmap::{IndexMap, IndexSet};
-/// Represents whether a constant expression's value is uniform or varies across
-/// partitions. Has two variants:
-/// - `Heterogeneous`: The constant expression may have different values for
-/// different partitions.
-/// - `Uniform(Option<ScalarValue>)`: The constant expression has the same value
-/// across all partitions, or is `None` if the value is unknown.
+/// How a constant expression's value varies across partitions.
+///
+/// See [Constants] for why the difference matters.
+///
+/// [Constants]: crate::equivalence::EquivalenceProperties#constants
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub enum AcrossPartitions {
+ /// The constant may have different values for different partitions.
#[default]
Heterogeneous,
+ /// The constant has the same value across all partitions.
+ ///
+ /// `Some` if the value is exactly known, `None` otherwise.
Uniform(Option<ScalarValue>),
}
@@ -52,20 +55,11 @@ impl Display for AcrossPartitions {
}
}
-/// A structure representing a expression known to be constant in a physical
-/// execution plan.
-///
-/// The `ConstExpr` struct encapsulates an expression that is constant during
-/// the execution of a query. For example if a filter like `A = 5` appears
-/// earlier in the plan, `A` would become a constant in subsequent operations.
-///
-/// # Fields
+/// An expression known to be constant during the execution of a query.
///
-/// - `expr`: Constant expression for a node in the physical plan.
-/// - `across_partitions`: A boolean flag indicating whether the constant
-/// expression is the same across partitions. If set to `true`, the constant
-/// expression has same value for all partitions. If set to `false`, the
-/// constant expression may have different values for different partitions.
+/// For example, after a filter like `A = 5` in the plan, the expression `A`
+/// is known to be constant in subsequent operations. See [Constants] for more
+/// details.
///
/// # Example
///
@@ -76,11 +70,13 @@ impl Display for AcrossPartitions {
/// // Create a constant expression from a physical expression:
/// let const_expr = ConstExpr::from(col);
/// ```
+///
+/// [Constants]: crate::equivalence::EquivalenceProperties#constants
#[derive(Clone, Debug)]
pub struct ConstExpr {
/// The expression that is known to be constant (e.g. a `Column`).
pub expr: Arc<dyn PhysicalExpr>,
- /// Indicates whether the constant have the same value across all partitions.
+ /// Does the constant have the same value across all partitions?
pub across_partitions: AcrossPartitions,
}
// TODO: The `ConstExpr` definition above can be in an inconsistent state where
@@ -157,13 +153,17 @@ impl From<Arc<dyn PhysicalExpr>> for ConstExpr {
}
}
-/// An `EquivalenceClass` is a set of [`Arc<dyn PhysicalExpr>`]s that are known
-/// to have the same value for all tuples in a relation. These are generated by
-/// equality predicates (e.g. `a = b`), typically equi-join conditions and
-/// equality conditions in filters.
+/// A set of [`Arc<dyn PhysicalExpr>`]s that are known
+/// to have the same value for all tuples in a relation.
+///
+/// `EquivalenceClass`es are generated by equality predicates such as `a = b`,
+/// typically equi-join conditions and conditions in filters. See
+/// [Equivalent expressions] for more detail.
+///
+/// Two `EquivalenceClass`es are equal if they contain the same expressions, regardless
+/// of their order.
///
-/// Two `EquivalenceClass`es are equal if they contains the same expressions in
-/// without any ordering.
+/// [Equivalent expressions]: crate::equivalence::EquivalenceProperties#equivalent-expressions
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct EquivalenceClass {
/// The expressions in this equivalence class. The order doesn't matter for
@@ -300,8 +300,12 @@ type AugmentedMapping<'a> = IndexMap<
(&'a ProjectionTargets, Option<&'a EquivalenceClass>),
>;
-/// A collection of distinct `EquivalenceClass`es. This object supports fast
-/// lookups of expressions and their equivalence classes.
+/// A set of distinct [`EquivalenceClass`]es.
+///
+/// `EquivalenceGroup` supports fast lookups of expressions and their
+/// equivalence classes. See [`EquivalenceProperties`] for more details.
+///
+/// [`EquivalenceProperties`]: crate::equivalence::EquivalenceProperties
#[derive(Clone, Debug, Default)]
pub struct EquivalenceGroup {
/// A mapping from expressions to their equivalence class key.
diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs
index ff57146efc..1f869cf06c 100644
--- a/datafusion/physical-expr/src/equivalence/mod.rs
+++ b/datafusion/physical-expr/src/equivalence/mod.rs
@@ -2,6 +2,15 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+//! Equivalence properties: the orderings, equality, constants and
+//! groupings known to hold for the output of a plan node, which the physical
+//! optimizer uses to avoid sorts and repartitions.
+//!
+//! # See Also
+//!
+//! * [`EquivalenceProperties`], which explains the concepts and how the
+//! pieces below fit together.
+
use std::borrow::Borrow;
use std::sync::Arc;
diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs
index 7de47767f6..a908cce4fe 100644
--- a/datafusion/physical-expr/src/equivalence/ordering.rs
+++ b/datafusion/physical-expr/src/equivalence/ordering.rs
@@ -15,8 +15,9 @@ use arrow::datatypes::SchemaRef;
use datafusion_common::Result;
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
-/// An `OrderingEquivalenceClass` keeps track of distinct alternative orderings
-/// than can describe a table. For example, consider the following table:
+/// Tracks distinct alternative orderings that describe a table.
+///
+/// For example, consider the following table:
///
/// ```text
/// ┌───┬───┬───┬───┐
@@ -30,16 +31,25 @@ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
/// ```
///
/// Here, both `[a ASC, b ASC]` and `[c DESC, d ASC]` describe the table
-/// ordering. In this case, we say that these orderings are equivalent.
+/// ordering, so these orderings are equivalent. See [Equivalent sort
+/// expressions] for more details.
+///
+/// An `OrderingEquivalenceClass` is a set of such equivalent orderings.
+///
+/// [`GroupingEquivalenceClass`] is used when groups of equal values are
+/// contiguous, but not in order.
///
-/// An `OrderingEquivalenceClass` is a set of such equivalent orderings, which
-/// is represented by a vector of `LexOrdering`s. The set does not store any
-/// redundant information by enforcing the invariant that no suffix of an
-/// ordering in the equivalence class is a prefix of another ordering in the
-/// equivalence class. The set can be empty, which means that there are no
-/// orderings that describe the table.
+/// [Equivalent sort expressions]: crate::equivalence::EquivalenceProperties#equivalent-sort-expressions
+/// [`GroupingEquivalenceClass`]: crate::equivalence::GroupingEquivalenceClass
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct OrderingEquivalenceClass {
+ /// Set of equivalent orderings
+ ///
+ /// Invariant: no suffix of an ordering in the equivalence class is a prefix
+ /// of another ordering in the equivalence class. This avoids redundant
+ /// information.
+ ///
+ /// If empty, means the table is unordered
orderings: Vec<LexOrdering>,
}
@@ -216,18 +226,11 @@ impl OrderingEquivalenceClass {
/// `ASC NULLS LAST` and `DESC NULLS FIRST`. Rows sorted both ways all have
/// the same value (or are all null).
///
- /// This is how a value that is constant only within groups of rows is
- /// recorded, since a `ConstExpr` is constant across all rows. For example,
- /// `SUM(x) OVER (PARTITION BY k)` has one value per `k`, so the window adds
- /// `[k, w]` once for each of the four directions of `w`.
- ///
- /// ### Future Improvements
+ /// This is how a whole-partition window aggregate records that its value
+ /// is constant within each window partition. See
+ /// [Per-partition constants from window functions].
///
- /// This function may become unnecessary if any of the following improvements
- /// are implemented:
- /// 1. `SortOptions` supports encoding constantness information.
- /// 2. `EquivalenceProperties` gains `FunctionalDependency` awareness, eliminating
- /// the need for `Constant` and `Constraints`.
+ /// [Per-partition constants from window functions]: crate::equivalence::EquivalenceProperties#example-window-functions-with-partition-by
pub fn is_expr_partial_const(&self, expr: &Arc<dyn PhysicalExpr>) -> bool {
is_partial_const(self.iter().map(|ordering| ordering.first()), expr)
}