Skip to content

Apache DataFusion SQL Query Engine

feat: add grouping equivalence properties (#24698)

xavlee320a7d28bc02
diff --git a/datafusion/physical-expr/src/equivalence/grouping.rs b/datafusion/physical-expr/src/equivalence/grouping.rs
new file mode 100644
index 0000000000..2fce6dad82
--- /dev/null
+++ b/datafusion/physical-expr/src/equivalence/grouping.rs
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+use std::fmt::Display;
+use std::ops::Deref;
+use std::sync::Arc;
+use std::vec::IntoIter;
+
+use crate::PhysicalExpr;
+use crate::expressions::with_new_schema;
+
+use arrow::datatypes::SchemaRef;
+use datafusion_common::{HashSet, Result};
+use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;
+
+/// A set of grouping tuples that are known to describe contiguous rows.
+///
+/// Each entry is a complete grouping tuple. For example, `[a, b]` means that,
+/// within each output partition, all rows having the same values for both `a`
+/// and `b` occur in one contiguous run. The runs themselves may occur in any
+/// order. Contiguity applies to the complete partition stream, not separately
+/// to each record batch.
+///
+/// Expression order within an entry is not significant: `[a, b]` and `[b, a]`
+/// describe the same grouping. Entries do not imply properties for subsets;
+/// `[a, b]` alone says nothing about whether all rows with the same `a` are
+/// contiguous.
+#[derive(Clone, Debug, Default)]
+pub struct GroupingEquivalenceClass {
+ groupings: Vec<Vec<Arc<dyn PhysicalExpr>>>,
+}
+
+impl GroupingEquivalenceClass {
+ /// Clears all groupings in this equivalence class.
+ pub fn clear(&mut self) {
+ self.groupings.clear();
+ }
+
+ /// Creates a grouping equivalence class, discarding empty and duplicate
+ /// entries and duplicate expressions within each entry.
+ pub fn new(
+ groupings: impl IntoIterator<Item = impl IntoIterator<Item = Arc<dyn PhysicalExpr>>>,
+ ) -> Self {
+ let mut result = Self::default();
+ result.add_groupings(groupings);
+ result
+ }
+
+ /// Adds grouping tuples to this equivalence class.
+ pub fn add_groupings(
+ &mut self,
+ groupings: impl IntoIterator<Item = impl IntoIterator<Item = Arc<dyn PhysicalExpr>>>,
+ ) {
+ for grouping in groupings {
+ let mut seen = HashSet::new();
+ let grouping = grouping
+ .into_iter()
+ .filter(|expr| seen.insert(Arc::clone(expr)))
+ .collect::<Vec<_>>();
+ if !grouping.is_empty() && !self.contains(&grouping) {
+ self.groupings.push(grouping);
+ }
+ }
+ }
+
+ /// Returns whether this class contains the complete grouping tuple.
+ pub fn contains(&self, grouping: &[Arc<dyn PhysicalExpr>]) -> bool {
+ self.groupings
+ .iter()
+ .any(|candidate| same_grouping(candidate, grouping))
+ }
+
+ /// Rewrites all expressions to reference an aligned schema.
+ pub fn with_new_schema(self, schema: &SchemaRef) -> Result<Self> {
+ let groupings = self.groupings.into_iter().map(|grouping| {
+ grouping
+ .into_iter()
+ .map(|expr| with_new_schema(expr, schema))
+ .collect::<Result<Vec<_>>>()
+ });
+ Ok(Self::new(groupings.collect::<Result<Vec<_>>>()?))
+ }
+}
+
+fn same_grouping(lhs: &[Arc<dyn PhysicalExpr>], rhs: &[Arc<dyn PhysicalExpr>]) -> bool {
+ lhs.iter()
+ .all(|lhs_expr| rhs.iter().any(|rhs_expr| lhs_expr.eq(rhs_expr)))
+ && rhs
+ .iter()
+ .all(|rhs_expr| lhs.iter().any(|lhs_expr| rhs_expr.eq(lhs_expr)))
+}
+
+impl PartialEq for GroupingEquivalenceClass {
+ fn eq(&self, other: &Self) -> bool {
+ self.groupings.len() == other.groupings.len()
+ && self
+ .groupings
+ .iter()
+ .all(|grouping| other.contains(grouping))
+ }
+}
+
+impl Eq for GroupingEquivalenceClass {}
+
+impl Deref for GroupingEquivalenceClass {
+ type Target = [Vec<Arc<dyn PhysicalExpr>>];
+
+ fn deref(&self) -> &Self::Target {
+ self.groupings.as_slice()
+ }
+}
+
+impl From<Vec<Vec<Arc<dyn PhysicalExpr>>>> for GroupingEquivalenceClass {
+ fn from(groupings: Vec<Vec<Arc<dyn PhysicalExpr>>>) -> Self {
+ Self::new(groupings)
+ }
+}
+
+/// Converts the grouping equivalence class into an iterator of complete
+/// grouping tuples.
+impl IntoIterator for GroupingEquivalenceClass {
+ type Item = Vec<Arc<dyn PhysicalExpr>>;
+ type IntoIter = IntoIter<Self::Item>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.groupings.into_iter()
+ }
+}
+
+impl From<GroupingEquivalenceClass> for Vec<Vec<Arc<dyn PhysicalExpr>>> {
+ fn from(geq_class: GroupingEquivalenceClass) -> Self {
+ geq_class.groupings
+ }
+}
+
+impl Display for GroupingEquivalenceClass {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "[")?;
+ let mut groupings = self.groupings.iter();
+ if let Some(grouping) = groupings.next() {
+ write!(f, "{}", format_physical_expr_list(grouping))?;
+ }
+ for grouping in groupings {
+ write!(f, ", {}", format_physical_expr_list(grouping))?;
+ }
+ write!(f, "]")
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::expressions::Column;
+
+ #[test]
+ fn grouping_equality_ignores_expression_and_entry_order() {
+ let a = Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>;
+ let b = Arc::new(Column::new("b", 1)) as Arc<dyn PhysicalExpr>;
+ let c = Arc::new(Column::new("c", 2)) as Arc<dyn PhysicalExpr>;
+
+ let lhs = GroupingEquivalenceClass::new([
+ vec![Arc::clone(&a), Arc::clone(&b)],
+ vec![Arc::clone(&c)],
+ ]);
+ let rhs = GroupingEquivalenceClass::new([
+ vec![Arc::clone(&c)],
+ vec![Arc::clone(&b), Arc::clone(&a)],
+ ]);
+
+ assert_eq!(lhs, rhs);
+ // The same expressions in different tuples describe different guarantees.
+ assert_ne!(lhs, GroupingEquivalenceClass::new([vec![a, c], vec![b]]));
+ }
+
+ #[test]
+ fn grouping_deduplicates_entries_and_expressions() {
+ let a = Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>;
+ let b = Arc::new(Column::new("b", 1)) as Arc<dyn PhysicalExpr>;
+
+ let groupings = GroupingEquivalenceClass::new([
+ vec![Arc::clone(&a), Arc::clone(&a), Arc::clone(&b)],
+ vec![Arc::clone(&b), Arc::clone(&a)],
+ vec![],
+ ]);
+
+ assert_eq!(groupings.len(), 1);
+ assert_eq!(groupings[0].len(), 2);
+ assert!(groupings.contains(&[Arc::clone(&a), Arc::clone(&a), Arc::clone(&b),]));
+ assert_eq!(groupings.to_string(), "[[a@0, b@1]]");
+ }
+}
diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs
index 1b878ee400..ff57146efc 100644
--- a/datafusion/physical-expr/src/equivalence/mod.rs
+++ b/datafusion/physical-expr/src/equivalence/mod.rs
@@ -11,10 +11,12 @@ use arrow::compute::SortOptions;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
mod class;
+mod grouping;
mod ordering;
mod properties;
pub use class::{AcrossPartitions, ConstExpr, EquivalenceClass, EquivalenceGroup};
+pub use grouping::GroupingEquivalenceClass;
pub use ordering::OrderingEquivalenceClass;
// Re-export for backwards compatibility, we recommend importing from
// datafusion_physical_expr::projection instead
@@ -44,7 +46,7 @@ pub fn convert_to_orderings<T: Borrow<Arc<dyn PhysicalExpr>>>(
#[cfg(test)]
mod tests {
use super::*;
- use crate::expressions::{Column, col};
+ use crate::expressions::{Column, cast, col};
use crate::{LexRequirement, PhysicalSortExpr};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
@@ -207,4 +209,234 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn grouping_satisfy_requires_the_complete_tuple() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let c = col("c", &schema)?;
+ let mut properties = EquivalenceProperties::new(schema);
+ properties.add_grouping([Arc::clone(&a), Arc::clone(&b)]);
+
+ assert!(properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&b)])?);
+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&a)])?);
+ assert!(!properties.grouping_satisfy([Arc::clone(&a)])?);
+ assert!(!properties.grouping_satisfy([Arc::clone(&b)])?);
+ assert!(!properties.grouping_satisfy([a, b, c])?);
+ Ok(())
+ }
+
+ #[test]
+ fn ordering_implies_grouped_prefixes() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let c = col("c", &schema)?;
+ let properties = EquivalenceProperties::new_with_orderings(
+ schema,
+ [[
+ PhysicalSortExpr::new_default(Arc::clone(&a)),
+ PhysicalSortExpr::new_default(Arc::clone(&b)),
+ PhysicalSortExpr::new_default(Arc::clone(&c)),
+ ]],
+ );
+
+ assert!(properties.grouping_satisfy([Arc::clone(&a)])?);
+ assert!(properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&b)])?);
+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&a)])?);
+ assert!(!properties.grouping_satisfy([Arc::clone(&b)])?);
+ assert!(!properties.grouping_satisfy([a, c])?);
+ Ok(())
+ }
+
+ #[test]
+ fn equivalent_orderings_can_jointly_satisfy_grouping() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let properties = EquivalenceProperties::new_with_orderings(
+ schema,
+ [
+ [PhysicalSortExpr::new_default(Arc::clone(&a))],
+ [PhysicalSortExpr::new_default(Arc::clone(&b))],
+ ],
+ );
+
+ assert!(properties.grouping_satisfy([b, a])?);
+ Ok(())
+ }
+
+ #[test]
+ fn reorder_clears_explicit_grouping() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let ordering = [PhysicalSortExpr::new_default(Arc::clone(&b))];
+ let mut properties =
+ EquivalenceProperties::new_with_orderings(schema, [ordering.clone()]);
+ properties.add_grouping([Arc::clone(&a)]);
+
+ // The ordering is already satisfied, but a physical sort may still
+ // rearrange equal `b` values and separate an `a` group.
+ assert!(!properties.reorder(ordering)?);
+ assert!(properties.geq_class().is_empty());
+ assert!(!properties.grouping_satisfy([a])?);
+ assert!(properties.grouping_satisfy([b])?);
+ Ok(())
+ }
+
+ #[test]
+ fn grouping_normalizes_equivalences_and_constants() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let c = col("c", &schema)?;
+ let mut properties = EquivalenceProperties::new(schema);
+ properties.add_grouping([Arc::clone(&b), Arc::clone(&c)]);
+
+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&c)])?);
+ assert!(!properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&c)])?);
+
+ // Making `a` canonical requires the cache to change from [b, c] to [a, c].
+ properties.add_equal_conditions(Arc::clone(&a), Arc::clone(&b))?;
+ assert!(properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&c)])?);
+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&c)])?);
+ assert!(!properties.grouping_satisfy([Arc::clone(&c)])?);
+
+ properties.add_constants([ConstExpr::from(Arc::clone(&a))])?;
+ assert!(properties.grouping_satisfy([Arc::clone(&c)])?);
+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&c)])?);
+
+ // Without the constant guarantee, the complete pair is required again.
+ properties.clear_per_partition_constants();
+ assert!(!properties.grouping_satisfy([Arc::clone(&c)])?);
+ assert!(properties.grouping_satisfy([b, c])?);
+ Ok(())
+ }
+
+ #[test]
+ fn grouping_is_rewritten_with_new_schema() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("c", DataType::Int32, true),
+ ]));
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let c = col("c", &schema)?;
+ let mut properties = EquivalenceProperties::new(schema);
+ properties.add_equal_conditions(a, Arc::clone(&b))?;
+ properties.add_grouping([b, c]);
+
+ let output_schema = Arc::new(Schema::new(vec![
+ Field::new("x", DataType::Int32, true),
+ Field::new("y", DataType::Int32, true),
+ Field::new("z", DataType::Int32, true),
+ ]));
+ let rewritten = properties.with_new_schema(Arc::clone(&output_schema))?;
+ let x = col("x", &output_schema)?;
+ let y = col("y", &output_schema)?;
+ let z = col("z", &output_schema)?;
+
+ // The declaration uses y; the cache uses x because x = y.
+ assert_eq!(
+ rewritten.geq_class(),
+ &GroupingEquivalenceClass::new([[Arc::clone(&y), Arc::clone(&z)]])
+ );
+ assert_eq!(
+ rewritten.normalized_geq_class(),
+ GroupingEquivalenceClass::new([[Arc::clone(&x), Arc::clone(&z)]])
+ );
+ assert!(rewritten.grouping_satisfy([y, Arc::clone(&z)])?);
+ assert!(rewritten.grouping_satisfy([x, Arc::clone(&z)])?);
+ assert!(!rewritten.grouping_satisfy([z])?);
+ Ok(())
+ }
+
+ #[test]
+ fn project_grouping_preserves_complete_tuple() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let c = col("c", &schema)?;
+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));
+ properties.add_grouping([Arc::clone(&a), Arc::clone(&b)]);
+
+ let output_schema = Arc::new(Schema::new(vec![
+ Field::new("z", DataType::Int32, true),
+ Field::new("x", DataType::Int32, true),
+ Field::new("y", DataType::Int32, true),
+ ]));
+ let mapping = ProjectionMapping::try_new(
+ [
+ (c, "z".to_string()),
+ (Arc::clone(&b), "x".to_string()),
+ (Arc::clone(&a), "y".to_string()),
+ ],
+ &schema,
+ )?;
+ let projected = properties.project(&mapping, Arc::clone(&output_schema));
+ let x = col("x", &output_schema)?;
+ let y = col("y", &output_schema)?;
+ assert!(projected.grouping_satisfy([x, y])?);
+ Ok(())
+ }
+
+ #[test]
+ fn project_grouping_drops_incomplete_tuple() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let c = col("c", &schema)?;
+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));
+ properties.add_grouping([Arc::clone(&a), b]);
+
+ let output_schema = Arc::new(Schema::new(vec![
+ Field::new("x", DataType::Int32, true),
+ Field::new("z", DataType::Int32, true),
+ ]));
+ let mapping = ProjectionMapping::try_new(
+ [(a, "x".to_string()), (c, "z".to_string())],
+ &schema,
+ )?;
+ let projected = properties.project(&mapping, Arc::clone(&output_schema));
+ assert!(projected.geq_class().is_empty());
+ assert!(!projected.grouping_satisfy([col("x", &output_schema)?])?);
+ Ok(())
+ }
+
+ #[test]
+ fn project_grouping_omits_constant_members() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let b = col("b", &schema)?;
+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));
+ properties.add_constants([ConstExpr::from(Arc::clone(&a))])?;
+ properties.add_grouping([a, Arc::clone(&b)]);
+
+ let output_schema =
+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, true)]));
+ let mapping = ProjectionMapping::try_new([(b, "x".to_string())], &schema)?;
+ let projected = properties.project(&mapping, Arc::clone(&output_schema));
+
+ assert!(projected.grouping_satisfy([col("x", &output_schema)?])?);
+ Ok(())
+ }
+
+ #[test]
+ fn project_grouping_preserves_explicit_derived_expression() -> Result<()> {
+ let schema = create_test_schema()?;
+ let a = col("a", &schema)?;
+ let widened = cast(a, &schema, DataType::Int64)?;
+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));
+ properties.add_grouping([Arc::clone(&widened)]);
+
+ let output_schema =
+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));
+ let mapping = ProjectionMapping::try_new([(widened, "x".to_string())], &schema)?;
+ let projected = properties.project(&mapping, Arc::clone(&output_schema));
+ assert!(projected.grouping_satisfy([col("x", &output_schema)?])?);
+ Ok(())
+ }
}
diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs
index 9e47fa70c6..1206315bbb 100644
--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs
@@ -20,7 +20,8 @@ use self::dependency::{
};
use self::walk::SortKeyWalk;
use crate::equivalence::{
- AcrossPartitions, EquivalenceGroup, OrderingEquivalenceClass, ProjectionMapping,
+ AcrossPartitions, EquivalenceGroup, GroupingEquivalenceClass,
+ OrderingEquivalenceClass, ProjectionMapping,
};
use crate::expressions::{Column, Literal, with_new_schema};
use crate::{
@@ -30,7 +31,7 @@ use crate::{
use arrow::compute::SortOptions;
use arrow::datatypes::SchemaRef;
-use datafusion_common::{Constraint, Constraints, HashMap, Result, plan_err};
+use datafusion_common::{Constraint, Constraints, HashMap, HashSet, Result, plan_err};
use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
@@ -40,6 +41,7 @@ use itertools::{Either, Itertools};
/// `EquivalenceProperties` stores information about the output of a plan node
/// that can be used to optimize the plan. Currently, it keeps track of:
/// - Sort expressions (orderings),
+/// - Complete expression tuples whose values are contiguous,
/// - Equivalent expressions; i.e. expressions known to have the same value.
/// - Constants expressions; i.e. expressions known to contain a single constant
/// value.
@@ -125,10 +127,16 @@ pub struct EquivalenceProperties {
eq_group: EquivalenceGroup,
/// Equivalent sort expressions (i.e. those define the same ordering).
oeq_class: OrderingEquivalenceClass,
+ /// Complete expression tuples whose values occupy one contiguous run
+ /// in each output partition.
+ geq_class: GroupingEquivalenceClass,
/// Cache storing equivalent sort expressions in normal form (i.e. without
/// constants/duplicates and in standard form) and a map associating leading
/// terms with full sort expressions.
oeq_cache: OrderingEquivalenceCache,
+ /// Grouping expressions in normal form (i.e. without constants,
+ /// duplicates, or non-canonical equivalent expressions).
+ geq_cache: GroupingEquivalenceClass,
/// Table constraints that factor in equivalence calculations. Unlike
/// constants, constraints have no per-partition form: each one must hold
/// across all output partitions, since operators that merge partitions
@@ -245,7 +253,9 @@ impl EquivalenceProperties {
Self {
eq_group: EquivalenceGroup::default(),
oeq_class: OrderingEquivalenceClass::default(),
+ geq_class: GroupingEquivalenceClass::default(),
oeq_cache: OrderingEquivalenceCache::default(),
+ geq_cache: GroupingEquivalenceClass::default(),
constraints: Constraints::default(),
schema,
}
@@ -278,6 +288,8 @@ impl EquivalenceProperties {
Self {
oeq_cache: OrderingEquivalenceCache::new(normal_orderings),
oeq_class,
+ geq_class: GroupingEquivalenceClass::default(),
+ geq_cache: GroupingEquivalenceClass::default(),
eq_group,
constraints: Constraints::default(),
schema,
@@ -294,6 +306,11 @@ impl EquivalenceProperties {
&self.oeq_class
}
+ /// Returns a reference to the grouping equivalence class within.
+ pub fn geq_class(&self) -> &GroupingEquivalenceClass {
+ &self.geq_class
+ }
+
/// Returns a reference to the equivalence group within.
pub fn eq_group(&self) -> &EquivalenceGroup {
&self.eq_group
@@ -329,6 +346,7 @@ impl EquivalenceProperties {
self.constraints.extend(other.constraints);
self.add_equivalence_group(other.eq_group)?;
self.add_orderings(other.oeq_class);
+ self.add_groupings(other.geq_class);
Ok(self)
}
@@ -339,6 +357,13 @@ impl EquivalenceProperties {
self.oeq_cache.clear();
}
+ /// Clears grouping information invalidated by an operation that changes
+ /// row sequence or combines input partitions.
+ pub fn clear_groupings(&mut self) {
+ self.geq_class.clear();
+ self.geq_cache.clear();
+ }
+
/// Removes constant expressions that may change across partitions.
/// This method should be used when merging data from different partitions.
pub fn clear_per_partition_constants(&mut self) {
@@ -350,6 +375,7 @@ impl EquivalenceProperties {
.cloned()
.map(|o| self.eq_group.normalize_sort_exprs(o));
self.oeq_cache = OrderingEquivalenceCache::new(normal_orderings);
+ self.update_geq_cache();
}
}
@@ -380,6 +406,61 @@ impl EquivalenceProperties {
self.add_orderings(std::iter::once(ordering));
}
+ /// Adds complete grouping tuples whose values are contiguous within
+ /// every output partition.
+ ///
+ /// This is a correctness assertion. DataFusion does not verify the row
+ /// layout, and consumers may act on a tuple as soon as its values change.
+ /// Callers must therefore only add tuples that hold for every output
+ /// partition.
+ ///
+ /// Expression order within a tuple is not significant. A tuple only
+ /// describes its complete set of expressions: grouping `[a, b]` does not
+ /// imply grouping `[a]` or `[b]`.
+ pub fn add_groupings(
+ &mut self,
+ groupings: impl IntoIterator<Item = impl IntoIterator<Item = Arc<dyn PhysicalExpr>>>,
+ ) {
+ for grouping in GroupingEquivalenceClass::new(groupings) {
+ let normal_grouping = self.normalize_grouping(grouping.iter().cloned());
+ self.geq_class.add_groupings(std::iter::once(grouping));
+ if !normal_grouping.is_empty() {
+ self.geq_cache
+ .add_groupings(std::iter::once(normal_grouping));
+ }
+ }
+ }
+
+ /// Adds one complete grouping tuple.
+ pub fn add_grouping(
+ &mut self,
+ grouping: impl IntoIterator<Item = Arc<dyn PhysicalExpr>>,
+ ) {
+ self.add_groupings(std::iter::once(grouping));
+ }
+
+ fn normalize_grouping(
+ &self,
+ grouping: impl IntoIterator<Item = Arc<dyn PhysicalExpr>>,
+ ) -> Vec<Arc<dyn PhysicalExpr>> {
+ let mut seen = HashSet::new();
+ grouping
+ .into_iter()
+ .map(|expr| self.eq_group.normalize_expr(expr))
+ .filter(|expr| self.eq_group.is_expr_constant(expr).is_none())
+ .filter(|expr| seen.insert(Arc::clone(expr)))
+ .collect()
+ }
+
+ fn update_geq_cache(&mut self) {
+ let groupings = self
+ .geq_class
+ .iter()
+ .map(|grouping| self.normalize_grouping(grouping.iter().cloned()))
+ .collect::<Vec<_>>();
+ self.geq_cache = GroupingEquivalenceClass::new(groupings);
+ }
+
fn update_oeq_cache(&mut self) -> Result<()> {
// Renormalize orderings if the equivalence group changes:
let normal_cls = mem::take(&mut self.oeq_cache.normal_cls);
@@ -405,6 +486,7 @@ impl EquivalenceProperties {
if !other_eq_group.is_empty() {
self.eq_group.extend(other_eq_group);
self.update_oeq_cache()?;
+ self.update_geq_cache();
}
Ok(())
}
@@ -421,6 +503,11 @@ impl EquivalenceProperties {
.into()
}
+ /// Returns the grouping equivalence class within in normal form.
+ pub fn normalized_geq_class(&self) -> GroupingEquivalenceClass {
+ self.geq_cache.clone()
+ }
+
/// Adds a new equality condition into the existing equivalence group.
/// If the given equality defines a new equivalence class, adds this new
/// equivalence class to the equivalence group.
@@ -434,6 +521,7 @@ impl EquivalenceProperties {
self.update_oeq_cache()?;
}
self.update_oeq_cache()?;
+ self.update_geq_cache();
Ok(())
}
@@ -456,6 +544,7 @@ impl EquivalenceProperties {
});
self.oeq_cache.normal_cls = OrderingEquivalenceClass::new(normal_orderings);
self.oeq_cache.update_map();
+ self.update_geq_cache();
// Discover any new orderings based on the constants:
let leading_exprs: Vec<_> = self.oeq_cache.leading_map.keys().cloned().collect();
for expr in leading_exprs {
@@ -549,15 +638,21 @@ impl EquivalenceProperties {
Ok(None)
}
- /// Updates the ordering equivalence class within assuming that the table
- /// is re-sorted according to the argument `ordering`, and returns whether
- /// this operation resulted in any change. Note that equivalence classes
- /// (and constants) do not change as they are unaffected by a re-sort. If
- /// the given ordering is already satisfied, the function does nothing.
+ /// Updates the ordering equivalence class assuming that the table is
+ /// re-sorted according to `ordering`, and returns whether the ordering
+ /// class changed. Equivalence classes and constants are unaffected by a
+ /// re-sort. Explicit grouping assertions are cleared because a sort may
+ /// rearrange rows that compare equally, even when `ordering` was already
+ /// satisfied.
pub fn reorder(
&mut self,
ordering: impl IntoIterator<Item = PhysicalSortExpr>,
) -> Result<bool> {
+ // A sort may reorder rows that compare equally under `ordering`, so
+ // explicit grouping assertions do not necessarily survive even when
+ // the requested ordering is already satisfied. Groupings implied by
+ // the output ordering remain discoverable through `oeq_class`.
+ self.clear_groupings();
let (ordering, ordering_tee) = ordering.into_iter().tee();
// First, standardize the given ordering:
let Some(normal_ordering) = self.normalize_sort_exprs(ordering) else {
@@ -595,6 +690,27 @@ impl EquivalenceProperties {
LexRequirement::new(self.eq_group.normalize_sort_requirements(sort_reqs))
}
+ /// Returns whether the given complete expression tuple is known to be
+ /// grouped within every output partition.
+ ///
+ /// A tuple is grouped when all rows with the same values for the complete
+ /// tuple occur in one contiguous run. The order of expressions within a
+ /// grouping tuple does not matter: `[a, b]` and `[b, a]` describe the same
+ /// groups. A lexicographical ordering also satisfies grouping for each of
+ /// its prefixes, regardless of sort direction.
+ pub fn grouping_satisfy(
+ &self,
+ given: impl IntoIterator<Item = Arc<dyn PhysicalExpr>>,
+ ) -> Result<bool> {
+ let normal_grouping = self.normalize_grouping(given);
+ if normal_grouping.is_empty() || self.geq_cache.contains(&normal_grouping) {
+ return Ok(true);
+ }
+
+ let (_, indices) = self.find_longest_permutation(&normal_grouping)?;
+ Ok(indices.len() == normal_grouping.len())
+ }
+
/// Iteratively checks whether the given ordering is satisfied by any of
/// the existing orderings. See [`Self::ordering_satisfy_requirement`] for
/// more details and examples.
@@ -1095,6 +1211,21 @@ impl EquivalenceProperties {
orderings.chain(projected_orderings).collect()
}
+ /// Projects grouping tuples through `mapping`, dropping a tuple unless all
+ /// of its expressions can be represented by the projection.
+ fn projected_groupings(
+ &self,
+ mapping: &ProjectionMapping,
+ ) -> Vec<Vec<Arc<dyn PhysicalExpr>>> {
+ self.geq_cache
+ .iter()
+ .filter_map(|grouping| {
+ self.project_expressions(grouping.iter(), mapping)
+ .collect::<Option<Vec<_>>>()
+ })
+ .collect()
+ }
+
/// Projects constraints according to the given projection mapping.
///
/// This function takes a projection mapping and extracts column indices of
@@ -1138,6 +1269,8 @@ impl EquivalenceProperties {
/// preserved through `c + 1` but dropped through `abs(c)`. Orderings
/// implied by the mapping are also derived, e.g. an ordering on `a + b`
/// yields one on the projected `a_new + b_new`.
+ /// - Groupings: a complete grouping tuple is carried only when every
+ /// expression in the tuple can be represented in the output.
/// - Equivalence group: each class is re-expressed on the output columns.
/// - Constraints: projected onto the surviving column indices.
///
@@ -1189,17 +1322,22 @@ impl EquivalenceProperties {
) -> Self {
let orderings =
self.projected_orderings(mapping, self.oeq_cache.normal_cls.clone());
+ let groupings = self.projected_groupings(mapping);
let normal_orderings = orderings
.iter()
.cloned()
.map(|o| eq_group.normalize_sort_exprs(o));
- Self {
+ let mut result = Self {
oeq_cache: OrderingEquivalenceCache::new(normal_orderings),
oeq_class: OrderingEquivalenceClass::new(orderings),
+ geq_class: GroupingEquivalenceClass::default(),
+ geq_cache: GroupingEquivalenceClass::default(),
constraints: self.projected_constraints(mapping).unwrap_or_default(),
schema: output_schema,
eq_group,
- }
+ };
+ result.add_groupings(groupings);
+ result
}
/// Returns the longest (potentially partial) permutation satisfying the
@@ -1404,6 +1542,11 @@ impl EquivalenceProperties {
self.oeq_class = self.oeq_class.with_new_schema(&schema)?;
self.oeq_cache.normal_cls = self.oeq_cache.normal_cls.with_new_schema(&schema)?;
+ // Rewrite grouping expressions according to new schema and rebuild
+ // their normalized form against the rewritten equivalence group.
+ self.geq_class = self.geq_class.with_new_schema(&schema)?;
+ self.update_geq_cache();
+
// Update the schema:
self.schema = schema;
@@ -1421,21 +1564,28 @@ impl From<EquivalenceProperties> for OrderingEquivalenceClass {
///
/// Format:
/// ```text
-/// order: [[b@1 ASC NULLS LAST]], eq: [{members: [a@0], constant: (heterogeneous)}]
+/// order: [[b@1 ASC NULLS LAST]], group: [[a@0, b@1]], eq: [{members: [a@0], constant: (heterogeneous)}]
/// ```
impl Display for EquivalenceProperties {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let empty_eq_group = self.eq_group.is_empty();
let empty_oeq_class = self.oeq_class.is_empty();
- if empty_oeq_class && empty_eq_group {
- write!(f, "No properties")?;
- } else if !empty_oeq_class {
+ let empty_geq_class = self.geq_class.is_empty();
+ if empty_oeq_class && empty_geq_class && empty_eq_group {
+ return write!(f, "No properties");
+ }
+
+ let mut separator = "";
+ if !empty_oeq_class {
write!(f, "order: {}", self.oeq_class)?;
- if !empty_eq_group {
- write!(f, ", eq: {}", self.eq_group)?;
- }
- } else {
- write!(f, "eq: {}", self.eq_group)?;
+ separator = ", ";
+ }
+ if !empty_geq_class {
+ write!(f, "{separator}group: {}", self.geq_class)?;
+ separator = ", ";
+ }
+ if !empty_eq_group {
+ write!(f, "{separator}eq: {}", self.eq_group)?;
}
Ok(())
}
diff --git a/datafusion/physical-expr/src/equivalence/properties/union.rs b/datafusion/physical-expr/src/equivalence/properties/union.rs
index ed2beb377b..a3d5c1c454 100644
--- a/datafusion/physical-expr/src/equivalence/properties/union.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/union.rs
@@ -6,6 +6,7 @@ use std::iter::Peekable;
use std::sync::Arc;
use super::EquivalenceProperties;
+use crate::equivalence::GroupingEquivalenceClass;
use crate::equivalence::class::AcrossPartitions;
use crate::{ConstExpr, PhysicalSortExpr};
@@ -24,13 +25,8 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;
/// partitions.
fn calculate_union_binary(
lhs: EquivalenceProperties,
- mut rhs: EquivalenceProperties,
+ rhs: &EquivalenceProperties,
) -> Result<EquivalenceProperties> {
- // Harmonize the schema of the rhs with the schema of the lhs (which is the accumulator schema):
- if !rhs.schema.eq(&lhs.schema) {
- rhs = rhs.with_new_schema(Arc::clone(&lhs.schema))?;
- }
-
// First, calculate valid constants for the union. An expression is constant
// at the output of the union if it is constant in both sides with matching values.
let rhs_constants = rhs.constants();
@@ -61,8 +57,8 @@ fn calculate_union_binary(
// Next, calculate valid orderings for the union by searching for prefixes
// in both sides.
let mut orderings = UnionEquivalentOrderingBuilder::new();
- orderings.add_satisfied_orderings(&lhs, &rhs)?;
- orderings.add_satisfied_orderings(&rhs, &lhs)?;
+ orderings.add_satisfied_orderings(&lhs, rhs)?;
+ orderings.add_satisfied_orderings(rhs, &lhs)?;
let orderings = orderings.build();
let mut eq_properties = EquivalenceProperties::new(lhs.schema);
@@ -81,6 +77,33 @@ pub fn calculate_union(
) -> Result<EquivalenceProperties> {
// TODO: In some cases, we should be able to preserve some equivalence
// classes. Add support for such cases.
+ let eqps = eqps
+ .into_iter()
+ .map(move |props| {
+ if props.schema.eq(&schema) {
+ Ok(props)
+ } else {
+ props.with_new_schema(Arc::clone(&schema))
+ }
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ // Each output partition is one input partition, so a grouping holds when
+ // every input satisfies it. Check the original inputs: folding orderings
+ // first can lose grouping evidence when their sort directions differ.
+ let mut groupings = GroupingEquivalenceClass::default();
+ 'grouping: for grouping in eqps.iter().flat_map(|props| props.geq_class().iter()) {
+ if groupings.contains(grouping) {
+ continue;
+ }
+ for props in &eqps {
+ if !props.grouping_satisfy(grouping.iter().cloned())? {
+ continue 'grouping;
+ }
+ }
+ groupings.add_groupings(std::iter::once(grouping.iter().cloned()));
+ }
+
let mut iter = eqps.into_iter();
let Some(mut acc) = iter.next() else {
return internal_err!(
@@ -88,14 +111,11 @@ pub fn calculate_union(
);
};
- // Harmonize the schema of the init with the schema of the union:
- if !acc.schema.eq(&schema) {
- acc = acc.with_new_schema(schema)?;
- }
// Fold in the rest of the EquivalenceProperties:
for props in iter {
- acc = calculate_union_binary(acc, props)?;
+ acc = calculate_union_binary(acc, &props)?;
}
+ acc.add_groupings(groupings);
Ok(acc)
}
@@ -676,6 +696,84 @@ mod tests {
.run()
}
+ #[test]
+ fn test_union_preserves_only_common_groupings() -> Result<()> {
+ let schema = create_test_schema()?;
+ UnionEquivalenceTest::new(&schema)
+ .with_child_grouping(
+ vec![vec!["e"], vec!["a", "b"], vec!["c", "d"]],
+ &schema,
+ )?
+ .with_child_grouping(
+ vec![vec!["f"], vec!["d", "c"], vec!["b", "a"]],
+ &schema,
+ )?
+ .with_expected_sort(vec![])?
+ .with_expected_grouping(vec![vec!["a", "b"], vec!["c", "d"]])?
+ .run()
+ }
+
+ #[test]
+ fn test_union_drops_grouping_missing_from_an_input() -> Result<()> {
+ let schema = create_test_schema()?;
+ for add_ordered_input in [false, true] {
+ let mut test = UnionEquivalenceTest::new(&schema)
+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?
+ .with_child_sort(vec![], &schema)?;
+ if add_ordered_input {
+ test = test.with_child_sort(vec![vec!["a", "b"]], &schema)?;
+ }
+ test.with_expected_sort(vec![])?.run()?;
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn test_union_preserves_grouping_implied_by_ordering() -> Result<()> {
+ let schema = create_test_schema()?;
+ UnionEquivalenceTest::new(&schema)
+ .with_child_sort(vec![vec!["a", "b"]], &schema)?
+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?
+ .with_expected_sort(vec![])?
+ .with_expected_grouping(vec![vec!["a", "b"]])?
+ .run()
+ }
+
+ #[test]
+ fn test_union_preserves_grouping_with_different_schemas() -> Result<()> {
+ let schema = create_test_schema()?;
+ let schema2 = append_fields(&schema, "1");
+ UnionEquivalenceTest::new(&schema)
+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?
+ .with_child_grouping(vec![vec!["b1", "a1"]], &schema2)?
+ .with_expected_sort(vec![])?
+ .with_expected_grouping(vec![vec!["a", "b"]])?
+ .run()
+ }
+
+ #[test]
+ fn test_union_grouping_is_independent_of_input_order() -> Result<()> {
+ let schema = create_test_schema()?;
+ // The ordered inputs have no common ordering, but both are grouped.
+ UnionEquivalenceTest::new(&schema)
+ .with_child_sort(vec![vec!["a", "b"]], &schema)?
+ .with_child_sort(vec![vec!["a DESC", "b DESC"]], &schema)?
+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?
+ .with_expected_sort(vec![])?
+ .with_expected_grouping(vec![vec!["a", "b"]])?
+ .run()
+ }
+
+ #[test]
+ fn test_union_does_not_infer_grouping_prefixes() -> Result<()> {
+ let schema = create_test_schema()?;
+ UnionEquivalenceTest::new(&schema)
+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?
+ .with_child_grouping(vec![vec!["a"]], &schema)?
+ .with_expected_sort(vec![])?
+ .run()
+ }
+
// TODO tests with multiple constants
#[derive(Debug)]
@@ -698,6 +796,40 @@ mod tests {
}
}
+ fn with_child_grouping(
+ mut self,
+ groupings: Vec<Vec<&str>>,
+ schema: &SchemaRef,
+ ) -> Result<Self> {
+ let mut properties = EquivalenceProperties::new(Arc::clone(schema));
+ for grouping in groupings {
+ properties.add_grouping(
+ grouping
+ .into_iter()
+ .map(|name| col(name, schema))
+ .collect::<Result<Vec<_>>>()?,
+ );
+ }
+ self.child_properties.push(properties);
+ Ok(self)
+ }
+
+ fn with_expected_grouping(mut self, groupings: Vec<Vec<&str>>) -> Result<Self> {
+ let properties = self
+ .expected_properties
+ .as_mut()
+ .expect("expected_properties not set");
+ for grouping in groupings {
+ properties.add_grouping(
+ grouping
+ .into_iter()
+ .map(|name| col(name, &self.output_schema))
+ .collect::<Result<Vec<_>>>()?,
+ );
+ }
+ Ok(self)
+ }
+
/// Add a union input with the specified orderings
///
/// See [`Self::make_props`] for the format of the strings in `orderings`
@@ -794,6 +926,15 @@ mod tests {
rhs: &EquivalenceProperties,
err_msg: String,
) {
+ assert_eq!(lhs.geq_class(), rhs.geq_class(), "{err_msg}");
+ assert_eq!(
+ lhs.normalized_geq_class(),
+ rhs.normalized_geq_class(),
+ "{err_msg}"
+ );
+ for grouping in rhs.geq_class().iter() {
+ assert!(lhs.grouping_satisfy(grouping.iter().cloned()).unwrap());
+ }
// Check whether constants are same
let lhs_constants = lhs.constants();
let rhs_constants = rhs.constants();
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index d3c85deb22..369cf77ae5 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -1445,6 +1445,9 @@ impl AggregateExec {
.equivalence_properties()
.project(group_expr_mapping, schema);
+ // Aggregation can reorder groups, including during spill replay.
+ eq_properties.clear_groupings();
+
// Only the clustered paths preserve existing ordering on group keys.
// Project the input's actual sort expressions; clustering alone does
// not establish an output ordering. Keep this consistent with
@@ -5328,6 +5331,40 @@ mod tests {
Ok(())
}
+ #[rstest::rstest]
+ fn aggregate_clears_explicit_grouping(
+ #[values(AggregateMode::Partial, AggregateMode::Single)] mode: AggregateMode,
+ ) -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("value", DataType::Int64, true),
+ ]));
+ let input = TestMemoryExec::try_new(&[vec![]], Arc::clone(&schema), None)?
+ .try_with_grouping_information(vec![vec![col("a", &schema)?]])?;
+ let aggregate = AggregateExec::try_new(
+ mode,
+ PhysicalGroupBy::new_single(vec![
+ (col("b", &schema)?, "group_b".to_string()),
+ (col("a", &schema)?, "group_a".to_string()),
+ ]),
+ vec![Arc::new(
+ AggregateExprBuilder::new(sum_udaf(), vec![col("value", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("SUM(value)")
+ .build()?,
+ )],
+ vec![None],
+ Arc::new(input),
+ schema,
+ )?;
+
+ let properties = aggregate.properties().equivalence_properties();
+ assert!(properties.geq_class().is_empty());
+ assert!(!properties.grouping_satisfy([col("group_a", &aggregate.schema())?])?);
+ Ok(())
+ }
+
#[rstest::rstest]
#[case::full(true)]
#[case::partial(false)]
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 3cd4e8c94f..e0d8ae173e 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -84,6 +84,9 @@ impl CoalescePartitionsExec {
// Coalescing partitions loses existing orderings:
let mut eq_properties = input.equivalence_properties().clone();
eq_properties.clear_orderings();
+ if input_partitions > 1 {
+ eq_properties.clear_groupings();
+ }
eq_properties.clear_per_partition_constants();
PlanProperties::new(
eq_properties, // Equivalence Properties
@@ -468,9 +471,40 @@ mod tests {
use arrow::array::RecordBatch;
use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_physical_expr::expressions::col;
use futures::FutureExt;
+ #[test]
+ fn grouping_is_cleared_only_when_partitions_are_merged() -> Result<()> {
+ let input = test::mem_exec(2);
+ let grouping = col("i", &input.schema())?;
+ let input = Arc::new(
+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,
+ );
+ assert!(
+ input
+ .properties()
+ .equivalence_properties()
+ .grouping_satisfy([Arc::clone(&grouping)])?
+ );
+ let merge = CoalescePartitionsExec::new(input);
+ let properties = merge.properties().equivalence_properties();
+ assert!(properties.geq_class().is_empty());
+ assert!(!properties.grouping_satisfy([grouping])?);
+
+ let input = test::mem_exec(1);
+ let grouping = col("i", &input.schema())?;
+ let input = Arc::new(
+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,
+ );
+ let merge = CoalescePartitionsExec::new(input);
+ let properties = merge.properties().equivalence_properties();
+ assert_eq!(properties.geq_class().len(), 1);
+ assert!(properties.grouping_satisfy([grouping])?);
+ Ok(())
+ }
+
#[tokio::test]
async fn merge() -> Result<()> {
let task_ctx = Arc::new(TaskContext::default());
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index cc346c02a6..51ead774c5 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -91,9 +91,13 @@ impl GlobalLimitExec {
/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.
fn compute_properties(input: &Arc<dyn ExecutionPlan>) -> PlanProperties {
+ let mut eq_properties = input.equivalence_properties().clone();
+ if input.output_partitioning().partition_count() > 1 {
+ eq_properties.clear_groupings();
+ }
PlanProperties::new(
- input.equivalence_properties().clone(), // Equivalence Properties
- Partitioning::UnknownPartitioning(1), // Output Partitioning
+ eq_properties, // Equivalence Properties
+ Partitioning::UnknownPartitioning(1), // Output Partitioning
input.pipeline_behavior(),
// Limit operations are always bounded since they output a finite number of rows
Boundedness::Bounded,
@@ -771,6 +775,41 @@ mod tests {
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::expressions::col;
+ #[test]
+ fn limits_preserve_grouping_without_merging_partitions() -> Result<()> {
+ let input = test::mem_exec(2);
+ let grouping = col("i", &input.schema())?;
+ let input: Arc<dyn ExecutionPlan> = Arc::new(
+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,
+ );
+ assert!(
+ input
+ .equivalence_properties()
+ .grouping_satisfy([Arc::clone(&grouping)])?
+ );
+
+ let local = LocalLimitExec::new(Arc::clone(&input), 10);
+ let properties = local.properties().equivalence_properties();
+ assert_eq!(properties.geq_class().len(), 1);
+ assert!(properties.grouping_satisfy([Arc::clone(&grouping)])?);
+
+ let global = GlobalLimitExec::new(input, 0, Some(10));
+ let properties = global.properties().equivalence_properties();
+ assert!(properties.geq_class().is_empty());
+ assert!(!properties.grouping_satisfy([grouping])?);
+
+ let input = test::mem_exec(1);
+ let grouping = col("i", &input.schema())?;
+ let input = Arc::new(
+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,
+ );
+ let global = GlobalLimitExec::new(input, 0, Some(10));
+ let properties = global.properties().equivalence_properties();
+ assert_eq!(properties.geq_class().len(), 1);
+ assert!(properties.grouping_satisfy([grouping])?);
+ Ok(())
+ }
+
#[tokio::test]
async fn limit() -> Result<()> {
let task_ctx = Arc::new(TaskContext::default());
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index 929f52fd15..13274d26e1 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -2209,6 +2209,7 @@ impl RepartitionExec {
if preserve_order && let Some(ordering) = input.output_ordering() {
eq_properties.add_ordering(ordering.clone());
}
+ eq_properties.clear_groupings();
eq_properties.clear_per_partition_constants();
}
eq_properties
@@ -2677,6 +2678,38 @@ mod tests {
};
use insta::assert_snapshot;
+ #[test]
+ fn grouping_is_cleared_only_when_input_partitions_are_merged() -> Result<()> {
+ let input = crate::test::mem_exec(2);
+ let grouping = col("i", &input.schema())?;
+ let input: Arc<dyn ExecutionPlan> = Arc::new(
+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,
+ );
+ assert!(
+ input
+ .equivalence_properties()
+ .grouping_satisfy([Arc::clone(&grouping)])?
+ );
+ let repartition =
+ RepartitionExec::try_new(input, Partitioning::RoundRobinBatch(2))?;
+ let properties = repartition.properties().equivalence_properties();
+ assert!(properties.geq_class().is_empty());
+ assert!(!properties.grouping_satisfy([grouping])?);
+
+ // Splitting one stream preserves contiguity in each output subsequence.
+ let input = crate::test::mem_exec(1);
+ let grouping = col("i", &input.schema())?;
+ let input: Arc<dyn ExecutionPlan> = Arc::new(
+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,
+ );
+ let repartition =
+ RepartitionExec::try_new(input, Partitioning::RoundRobinBatch(2))?;
+ let properties = repartition.properties().equivalence_properties();
+ assert_eq!(properties.geq_class().len(), 1);
+ assert!(properties.grouping_satisfy([grouping])?);
+ Ok(())
+ }
+
#[derive(Debug)]
struct UnboundedTestPartition {
schema: SchemaRef,
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 4cfe4e5098..641e932c28 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -164,6 +164,7 @@ impl SortPreservingMergeExec {
// Other input orderings hold only within each partition:
if input_partitions > 1 {
eq_properties.clear_orderings();
+ eq_properties.clear_groupings();
}
eq_properties.clear_per_partition_constants();
eq_properties.add_ordering(ordering);
@@ -884,6 +885,42 @@ mod tests {
use insta::assert_snapshot;
use tokio::time::timeout;
+ #[test]
+ fn merging_sorted_partitions_clears_explicit_grouping() -> Result<()> {
+ // Each partition is grouped on `a`, but merging on `b` splits the `a` runs.
+ let first = test::build_table_i32_two_cols(
+ ("a", &vec![1, 1, 2, 2]),
+ ("b", &vec![1, 3, 5, 7]),
+ );
+ let second = test::build_table_i32_two_cols(
+ ("a", &vec![2, 2, 1, 1]),
+ ("b", &vec![2, 4, 6, 8]),
+ );
+ let schema = first.schema();
+ let grouping = col("a", &schema)?;
+ let sort_key = col("b", &schema)?;
+ let ordering: LexOrdering =
+ [PhysicalSortExpr::new_default(Arc::clone(&sort_key))].into();
+ let input = TestMemoryExec::try_new(&[vec![first], vec![second]], schema, None)?
+ .try_with_sort_information(vec![ordering.clone()])?
+ .try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?;
+ assert!(
+ input
+ .properties()
+ .equivalence_properties()
+ .grouping_satisfy([Arc::clone(&grouping)])?
+ );
+ let merge = SortPreservingMergeExec::new(ordering.clone(), Arc::new(input));
+ let properties = merge.properties().equivalence_properties();
+
+ assert!(properties.geq_class().is_empty());
+ assert!(!properties.grouping_satisfy([grouping])?);
+ // The merge key remains ordered, and therefore grouped.
+ assert!(properties.ordering_satisfy(ordering)?);
+ assert!(properties.grouping_satisfy([sort_key])?);
+ Ok(())
+ }
+
// The number in the function is highly related to the memory limit we are testing
// any change of the constant should be aware of
fn generate_task_ctx_for_round_robin_tie_breaker(
diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rs
index 25125fba83..47c982744d 100644
--- a/datafusion/physical-plan/src/test.rs
+++ b/datafusion/physical-plan/src/test.rs
@@ -60,6 +60,8 @@ pub struct TestMemoryExec {
projection: Option<Vec<usize>>,
/// Sort information: one or more equivalent orderings
sort_information: Vec<LexOrdering>,
+ /// Complete expression tuples whose values are contiguous.
+ grouping_information: Vec<Vec<Arc<dyn PhysicalExpr>>>,
/// if partition sizes should be displayed
show_sizes: bool,
/// The maximum number of records to read from this plan. If `None`,
@@ -220,10 +222,12 @@ impl TestMemoryExec {
}
fn eq_properties(&self) -> EquivalenceProperties {
- EquivalenceProperties::new_with_orderings(
+ let mut properties = EquivalenceProperties::new_with_orderings(
Arc::clone(&self.projected_schema),
self.sort_information.clone(),
- )
+ );
+ properties.add_groupings(self.grouping_information.clone());
+ properties
}
fn statistics_inner(&self) -> Result<Statistics> {
@@ -255,6 +259,7 @@ impl TestMemoryExec {
projected_schema,
projection,
sort_information: vec![],
+ grouping_information: vec![],
show_sizes: true,
fetch: None,
})
@@ -350,6 +355,52 @@ impl TestMemoryExec {
Ok(self)
}
+ /// Adds grouping information to this source.
+ pub fn try_with_grouping_information(
+ mut self,
+ mut grouping_information: Vec<Vec<Arc<dyn PhysicalExpr>>>,
+ ) -> Result<Self> {
+ // All grouping expressions must refer to the original schema.
+ let fields = self.schema.fields();
+ let ambiguous_column = grouping_information
+ .iter()
+ .flatten()
+ .flat_map(collect_columns)
+ .find(|col| {
+ fields
+ .get(col.index())
+ .map(|field| field.name() != col.name())
+ .unwrap_or(true)
+ });
+ assert_or_internal_err!(
+ ambiguous_column.is_none(),
+ "Column {:?} is not found in the original schema of the TestMemoryExec",
+ ambiguous_column.as_ref().unwrap()
+ );
+
+ if let Some(projection) = &self.projection {
+ let base_schema = self.original_schema();
+ let proj_exprs = projection.iter().map(|idx| {
+ let name = base_schema.field(*idx).name();
+ (Arc::new(Column::new(name, *idx)) as _, name.to_string())
+ });
+ let projection_mapping =
+ ProjectionMapping::try_new(proj_exprs, &base_schema)?;
+ let mut base_eqp = EquivalenceProperties::new(base_schema);
+ base_eqp.add_groupings(grouping_information);
+ grouping_information = base_eqp
+ .project(&projection_mapping, Arc::clone(&self.projected_schema))
+ .geq_class()
+ .iter()
+ .cloned()
+ .collect();
+ }
+
+ self.grouping_information = grouping_information;
+ self.cache = Arc::new(self.compute_properties());
+ Ok(self)
+ }
+
/// Arc clone of ref to original schema
pub fn original_schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index 3b4cebc67c..042f3217e2 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -1216,6 +1216,74 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn test_union_preserves_grouped_partition_streams() -> Result<()> {
+ let first =
+ test::build_table_i32_two_cols(("a", &vec![2, 2, 1]), ("b", &vec![0, 0, 1]));
+ let second =
+ test::build_table_i32_two_cols(("a", &vec![1, 2, 1]), ("b", &vec![1, 1, 0]));
+ let schema = first.schema();
+ let grouping = vec![col("a", &schema)?, col("b", &schema)?];
+ // Only the complete tuple is contiguous, and one run spans batches.
+ let left_partitions = vec![vec![first, second], vec![]];
+ let right_partitions = vec![vec![
+ test::build_table_i32_two_cols(("a", &vec![1, 2, 1]), ("b", &vec![0, 1, 1])),
+ test::build_table_i32_two_cols(("a", &vec![1, 2, 2]), ("b", &vec![1, 0, 0])),
+ ]];
+ let left = TestMemoryExec::try_new(&left_partitions, Arc::clone(&schema), None)?
+ .try_with_grouping_information(vec![grouping.clone()])?;
+ let right = TestMemoryExec::try_new(&right_partitions, schema, None)?
+ .try_with_grouping_information(vec![grouping.clone()])?;
+ let union = UnionExec::try_new(vec![Arc::new(left), Arc::new(right)])?;
+ let properties = union.equivalence_properties();
+ assert!(properties.grouping_satisfy(grouping)?);
+ assert!(!properties.grouping_satisfy([col("a", &union.schema())?])?);
+ assert!(!properties.grouping_satisfy([col("b", &union.schema())?])?);
+ assert!(union.output_ordering().is_none());
+ assert!(matches!(
+ union.output_partitioning(),
+ Partitioning::UnknownPartitioning(3)
+ ));
+
+ let context = Arc::new(TaskContext::default());
+ for (partition, expected) in
+ left_partitions.iter().chain(&right_partitions).enumerate()
+ {
+ let actual =
+ crate::common::collect(union.execute(partition, Arc::clone(&context))?)
+ .await?;
+ assert_eq!(&actual, expected);
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn test_interleave_drops_grouping() -> Result<()> {
+ let schema = create_test_schema()?;
+ let grouping = vec![col("a", &schema)?, col("b", &schema)?];
+ let source = TestMemoryExec::try_new(&[vec![]], schema, None)?
+ .try_with_grouping_information(vec![grouping.clone()])?;
+ let input: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(
+ Arc::new(source),
+ Partitioning::Hash(grouping.clone(), 1),
+ )?);
+ assert!(
+ input
+ .equivalence_properties()
+ .grouping_satisfy(grouping.clone())?
+ );
+
+ let interleave: Arc<dyn ExecutionPlan> =
+ Arc::new(InterleaveExec::try_new(vec![Arc::clone(&input), input])?);
+ assert!(interleave.equivalence_properties().geq_class().is_empty());
+ assert!(
+ !interleave
+ .equivalence_properties()
+ .grouping_satisfy(grouping)?
+ );
+ Ok(())
+ }
+
fn stats_merge_inputs() -> (SchemaRef, Statistics, Statistics, Statistics) {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)]));