Apache DataFusion SQL Query Engine
feat: add grouping equivalence properties (#24698)
xavlee320a7d28bc02
diff --git a/datafusion/physical-expr/src/equivalence/grouping.rs b/datafusion/physical-expr/src/equivalence/grouping.rsnew file mode 100644index 0000000000..2fce6dad82--- /dev/null+++ b/datafusion/physical-expr/src/equivalence/grouping.rs@@ -0,0 +1,192 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++use std::fmt::Display;+use std::ops::Deref;+use std::sync::Arc;+use std::vec::IntoIter;++use crate::PhysicalExpr;+use crate::expressions::with_new_schema;++use arrow::datatypes::SchemaRef;+use datafusion_common::{HashSet, Result};+use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;++/// A set of grouping tuples that are known to describe contiguous rows.+///+/// Each entry is a complete grouping tuple. For example, `[a, b]` means that,+/// within each output partition, all rows having the same values for both `a`+/// and `b` occur in one contiguous run. The runs themselves may occur in any+/// order. Contiguity applies to the complete partition stream, not separately+/// to each record batch.+///+/// Expression order within an entry is not significant: `[a, b]` and `[b, a]`+/// describe the same grouping. Entries do not imply properties for subsets;+/// `[a, b]` alone says nothing about whether all rows with the same `a` are+/// contiguous.+#[derive(Clone, Debug, Default)]+pub struct GroupingEquivalenceClass {+ groupings: Vec<Vec<Arc<dyn PhysicalExpr>>>,+}++impl GroupingEquivalenceClass {+ /// Clears all groupings in this equivalence class.+ pub fn clear(&mut self) {+ self.groupings.clear();+ }++ /// Creates a grouping equivalence class, discarding empty and duplicate+ /// entries and duplicate expressions within each entry.+ pub fn new(+ groupings: impl IntoIterator<Item = impl IntoIterator<Item = Arc<dyn PhysicalExpr>>>,+ ) -> Self {+ let mut result = Self::default();+ result.add_groupings(groupings);+ result+ }++ /// Adds grouping tuples to this equivalence class.+ pub fn add_groupings(+ &mut self,+ groupings: impl IntoIterator<Item = impl IntoIterator<Item = Arc<dyn PhysicalExpr>>>,+ ) {+ for grouping in groupings {+ let mut seen = HashSet::new();+ let grouping = grouping+ .into_iter()+ .filter(|expr| seen.insert(Arc::clone(expr)))+ .collect::<Vec<_>>();+ if !grouping.is_empty() && !self.contains(&grouping) {+ self.groupings.push(grouping);+ }+ }+ }++ /// Returns whether this class contains the complete grouping tuple.+ pub fn contains(&self, grouping: &[Arc<dyn PhysicalExpr>]) -> bool {+ self.groupings+ .iter()+ .any(|candidate| same_grouping(candidate, grouping))+ }++ /// Rewrites all expressions to reference an aligned schema.+ pub fn with_new_schema(self, schema: &SchemaRef) -> Result<Self> {+ let groupings = self.groupings.into_iter().map(|grouping| {+ grouping+ .into_iter()+ .map(|expr| with_new_schema(expr, schema))+ .collect::<Result<Vec<_>>>()+ });+ Ok(Self::new(groupings.collect::<Result<Vec<_>>>()?))+ }+}++fn same_grouping(lhs: &[Arc<dyn PhysicalExpr>], rhs: &[Arc<dyn PhysicalExpr>]) -> bool {+ lhs.iter()+ .all(|lhs_expr| rhs.iter().any(|rhs_expr| lhs_expr.eq(rhs_expr)))+ && rhs+ .iter()+ .all(|rhs_expr| lhs.iter().any(|lhs_expr| rhs_expr.eq(lhs_expr)))+}++impl PartialEq for GroupingEquivalenceClass {+ fn eq(&self, other: &Self) -> bool {+ self.groupings.len() == other.groupings.len()+ && self+ .groupings+ .iter()+ .all(|grouping| other.contains(grouping))+ }+}++impl Eq for GroupingEquivalenceClass {}++impl Deref for GroupingEquivalenceClass {+ type Target = [Vec<Arc<dyn PhysicalExpr>>];++ fn deref(&self) -> &Self::Target {+ self.groupings.as_slice()+ }+}++impl From<Vec<Vec<Arc<dyn PhysicalExpr>>>> for GroupingEquivalenceClass {+ fn from(groupings: Vec<Vec<Arc<dyn PhysicalExpr>>>) -> Self {+ Self::new(groupings)+ }+}++/// Converts the grouping equivalence class into an iterator of complete+/// grouping tuples.+impl IntoIterator for GroupingEquivalenceClass {+ type Item = Vec<Arc<dyn PhysicalExpr>>;+ type IntoIter = IntoIter<Self::Item>;++ fn into_iter(self) -> Self::IntoIter {+ self.groupings.into_iter()+ }+}++impl From<GroupingEquivalenceClass> for Vec<Vec<Arc<dyn PhysicalExpr>>> {+ fn from(geq_class: GroupingEquivalenceClass) -> Self {+ geq_class.groupings+ }+}++impl Display for GroupingEquivalenceClass {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "[")?;+ let mut groupings = self.groupings.iter();+ if let Some(grouping) = groupings.next() {+ write!(f, "{}", format_physical_expr_list(grouping))?;+ }+ for grouping in groupings {+ write!(f, ", {}", format_physical_expr_list(grouping))?;+ }+ write!(f, "]")+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::expressions::Column;++ #[test]+ fn grouping_equality_ignores_expression_and_entry_order() {+ let a = Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>;+ let b = Arc::new(Column::new("b", 1)) as Arc<dyn PhysicalExpr>;+ let c = Arc::new(Column::new("c", 2)) as Arc<dyn PhysicalExpr>;++ let lhs = GroupingEquivalenceClass::new([+ vec![Arc::clone(&a), Arc::clone(&b)],+ vec![Arc::clone(&c)],+ ]);+ let rhs = GroupingEquivalenceClass::new([+ vec![Arc::clone(&c)],+ vec![Arc::clone(&b), Arc::clone(&a)],+ ]);++ assert_eq!(lhs, rhs);+ // The same expressions in different tuples describe different guarantees.+ assert_ne!(lhs, GroupingEquivalenceClass::new([vec![a, c], vec![b]]));+ }++ #[test]+ fn grouping_deduplicates_entries_and_expressions() {+ let a = Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>;+ let b = Arc::new(Column::new("b", 1)) as Arc<dyn PhysicalExpr>;++ let groupings = GroupingEquivalenceClass::new([+ vec![Arc::clone(&a), Arc::clone(&a), Arc::clone(&b)],+ vec![Arc::clone(&b), Arc::clone(&a)],+ vec![],+ ]);++ assert_eq!(groupings.len(), 1);+ assert_eq!(groupings[0].len(), 2);+ assert!(groupings.contains(&[Arc::clone(&a), Arc::clone(&a), Arc::clone(&b),]));+ assert_eq!(groupings.to_string(), "[[a@0, b@1]]");+ }+}diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rsindex 1b878ee400..ff57146efc 100644--- a/datafusion/physical-expr/src/equivalence/mod.rs+++ b/datafusion/physical-expr/src/equivalence/mod.rs@@ -11,10 +11,12 @@ use arrow::compute::SortOptions;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};mod class;+mod grouping;mod ordering;mod properties;pub use class::{AcrossPartitions, ConstExpr, EquivalenceClass, EquivalenceGroup};+pub use grouping::GroupingEquivalenceClass;pub use ordering::OrderingEquivalenceClass;// Re-export for backwards compatibility, we recommend importing from// datafusion_physical_expr::projection instead@@ -44,7 +46,7 @@ pub fn convert_to_orderings<T: Borrow<Arc<dyn PhysicalExpr>>>(#[cfg(test)]mod tests {use super::*;- use crate::expressions::{Column, col};+ use crate::expressions::{Column, cast, col};use crate::{LexRequirement, PhysicalSortExpr};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};@@ -207,4 +209,234 @@ mod tests {Ok(())}++ #[test]+ fn grouping_satisfy_requires_the_complete_tuple() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let c = col("c", &schema)?;+ let mut properties = EquivalenceProperties::new(schema);+ properties.add_grouping([Arc::clone(&a), Arc::clone(&b)]);++ assert!(properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&b)])?);+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&a)])?);+ assert!(!properties.grouping_satisfy([Arc::clone(&a)])?);+ assert!(!properties.grouping_satisfy([Arc::clone(&b)])?);+ assert!(!properties.grouping_satisfy([a, b, c])?);+ Ok(())+ }++ #[test]+ fn ordering_implies_grouped_prefixes() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let c = col("c", &schema)?;+ let properties = EquivalenceProperties::new_with_orderings(+ schema,+ [[+ PhysicalSortExpr::new_default(Arc::clone(&a)),+ PhysicalSortExpr::new_default(Arc::clone(&b)),+ PhysicalSortExpr::new_default(Arc::clone(&c)),+ ]],+ );++ assert!(properties.grouping_satisfy([Arc::clone(&a)])?);+ assert!(properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&b)])?);+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&a)])?);+ assert!(!properties.grouping_satisfy([Arc::clone(&b)])?);+ assert!(!properties.grouping_satisfy([a, c])?);+ Ok(())+ }++ #[test]+ fn equivalent_orderings_can_jointly_satisfy_grouping() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let properties = EquivalenceProperties::new_with_orderings(+ schema,+ [+ [PhysicalSortExpr::new_default(Arc::clone(&a))],+ [PhysicalSortExpr::new_default(Arc::clone(&b))],+ ],+ );++ assert!(properties.grouping_satisfy([b, a])?);+ Ok(())+ }++ #[test]+ fn reorder_clears_explicit_grouping() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let ordering = [PhysicalSortExpr::new_default(Arc::clone(&b))];+ let mut properties =+ EquivalenceProperties::new_with_orderings(schema, [ordering.clone()]);+ properties.add_grouping([Arc::clone(&a)]);++ // The ordering is already satisfied, but a physical sort may still+ // rearrange equal `b` values and separate an `a` group.+ assert!(!properties.reorder(ordering)?);+ assert!(properties.geq_class().is_empty());+ assert!(!properties.grouping_satisfy([a])?);+ assert!(properties.grouping_satisfy([b])?);+ Ok(())+ }++ #[test]+ fn grouping_normalizes_equivalences_and_constants() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let c = col("c", &schema)?;+ let mut properties = EquivalenceProperties::new(schema);+ properties.add_grouping([Arc::clone(&b), Arc::clone(&c)]);++ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&c)])?);+ assert!(!properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&c)])?);++ // Making `a` canonical requires the cache to change from [b, c] to [a, c].+ properties.add_equal_conditions(Arc::clone(&a), Arc::clone(&b))?;+ assert!(properties.grouping_satisfy([Arc::clone(&a), Arc::clone(&c)])?);+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&c)])?);+ assert!(!properties.grouping_satisfy([Arc::clone(&c)])?);++ properties.add_constants([ConstExpr::from(Arc::clone(&a))])?;+ assert!(properties.grouping_satisfy([Arc::clone(&c)])?);+ assert!(properties.grouping_satisfy([Arc::clone(&b), Arc::clone(&c)])?);++ // Without the constant guarantee, the complete pair is required again.+ properties.clear_per_partition_constants();+ assert!(!properties.grouping_satisfy([Arc::clone(&c)])?);+ assert!(properties.grouping_satisfy([b, c])?);+ Ok(())+ }++ #[test]+ fn grouping_is_rewritten_with_new_schema() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ Field::new("c", DataType::Int32, true),+ ]));+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let c = col("c", &schema)?;+ let mut properties = EquivalenceProperties::new(schema);+ properties.add_equal_conditions(a, Arc::clone(&b))?;+ properties.add_grouping([b, c]);++ let output_schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Int32, true),+ Field::new("z", DataType::Int32, true),+ ]));+ let rewritten = properties.with_new_schema(Arc::clone(&output_schema))?;+ let x = col("x", &output_schema)?;+ let y = col("y", &output_schema)?;+ let z = col("z", &output_schema)?;++ // The declaration uses y; the cache uses x because x = y.+ assert_eq!(+ rewritten.geq_class(),+ &GroupingEquivalenceClass::new([[Arc::clone(&y), Arc::clone(&z)]])+ );+ assert_eq!(+ rewritten.normalized_geq_class(),+ GroupingEquivalenceClass::new([[Arc::clone(&x), Arc::clone(&z)]])+ );+ assert!(rewritten.grouping_satisfy([y, Arc::clone(&z)])?);+ assert!(rewritten.grouping_satisfy([x, Arc::clone(&z)])?);+ assert!(!rewritten.grouping_satisfy([z])?);+ Ok(())+ }++ #[test]+ fn project_grouping_preserves_complete_tuple() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let c = col("c", &schema)?;+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));+ properties.add_grouping([Arc::clone(&a), Arc::clone(&b)]);++ let output_schema = Arc::new(Schema::new(vec![+ Field::new("z", DataType::Int32, true),+ Field::new("x", DataType::Int32, true),+ Field::new("y", DataType::Int32, true),+ ]));+ let mapping = ProjectionMapping::try_new(+ [+ (c, "z".to_string()),+ (Arc::clone(&b), "x".to_string()),+ (Arc::clone(&a), "y".to_string()),+ ],+ &schema,+ )?;+ let projected = properties.project(&mapping, Arc::clone(&output_schema));+ let x = col("x", &output_schema)?;+ let y = col("y", &output_schema)?;+ assert!(projected.grouping_satisfy([x, y])?);+ Ok(())+ }++ #[test]+ fn project_grouping_drops_incomplete_tuple() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let c = col("c", &schema)?;+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));+ properties.add_grouping([Arc::clone(&a), b]);++ let output_schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Int32, true),+ Field::new("z", DataType::Int32, true),+ ]));+ let mapping = ProjectionMapping::try_new(+ [(a, "x".to_string()), (c, "z".to_string())],+ &schema,+ )?;+ let projected = properties.project(&mapping, Arc::clone(&output_schema));+ assert!(projected.geq_class().is_empty());+ assert!(!projected.grouping_satisfy([col("x", &output_schema)?])?);+ Ok(())+ }++ #[test]+ fn project_grouping_omits_constant_members() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let b = col("b", &schema)?;+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));+ properties.add_constants([ConstExpr::from(Arc::clone(&a))])?;+ properties.add_grouping([a, Arc::clone(&b)]);++ let output_schema =+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, true)]));+ let mapping = ProjectionMapping::try_new([(b, "x".to_string())], &schema)?;+ let projected = properties.project(&mapping, Arc::clone(&output_schema));++ assert!(projected.grouping_satisfy([col("x", &output_schema)?])?);+ Ok(())+ }++ #[test]+ fn project_grouping_preserves_explicit_derived_expression() -> Result<()> {+ let schema = create_test_schema()?;+ let a = col("a", &schema)?;+ let widened = cast(a, &schema, DataType::Int64)?;+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));+ properties.add_grouping([Arc::clone(&widened)]);++ let output_schema =+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]));+ let mapping = ProjectionMapping::try_new([(widened, "x".to_string())], &schema)?;+ let projected = properties.project(&mapping, Arc::clone(&output_schema));+ assert!(projected.grouping_satisfy([col("x", &output_schema)?])?);+ Ok(())+ }}diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rsindex 9e47fa70c6..1206315bbb 100644--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs@@ -20,7 +20,8 @@ use self::dependency::{};use self::walk::SortKeyWalk;use crate::equivalence::{- AcrossPartitions, EquivalenceGroup, OrderingEquivalenceClass, ProjectionMapping,+ AcrossPartitions, EquivalenceGroup, GroupingEquivalenceClass,+ OrderingEquivalenceClass, ProjectionMapping,};use crate::expressions::{Column, Literal, with_new_schema};use crate::{@@ -30,7 +31,7 @@ use crate::{use arrow::compute::SortOptions;use arrow::datatypes::SchemaRef;-use datafusion_common::{Constraint, Constraints, HashMap, Result, plan_err};+use datafusion_common::{Constraint, Constraints, HashMap, HashSet, Result, plan_err};use datafusion_expr::interval_arithmetic::Interval;use datafusion_expr::sort_properties::{ExprProperties, SortProperties};@@ -40,6 +41,7 @@ use itertools::{Either, Itertools};/// `EquivalenceProperties` stores information about the output of a plan node/// that can be used to optimize the plan. Currently, it keeps track of:/// - Sort expressions (orderings),+/// - Complete expression tuples whose values are contiguous,/// - Equivalent expressions; i.e. expressions known to have the same value./// - Constants expressions; i.e. expressions known to contain a single constant/// value.@@ -125,10 +127,16 @@ pub struct EquivalenceProperties {eq_group: EquivalenceGroup,/// Equivalent sort expressions (i.e. those define the same ordering).oeq_class: OrderingEquivalenceClass,+ /// Complete expression tuples whose values occupy one contiguous run+ /// in each output partition.+ geq_class: GroupingEquivalenceClass,/// Cache storing equivalent sort expressions in normal form (i.e. without/// constants/duplicates and in standard form) and a map associating leading/// terms with full sort expressions.oeq_cache: OrderingEquivalenceCache,+ /// Grouping expressions in normal form (i.e. without constants,+ /// duplicates, or non-canonical equivalent expressions).+ geq_cache: GroupingEquivalenceClass,/// Table constraints that factor in equivalence calculations. Unlike/// constants, constraints have no per-partition form: each one must hold/// across all output partitions, since operators that merge partitions@@ -245,7 +253,9 @@ impl EquivalenceProperties {Self {eq_group: EquivalenceGroup::default(),oeq_class: OrderingEquivalenceClass::default(),+ geq_class: GroupingEquivalenceClass::default(),oeq_cache: OrderingEquivalenceCache::default(),+ geq_cache: GroupingEquivalenceClass::default(),constraints: Constraints::default(),schema,}@@ -278,6 +288,8 @@ impl EquivalenceProperties {Self {oeq_cache: OrderingEquivalenceCache::new(normal_orderings),oeq_class,+ geq_class: GroupingEquivalenceClass::default(),+ geq_cache: GroupingEquivalenceClass::default(),eq_group,constraints: Constraints::default(),schema,@@ -294,6 +306,11 @@ impl EquivalenceProperties {&self.oeq_class}+ /// Returns a reference to the grouping equivalence class within.+ pub fn geq_class(&self) -> &GroupingEquivalenceClass {+ &self.geq_class+ }+/// Returns a reference to the equivalence group within.pub fn eq_group(&self) -> &EquivalenceGroup {&self.eq_group@@ -329,6 +346,7 @@ impl EquivalenceProperties {self.constraints.extend(other.constraints);self.add_equivalence_group(other.eq_group)?;self.add_orderings(other.oeq_class);+ self.add_groupings(other.geq_class);Ok(self)}@@ -339,6 +357,13 @@ impl EquivalenceProperties {self.oeq_cache.clear();}+ /// Clears grouping information invalidated by an operation that changes+ /// row sequence or combines input partitions.+ pub fn clear_groupings(&mut self) {+ self.geq_class.clear();+ self.geq_cache.clear();+ }+/// Removes constant expressions that may change across partitions./// This method should be used when merging data from different partitions.pub fn clear_per_partition_constants(&mut self) {@@ -350,6 +375,7 @@ impl EquivalenceProperties {.cloned().map(|o| self.eq_group.normalize_sort_exprs(o));self.oeq_cache = OrderingEquivalenceCache::new(normal_orderings);+ self.update_geq_cache();}}@@ -380,6 +406,61 @@ impl EquivalenceProperties {self.add_orderings(std::iter::once(ordering));}+ /// Adds complete grouping tuples whose values are contiguous within+ /// every output partition.+ ///+ /// This is a correctness assertion. DataFusion does not verify the row+ /// layout, and consumers may act on a tuple as soon as its values change.+ /// Callers must therefore only add tuples that hold for every output+ /// partition.+ ///+ /// Expression order within a tuple is not significant. A tuple only+ /// describes its complete set of expressions: grouping `[a, b]` does not+ /// imply grouping `[a]` or `[b]`.+ pub fn add_groupings(+ &mut self,+ groupings: impl IntoIterator<Item = impl IntoIterator<Item = Arc<dyn PhysicalExpr>>>,+ ) {+ for grouping in GroupingEquivalenceClass::new(groupings) {+ let normal_grouping = self.normalize_grouping(grouping.iter().cloned());+ self.geq_class.add_groupings(std::iter::once(grouping));+ if !normal_grouping.is_empty() {+ self.geq_cache+ .add_groupings(std::iter::once(normal_grouping));+ }+ }+ }++ /// Adds one complete grouping tuple.+ pub fn add_grouping(+ &mut self,+ grouping: impl IntoIterator<Item = Arc<dyn PhysicalExpr>>,+ ) {+ self.add_groupings(std::iter::once(grouping));+ }++ fn normalize_grouping(+ &self,+ grouping: impl IntoIterator<Item = Arc<dyn PhysicalExpr>>,+ ) -> Vec<Arc<dyn PhysicalExpr>> {+ let mut seen = HashSet::new();+ grouping+ .into_iter()+ .map(|expr| self.eq_group.normalize_expr(expr))+ .filter(|expr| self.eq_group.is_expr_constant(expr).is_none())+ .filter(|expr| seen.insert(Arc::clone(expr)))+ .collect()+ }++ fn update_geq_cache(&mut self) {+ let groupings = self+ .geq_class+ .iter()+ .map(|grouping| self.normalize_grouping(grouping.iter().cloned()))+ .collect::<Vec<_>>();+ self.geq_cache = GroupingEquivalenceClass::new(groupings);+ }+fn update_oeq_cache(&mut self) -> Result<()> {// Renormalize orderings if the equivalence group changes:let normal_cls = mem::take(&mut self.oeq_cache.normal_cls);@@ -405,6 +486,7 @@ impl EquivalenceProperties {if !other_eq_group.is_empty() {self.eq_group.extend(other_eq_group);self.update_oeq_cache()?;+ self.update_geq_cache();}Ok(())}@@ -421,6 +503,11 @@ impl EquivalenceProperties {.into()}+ /// Returns the grouping equivalence class within in normal form.+ pub fn normalized_geq_class(&self) -> GroupingEquivalenceClass {+ self.geq_cache.clone()+ }+/// Adds a new equality condition into the existing equivalence group./// If the given equality defines a new equivalence class, adds this new/// equivalence class to the equivalence group.@@ -434,6 +521,7 @@ impl EquivalenceProperties {self.update_oeq_cache()?;}self.update_oeq_cache()?;+ self.update_geq_cache();Ok(())}@@ -456,6 +544,7 @@ impl EquivalenceProperties {});self.oeq_cache.normal_cls = OrderingEquivalenceClass::new(normal_orderings);self.oeq_cache.update_map();+ self.update_geq_cache();// Discover any new orderings based on the constants:let leading_exprs: Vec<_> = self.oeq_cache.leading_map.keys().cloned().collect();for expr in leading_exprs {@@ -549,15 +638,21 @@ impl EquivalenceProperties {Ok(None)}- /// Updates the ordering equivalence class within assuming that the table- /// is re-sorted according to the argument `ordering`, and returns whether- /// this operation resulted in any change. Note that equivalence classes- /// (and constants) do not change as they are unaffected by a re-sort. If- /// the given ordering is already satisfied, the function does nothing.+ /// Updates the ordering equivalence class assuming that the table is+ /// re-sorted according to `ordering`, and returns whether the ordering+ /// class changed. Equivalence classes and constants are unaffected by a+ /// re-sort. Explicit grouping assertions are cleared because a sort may+ /// rearrange rows that compare equally, even when `ordering` was already+ /// satisfied.pub fn reorder(&mut self,ordering: impl IntoIterator<Item = PhysicalSortExpr>,) -> Result<bool> {+ // A sort may reorder rows that compare equally under `ordering`, so+ // explicit grouping assertions do not necessarily survive even when+ // the requested ordering is already satisfied. Groupings implied by+ // the output ordering remain discoverable through `oeq_class`.+ self.clear_groupings();let (ordering, ordering_tee) = ordering.into_iter().tee();// First, standardize the given ordering:let Some(normal_ordering) = self.normalize_sort_exprs(ordering) else {@@ -595,6 +690,27 @@ impl EquivalenceProperties {LexRequirement::new(self.eq_group.normalize_sort_requirements(sort_reqs))}+ /// Returns whether the given complete expression tuple is known to be+ /// grouped within every output partition.+ ///+ /// A tuple is grouped when all rows with the same values for the complete+ /// tuple occur in one contiguous run. The order of expressions within a+ /// grouping tuple does not matter: `[a, b]` and `[b, a]` describe the same+ /// groups. A lexicographical ordering also satisfies grouping for each of+ /// its prefixes, regardless of sort direction.+ pub fn grouping_satisfy(+ &self,+ given: impl IntoIterator<Item = Arc<dyn PhysicalExpr>>,+ ) -> Result<bool> {+ let normal_grouping = self.normalize_grouping(given);+ if normal_grouping.is_empty() || self.geq_cache.contains(&normal_grouping) {+ return Ok(true);+ }++ let (_, indices) = self.find_longest_permutation(&normal_grouping)?;+ Ok(indices.len() == normal_grouping.len())+ }+/// Iteratively checks whether the given ordering is satisfied by any of/// the existing orderings. See [`Self::ordering_satisfy_requirement`] for/// more details and examples.@@ -1095,6 +1211,21 @@ impl EquivalenceProperties {orderings.chain(projected_orderings).collect()}+ /// Projects grouping tuples through `mapping`, dropping a tuple unless all+ /// of its expressions can be represented by the projection.+ fn projected_groupings(+ &self,+ mapping: &ProjectionMapping,+ ) -> Vec<Vec<Arc<dyn PhysicalExpr>>> {+ self.geq_cache+ .iter()+ .filter_map(|grouping| {+ self.project_expressions(grouping.iter(), mapping)+ .collect::<Option<Vec<_>>>()+ })+ .collect()+ }+/// Projects constraints according to the given projection mapping.////// This function takes a projection mapping and extracts column indices of@@ -1138,6 +1269,8 @@ impl EquivalenceProperties {/// preserved through `c + 1` but dropped through `abs(c)`. Orderings/// implied by the mapping are also derived, e.g. an ordering on `a + b`/// yields one on the projected `a_new + b_new`.+ /// - Groupings: a complete grouping tuple is carried only when every+ /// expression in the tuple can be represented in the output./// - Equivalence group: each class is re-expressed on the output columns./// - Constraints: projected onto the surviving column indices.///@@ -1189,17 +1322,22 @@ impl EquivalenceProperties {) -> Self {let orderings =self.projected_orderings(mapping, self.oeq_cache.normal_cls.clone());+ let groupings = self.projected_groupings(mapping);let normal_orderings = orderings.iter().cloned().map(|o| eq_group.normalize_sort_exprs(o));- Self {+ let mut result = Self {oeq_cache: OrderingEquivalenceCache::new(normal_orderings),oeq_class: OrderingEquivalenceClass::new(orderings),+ geq_class: GroupingEquivalenceClass::default(),+ geq_cache: GroupingEquivalenceClass::default(),constraints: self.projected_constraints(mapping).unwrap_or_default(),schema: output_schema,eq_group,- }+ };+ result.add_groupings(groupings);+ result}/// Returns the longest (potentially partial) permutation satisfying the@@ -1404,6 +1542,11 @@ impl EquivalenceProperties {self.oeq_class = self.oeq_class.with_new_schema(&schema)?;self.oeq_cache.normal_cls = self.oeq_cache.normal_cls.with_new_schema(&schema)?;+ // Rewrite grouping expressions according to new schema and rebuild+ // their normalized form against the rewritten equivalence group.+ self.geq_class = self.geq_class.with_new_schema(&schema)?;+ self.update_geq_cache();+// Update the schema:self.schema = schema;@@ -1421,21 +1564,28 @@ impl From<EquivalenceProperties> for OrderingEquivalenceClass {////// Format:/// ```text-/// order: [[b@1 ASC NULLS LAST]], eq: [{members: [a@0], constant: (heterogeneous)}]+/// order: [[b@1 ASC NULLS LAST]], group: [[a@0, b@1]], eq: [{members: [a@0], constant: (heterogeneous)}]/// ```impl Display for EquivalenceProperties {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {let empty_eq_group = self.eq_group.is_empty();let empty_oeq_class = self.oeq_class.is_empty();- if empty_oeq_class && empty_eq_group {- write!(f, "No properties")?;- } else if !empty_oeq_class {+ let empty_geq_class = self.geq_class.is_empty();+ if empty_oeq_class && empty_geq_class && empty_eq_group {+ return write!(f, "No properties");+ }++ let mut separator = "";+ if !empty_oeq_class {write!(f, "order: {}", self.oeq_class)?;- if !empty_eq_group {- write!(f, ", eq: {}", self.eq_group)?;- }- } else {- write!(f, "eq: {}", self.eq_group)?;+ separator = ", ";+ }+ if !empty_geq_class {+ write!(f, "{separator}group: {}", self.geq_class)?;+ separator = ", ";+ }+ if !empty_eq_group {+ write!(f, "{separator}eq: {}", self.eq_group)?;}Ok(())}diff --git a/datafusion/physical-expr/src/equivalence/properties/union.rs b/datafusion/physical-expr/src/equivalence/properties/union.rsindex ed2beb377b..a3d5c1c454 100644--- a/datafusion/physical-expr/src/equivalence/properties/union.rs+++ b/datafusion/physical-expr/src/equivalence/properties/union.rs@@ -6,6 +6,7 @@ use std::iter::Peekable;use std::sync::Arc;use super::EquivalenceProperties;+use crate::equivalence::GroupingEquivalenceClass;use crate::equivalence::class::AcrossPartitions;use crate::{ConstExpr, PhysicalSortExpr};@@ -24,13 +25,8 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;/// partitions.fn calculate_union_binary(lhs: EquivalenceProperties,- mut rhs: EquivalenceProperties,+ rhs: &EquivalenceProperties,) -> Result<EquivalenceProperties> {- // Harmonize the schema of the rhs with the schema of the lhs (which is the accumulator schema):- if !rhs.schema.eq(&lhs.schema) {- rhs = rhs.with_new_schema(Arc::clone(&lhs.schema))?;- }-// First, calculate valid constants for the union. An expression is constant// at the output of the union if it is constant in both sides with matching values.let rhs_constants = rhs.constants();@@ -61,8 +57,8 @@ fn calculate_union_binary(// Next, calculate valid orderings for the union by searching for prefixes// in both sides.let mut orderings = UnionEquivalentOrderingBuilder::new();- orderings.add_satisfied_orderings(&lhs, &rhs)?;- orderings.add_satisfied_orderings(&rhs, &lhs)?;+ orderings.add_satisfied_orderings(&lhs, rhs)?;+ orderings.add_satisfied_orderings(rhs, &lhs)?;let orderings = orderings.build();let mut eq_properties = EquivalenceProperties::new(lhs.schema);@@ -81,6 +77,33 @@ pub fn calculate_union() -> Result<EquivalenceProperties> {// TODO: In some cases, we should be able to preserve some equivalence// classes. Add support for such cases.+ let eqps = eqps+ .into_iter()+ .map(move |props| {+ if props.schema.eq(&schema) {+ Ok(props)+ } else {+ props.with_new_schema(Arc::clone(&schema))+ }+ })+ .collect::<Result<Vec<_>>>()?;++ // Each output partition is one input partition, so a grouping holds when+ // every input satisfies it. Check the original inputs: folding orderings+ // first can lose grouping evidence when their sort directions differ.+ let mut groupings = GroupingEquivalenceClass::default();+ 'grouping: for grouping in eqps.iter().flat_map(|props| props.geq_class().iter()) {+ if groupings.contains(grouping) {+ continue;+ }+ for props in &eqps {+ if !props.grouping_satisfy(grouping.iter().cloned())? {+ continue 'grouping;+ }+ }+ groupings.add_groupings(std::iter::once(grouping.iter().cloned()));+ }+let mut iter = eqps.into_iter();let Some(mut acc) = iter.next() else {return internal_err!(@@ -88,14 +111,11 @@ pub fn calculate_union();};- // Harmonize the schema of the init with the schema of the union:- if !acc.schema.eq(&schema) {- acc = acc.with_new_schema(schema)?;- }// Fold in the rest of the EquivalenceProperties:for props in iter {- acc = calculate_union_binary(acc, props)?;+ acc = calculate_union_binary(acc, &props)?;}+ acc.add_groupings(groupings);Ok(acc)}@@ -676,6 +696,84 @@ mod tests {.run()}+ #[test]+ fn test_union_preserves_only_common_groupings() -> Result<()> {+ let schema = create_test_schema()?;+ UnionEquivalenceTest::new(&schema)+ .with_child_grouping(+ vec![vec!["e"], vec!["a", "b"], vec!["c", "d"]],+ &schema,+ )?+ .with_child_grouping(+ vec![vec!["f"], vec!["d", "c"], vec!["b", "a"]],+ &schema,+ )?+ .with_expected_sort(vec![])?+ .with_expected_grouping(vec![vec!["a", "b"], vec!["c", "d"]])?+ .run()+ }++ #[test]+ fn test_union_drops_grouping_missing_from_an_input() -> Result<()> {+ let schema = create_test_schema()?;+ for add_ordered_input in [false, true] {+ let mut test = UnionEquivalenceTest::new(&schema)+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?+ .with_child_sort(vec![], &schema)?;+ if add_ordered_input {+ test = test.with_child_sort(vec![vec!["a", "b"]], &schema)?;+ }+ test.with_expected_sort(vec![])?.run()?;+ }+ Ok(())+ }++ #[test]+ fn test_union_preserves_grouping_implied_by_ordering() -> Result<()> {+ let schema = create_test_schema()?;+ UnionEquivalenceTest::new(&schema)+ .with_child_sort(vec![vec!["a", "b"]], &schema)?+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?+ .with_expected_sort(vec![])?+ .with_expected_grouping(vec![vec!["a", "b"]])?+ .run()+ }++ #[test]+ fn test_union_preserves_grouping_with_different_schemas() -> Result<()> {+ let schema = create_test_schema()?;+ let schema2 = append_fields(&schema, "1");+ UnionEquivalenceTest::new(&schema)+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?+ .with_child_grouping(vec![vec!["b1", "a1"]], &schema2)?+ .with_expected_sort(vec![])?+ .with_expected_grouping(vec![vec!["a", "b"]])?+ .run()+ }++ #[test]+ fn test_union_grouping_is_independent_of_input_order() -> Result<()> {+ let schema = create_test_schema()?;+ // The ordered inputs have no common ordering, but both are grouped.+ UnionEquivalenceTest::new(&schema)+ .with_child_sort(vec![vec!["a", "b"]], &schema)?+ .with_child_sort(vec![vec!["a DESC", "b DESC"]], &schema)?+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?+ .with_expected_sort(vec![])?+ .with_expected_grouping(vec![vec!["a", "b"]])?+ .run()+ }++ #[test]+ fn test_union_does_not_infer_grouping_prefixes() -> Result<()> {+ let schema = create_test_schema()?;+ UnionEquivalenceTest::new(&schema)+ .with_child_grouping(vec![vec!["a", "b"]], &schema)?+ .with_child_grouping(vec![vec!["a"]], &schema)?+ .with_expected_sort(vec![])?+ .run()+ }+// TODO tests with multiple constants#[derive(Debug)]@@ -698,6 +796,40 @@ mod tests {}}+ fn with_child_grouping(+ mut self,+ groupings: Vec<Vec<&str>>,+ schema: &SchemaRef,+ ) -> Result<Self> {+ let mut properties = EquivalenceProperties::new(Arc::clone(schema));+ for grouping in groupings {+ properties.add_grouping(+ grouping+ .into_iter()+ .map(|name| col(name, schema))+ .collect::<Result<Vec<_>>>()?,+ );+ }+ self.child_properties.push(properties);+ Ok(self)+ }++ fn with_expected_grouping(mut self, groupings: Vec<Vec<&str>>) -> Result<Self> {+ let properties = self+ .expected_properties+ .as_mut()+ .expect("expected_properties not set");+ for grouping in groupings {+ properties.add_grouping(+ grouping+ .into_iter()+ .map(|name| col(name, &self.output_schema))+ .collect::<Result<Vec<_>>>()?,+ );+ }+ Ok(self)+ }+/// Add a union input with the specified orderings////// See [`Self::make_props`] for the format of the strings in `orderings`@@ -794,6 +926,15 @@ mod tests {rhs: &EquivalenceProperties,err_msg: String,) {+ assert_eq!(lhs.geq_class(), rhs.geq_class(), "{err_msg}");+ assert_eq!(+ lhs.normalized_geq_class(),+ rhs.normalized_geq_class(),+ "{err_msg}"+ );+ for grouping in rhs.geq_class().iter() {+ assert!(lhs.grouping_satisfy(grouping.iter().cloned()).unwrap());+ }// Check whether constants are samelet lhs_constants = lhs.constants();let rhs_constants = rhs.constants();diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex d3c85deb22..369cf77ae5 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1445,6 +1445,9 @@ impl AggregateExec {.equivalence_properties().project(group_expr_mapping, schema);+ // Aggregation can reorder groups, including during spill replay.+ eq_properties.clear_groupings();+// Only the clustered paths preserve existing ordering on group keys.// Project the input's actual sort expressions; clustering alone does// not establish an output ordering. Keep this consistent with@@ -5328,6 +5331,40 @@ mod tests {Ok(())}+ #[rstest::rstest]+ fn aggregate_clears_explicit_grouping(+ #[values(AggregateMode::Partial, AggregateMode::Single)] mode: AggregateMode,+ ) -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ Field::new("value", DataType::Int64, true),+ ]));+ let input = TestMemoryExec::try_new(&[vec![]], Arc::clone(&schema), None)?+ .try_with_grouping_information(vec![vec![col("a", &schema)?]])?;+ let aggregate = AggregateExec::try_new(+ mode,+ PhysicalGroupBy::new_single(vec![+ (col("b", &schema)?, "group_b".to_string()),+ (col("a", &schema)?, "group_a".to_string()),+ ]),+ vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("value", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("SUM(value)")+ .build()?,+ )],+ vec![None],+ Arc::new(input),+ schema,+ )?;++ let properties = aggregate.properties().equivalence_properties();+ assert!(properties.geq_class().is_empty());+ assert!(!properties.grouping_satisfy([col("group_a", &aggregate.schema())?])?);+ Ok(())+ }+#[rstest::rstest]#[case::full(true)]#[case::partial(false)]diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 3cd4e8c94f..e0d8ae173e 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -84,6 +84,9 @@ impl CoalescePartitionsExec {// Coalescing partitions loses existing orderings:let mut eq_properties = input.equivalence_properties().clone();eq_properties.clear_orderings();+ if input_partitions > 1 {+ eq_properties.clear_groupings();+ }eq_properties.clear_per_partition_constants();PlanProperties::new(eq_properties, // Equivalence Properties@@ -468,9 +471,40 @@ mod tests {use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_physical_expr::expressions::col;use futures::FutureExt;+ #[test]+ fn grouping_is_cleared_only_when_partitions_are_merged() -> Result<()> {+ let input = test::mem_exec(2);+ let grouping = col("i", &input.schema())?;+ let input = Arc::new(+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,+ );+ assert!(+ input+ .properties()+ .equivalence_properties()+ .grouping_satisfy([Arc::clone(&grouping)])?+ );+ let merge = CoalescePartitionsExec::new(input);+ let properties = merge.properties().equivalence_properties();+ assert!(properties.geq_class().is_empty());+ assert!(!properties.grouping_satisfy([grouping])?);++ let input = test::mem_exec(1);+ let grouping = col("i", &input.schema())?;+ let input = Arc::new(+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,+ );+ let merge = CoalescePartitionsExec::new(input);+ let properties = merge.properties().equivalence_properties();+ assert_eq!(properties.geq_class().len(), 1);+ assert!(properties.grouping_satisfy([grouping])?);+ Ok(())+ }+#[tokio::test]async fn merge() -> Result<()> {let task_ctx = Arc::new(TaskContext::default());diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex cc346c02a6..51ead774c5 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -91,9 +91,13 @@ impl GlobalLimitExec {/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.fn compute_properties(input: &Arc<dyn ExecutionPlan>) -> PlanProperties {+ let mut eq_properties = input.equivalence_properties().clone();+ if input.output_partitioning().partition_count() > 1 {+ eq_properties.clear_groupings();+ }PlanProperties::new(- input.equivalence_properties().clone(), // Equivalence Properties- Partitioning::UnknownPartitioning(1), // Output Partitioning+ eq_properties, // Equivalence Properties+ Partitioning::UnknownPartitioning(1), // Output Partitioninginput.pipeline_behavior(),// Limit operations are always bounded since they output a finite number of rowsBoundedness::Bounded,@@ -771,6 +775,41 @@ mod tests {use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::expressions::col;+ #[test]+ fn limits_preserve_grouping_without_merging_partitions() -> Result<()> {+ let input = test::mem_exec(2);+ let grouping = col("i", &input.schema())?;+ let input: Arc<dyn ExecutionPlan> = Arc::new(+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,+ );+ assert!(+ input+ .equivalence_properties()+ .grouping_satisfy([Arc::clone(&grouping)])?+ );++ let local = LocalLimitExec::new(Arc::clone(&input), 10);+ let properties = local.properties().equivalence_properties();+ assert_eq!(properties.geq_class().len(), 1);+ assert!(properties.grouping_satisfy([Arc::clone(&grouping)])?);++ let global = GlobalLimitExec::new(input, 0, Some(10));+ let properties = global.properties().equivalence_properties();+ assert!(properties.geq_class().is_empty());+ assert!(!properties.grouping_satisfy([grouping])?);++ let input = test::mem_exec(1);+ let grouping = col("i", &input.schema())?;+ let input = Arc::new(+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,+ );+ let global = GlobalLimitExec::new(input, 0, Some(10));+ let properties = global.properties().equivalence_properties();+ assert_eq!(properties.geq_class().len(), 1);+ assert!(properties.grouping_satisfy([grouping])?);+ Ok(())+ }+#[tokio::test]async fn limit() -> Result<()> {let task_ctx = Arc::new(TaskContext::default());diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 929f52fd15..13274d26e1 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -2209,6 +2209,7 @@ impl RepartitionExec {if preserve_order && let Some(ordering) = input.output_ordering() {eq_properties.add_ordering(ordering.clone());}+ eq_properties.clear_groupings();eq_properties.clear_per_partition_constants();}eq_properties@@ -2677,6 +2678,38 @@ mod tests {};use insta::assert_snapshot;+ #[test]+ fn grouping_is_cleared_only_when_input_partitions_are_merged() -> Result<()> {+ let input = crate::test::mem_exec(2);+ let grouping = col("i", &input.schema())?;+ let input: Arc<dyn ExecutionPlan> = Arc::new(+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,+ );+ assert!(+ input+ .equivalence_properties()+ .grouping_satisfy([Arc::clone(&grouping)])?+ );+ let repartition =+ RepartitionExec::try_new(input, Partitioning::RoundRobinBatch(2))?;+ let properties = repartition.properties().equivalence_properties();+ assert!(properties.geq_class().is_empty());+ assert!(!properties.grouping_satisfy([grouping])?);++ // Splitting one stream preserves contiguity in each output subsequence.+ let input = crate::test::mem_exec(1);+ let grouping = col("i", &input.schema())?;+ let input: Arc<dyn ExecutionPlan> = Arc::new(+ input.try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?,+ );+ let repartition =+ RepartitionExec::try_new(input, Partitioning::RoundRobinBatch(2))?;+ let properties = repartition.properties().equivalence_properties();+ assert_eq!(properties.geq_class().len(), 1);+ assert!(properties.grouping_satisfy([grouping])?);+ Ok(())+ }+#[derive(Debug)]struct UnboundedTestPartition {schema: SchemaRef,diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex 4cfe4e5098..641e932c28 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -164,6 +164,7 @@ impl SortPreservingMergeExec {// Other input orderings hold only within each partition:if input_partitions > 1 {eq_properties.clear_orderings();+ eq_properties.clear_groupings();}eq_properties.clear_per_partition_constants();eq_properties.add_ordering(ordering);@@ -884,6 +885,42 @@ mod tests {use insta::assert_snapshot;use tokio::time::timeout;+ #[test]+ fn merging_sorted_partitions_clears_explicit_grouping() -> Result<()> {+ // Each partition is grouped on `a`, but merging on `b` splits the `a` runs.+ let first = test::build_table_i32_two_cols(+ ("a", &vec![1, 1, 2, 2]),+ ("b", &vec![1, 3, 5, 7]),+ );+ let second = test::build_table_i32_two_cols(+ ("a", &vec![2, 2, 1, 1]),+ ("b", &vec![2, 4, 6, 8]),+ );+ let schema = first.schema();+ let grouping = col("a", &schema)?;+ let sort_key = col("b", &schema)?;+ let ordering: LexOrdering =+ [PhysicalSortExpr::new_default(Arc::clone(&sort_key))].into();+ let input = TestMemoryExec::try_new(&[vec![first], vec![second]], schema, None)?+ .try_with_sort_information(vec![ordering.clone()])?+ .try_with_grouping_information(vec![vec![Arc::clone(&grouping)]])?;+ assert!(+ input+ .properties()+ .equivalence_properties()+ .grouping_satisfy([Arc::clone(&grouping)])?+ );+ let merge = SortPreservingMergeExec::new(ordering.clone(), Arc::new(input));+ let properties = merge.properties().equivalence_properties();++ assert!(properties.geq_class().is_empty());+ assert!(!properties.grouping_satisfy([grouping])?);+ // The merge key remains ordered, and therefore grouped.+ assert!(properties.ordering_satisfy(ordering)?);+ assert!(properties.grouping_satisfy([sort_key])?);+ Ok(())+ }+// The number in the function is highly related to the memory limit we are testing// any change of the constant should be aware offn generate_task_ctx_for_round_robin_tie_breaker(diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rsindex 25125fba83..47c982744d 100644--- a/datafusion/physical-plan/src/test.rs+++ b/datafusion/physical-plan/src/test.rs@@ -60,6 +60,8 @@ pub struct TestMemoryExec {projection: Option<Vec<usize>>,/// Sort information: one or more equivalent orderingssort_information: Vec<LexOrdering>,+ /// Complete expression tuples whose values are contiguous.+ grouping_information: Vec<Vec<Arc<dyn PhysicalExpr>>>,/// if partition sizes should be displayedshow_sizes: bool,/// The maximum number of records to read from this plan. If `None`,@@ -220,10 +222,12 @@ impl TestMemoryExec {}fn eq_properties(&self) -> EquivalenceProperties {- EquivalenceProperties::new_with_orderings(+ let mut properties = EquivalenceProperties::new_with_orderings(Arc::clone(&self.projected_schema),self.sort_information.clone(),- )+ );+ properties.add_groupings(self.grouping_information.clone());+ properties}fn statistics_inner(&self) -> Result<Statistics> {@@ -255,6 +259,7 @@ impl TestMemoryExec {projected_schema,projection,sort_information: vec![],+ grouping_information: vec![],show_sizes: true,fetch: None,})@@ -350,6 +355,52 @@ impl TestMemoryExec {Ok(self)}+ /// Adds grouping information to this source.+ pub fn try_with_grouping_information(+ mut self,+ mut grouping_information: Vec<Vec<Arc<dyn PhysicalExpr>>>,+ ) -> Result<Self> {+ // All grouping expressions must refer to the original schema.+ let fields = self.schema.fields();+ let ambiguous_column = grouping_information+ .iter()+ .flatten()+ .flat_map(collect_columns)+ .find(|col| {+ fields+ .get(col.index())+ .map(|field| field.name() != col.name())+ .unwrap_or(true)+ });+ assert_or_internal_err!(+ ambiguous_column.is_none(),+ "Column {:?} is not found in the original schema of the TestMemoryExec",+ ambiguous_column.as_ref().unwrap()+ );++ if let Some(projection) = &self.projection {+ let base_schema = self.original_schema();+ let proj_exprs = projection.iter().map(|idx| {+ let name = base_schema.field(*idx).name();+ (Arc::new(Column::new(name, *idx)) as _, name.to_string())+ });+ let projection_mapping =+ ProjectionMapping::try_new(proj_exprs, &base_schema)?;+ let mut base_eqp = EquivalenceProperties::new(base_schema);+ base_eqp.add_groupings(grouping_information);+ grouping_information = base_eqp+ .project(&projection_mapping, Arc::clone(&self.projected_schema))+ .geq_class()+ .iter()+ .cloned()+ .collect();+ }++ self.grouping_information = grouping_information;+ self.cache = Arc::new(self.compute_properties());+ Ok(self)+ }+/// Arc clone of ref to original schemapub fn original_schema(&self) -> SchemaRef {Arc::clone(&self.schema)diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 3b4cebc67c..042f3217e2 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -1216,6 +1216,74 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_union_preserves_grouped_partition_streams() -> Result<()> {+ let first =+ test::build_table_i32_two_cols(("a", &vec![2, 2, 1]), ("b", &vec![0, 0, 1]));+ let second =+ test::build_table_i32_two_cols(("a", &vec![1, 2, 1]), ("b", &vec![1, 1, 0]));+ let schema = first.schema();+ let grouping = vec![col("a", &schema)?, col("b", &schema)?];+ // Only the complete tuple is contiguous, and one run spans batches.+ let left_partitions = vec![vec![first, second], vec![]];+ let right_partitions = vec![vec![+ test::build_table_i32_two_cols(("a", &vec![1, 2, 1]), ("b", &vec![0, 1, 1])),+ test::build_table_i32_two_cols(("a", &vec![1, 2, 2]), ("b", &vec![1, 0, 0])),+ ]];+ let left = TestMemoryExec::try_new(&left_partitions, Arc::clone(&schema), None)?+ .try_with_grouping_information(vec![grouping.clone()])?;+ let right = TestMemoryExec::try_new(&right_partitions, schema, None)?+ .try_with_grouping_information(vec![grouping.clone()])?;+ let union = UnionExec::try_new(vec![Arc::new(left), Arc::new(right)])?;+ let properties = union.equivalence_properties();+ assert!(properties.grouping_satisfy(grouping)?);+ assert!(!properties.grouping_satisfy([col("a", &union.schema())?])?);+ assert!(!properties.grouping_satisfy([col("b", &union.schema())?])?);+ assert!(union.output_ordering().is_none());+ assert!(matches!(+ union.output_partitioning(),+ Partitioning::UnknownPartitioning(3)+ ));++ let context = Arc::new(TaskContext::default());+ for (partition, expected) in+ left_partitions.iter().chain(&right_partitions).enumerate()+ {+ let actual =+ crate::common::collect(union.execute(partition, Arc::clone(&context))?)+ .await?;+ assert_eq!(&actual, expected);+ }+ Ok(())+ }++ #[test]+ fn test_interleave_drops_grouping() -> Result<()> {+ let schema = create_test_schema()?;+ let grouping = vec![col("a", &schema)?, col("b", &schema)?];+ let source = TestMemoryExec::try_new(&[vec![]], schema, None)?+ .try_with_grouping_information(vec![grouping.clone()])?;+ let input: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(+ Arc::new(source),+ Partitioning::Hash(grouping.clone(), 1),+ )?);+ assert!(+ input+ .equivalence_properties()+ .grouping_satisfy(grouping.clone())?+ );++ let interleave: Arc<dyn ExecutionPlan> =+ Arc::new(InterleaveExec::try_new(vec![Arc::clone(&input), input])?);+ assert!(interleave.equivalence_properties().geq_class().is_empty());+ assert!(+ !interleave+ .equivalence_properties()+ .grouping_satisfy(grouping)?+ );+ Ok(())+ }+fn stats_merge_inputs() -> (SchemaRef, Statistics, Statistics, Statistics) {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)]));