Apache DataFusion SQL Query Engine
fix: keep a constant sort key when a projection reads a duplicate alias (#26397)
Peter L56eca4a7c5f8
diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rsindex 5e2f4784cb..b0f00c359e 100644--- a/datafusion/physical-expr/src/equivalence/class.rs+++ b/datafusion/physical-expr/src/equivalence/class.rs@@ -670,7 +670,17 @@ impl EquivalenceGroup {mapping.iter().map(|(k, v)| {- let eq_class = self.get_equivalence_class(k);+ // A source expression may only be in a class in its normalized+ // form. For example, if `a == b`, `a + c` is in a class, and+ // the mapping has `b + c`, then the class of `b + c` is found+ // only through `a + c`. A leaf that is in no class normalizes+ // to itself, so only expressions with children are normalized.+ let eq_class = self.get_equivalence_class(k).or_else(|| {+ if k.children().is_empty() {+ return None;+ }+ self.get_equivalence_class(&self.normalize_expr(Arc::clone(k)))+ });(k, (v, eq_class))}).collect()@@ -1276,6 +1286,106 @@ mod tests {Ok(())}+ #[test]+ fn test_project_constant_of_equivalent_source() -> Result<()> {+ // - columns: [a, b, c].+ // - "a" and "b" in the same equivalence class, and a + c = 7.+ // - then after a b + c projection, col(0) must be the constant 7.+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ Field::new("c", DataType::Int32, false),+ ]));+ let mut group = EquivalenceGroup::default();+ group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);+ group.add_equal_conditions(+ binary(+ col("a", &schema)?,+ Operator::Plus,+ col("c", &schema)?,+ &schema,+ )?,+ lit(7),+ );++ let projected_schema =+ Arc::new(Schema::new(vec![Field::new("b+c", DataType::Int32, false)]));+ let mapping = std::iter::once((+ binary(+ col("b", &schema)?,+ Operator::Plus,+ col("c", &schema)?,+ &schema,+ )?,+ vec![(col("b+c", &projected_schema)?, 0)].into(),+ ))+ .collect::<ProjectionMapping>();++ let projected = group.project(&mapping);++ assert_eq!(+ projected.is_expr_constant(&col("b+c", &projected_schema)?),+ Some(AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(7)))))+ );++ Ok(())+ }++ #[test]+ fn test_project_class_of_equivalent_source() -> Result<()> {+ // - columns: [a, b, c, d].+ // - "a" and "b" in the same equivalence class, and a + c = d * 2.+ // - then after a b + c, d projection, col(0) and col(1) * 2 must be+ // in the same class.+ let schema = abc_schema();+ let mut group = EquivalenceGroup::default();+ group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);+ group.add_equal_conditions(+ binary(+ col("a", &schema)?,+ Operator::Plus,+ col("c", &schema)?,+ &schema,+ )?,+ binary(col("d", &schema)?, Operator::Multiply, lit(2), &schema)?,+ );++ let projected_schema = Arc::new(Schema::new(vec![+ Field::new("b+c", DataType::Int32, false),+ Field::new("d", DataType::Int32, false),+ ]));+ let mapping = [+ (+ binary(+ col("b", &schema)?,+ Operator::Plus,+ col("c", &schema)?,+ &schema,+ )?,+ vec![(col("b+c", &projected_schema)?, 0)].into(),+ ),+ (+ col("d", &schema)?,+ vec![(col("d", &projected_schema)?, 1)].into(),+ ),+ ]+ .into_iter()+ .collect::<ProjectionMapping>();++ let projected = group.project(&mapping);++ let first_normalized = projected.normalize_expr(col("b+c", &projected_schema)?);+ let second_normalized = projected.normalize_expr(binary(+ col("d", &projected_schema)?,+ Operator::Multiply,+ lit(2),+ &projected_schema,+ )?);+ assert!(first_normalized.eq(&second_normalized));++ Ok(())+ }+/// Builds a group from a list of equated column pairs.fn group_of(schema: &SchemaRef, pairs: &[(&str, &str)]) -> Result<EquivalenceGroup> {let mut group = EquivalenceGroup::default();diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex 9e19344901..b178a33da7 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -2622,3 +2622,38 @@ DROP TABLE union_leaf_c;statement okDROP TABLE union_leaf_d;++#####################+# A sort key that a filter makes constant, read through a duplicate extraction alias+# See https://github.com/apache/datafusion/issues/26391+#####################++# The filter and the projection each extract `s['k']`, under two aliases. The+# filter moves into the scan, and the scan computes both aliases. The sort key+# `c` reads the second alias, and the filter's constant reads the first. The+# planner must still know that `c` is a constant after the projections merge.+statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++statement ok+COPY (SELECT * FROM (VALUES ({k: 'a'}, 1), ({k: 'b'}, 2)) AS v(s, id))+TO 'test_files/scratch/projection_pushdown/duplicate_alias.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE duplicate_alias STORED AS PARQUET+LOCATION 'test_files/scratch/projection_pushdown/duplicate_alias.parquet';++query TI+SELECT upper(s['k']) AS c, synth+FROM (SELECT s, id + 1 AS synth FROM duplicate_alias) AS v+WHERE upper(s['k']) = 'A'+ORDER BY synth, c;+----+A 2++statement ok+DROP TABLE duplicate_alias;++statement ok+RESET datafusion.execution.parquet.pushdown_filters;