Skip to content

Apache DataFusion SQL Query Engine

fix: keep a constant sort key when a projection reads a duplicate alias (#26397)

Peter L56eca4a7c5f8
diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rs
index 5e2f4784cb..b0f00c359e 100644
--- a/datafusion/physical-expr/src/equivalence/class.rs
+++ b/datafusion/physical-expr/src/equivalence/class.rs
@@ -670,7 +670,17 @@ impl EquivalenceGroup {
mapping
.iter()
.map(|(k, v)| {
- let eq_class = self.get_equivalence_class(k);
+ // A source expression may only be in a class in its normalized
+ // form. For example, if `a == b`, `a + c` is in a class, and
+ // the mapping has `b + c`, then the class of `b + c` is found
+ // only through `a + c`. A leaf that is in no class normalizes
+ // to itself, so only expressions with children are normalized.
+ let eq_class = self.get_equivalence_class(k).or_else(|| {
+ if k.children().is_empty() {
+ return None;
+ }
+ self.get_equivalence_class(&self.normalize_expr(Arc::clone(k)))
+ });
(k, (v, eq_class))
})
.collect()
@@ -1276,6 +1286,106 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_project_constant_of_equivalent_source() -> Result<()> {
+ // - columns: [a, b, c].
+ // - "a" and "b" in the same equivalence class, and a + c = 7.
+ // - then after a b + c projection, col(0) must be the constant 7.
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ Field::new("c", DataType::Int32, false),
+ ]));
+ let mut group = EquivalenceGroup::default();
+ group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);
+ group.add_equal_conditions(
+ binary(
+ col("a", &schema)?,
+ Operator::Plus,
+ col("c", &schema)?,
+ &schema,
+ )?,
+ lit(7),
+ );
+
+ let projected_schema =
+ Arc::new(Schema::new(vec![Field::new("b+c", DataType::Int32, false)]));
+ let mapping = std::iter::once((
+ binary(
+ col("b", &schema)?,
+ Operator::Plus,
+ col("c", &schema)?,
+ &schema,
+ )?,
+ vec![(col("b+c", &projected_schema)?, 0)].into(),
+ ))
+ .collect::<ProjectionMapping>();
+
+ let projected = group.project(&mapping);
+
+ assert_eq!(
+ projected.is_expr_constant(&col("b+c", &projected_schema)?),
+ Some(AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(7)))))
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_project_class_of_equivalent_source() -> Result<()> {
+ // - columns: [a, b, c, d].
+ // - "a" and "b" in the same equivalence class, and a + c = d * 2.
+ // - then after a b + c, d projection, col(0) and col(1) * 2 must be
+ // in the same class.
+ let schema = abc_schema();
+ let mut group = EquivalenceGroup::default();
+ group.add_equal_conditions(col("a", &schema)?, col("b", &schema)?);
+ group.add_equal_conditions(
+ binary(
+ col("a", &schema)?,
+ Operator::Plus,
+ col("c", &schema)?,
+ &schema,
+ )?,
+ binary(col("d", &schema)?, Operator::Multiply, lit(2), &schema)?,
+ );
+
+ let projected_schema = Arc::new(Schema::new(vec![
+ Field::new("b+c", DataType::Int32, false),
+ Field::new("d", DataType::Int32, false),
+ ]));
+ let mapping = [
+ (
+ binary(
+ col("b", &schema)?,
+ Operator::Plus,
+ col("c", &schema)?,
+ &schema,
+ )?,
+ vec![(col("b+c", &projected_schema)?, 0)].into(),
+ ),
+ (
+ col("d", &schema)?,
+ vec![(col("d", &projected_schema)?, 1)].into(),
+ ),
+ ]
+ .into_iter()
+ .collect::<ProjectionMapping>();
+
+ let projected = group.project(&mapping);
+
+ let first_normalized = projected.normalize_expr(col("b+c", &projected_schema)?);
+ let second_normalized = projected.normalize_expr(binary(
+ col("d", &projected_schema)?,
+ Operator::Multiply,
+ lit(2),
+ &projected_schema,
+ )?);
+ assert!(first_normalized.eq(&second_normalized));
+
+ Ok(())
+ }
+
/// Builds a group from a list of equated column pairs.
fn group_of(schema: &SchemaRef, pairs: &[(&str, &str)]) -> Result<EquivalenceGroup> {
let mut group = EquivalenceGroup::default();
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index 9e19344901..b178a33da7 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -2622,3 +2622,38 @@ DROP TABLE union_leaf_c;
statement ok
DROP TABLE union_leaf_d;
+
+#####################
+# A sort key that a filter makes constant, read through a duplicate extraction alias
+# See https://github.com/apache/datafusion/issues/26391
+#####################
+
+# The filter and the projection each extract `s['k']`, under two aliases. The
+# filter moves into the scan, and the scan computes both aliases. The sort key
+# `c` reads the second alias, and the filter's constant reads the first. The
+# planner must still know that `c` is a constant after the projections merge.
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+statement ok
+COPY (SELECT * FROM (VALUES ({k: 'a'}, 1), ({k: 'b'}, 2)) AS v(s, id))
+TO 'test_files/scratch/projection_pushdown/duplicate_alias.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE duplicate_alias STORED AS PARQUET
+LOCATION 'test_files/scratch/projection_pushdown/duplicate_alias.parquet';
+
+query TI
+SELECT upper(s['k']) AS c, synth
+FROM (SELECT s, id + 1 AS synth FROM duplicate_alias) AS v
+WHERE upper(s['k']) = 'A'
+ORDER BY synth, c;
+----
+A 2
+
+statement ok
+DROP TABLE duplicate_alias;
+
+statement ok
+RESET datafusion.execution.parquet.pushdown_filters;