Apache DataFusion SQL Query Engine
Fix projection functional dependency remapping (#23028)
Huang Qiwei12b93675997b
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex e3c151486f..c154bc7c92 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -4137,8 +4137,12 @@ fn calc_func_dependencies_for_project(exprs: &[Expr],input: &LogicalPlan,) -> Result<FunctionalDependencies> {+ // Sentinel for projection outputs that do not map back to any input field.+ const COMPUTED_EXPR_INDEX: usize = usize::MAX;+let input_fields = input.schema().field_names();- // Calculate expression indices (if present) in the input schema.+ // Map each projection output position to its input column index.+ // A projection expression can produce multiple output columns, such as `*`.let proj_indices = exprs.iter().map(|expr| match expr {@@ -4154,30 +4158,33 @@ fn calc_func_dependencies_for_project(Ok::<_, DataFusionError>(wildcard_fields.into_iter()- .filter_map(|(qualifier, f)| {+ .map(|(qualifier, f)| {let flat_name = qualifier.map(|t| format!("{}.{}", t, f.name())).unwrap_or_else(|| f.name().clone());- input_fields.iter().position(|item| *item == flat_name)+ input_fields+ .iter()+ .position(|item| *item == flat_name)+ .unwrap_or(COMPUTED_EXPR_INDEX)}).collect::<Vec<_>>(),)}Expr::Alias(alias) => {let name = format!("{}", alias.expr);- Ok(input_fields+ let input_index = input_fields.iter().position(|item| *item == name)- .map(|i| vec![i])- .unwrap_or(vec![]))+ .unwrap_or(COMPUTED_EXPR_INDEX);+ Ok(vec![input_index])}_ => {let name = format!("{expr}");- Ok(input_fields+ let input_index = input_fields.iter().position(|item| *item == name)- .map(|i| vec![i])- .unwrap_or(vec![]))+ .unwrap_or(COMPUTED_EXPR_INDEX);+ Ok(vec![input_index])}}).collect::<Result<Vec<_>>>()?@@ -4940,6 +4947,82 @@ mod tests {])}+ #[test]+ fn projection_with_leading_computed_column_preserves_pk() -> Result<()> {+ let constraints =+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])]);+ let source = Arc::new(+ LogicalTableSource::new(Arc::new(employee_schema()))+ .with_constraints(constraints),+ );+ let plan = LogicalPlanBuilder::scan("employee_csv", source, None)?+ .project(vec![+ lit(1i32).alias("__common_expr_1"),+ col("id"),+ col("first_name"),+ col("salary"),+ ])?+ .build()?;++ let deps = plan.schema().functional_dependencies();+ assert_eq!(deps.len(), 1);+ assert_eq!(deps[0].source_indices, vec![1]);++ Ok(())+ }++ #[test]+ fn projection_with_leading_computed_column_and_wildcard_preserves_pk() -> Result<()> {+ let constraints =+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])]);+ let source = Arc::new(+ LogicalTableSource::new(Arc::new(employee_schema()))+ .with_constraints(constraints),+ );+ let plan = LogicalPlanBuilder::scan("employee_csv", source, None)?+ .project(vec![+ SelectExpr::Expression(lit(1i32).alias("__common_expr_1")),+ SelectExpr::Wildcard(Default::default()),+ ])?+ .build()?;++ let deps = plan.schema().functional_dependencies();+ assert_eq!(plan.schema().fields().len(), 6);+ assert_eq!(deps.len(), 1);+ assert_eq!(deps[0].source_indices, vec![1]);+ assert_eq!(deps[0].target_indices, vec![0, 1, 2, 3, 4, 5]);++ Ok(())+ }++ #[test]+ fn projection_with_wildcard_expr_before_pk_preserves_pk() -> Result<()> {+ let constraints =+ Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])]);+ let source = Arc::new(+ LogicalTableSource::new(Arc::new(employee_schema()))+ .with_constraints(constraints),+ );+ let input = LogicalPlanBuilder::scan("employee_csv", source, None)?.build()?;+ #[expect(deprecated)]+ let projection = Projection::try_new(+ vec![+ Expr::Wildcard {+ qualifier: None,+ options: Box::new(crate::expr::WildcardOptions::default()),+ },+ col("employee_csv.id"),+ ],+ Arc::new(input),+ )?;++ let deps = projection.schema.functional_dependencies();+ assert_eq!(deps.len(), 1);+ assert_eq!(deps[0].source_indices, vec![1]);++ Ok(())+ }+fn i32_split_point(value: i32) -> SplitPoint {SplitPoint::new(vec![ScalarValue::Int32(Some(value))])}diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.partindex ad23cd9079..e678f8b440 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part@@ -39,7 +39,7 @@ logical_plan01)Projection: CAST(sum(lineitem.l_extendedprice) AS Float64) / Float64(7) AS avg_yearly02)--Aggregate: groupBy=[[]], aggr=[[sum(lineitem.l_extendedprice)]]03)----Projection: lineitem.l_extendedprice-04)------Inner Join: part.p_partkey = __scalar_sq_1.l_partkey Filter: CAST(lineitem.l_quantity AS Decimal128(30, 15)) < __scalar_sq_1.Float64(0.2) * avg(lineitem.l_quantity)+04)------LeftSemi Join: part.p_partkey = __scalar_sq_1.l_partkey Filter: CAST(lineitem.l_quantity AS Decimal128(30, 15)) < __scalar_sq_1.Float64(0.2) * avg(lineitem.l_quantity)05)--------Projection: lineitem.l_quantity, lineitem.l_extendedprice, part.p_partkey06)----------Inner Join: lineitem.l_partkey = part.p_partkey07)------------TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice]@@ -55,7 +55,7 @@ physical_plan02)--AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)]03)----CoalescePartitionsExec04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)]-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1]+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1]06)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_quantity@1, l_extendedprice@2, p_partkey@3]07)------------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=408)--------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_partkey, l_quantity, l_extendedprice], constraints=[PrimaryKey([0, 3])], file_type=csv, has_header=false