Apache DataFusion SQL Query Engine
Clean up expression placement UDF usage in tests (#20272)
Adrian Garcia Badaraccof9d37cf76945
diff --git a/datafusion/optimizer/src/extract_leaf_expressions.rs b/datafusion/optimizer/src/extract_leaf_expressions.rsindex 151bca8278..d04261456d 100644--- a/datafusion/optimizer/src/extract_leaf_expressions.rs+++ b/datafusion/optimizer/src/extract_leaf_expressions.rs@@ -167,72 +167,22 @@ mod tests {use super::*;use crate::optimize_projections::OptimizeProjections;+ use crate::test::udfs::PlacementTestUDF;use crate::test::*;use crate::{Optimizer, OptimizerContext};- use arrow::datatypes::DataType;use datafusion_common::Result;use datafusion_expr::expr::ScalarFunction;+ use datafusion_expr::{Expr, ExpressionPlacement};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,- TypeSignature, col, lit, logical_plan::builder::LogicalPlanBuilder,+ ScalarUDF, col, lit, logical_plan::builder::LogicalPlanBuilder,};- use datafusion_expr::{Expr, ExpressionPlacement};-- /// A mock UDF that simulates a leaf-pushable function like `get_field`.- /// It returns `MoveTowardsLeafNodes` when its first argument is Column or MoveTowardsLeafNodes.- #[derive(Debug, PartialEq, Eq, Hash)]- struct MockLeafFunc {- signature: Signature,- }-- impl MockLeafFunc {- fn new() -> Self {- Self {- signature: Signature::new(- TypeSignature::Any(2),- datafusion_expr::Volatility::Immutable,- ),- }- }- }-- impl ScalarUDFImpl for MockLeafFunc {- fn as_any(&self) -> &dyn std::any::Any {- self- }-- fn name(&self) -> &str {- "mock_leaf"- }-- fn signature(&self) -> &Signature {- &self.signature- }- fn return_type(&self, _args: &[DataType]) -> Result<DataType> {- Ok(DataType::Utf8)- }-- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {- unimplemented!("This is only used for testing optimization")- }-- fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {- // Return MoveTowardsLeafNodes if first arg is Column or MoveTowardsLeafNodes- // (like get_field does)- match args.first() {- Some(ExpressionPlacement::Column)- | Some(ExpressionPlacement::MoveTowardsLeafNodes) => {- ExpressionPlacement::MoveTowardsLeafNodes- }- _ => ExpressionPlacement::KeepInPlace,- }- }- }-- fn mock_leaf(expr: Expr, name: &str) -> Expr {+ fn leaf_udf(expr: Expr, name: &str) -> Expr {Expr::ScalarFunction(ScalarFunction::new_udf(- Arc::new(ScalarUDF::new_from_impl(MockLeafFunc::new())),+ Arc::new(ScalarUDF::new_from_impl(+ PlacementTestUDF::new()+ .with_placement(ExpressionPlacement::MoveTowardsLeafNodes),+ )),vec![expr, lit(name)],))}@@ -251,9 +201,8 @@ mod tests {/// 3. **After Pushdown** - + PushDownLeafProjections/// 4. **Optimized** - + final OptimizeProjectionsfn format_optimization_stages(plan: &LogicalPlan) -> Result<String> {- let ctx = OptimizerContext::new().with_max_passes(1);-let run = |rules: Vec<Arc<dyn OptimizerRule + Send + Sync>>| -> Result<String> {+ let ctx = OptimizerContext::new().with_max_passes(1);let optimizer = Optimizer::with_rules(rules);let optimized = optimizer.optimize(plan.clone(), &ctx, |_, _| {})?;Ok(format!("{optimized}"))@@ -318,7 +267,7 @@ mod tests {fn test_extract_from_filter() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan.clone())- .filter(mock_leaf(col("user"), "status").eq(lit("active")))?+ .filter(leaf_udf(col("user"), "status").eq(lit("active")))?.select(vec![table_scan.schema()@@ -330,7 +279,7 @@ mod tests {assert_stages!(plan, @r#"## Original PlanProjection: test.id- Filter: mock_leaf(test.user, Utf8("status")) = Utf8("active")+ Filter: leaf_udf(test.user, Utf8("status")) = Utf8("active")TableScan: test projection=[id, user]## After Extraction@@ -371,12 +320,12 @@ mod tests {fn test_extract_from_projection() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .project(vec![mock_leaf(col("user"), "name")])?+ .project(vec![leaf_udf(col("user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name"))+ Projection: leaf_udf(test.user, Utf8("name"))TableScan: test projection=[user]## After Extraction@@ -395,7 +344,7 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).project(vec![- mock_leaf(col("user"), "name")+ leaf_udf(col("user"), "name").is_not_null().alias("has_name"),])?@@ -403,7 +352,7 @@ mod tests {assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name")) IS NOT NULL AS has_name+ Projection: leaf_udf(test.user, Utf8("name")) IS NOT NULL AS has_nameTableScan: test projection=[user]## After Extraction@@ -442,7 +391,7 @@ mod tests {#[test]fn test_filter_with_deduplication() -> Result<()> {let table_scan = test_table_scan_with_struct()?;- let field_access = mock_leaf(col("user"), "name");+ let field_access = leaf_udf(col("user"), "name");// Filter with the same expression used twicelet plan = LogicalPlanBuilder::from(table_scan).filter(@@ -455,7 +404,7 @@ mod tests {assert_stages!(plan, @r#"## Original Plan- Filter: mock_leaf(test.user, Utf8("name")) IS NOT NULL AND mock_leaf(test.user, Utf8("name")) IS NULL+ Filter: leaf_udf(test.user, Utf8("name")) IS NOT NULL AND leaf_udf(test.user, Utf8("name")) IS NULLTableScan: test projection=[id, user]## After Extraction@@ -473,12 +422,12 @@ mod tests {fn test_already_leaf_expression_in_filter() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "name").eq(lit("test")))?+ .filter(leaf_udf(col("user"), "name").eq(lit("test")))?.build()?;assert_stages!(plan, @r#"## Original Plan- Filter: mock_leaf(test.user, Utf8("name")) = Utf8("test")+ Filter: leaf_udf(test.user, Utf8("name")) = Utf8("test")TableScan: test projection=[id, user]## After Extraction@@ -498,12 +447,12 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .aggregate(vec![mock_leaf(col("user"), "status")], vec![count(lit(1))])?+ .aggregate(vec![leaf_udf(col("user"), "status")], vec![count(lit(1))])?.build()?;assert_stages!(plan, @r#"## Original Plan- Aggregate: groupBy=[[mock_leaf(test.user, Utf8("status"))]], aggr=[[COUNT(Int32(1))]]+ Aggregate: groupBy=[[leaf_udf(test.user, Utf8("status"))]], aggr=[[COUNT(Int32(1))]]TableScan: test projection=[user]## After Extraction@@ -525,13 +474,13 @@ mod tests {let plan = LogicalPlanBuilder::from(table_scan).aggregate(vec![col("user")],- vec![count(mock_leaf(col("user"), "value"))],+ vec![count(leaf_udf(col("user"), "value"))],)?.build()?;assert_stages!(plan, @r#"## Original Plan- Aggregate: groupBy=[[test.user]], aggr=[[COUNT(mock_leaf(test.user, Utf8("value")))]]+ Aggregate: groupBy=[[test.user]], aggr=[[COUNT(leaf_udf(test.user, Utf8("value")))]]TableScan: test projection=[user]## After Extraction@@ -549,14 +498,14 @@ mod tests {fn test_projection_with_filter_combined() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "status").eq(lit("active")))?- .project(vec![mock_leaf(col("user"), "name")])?+ .filter(leaf_udf(col("user"), "status").eq(lit("active")))?+ .project(vec![leaf_udf(col("user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name"))- Filter: mock_leaf(test.user, Utf8("status")) = Utf8("active")+ Projection: leaf_udf(test.user, Utf8("name"))+ Filter: leaf_udf(test.user, Utf8("status")) = Utf8("active")TableScan: test projection=[user]## After Extraction@@ -574,12 +523,12 @@ mod tests {fn test_projection_preserves_alias() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .project(vec![mock_leaf(col("user"), "name").alias("username")])?+ .project(vec![leaf_udf(col("user"), "name").alias("username")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name")) AS username+ Projection: leaf_udf(test.user, Utf8("name")) AS usernameTableScan: test projection=[user]## After Extraction@@ -600,14 +549,14 @@ mod tests {fn test_projection_different_field_from_filter() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "value").gt(lit(150)))?- .project(vec![col("user"), mock_leaf(col("user"), "label")])?+ .filter(leaf_udf(col("user"), "value").gt(lit(150)))?+ .project(vec![col("user"), leaf_udf(col("user"), "label")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: test.user, mock_leaf(test.user, Utf8("label"))- Filter: mock_leaf(test.user, Utf8("value")) > Int32(150)+ Projection: test.user, leaf_udf(test.user, Utf8("label"))+ Filter: leaf_udf(test.user, Utf8("value")) > Int32(150)TableScan: test projection=[user]## After Extraction@@ -624,14 +573,14 @@ mod tests {#[test]fn test_projection_deduplication() -> Result<()> {let table_scan = test_table_scan_with_struct()?;- let field = mock_leaf(col("user"), "name");+ let field = leaf_udf(col("user"), "name");let plan = LogicalPlanBuilder::from(table_scan).project(vec![field.clone(), field.clone().alias("name2")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name")), mock_leaf(test.user, Utf8("name")) AS name2+ Projection: leaf_udf(test.user, Utf8("name")), leaf_udf(test.user, Utf8("name")) AS name2TableScan: test projection=[user]## After Extraction@@ -655,12 +604,12 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).sort(vec![col("user").sort(true, true)])?- .project(vec![mock_leaf(col("user"), "name")])?+ .project(vec![leaf_udf(col("user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name"))+ Projection: leaf_udf(test.user, Utf8("name"))Sort: test.user ASC NULLS FIRSTTableScan: test projection=[user]@@ -681,12 +630,12 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).limit(0, Some(10))?- .project(vec![mock_leaf(col("user"), "name")])?+ .project(vec![leaf_udf(col("user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name"))+ Projection: leaf_udf(test.user, Utf8("name"))Limit: skip=0, fetch=10TableScan: test projection=[user]@@ -710,13 +659,13 @@ mod tests {let plan = LogicalPlanBuilder::from(table_scan).aggregate(vec![col("user")],- vec![count(mock_leaf(col("user"), "value")).alias("cnt")],+ vec![count(leaf_udf(col("user"), "value")).alias("cnt")],)?.build()?;assert_stages!(plan, @r#"## Original Plan- Aggregate: groupBy=[[test.user]], aggr=[[COUNT(mock_leaf(test.user, Utf8("value"))) AS cnt]]+ Aggregate: groupBy=[[test.user]], aggr=[[COUNT(leaf_udf(test.user, Utf8("value"))) AS cnt]]TableScan: test projection=[user]## After Extraction@@ -762,14 +711,14 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).project(vec![- mock_leaf(col("user"), "name").alias("__datafusion_extracted_manual"),+ leaf_udf(col("user"), "name").alias("__datafusion_extracted_manual"),col("user"),])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name")) AS __datafusion_extracted_manual, test.user+ Projection: leaf_udf(test.user, Utf8("name")) AS __datafusion_extracted_manual, test.userTableScan: test projection=[user]## After Extraction@@ -788,14 +737,14 @@ mod tests {fn test_merge_into_existing_extracted_projection() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "status").eq(lit("active")))?- .filter(mock_leaf(col("user"), "name").is_not_null())?+ .filter(leaf_udf(col("user"), "status").eq(lit("active")))?+ .filter(leaf_udf(col("user"), "name").is_not_null())?.build()?;assert_stages!(plan, @r#"## Original Plan- Filter: mock_leaf(test.user, Utf8("name")) IS NOT NULL- Filter: mock_leaf(test.user, Utf8("status")) = Utf8("active")+ Filter: leaf_udf(test.user, Utf8("name")) IS NOT NULL+ Filter: leaf_udf(test.user, Utf8("status")) = Utf8("active")TableScan: test projection=[id, user]## After Extraction@@ -815,12 +764,12 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).project(vec![col("user")])?- .project(vec![mock_leaf(col("user"), "name")])?+ .project(vec![leaf_udf(col("user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("name"))+ Projection: leaf_udf(test.user, Utf8("name"))TableScan: test projection=[user]## After Extraction@@ -889,14 +838,14 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "status").eq(lit("active")))?- .aggregate(vec![mock_leaf(col("user"), "name")], vec![count(lit(1))])?+ .filter(leaf_udf(col("user"), "status").eq(lit("active")))?+ .aggregate(vec![leaf_udf(col("user"), "name")], vec![count(lit(1))])?.build()?;assert_stages!(plan, @r#"## Original Plan- Aggregate: groupBy=[[mock_leaf(test.user, Utf8("name"))]], aggr=[[COUNT(Int32(1))]]- Filter: mock_leaf(test.user, Utf8("status")) = Utf8("active")+ Aggregate: groupBy=[[leaf_udf(test.user, Utf8("name"))]], aggr=[[COUNT(Int32(1))]]+ Filter: leaf_udf(test.user, Utf8("status")) = Utf8("active")TableScan: test projection=[user]## After Extraction@@ -915,14 +864,14 @@ mod tests {fn test_merge_with_new_columns() -> Result<()> {let table_scan = test_table_scan()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("a"), "x").eq(lit(1)))?- .filter(mock_leaf(col("b"), "y").eq(lit(2)))?+ .filter(leaf_udf(col("a"), "x").eq(lit(1)))?+ .filter(leaf_udf(col("b"), "y").eq(lit(2)))?.build()?;assert_stages!(plan, @r#"## Original Plan- Filter: mock_leaf(test.b, Utf8("y")) = Int32(2)- Filter: mock_leaf(test.a, Utf8("x")) = Int32(1)+ Filter: leaf_udf(test.b, Utf8("y")) = Int32(2)+ Filter: leaf_udf(test.a, Utf8("x")) = Int32(1)TableScan: test projection=[a, b, c]## After Extraction@@ -960,8 +909,8 @@ mod tests {right,JoinType::Inner,(- vec![mock_leaf(col("user"), "id")],- vec![mock_leaf(col("user"), "id")],+ vec![leaf_udf(col("user"), "id")],+ vec![leaf_udf(col("user"), "id")],),None,)?@@ -969,7 +918,7 @@ mod tests {assert_stages!(plan, @r#"## Original Plan- Inner Join: mock_leaf(test.user, Utf8("id")) = mock_leaf(right.user, Utf8("id"))+ Inner Join: leaf_udf(test.user, Utf8("id")) = leaf_udf(right.user, Utf8("id"))TableScan: test projection=[id, user]TableScan: right projection=[id, user]@@ -998,14 +947,14 @@ mod tests {JoinType::Inner,vec![col("test.user").eq(col("right.user")),- mock_leaf(col("test.user"), "status").eq(lit("active")),+ leaf_udf(col("test.user"), "status").eq(lit("active")),],)?.build()?;assert_stages!(plan, @r#"## Original Plan- Inner Join: Filter: test.user = right.user AND mock_leaf(test.user, Utf8("status")) = Utf8("active")+ Inner Join: Filter: test.user = right.user AND leaf_udf(test.user, Utf8("status")) = Utf8("active")TableScan: test projection=[id, user]TableScan: right projection=[id, user]@@ -1034,15 +983,15 @@ mod tests {JoinType::Inner,vec![col("test.user").eq(col("right.user")),- mock_leaf(col("test.user"), "status").eq(lit("active")),- mock_leaf(col("right.user"), "role").eq(lit("admin")),+ leaf_udf(col("test.user"), "status").eq(lit("active")),+ leaf_udf(col("right.user"), "role").eq(lit("admin")),],)?.build()?;assert_stages!(plan, @r#"## Original Plan- Inner Join: Filter: test.user = right.user AND mock_leaf(test.user, Utf8("status")) = Utf8("active") AND mock_leaf(right.user, Utf8("role")) = Utf8("admin")+ Inner Join: Filter: test.user = right.user AND leaf_udf(test.user, Utf8("status")) = Utf8("active") AND leaf_udf(right.user, Utf8("role")) = Utf8("admin")TableScan: test projection=[id, user]TableScan: right projection=[id, user]@@ -1099,18 +1048,18 @@ mod tests {right,JoinType::Inner,(- vec![mock_leaf(col("user"), "id")],- vec![mock_leaf(col("user"), "id")],+ vec![leaf_udf(col("user"), "id")],+ vec![leaf_udf(col("user"), "id")],),None,)?- .filter(mock_leaf(col("test.user"), "status").eq(lit("active")))?+ .filter(leaf_udf(col("test.user"), "status").eq(lit("active")))?.build()?;assert_stages!(plan, @r#"## Original Plan- Filter: mock_leaf(test.user, Utf8("status")) = Utf8("active")- Inner Join: mock_leaf(test.user, Utf8("id")) = mock_leaf(right.user, Utf8("id"))+ Filter: leaf_udf(test.user, Utf8("status")) = Utf8("active")+ Inner Join: leaf_udf(test.user, Utf8("id")) = leaf_udf(right.user, Utf8("id"))TableScan: test projection=[id, user]TableScan: right projection=[id, user]@@ -1137,14 +1086,14 @@ mod tests {let plan = LogicalPlanBuilder::from(left).join(right, JoinType::Inner, (vec!["id"], vec!["id"]), None)?.project(vec![- mock_leaf(col("test.user"), "status"),- mock_leaf(col("right.user"), "role"),+ leaf_udf(col("test.user"), "status"),+ leaf_udf(col("right.user"), "role"),])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(test.user, Utf8("status")), mock_leaf(right.user, Utf8("role"))+ Projection: leaf_udf(test.user, Utf8("status")), leaf_udf(right.user, Utf8("role"))Inner Join: test.id = right.idTableScan: test projection=[id, user]TableScan: right projection=[id, user]@@ -1171,12 +1120,12 @@ mod tests {let plan = LogicalPlanBuilder::from(table_scan).project(vec![col("user").alias("x")])?.filter(col("x").is_not_null())?- .project(vec![mock_leaf(col("x"), "a")])?+ .project(vec![leaf_udf(col("x"), "a")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(x, Utf8("a"))+ Projection: leaf_udf(x, Utf8("a"))Filter: x IS NOT NULLProjection: test.user AS xTableScan: test projection=[user]@@ -1199,12 +1148,12 @@ mod tests {let plan = LogicalPlanBuilder::from(table_scan).project(vec![col("user").alias("x")])?.filter(col("x").is_not_null())?- .project(vec![mock_leaf(col("x"), "a").is_not_null()])?+ .project(vec![leaf_udf(col("x"), "a").is_not_null()])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(x, Utf8("a")) IS NOT NULL+ Projection: leaf_udf(x, Utf8("a")) IS NOT NULLFilter: x IS NOT NULLProjection: test.user AS xTableScan: test projection=[user]@@ -1226,12 +1175,12 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).project(vec![col("user").alias("x")])?- .filter(mock_leaf(col("x"), "a").eq(lit("active")))?+ .filter(leaf_udf(col("x"), "a").eq(lit("active")))?.build()?;assert_stages!(plan, @r#"## Original Plan- Filter: mock_leaf(x, Utf8("a")) = Utf8("active")+ Filter: leaf_udf(x, Utf8("a")) = Utf8("active")Projection: test.user AS xTableScan: test projection=[user]@@ -1256,12 +1205,12 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).alias("sub")?- .project(vec![mock_leaf(col("sub.user"), "name")])?+ .project(vec![leaf_udf(col("sub.user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(sub.user, Utf8("name"))+ Projection: leaf_udf(sub.user, Utf8("name"))SubqueryAlias: subTableScan: test projection=[user]@@ -1282,14 +1231,14 @@ mod tests {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan).alias("sub")?- .filter(mock_leaf(col("sub.user"), "status").eq(lit("active")))?- .project(vec![mock_leaf(col("sub.user"), "name")])?+ .filter(leaf_udf(col("sub.user"), "status").eq(lit("active")))?+ .project(vec![leaf_udf(col("sub.user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(sub.user, Utf8("name"))- Filter: mock_leaf(sub.user, Utf8("status")) = Utf8("active")+ Projection: leaf_udf(sub.user, Utf8("name"))+ Filter: leaf_udf(sub.user, Utf8("status")) = Utf8("active")SubqueryAlias: subTableScan: test projection=[user]@@ -1311,12 +1260,12 @@ mod tests {let plan = LogicalPlanBuilder::from(table_scan).alias("inner_sub")?.alias("outer_sub")?- .project(vec![mock_leaf(col("outer_sub.user"), "name")])?+ .project(vec![leaf_udf(col("outer_sub.user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: mock_leaf(outer_sub.user, Utf8("name"))+ Projection: leaf_udf(outer_sub.user, Utf8("name"))SubqueryAlias: outer_subSubqueryAlias: inner_subTableScan: test projection=[user]@@ -1357,63 +1306,21 @@ mod tests {")}- /// A variant of MockLeafFunc with the same `name()` but a different concrete type.- /// Used to verify that deduplication uses `Expr` equality, not `schema_name`.- #[derive(Debug, PartialEq, Eq, Hash)]- struct MockLeafFuncVariant {- signature: Signature,- }-- impl MockLeafFuncVariant {- fn new() -> Self {- Self {- signature: Signature::new(- TypeSignature::Any(2),- datafusion_expr::Volatility::Immutable,- ),- }- }- }-- impl ScalarUDFImpl for MockLeafFuncVariant {- fn as_any(&self) -> &dyn std::any::Any {- self- }-- fn name(&self) -> &str {- "mock_leaf"- }-- fn signature(&self) -> &Signature {- &self.signature- }-- fn return_type(&self, _args: &[DataType]) -> Result<DataType> {- Ok(DataType::Utf8)- }-- fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {- unimplemented!("This is only used for testing optimization")- }-- fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement {- match args.first() {- Some(ExpressionPlacement::Column)- | Some(ExpressionPlacement::MoveTowardsLeafNodes) => {- ExpressionPlacement::MoveTowardsLeafNodes- }- _ => ExpressionPlacement::KeepInPlace,- }- }- }-/// Two UDFs with the same `name()` but different concrete types should NOT be/// deduplicated -- they are semantically different expressions that happen to/// collide on `schema_name()`.#[test]fn test_different_udfs_same_schema_name_not_deduplicated() -> Result<()> {- let udf_a = Arc::new(ScalarUDF::new_from_impl(MockLeafFunc::new()));- let udf_b = Arc::new(ScalarUDF::new_from_impl(MockLeafFuncVariant::new()));+ let udf_a = Arc::new(ScalarUDF::new_from_impl(+ PlacementTestUDF::new()+ .with_placement(ExpressionPlacement::MoveTowardsLeafNodes)+ .with_id(1),+ ));+ let udf_b = Arc::new(ScalarUDF::new_from_impl(+ PlacementTestUDF::new()+ .with_placement(ExpressionPlacement::MoveTowardsLeafNodes)+ .with_id(2),+ ));let expr_a = Expr::ScalarFunction(ScalarFunction::new_udf(udf_a,@@ -1449,7 +1356,7 @@ mod tests {assert_stages!(plan, @r#"## Original PlanProjection: test.id- Filter: mock_leaf(test.user, Utf8("field")) = Utf8("a") AND mock_leaf(test.user, Utf8("field")) = Utf8("b")+ Filter: leaf_udf(test.user, Utf8("field")) = Utf8("a") AND leaf_udf(test.user, Utf8("field")) = Utf8("b")TableScan: test projection=[id, user]## After Extraction@@ -1473,14 +1380,14 @@ mod tests {fn test_extraction_pushdown_through_filter_with_extracted_predicate() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "status").eq(lit("active")))?- .project(vec![col("id"), mock_leaf(col("user"), "name")])?+ .filter(leaf_udf(col("user"), "status").eq(lit("active")))?+ .project(vec![col("id"), leaf_udf(col("user"), "name")])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: test.id, mock_leaf(test.user, Utf8("name"))- Filter: mock_leaf(test.user, Utf8("status")) = Utf8("active")+ Projection: test.id, leaf_udf(test.user, Utf8("name"))+ Filter: leaf_udf(test.user, Utf8("status")) = Utf8("active")TableScan: test projection=[id, user]## After Extraction@@ -1498,7 +1405,7 @@ mod tests {#[test]fn test_extraction_pushdown_same_expr_in_filter_and_projection() -> Result<()> {let table_scan = test_table_scan_with_struct()?;- let field_expr = mock_leaf(col("user"), "status");+ let field_expr = leaf_udf(col("user"), "status");let plan = LogicalPlanBuilder::from(table_scan).filter(field_expr.clone().gt(lit(5)))?.project(vec![col("id"), field_expr])?@@ -1506,8 +1413,8 @@ mod tests {assert_stages!(plan, @r#"## Original Plan- Projection: test.id, mock_leaf(test.user, Utf8("status"))- Filter: mock_leaf(test.user, Utf8("status")) > Int32(5)+ Projection: test.id, leaf_udf(test.user, Utf8("status"))+ Filter: leaf_udf(test.user, Utf8("status")) > Int32(5)TableScan: test projection=[id, user]## After Extraction@@ -1536,20 +1443,20 @@ mod tests {JoinType::Left,vec![col("test.id").eq(col("right.id")),- mock_leaf(col("right.user"), "status").gt(lit(5)),+ leaf_udf(col("right.user"), "status").gt(lit(5)),],)?.project(vec![col("test.id"),- mock_leaf(col("test.user"), "name"),- mock_leaf(col("right.user"), "status"),+ leaf_udf(col("test.user"), "name"),+ leaf_udf(col("right.user"), "status"),])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: test.id, mock_leaf(test.user, Utf8("name")), mock_leaf(right.user, Utf8("status"))- Left Join: Filter: test.id = right.id AND mock_leaf(right.user, Utf8("status")) > Int32(5)+ Projection: test.id, leaf_udf(test.user, Utf8("name")), leaf_udf(right.user, Utf8("status"))+ Left Join: Filter: test.id = right.id AND leaf_udf(right.user, Utf8("status")) > Int32(5)TableScan: test projection=[id, user]TableScan: right projection=[id, user]@@ -1570,18 +1477,18 @@ mod tests {fn test_pure_extraction_proj_push_through_filter() -> Result<()> {let table_scan = test_table_scan_with_struct()?;let plan = LogicalPlanBuilder::from(table_scan)- .filter(mock_leaf(col("user"), "status").gt(lit(5)))?+ .filter(leaf_udf(col("user"), "status").gt(lit(5)))?.project(vec![col("id"),- mock_leaf(col("user"), "name"),- mock_leaf(col("user"), "status"),+ leaf_udf(col("user"), "name"),+ leaf_udf(col("user"), "status"),])?.build()?;assert_stages!(plan, @r#"## Original Plan- Projection: test.id, mock_leaf(test.user, Utf8("name")), mock_leaf(test.user, Utf8("status"))- Filter: mock_leaf(test.user, Utf8("status")) > Int32(5)+ Projection: test.id, leaf_udf(test.user, Utf8("name")), leaf_udf(test.user, Utf8("status"))+ Filter: leaf_udf(test.user, Utf8("status")) > Int32(5)TableScan: test projection=[id, user]## After Extractiondiff --git a/datafusion/optimizer/src/test/udfs.rs b/datafusion/optimizer/src/test/udfs.rsindex 0e68568dec..9164603dba 100644--- a/datafusion/optimizer/src/test/udfs.rs+++ b/datafusion/optimizer/src/test/udfs.rs@@ -21,7 +21,7 @@ use arrow::datatypes::DataType;use datafusion_common::Result;use datafusion_expr::{ColumnarValue, Expr, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF,- ScalarUDFImpl, Signature, Volatility,+ ScalarUDFImpl, Signature, TypeSignature,};/// A configurable test UDF for optimizer tests.@@ -30,6 +30,7 @@ use datafusion_expr::{pub struct PlacementTestUDF {signature: Signature,placement: ExpressionPlacement,+ id: usize,}impl Default for PlacementTestUDF {@@ -41,15 +42,30 @@ impl Default for PlacementTestUDF {impl PlacementTestUDF {pub fn new() -> Self {Self {- signature: Signature::exact(vec![DataType::UInt32], Volatility::Immutable),+ // Accept any one or two arguments and return UInt32 for testing purposes.+ // The actual types don't matter since this UDF is not intended for execution.+ signature: Signature::new(+ TypeSignature::OneOf(vec![TypeSignature::Any(1), TypeSignature::Any(2)]),+ datafusion_expr::Volatility::Immutable,+ ),placement: ExpressionPlacement::MoveTowardsLeafNodes,+ id: 0,}}+ /// Set the expression placement for this UDF, which is used by optimizer rules to determine where in the plan the expression should be placed.+ /// This also resets the name of the UDF to a default based on the placement.pub fn with_placement(mut self, placement: ExpressionPlacement) -> Self {self.placement = placement;self}++ /// Set the id of the UDF.+ /// This is an arbitrary made up field to allow creating multiple distinct UDFs with the same placement.+ pub fn with_id(mut self, id: usize) -> Self {+ self.id = id;+ self+ }}impl ScalarUDFImpl for PlacementTestUDF {