Apache DataFusion SQL Query Engine
feat: support pushdown alias on dynamic filter with `ProjectionExec` (#19404)
discord93b44bbf60c47
diff --git a/Cargo.lock b/Cargo.lockindex 3dc276d7c2..63460999bb 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1764,6 +1764,7 @@ dependencies = ["parking_lot","parquet","paste",+ "pretty_assertions","rand 0.9.2","rand_distr","recursive",diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex 5c7e944e59..89a22ac4ab 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -175,6 +175,7 @@ env_logger = { workspace = true }glob = { workspace = true }insta = { workspace = true }paste = { workspace = true }+pretty_assertions = "1.0"rand = { workspace = true, features = ["small_rng"] }rand_distr = "0.5"recursive = { workspace = true }diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rsindex 5f1971f649..bb9e03c837 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs@@ -59,6 +59,7 @@ use datafusion_physical_plan::{coalesce_partitions::CoalescePartitionsExec,collect,filter::{FilterExec, FilterExecBuilder},+ projection::ProjectionExec,repartition::RepartitionExec,sorts::sort::SortExec,};@@ -1826,6 +1827,234 @@ fn schema() -> SchemaRef {Arc::clone(&TEST_SCHEMA)}+struct ProjectionDynFilterTestCase {+ schema: SchemaRef,+ batches: Vec<RecordBatch>,+ projection: Vec<(Arc<dyn PhysicalExpr>, String)>,+ sort_expr: PhysicalSortExpr,+ expected_plans: Vec<String>,+}++async fn run_projection_dyn_filter_case(case: ProjectionDynFilterTestCase) {+ let ProjectionDynFilterTestCase {+ schema,+ batches,+ projection,+ sort_expr,+ expected_plans,+ } = case;++ let scan = TestScanBuilder::new(Arc::clone(&schema))+ .with_support(true)+ .with_batches(batches)+ .build();++ let projection_exec = Arc::new(ProjectionExec::try_new(projection, scan).unwrap());++ let sort = Arc::new(+ SortExec::new(LexOrdering::new(vec![sort_expr]).unwrap(), projection_exec)+ .with_fetch(Some(2)),+ ) as Arc<dyn ExecutionPlan>;++ let mut config = ConfigOptions::default();+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_dynamic_filter_pushdown = true;++ let optimized_plan = FilterPushdown::new_post_optimization()+ .optimize(Arc::clone(&sort), &config)+ .unwrap();++ pretty_assertions::assert_eq!(+ format_plan_for_test(&optimized_plan).trim(),+ expected_plans[0].trim()+ );++ let config = SessionConfig::new().with_batch_size(2);+ let session_ctx = SessionContext::new_with_config(config);+ session_ctx.register_object_store(+ ObjectStoreUrl::parse("test://").unwrap().as_ref(),+ Arc::new(InMemory::new()),+ );+ let state = session_ctx.state();+ let task_ctx = state.task_ctx();+ let mut stream = optimized_plan.execute(0, Arc::clone(&task_ctx)).unwrap();+ for (idx, expected_plan) in expected_plans.iter().enumerate().skip(1) {+ stream.next().await.unwrap().unwrap();+ let formatted_plan = format_plan_for_test(&optimized_plan);+ pretty_assertions::assert_eq!(+ formatted_plan.trim(),+ expected_plan.trim(),+ "Mismatch at iteration {}",+ idx+ );+ }+}++#[tokio::test]+async fn test_topk_with_projection_transformation_on_dyn_filter() {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Utf8, false),+ Field::new("c", DataType::Float64, false),+ ]));+ let simple_abc = vec![+ record_batch!(+ ("a", Int32, [1, 2, 3]),+ ("b", Utf8, ["x", "y", "z"]),+ ("c", Float64, [1.0, 2.0, 3.0])+ )+ .unwrap(),+ ];++ // Case 1: Reordering [b, a]+ run_projection_dyn_filter_case(ProjectionDynFilterTestCase {+ schema: Arc::clone(&schema),+ batches: simple_abc.clone(),+ projection: vec![+ (col("b", &schema).unwrap(), "b".to_string()),+ (col("a", &schema).unwrap(), "a".to_string()),+ ],+ sort_expr: PhysicalSortExpr::new(+ Arc::new(Column::new("a", 1)),+ SortOptions::default(),+ ),+ expected_plans: vec![+r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false]+ - ProjectionExec: expr=[b@1 as b, a@0 as a]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),+r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false], filter=[a@1 IS NULL OR a@1 < 2]+ - ProjectionExec: expr=[b@1 as b, a@0 as a]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string()]+ })+ .await;++ // Case 2: Pruning [a]+ run_projection_dyn_filter_case(ProjectionDynFilterTestCase {+ schema: Arc::clone(&schema),+ batches: simple_abc.clone(),+ projection: vec![(col("a", &schema).unwrap(), "a".to_string())],+ sort_expr: PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ SortOptions::default(),+ ),+ expected_plans: vec![+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]+ - ProjectionExec: expr=[a@0 as a]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),+ ],+ })+ .await;++ // Case 3: Identity [a, b]+ run_projection_dyn_filter_case(ProjectionDynFilterTestCase {+ schema: Arc::clone(&schema),+ batches: simple_abc.clone(),+ projection: vec![+ (col("a", &schema).unwrap(), "a".to_string()),+ (col("b", &schema).unwrap(), "b".to_string()),+ ],+ sort_expr: PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ SortOptions::default(),+ ),+ expected_plans: vec![+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),+ ],+ })+ .await;++ // Case 4: Expressions [a + 1, b]+ run_projection_dyn_filter_case(ProjectionDynFilterTestCase {+ schema: Arc::clone(&schema),+ batches: simple_abc.clone(),+ projection: vec![+ (+ Arc::new(BinaryExpr::new(+ col("a", &schema).unwrap(),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ )),+ "a_plus_1".to_string(),+ ),+ (col("b", &schema).unwrap(), "b".to_string()),+ ],+ sort_expr: PhysicalSortExpr::new(+ Arc::new(Column::new("a_plus_1", 0)),+ SortOptions::default(),+ ),+ expected_plans: vec![+ r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),+ r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false], filter=[a_plus_1@0 IS NULL OR a_plus_1@0 < 3]+ - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),+ ],+ })+ .await;++ // Case 5: [a as b, b as a] (swapped columns)+ run_projection_dyn_filter_case(ProjectionDynFilterTestCase {+ schema: Arc::clone(&schema),+ batches: simple_abc.clone(),+ projection: vec![+ (col("a", &schema).unwrap(), "b".to_string()),+ (col("b", &schema).unwrap(), "a".to_string()),+ ],+ sort_expr: PhysicalSortExpr::new(+ Arc::new(Column::new("b", 0)),+ SortOptions::default(),+ ),+ expected_plans: vec![+ r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as b, b@1 as a]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),+ r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false], filter=[b@0 IS NULL OR b@0 < 2]+ - ProjectionExec: expr=[a@0 as b, b@1 as a]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),+ ],+ })+ .await;++ // Case 6: Confusing expr [a + 1 as a, b]+ run_projection_dyn_filter_case(ProjectionDynFilterTestCase {+ schema: Arc::clone(&schema),+ batches: simple_abc.clone(),+ projection: vec![+ (+ Arc::new(BinaryExpr::new(+ col("a", &schema).unwrap(),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ )),+ "a".to_string(),+ ),+ (col("b", &schema).unwrap(), "b".to_string()),+ ],+ sort_expr: PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ SortOptions::default(),+ ),+ expected_plans: vec![+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),+ r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 3]+ - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),+ ],+ })+ .await;+}+/// Returns a predicate that is a binary expression col = litfn col_lit_predicate(column_name: &str,diff --git a/datafusion/physical-plan/src/column_rewriter.rs b/datafusion/physical-plan/src/column_rewriter.rsnew file mode 100644index 0000000000..7cd8656304--- /dev/null+++ b/datafusion/physical-plan/src/column_rewriter.rs@@ -0,0 +1,383 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::sync::Arc;++use datafusion_common::{+ DataFusionError, HashMap,+ tree_node::{Transformed, TreeNodeRecursion, TreeNodeRewriter},+};+use datafusion_physical_expr::{PhysicalExpr, expressions::Column};++/// Rewrite column references in a physical expr according to a mapping.+///+/// This rewriter traverses the expression tree and replaces [`Column`] nodes+/// with the corresponding expression found in the `column_map`.+///+/// If a column is found in the map, it is replaced by the mapped expression.+/// If a column is NOT found in the map, a `DataFusionError::Internal` is+/// returned.+pub struct PhysicalColumnRewriter<'a> {+ /// Mapping from original column to new column.+ pub column_map: &'a HashMap<Column, Arc<dyn PhysicalExpr>>,+}++impl<'a> PhysicalColumnRewriter<'a> {+ /// Create a new PhysicalColumnRewriter with the given column mapping.+ pub fn new(column_map: &'a HashMap<Column, Arc<dyn PhysicalExpr>>) -> Self {+ Self { column_map }+ }+}++impl<'a> TreeNodeRewriter for PhysicalColumnRewriter<'a> {+ type Node = Arc<dyn PhysicalExpr>;++ fn f_down(+ &mut self,+ node: Self::Node,+ ) -> datafusion_common::Result<Transformed<Self::Node>> {+ if let Some(column) = node.as_any().downcast_ref::<Column>() {+ if let Some(new_column) = self.column_map.get(column) {+ // jump to prevent rewriting the new sub-expression again+ return Ok(Transformed::new(+ Arc::clone(new_column),+ true,+ TreeNodeRecursion::Jump,+ ));+ } else {+ // Column not found in mapping+ return Err(DataFusionError::Internal(format!(+ "Column {column:?} not found in column mapping {:?}",+ self.column_map+ )));+ }+ }+ Ok(Transformed::no(node))+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::{DataFusionError, Result, tree_node::TreeNode};+ use datafusion_physical_expr::{+ PhysicalExpr,+ expressions::{Column, binary, col, lit},+ };+ use std::sync::Arc;++ /// Helper function to create a test schema+ fn create_test_schema() -> Arc<Schema> {+ Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ Field::new("c", DataType::Int32, true),+ Field::new("d", DataType::Int32, true),+ Field::new("e", DataType::Int32, true),+ Field::new("new_col", DataType::Int32, true),+ Field::new("inner_col", DataType::Int32, true),+ Field::new("another_col", DataType::Int32, true),+ ]))+ }++ /// Helper function to create a complex nested expression with multiple columns+ /// Create: (col_a + col_b) * (col_c - col_d) + col_e+ fn create_complex_expression(schema: &Schema) -> Arc<dyn PhysicalExpr> {+ let col_a = col("a", schema).unwrap();+ let col_b = col("b", schema).unwrap();+ let col_c = col("c", schema).unwrap();+ let col_d = col("d", schema).unwrap();+ let col_e = col("e", schema).unwrap();++ let add_expr =+ binary(col_a, datafusion_expr::Operator::Plus, col_b, schema).unwrap();+ let sub_expr =+ binary(col_c, datafusion_expr::Operator::Minus, col_d, schema).unwrap();+ let mul_expr = binary(+ add_expr,+ datafusion_expr::Operator::Multiply,+ sub_expr,+ schema,+ )+ .unwrap();+ binary(mul_expr, datafusion_expr::Operator::Plus, col_e, schema).unwrap()+ }++ /// Helper function to create a deeply nested expression+ /// Create: col_a + (col_b + (col_c + (col_d + col_e)))+ fn create_deeply_nested_expression(schema: &Schema) -> Arc<dyn PhysicalExpr> {+ let col_a = col("a", schema).unwrap();+ let col_b = col("b", schema).unwrap();+ let col_c = col("c", schema).unwrap();+ let col_d = col("d", schema).unwrap();+ let col_e = col("e", schema).unwrap();++ let inner1 =+ binary(col_d, datafusion_expr::Operator::Plus, col_e, schema).unwrap();+ let inner2 =+ binary(col_c, datafusion_expr::Operator::Plus, inner1, schema).unwrap();+ let inner3 =+ binary(col_b, datafusion_expr::Operator::Plus, inner2, schema).unwrap();+ binary(col_a, datafusion_expr::Operator::Plus, inner3, schema).unwrap()+ }++ #[test]+ fn test_simple_column_replacement_with_jump() -> Result<()> {+ let schema = create_test_schema();++ // Test that Jump prevents re-processing of replaced columns+ let mut column_map = HashMap::new();+ column_map.insert(Column::new_with_schema("a", &schema).unwrap(), lit(42i32));+ column_map.insert(+ Column::new_with_schema("b", &schema).unwrap(),+ lit("replaced_b"),+ );+ column_map.insert(+ Column::new_with_schema("c", &schema).unwrap(),+ col("c", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("d", &schema).unwrap(),+ col("d", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("e", &schema).unwrap(),+ col("e", &schema).unwrap(),+ );++ let mut rewriter = PhysicalColumnRewriter::new(&column_map);+ let expr = create_complex_expression(&schema);++ let result = expr.rewrite(&mut rewriter)?;++ // Verify the transformation occurred+ assert!(result.transformed);++ assert_eq!(+ format!("{}", result.data),+ "(42 + replaced_b) * (c@2 - d@3) + e@4"+ );++ Ok(())+ }++ #[test]+ fn test_nested_column_replacement_with_jump() -> Result<()> {+ let schema = create_test_schema();+ // Test Jump behavior with deeply nested expressions+ let mut column_map = HashMap::new();+ // Replace col_c with a complex expression containing new columns+ let replacement_expr = binary(+ lit(100i32),+ datafusion_expr::Operator::Plus,+ col("new_col", &schema).unwrap(),+ &schema,+ )+ .unwrap();+ column_map.insert(+ Column::new_with_schema("c", &schema).unwrap(),+ replacement_expr,+ );+ column_map.insert(+ Column::new_with_schema("a", &schema).unwrap(),+ col("a", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("b", &schema).unwrap(),+ col("b", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("d", &schema).unwrap(),+ col("d", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("e", &schema).unwrap(),+ col("e", &schema).unwrap(),+ );++ let mut rewriter = PhysicalColumnRewriter::new(&column_map);+ let expr = create_deeply_nested_expression(&schema);++ let result = expr.rewrite(&mut rewriter)?;++ // Verify transformation occurred+ assert!(result.transformed);++ assert_eq!(+ format!("{}", result.data),+ "a@0 + b@1 + 100 + new_col@5 + d@3 + e@4"+ );++ Ok(())+ }++ #[test]+ fn test_circular_reference_prevention() -> Result<()> {+ let schema = create_test_schema();+ // Test that Jump prevents infinite recursion with circular references+ let mut column_map = HashMap::new();++ // Create a circular reference: col_a -> col_b -> col_a (but Jump should prevent the second visit)+ column_map.insert(+ Column::new_with_schema("a", &schema).unwrap(),+ col("b", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("b", &schema).unwrap(),+ col("a", &schema).unwrap(),+ );++ let mut rewriter = PhysicalColumnRewriter::new(&column_map);++ // Start with an expression containing col_a+ let expr = binary(+ col("a", &schema).unwrap(),+ datafusion_expr::Operator::Plus,+ col("b", &schema).unwrap(),+ &schema,+ )+ .unwrap();++ let result = expr.rewrite(&mut rewriter)?;++ // Verify transformation occurred+ assert!(result.transformed);++ assert_eq!(format!("{}", result.data), "b@1 + a@0");++ Ok(())+ }++ #[test]+ fn test_multiple_replacements_in_same_expression() -> Result<()> {+ let schema = create_test_schema();+ // Test multiple column replacements in the same complex expression+ let mut column_map = HashMap::new();++ // Replace multiple columns with literals+ column_map.insert(Column::new_with_schema("a", &schema).unwrap(), lit(10i32));+ column_map.insert(Column::new_with_schema("c", &schema).unwrap(), lit(20i32));+ column_map.insert(Column::new_with_schema("e", &schema).unwrap(), lit(30i32));+ column_map.insert(+ Column::new_with_schema("b", &schema).unwrap(),+ col("b", &schema).unwrap(),+ );+ column_map.insert(+ Column::new_with_schema("d", &schema).unwrap(),+ col("d", &schema).unwrap(),+ );++ let mut rewriter = PhysicalColumnRewriter::new(&column_map);+ let expr = create_complex_expression(&schema); // (col_a + col_b) * (col_c - col_d) + col_e++ let result = expr.rewrite(&mut rewriter)?;++ // Verify transformation occurred+ assert!(result.transformed);+ assert_eq!(format!("{}", result.data), "(10 + b@1) * (20 - d@3) + 30");++ Ok(())+ }++ #[test]+ fn test_jump_with_complex_replacement_expression() -> Result<()> {+ let schema = create_test_schema();+ // Test Jump behavior when replacing with very complex expressions+ let mut column_map = HashMap::new();++ // Replace col_a with a complex nested expression+ let inner_expr = binary(+ lit(5i32),+ datafusion_expr::Operator::Multiply,+ col("a", &schema).unwrap(),+ &schema,+ )+ .unwrap();+ let middle_expr = binary(+ inner_expr,+ datafusion_expr::Operator::Plus,+ lit(3i32),+ &schema,+ )+ .unwrap();+ let complex_replacement = binary(+ middle_expr,+ datafusion_expr::Operator::Minus,+ col("another_col", &schema).unwrap(),+ &schema,+ )+ .unwrap();++ column_map.insert(+ Column::new_with_schema("a", &schema).unwrap(),+ complex_replacement,+ );+ column_map.insert(+ Column::new_with_schema("b", &schema).unwrap(),+ col("b", &schema).unwrap(),+ );++ let mut rewriter = PhysicalColumnRewriter::new(&column_map);++ // Create expression: col_a + col_b+ let expr = binary(+ col("a", &schema).unwrap(),+ datafusion_expr::Operator::Plus,+ col("b", &schema).unwrap(),+ &schema,+ )+ .unwrap();++ let result = expr.rewrite(&mut rewriter)?;++ assert_eq!(+ format!("{}", result.data),+ "5 * a@0 + 3 - another_col@7 + b@1"+ );++ // Verify transformation occurred+ assert!(result.transformed);++ Ok(())+ }++ #[test]+ fn test_unmapped_columns_detection() -> Result<()> {+ let schema = create_test_schema();+ let mut column_map = HashMap::new();++ // Only map col_a, leave col_b unmapped+ column_map.insert(Column::new_with_schema("a", &schema).unwrap(), lit(42i32));++ let mut rewriter = PhysicalColumnRewriter::new(&column_map);++ // Create expression: col_a + col_b+ let expr = binary(+ col("a", &schema).unwrap(),+ datafusion_expr::Operator::Plus,+ col("b", &schema).unwrap(),+ &schema,+ )+ .unwrap();++ let err = expr.rewrite(&mut rewriter).unwrap_err();+ assert!(matches!(err, DataFusionError::Internal(_)));++ Ok(())+ }+}diff --git a/datafusion/physical-plan/src/filter_pushdown.rs b/datafusion/physical-plan/src/filter_pushdown.rsindex 1274e954ea..37cbd68490 100644--- a/datafusion/physical-plan/src/filter_pushdown.rs+++ b/datafusion/physical-plan/src/filter_pushdown.rs@@ -37,8 +37,12 @@use std::collections::HashSet;use std::sync::Arc;-use datafusion_common::Result;-use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};+use arrow_schema::Schema;+use datafusion_common::{+ Result,+ tree_node::{TreeNode, TreeNodeRecursion},+};+use datafusion_physical_expr::{expressions::Column, utils::reassign_expr_columns};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use itertools::Itertools;@@ -306,6 +310,56 @@ pub struct ChildFilterDescription {pub(crate) self_filters: Vec<Arc<dyn PhysicalExpr>>,}+/// A utility for checking whether a filter expression can be pushed down+/// to a child node based on column availability.+///+/// This checker validates that all columns referenced in a filter expression+/// exist in the target schema. If any column in the filter is not present+/// in the schema, the filter cannot be pushed down to that child.+pub(crate) struct FilterColumnChecker<'a> {+ column_names: HashSet<&'a str>,+}++impl<'a> FilterColumnChecker<'a> {+ /// Creates a new [`FilterColumnChecker`] from the given schema.+ ///+ /// Extracts all column names from the schema's fields to build+ /// a lookup set for efficient column existence checks.+ pub(crate) fn new(input_schema: &'a Schema) -> Self {+ let column_names: HashSet<&str> = input_schema+ .fields()+ .iter()+ .map(|f| f.name().as_str())+ .collect();+ Self { column_names }+ }++ /// Checks whether a filter expression can be pushed down to the child+ /// whose schema was used to create this checker.+ ///+ /// Returns `true` if all [`Column`] references in the filter expression+ /// exist in the target schema, `false` otherwise.+ ///+ /// This method traverses the entire expression tree, checking each+ /// column reference against the available column names.+ pub(crate) fn can_pushdown(&self, filter: &Arc<dyn PhysicalExpr>) -> bool {+ let mut can_apply = true;+ filter+ .apply(|expr| {+ if let Some(column) = expr.as_any().downcast_ref::<Column>()+ && !self.column_names.contains(column.name())+ {+ can_apply = false;+ return Ok(TreeNodeRecursion::Stop);+ }++ Ok(TreeNodeRecursion::Continue)+ })+ .expect("infallible traversal");+ can_apply+ }+}+impl ChildFilterDescription {/// Build a child filter description by analyzing which parent filters can be pushed to a specific child.///@@ -320,26 +374,14 @@ impl ChildFilterDescription {) -> Result<Self> {let child_schema = child.schema();- // Get column names from child schema for quick lookup- let child_column_names: HashSet<&str> = child_schema- .fields()- .iter()- .map(|f| f.name().as_str())- .collect();+ // Build a set of column names in the child schema for quick lookup+ let checker = FilterColumnChecker::new(&child_schema);// Analyze each parent filterlet mut child_parent_filters = Vec::with_capacity(parent_filters.len());for filter in parent_filters {- // Check which columns the filter references- let referenced_columns = collect_columns(filter);-- // Check if all referenced columns exist in the child schema- let all_columns_exist = referenced_columns- .iter()- .all(|col| child_column_names.contains(col.name()));-- if all_columns_exist {+ if checker.can_pushdown(filter) {// All columns exist in child - we can push down// Need to reassign column indices to match child schemalet reassigned_filter =diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex ec8e154cae..9352a143c1 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -68,6 +68,7 @@ pub mod async_func;pub mod coalesce;pub mod coalesce_batches;pub mod coalesce_partitions;+pub mod column_rewriter;pub mod common;pub mod coop;pub mod display;diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 8f2f2219f4..8d4c775f87 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -26,10 +26,11 @@ use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream,SendableRecordBatchStream, SortOrderPushdownResult, Statistics,};+use crate::column_rewriter::PhysicalColumnRewriter;use crate::execution_plan::CardinalityEffect;use crate::filter_pushdown::{- ChildPushdownResult, FilterDescription, FilterPushdownPhase,- FilterPushdownPropagation,+ ChildFilterDescription, ChildPushdownResult, FilterColumnChecker, FilterDescription,+ FilterPushdownPhase, FilterPushdownPropagation, PushedDownPredicate,};use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn, JoinOnRef};use crate::{DisplayFormatType, ExecutionPlan, PhysicalExpr};@@ -45,11 +46,11 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};-use datafusion_common::{JoinSide, Result, internal_err};+use datafusion_common::{DataFusionError, JoinSide, Result, internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::projection::Projector;-use datafusion_physical_expr::utils::collect_columns;+use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};use datafusion_physical_expr_common::physical_expr::{PhysicalExprRef, fmt_sql};use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement, PhysicalSortExpr,@@ -192,6 +193,29 @@ impl ProjectionExec {input.boundedness(),))}++ /// Collect reverse alias mapping from projection expressions.+ /// The result hash map is a map from aliased Column in parent to original expr.+ fn collect_reverse_alias(+ &self,+ ) -> Result<datafusion_common::HashMap<Column, Arc<dyn PhysicalExpr>>> {+ let mut alias_map = datafusion_common::HashMap::new();+ for projection in self.projection_expr().iter() {+ let (aliased_index, _output_field) = self+ .projector+ .output_schema()+ .column_with_name(&projection.alias)+ .ok_or_else(|| {+ DataFusionError::Internal(format!(+ "Expr {} with alias {} not found in output schema",+ projection.expr, projection.alias+ ))+ })?;+ let aliased_col = Column::new(&projection.alias, aliased_index);+ alias_map.insert(aliased_col, Arc::clone(&projection.expr));+ }+ Ok(alias_map)+ }}impl DisplayAs for ProjectionExec {@@ -347,10 +371,31 @@ impl ExecutionPlan for ProjectionExec {parent_filters: Vec<Arc<dyn PhysicalExpr>>,_config: &ConfigOptions,) -> Result<FilterDescription> {- // TODO: In future, we can try to handle inverting aliases here.- // For the time being, we pass through untransformed filters, so filters on aliases are not handled.- // https://github.com/apache/datafusion/issues/17246- FilterDescription::from_children(parent_filters, &self.children())+ // expand alias column to original expr in parent filters+ let invert_alias_map = self.collect_reverse_alias()?;+ let output_schema = self.schema();+ let checker = FilterColumnChecker::new(&output_schema);+ let mut child_parent_filters = Vec::with_capacity(parent_filters.len());++ for filter in parent_filters {+ if !checker.can_pushdown(&filter) {+ child_parent_filters.push(PushedDownPredicate::unsupported(filter));+ continue;+ }+ // All columns exist in child - we can push down+ // Need to reassign column indices to match child schema+ let reassigned_filter = reassign_expr_columns(filter, &output_schema)?;+ // rewrite filter expression using invert alias map+ let mut rewriter = PhysicalColumnRewriter::new(&invert_alias_map);+ let rewritten = reassigned_filter.rewrite(&mut rewriter)?.data;++ child_parent_filters.push(PushedDownPredicate::supported(rewritten));+ }++ Ok(FilterDescription::new().with_child(ChildFilterDescription {+ parent_filters: child_parent_filters,+ self_filters: vec![],+ }))}fn handle_child_pushdown_result(@@ -1086,6 +1131,7 @@ mod tests {use crate::common::collect;+ use crate::filter_pushdown::PushedDown;use crate::test;use crate::test::exec::StatisticsExec;@@ -1094,7 +1140,9 @@ mod tests {use datafusion_common::stats::{ColumnStatistics, Precision, Statistics};use datafusion_expr::Operator;- use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal, col};+ use datafusion_physical_expr::expressions::{+ BinaryExpr, Column, DynamicFilterPhysicalExpr, Literal, binary, col, lit,+ };#[test]fn test_collect_column_indices() -> Result<()> {@@ -1283,4 +1331,431 @@ mod tests {);assert!(stats.total_byte_size.is_exact().unwrap_or(false));}++ #[test]+ fn test_filter_pushdown_with_alias() -> Result<()> {+ let input_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let input = Arc::new(StatisticsExec::new(+ Statistics::new_unknown(&input_schema),+ input_schema.clone(),+ ));++ // project "a" as "b"+ let projection = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "b".to_string(),+ }],+ input,+ )?;++ // filter "b > 5"+ let filter = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("b", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![filter],+ &ConfigOptions::default(),+ )?;++ // Should be converted to "a > 5"+ // "a" is index 0 in input+ let expected_filter = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ assert_eq!(description.self_filters(), vec![vec![]]);+ let pushed_filters = &description.parent_filters()[0];+ assert_eq!(+ format!("{}", pushed_filters[0].predicate),+ format!("{}", expected_filter)+ );+ // Verify the predicate was actually pushed down+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::Yes));++ Ok(())+ }++ #[test]+ fn test_filter_pushdown_with_multiple_aliases() -> Result<()> {+ let input_schema = Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ column_statistics: vec![Default::default(); input_schema.fields().len()],+ ..Default::default()+ },+ input_schema.clone(),+ ));++ // project "a" as "x", "b" as "y"+ let projection = ProjectionExec::try_new(+ vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "x".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "y".to_string(),+ },+ ],+ input,+ )?;++ // filter "x > 5"+ let filter1 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("x", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ // filter "y < 10"+ let filter2 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("y", 1)),+ Operator::Lt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),+ )) as Arc<dyn PhysicalExpr>;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![filter1, filter2],+ &ConfigOptions::default(),+ )?;++ // Should be converted to "a > 5" and "b < 10"+ let expected_filter1 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ let expected_filter2 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("b", 1)),+ Operator::Lt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),+ )) as Arc<dyn PhysicalExpr>;++ let pushed_filters = &description.parent_filters()[0];+ assert_eq!(pushed_filters.len(), 2);+ // Note: The order of filters is preserved+ assert_eq!(+ format!("{}", pushed_filters[0].predicate),+ format!("{}", expected_filter1)+ );+ assert_eq!(+ format!("{}", pushed_filters[1].predicate),+ format!("{}", expected_filter2)+ );+ // Verify the predicates were actually pushed down+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::Yes));+ assert!(matches!(pushed_filters[1].discriminant, PushedDown::Yes));++ Ok(())+ }++ #[test]+ fn test_filter_pushdown_with_swapped_aliases() -> Result<()> {+ let input_schema = Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ column_statistics: vec![Default::default(); input_schema.fields().len()],+ ..Default::default()+ },+ input_schema.clone(),+ ));++ // project "a" as "b", "b" as "a"+ let projection = ProjectionExec::try_new(+ vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "b".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "a".to_string(),+ },+ ],+ input,+ )?;++ // filter "b > 5" (output column 0, which is "a" in input)+ let filter1 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("b", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ // filter "a < 10" (output column 1, which is "b" in input)+ let filter2 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 1)),+ Operator::Lt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),+ )) as Arc<dyn PhysicalExpr>;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![filter1, filter2],+ &ConfigOptions::default(),+ )?;++ let pushed_filters = &description.parent_filters()[0];+ assert_eq!(pushed_filters.len(), 2);++ // "b" (output index 0) -> "a" (input index 0)+ let expected_filter1 = "a@0 > 5";+ // "a" (output index 1) -> "b" (input index 1)+ let expected_filter2 = "b@1 < 10";++ assert_eq!(format!("{}", pushed_filters[0].predicate), expected_filter1);+ assert_eq!(format!("{}", pushed_filters[1].predicate), expected_filter2);+ // Verify the predicates were actually pushed down+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::Yes));+ assert!(matches!(pushed_filters[1].discriminant, PushedDown::Yes));++ Ok(())+ }++ #[test]+ fn test_filter_pushdown_with_mixed_columns() -> Result<()> {+ let input_schema = Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ column_statistics: vec![Default::default(); input_schema.fields().len()],+ ..Default::default()+ },+ input_schema.clone(),+ ));++ // project "a" as "x", "b" as "b" (pass through)+ let projection = ProjectionExec::try_new(+ vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "x".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("b", 1)),+ alias: "b".to_string(),+ },+ ],+ input,+ )?;++ // filter "x > 5"+ let filter1 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("x", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ // filter "b < 10" (using output index 1 which corresponds to 'b')+ let filter2 = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("b", 1)),+ Operator::Lt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),+ )) as Arc<dyn PhysicalExpr>;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![filter1, filter2],+ &ConfigOptions::default(),+ )?;++ let pushed_filters = &description.parent_filters()[0];+ assert_eq!(pushed_filters.len(), 2);+ // "x" -> "a" (index 0)+ let expected_filter1 = "a@0 > 5";+ // "b" -> "b" (index 1)+ let expected_filter2 = "b@1 < 10";++ assert_eq!(format!("{}", pushed_filters[0].predicate), expected_filter1);+ assert_eq!(format!("{}", pushed_filters[1].predicate), expected_filter2);+ // Verify the predicates were actually pushed down+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::Yes));+ assert!(matches!(pushed_filters[1].discriminant, PushedDown::Yes));++ Ok(())+ }++ #[test]+ fn test_filter_pushdown_with_complex_expression() -> Result<()> {+ let input_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ column_statistics: vec![Default::default(); input_schema.fields().len()],+ ..Default::default()+ },+ input_schema.clone(),+ ));++ // project "a + 1" as "z"+ let projection = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ )),+ alias: "z".to_string(),+ }],+ input,+ )?;++ // filter "z > 10"+ let filter = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("z", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),+ )) as Arc<dyn PhysicalExpr>;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![filter],+ &ConfigOptions::default(),+ )?;++ // expand to `a + 1 > 10`+ let pushed_filters = &description.parent_filters()[0];+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::Yes));+ assert_eq!(format!("{}", pushed_filters[0].predicate), "a@0 + 1 > 10");++ Ok(())+ }++ #[test]+ fn test_filter_pushdown_with_unknown_column() -> Result<()> {+ let input_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ column_statistics: vec![Default::default(); input_schema.fields().len()],+ ..Default::default()+ },+ input_schema.clone(),+ ));++ // project "a" as "a"+ let projection = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(Column::new("a", 0)),+ alias: "a".to_string(),+ }],+ input,+ )?;++ // filter "unknown_col > 5" - using a column name that doesn't exist in projection output+ // Column constructor: name, index. Index 1 doesn't exist.+ let filter = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("unknown_col", 1)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ )) as Arc<dyn PhysicalExpr>;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![filter],+ &ConfigOptions::default(),+ )?;++ let pushed_filters = &description.parent_filters()[0];+ assert!(matches!(pushed_filters[0].discriminant, PushedDown::No));+ // The column shouldn't be found in the alias map, so it remains unchanged with its index+ assert_eq!(+ format!("{}", pushed_filters[0].predicate),+ "unknown_col@1 > 5"+ );++ Ok(())+ }++ /// Basic test for `DynamicFilterPhysicalExpr` can correctly update its child expression+ /// i.e. starting with lit(true) and after update it becomes `a > 5`+ /// with projection [b - 1 as a], the pushed down filter should be `b - 1 > 5`+ #[test]+ fn test_basic_dyn_filter_projection_pushdown_update_child() -> Result<()> {+ let input_schema =+ Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, false)]));++ let input = Arc::new(StatisticsExec::new(+ Statistics {+ column_statistics: vec![Default::default(); input_schema.fields().len()],+ ..Default::default()+ },+ input_schema.as_ref().clone(),+ ));++ // project "b" - 1 as "a"+ let projection = ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: binary(+ Arc::new(Column::new("b", 0)),+ Operator::Minus,+ lit(1),+ &input_schema,+ )+ .unwrap(),+ alias: "a".to_string(),+ }],+ input,+ )?;++ // simulate projection's parent create a dynamic filter on "a"+ let projected_schema = projection.schema();+ let col_a = col("a", &projected_schema)?;+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::clone(&col_a)],+ lit(true),+ ));+ // Initial state should be lit(true)+ let current = dynamic_filter.current()?;+ assert_eq!(format!("{current}"), "true");++ let dyn_phy_expr: Arc<dyn PhysicalExpr> = Arc::clone(&dynamic_filter) as _;++ let description = projection.gather_filters_for_pushdown(+ FilterPushdownPhase::Post,+ vec![dyn_phy_expr],+ &ConfigOptions::default(),+ )?;++ let pushed_filters = &description.parent_filters()[0][0];++ // Check currently pushed_filters is lit(true)+ assert_eq!(+ format!("{}", pushed_filters.predicate),+ "DynamicFilter [ empty ]"+ );++ // Update to a > 5 (after projection, b is now called a)+ let new_expr =+ Arc::new(BinaryExpr::new(Arc::clone(&col_a), Operator::Gt, lit(5i32)));+ dynamic_filter.update(new_expr)?;++ // Now it should be a > 5+ let current = dynamic_filter.current()?;+ assert_eq!(format!("{current}"), "a@0 > 5");++ // Check currently pushed_filters is b - 1 > 5 (because b - 1 is projected as a)+ assert_eq!(+ format!("{}", pushed_filters.predicate),+ "DynamicFilter [ b@0 - 1 > 5 ]"+ );++ Ok(())+ }}diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex 3e403171e0..38a5b11870 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -92,6 +92,30 @@ physical_plan01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ]+statement ok+set datafusion.explain.analyze_level = summary;++query TT+EXPLAIN ANALYZE SELECT id, value AS v, value + id as name FROM test_parquet where value > 3 ORDER BY v DESC LIMIT 3;+----+Plan with Metrics+01)SortPreservingMergeExec: [v@1 DESC], fetch=3, metrics=[output_rows=3, <slt:ignore>]+02)--SortExec: TopK(fetch=3), expr=[v@1 DESC], preserve_partitioning=[true], filter=[v@1 IS NULL OR v@1 > 800], metrics=[output_rows=3, <slt:ignore>]+03)----ProjectionExec: expr=[id@0 as id, value@1 as v, value@1 + id@0 as name], metrics=[output_rows=10, <slt:ignore>]+04)------FilterExec: value@1 > 3, metrics=[output_rows=10, <slt:ignore>, selectivity=100% (10/10)]+05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, metrics=[output_rows=10, <slt:ignore>]+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, <slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_rows_pruned=10 total → 10 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18% (210/1.16 K)]++statement ok+set datafusion.explain.analyze_level = dev;++query III+SELECT id, value AS v, value + id as name FROM test_parquet where value > 3 ORDER BY v DESC LIMIT 3;+----+10 1000 1010+9 900 909+8 800 808+# Disable TopK dynamic filter pushdownstatement okSET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = false;@@ -106,6 +130,13 @@ physical_plan01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet+query IIT+SELECT id, value AS v, name FROM (SELECT * FROM test_parquet UNION ALL SELECT * FROM test_parquet) ORDER BY v DESC LIMIT 3;+----+10 1000 j+10 1000 j+9 900 i+# Re-enable for next testsstatement okSET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = true;diff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.sltindex aba468d21f..8a1fef0722 100644--- a/datafusion/sqllogictest/test_files/topk.slt+++ b/datafusion/sqllogictest/test_files/topk.slt@@ -383,7 +383,7 @@ physical_plan03)----ProjectionExec: expr=[__common_expr_1@0 as number_plus, number@1 as number, __common_expr_1@0 as other_number_plus, age@2 as age]04)------ProjectionExec: expr=[CAST(number@0 AS Int64) + 1 as __common_expr_1, number@0 as number, age@1 as age]05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, age], output_ordering=[number@0 DESC], file_type=parquet+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, age], output_ordering=[number@0 DESC], file_type=parquet, predicate=DynamicFilter [ empty ]# Cleanupstatement ok