Apache DataFusion SQL Query Engine
fix: Handle `input_file_name()` pushdown into `ParquetSource` with filter pushdown enabled (#23638)
Adam Gutglickdd4dcbc2e425
diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rsindex c9d8beab14..96c99ab207 100644--- a/datafusion/datasource-parquet/src/projection_read_plan.rs+++ b/datafusion/datasource-parquet/src/projection_read_plan.rs@@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet};use std::sync::Arc;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion_functions::core::input_file_name::InputFileNameFunc;use parquet::arrow::ProjectionMask;use parquet::schema::types::SchemaDescriptor;@@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {return Ok(recursion);}- if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())+ if ScalarFunctionExpr::try_downcast_func::<InputFileNameFunc>(node.as_ref()).is_some()+ || ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())+ .is_some(){self.has_unpushable_udfs = true;return Ok(TreeNodeRecursion::Jump);diff --git a/datafusion/sqllogictest/test_files/input_file_name.slt b/datafusion/sqllogictest/test_files/input_file_name.sltindex 8fb72d4a9d..32110aa2d6 100644--- a/datafusion/sqllogictest/test_files/input_file_name.slt+++ b/datafusion/sqllogictest/test_files/input_file_name.slt@@ -121,8 +121,7 @@ physical_plan01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST]02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true]03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1]-04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2-05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquetstatement ok-DROP TABLE pq_table;\ No newline at end of file+DROP TABLE pq_table;diff --git a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt b/datafusion/sqllogictest/test_files/parquet_metadata_functions.sltindex 773ab6761f..25a3c4eb4c 100644--- a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt+++ b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt@@ -52,5 +52,33 @@ logical_plan02)--TableScan: test_table projection=[column1]physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet++# Make sure it also behaves consistently regardless of filter pushdown++statement ok+SET datafusion.execution.parquet.pushdown_filters = false;++query TII rowsort+SELECT input_file_name(), file_row_index(), column1+FROM test_table+WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';+----+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++query TII rowsort+SELECT input_file_name(), file_row_index(), column1+FROM test_table+WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';+----+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60++statement ok+RESET datafusion.execution.parquet.pushdown_filters;+statement okDROP TABLE test_table;