Apache DataFusion SQL Query Engine
fix: derive scan equivalences only from filters the source applies exactly (#25780)
Adrian Garcia Badaracco39ca2c74a9f6
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 32fc8c9f3d..42e7116dff 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -50,6 +50,7 @@ use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_functions::core::file_row_index::FileRowIndexFunc;use datafusion_physical_expr::expressions::{Column, DynamicFilterTracking};use datafusion_physical_expr::projection::ProjectionExprs;+use datafusion_physical_expr::utils::split_conjunction;use datafusion_physical_expr::{EquivalenceProperties, conjunction};use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;use datafusion_physical_expr_adapter::rewrite::{@@ -702,6 +703,23 @@ impl FileSource for ParquetSource {self.predicate.clone()}+ /// The predicate is applied to every row only when filter pushdown is+ /// enabled, and then only the conjuncts that can become a `RowFilter`.+ /// Otherwise the predicate is used only for pruning.+ fn exact_filter(&self) -> Option<Arc<dyn PhysicalExpr>> {+ if !self.pushdown_filters() {+ return None;+ }+ let predicate = self.predicate.as_ref()?;+ let pushable_schema = self.table_schema.schema_without_virtual_columns();+ let exact = split_conjunction(predicate)+ .into_iter()+ .filter(|expr| can_expr_be_pushed_down_with_schemas(expr, pushable_schema))+ .cloned()+ .collect::<Vec<_>>();+ (!exact.is_empty()).then(|| conjunction(exact))+ }+fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.batch_size = Some(batch_size);diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex f1a94f2e12..d0a1595692 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -114,6 +114,21 @@ pub trait FileSource: Any + Send + Sync {None}+ /// Returns the part of [`Self::filter`] that every output row is+ /// guaranteed to satisfy.+ ///+ /// [`FileScanConfig`] derives equivalence properties (constants and equal+ /// columns) from this filter. A filter that the source uses only to skip+ /// data (for example, row group or page pruning with statistics) does not+ /// remove every non-matching row, so it must not be returned here.+ ///+ /// Returns `None` by default.+ ///+ /// [`FileScanConfig`]: crate::file_scan_config::FileScanConfig+ fn exact_filter(&self) -> Option<Arc<dyn PhysicalExpr>> {+ None+ }+/// Return the projection that will be applied to the output stream on top/// of [`Self::table_schema`].///diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 61b41fd1e9..18f952172a 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -910,7 +910,9 @@ impl DataSource for FileScanConfig {).with_constraints(self.constraints.clone());- if let Some(filter) = self.file_source.filter() {+ // Only a filter that the source applies exactly gives valid equivalences.+ // A filter that is used only for pruning lets non-matching rows through.+ if let Some(filter) = self.file_source.exact_filter() {// We need to remap column indexes to match the projected schema since that's what the equivalence properties deal with.// Note that this will *ignore* any non-projected columns: these don't factor into ordering / equivalence.match Self::add_filter_equivalence_info(&filter, &mut eq_properties, schema) {diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex 289134d9f1..376fb82a27 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -1700,7 +1700,7 @@ physical_plan08)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=109)--------------WorkTableExec: name=r10)--FilterExec: k@0 = 2, projection=[v2@1]-11)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+11)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true12)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/cte/test.parquet]]}, projection=[k, v2], output_ordering=[k@0 ASC NULLS LAST], file_type=parquet, predicate=k@0 = 2, pruning_predicate=k_null_count@2 != row_count@3 AND k_min@0 <= 2 AND 2 <= k_max@1, required_guarantees=[k in (2)]query IIdiff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex 8dd763205f..674f866fc6 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -1336,3 +1336,111 @@ RESET datafusion.execution.parquet.pushdown_filters;statement okdrop table t;++# A predicate that the Parquet scan keeps only for pruning+# (pushdown_filters = false) must not make the scan claim equivalences.+# Each file is sorted by (a, b). If the scan claims that `a` is constant,+# the order-preserving repartition below the FilterExec merges on `b` only+# and the rows with a = 5 come out in the wrong order.++statement ok+COPY (SELECT * FROM (VALUES (3, 9), (5, 1)) v(a, b))+TO 'test_files/scratch/push_down_filter_parquet/pruning_only_eq/f1.parquet'+STORED AS PARQUET;++statement ok+COPY (SELECT * FROM (VALUES (5, 2)) v(a, b))+TO 'test_files/scratch/push_down_filter_parquet/pruning_only_eq/f2.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE pruning_only_eq (a INT, b INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/pruning_only_eq/'+WITH ORDER (a ASC, b ASC);++statement ok+set datafusion.optimizer.repartition_file_scans = false;++statement ok+set datafusion.optimizer.prefer_existing_sort = true;++statement ok+set datafusion.execution.parquet.pushdown_filters = false;++statement ok+set datafusion.explain.physical_plan_only = true;++# The repartition merges on (a, b), the real file ordering+query TT+EXPLAIN SELECT b FROM pruning_only_eq WHERE a = 5 ORDER BY b LIMIT 1;+----+physical_plan+01)SortPreservingMergeExec: [b@0 ASC NULLS LAST], fetch=1+02)--FilterExec: a@0 = 5, projection=[b@1], fetch=1+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, preserve_order=true, sort_exprs=a@0 ASC NULLS LAST, b@1 ASC NULLS LAST+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/pruning_only_eq/f1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/pruning_only_eq/f2.parquet]]}, projection=[a, b], output_ordering=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], file_type=parquet, predicate=a@0 = 5, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= 5 AND 5 <= a_max@1, required_guarantees=[a in (5)]++query I+SELECT b FROM pruning_only_eq WHERE a = 5 ORDER BY b LIMIT 1;+----+1++query II+SELECT a, b FROM pruning_only_eq WHERE a = 5 ORDER BY b;+----+5 1+5 2++statement ok+set datafusion.optimizer.prefer_existing_sort = false;++query III+SELECT a, b, count(*) FROM pruning_only_eq WHERE a = 5 GROUP BY a, b ORDER BY b;+----+5 1 1+5 2 1++# The same filter, with columns `a` and `b` in the opposite roles+statement ok+set datafusion.optimizer.prefer_existing_sort = true;++statement ok+COPY (SELECT * FROM (VALUES (1, 9), (3, 3)) v(a, b))+TO 'test_files/scratch/push_down_filter_parquet/pruning_only_col_eq/f1.parquet'+STORED AS PARQUET;++statement ok+COPY (SELECT * FROM (VALUES (5, 5)) v(a, b))+TO 'test_files/scratch/push_down_filter_parquet/pruning_only_col_eq/f2.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE pruning_only_col_eq (a INT, b INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/pruning_only_col_eq/'+WITH ORDER (a ASC);++query II+SELECT a, b FROM pruning_only_col_eq WHERE a = b ORDER BY a;+----+3 3+5 5++statement ok+drop table pruning_only_eq;++statement ok+drop table pruning_only_col_eq;++statement ok+RESET datafusion.optimizer.repartition_file_scans;++statement ok+RESET datafusion.optimizer.prefer_existing_sort;++statement ok+RESET datafusion.execution.parquet.pushdown_filters;++statement ok+RESET datafusion.explain.physical_plan_only;diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex cad4bc71bc..79067402a7 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -632,3 +632,31 @@ Wire compatibility is directional:- A 55.0 reader must not consume an alias-preserving 56.0 MERGE payload. Itignores the unknown field but cannot preserve the qualifier required by theexpressions, which can cause resolution failure or incorrect rebinding.++### `FileSource::exact_filter` controls scan equivalence properties++`FileScanConfig` no longer derives equivalence properties (constant columns+and equal columns) from `FileSource::filter`. It uses the new+`FileSource::exact_filter` method instead. This method returns the part of the+filter that every output row satisfies. A filter that the source uses only for+pruning (for example, Parquet row group statistics when+`datafusion.execution.parquet.pushdown_filters` is `false`) lets+non-matching rows through, and equivalences derived from it gave wrong query+results.++`exact_filter` returns `None` by default. `ParquetSource` returns the pushed-down+conjuncts when `pushdown_filters` is enabled.++**Migration guide:**++If your custom `FileSource` applies its filter to every row, implement+`exact_filter` to keep the equivalence properties of the scan:++```rust,ignore+impl FileSource for MySource {+ fn exact_filter(&self) -> Option<Arc<dyn PhysicalExpr>> {+ self.filter()+ }+ // ...+}+```