Apache DataFusion SQL Query Engine
Support `columns_sorted` in row_filters (#20497)
Konstantin Tarasovab2823475d0c
diff --git a/benchmarks/README.md b/benchmarks/README.mdindex 7e9818aef2..3aa4f4bb86 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -879,13 +879,13 @@ The benchmark includes queries that:The sorted dataset is automatically generated from the ClickBench partitioned dataset. You can configure the memory used during the sorting process with the `DATAFUSION_MEMORY_GB` environment variable. The default memory limit is 12GB.```bash-./bench.sh data data_sorted_clickbench+./bench.sh data clickbench_sorted```To create the sorted dataset, for example with 16GB of memory, run:```bash-DATAFUSION_MEMORY_GB=16 ./bench.sh data data_sorted_clickbench+DATAFUSION_MEMORY_GB=16 ./bench.sh data clickbench_sorted```This command will:@@ -896,7 +896,7 @@ This command will:#### Running the Benchmark```bash-./bench.sh run data_sorted_clickbench+./bench.sh run clickbench_sorted```This runs queries against the pre-sorted dataset with the `--sorted-by EventTime` flag, which informs DataFusion that the data is pre-sorted, allowing it to optimize away redundant sort operations.diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex 42bdbfb992..c57bbdaeba 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -50,14 +50,9 @@//! 2. Determine whether each predicate can be evaluated as an `ArrowPredicate`.//! 3. Determine, for each predicate, the total compressed size of all//! columns required to evaluate the predicate.-//! 4. Determine, for each predicate, whether all columns required to-//! evaluate the expression are sorted.-//! 5. Re-order the predicate by total size (from step 3).-//! 6. Partition the predicates according to whether they are sorted (from step 4)-//! 7. "Compile" each predicate `Expr` to a `DatafusionArrowPredicate`.-//! 8. Build the `RowFilter` with the sorted predicates followed by-//! the unsorted predicates. Within each partition, predicates are-//! still be sorted by size.+//! 4. Re-order predicates by total size (from step 3).+//! 5. "Compile" each predicate `Expr` to a `DatafusionArrowPredicate`.+//! 6. Build the `RowFilter` from the ordered predicates.//!//! List-aware predicates (for example, `array_has`, `array_has_all`, and//! `array_has_any`) can be evaluated directly during Parquet decoding.@@ -70,7 +65,6 @@//! - `WHERE s['value'] > 5` — pushed down (accesses a primitive leaf)//! - `WHERE s IS NOT NULL` — not pushed down (references the whole struct)-use std::cmp::Ordering;use std::collections::BTreeSet;use std::sync::Arc;@@ -197,8 +191,6 @@ pub(crate) struct FilterCandidate {/// the filter and to order the filters when `reorder_predicates` is true./// This is generated by summing the compressed size of all columns that the filter references.required_bytes: usize,- /// Can this filter use an index (e.g. a page index) to prune rows?- can_use_index: bool,/// Column indices into the parquet file schema required to evaluate this filter.projection: LeafProjection,/// The Arrow schema containing only the columns required by this filter,@@ -273,12 +265,9 @@ impl FilterCandidateBuilder {);let required_bytes = size_of_columns(&leaf_indices, metadata)?;- let can_use_index = columns_sorted(&leaf_indices, metadata)?;-Ok(Some(FilterCandidate {expr: self.expr,required_bytes,- can_use_index,projection: LeafProjection { leaf_indices },filter_schema: projected_schema,}))@@ -817,16 +806,6 @@ fn size_of_columns(columns: &[usize], metadata: &ParquetMetaData) -> Result<usizOk(total_size)}-/// For a given set of `Column`s required for predicate `Expr` determine whether-/// all columns are sorted.-///-/// Sorted columns may be queried more efficiently in the presence of-/// a PageIndex.-fn columns_sorted(_columns: &[usize], _metadata: &ParquetMetaData) -> Result<bool> {- // TODO How do we know this?- Ok(false)-}-/// Build a [`RowFilter`] from the given predicate expression if possible.////// # Arguments@@ -881,12 +860,7 @@ pub fn build_row_filter(}if reorder_predicates {- candidates.sort_unstable_by(|c1, c2| {- match c1.can_use_index.cmp(&c2.can_use_index) {- Ordering::Equal => c1.required_bytes.cmp(&c2.required_bytes),- ord => ord,- }- });+ candidates.sort_unstable_by_key(|c| c.required_bytes);}// To avoid double-counting metrics when multiple predicates are used: