Skip to content

Apache DataFusion SQL Query Engine

Row group limit pruning for row groups that entirely match predicates (#18868)

xudong.wca904b30861c
diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs
index 44c9a2393e..4d52521d62 100644
--- a/datafusion/core/tests/parquet/mod.rs
+++ b/datafusion/core/tests/parquet/mod.rs
@@ -30,6 +30,7 @@ use arrow::{
record_batch::RecordBatch,
util::pretty::pretty_format_batches,
};
+use arrow_schema::SchemaRef;
use chrono::{Datelike, Duration, TimeDelta};
use datafusion::{
datasource::{TableProvider, provider_as_source},
@@ -110,6 +111,26 @@ struct ContextWithParquet {
ctx: SessionContext,
}
+struct PruningMetric {
+ total_pruned: usize,
+ total_matched: usize,
+ total_fully_matched: usize,
+}
+
+impl PruningMetric {
+ pub fn total_pruned(&self) -> usize {
+ self.total_pruned
+ }
+
+ pub fn total_matched(&self) -> usize {
+ self.total_matched
+ }
+
+ pub fn total_fully_matched(&self) -> usize {
+ self.total_fully_matched
+ }
+}
+
/// The output of running one of the test cases
struct TestOutput {
/// The input query SQL
@@ -127,8 +148,8 @@ struct TestOutput {
impl TestOutput {
/// retrieve the value of the named metric, if any
fn metric_value(&self, metric_name: &str) -> Option<usize> {
- if let Some((pruned, _matched)) = self.pruning_metric(metric_name) {
- return Some(pruned);
+ if let Some(pm) = self.pruning_metric(metric_name) {
+ return Some(pm.total_pruned());
}
self.parquet_metrics
@@ -141,9 +162,10 @@ impl TestOutput {
})
}
- fn pruning_metric(&self, metric_name: &str) -> Option<(usize, usize)> {
+ fn pruning_metric(&self, metric_name: &str) -> Option<PruningMetric> {
let mut total_pruned = 0;
let mut total_matched = 0;
+ let mut total_fully_matched = 0;
let mut found = false;
for metric in self.parquet_metrics.iter() {
@@ -155,12 +177,18 @@ impl TestOutput {
{
total_pruned += pruning_metrics.pruned();
total_matched += pruning_metrics.matched();
+ total_fully_matched += pruning_metrics.fully_matched();
+
found = true;
}
}
if found {
- Some((total_pruned, total_matched))
+ Some(PruningMetric {
+ total_pruned,
+ total_matched,
+ total_fully_matched,
+ })
} else {
None
}
@@ -172,27 +200,33 @@ impl TestOutput {
}
/// The number of row_groups pruned / matched by bloom filter
- fn row_groups_bloom_filter(&self) -> Option<(usize, usize)> {
+ fn row_groups_bloom_filter(&self) -> Option<PruningMetric> {
self.pruning_metric("row_groups_pruned_bloom_filter")
}
/// The number of row_groups matched by statistics
fn row_groups_matched_statistics(&self) -> Option<usize> {
self.pruning_metric("row_groups_pruned_statistics")
- .map(|(_pruned, matched)| matched)
+ .map(|pm| pm.total_matched())
+ }
+
+ /// The number of row_groups fully matched by statistics
+ fn row_groups_fully_matched_statistics(&self) -> Option<usize> {
+ self.pruning_metric("row_groups_pruned_statistics")
+ .map(|pm| pm.total_fully_matched())
}
/// The number of row_groups pruned by statistics
fn row_groups_pruned_statistics(&self) -> Option<usize> {
self.pruning_metric("row_groups_pruned_statistics")
- .map(|(pruned, _matched)| pruned)
+ .map(|pm| pm.total_pruned())
}
/// Metric `files_ranges_pruned_statistics` tracks both pruned and matched count,
/// for testing purpose, here it only aggregate the `pruned` count.
fn files_ranges_pruned_statistics(&self) -> Option<usize> {
self.pruning_metric("files_ranges_pruned_statistics")
- .map(|(pruned, _matched)| pruned)
+ .map(|pm| pm.total_pruned())
}
/// The number of row_groups matched by bloom filter or statistics
@@ -201,14 +235,13 @@ impl TestOutput {
/// filter: 7 total -> 3 matched, this function returns 3 for the final matched
/// count.
fn row_groups_matched(&self) -> Option<usize> {
- self.row_groups_bloom_filter()
- .map(|(_pruned, matched)| matched)
+ self.row_groups_bloom_filter().map(|pm| pm.total_matched())
}
/// The number of row_groups pruned
fn row_groups_pruned(&self) -> Option<usize> {
self.row_groups_bloom_filter()
- .map(|(pruned, _matched)| pruned)
+ .map(|pm| pm.total_pruned())
.zip(self.row_groups_pruned_statistics())
.map(|(a, b)| a + b)
}
@@ -216,7 +249,13 @@ impl TestOutput {
/// The number of row pages pruned
fn row_pages_pruned(&self) -> Option<usize> {
self.pruning_metric("page_index_rows_pruned")
- .map(|(pruned, _matched)| pruned)
+ .map(|pm| pm.total_pruned())
+ }
+
+ /// The number of row groups pruned by limit pruning
+ fn limit_pruned_row_groups(&self) -> Option<usize> {
+ self.pruning_metric("limit_pruned_row_groups")
+ .map(|pm| pm.total_pruned())
}
fn description(&self) -> String {
@@ -232,20 +271,41 @@ impl TestOutput {
/// and the appropriate scenario
impl ContextWithParquet {
async fn new(scenario: Scenario, unit: Unit) -> Self {
- Self::with_config(scenario, unit, SessionConfig::new()).await
+ Self::with_config(scenario, unit, SessionConfig::new(), None, None).await
+ }
+
+ /// Set custom schema and batches for the test
+ pub async fn with_custom_data(
+ scenario: Scenario,
+ unit: Unit,
+ schema: Arc<Schema>,
+ batches: Vec<RecordBatch>,
+ ) -> Self {
+ Self::with_config(
+ scenario,
+ unit,
+ SessionConfig::new(),
+ Some(schema),
+ Some(batches),
+ )
+ .await
}
async fn with_config(
scenario: Scenario,
unit: Unit,
mut config: SessionConfig,
+ custom_schema: Option<SchemaRef>,
+ custom_batches: Option<Vec<RecordBatch>>,
) -> Self {
// Use a single partition for deterministic results no matter how many CPUs the host has
config = config.with_target_partitions(1);
let file = match unit {
Unit::RowGroup(row_per_group) => {
config = config.with_parquet_bloom_filter_pruning(true);
- make_test_file_rg(scenario, row_per_group).await
+ config.options_mut().execution.parquet.pushdown_filters = true;
+ make_test_file_rg(scenario, row_per_group, custom_schema, custom_batches)
+ .await
}
Unit::Page(row_per_page) => {
config = config.with_parquet_page_index_pruning(true);
@@ -1075,7 +1135,12 @@ fn create_data_batch(scenario: Scenario) -> Vec<RecordBatch> {
}
/// Create a test parquet file with various data types
-async fn make_test_file_rg(scenario: Scenario, row_per_group: usize) -> NamedTempFile {
+async fn make_test_file_rg(
+ scenario: Scenario,
+ row_per_group: usize,
+ custom_schema: Option<SchemaRef>,
+ custom_batches: Option<Vec<RecordBatch>>,
+) -> NamedTempFile {
let mut output_file = tempfile::Builder::new()
.prefix("parquet_pruning")
.suffix(".parquet")
@@ -1088,8 +1153,14 @@ async fn make_test_file_rg(scenario: Scenario, row_per_group: usize) -> NamedTem
.set_statistics_enabled(EnabledStatistics::Page)
.build();
- let batches = create_data_batch(scenario);
- let schema = batches[0].schema();
+ let (batches, schema) =
+ if let (Some(schema), Some(batches)) = (custom_schema, custom_batches) {
+ (batches, schema)
+ } else {
+ let batches = create_data_batch(scenario);
+ let schema = batches[0].schema();
+ (batches, schema)
+ };
let mut writer = ArrowWriter::try_new(&mut output_file, schema, Some(props)).unwrap();
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index 0411298055..445ae7e97f 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -18,8 +18,12 @@
//! This file contains an end to end test of parquet pruning. It writes
//! data into a parquet file and then verifies row groups are pruned as
//! expected.
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, RecordBatch};
+use arrow_schema::{DataType, Field, Schema};
use datafusion::prelude::SessionConfig;
-use datafusion_common::ScalarValue;
+use datafusion_common::{DataFusionError, ScalarValue};
use itertools::Itertools;
use crate::parquet::Unit::RowGroup;
@@ -30,10 +34,12 @@ struct RowGroupPruningTest {
query: String,
expected_errors: Option<usize>,
expected_row_group_matched_by_statistics: Option<usize>,
+ expected_row_group_fully_matched_by_statistics: Option<usize>,
expected_row_group_pruned_by_statistics: Option<usize>,
expected_files_pruned_by_statistics: Option<usize>,
expected_row_group_matched_by_bloom_filter: Option<usize>,
expected_row_group_pruned_by_bloom_filter: Option<usize>,
+ expected_limit_pruned_row_groups: Option<usize>,
expected_rows: usize,
}
impl RowGroupPruningTest {
@@ -45,9 +51,11 @@ impl RowGroupPruningTest {
expected_errors: None,
expected_row_group_matched_by_statistics: None,
expected_row_group_pruned_by_statistics: None,
+ expected_row_group_fully_matched_by_statistics: None,
expected_files_pruned_by_statistics: None,
expected_row_group_matched_by_bloom_filter: None,
expected_row_group_pruned_by_bloom_filter: None,
+ expected_limit_pruned_row_groups: None,
expected_rows: 0,
}
}
@@ -76,6 +84,15 @@ impl RowGroupPruningTest {
self
}
+ // Set the expected fully matched row groups by statistics
+ fn with_fully_matched_by_stats(
+ mut self,
+ fully_matched_by_stats: Option<usize>,
+ ) -> Self {
+ self.expected_row_group_fully_matched_by_statistics = fully_matched_by_stats;
+ self
+ }
+
// Set the expected pruned row groups by statistics
fn with_pruned_by_stats(mut self, pruned_by_stats: Option<usize>) -> Self {
self.expected_row_group_pruned_by_statistics = pruned_by_stats;
@@ -99,6 +116,11 @@ impl RowGroupPruningTest {
self
}
+ fn with_limit_pruned_row_groups(mut self, pruned_by_limit: Option<usize>) -> Self {
+ self.expected_limit_pruned_row_groups = pruned_by_limit;
+ self
+ }
+
/// Set the number of expected rows from the output of this test
fn with_expected_rows(mut self, rows: usize) -> Self {
self.expected_rows = rows;
@@ -135,15 +157,74 @@ impl RowGroupPruningTest {
);
let bloom_filter_metrics = output.row_groups_bloom_filter();
assert_eq!(
- bloom_filter_metrics.map(|(_pruned, matched)| matched),
+ bloom_filter_metrics.as_ref().map(|pm| pm.total_matched()),
self.expected_row_group_matched_by_bloom_filter,
"mismatched row_groups_matched_bloom_filter",
);
assert_eq!(
- bloom_filter_metrics.map(|(pruned, _matched)| pruned),
+ bloom_filter_metrics.map(|pm| pm.total_pruned()),
self.expected_row_group_pruned_by_bloom_filter,
"mismatched row_groups_pruned_bloom_filter",
);
+
+ assert_eq!(
+ output.result_rows,
+ self.expected_rows,
+ "Expected {} rows, got {}: {}",
+ output.result_rows,
+ self.expected_rows,
+ output.description(),
+ );
+ }
+
+ // Execute the test with the current configuration
+ async fn test_row_group_prune_with_custom_data(
+ self,
+ schema: Arc<Schema>,
+ batches: Vec<RecordBatch>,
+ max_row_per_group: usize,
+ ) {
+ let output = ContextWithParquet::with_custom_data(
+ self.scenario,
+ RowGroup(max_row_per_group),
+ schema,
+ batches,
+ )
+ .await
+ .query(&self.query)
+ .await;
+
+ println!("{}", output.description());
+ assert_eq!(
+ output.predicate_evaluation_errors(),
+ self.expected_errors,
+ "mismatched predicate_evaluation error"
+ );
+ assert_eq!(
+ output.row_groups_matched_statistics(),
+ self.expected_row_group_matched_by_statistics,
+ "mismatched row_groups_matched_statistics",
+ );
+ assert_eq!(
+ output.row_groups_fully_matched_statistics(),
+ self.expected_row_group_fully_matched_by_statistics,
+ "mismatched row_groups_fully_matched_statistics",
+ );
+ assert_eq!(
+ output.row_groups_pruned_statistics(),
+ self.expected_row_group_pruned_by_statistics,
+ "mismatched row_groups_pruned_statistics",
+ );
+ assert_eq!(
+ output.files_ranges_pruned_statistics(),
+ self.expected_files_pruned_by_statistics,
+ "mismatched files_ranges_pruned_statistics",
+ );
+ assert_eq!(
+ output.limit_pruned_row_groups(),
+ self.expected_limit_pruned_row_groups,
+ "mismatched limit_pruned_row_groups",
+ );
assert_eq!(
output.result_rows,
self.expected_rows,
@@ -289,11 +370,16 @@ async fn prune_disabled() {
let expected_rows = 10;
let config = SessionConfig::new().with_parquet_pruning(false);
- let output =
- ContextWithParquet::with_config(Scenario::Timestamps, RowGroup(5), config)
- .await
- .query(query)
- .await;
+ let output = ContextWithParquet::with_config(
+ Scenario::Timestamps,
+ RowGroup(5),
+ config,
+ None,
+ None,
+ )
+ .await
+ .query(query)
+ .await;
println!("{}", output.description());
// This should not prune any
@@ -1636,3 +1722,240 @@ async fn test_bloom_filter_decimal_dict() {
.test_row_group_prune()
.await;
}
+
+// Helper function to create a batch with a single Int32 column.
+fn make_i32_batch(
+ name: &str,
+ values: Vec<i32>,
+) -> datafusion_common::error::Result<RecordBatch> {
+ let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Int32, false)]));
+ let array: ArrayRef = Arc::new(Int32Array::from(values));
+ RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
+}
+
+// Helper function to create a batch with two Int32 columns
+fn make_two_col_i32_batch(
+ name_a: &str,
+ name_b: &str,
+ values_a: Vec<i32>,
+ values_b: Vec<i32>,
+) -> datafusion_common::error::Result<RecordBatch> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new(name_a, DataType::Int32, false),
+ Field::new(name_b, DataType::Int32, false),
+ ]));
+ let array_a: ArrayRef = Arc::new(Int32Array::from(values_a));
+ let array_b: ArrayRef = Arc::new(Int32Array::from(values_b));
+ RecordBatch::try_new(schema, vec![array_a, array_b]).map_err(DataFusionError::from)
+}
+
+#[tokio::test]
+async fn test_limit_pruning_basic() -> datafusion_common::error::Result<()> {
+ // Scenario: Simple integer column, multiple row groups
+ // Query: SELECT c1 FROM t WHERE c1 = 0 LIMIT 2
+ // We expect 2 rows in total.
+
+ // Row Group 0: c1 = [0, -2] -> Partially matched, 1 row
+ // Row Group 1: c1 = [1, 2] -> Fully matched, 2 rows
+ // Row Group 2: c1 = [3, 4] -> Fully matched, 2 rows
+ // Row Group 3: c1 = [5, 6] -> Fully matched, 2 rows
+ // Row Group 4: c1 = [-1, -2] -> Not matched
+
+ // If limit = 2, and RG1 is fully matched and has 2 rows, we should
+ // only scan RG1 and prune other row groups
+ // RG4 is pruned by statistics. RG2 and RG3 are pruned by limit.
+ // So 2 row groups are effectively pruned due to limit pruning.
+
+ let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));
+ let query = "SELECT c1 FROM t WHERE c1 >= 0 LIMIT 2";
+
+ let batches = vec![
+ make_i32_batch("c1", vec![0, -2])?,
+ make_i32_batch("c1", vec![0, 0])?,
+ make_i32_batch("c1", vec![0, 0])?,
+ make_i32_batch("c1", vec![0, 0])?,
+ make_i32_batch("c1", vec![-1, -2])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int) // Assuming Scenario::Int can handle this data
+ .with_query(query)
+ .with_expected_errors(Some(0))
+ .with_expected_rows(2)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4))
+ .with_fully_matched_by_stats(Some(3))
+ .with_pruned_by_stats(Some(1))
+ .with_limit_pruned_row_groups(Some(3))
+ .test_row_group_prune_with_custom_data(schema, batches, 2)
+ .await;
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_limit_pruning_complex_filter() -> datafusion_common::error::Result<()> {
+ // Test Case 1: Complex filter with two columns (a = 1 AND b > 1 AND b < 4)
+ // Row Group 0: a=[1,1,1], b=[0,2,3] -> Partially matched, 2 rows match (b=2,3)
+ // Row Group 1: a=[1,1,1], b=[2,2,2] -> Fully matched, 3 rows
+ // Row Group 2: a=[1,1,1], b=[2,3,3] -> Fully matched, 3 rows
+ // Row Group 3: a=[1,1,1], b=[2,2,3] -> Fully matched, 3 rows
+ // Row Group 4: a=[2,2,2], b=[2,2,2] -> Not matched (a != 1)
+ // Row Group 5: a=[1,1,1], b=[5,6,7] -> Not matched (b >= 4)
+
+ // With LIMIT 5, we need RG1 (3 rows) + RG2 (2 rows from 3) = 5 rows
+ // RG4 and RG5 should be pruned by statistics
+ // RG3 should be pruned by limit
+ // RG0 is partially matched, so it depends on the order
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ]));
+ let query = "SELECT a, b FROM t WHERE a = 1 AND b > 1 AND b < 4 LIMIT 5";
+
+ let batches = vec![
+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![0, 2, 3])?,
+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 2, 2])?,
+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 3, 3])?,
+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 2, 3])?,
+ make_two_col_i32_batch("a", "b", vec![2, 2, 2], vec![2, 2, 2])?,
+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![5, 6, 7])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query(query)
+ .with_expected_errors(Some(0))
+ .with_expected_rows(5)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 are matched
+ .with_fully_matched_by_stats(Some(3))
+ .with_pruned_by_stats(Some(2)) // RG4,5 are pruned
+ .with_limit_pruned_row_groups(Some(2)) // RG0, RG3 is pruned by limit
+ .test_row_group_prune_with_custom_data(schema, batches, 3)
+ .await;
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_limit_pruning_multiple_fully_matched()
+-> datafusion_common::error::Result<()> {
+ // Test Case 2: Limit requires multiple fully matched row groups
+ // Row Group 0: a=[5,5,5,5] -> Fully matched, 4 rows
+ // Row Group 1: a=[5,5,5,5] -> Fully matched, 4 rows
+ // Row Group 2: a=[5,5,5,5] -> Fully matched, 4 rows
+ // Row Group 3: a=[5,5,5,5] -> Fully matched, 4 rows
+ // Row Group 4: a=[1,2,3,4] -> Not matched
+
+ // With LIMIT 8, we need RG0 (4 rows) + RG1 (4 rows) 8 rows
+ // RG2,3 should be pruned by limit
+ // RG4 should be pruned by statistics
+
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let query = "SELECT a FROM t WHERE a = 5 LIMIT 8";
+
+ let batches = vec![
+ make_i32_batch("a", vec![5, 5, 5, 5])?,
+ make_i32_batch("a", vec![5, 5, 5, 5])?,
+ make_i32_batch("a", vec![5, 5, 5, 5])?,
+ make_i32_batch("a", vec![5, 5, 5, 5])?,
+ make_i32_batch("a", vec![1, 2, 3, 4])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query(query)
+ .with_expected_errors(Some(0))
+ .with_expected_rows(8)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched
+ .with_fully_matched_by_stats(Some(4))
+ .with_pruned_by_stats(Some(1)) // RG4 pruned
+ .with_limit_pruned_row_groups(Some(2)) // RG2,3 pruned by limit
+ .test_row_group_prune_with_custom_data(schema, batches, 4)
+ .await;
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_limit_pruning_no_fully_matched() -> datafusion_common::error::Result<()> {
+ // Test Case 3: No fully matched row groups - all are partially matched
+ // Row Group 0: a=[1,2,3] -> Partially matched, 1 row (a=2)
+ // Row Group 1: a=[2,3,4] -> Partially matched, 1 row (a=2)
+ // Row Group 2: a=[2,5,6] -> Partially matched, 1 row (a=2)
+ // Row Group 3: a=[2,7,8] -> Partially matched, 1 row (a=2)
+ // Row Group 4: a=[9,10,11] -> Not matched
+
+ // With LIMIT 3, we need to scan RG0,1,2 to get 3 matching rows
+ // Cannot prune much by limit since all matching RGs are partial
+ // RG4 should be pruned by statistics
+
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let query = "SELECT a FROM t WHERE a = 2 LIMIT 3";
+
+ let batches = vec![
+ make_i32_batch("a", vec![1, 2, 3])?,
+ make_i32_batch("a", vec![2, 3, 4])?,
+ make_i32_batch("a", vec![2, 5, 6])?,
+ make_i32_batch("a", vec![2, 7, 8])?,
+ make_i32_batch("a", vec![9, 10, 11])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query(query)
+ .with_expected_errors(Some(0))
+ .with_expected_rows(3)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched
+ .with_fully_matched_by_stats(Some(0))
+ .with_pruned_by_stats(Some(1)) // RG4 pruned
+ .with_limit_pruned_row_groups(Some(0)) // RG3 pruned by limit
+ .test_row_group_prune_with_custom_data(schema, batches, 3)
+ .await;
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_limit_pruning_exceeds_fully_matched() -> datafusion_common::error::Result<()>
+{
+ // Test Case 4: Limit exceeds all fully matched rows, need partially matched
+ // Row Group 0: a=[10,11,12,12] -> Partially matched, 1 row (a=10)
+ // Row Group 1: a=[10,10,10,10] -> Fully matched, 4 rows
+ // Row Group 2: a=[10,10,10,10] -> Fully matched, 4 rows
+ // Row Group 3: a=[10,13,14,11] -> Partially matched, 1 row (a=10)
+ // Row Group 4: a=[20,21,22,22] -> Not matched
+
+ // With LIMIT 10, we need RG1 (4) + RG2 (4) = 8 from fully matched
+ // Still need 2 more, so we need to scan partially matched RG0 and RG3
+ // All matching row groups should be scanned, only RG4 pruned by statistics
+
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let query = "SELECT a FROM t WHERE a = 10 LIMIT 10";
+
+ let batches = vec![
+ make_i32_batch("a", vec![10, 11, 12, 12])?,
+ make_i32_batch("a", vec![10, 10, 10, 10])?,
+ make_i32_batch("a", vec![10, 10, 10, 10])?,
+ make_i32_batch("a", vec![10, 13, 14, 11])?,
+ make_i32_batch("a", vec![20, 21, 22, 22])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query(query)
+ .with_expected_errors(Some(0))
+ .with_expected_rows(10) // Total: 1 + 4 + 4 + 1 = 10
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched
+ .with_fully_matched_by_stats(Some(2))
+ .with_pruned_by_stats(Some(1)) // RG4 pruned
+ .with_limit_pruned_row_groups(Some(0)) // No limit pruning since we need all RGs
+ .test_row_group_prune_with_custom_data(schema, batches, 4)
+ .await;
+ Ok(())
+}
diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rs
index 8ce3a081a2..fbb14d9a6d 100644
--- a/datafusion/datasource-parquet/src/metrics.rs
+++ b/datafusion/datasource-parquet/src/metrics.rs
@@ -45,9 +45,11 @@ pub struct ParquetFileMetrics {
pub files_ranges_pruned_statistics: PruningMetrics,
/// Number of times the predicate could not be evaluated
pub predicate_evaluation_errors: Count,
- /// Number of row groups whose bloom filters were checked, tracked with matched/pruned counts
+ /// Number of row groups pruned by bloom filters
pub row_groups_pruned_bloom_filter: PruningMetrics,
- /// Number of row groups whose statistics were checked, tracked with matched/pruned counts
+ /// Number of row groups pruned due to limit pruning.
+ pub limit_pruned_row_groups: PruningMetrics,
+ /// Number of row groups pruned by statistics
pub row_groups_pruned_statistics: PruningMetrics,
/// Total number of bytes scanned
pub bytes_scanned: Count,
@@ -99,6 +101,11 @@ impl ParquetFileMetrics {
.with_type(MetricType::SUMMARY)
.pruning_metrics("row_groups_pruned_bloom_filter", partition);
+ let limit_pruned_row_groups = MetricBuilder::new(metrics)
+ .with_new_label("filename", filename.to_string())
+ .with_type(MetricType::SUMMARY)
+ .pruning_metrics("limit_pruned_row_groups", partition);
+
let row_groups_pruned_statistics = MetricBuilder::new(metrics)
.with_new_label("filename", filename.to_string())
.with_type(MetricType::SUMMARY)
@@ -173,6 +180,7 @@ impl ParquetFileMetrics {
predicate_evaluation_errors,
row_groups_pruned_bloom_filter,
row_groups_pruned_statistics,
+ limit_pruned_row_groups,
bytes_scanned,
pushdown_rows_pruned,
pushdown_rows_matched,
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index 570f9b4412..8f31d2df24 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -69,13 +69,15 @@ use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMe
/// Implements [`FileOpener`] for a parquet file
pub(super) struct ParquetOpener {
/// Execution partition index
- pub partition_index: usize,
+ pub(crate) partition_index: usize,
/// Projection to apply on top of the table schema (i.e. can reference partition columns).
pub projection: ProjectionExprs,
/// Target number of rows in each output RecordBatch
pub batch_size: usize,
/// Optional limit on the number of rows to read
- pub limit: Option<usize>,
+ pub(crate) limit: Option<usize>,
+ /// If should keep the output rows in order
+ pub preserve_order: bool,
/// Optional predicate to apply during the scan
pub predicate: Option<Arc<dyn PhysicalExpr>>,
/// Table schema, including partition columns.
@@ -277,6 +279,8 @@ impl FileOpener for ParquetOpener {
let max_predicate_cache_size = self.max_predicate_cache_size;
let reverse_row_groups = self.reverse_row_groups;
+ let preserve_order = self.preserve_order;
+
Ok(Box::pin(async move {
#[cfg(feature = "parquet_encryption")]
let file_decryption_properties = encryption_context
@@ -545,11 +549,15 @@ impl FileOpener for ParquetOpener {
.add_matched(n_remaining_row_groups);
}
- let mut access_plan = row_groups.build();
+ // Prune by limit if limit is set and limit order is not sensitive
+ if let (Some(limit), false) = (limit, preserve_order) {
+ row_groups.prune_by_limit(limit, rg_metadata, &file_metrics);
+ }
// --------------------------------------------------------
// Step: prune pages from the kept row groups
//
+ let mut access_plan = row_groups.build();
// page index pruning: if all data on individual pages can
// be ruled using page metadata, rows from other columns
// with that range can be skipped as well
@@ -1051,6 +1059,7 @@ mod test {
coerce_int96: Option<arrow::datatypes::TimeUnit>,
max_predicate_cache_size: Option<usize>,
reverse_row_groups: bool,
+ preserve_order: bool,
}
impl ParquetOpenerBuilder {
@@ -1076,6 +1085,7 @@ mod test {
coerce_int96: None,
max_predicate_cache_size: None,
reverse_row_groups: false,
+ preserve_order: false,
}
}
@@ -1183,6 +1193,7 @@ mod test {
encryption_factory: None,
max_predicate_cache_size: self.max_predicate_cache_size,
reverse_row_groups: self.reverse_row_groups,
+ preserve_order: self.preserve_order,
}
}
}
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 046379cc25..7eea8285ad 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -24,6 +24,8 @@ use arrow::datatypes::Schema;
use datafusion_common::pruning::PruningStatistics;
use datafusion_common::{Column, Result, ScalarValue};
use datafusion_datasource::FileRange;
+use datafusion_physical_expr::PhysicalExprSimplifier;
+use datafusion_physical_expr::expressions::NotExpr;
use datafusion_pruning::PruningPredicate;
use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
use parquet::arrow::parquet_column;
@@ -46,13 +48,20 @@ use parquet::{
pub struct RowGroupAccessPlanFilter {
/// which row groups should be accessed
access_plan: ParquetAccessPlan,
+ /// Row groups where ALL rows are known to match the pruning predicate
+ /// (the predicate does not filter any rows)
+ is_fully_matched: Vec<bool>,
}
impl RowGroupAccessPlanFilter {
/// Create a new `RowGroupPlanBuilder` for pruning out the groups to scan
/// based on metadata and statistics
pub fn new(access_plan: ParquetAccessPlan) -> Self {
- Self { access_plan }
+ let num_row_groups = access_plan.len();
+ Self {
+ access_plan,
+ is_fully_matched: vec![false; num_row_groups],
+ }
}
/// Return true if there are no row groups
@@ -70,6 +79,139 @@ impl RowGroupAccessPlanFilter {
self.access_plan
}
+ /// Returns the is_fully_matched vector
+ pub fn is_fully_matched(&self) -> &Vec<bool> {
+ &self.is_fully_matched
+ }
+
+ /// Prunes the access plan based on the limit and fully contained row groups.
+ ///
+ /// The pruning works by leveraging the concept of fully matched row groups. Consider a query like:
+ /// `WHERE species LIKE 'Alpine%' AND s >= 50 LIMIT N`
+ ///
+ /// After initial filtering, row groups can be classified into three states:
+ ///
+ /// 1. Not Matching / Pruned
+ /// 2. Partially Matching (Row Group/Page contains some matches)
+ /// 3. Fully Matching (Entire range is within predicate)
+ ///
+ /// +-----------------------------------------------------------------------+
+ /// | NOT MATCHING |
+ /// | Row group 1 |
+ /// | +-----------------------------------+-----------------------------+ |
+ /// | | SPECIES | S | |
+ /// | +-----------------------------------+-----------------------------+ |
+ /// | | Snow Vole | 7 | |
+ /// | | Brown Bear | 133 ✅ | |
+ /// | | Gray Wolf | 82 ✅ | |
+ /// | +-----------------------------------+-----------------------------+ |
+ /// +-----------------------------------------------------------------------+
+ ///
+ /// +---------------------------------------------------------------------------+
+ /// | PARTIALLY MATCHING |
+ /// | |
+ /// | Row group 2 Row group 4 |
+ /// | +------------------+--------------+ +------------------+----------+ |
+ /// | | SPECIES | S | | SPECIES | S | |
+ /// | +------------------+--------------+ +------------------+----------+ |
+ /// | | Lynx | 71 ✅ | | Europ. Mole | 4 | |
+ /// | | Red Fox | 40 | | Polecat | 16 | |
+ /// | | Alpine Bat ✅ | 6 | | Alpine Ibex ✅ | 97 ✅ | |
+ /// | +------------------+--------------+ +------------------+----------+ |
+ /// +---------------------------------------------------------------------------+
+ ///
+ /// +-----------------------------------------------------------------------+
+ /// | FULLY MATCHING |
+ /// | Row group 3 |
+ /// | +-----------------------------------+-----------------------------+ |
+ /// | | SPECIES | S | |
+ /// | +-----------------------------------+-----------------------------+ |
+ /// | | Alpine Ibex ✅ | 101 ✅ | |
+ /// | | Alpine Goat ✅ | 76 ✅ | |
+ /// | | Alpine Sheep ✅ | 83 ✅ | |
+ /// | +-----------------------------------+-----------------------------+ |
+ /// +-----------------------------------------------------------------------+
+ ///
+ /// ### Identification of Fully Matching Row Groups
+ ///
+ /// DataFusion identifies row groups where ALL rows satisfy the filter by inverting the
+ /// predicate and checking if statistics prove the inverted version is false for the group.
+ ///
+ /// For example, prefix matches like `species LIKE 'Alpine%'` are pruned using ranges:
+ /// 1. Candidate Range: `species >= 'Alpine' AND species < 'Alpinf'`
+ /// 2. Inverted Condition (to prove full match): `species < 'Alpine' OR species >= 'Alpinf'`
+ /// 3. Statistical Evaluation (check if any row *could* satisfy the inverted condition):
+ /// `min < 'Alpine' OR max >= 'Alpinf'`
+ ///
+ /// If this evaluation is **false**, it proves no row can fail the original filter,
+ /// so the row group is **FULLY MATCHING**.
+ ///
+ /// ### Impact of Statistics Truncation
+ ///
+ /// The precision of pruning depends on the metadata quality. Truncated statistics
+ /// may prevent the system from proving a full match.
+ ///
+ /// **Example**: `WHERE species LIKE 'Alpine%'` (Target range: `['Alpine', 'Alpinf')`)
+ ///
+ /// | Truncation Length | min / max | Inverted Evaluation | Status |
+ /// |-------------------|---------------------|---------------------------------------------------------------------|------------------------|
+ /// | **Length 6** | `Alpine` / `Alpine` | `"Alpine" < "Alpine" (F) OR "Alpine" >= "Alpinf" (F)` -> **false** | **FULLY MATCHING** |
+ /// | **Length 3** | `Alp` / `Alq` | `"Alp" < "Alpine" (T) OR "Alq" >= "Alpinf" (T)` -> **true** | **PARTIALLY MATCHING** |
+ ///
+ /// Even though Row Group 3 only contains matching rows, truncation to length 3 makes
+ /// the statistics `[Alp, Alq]` too broad to prove it (they could include "Alpha").
+ /// The system must conservatively scan the group.
+ ///
+ /// Without limit pruning: Scan Partition 2 → Partition 3 → Partition 4 (until limit reached)
+ /// With limit pruning: If Partition 3 contains enough rows to satisfy the limit,
+ /// skip Partitions 2 and 4 entirely and go directly to Partition 3.
+ ///
+ /// This optimization is particularly effective when:
+ /// - The limit is small relative to the total dataset size
+ /// - There are row groups that are fully matched by the filter predicates
+ /// - The fully matched row groups contain sufficient rows to satisfy the limit
+ ///
+ /// For more information, see the [paper](https://arxiv.org/pdf/2504.11540)'s "Pruning for LIMIT Queries" part
+ pub fn prune_by_limit(
+ &mut self,
+ limit: usize,
+ rg_metadata: &[RowGroupMetaData],
+ metrics: &ParquetFileMetrics,
+ ) {
+ let mut fully_matched_row_group_indexes: Vec<usize> = Vec::new();
+ let mut fully_matched_rows_count: usize = 0;
+
+ // Iterate through the currently accessible row groups and try to
+ // find a set of matching row groups that can satisfy the limit
+ for &idx in self.access_plan.row_group_indexes().iter() {
+ if self.is_fully_matched[idx] {
+ let row_group_row_count = rg_metadata[idx].num_rows() as usize;
+ fully_matched_row_group_indexes.push(idx);
+ fully_matched_rows_count += row_group_row_count;
+ if fully_matched_rows_count >= limit {
+ break;
+ }
+ }
+ }
+
+ // If we can satisfy the limit with fully matching row groups,
+ // rewrite the plan to do so
+ if fully_matched_rows_count >= limit {
+ let original_num_accessible_row_groups =
+ self.access_plan.row_group_indexes().len();
+ let new_num_accessible_row_groups = fully_matched_row_group_indexes.len();
+ let pruned_count = original_num_accessible_row_groups
+ .saturating_sub(new_num_accessible_row_groups);
+ metrics.limit_pruned_row_groups.add_pruned(pruned_count);
+
+ let mut new_access_plan = ParquetAccessPlan::new_none(rg_metadata.len());
+ for &idx in &fully_matched_row_group_indexes {
+ new_access_plan.scan(idx);
+ }
+ self.access_plan = new_access_plan;
+ }
+ }
+
/// Prune remaining row groups to only those within the specified range.
///
/// Updates this set to mark row groups that should not be scanned
@@ -135,15 +277,26 @@ impl RowGroupAccessPlanFilter {
// try to prune the row groups in a single call
match predicate.prune(&pruning_stats) {
Ok(values) => {
- // values[i] is false means the predicate could not be true for row group i
+ let mut fully_contained_candidates_original_idx: Vec<usize> = Vec::new();
for (idx, &value) in row_group_indexes.iter().zip(values.iter()) {
if !value {
self.access_plan.skip(*idx);
metrics.row_groups_pruned_statistics.add_pruned(1);
} else {
metrics.row_groups_pruned_statistics.add_matched(1);
+ fully_contained_candidates_original_idx.push(*idx);
}
}
+
+ // Check if any of the matched row groups are fully contained by the predicate
+ self.identify_fully_matched_row_groups(
+ &fully_contained_candidates_original_idx,
+ arrow_schema,
+ parquet_schema,
+ groups,
+ predicate,
+ metrics,
+ );
}
// stats filter array could not be built, so we can't prune
Err(e) => {
@@ -153,6 +306,68 @@ impl RowGroupAccessPlanFilter {
}
}
+ /// Identifies row groups that are fully matched by the predicate.
+ ///
+ /// This optimization checks whether all rows in a row group satisfy the predicate
+ /// by inverting the predicate and checking if it prunes the row group. If the
+ /// inverted predicate prunes a row group, it means no rows match the inverted
+ /// predicate, which implies all rows match the original predicate.
+ ///
+ /// Note: This optimization is relatively inexpensive for a limited number of row groups.
+ fn identify_fully_matched_row_groups(
+ &mut self,
+ candidate_row_group_indices: &[usize],
+ arrow_schema: &Schema,
+ parquet_schema: &SchemaDescriptor,
+ groups: &[RowGroupMetaData],
+ predicate: &PruningPredicate,
+ metrics: &ParquetFileMetrics,
+ ) {
+ if candidate_row_group_indices.is_empty() {
+ return;
+ }
+
+ // Use NotExpr to create the inverted predicate
+ let inverted_expr = Arc::new(NotExpr::new(Arc::clone(predicate.orig_expr())));
+
+ // Simplify the NOT expression (e.g., NOT(c1 = 0) -> c1 != 0)
+ // before building the pruning predicate
+ let simplifier = PhysicalExprSimplifier::new(arrow_schema);
+ let Ok(inverted_expr) = simplifier.simplify(inverted_expr) else {
+ return;
+ };
+
+ let Ok(inverted_predicate) =
+ PruningPredicate::try_new(inverted_expr, Arc::clone(predicate.schema()))
+ else {
+ return;
+ };
+
+ let inverted_pruning_stats = RowGroupPruningStatistics {
+ parquet_schema,
+ row_group_metadatas: candidate_row_group_indices
+ .iter()
+ .map(|&i| &groups[i])
+ .collect::<Vec<_>>(),
+ arrow_schema,
+ };
+
+ let Ok(inverted_values) = inverted_predicate.prune(&inverted_pruning_stats)
+ else {
+ return;
+ };
+
+ for (i, &original_row_group_idx) in candidate_row_group_indices.iter().enumerate()
+ {
+ // If the inverted predicate *also* prunes this row group (meaning inverted_values[i] is false),
+ // it implies that *all* rows in this group satisfy the original predicate.
+ if !inverted_values[i] {
+ self.is_fully_matched[original_row_group_idx] = true;
+ metrics.row_groups_pruned_statistics.add_fully_matched(1);
+ }
+ }
+ }
+
/// Prune remaining row groups using available bloom filters and the
/// [`PruningPredicate`].
///
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 2e0919b144..07f58db185 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -548,6 +548,7 @@ impl FileSource for ParquetSource {
.batch_size
.expect("Batch size must set before creating ParquetOpener"),
limit: base_config.limit,
+ preserve_order: base_config.preserve_order,
predicate: self.predicate.clone(),
table_schema: self.table_schema.clone(),
metadata_size_hint: self.metadata_size_hint,
diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs
index 1f7c37315c..51b9ba9e06 100644
--- a/datafusion/datasource/src/file_scan_config.rs
+++ b/datafusion/datasource/src/file_scan_config.rs
@@ -152,6 +152,11 @@ pub struct FileScanConfig {
/// The maximum number of records to read from this plan. If `None`,
/// all records after filtering are returned.
pub limit: Option<usize>,
+ /// Whether the scan's limit is order sensitive
+ /// When `true`, files must be read in the exact order specified to produce
+ /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,
+ /// DataFusion may reorder file processing for optimization without affecting correctness.
+ pub preserve_order: bool,
/// All equivalent lexicographical orderings that describe the schema.
pub output_ordering: Vec<LexOrdering>,
/// File compression type
@@ -240,6 +245,7 @@ pub struct FileScanConfigBuilder {
object_store_url: ObjectStoreUrl,
file_source: Arc<dyn FileSource>,
limit: Option<usize>,
+ preserve_order: bool,
constraints: Option<Constraints>,
file_groups: Vec<FileGroup>,
statistics: Option<Statistics>,
@@ -269,6 +275,7 @@ impl FileScanConfigBuilder {
output_ordering: vec![],
file_compression_type: None,
limit: None,
+ preserve_order: false,
constraints: None,
batch_size: None,
expr_adapter_factory: None,
@@ -283,6 +290,15 @@ impl FileScanConfigBuilder {
self
}
+ /// Set whether the limit should be order-sensitive.
+ /// When `true`, files must be read in the exact order specified to produce
+ /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,
+ /// DataFusion may reorder file processing for optimization without affecting correctness.
+ pub fn with_preserve_order(mut self, order_sensitive: bool) -> Self {
+ self.preserve_order = order_sensitive;
+ self
+ }
+
/// Set the file source for scanning files.
///
/// This method allows you to change the file source implementation (e.g. ParquetSource, CsvSource, etc.)
@@ -450,6 +466,7 @@ impl FileScanConfigBuilder {
object_store_url,
file_source,
limit,
+ preserve_order,
constraints,
file_groups,
statistics,
@@ -467,10 +484,14 @@ impl FileScanConfigBuilder {
let file_compression_type =
file_compression_type.unwrap_or(FileCompressionType::UNCOMPRESSED);
+ // If there is an output ordering, we should preserve it.
+ let preserve_order = preserve_order || !output_ordering.is_empty();
+
FileScanConfig {
object_store_url,
file_source,
limit,
+ preserve_order,
constraints,
file_groups,
output_ordering,
@@ -493,6 +514,7 @@ impl From<FileScanConfig> for FileScanConfigBuilder {
output_ordering: config.output_ordering,
file_compression_type: Some(config.file_compression_type),
limit: config.limit,
+ preserve_order: config.preserve_order,
constraints: Some(config.constraints),
batch_size: config.batch_size,
expr_adapter_factory: config.expr_adapter_factory,
@@ -850,6 +872,18 @@ impl DataSource for FileScanConfig {
}
}
}
+
+ fn with_preserve_order(&self, preserve_order: bool) -> Option<Arc<dyn DataSource>> {
+ if self.preserve_order == preserve_order {
+ return Some(Arc::new(self.clone()));
+ }
+
+ let new_config = FileScanConfig {
+ preserve_order,
+ ..self.clone()
+ };
+ Some(Arc::new(new_config))
+ }
}
impl FileScanConfig {
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index a3892dfac9..de18b6be22 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -210,6 +210,11 @@ pub trait DataSource: Send + Sync + Debug {
) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {
Ok(SortOrderPushdownResult::Unsupported)
}
+
+ /// Returns a variant of this `DataSource` that is aware of order-sensitivity.
+ fn with_preserve_order(&self, _preserve_order: bool) -> Option<Arc<dyn DataSource>> {
+ None
+ }
}
/// [`ExecutionPlan`] that reads one or more files
@@ -393,6 +398,18 @@ impl ExecutionPlan for DataSourceExec {
Ok(Arc::new(new_exec) as Arc<dyn ExecutionPlan>)
})
}
+
+ fn with_preserve_order(
+ &self,
+ preserve_order: bool,
+ ) -> Option<Arc<dyn ExecutionPlan>> {
+ self.data_source
+ .with_preserve_order(preserve_order)
+ .map(|new_data_source| {
+ Arc::new(self.clone().with_data_source(new_data_source))
+ as Arc<dyn ExecutionPlan>
+ })
+ }
}
impl DataSourceExec {
diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs
index 548eadffa2..f97b05ea68 100644
--- a/datafusion/optimizer/src/optimize_projections/mod.rs
+++ b/datafusion/optimizer/src/optimize_projections/mod.rs
@@ -268,15 +268,10 @@ fn optimize_projections(
Some(projection) => indices.into_mapped_indices(|idx| projection[idx]),
None => indices.into_inner(),
};
- return TableScan::try_new(
- table_name,
- source,
- Some(projection),
- filters,
- fetch,
- )
- .map(LogicalPlan::TableScan)
- .map(Transformed::yes);
+ let new_scan =
+ TableScan::try_new(table_name, source, Some(projection), filters, fetch)?;
+
+ return Ok(Transformed::yes(LogicalPlan::TableScan(new_scan)));
}
// Other node types are handled below
_ => {}
diff --git a/datafusion/physical-expr-common/src/metrics/value.rs b/datafusion/physical-expr-common/src/metrics/value.rs
index 9a14b804a2..5ecaa86fc3 100644
--- a/datafusion/physical-expr-common/src/metrics/value.rs
+++ b/datafusion/physical-expr-common/src/metrics/value.rs
@@ -372,19 +372,31 @@ impl Drop for ScopedTimerGuard<'_> {
pub struct PruningMetrics {
pruned: Arc<AtomicUsize>,
matched: Arc<AtomicUsize>,
+ fully_matched: Arc<AtomicUsize>,
}
impl Display for PruningMetrics {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let matched = self.matched.load(Ordering::Relaxed);
let total = self.pruned.load(Ordering::Relaxed) + matched;
+ let fully_matched = self.fully_matched.load(Ordering::Relaxed);
- write!(
- f,
- "{} total → {} matched",
- human_readable_count(total),
- human_readable_count(matched)
- )
+ if fully_matched != 0 {
+ write!(
+ f,
+ "{} total → {} matched -> {} fully matched",
+ human_readable_count(total),
+ human_readable_count(matched),
+ human_readable_count(fully_matched)
+ )
+ } else {
+ write!(
+ f,
+ "{} total → {} matched",
+ human_readable_count(total),
+ human_readable_count(matched)
+ )
+ }
}
}
@@ -400,6 +412,7 @@ impl PruningMetrics {
Self {
pruned: Arc::new(AtomicUsize::new(0)),
matched: Arc::new(AtomicUsize::new(0)),
+ fully_matched: Arc::new(AtomicUsize::new(0)),
}
}
@@ -417,6 +430,13 @@ impl PruningMetrics {
self.matched.fetch_add(n, Ordering::Relaxed);
}
+ /// Add `n` to the metric's fully matched value
+ pub fn add_fully_matched(&self, n: usize) {
+ // relaxed ordering for operations on `value` poses no issues
+ // we're purely using atomic ops with no associated memory ops
+ self.fully_matched.fetch_add(n, Ordering::Relaxed);
+ }
+
/// Subtract `n` to the metric's matched value.
pub fn subtract_matched(&self, n: usize) {
// relaxed ordering for operations on `value` poses no issues
@@ -433,6 +453,11 @@ impl PruningMetrics {
pub fn matched(&self) -> usize {
self.matched.load(Ordering::Relaxed)
}
+
+ /// Number of items fully matched
+ pub fn fully_matched(&self) -> usize {
+ self.fully_matched.load(Ordering::Relaxed)
+ }
}
/// Counters tracking ratio metrics (e.g. matched vs total)
@@ -906,8 +931,11 @@ impl MetricValue {
) => {
let pruned = other_pruning_metrics.pruned.load(Ordering::Relaxed);
let matched = other_pruning_metrics.matched.load(Ordering::Relaxed);
+ let fully_matched =
+ other_pruning_metrics.fully_matched.load(Ordering::Relaxed);
pruning_metrics.add_pruned(pruned);
pruning_metrics.add_matched(matched);
+ pruning_metrics.add_fully_matched(fully_matched);
}
(
Self::Ratio { ratio_metrics, .. },
diff --git a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs
index a5fafb9e87..247ebb2785 100644
--- a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs
+++ b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs
@@ -581,11 +581,17 @@ fn analyze_immediate_sort_removal(
// Remove the sort:
node.children = node.children.swap_remove(0).children;
if let Some(fetch) = sort_exec.fetch() {
+ let required_ordering = sort_exec.properties().output_ordering().cloned();
// If the sort has a fetch, we need to add a limit:
if properties.output_partitioning().partition_count() == 1 {
- Arc::new(GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch)))
+ let mut global_limit =
+ GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch));
+ global_limit.set_required_ordering(required_ordering);
+ Arc::new(global_limit)
} else {
- Arc::new(LocalLimitExec::new(Arc::clone(sort_input), fetch))
+ let mut local_limit = LocalLimitExec::new(Arc::clone(sort_input), fetch);
+ local_limit.set_required_ordering(required_ordering);
+ Arc::new(local_limit)
}
} else {
Arc::clone(sort_input)
diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rs
index 4cb3abe30b..a4dac81dba 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs
@@ -50,6 +50,7 @@ pub struct GlobalRequirements {
fetch: Option<usize>,
skip: usize,
satisfied: bool,
+ preserve_order: bool,
}
impl LimitPushdown {
@@ -69,6 +70,7 @@ impl PhysicalOptimizerRule for LimitPushdown {
fetch: None,
skip: 0,
satisfied: false,
+ preserve_order: false,
};
pushdown_limits(plan, global_state)
}
@@ -111,6 +113,13 @@ impl LimitExec {
Self::Local(_) => 0,
}
}
+
+ fn preserve_order(&self) -> bool {
+ match self {
+ Self::Global(global) => global.required_ordering().is_some(),
+ Self::Local(local) => local.required_ordering().is_some(),
+ }
+ }
}
impl From<LimitExec> for Arc<dyn ExecutionPlan> {
@@ -145,6 +154,7 @@ pub fn pushdown_limit_helper(
);
global_state.skip = skip;
global_state.fetch = fetch;
+ global_state.preserve_order = limit_exec.preserve_order();
// Now the global state has the most recent information, we can remove
// the `LimitExec` plan. We will decide later if we should add it again
@@ -241,17 +251,28 @@ pub fn pushdown_limit_helper(
let maybe_fetchable = pushdown_plan.with_fetch(skip_and_fetch);
if global_state.satisfied {
if let Some(plan_with_fetch) = maybe_fetchable {
- Ok((Transformed::yes(plan_with_fetch), global_state))
+ let plan_with_preserve_order = plan_with_fetch
+ .with_preserve_order(global_state.preserve_order)
+ .unwrap_or(plan_with_fetch);
+ Ok((Transformed::yes(plan_with_preserve_order), global_state))
} else {
Ok((Transformed::no(pushdown_plan), global_state))
}
} else {
global_state.satisfied = true;
pushdown_plan = if let Some(plan_with_fetch) = maybe_fetchable {
+ let plan_with_preserve_order = plan_with_fetch
+ .with_preserve_order(global_state.preserve_order)
+ .unwrap_or(plan_with_fetch);
+
if global_skip > 0 {
- add_global_limit(plan_with_fetch, global_skip, Some(global_fetch))
+ add_global_limit(
+ plan_with_preserve_order,
+ global_skip,
+ Some(global_fetch),
+ )
} else {
- plan_with_fetch
+ plan_with_preserve_order
}
} else {
add_limit(pushdown_plan, global_skip, global_fetch)
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index d83f90eb3d..22dcc85d6e 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -278,6 +278,19 @@ impl ExecutionPlan for CoalescePartitionsExec {
}))
}
+ fn with_preserve_order(
+ &self,
+ preserve_order: bool,
+ ) -> Option<Arc<dyn ExecutionPlan>> {
+ self.input
+ .with_preserve_order(preserve_order)
+ .and_then(|new_input| {
+ Arc::new(self.clone())
+ .with_new_children(vec![new_input])
+ .ok()
+ })
+ }
+
fn gather_filters_for_pushdown(
&self,
_phase: FilterPushdownPhase,
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index 8d72921d06..52f4829127 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -709,6 +709,19 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {
) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {
Ok(SortOrderPushdownResult::Unsupported)
}
+
+ /// Returns a variant of this `ExecutionPlan` that is aware of order-sensitivity.
+ ///
+ /// This is used to signal to data sources that the output ordering must be
+ /// preserved, even if it might be more efficient to ignore it (e.g. by
+ /// skipping some row groups in Parquet).
+ ///
+ fn with_preserve_order(
+ &self,
+ _preserve_order: bool,
+ ) -> Option<Arc<dyn ExecutionPlan>> {
+ None
+ }
}
/// [`ExecutionPlan`] Invariant Level
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index e78e50c986..76516c25ad 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -614,6 +614,19 @@ impl ExecutionPlan for FilterExec {
fetch,
}))
}
+
+ fn with_preserve_order(
+ &self,
+ preserve_order: bool,
+ ) -> Option<Arc<dyn ExecutionPlan>> {
+ self.input
+ .with_preserve_order(preserve_order)
+ .and_then(|new_input| {
+ Arc::new(self.clone())
+ .with_new_children(vec![new_input])
+ .ok()
+ })
+ }
}
impl EmbeddedProjection for FilterExec {
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index 05d6882821..fea7acb221 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -35,6 +35,7 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
+use datafusion_physical_expr::LexOrdering;
use futures::stream::{Stream, StreamExt};
use log::trace;
@@ -51,6 +52,9 @@ pub struct GlobalLimitExec {
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
cache: PlanProperties,
+ /// Does the limit have to preserve the order of its input, and if so what is it?
+ /// Some optimizations may reorder the input if no particular sort is required
+ required_ordering: Option<LexOrdering>,
}
impl GlobalLimitExec {
@@ -63,6 +67,7 @@ impl GlobalLimitExec {
fetch,
metrics: ExecutionPlanMetricsSet::new(),
cache,
+ required_ordering: None,
}
}
@@ -91,6 +96,16 @@ impl GlobalLimitExec {
Boundedness::Bounded,
)
}
+
+ /// Get the required ordering from limit
+ pub fn required_ordering(&self) -> &Option<LexOrdering> {
+ &self.required_ordering
+ }
+
+ /// Set the required ordering for limit
+ pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {
+ self.required_ordering = required_ordering;
+ }
}
impl DisplayAs for GlobalLimitExec {
@@ -223,6 +238,9 @@ pub struct LocalLimitExec {
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
cache: PlanProperties,
+ /// If the child plan is a sort node, after the sort node is removed during
+ /// physical optimization, we should add the required ordering to the limit node
+ required_ordering: Option<LexOrdering>,
}
impl LocalLimitExec {
@@ -234,6 +252,7 @@ impl LocalLimitExec {
fetch,
metrics: ExecutionPlanMetricsSet::new(),
cache,
+ required_ordering: None,
}
}
@@ -257,6 +276,16 @@ impl LocalLimitExec {
Boundedness::Bounded,
)
}
+
+ /// Get the required ordering from limit
+ pub fn required_ordering(&self) -> &Option<LexOrdering> {
+ &self.required_ordering
+ }
+
+ /// Set the required ordering for limit
+ pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {
+ self.required_ordering = required_ordering;
+ }
}
impl DisplayAs for LocalLimitExec {
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index e8608f17a1..8f2f2219f4 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -427,6 +427,19 @@ impl ExecutionPlan for ProjectionExec {
}
}
}
+
+ fn with_preserve_order(
+ &self,
+ preserve_order: bool,
+ ) -> Option<Arc<dyn ExecutionPlan>> {
+ self.input
+ .with_preserve_order(preserve_order)
+ .and_then(|new_input| {
+ Arc::new(self.clone())
+ .with_new_children(vec![new_input])
+ .ok()
+ })
+ }
}
impl ProjectionStream {
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 0ddea90a98..68c457a0d8 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -245,6 +245,19 @@ impl ExecutionPlan for SortPreservingMergeExec {
}))
}
+ fn with_preserve_order(
+ &self,
+ preserve_order: bool,
+ ) -> Option<Arc<dyn ExecutionPlan>> {
+ self.input
+ .with_preserve_order(preserve_order)
+ .and_then(|new_input| {
+ Arc::new(self.clone())
+ .with_new_children(vec![new_input])
+ .ok()
+ })
+ }
+
fn required_input_distribution(&self) -> Vec<Distribution> {
vec![Distribution::UnspecifiedDistribution]
}
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index b5b8267d7f..5f1b4233b5 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -492,7 +492,6 @@ impl PruningPredicate {
// Simplify the newly created predicate to get rid of redundant casts, comparisons, etc.
let predicate_expr =
PhysicalExprSimplifier::new(&predicate_schema).simplify(predicate_expr)?;
-
let literal_guarantees = LiteralGuarantee::analyze(&expr);
Ok(Self {
diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.slt
new file mode 100644
index 0000000000..8a94bf8adc
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+statement ok
+set datafusion.execution.parquet.pushdown_filters = true;
+
+
+statement ok
+CREATE TABLE tracking_data AS VALUES
+-- ***** Row Group 0 *****
+ ('Anow Vole', 7),
+ ('Brown Bear', 133),
+ ('Gray Wolf', 82),
+-- ***** Row Group 1 *****
+ ('Lynx', 71),
+ ('Red Fox', 40),
+ ('Alpine Bat', 6),
+-- ***** Row Group 2 *****
+ ('Nlpine Ibex', 101),
+ ('Nlpine Goat', 76),
+ ('Nlpine Sheep', 83),
+-- ***** Row Group 3 *****
+ ('Europ. Mole', 4),
+ ('Polecat', 16),
+ ('Alpine Ibex', 97);
+
+statement ok
+COPY (SELECT column1 as species, column2 as s FROM tracking_data)
+TO 'test_files/scratch/limit_pruning/data.parquet'
+STORED AS PARQUET
+OPTIONS (
+ 'format.max_row_group_size' '3'
+);
+
+statement ok
+drop table tracking_data;
+
+statement ok
+CREATE EXTERNAL TABLE tracking_data
+STORED AS PARQUET
+LOCATION 'test_files/scratch/limit_pruning/data.parquet';
+
+
+statement ok
+set datafusion.explain.analyze_level = summary;
+
+# row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched
+# limit_pruned_row_groups=2 total → 0 matched
+query TT
+explain analyze select * from tracking_data where species > 'M' AND s >= 50 limit 3;
+----
+Plan with Metrics DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], limit=3, file_type=parquet, predicate=species@0 > M AND s@1 >= 50, pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_rows_pruned=3 total → 3 matched, limit_pruned_row_groups=2 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (171/2.35 K)]
+
+# limit_pruned_row_groups=0 total → 0 matched
+# because of order by, scan needs to preserve sort, so limit pruning is disabled
+query TT
+explain analyze select * from tracking_data where species > 'M' AND s >= 50 order by species limit 3;
+----
+Plan with Metrics
+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_rows_pruned=9 total → 9 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]
+
+statement ok
+drop table tracking_data;
+
+statement ok
+reset datafusion.explain.analyze_level;
diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.md
index 5a1184539c..8fe8316381 100644
--- a/docs/source/user-guide/explain-usage.md
+++ b/docs/source/user-guide/explain-usage.md
@@ -228,6 +228,7 @@ When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains
- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).
- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.
- `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched.
+- `limit_pruned_row_groups`: number of row groups pruned by the limit.
- `pushdown_rows_matched`: rows that were tested by any of the above filters, and passed all of them.
- `pushdown_rows_pruned`: rows that were tested by any of the above filters, and did not pass at least one of them.
- `predicate_evaluation_errors`: number of times evaluating the filter expression failed (expected to be zero in normal operation)