Apache DataFusion SQL Query Engine
Row group limit pruning for row groups that entirely match predicates (#18868)
xudong.wca904b30861c
diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rsindex 44c9a2393e..4d52521d62 100644--- a/datafusion/core/tests/parquet/mod.rs+++ b/datafusion/core/tests/parquet/mod.rs@@ -30,6 +30,7 @@ use arrow::{record_batch::RecordBatch,util::pretty::pretty_format_batches,};+use arrow_schema::SchemaRef;use chrono::{Datelike, Duration, TimeDelta};use datafusion::{datasource::{TableProvider, provider_as_source},@@ -110,6 +111,26 @@ struct ContextWithParquet {ctx: SessionContext,}+struct PruningMetric {+ total_pruned: usize,+ total_matched: usize,+ total_fully_matched: usize,+}++impl PruningMetric {+ pub fn total_pruned(&self) -> usize {+ self.total_pruned+ }++ pub fn total_matched(&self) -> usize {+ self.total_matched+ }++ pub fn total_fully_matched(&self) -> usize {+ self.total_fully_matched+ }+}+/// The output of running one of the test casesstruct TestOutput {/// The input query SQL@@ -127,8 +148,8 @@ struct TestOutput {impl TestOutput {/// retrieve the value of the named metric, if anyfn metric_value(&self, metric_name: &str) -> Option<usize> {- if let Some((pruned, _matched)) = self.pruning_metric(metric_name) {- return Some(pruned);+ if let Some(pm) = self.pruning_metric(metric_name) {+ return Some(pm.total_pruned());}self.parquet_metrics@@ -141,9 +162,10 @@ impl TestOutput {})}- fn pruning_metric(&self, metric_name: &str) -> Option<(usize, usize)> {+ fn pruning_metric(&self, metric_name: &str) -> Option<PruningMetric> {let mut total_pruned = 0;let mut total_matched = 0;+ let mut total_fully_matched = 0;let mut found = false;for metric in self.parquet_metrics.iter() {@@ -155,12 +177,18 @@ impl TestOutput {{total_pruned += pruning_metrics.pruned();total_matched += pruning_metrics.matched();+ total_fully_matched += pruning_metrics.fully_matched();+found = true;}}if found {- Some((total_pruned, total_matched))+ Some(PruningMetric {+ total_pruned,+ total_matched,+ total_fully_matched,+ })} else {None}@@ -172,27 +200,33 @@ impl TestOutput {}/// The number of row_groups pruned / matched by bloom filter- fn row_groups_bloom_filter(&self) -> Option<(usize, usize)> {+ fn row_groups_bloom_filter(&self) -> Option<PruningMetric> {self.pruning_metric("row_groups_pruned_bloom_filter")}/// The number of row_groups matched by statisticsfn row_groups_matched_statistics(&self) -> Option<usize> {self.pruning_metric("row_groups_pruned_statistics")- .map(|(_pruned, matched)| matched)+ .map(|pm| pm.total_matched())+ }++ /// The number of row_groups fully matched by statistics+ fn row_groups_fully_matched_statistics(&self) -> Option<usize> {+ self.pruning_metric("row_groups_pruned_statistics")+ .map(|pm| pm.total_fully_matched())}/// The number of row_groups pruned by statisticsfn row_groups_pruned_statistics(&self) -> Option<usize> {self.pruning_metric("row_groups_pruned_statistics")- .map(|(pruned, _matched)| pruned)+ .map(|pm| pm.total_pruned())}/// Metric `files_ranges_pruned_statistics` tracks both pruned and matched count,/// for testing purpose, here it only aggregate the `pruned` count.fn files_ranges_pruned_statistics(&self) -> Option<usize> {self.pruning_metric("files_ranges_pruned_statistics")- .map(|(pruned, _matched)| pruned)+ .map(|pm| pm.total_pruned())}/// The number of row_groups matched by bloom filter or statistics@@ -201,14 +235,13 @@ impl TestOutput {/// filter: 7 total -> 3 matched, this function returns 3 for the final matched/// count.fn row_groups_matched(&self) -> Option<usize> {- self.row_groups_bloom_filter()- .map(|(_pruned, matched)| matched)+ self.row_groups_bloom_filter().map(|pm| pm.total_matched())}/// The number of row_groups prunedfn row_groups_pruned(&self) -> Option<usize> {self.row_groups_bloom_filter()- .map(|(pruned, _matched)| pruned)+ .map(|pm| pm.total_pruned()).zip(self.row_groups_pruned_statistics()).map(|(a, b)| a + b)}@@ -216,7 +249,13 @@ impl TestOutput {/// The number of row pages prunedfn row_pages_pruned(&self) -> Option<usize> {self.pruning_metric("page_index_rows_pruned")- .map(|(pruned, _matched)| pruned)+ .map(|pm| pm.total_pruned())+ }++ /// The number of row groups pruned by limit pruning+ fn limit_pruned_row_groups(&self) -> Option<usize> {+ self.pruning_metric("limit_pruned_row_groups")+ .map(|pm| pm.total_pruned())}fn description(&self) -> String {@@ -232,20 +271,41 @@ impl TestOutput {/// and the appropriate scenarioimpl ContextWithParquet {async fn new(scenario: Scenario, unit: Unit) -> Self {- Self::with_config(scenario, unit, SessionConfig::new()).await+ Self::with_config(scenario, unit, SessionConfig::new(), None, None).await+ }++ /// Set custom schema and batches for the test+ pub async fn with_custom_data(+ scenario: Scenario,+ unit: Unit,+ schema: Arc<Schema>,+ batches: Vec<RecordBatch>,+ ) -> Self {+ Self::with_config(+ scenario,+ unit,+ SessionConfig::new(),+ Some(schema),+ Some(batches),+ )+ .await}async fn with_config(scenario: Scenario,unit: Unit,mut config: SessionConfig,+ custom_schema: Option<SchemaRef>,+ custom_batches: Option<Vec<RecordBatch>>,) -> Self {// Use a single partition for deterministic results no matter how many CPUs the host hasconfig = config.with_target_partitions(1);let file = match unit {Unit::RowGroup(row_per_group) => {config = config.with_parquet_bloom_filter_pruning(true);- make_test_file_rg(scenario, row_per_group).await+ config.options_mut().execution.parquet.pushdown_filters = true;+ make_test_file_rg(scenario, row_per_group, custom_schema, custom_batches)+ .await}Unit::Page(row_per_page) => {config = config.with_parquet_page_index_pruning(true);@@ -1075,7 +1135,12 @@ fn create_data_batch(scenario: Scenario) -> Vec<RecordBatch> {}/// Create a test parquet file with various data types-async fn make_test_file_rg(scenario: Scenario, row_per_group: usize) -> NamedTempFile {+async fn make_test_file_rg(+ scenario: Scenario,+ row_per_group: usize,+ custom_schema: Option<SchemaRef>,+ custom_batches: Option<Vec<RecordBatch>>,+) -> NamedTempFile {let mut output_file = tempfile::Builder::new().prefix("parquet_pruning").suffix(".parquet")@@ -1088,8 +1153,14 @@ async fn make_test_file_rg(scenario: Scenario, row_per_group: usize) -> NamedTem.set_statistics_enabled(EnabledStatistics::Page).build();- let batches = create_data_batch(scenario);- let schema = batches[0].schema();+ let (batches, schema) =+ if let (Some(schema), Some(batches)) = (custom_schema, custom_batches) {+ (batches, schema)+ } else {+ let batches = create_data_batch(scenario);+ let schema = batches[0].schema();+ (batches, schema)+ };let mut writer = ArrowWriter::try_new(&mut output_file, schema, Some(props)).unwrap();diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rsindex 0411298055..445ae7e97f 100644--- a/datafusion/core/tests/parquet/row_group_pruning.rs+++ b/datafusion/core/tests/parquet/row_group_pruning.rs@@ -18,8 +18,12 @@//! This file contains an end to end test of parquet pruning. It writes//! data into a parquet file and then verifies row groups are pruned as//! expected.+use std::sync::Arc;++use arrow::array::{ArrayRef, Int32Array, RecordBatch};+use arrow_schema::{DataType, Field, Schema};use datafusion::prelude::SessionConfig;-use datafusion_common::ScalarValue;+use datafusion_common::{DataFusionError, ScalarValue};use itertools::Itertools;use crate::parquet::Unit::RowGroup;@@ -30,10 +34,12 @@ struct RowGroupPruningTest {query: String,expected_errors: Option<usize>,expected_row_group_matched_by_statistics: Option<usize>,+ expected_row_group_fully_matched_by_statistics: Option<usize>,expected_row_group_pruned_by_statistics: Option<usize>,expected_files_pruned_by_statistics: Option<usize>,expected_row_group_matched_by_bloom_filter: Option<usize>,expected_row_group_pruned_by_bloom_filter: Option<usize>,+ expected_limit_pruned_row_groups: Option<usize>,expected_rows: usize,}impl RowGroupPruningTest {@@ -45,9 +51,11 @@ impl RowGroupPruningTest {expected_errors: None,expected_row_group_matched_by_statistics: None,expected_row_group_pruned_by_statistics: None,+ expected_row_group_fully_matched_by_statistics: None,expected_files_pruned_by_statistics: None,expected_row_group_matched_by_bloom_filter: None,expected_row_group_pruned_by_bloom_filter: None,+ expected_limit_pruned_row_groups: None,expected_rows: 0,}}@@ -76,6 +84,15 @@ impl RowGroupPruningTest {self}+ // Set the expected fully matched row groups by statistics+ fn with_fully_matched_by_stats(+ mut self,+ fully_matched_by_stats: Option<usize>,+ ) -> Self {+ self.expected_row_group_fully_matched_by_statistics = fully_matched_by_stats;+ self+ }+// Set the expected pruned row groups by statisticsfn with_pruned_by_stats(mut self, pruned_by_stats: Option<usize>) -> Self {self.expected_row_group_pruned_by_statistics = pruned_by_stats;@@ -99,6 +116,11 @@ impl RowGroupPruningTest {self}+ fn with_limit_pruned_row_groups(mut self, pruned_by_limit: Option<usize>) -> Self {+ self.expected_limit_pruned_row_groups = pruned_by_limit;+ self+ }+/// Set the number of expected rows from the output of this testfn with_expected_rows(mut self, rows: usize) -> Self {self.expected_rows = rows;@@ -135,15 +157,74 @@ impl RowGroupPruningTest {);let bloom_filter_metrics = output.row_groups_bloom_filter();assert_eq!(- bloom_filter_metrics.map(|(_pruned, matched)| matched),+ bloom_filter_metrics.as_ref().map(|pm| pm.total_matched()),self.expected_row_group_matched_by_bloom_filter,"mismatched row_groups_matched_bloom_filter",);assert_eq!(- bloom_filter_metrics.map(|(pruned, _matched)| pruned),+ bloom_filter_metrics.map(|pm| pm.total_pruned()),self.expected_row_group_pruned_by_bloom_filter,"mismatched row_groups_pruned_bloom_filter",);++ assert_eq!(+ output.result_rows,+ self.expected_rows,+ "Expected {} rows, got {}: {}",+ output.result_rows,+ self.expected_rows,+ output.description(),+ );+ }++ // Execute the test with the current configuration+ async fn test_row_group_prune_with_custom_data(+ self,+ schema: Arc<Schema>,+ batches: Vec<RecordBatch>,+ max_row_per_group: usize,+ ) {+ let output = ContextWithParquet::with_custom_data(+ self.scenario,+ RowGroup(max_row_per_group),+ schema,+ batches,+ )+ .await+ .query(&self.query)+ .await;++ println!("{}", output.description());+ assert_eq!(+ output.predicate_evaluation_errors(),+ self.expected_errors,+ "mismatched predicate_evaluation error"+ );+ assert_eq!(+ output.row_groups_matched_statistics(),+ self.expected_row_group_matched_by_statistics,+ "mismatched row_groups_matched_statistics",+ );+ assert_eq!(+ output.row_groups_fully_matched_statistics(),+ self.expected_row_group_fully_matched_by_statistics,+ "mismatched row_groups_fully_matched_statistics",+ );+ assert_eq!(+ output.row_groups_pruned_statistics(),+ self.expected_row_group_pruned_by_statistics,+ "mismatched row_groups_pruned_statistics",+ );+ assert_eq!(+ output.files_ranges_pruned_statistics(),+ self.expected_files_pruned_by_statistics,+ "mismatched files_ranges_pruned_statistics",+ );+ assert_eq!(+ output.limit_pruned_row_groups(),+ self.expected_limit_pruned_row_groups,+ "mismatched limit_pruned_row_groups",+ );assert_eq!(output.result_rows,self.expected_rows,@@ -289,11 +370,16 @@ async fn prune_disabled() {let expected_rows = 10;let config = SessionConfig::new().with_parquet_pruning(false);- let output =- ContextWithParquet::with_config(Scenario::Timestamps, RowGroup(5), config)- .await- .query(query)- .await;+ let output = ContextWithParquet::with_config(+ Scenario::Timestamps,+ RowGroup(5),+ config,+ None,+ None,+ )+ .await+ .query(query)+ .await;println!("{}", output.description());// This should not prune any@@ -1636,3 +1722,240 @@ async fn test_bloom_filter_decimal_dict() {.test_row_group_prune().await;}++// Helper function to create a batch with a single Int32 column.+fn make_i32_batch(+ name: &str,+ values: Vec<i32>,+) -> datafusion_common::error::Result<RecordBatch> {+ let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Int32, false)]));+ let array: ArrayRef = Arc::new(Int32Array::from(values));+ RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)+}++// Helper function to create a batch with two Int32 columns+fn make_two_col_i32_batch(+ name_a: &str,+ name_b: &str,+ values_a: Vec<i32>,+ values_b: Vec<i32>,+) -> datafusion_common::error::Result<RecordBatch> {+ let schema = Arc::new(Schema::new(vec![+ Field::new(name_a, DataType::Int32, false),+ Field::new(name_b, DataType::Int32, false),+ ]));+ let array_a: ArrayRef = Arc::new(Int32Array::from(values_a));+ let array_b: ArrayRef = Arc::new(Int32Array::from(values_b));+ RecordBatch::try_new(schema, vec![array_a, array_b]).map_err(DataFusionError::from)+}++#[tokio::test]+async fn test_limit_pruning_basic() -> datafusion_common::error::Result<()> {+ // Scenario: Simple integer column, multiple row groups+ // Query: SELECT c1 FROM t WHERE c1 = 0 LIMIT 2+ // We expect 2 rows in total.++ // Row Group 0: c1 = [0, -2] -> Partially matched, 1 row+ // Row Group 1: c1 = [1, 2] -> Fully matched, 2 rows+ // Row Group 2: c1 = [3, 4] -> Fully matched, 2 rows+ // Row Group 3: c1 = [5, 6] -> Fully matched, 2 rows+ // Row Group 4: c1 = [-1, -2] -> Not matched++ // If limit = 2, and RG1 is fully matched and has 2 rows, we should+ // only scan RG1 and prune other row groups+ // RG4 is pruned by statistics. RG2 and RG3 are pruned by limit.+ // So 2 row groups are effectively pruned due to limit pruning.++ let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)]));+ let query = "SELECT c1 FROM t WHERE c1 >= 0 LIMIT 2";++ let batches = vec![+ make_i32_batch("c1", vec![0, -2])?,+ make_i32_batch("c1", vec![0, 0])?,+ make_i32_batch("c1", vec![0, 0])?,+ make_i32_batch("c1", vec![0, 0])?,+ make_i32_batch("c1", vec![-1, -2])?,+ ];++ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int) // Assuming Scenario::Int can handle this data+ .with_query(query)+ .with_expected_errors(Some(0))+ .with_expected_rows(2)+ .with_pruned_files(Some(0))+ .with_matched_by_stats(Some(4))+ .with_fully_matched_by_stats(Some(3))+ .with_pruned_by_stats(Some(1))+ .with_limit_pruned_row_groups(Some(3))+ .test_row_group_prune_with_custom_data(schema, batches, 2)+ .await;++ Ok(())+}++#[tokio::test]+async fn test_limit_pruning_complex_filter() -> datafusion_common::error::Result<()> {+ // Test Case 1: Complex filter with two columns (a = 1 AND b > 1 AND b < 4)+ // Row Group 0: a=[1,1,1], b=[0,2,3] -> Partially matched, 2 rows match (b=2,3)+ // Row Group 1: a=[1,1,1], b=[2,2,2] -> Fully matched, 3 rows+ // Row Group 2: a=[1,1,1], b=[2,3,3] -> Fully matched, 3 rows+ // Row Group 3: a=[1,1,1], b=[2,2,3] -> Fully matched, 3 rows+ // Row Group 4: a=[2,2,2], b=[2,2,2] -> Not matched (a != 1)+ // Row Group 5: a=[1,1,1], b=[5,6,7] -> Not matched (b >= 4)++ // With LIMIT 5, we need RG1 (3 rows) + RG2 (2 rows from 3) = 5 rows+ // RG4 and RG5 should be pruned by statistics+ // RG3 should be pruned by limit+ // RG0 is partially matched, so it depends on the order++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]));+ let query = "SELECT a, b FROM t WHERE a = 1 AND b > 1 AND b < 4 LIMIT 5";++ let batches = vec![+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![0, 2, 3])?,+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 2, 2])?,+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 3, 3])?,+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 2, 3])?,+ make_two_col_i32_batch("a", "b", vec![2, 2, 2], vec![2, 2, 2])?,+ make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![5, 6, 7])?,+ ];++ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(query)+ .with_expected_errors(Some(0))+ .with_expected_rows(5)+ .with_pruned_files(Some(0))+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 are matched+ .with_fully_matched_by_stats(Some(3))+ .with_pruned_by_stats(Some(2)) // RG4,5 are pruned+ .with_limit_pruned_row_groups(Some(2)) // RG0, RG3 is pruned by limit+ .test_row_group_prune_with_custom_data(schema, batches, 3)+ .await;++ Ok(())+}++#[tokio::test]+async fn test_limit_pruning_multiple_fully_matched()+-> datafusion_common::error::Result<()> {+ // Test Case 2: Limit requires multiple fully matched row groups+ // Row Group 0: a=[5,5,5,5] -> Fully matched, 4 rows+ // Row Group 1: a=[5,5,5,5] -> Fully matched, 4 rows+ // Row Group 2: a=[5,5,5,5] -> Fully matched, 4 rows+ // Row Group 3: a=[5,5,5,5] -> Fully matched, 4 rows+ // Row Group 4: a=[1,2,3,4] -> Not matched++ // With LIMIT 8, we need RG0 (4 rows) + RG1 (4 rows) 8 rows+ // RG2,3 should be pruned by limit+ // RG4 should be pruned by statistics++ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let query = "SELECT a FROM t WHERE a = 5 LIMIT 8";++ let batches = vec![+ make_i32_batch("a", vec![5, 5, 5, 5])?,+ make_i32_batch("a", vec![5, 5, 5, 5])?,+ make_i32_batch("a", vec![5, 5, 5, 5])?,+ make_i32_batch("a", vec![5, 5, 5, 5])?,+ make_i32_batch("a", vec![1, 2, 3, 4])?,+ ];++ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(query)+ .with_expected_errors(Some(0))+ .with_expected_rows(8)+ .with_pruned_files(Some(0))+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched+ .with_fully_matched_by_stats(Some(4))+ .with_pruned_by_stats(Some(1)) // RG4 pruned+ .with_limit_pruned_row_groups(Some(2)) // RG2,3 pruned by limit+ .test_row_group_prune_with_custom_data(schema, batches, 4)+ .await;++ Ok(())+}++#[tokio::test]+async fn test_limit_pruning_no_fully_matched() -> datafusion_common::error::Result<()> {+ // Test Case 3: No fully matched row groups - all are partially matched+ // Row Group 0: a=[1,2,3] -> Partially matched, 1 row (a=2)+ // Row Group 1: a=[2,3,4] -> Partially matched, 1 row (a=2)+ // Row Group 2: a=[2,5,6] -> Partially matched, 1 row (a=2)+ // Row Group 3: a=[2,7,8] -> Partially matched, 1 row (a=2)+ // Row Group 4: a=[9,10,11] -> Not matched++ // With LIMIT 3, we need to scan RG0,1,2 to get 3 matching rows+ // Cannot prune much by limit since all matching RGs are partial+ // RG4 should be pruned by statistics++ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let query = "SELECT a FROM t WHERE a = 2 LIMIT 3";++ let batches = vec![+ make_i32_batch("a", vec![1, 2, 3])?,+ make_i32_batch("a", vec![2, 3, 4])?,+ make_i32_batch("a", vec![2, 5, 6])?,+ make_i32_batch("a", vec![2, 7, 8])?,+ make_i32_batch("a", vec![9, 10, 11])?,+ ];++ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(query)+ .with_expected_errors(Some(0))+ .with_expected_rows(3)+ .with_pruned_files(Some(0))+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched+ .with_fully_matched_by_stats(Some(0))+ .with_pruned_by_stats(Some(1)) // RG4 pruned+ .with_limit_pruned_row_groups(Some(0)) // RG3 pruned by limit+ .test_row_group_prune_with_custom_data(schema, batches, 3)+ .await;++ Ok(())+}++#[tokio::test]+async fn test_limit_pruning_exceeds_fully_matched() -> datafusion_common::error::Result<()>+{+ // Test Case 4: Limit exceeds all fully matched rows, need partially matched+ // Row Group 0: a=[10,11,12,12] -> Partially matched, 1 row (a=10)+ // Row Group 1: a=[10,10,10,10] -> Fully matched, 4 rows+ // Row Group 2: a=[10,10,10,10] -> Fully matched, 4 rows+ // Row Group 3: a=[10,13,14,11] -> Partially matched, 1 row (a=10)+ // Row Group 4: a=[20,21,22,22] -> Not matched++ // With LIMIT 10, we need RG1 (4) + RG2 (4) = 8 from fully matched+ // Still need 2 more, so we need to scan partially matched RG0 and RG3+ // All matching row groups should be scanned, only RG4 pruned by statistics++ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let query = "SELECT a FROM t WHERE a = 10 LIMIT 10";++ let batches = vec![+ make_i32_batch("a", vec![10, 11, 12, 12])?,+ make_i32_batch("a", vec![10, 10, 10, 10])?,+ make_i32_batch("a", vec![10, 10, 10, 10])?,+ make_i32_batch("a", vec![10, 13, 14, 11])?,+ make_i32_batch("a", vec![20, 21, 22, 22])?,+ ];++ RowGroupPruningTest::new()+ .with_scenario(Scenario::Int)+ .with_query(query)+ .with_expected_errors(Some(0))+ .with_expected_rows(10) // Total: 1 + 4 + 4 + 1 = 10+ .with_pruned_files(Some(0))+ .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched+ .with_fully_matched_by_stats(Some(2))+ .with_pruned_by_stats(Some(1)) // RG4 pruned+ .with_limit_pruned_row_groups(Some(0)) // No limit pruning since we need all RGs+ .test_row_group_prune_with_custom_data(schema, batches, 4)+ .await;+ Ok(())+}diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex 8ce3a081a2..fbb14d9a6d 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -45,9 +45,11 @@ pub struct ParquetFileMetrics {pub files_ranges_pruned_statistics: PruningMetrics,/// Number of times the predicate could not be evaluatedpub predicate_evaluation_errors: Count,- /// Number of row groups whose bloom filters were checked, tracked with matched/pruned counts+ /// Number of row groups pruned by bloom filterspub row_groups_pruned_bloom_filter: PruningMetrics,- /// Number of row groups whose statistics were checked, tracked with matched/pruned counts+ /// Number of row groups pruned due to limit pruning.+ pub limit_pruned_row_groups: PruningMetrics,+ /// Number of row groups pruned by statisticspub row_groups_pruned_statistics: PruningMetrics,/// Total number of bytes scannedpub bytes_scanned: Count,@@ -99,6 +101,11 @@ impl ParquetFileMetrics {.with_type(MetricType::SUMMARY).pruning_metrics("row_groups_pruned_bloom_filter", partition);+ let limit_pruned_row_groups = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .with_type(MetricType::SUMMARY)+ .pruning_metrics("limit_pruned_row_groups", partition);+let row_groups_pruned_statistics = MetricBuilder::new(metrics).with_new_label("filename", filename.to_string()).with_type(MetricType::SUMMARY)@@ -173,6 +180,7 @@ impl ParquetFileMetrics {predicate_evaluation_errors,row_groups_pruned_bloom_filter,row_groups_pruned_statistics,+ limit_pruned_row_groups,bytes_scanned,pushdown_rows_pruned,pushdown_rows_matched,diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 570f9b4412..8f31d2df24 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -69,13 +69,15 @@ use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMe/// Implements [`FileOpener`] for a parquet filepub(super) struct ParquetOpener {/// Execution partition index- pub partition_index: usize,+ pub(crate) partition_index: usize,/// Projection to apply on top of the table schema (i.e. can reference partition columns).pub projection: ProjectionExprs,/// Target number of rows in each output RecordBatchpub batch_size: usize,/// Optional limit on the number of rows to read- pub limit: Option<usize>,+ pub(crate) limit: Option<usize>,+ /// If should keep the output rows in order+ pub preserve_order: bool,/// Optional predicate to apply during the scanpub predicate: Option<Arc<dyn PhysicalExpr>>,/// Table schema, including partition columns.@@ -277,6 +279,8 @@ impl FileOpener for ParquetOpener {let max_predicate_cache_size = self.max_predicate_cache_size;let reverse_row_groups = self.reverse_row_groups;+ let preserve_order = self.preserve_order;+Ok(Box::pin(async move {#[cfg(feature = "parquet_encryption")]let file_decryption_properties = encryption_context@@ -545,11 +549,15 @@ impl FileOpener for ParquetOpener {.add_matched(n_remaining_row_groups);}- let mut access_plan = row_groups.build();+ // Prune by limit if limit is set and limit order is not sensitive+ if let (Some(limit), false) = (limit, preserve_order) {+ row_groups.prune_by_limit(limit, rg_metadata, &file_metrics);+ }// --------------------------------------------------------// Step: prune pages from the kept row groups//+ let mut access_plan = row_groups.build();// page index pruning: if all data on individual pages can// be ruled using page metadata, rows from other columns// with that range can be skipped as well@@ -1051,6 +1059,7 @@ mod test {coerce_int96: Option<arrow::datatypes::TimeUnit>,max_predicate_cache_size: Option<usize>,reverse_row_groups: bool,+ preserve_order: bool,}impl ParquetOpenerBuilder {@@ -1076,6 +1085,7 @@ mod test {coerce_int96: None,max_predicate_cache_size: None,reverse_row_groups: false,+ preserve_order: false,}}@@ -1183,6 +1193,7 @@ mod test {encryption_factory: None,max_predicate_cache_size: self.max_predicate_cache_size,reverse_row_groups: self.reverse_row_groups,+ preserve_order: self.preserve_order,}}}diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 046379cc25..7eea8285ad 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -24,6 +24,8 @@ use arrow::datatypes::Schema;use datafusion_common::pruning::PruningStatistics;use datafusion_common::{Column, Result, ScalarValue};use datafusion_datasource::FileRange;+use datafusion_physical_expr::PhysicalExprSimplifier;+use datafusion_physical_expr::expressions::NotExpr;use datafusion_pruning::PruningPredicate;use parquet::arrow::arrow_reader::statistics::StatisticsConverter;use parquet::arrow::parquet_column;@@ -46,13 +48,20 @@ use parquet::{pub struct RowGroupAccessPlanFilter {/// which row groups should be accessedaccess_plan: ParquetAccessPlan,+ /// Row groups where ALL rows are known to match the pruning predicate+ /// (the predicate does not filter any rows)+ is_fully_matched: Vec<bool>,}impl RowGroupAccessPlanFilter {/// Create a new `RowGroupPlanBuilder` for pruning out the groups to scan/// based on metadata and statisticspub fn new(access_plan: ParquetAccessPlan) -> Self {- Self { access_plan }+ let num_row_groups = access_plan.len();+ Self {+ access_plan,+ is_fully_matched: vec![false; num_row_groups],+ }}/// Return true if there are no row groups@@ -70,6 +79,139 @@ impl RowGroupAccessPlanFilter {self.access_plan}+ /// Returns the is_fully_matched vector+ pub fn is_fully_matched(&self) -> &Vec<bool> {+ &self.is_fully_matched+ }++ /// Prunes the access plan based on the limit and fully contained row groups.+ ///+ /// The pruning works by leveraging the concept of fully matched row groups. Consider a query like:+ /// `WHERE species LIKE 'Alpine%' AND s >= 50 LIMIT N`+ ///+ /// After initial filtering, row groups can be classified into three states:+ ///+ /// 1. Not Matching / Pruned+ /// 2. Partially Matching (Row Group/Page contains some matches)+ /// 3. Fully Matching (Entire range is within predicate)+ ///+ /// +-----------------------------------------------------------------------++ /// | NOT MATCHING |+ /// | Row group 1 |+ /// | +-----------------------------------+-----------------------------+ |+ /// | | SPECIES | S | |+ /// | +-----------------------------------+-----------------------------+ |+ /// | | Snow Vole | 7 | |+ /// | | Brown Bear | 133 ✅ | |+ /// | | Gray Wolf | 82 ✅ | |+ /// | +-----------------------------------+-----------------------------+ |+ /// +-----------------------------------------------------------------------++ ///+ /// +---------------------------------------------------------------------------++ /// | PARTIALLY MATCHING |+ /// | |+ /// | Row group 2 Row group 4 |+ /// | +------------------+--------------+ +------------------+----------+ |+ /// | | SPECIES | S | | SPECIES | S | |+ /// | +------------------+--------------+ +------------------+----------+ |+ /// | | Lynx | 71 ✅ | | Europ. Mole | 4 | |+ /// | | Red Fox | 40 | | Polecat | 16 | |+ /// | | Alpine Bat ✅ | 6 | | Alpine Ibex ✅ | 97 ✅ | |+ /// | +------------------+--------------+ +------------------+----------+ |+ /// +---------------------------------------------------------------------------++ ///+ /// +-----------------------------------------------------------------------++ /// | FULLY MATCHING |+ /// | Row group 3 |+ /// | +-----------------------------------+-----------------------------+ |+ /// | | SPECIES | S | |+ /// | +-----------------------------------+-----------------------------+ |+ /// | | Alpine Ibex ✅ | 101 ✅ | |+ /// | | Alpine Goat ✅ | 76 ✅ | |+ /// | | Alpine Sheep ✅ | 83 ✅ | |+ /// | +-----------------------------------+-----------------------------+ |+ /// +-----------------------------------------------------------------------++ ///+ /// ### Identification of Fully Matching Row Groups+ ///+ /// DataFusion identifies row groups where ALL rows satisfy the filter by inverting the+ /// predicate and checking if statistics prove the inverted version is false for the group.+ ///+ /// For example, prefix matches like `species LIKE 'Alpine%'` are pruned using ranges:+ /// 1. Candidate Range: `species >= 'Alpine' AND species < 'Alpinf'`+ /// 2. Inverted Condition (to prove full match): `species < 'Alpine' OR species >= 'Alpinf'`+ /// 3. Statistical Evaluation (check if any row *could* satisfy the inverted condition):+ /// `min < 'Alpine' OR max >= 'Alpinf'`+ ///+ /// If this evaluation is **false**, it proves no row can fail the original filter,+ /// so the row group is **FULLY MATCHING**.+ ///+ /// ### Impact of Statistics Truncation+ ///+ /// The precision of pruning depends on the metadata quality. Truncated statistics+ /// may prevent the system from proving a full match.+ ///+ /// **Example**: `WHERE species LIKE 'Alpine%'` (Target range: `['Alpine', 'Alpinf')`)+ ///+ /// | Truncation Length | min / max | Inverted Evaluation | Status |+ /// |-------------------|---------------------|---------------------------------------------------------------------|------------------------|+ /// | **Length 6** | `Alpine` / `Alpine` | `"Alpine" < "Alpine" (F) OR "Alpine" >= "Alpinf" (F)` -> **false** | **FULLY MATCHING** |+ /// | **Length 3** | `Alp` / `Alq` | `"Alp" < "Alpine" (T) OR "Alq" >= "Alpinf" (T)` -> **true** | **PARTIALLY MATCHING** |+ ///+ /// Even though Row Group 3 only contains matching rows, truncation to length 3 makes+ /// the statistics `[Alp, Alq]` too broad to prove it (they could include "Alpha").+ /// The system must conservatively scan the group.+ ///+ /// Without limit pruning: Scan Partition 2 → Partition 3 → Partition 4 (until limit reached)+ /// With limit pruning: If Partition 3 contains enough rows to satisfy the limit,+ /// skip Partitions 2 and 4 entirely and go directly to Partition 3.+ ///+ /// This optimization is particularly effective when:+ /// - The limit is small relative to the total dataset size+ /// - There are row groups that are fully matched by the filter predicates+ /// - The fully matched row groups contain sufficient rows to satisfy the limit+ ///+ /// For more information, see the [paper](https://arxiv.org/pdf/2504.11540)'s "Pruning for LIMIT Queries" part+ pub fn prune_by_limit(+ &mut self,+ limit: usize,+ rg_metadata: &[RowGroupMetaData],+ metrics: &ParquetFileMetrics,+ ) {+ let mut fully_matched_row_group_indexes: Vec<usize> = Vec::new();+ let mut fully_matched_rows_count: usize = 0;++ // Iterate through the currently accessible row groups and try to+ // find a set of matching row groups that can satisfy the limit+ for &idx in self.access_plan.row_group_indexes().iter() {+ if self.is_fully_matched[idx] {+ let row_group_row_count = rg_metadata[idx].num_rows() as usize;+ fully_matched_row_group_indexes.push(idx);+ fully_matched_rows_count += row_group_row_count;+ if fully_matched_rows_count >= limit {+ break;+ }+ }+ }++ // If we can satisfy the limit with fully matching row groups,+ // rewrite the plan to do so+ if fully_matched_rows_count >= limit {+ let original_num_accessible_row_groups =+ self.access_plan.row_group_indexes().len();+ let new_num_accessible_row_groups = fully_matched_row_group_indexes.len();+ let pruned_count = original_num_accessible_row_groups+ .saturating_sub(new_num_accessible_row_groups);+ metrics.limit_pruned_row_groups.add_pruned(pruned_count);++ let mut new_access_plan = ParquetAccessPlan::new_none(rg_metadata.len());+ for &idx in &fully_matched_row_group_indexes {+ new_access_plan.scan(idx);+ }+ self.access_plan = new_access_plan;+ }+ }+/// Prune remaining row groups to only those within the specified range.////// Updates this set to mark row groups that should not be scanned@@ -135,15 +277,26 @@ impl RowGroupAccessPlanFilter {// try to prune the row groups in a single callmatch predicate.prune(&pruning_stats) {Ok(values) => {- // values[i] is false means the predicate could not be true for row group i+ let mut fully_contained_candidates_original_idx: Vec<usize> = Vec::new();for (idx, &value) in row_group_indexes.iter().zip(values.iter()) {if !value {self.access_plan.skip(*idx);metrics.row_groups_pruned_statistics.add_pruned(1);} else {metrics.row_groups_pruned_statistics.add_matched(1);+ fully_contained_candidates_original_idx.push(*idx);}}++ // Check if any of the matched row groups are fully contained by the predicate+ self.identify_fully_matched_row_groups(+ &fully_contained_candidates_original_idx,+ arrow_schema,+ parquet_schema,+ groups,+ predicate,+ metrics,+ );}// stats filter array could not be built, so we can't pruneErr(e) => {@@ -153,6 +306,68 @@ impl RowGroupAccessPlanFilter {}}+ /// Identifies row groups that are fully matched by the predicate.+ ///+ /// This optimization checks whether all rows in a row group satisfy the predicate+ /// by inverting the predicate and checking if it prunes the row group. If the+ /// inverted predicate prunes a row group, it means no rows match the inverted+ /// predicate, which implies all rows match the original predicate.+ ///+ /// Note: This optimization is relatively inexpensive for a limited number of row groups.+ fn identify_fully_matched_row_groups(+ &mut self,+ candidate_row_group_indices: &[usize],+ arrow_schema: &Schema,+ parquet_schema: &SchemaDescriptor,+ groups: &[RowGroupMetaData],+ predicate: &PruningPredicate,+ metrics: &ParquetFileMetrics,+ ) {+ if candidate_row_group_indices.is_empty() {+ return;+ }++ // Use NotExpr to create the inverted predicate+ let inverted_expr = Arc::new(NotExpr::new(Arc::clone(predicate.orig_expr())));++ // Simplify the NOT expression (e.g., NOT(c1 = 0) -> c1 != 0)+ // before building the pruning predicate+ let simplifier = PhysicalExprSimplifier::new(arrow_schema);+ let Ok(inverted_expr) = simplifier.simplify(inverted_expr) else {+ return;+ };++ let Ok(inverted_predicate) =+ PruningPredicate::try_new(inverted_expr, Arc::clone(predicate.schema()))+ else {+ return;+ };++ let inverted_pruning_stats = RowGroupPruningStatistics {+ parquet_schema,+ row_group_metadatas: candidate_row_group_indices+ .iter()+ .map(|&i| &groups[i])+ .collect::<Vec<_>>(),+ arrow_schema,+ };++ let Ok(inverted_values) = inverted_predicate.prune(&inverted_pruning_stats)+ else {+ return;+ };++ for (i, &original_row_group_idx) in candidate_row_group_indices.iter().enumerate()+ {+ // If the inverted predicate *also* prunes this row group (meaning inverted_values[i] is false),+ // it implies that *all* rows in this group satisfy the original predicate.+ if !inverted_values[i] {+ self.is_fully_matched[original_row_group_idx] = true;+ metrics.row_groups_pruned_statistics.add_fully_matched(1);+ }+ }+ }+/// Prune remaining row groups using available bloom filters and the/// [`PruningPredicate`].///diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 2e0919b144..07f58db185 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -548,6 +548,7 @@ impl FileSource for ParquetSource {.batch_size.expect("Batch size must set before creating ParquetOpener"),limit: base_config.limit,+ preserve_order: base_config.preserve_order,predicate: self.predicate.clone(),table_schema: self.table_schema.clone(),metadata_size_hint: self.metadata_size_hint,diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex 1f7c37315c..51b9ba9e06 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -152,6 +152,11 @@ pub struct FileScanConfig {/// The maximum number of records to read from this plan. If `None`,/// all records after filtering are returned.pub limit: Option<usize>,+ /// Whether the scan's limit is order sensitive+ /// When `true`, files must be read in the exact order specified to produce+ /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,+ /// DataFusion may reorder file processing for optimization without affecting correctness.+ pub preserve_order: bool,/// All equivalent lexicographical orderings that describe the schema.pub output_ordering: Vec<LexOrdering>,/// File compression type@@ -240,6 +245,7 @@ pub struct FileScanConfigBuilder {object_store_url: ObjectStoreUrl,file_source: Arc<dyn FileSource>,limit: Option<usize>,+ preserve_order: bool,constraints: Option<Constraints>,file_groups: Vec<FileGroup>,statistics: Option<Statistics>,@@ -269,6 +275,7 @@ impl FileScanConfigBuilder {output_ordering: vec![],file_compression_type: None,limit: None,+ preserve_order: false,constraints: None,batch_size: None,expr_adapter_factory: None,@@ -283,6 +290,15 @@ impl FileScanConfigBuilder {self}+ /// Set whether the limit should be order-sensitive.+ /// When `true`, files must be read in the exact order specified to produce+ /// correct results (e.g., for `ORDER BY ... LIMIT` queries). When `false`,+ /// DataFusion may reorder file processing for optimization without affecting correctness.+ pub fn with_preserve_order(mut self, order_sensitive: bool) -> Self {+ self.preserve_order = order_sensitive;+ self+ }+/// Set the file source for scanning files.////// This method allows you to change the file source implementation (e.g. ParquetSource, CsvSource, etc.)@@ -450,6 +466,7 @@ impl FileScanConfigBuilder {object_store_url,file_source,limit,+ preserve_order,constraints,file_groups,statistics,@@ -467,10 +484,14 @@ impl FileScanConfigBuilder {let file_compression_type =file_compression_type.unwrap_or(FileCompressionType::UNCOMPRESSED);+ // If there is an output ordering, we should preserve it.+ let preserve_order = preserve_order || !output_ordering.is_empty();+FileScanConfig {object_store_url,file_source,limit,+ preserve_order,constraints,file_groups,output_ordering,@@ -493,6 +514,7 @@ impl From<FileScanConfig> for FileScanConfigBuilder {output_ordering: config.output_ordering,file_compression_type: Some(config.file_compression_type),limit: config.limit,+ preserve_order: config.preserve_order,constraints: Some(config.constraints),batch_size: config.batch_size,expr_adapter_factory: config.expr_adapter_factory,@@ -850,6 +872,18 @@ impl DataSource for FileScanConfig {}}}++ fn with_preserve_order(&self, preserve_order: bool) -> Option<Arc<dyn DataSource>> {+ if self.preserve_order == preserve_order {+ return Some(Arc::new(self.clone()));+ }++ let new_config = FileScanConfig {+ preserve_order,+ ..self.clone()+ };+ Some(Arc::new(new_config))+ }}impl FileScanConfig {diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex a3892dfac9..de18b6be22 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -210,6 +210,11 @@ pub trait DataSource: Send + Sync + Debug {) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {Ok(SortOrderPushdownResult::Unsupported)}++ /// Returns a variant of this `DataSource` that is aware of order-sensitivity.+ fn with_preserve_order(&self, _preserve_order: bool) -> Option<Arc<dyn DataSource>> {+ None+ }}/// [`ExecutionPlan`] that reads one or more files@@ -393,6 +398,18 @@ impl ExecutionPlan for DataSourceExec {Ok(Arc::new(new_exec) as Arc<dyn ExecutionPlan>)})}++ fn with_preserve_order(+ &self,+ preserve_order: bool,+ ) -> Option<Arc<dyn ExecutionPlan>> {+ self.data_source+ .with_preserve_order(preserve_order)+ .map(|new_data_source| {+ Arc::new(self.clone().with_data_source(new_data_source))+ as Arc<dyn ExecutionPlan>+ })+ }}impl DataSourceExec {diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex 548eadffa2..f97b05ea68 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -268,15 +268,10 @@ fn optimize_projections(Some(projection) => indices.into_mapped_indices(|idx| projection[idx]),None => indices.into_inner(),};- return TableScan::try_new(- table_name,- source,- Some(projection),- filters,- fetch,- )- .map(LogicalPlan::TableScan)- .map(Transformed::yes);+ let new_scan =+ TableScan::try_new(table_name, source, Some(projection), filters, fetch)?;++ return Ok(Transformed::yes(LogicalPlan::TableScan(new_scan)));}// Other node types are handled below_ => {}diff --git a/datafusion/physical-expr-common/src/metrics/value.rs b/datafusion/physical-expr-common/src/metrics/value.rsindex 9a14b804a2..5ecaa86fc3 100644--- a/datafusion/physical-expr-common/src/metrics/value.rs+++ b/datafusion/physical-expr-common/src/metrics/value.rs@@ -372,19 +372,31 @@ impl Drop for ScopedTimerGuard<'_> {pub struct PruningMetrics {pruned: Arc<AtomicUsize>,matched: Arc<AtomicUsize>,+ fully_matched: Arc<AtomicUsize>,}impl Display for PruningMetrics {fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {let matched = self.matched.load(Ordering::Relaxed);let total = self.pruned.load(Ordering::Relaxed) + matched;+ let fully_matched = self.fully_matched.load(Ordering::Relaxed);- write!(- f,- "{} total → {} matched",- human_readable_count(total),- human_readable_count(matched)- )+ if fully_matched != 0 {+ write!(+ f,+ "{} total → {} matched -> {} fully matched",+ human_readable_count(total),+ human_readable_count(matched),+ human_readable_count(fully_matched)+ )+ } else {+ write!(+ f,+ "{} total → {} matched",+ human_readable_count(total),+ human_readable_count(matched)+ )+ }}}@@ -400,6 +412,7 @@ impl PruningMetrics {Self {pruned: Arc::new(AtomicUsize::new(0)),matched: Arc::new(AtomicUsize::new(0)),+ fully_matched: Arc::new(AtomicUsize::new(0)),}}@@ -417,6 +430,13 @@ impl PruningMetrics {self.matched.fetch_add(n, Ordering::Relaxed);}+ /// Add `n` to the metric's fully matched value+ pub fn add_fully_matched(&self, n: usize) {+ // relaxed ordering for operations on `value` poses no issues+ // we're purely using atomic ops with no associated memory ops+ self.fully_matched.fetch_add(n, Ordering::Relaxed);+ }+/// Subtract `n` to the metric's matched value.pub fn subtract_matched(&self, n: usize) {// relaxed ordering for operations on `value` poses no issues@@ -433,6 +453,11 @@ impl PruningMetrics {pub fn matched(&self) -> usize {self.matched.load(Ordering::Relaxed)}++ /// Number of items fully matched+ pub fn fully_matched(&self) -> usize {+ self.fully_matched.load(Ordering::Relaxed)+ }}/// Counters tracking ratio metrics (e.g. matched vs total)@@ -906,8 +931,11 @@ impl MetricValue {) => {let pruned = other_pruning_metrics.pruned.load(Ordering::Relaxed);let matched = other_pruning_metrics.matched.load(Ordering::Relaxed);+ let fully_matched =+ other_pruning_metrics.fully_matched.load(Ordering::Relaxed);pruning_metrics.add_pruned(pruned);pruning_metrics.add_matched(matched);+ pruning_metrics.add_fully_matched(fully_matched);}(Self::Ratio { ratio_metrics, .. },diff --git a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/enforce_sorting/mod.rsindex a5fafb9e87..247ebb2785 100644--- a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs+++ b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs@@ -581,11 +581,17 @@ fn analyze_immediate_sort_removal(// Remove the sort:node.children = node.children.swap_remove(0).children;if let Some(fetch) = sort_exec.fetch() {+ let required_ordering = sort_exec.properties().output_ordering().cloned();// If the sort has a fetch, we need to add a limit:if properties.output_partitioning().partition_count() == 1 {- Arc::new(GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch)))+ let mut global_limit =+ GlobalLimitExec::new(Arc::clone(sort_input), 0, Some(fetch));+ global_limit.set_required_ordering(required_ordering);+ Arc::new(global_limit)} else {- Arc::new(LocalLimitExec::new(Arc::clone(sort_input), fetch))+ let mut local_limit = LocalLimitExec::new(Arc::clone(sort_input), fetch);+ local_limit.set_required_ordering(required_ordering);+ Arc::new(local_limit)}} else {Arc::clone(sort_input)diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rsindex 4cb3abe30b..a4dac81dba 100644--- a/datafusion/physical-optimizer/src/limit_pushdown.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs@@ -50,6 +50,7 @@ pub struct GlobalRequirements {fetch: Option<usize>,skip: usize,satisfied: bool,+ preserve_order: bool,}impl LimitPushdown {@@ -69,6 +70,7 @@ impl PhysicalOptimizerRule for LimitPushdown {fetch: None,skip: 0,satisfied: false,+ preserve_order: false,};pushdown_limits(plan, global_state)}@@ -111,6 +113,13 @@ impl LimitExec {Self::Local(_) => 0,}}++ fn preserve_order(&self) -> bool {+ match self {+ Self::Global(global) => global.required_ordering().is_some(),+ Self::Local(local) => local.required_ordering().is_some(),+ }+ }}impl From<LimitExec> for Arc<dyn ExecutionPlan> {@@ -145,6 +154,7 @@ pub fn pushdown_limit_helper();global_state.skip = skip;global_state.fetch = fetch;+ global_state.preserve_order = limit_exec.preserve_order();// Now the global state has the most recent information, we can remove// the `LimitExec` plan. We will decide later if we should add it again@@ -241,17 +251,28 @@ pub fn pushdown_limit_helper(let maybe_fetchable = pushdown_plan.with_fetch(skip_and_fetch);if global_state.satisfied {if let Some(plan_with_fetch) = maybe_fetchable {- Ok((Transformed::yes(plan_with_fetch), global_state))+ let plan_with_preserve_order = plan_with_fetch+ .with_preserve_order(global_state.preserve_order)+ .unwrap_or(plan_with_fetch);+ Ok((Transformed::yes(plan_with_preserve_order), global_state))} else {Ok((Transformed::no(pushdown_plan), global_state))}} else {global_state.satisfied = true;pushdown_plan = if let Some(plan_with_fetch) = maybe_fetchable {+ let plan_with_preserve_order = plan_with_fetch+ .with_preserve_order(global_state.preserve_order)+ .unwrap_or(plan_with_fetch);+if global_skip > 0 {- add_global_limit(plan_with_fetch, global_skip, Some(global_fetch))+ add_global_limit(+ plan_with_preserve_order,+ global_skip,+ Some(global_fetch),+ )} else {- plan_with_fetch+ plan_with_preserve_order}} else {add_limit(pushdown_plan, global_skip, global_fetch)diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex d83f90eb3d..22dcc85d6e 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -278,6 +278,19 @@ impl ExecutionPlan for CoalescePartitionsExec {}))}+ fn with_preserve_order(+ &self,+ preserve_order: bool,+ ) -> Option<Arc<dyn ExecutionPlan>> {+ self.input+ .with_preserve_order(preserve_order)+ .and_then(|new_input| {+ Arc::new(self.clone())+ .with_new_children(vec![new_input])+ .ok()+ })+ }+fn gather_filters_for_pushdown(&self,_phase: FilterPushdownPhase,diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex 8d72921d06..52f4829127 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -709,6 +709,19 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {Ok(SortOrderPushdownResult::Unsupported)}++ /// Returns a variant of this `ExecutionPlan` that is aware of order-sensitivity.+ ///+ /// This is used to signal to data sources that the output ordering must be+ /// preserved, even if it might be more efficient to ignore it (e.g. by+ /// skipping some row groups in Parquet).+ ///+ fn with_preserve_order(+ &self,+ _preserve_order: bool,+ ) -> Option<Arc<dyn ExecutionPlan>> {+ None+ }}/// [`ExecutionPlan`] Invariant Leveldiff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex e78e50c986..76516c25ad 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -614,6 +614,19 @@ impl ExecutionPlan for FilterExec {fetch,}))}++ fn with_preserve_order(+ &self,+ preserve_order: bool,+ ) -> Option<Arc<dyn ExecutionPlan>> {+ self.input+ .with_preserve_order(preserve_order)+ .and_then(|new_input| {+ Arc::new(self.clone())+ .with_new_children(vec![new_input])+ .ok()+ })+ }}impl EmbeddedProjection for FilterExec {diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex 05d6882821..fea7acb221 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -35,6 +35,7 @@ use arrow::record_batch::RecordBatch;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;+use datafusion_physical_expr::LexOrdering;use futures::stream::{Stream, StreamExt};use log::trace;@@ -51,6 +52,9 @@ pub struct GlobalLimitExec {/// Execution metricsmetrics: ExecutionPlanMetricsSet,cache: PlanProperties,+ /// Does the limit have to preserve the order of its input, and if so what is it?+ /// Some optimizations may reorder the input if no particular sort is required+ required_ordering: Option<LexOrdering>,}impl GlobalLimitExec {@@ -63,6 +67,7 @@ impl GlobalLimitExec {fetch,metrics: ExecutionPlanMetricsSet::new(),cache,+ required_ordering: None,}}@@ -91,6 +96,16 @@ impl GlobalLimitExec {Boundedness::Bounded,)}++ /// Get the required ordering from limit+ pub fn required_ordering(&self) -> &Option<LexOrdering> {+ &self.required_ordering+ }++ /// Set the required ordering for limit+ pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {+ self.required_ordering = required_ordering;+ }}impl DisplayAs for GlobalLimitExec {@@ -223,6 +238,9 @@ pub struct LocalLimitExec {/// Execution metricsmetrics: ExecutionPlanMetricsSet,cache: PlanProperties,+ /// If the child plan is a sort node, after the sort node is removed during+ /// physical optimization, we should add the required ordering to the limit node+ required_ordering: Option<LexOrdering>,}impl LocalLimitExec {@@ -234,6 +252,7 @@ impl LocalLimitExec {fetch,metrics: ExecutionPlanMetricsSet::new(),cache,+ required_ordering: None,}}@@ -257,6 +276,16 @@ impl LocalLimitExec {Boundedness::Bounded,)}++ /// Get the required ordering from limit+ pub fn required_ordering(&self) -> &Option<LexOrdering> {+ &self.required_ordering+ }++ /// Set the required ordering for limit+ pub fn set_required_ordering(&mut self, required_ordering: Option<LexOrdering>) {+ self.required_ordering = required_ordering;+ }}impl DisplayAs for LocalLimitExec {diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex e8608f17a1..8f2f2219f4 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -427,6 +427,19 @@ impl ExecutionPlan for ProjectionExec {}}}++ fn with_preserve_order(+ &self,+ preserve_order: bool,+ ) -> Option<Arc<dyn ExecutionPlan>> {+ self.input+ .with_preserve_order(preserve_order)+ .and_then(|new_input| {+ Arc::new(self.clone())+ .with_new_children(vec![new_input])+ .ok()+ })+ }}impl ProjectionStream {diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex 0ddea90a98..68c457a0d8 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -245,6 +245,19 @@ impl ExecutionPlan for SortPreservingMergeExec {}))}+ fn with_preserve_order(+ &self,+ preserve_order: bool,+ ) -> Option<Arc<dyn ExecutionPlan>> {+ self.input+ .with_preserve_order(preserve_order)+ .and_then(|new_input| {+ Arc::new(self.clone())+ .with_new_children(vec![new_input])+ .ok()+ })+ }+fn required_input_distribution(&self) -> Vec<Distribution> {vec![Distribution::UnspecifiedDistribution]}diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rsindex b5b8267d7f..5f1b4233b5 100644--- a/datafusion/pruning/src/pruning_predicate.rs+++ b/datafusion/pruning/src/pruning_predicate.rs@@ -492,7 +492,6 @@ impl PruningPredicate {// Simplify the newly created predicate to get rid of redundant casts, comparisons, etc.let predicate_expr =PhysicalExprSimplifier::new(&predicate_schema).simplify(predicate_expr)?;-let literal_guarantees = LiteralGuarantee::analyze(&expr);Ok(Self {diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.sltnew file mode 100644index 0000000000..8a94bf8adc--- /dev/null+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt@@ -0,0 +1,81 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++statement ok+set datafusion.execution.parquet.pushdown_filters = true;+++statement ok+CREATE TABLE tracking_data AS VALUES+-- ***** Row Group 0 *****+ ('Anow Vole', 7),+ ('Brown Bear', 133),+ ('Gray Wolf', 82),+-- ***** Row Group 1 *****+ ('Lynx', 71),+ ('Red Fox', 40),+ ('Alpine Bat', 6),+-- ***** Row Group 2 *****+ ('Nlpine Ibex', 101),+ ('Nlpine Goat', 76),+ ('Nlpine Sheep', 83),+-- ***** Row Group 3 *****+ ('Europ. Mole', 4),+ ('Polecat', 16),+ ('Alpine Ibex', 97);++statement ok+COPY (SELECT column1 as species, column2 as s FROM tracking_data)+TO 'test_files/scratch/limit_pruning/data.parquet'+STORED AS PARQUET+OPTIONS (+ 'format.max_row_group_size' '3'+);++statement ok+drop table tracking_data;++statement ok+CREATE EXTERNAL TABLE tracking_data+STORED AS PARQUET+LOCATION 'test_files/scratch/limit_pruning/data.parquet';+++statement ok+set datafusion.explain.analyze_level = summary;++# row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched+# limit_pruned_row_groups=2 total → 0 matched+query TT+explain analyze select * from tracking_data where species > 'M' AND s >= 50 limit 3;+----+Plan with Metrics DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], limit=3, file_type=parquet, predicate=species@0 > M AND s@1 >= 50, pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_rows_pruned=3 total → 3 matched, limit_pruned_row_groups=2 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (171/2.35 K)]++# limit_pruned_row_groups=0 total → 0 matched+# because of order by, scan needs to preserve sort, so limit pruning is disabled+query TT+explain analyze select * from tracking_data where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_rows_pruned=9 total → 9 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]++statement ok+drop table tracking_data;++statement ok+reset datafusion.explain.analyze_level;diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.mdindex 5a1184539c..8fe8316381 100644--- a/docs/source/user-guide/explain-usage.md+++ b/docs/source/user-guide/explain-usage.md@@ -228,6 +228,7 @@ When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.- `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched.+- `limit_pruned_row_groups`: number of row groups pruned by the limit.- `pushdown_rows_matched`: rows that were tested by any of the above filters, and passed all of them.- `pushdown_rows_pruned`: rows that were tested by any of the above filters, and did not pass at least one of them.- `predicate_evaluation_errors`: number of times evaluating the filter expression failed (expected to be zero in normal operation)