Skip to content

Apache DataFusion SQL Query Engine

Skip RowFilter and page pruning for fully matched row groups (#21637)

xudong.w3f501f4b9e6b
diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rs
index ca4d097c37..295665055b 100644
--- a/datafusion/datasource-parquet/src/access_plan.rs
+++ b/datafusion/datasource-parquet/src/access_plan.rs
@@ -91,6 +91,12 @@ use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};
pub struct ParquetAccessPlan {
/// How to access the i-th row group
row_groups: Vec<RowGroupAccess>,
+ /// Whether all rows in the i-th row group are known to match the predicate.
+ ///
+ /// This is tracked separately from [`RowGroupAccess`] because it describes
+ /// whether row-level filter evaluation can be skipped, not which rows should
+ /// be read.
+ fully_matched: Vec<bool>,
}
/// Describes how the parquet reader will access a row group
@@ -104,6 +110,24 @@ pub enum RowGroupAccess {
Selection(RowSelection),
}
+/// A consecutive set of row groups that share the same row filter requirement.
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct RowGroupRun {
+ /// True if this run needs row filter evaluation.
+ pub(crate) needs_filter: bool,
+ /// The access plan for this run.
+ pub(crate) access_plan: ParquetAccessPlan,
+}
+
+impl RowGroupRun {
+ fn new(needs_filter: bool, access_plan: ParquetAccessPlan) -> Self {
+ Self {
+ needs_filter,
+ access_plan,
+ }
+ }
+}
+
impl RowGroupAccess {
/// Return true if this row group should be scanned
pub fn should_scan(&self) -> bool {
@@ -119,6 +143,7 @@ impl ParquetAccessPlan {
pub fn new_all(row_group_count: usize) -> Self {
Self {
row_groups: vec![RowGroupAccess::Scan; row_group_count],
+ fully_matched: vec![false; row_group_count],
}
}
@@ -126,17 +151,26 @@ impl ParquetAccessPlan {
pub fn new_none(row_group_count: usize) -> Self {
Self {
row_groups: vec![RowGroupAccess::Skip; row_group_count],
+ fully_matched: vec![false; row_group_count],
}
}
/// Create a new `ParquetAccessPlan` from the specified [`RowGroupAccess`]es
pub fn new(row_groups: Vec<RowGroupAccess>) -> Self {
- Self { row_groups }
+ let row_group_count = row_groups.len();
+ Self {
+ row_groups,
+ fully_matched: vec![false; row_group_count],
+ }
}
/// Set the i-th row group to the specified [`RowGroupAccess`]
pub fn set(&mut self, idx: usize, access: RowGroupAccess) {
+ let should_scan = access.should_scan();
self.row_groups[idx] = access;
+ if !should_scan {
+ self.fully_matched[idx] = false;
+ }
}
/// skips the i-th row group (should not be scanned)
@@ -154,6 +188,32 @@ impl ParquetAccessPlan {
self.row_groups[idx].should_scan()
}
+ /// Marks the i-th row group as fully matched.
+ ///
+ /// Fully matched row groups are still read according to their
+ /// [`RowGroupAccess`], but row-level filter evaluation can be skipped.
+ pub(crate) fn mark_fully_matched(&mut self, idx: usize) {
+ if self.should_scan(idx) {
+ self.fully_matched[idx] = true;
+ }
+ }
+
+ /// Return true if the i-th row group is fully matched and scanned.
+ pub(crate) fn is_fully_matched(&self, idx: usize) -> bool {
+ self.should_scan(idx) && self.fully_matched[idx]
+ }
+
+ /// Returns the fully matched row group flags.
+ pub(crate) fn fully_matched(&self) -> &Vec<bool> {
+ &self.fully_matched
+ }
+
+ /// Return true if any scanned row group is fully matched.
+ fn has_fully_matched(&self) -> bool {
+ self.row_group_index_iter()
+ .any(|idx| self.is_fully_matched(idx))
+ }
+
/// Set to scan only the [`RowSelection`] in the specified row group.
///
/// Behavior is different depending on the existing access
@@ -339,6 +399,54 @@ impl ParquetAccessPlan {
self.row_groups
}
+ /// Split this plan into consecutive row group runs that share the same row
+ /// filter requirement.
+ pub(crate) fn split_runs(self, needs_filter: bool) -> Vec<RowGroupRun> {
+ if !needs_filter || !self.has_fully_matched() {
+ return vec![RowGroupRun::new(needs_filter, self)];
+ }
+
+ let num_row_groups = self.row_groups.len();
+ let row_groups = self.row_groups;
+ let fully_matched = self.fully_matched;
+ let mut runs: Vec<RowGroupRun> = Vec::new();
+
+ for (idx, (access, fully_matched)) in
+ row_groups.into_iter().zip(fully_matched).enumerate()
+ {
+ if !access.should_scan() {
+ continue;
+ }
+
+ let row_group_needs_filter = !fully_matched;
+ if let Some(run) = runs
+ .last_mut()
+ .filter(|run| run.needs_filter == row_group_needs_filter)
+ {
+ run.access_plan.set(idx, access);
+ if fully_matched {
+ run.access_plan.mark_fully_matched(idx);
+ }
+ } else {
+ let mut run_plan = ParquetAccessPlan::new_none(num_row_groups);
+ run_plan.set(idx, access);
+ if fully_matched {
+ run_plan.mark_fully_matched(idx);
+ }
+ runs.push(RowGroupRun::new(row_group_needs_filter, run_plan));
+ }
+ }
+
+ if runs.is_empty() {
+ vec![RowGroupRun::new(
+ needs_filter,
+ ParquetAccessPlan::new_none(num_row_groups),
+ )]
+ } else {
+ runs
+ }
+ }
+
/// Prepare this plan and resolve to the final `PreparedAccessPlan`
pub(crate) fn prepare(
self,
diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rs
index 8eb5912b91..262dde024a 100644
--- a/datafusion/datasource-parquet/src/metrics.rs
+++ b/datafusion/datasource-parquet/src/metrics.rs
@@ -213,4 +213,28 @@ impl ParquetFileMetrics {
predicate_cache_records,
}
}
+
+ /// Record pages whose page-index pruning was skipped because the containing
+ /// row group was fully matched by row-group statistics.
+ ///
+ /// The counter is only registered when there is a non-zero value. This keeps
+ /// [`ParquetFileMetrics::new`] from cloning the filename and metrics set for
+ /// files that never use this metric.
+ pub(crate) fn add_page_index_pages_skipped_by_fully_matched(
+ metrics: &ExecutionPlanMetricsSet,
+ partition: usize,
+ filename: &str,
+ n: usize,
+ ) {
+ if n == 0 {
+ return;
+ }
+
+ let count = MetricBuilder::new(metrics)
+ .with_new_label("filename", filename.to_string())
+ .with_type(MetricType::Summary)
+ .with_category(MetricCategory::Rows)
+ .counter("page_index_pages_skipped_by_fully_matched", partition);
+ count.add(n);
+ }
}
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index bbbd298687..f46bfc62d0 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -17,6 +17,7 @@
//! [`ParquetMorselizer`] state machines for opening Parquet files
+use crate::access_plan::PreparedAccessPlan;
use crate::page_filter::PagePruningAccessPlanFilter;
use crate::row_filter::build_projection_read_plan;
use crate::row_group_filter::{BloomFilterStatistics, RowGroupAccessPlanFilter};
@@ -30,7 +31,7 @@ use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselize
use datafusion_physical_expr::projection::{ProjectionExprs, Projector};
use datafusion_physical_expr::utils::reassign_expr_columns;
use datafusion_physical_expr_adapter::replace_columns_with_literals;
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
use std::fmt;
use std::future::Future;
use std::mem;
@@ -68,7 +69,7 @@ use parquet::DecodeResult;
use parquet::arrow::ParquetRecordBatchStreamBuilder;
use parquet::arrow::arrow_reader::metrics::ArrowReaderMetrics;
use parquet::arrow::arrow_reader::{
- ArrowReaderMetadata, ArrowReaderOptions, RowSelectionPolicy,
+ ArrowReaderMetadata, ArrowReaderOptions, RowFilter, RowSelectionPolicy,
};
use parquet::arrow::async_reader::AsyncFileReader;
use parquet::arrow::parquet_column;
@@ -1075,62 +1076,85 @@ impl RowGroupsPrunedParquetOpen {
let file_metadata = Arc::clone(reader_metadata.metadata());
let rg_metadata = file_metadata.row_groups();
- // Filter pushdown: evaluate predicates during scan
- let row_filter = if let Some(predicate) = prepared
+ // Filter pushdown: evaluate predicates during scan.
+ // Keep the predicate around so we can rebuild RowFilter per decoder run
+ // when fully matched row groups split the scan into multiple decoders.
+ let pushdown_predicate = prepared
.pushdown_filters
.then_some(prepared.predicate.clone())
- .flatten()
- {
- let row_filter = row_filter::build_row_filter(
- &predicate,
- &prepared.physical_file_schema,
- file_metadata.as_ref(),
- prepared.reorder_predicates,
- &prepared.file_metrics,
- );
+ .flatten();
- match row_filter {
- Ok(Some(filter)) => Some(filter),
- Ok(None) => None,
- Err(e) => {
- debug!("Ignoring error building row filter for '{predicate:?}': {e}");
- None
+ let try_build_row_filter =
+ |predicate: &Arc<dyn PhysicalExpr>| -> Option<RowFilter> {
+ match row_filter::build_row_filter(
+ predicate,
+ &prepared.physical_file_schema,
+ file_metadata.as_ref(),
+ prepared.reorder_predicates,
+ &prepared.file_metrics,
+ ) {
+ Ok(Some(filter)) => Some(filter),
+ Ok(None) => None,
+ Err(e) => {
+ debug!(
+ "Ignoring error building row filter for '{predicate:?}': {e}"
+ );
+ None
+ }
}
- }
- } else {
- None
- };
+ };
+
+ // Build the first RowFilter eagerly; it will be reused for the first
+ // filtered decoder run and rebuilt from pushdown_predicate for any
+ // additional filtered runs.
+ let mut first_row_filter =
+ pushdown_predicate.as_ref().and_then(&try_build_row_filter);
+ let has_row_filter = first_row_filter.is_some();
// Prune by limit if limit is set and limit order is not sensitive
if let (Some(limit), false) = (prepared.limit, prepared.preserve_order) {
row_groups.prune_by_limit(limit, rg_metadata, &prepared.file_metrics);
}
+ // Build the access plan. Fully matched row groups have all rows
+ // satisfying the predicate, so page pruning and row filter evaluation
+ // can be skipped for them.
+ let mut access_plan = row_groups.build();
+
// Page index pruning: if all data on individual pages can
// be ruled using page metadata, rows from other columns
// with that range can be skipped as well.
- let mut access_plan = row_groups.build();
if prepared.enable_page_index
&& !access_plan.is_empty()
&& let Some(page_pruning_predicate) = page_pruning_predicate
{
- access_plan = page_pruning_predicate.prune_plan_with_page_index(
- access_plan,
- &prepared.physical_file_schema,
- reader_metadata.parquet_schema(),
- file_metadata.as_ref(),
- &prepared.file_metrics,
+ let (page_pruned_access_plan, pages_skipped_by_fully_matched) =
+ page_pruning_predicate.prune_plan_with_page_index(
+ access_plan,
+ &prepared.physical_file_schema,
+ reader_metadata.parquet_schema(),
+ file_metadata.as_ref(),
+ &prepared.file_metrics,
+ );
+ access_plan = page_pruned_access_plan;
+ ParquetFileMetrics::add_page_index_pages_skipped_by_fully_matched(
+ &prepared.metrics,
+ prepared.partition_index,
+ &prepared.file_name,
+ pages_skipped_by_fully_matched,
);
}
- // Prepare the access plan (extract row groups and row selection)
- let mut prepared_plan = access_plan.prepare(rg_metadata)?;
-
- // Potentially reverse the access plan for performance.
- // See `ParquetSource::try_pushdown_sort` for the rationale.
- if prepared.reverse_row_groups {
- prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;
- }
+ // Prepare access plans (extract row groups and row selection).
+ let prepare_access_plan =
+ |plan: ParquetAccessPlan| -> Result<PreparedAccessPlan> {
+ let mut prepared_access_plan = plan.prepare(rg_metadata)?;
+ if prepared.reverse_row_groups {
+ prepared_access_plan =
+ prepared_access_plan.reverse(file_metadata.as_ref())?;
+ }
+ Ok(prepared_access_plan)
+ };
let arrow_reader_metrics = ArrowReaderMetrics::enabled();
let read_plan = build_projection_read_plan(
@@ -1139,33 +1163,69 @@ impl RowGroupsPrunedParquetOpen {
reader_metadata.parquet_schema(),
);
- let mut decoder_builder =
- ParquetPushDecoderBuilder::new_with_metadata(reader_metadata)
- .with_projection(read_plan.projection_mask)
+ // Split into consecutive runs of row groups that share the same filter
+ // requirement. Fully matched row groups skip the RowFilter; others need it.
+ // Reverse the run order for reverse scans so the combined decoder stream
+ // preserves the requested global row group order.
+ let mut runs = access_plan.split_runs(has_row_filter);
+ if prepared.reverse_row_groups {
+ runs.reverse();
+ }
+ let run_count = runs.len();
+ let decoder_limit = prepared.limit.filter(|_| run_count == 1);
+ let remaining_limit = prepared.limit.filter(|_| run_count > 1);
+
+ // Helper: configure a decoder builder with shared options from
+ // the prepared plan.
+ let build_decoder = |prepared_access_plan: PreparedAccessPlan,
+ metadata: ArrowReaderMetadata|
+ -> Result<ParquetPushDecoderBuilder> {
+ let mut builder = ParquetPushDecoderBuilder::new_with_metadata(metadata)
+ .with_projection(read_plan.projection_mask.clone())
.with_batch_size(prepared.batch_size)
.with_metrics(arrow_reader_metrics.clone());
+ if prepared.force_filter_selections {
+ builder =
+ builder.with_row_selection_policy(RowSelectionPolicy::Selectors);
+ }
+ if let Some(row_selection) = prepared_access_plan.row_selection {
+ builder = builder.with_row_selection(row_selection);
+ }
+ builder = builder.with_row_groups(prepared_access_plan.row_group_indexes);
+ if let Some(limit) = decoder_limit {
+ builder = builder.with_limit(limit);
+ }
+ Ok(builder)
+ };
- if let Some(row_filter) = row_filter {
- decoder_builder = decoder_builder.with_row_filter(row_filter);
- }
- if prepared.force_filter_selections {
- decoder_builder =
- decoder_builder.with_row_selection_policy(RowSelectionPolicy::Selectors);
- }
- if let Some(row_selection) = prepared_plan.row_selection {
- decoder_builder = decoder_builder.with_row_selection(row_selection);
- }
- decoder_builder =
- decoder_builder.with_row_groups(prepared_plan.row_group_indexes);
- if let Some(limit) = prepared.limit {
- decoder_builder = decoder_builder.with_limit(limit);
- }
- if let Some(max_predicate_cache_size) = prepared.max_predicate_cache_size {
- decoder_builder =
- decoder_builder.with_max_predicate_cache_size(max_predicate_cache_size);
+ // Build a decoder per run.
+ let mut decoders = VecDeque::with_capacity(runs.len());
+ for run in runs {
+ let prepared_access_plan = prepare_access_plan(run.access_plan)?;
+ let mut builder =
+ build_decoder(prepared_access_plan, reader_metadata.clone())?;
+ if run.needs_filter {
+ // Reuse pre-built filter for the first filtered run,
+ // rebuild from the predicate for subsequent ones.
+ let row_filter = first_row_filter.take().or_else(|| {
+ pushdown_predicate.as_ref().and_then(&try_build_row_filter)
+ });
+ if let Some(row_filter) = row_filter {
+ builder = builder.with_row_filter(row_filter);
+ }
+ if let Some(max_predicate_cache_size) = prepared.max_predicate_cache_size
+ {
+ builder =
+ builder.with_max_predicate_cache_size(max_predicate_cache_size);
+ }
+ }
+ decoders.push_back(builder.build()?);
}
- let decoder = decoder_builder.build()?;
+ let decoder = decoders
+ .pop_front()
+ .expect("at least one decoder must be created");
+ let pending_decoders = decoders;
let predicate_cache_inner_records =
prepared.file_metrics.predicate_cache_inner_records.clone();
@@ -1190,6 +1250,8 @@ impl RowGroupsPrunedParquetOpen {
let stream = futures::stream::unfold(
PushDecoderStreamState {
decoder,
+ pending_decoders,
+ remaining_limit,
reader: prepared.async_file_reader,
projector,
output_schema,
@@ -1226,6 +1288,16 @@ impl RowGroupsPrunedParquetOpen {
/// fully consumed.
struct PushDecoderStreamState {
decoder: ParquetPushDecoder,
+ /// Additional decoders to process after the current one finishes.
+ /// Used when fully matched row groups split the scan into consecutive
+ /// runs with different filter configurations, maintaining original order.
+ pending_decoders: VecDeque<ParquetPushDecoder>,
+ /// Global remaining row limit across all decoder runs.
+ ///
+ /// Decoder-local limits are only safe for single-run scans. When the scan
+ /// is split across multiple decoders, the combined stream limit is enforced
+ /// here instead.
+ remaining_limit: Option<usize>,
reader: Box<dyn AsyncFileReader>,
projector: Projector,
output_schema: Arc<Schema>,
@@ -1252,6 +1324,9 @@ impl PushDecoderStreamState {
/// with `unfold`'s ownership across yield points.
async fn transition(mut self) -> Option<(Result<RecordBatch>, Self)> {
loop {
+ if self.remaining_limit == Some(0) {
+ return None;
+ }
match self.decoder.try_decode() {
Ok(DecodeResult::NeedsData(ranges)) => {
let data = self
@@ -1269,6 +1344,18 @@ impl PushDecoderStreamState {
}
}
Ok(DecodeResult::Data(batch)) => {
+ let batch = if let Some(remaining_limit) = self.remaining_limit {
+ if batch.num_rows() > remaining_limit {
+ self.remaining_limit = Some(0);
+ batch.slice(0, remaining_limit)
+ } else {
+ self.remaining_limit =
+ Some(remaining_limit - batch.num_rows());
+ batch
+ }
+ } else {
+ batch
+ };
let mut timer = self.baseline_metrics.elapsed_compute().timer();
self.copy_arrow_reader_metrics();
let result = self.project_batch(&batch);
@@ -1278,6 +1365,12 @@ impl PushDecoderStreamState {
return Some((result, self));
}
Ok(DecodeResult::Finished) => {
+ // If there are pending decoders (e.g. for consecutive runs
+ // with different filter configurations), switch to the next.
+ if let Some(next) = self.pending_decoders.pop_front() {
+ self.decoder = next;
+ continue;
+ }
return None;
}
Err(e) => {
@@ -1753,6 +1846,12 @@ mod test {
self
}
+ /// Set a row limit.
+ fn with_limit(mut self, limit: usize) -> Self {
+ self.limit = Some(limit);
+ self
+ }
+
/// Set reverse row groups flag.
fn with_reverse_row_groups(mut self, enable: bool) -> Self {
self.reverse_row_groups = enable;
@@ -2714,4 +2813,162 @@ mod test {
"without page index all rows are returned"
);
}
+
+ async fn fully_matched_split_test_file(
+ store: Arc<dyn ObjectStore>,
+ ) -> (SchemaRef, PartitionedFile) {
+ use parquet::file::properties::WriterProperties;
+
+ let batch0 =
+ record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
+ let batch1 =
+ record_batch!(("a", Int32, vec![Some(4), Some(5), Some(6)])).unwrap();
+ let batch2 =
+ record_batch!(("a", Int32, vec![Some(7), Some(1), Some(2)])).unwrap();
+
+ let props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(3))
+ .build();
+
+ let data_len = write_parquet_batches(
+ Arc::clone(&store),
+ "test.parquet",
+ vec![batch0.clone(), batch1, batch2],
+ Some(props),
+ )
+ .await;
+
+ let schema = batch0.schema();
+ let file = PartitionedFile::new(
+ "test.parquet".to_string(),
+ u64::try_from(data_len).unwrap(),
+ );
+ (schema, file)
+ }
+
+ #[tokio::test]
+ async fn test_fully_matched_runs_respect_global_limit() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (schema, file) = fully_matched_split_test_file(Arc::clone(&store)).await;
+ let predicate = logical2physical(&col("a").gt_eq(lit(3)), &schema);
+
+ let opener = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_predicate(predicate)
+ .with_pushdown_filters(true)
+ .with_row_group_stats_pruning(true)
+ .with_limit(4)
+ .build();
+
+ let values = collect_int32_values(open_file(&opener, file).await.unwrap()).await;
+ assert_eq!(values, vec![3, 4, 5, 6]);
+ }
+
+ #[tokio::test]
+ async fn test_fully_matched_runs_preserve_reverse_order() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let (schema, file) = fully_matched_split_test_file(Arc::clone(&store)).await;
+ let predicate = logical2physical(&col("a").gt_eq(lit(3)), &schema);
+
+ let opener = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_predicate(predicate)
+ .with_pushdown_filters(true)
+ .with_row_group_stats_pruning(true)
+ .with_reverse_row_groups(true)
+ .build();
+
+ let values = collect_int32_values(open_file(&opener, file).await.unwrap()).await;
+ assert_eq!(values, vec![7, 4, 5, 6, 3]);
+ }
+
+ #[test]
+ fn test_split_decoder_runs_no_fully_matched() {
+ // All row groups need filtering: single run.
+ let plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Scan,
+ RowGroupAccess::Scan,
+ ]);
+ let runs = plan.split_runs(true);
+ assert_eq!(runs.len(), 1);
+ assert!(runs[0].needs_filter);
+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0, 1, 2]);
+ }
+
+ #[test]
+ fn test_split_decoder_runs_all_fully_matched() {
+ // All row groups are fully matched: single run, no filter.
+ let mut plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Scan,
+ RowGroupAccess::Scan,
+ ]);
+ plan.mark_fully_matched(0);
+ plan.mark_fully_matched(1);
+ plan.mark_fully_matched(2);
+
+ let runs = plan.split_runs(true);
+ assert_eq!(runs.len(), 1);
+ assert!(!runs[0].needs_filter);
+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0, 1, 2]);
+ }
+
+ #[test]
+ fn test_split_decoder_runs_mixed() {
+ // [F, M, M, F, M] creates 4 runs preserving order.
+ let mut plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan, // 0: filtered
+ RowGroupAccess::Scan, // 1: matched
+ RowGroupAccess::Scan, // 2: matched
+ RowGroupAccess::Scan, // 3: filtered
+ RowGroupAccess::Scan, // 4: matched
+ ]);
+ plan.mark_fully_matched(1);
+ plan.mark_fully_matched(2);
+ plan.mark_fully_matched(4);
+
+ let runs = plan.split_runs(true);
+ assert_eq!(runs.len(), 4);
+
+ assert!(runs[0].needs_filter);
+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0]);
+
+ assert!(!runs[1].needs_filter);
+ assert_eq!(runs[1].access_plan.row_group_indexes(), vec![1, 2]);
+
+ assert!(runs[2].needs_filter);
+ assert_eq!(runs[2].access_plan.row_group_indexes(), vec![3]);
+
+ assert!(!runs[3].needs_filter);
+ assert_eq!(runs[3].access_plan.row_group_indexes(), vec![4]);
+ }
+
+ #[test]
+ fn test_split_decoder_runs_with_skipped_groups() {
+ // Skipped row groups are excluded from all runs.
+ let mut plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan, // 0: filtered
+ RowGroupAccess::Skip, // 1: pruned
+ RowGroupAccess::Scan, // 2: matched
+ RowGroupAccess::Scan, // 3: filtered
+ ]);
+ plan.mark_fully_matched(2);
+
+ let runs = plan.split_runs(true);
+ assert_eq!(runs.len(), 3);
+
+ assert!(runs[0].needs_filter);
+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0]);
+
+ assert!(!runs[1].needs_filter);
+ assert_eq!(runs[1].access_plan.row_group_indexes(), vec![2]);
+
+ assert!(runs[2].needs_filter);
+ assert_eq!(runs[2].access_plan.row_group_indexes(), vec![3]);
+ }
}
diff --git a/datafusion/datasource-parquet/src/page_filter.rs b/datafusion/datasource-parquet/src/page_filter.rs
index bc6551c977..fa3e2dd44d 100644
--- a/datafusion/datasource-parquet/src/page_filter.rs
+++ b/datafusion/datasource-parquet/src/page_filter.rs
@@ -160,18 +160,18 @@ impl PagePruningAccessPlanFilter {
parquet_schema: &SchemaDescriptor,
parquet_metadata: &ParquetMetaData,
file_metrics: &ParquetFileMetrics,
- ) -> ParquetAccessPlan {
+ ) -> (ParquetAccessPlan, usize) {
// scoped timer updates on drop
let _timer_guard = file_metrics.page_index_eval_time.timer();
if self.predicates.is_empty() {
- return access_plan;
+ return (access_plan, 0);
}
let page_index_predicates = &self.predicates;
let groups = parquet_metadata.row_groups();
if groups.is_empty() {
- return access_plan;
+ return (access_plan, 0);
}
if parquet_metadata.offset_index().is_none()
@@ -182,7 +182,7 @@ impl PagePruningAccessPlanFilter {
parquet_metadata.offset_index().is_some(),
parquet_metadata.column_index().is_some()
);
- return access_plan;
+ return (access_plan, 0);
};
// track the total number of rows that should be skipped
@@ -193,10 +193,22 @@ impl PagePruningAccessPlanFilter {
let mut total_pages_skip = 0;
// track the total number of pages that should not be skipped
let mut total_pages_select = 0;
+ // track pages for which page-index pruning was skipped because the
+ // containing row group was already proven fully matched by statistics
+ let mut total_pages_skipped_by_fully_matched = 0;
// for each row group specified in the access plan
let row_group_indexes = access_plan.row_group_indexes();
for row_group_index in row_group_indexes {
+ // Skip page pruning for fully matched row groups: all rows are
+ // known to satisfy the predicate, so page-level pruning is wasted work.
+ if access_plan.is_fully_matched(row_group_index) {
+ let page_count =
+ fully_matched_page_count(row_group_index, parquet_metadata);
+ total_pages_skipped_by_fully_matched += page_count;
+
+ continue;
+ }
// The selection for this particular row group
let mut overall_selection = None;
@@ -211,10 +223,13 @@ impl PagePruningAccessPlanFilter {
HashSet::from_iter(0..total_pages_in_group);
for predicate in page_index_predicates {
- let column = predicate
- .required_columns()
- .single_column()
- .expect("Page pruning requires single column predicates");
+ let Some(column) = predicate.required_columns().single_column() else {
+ debug!(
+ "Ignoring multi-column page pruning predicate: {:?}",
+ predicate.predicate_expr()
+ );
+ continue;
+ };
let converter = StatisticsConverter::try_new(
column.name(),
@@ -314,7 +329,7 @@ impl PagePruningAccessPlanFilter {
file_metrics
.page_index_pages_pruned
.add_matched(total_pages_select);
- access_plan
+ (access_plan, total_pages_skipped_by_fully_matched)
}
/// Returns the number of filters in the [`PagePruningAccessPlanFilter`]
@@ -333,6 +348,19 @@ fn update_selection(
}
}
+/// Returns the number of pages for which page-index pruning is skipped because
+/// the containing row group is fully matched by row-group statistics.
+fn fully_matched_page_count(
+ row_group_index: usize,
+ parquet_metadata: &ParquetMetaData,
+) -> usize {
+ parquet_metadata.offset_index().map_or(0, |offset_index| {
+ offset_index[row_group_index]
+ .first()
+ .map_or(0, |column| column.page_locations.len())
+ })
+}
+
/// Returns a [`RowSelection`] for the rows in this row group to scan, in addition to a vec of
/// booleans that state if each page was matched (true) or not (false).
///
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 9eb9024516..c45e69600f 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -45,20 +45,13 @@ use parquet::{bloom_filter::Sbbf, file::metadata::RowGroupMetaData};
pub struct RowGroupAccessPlanFilter {
/// which row groups should be accessed
access_plan: ParquetAccessPlan,
- /// Row groups where ALL rows are known to match the pruning predicate
- /// (the predicate does not filter any rows)
- is_fully_matched: Vec<bool>,
}
impl RowGroupAccessPlanFilter {
/// Create a new `RowGroupPlanBuilder` for pruning out the groups to scan
/// based on metadata and statistics
pub fn new(access_plan: ParquetAccessPlan) -> Self {
- let num_row_groups = access_plan.len();
- Self {
- access_plan,
- is_fully_matched: vec![false; num_row_groups],
- }
+ Self { access_plan }
}
/// Return true if there are no row groups
@@ -76,14 +69,14 @@ impl RowGroupAccessPlanFilter {
self.access_plan.row_group_index_iter()
}
- /// Returns the inner access plan
+ /// Returns the inner access plan.
pub fn build(self) -> ParquetAccessPlan {
self.access_plan
}
- /// Returns the is_fully_matched vector
+ /// Returns the is_fully_matched vector.
pub fn is_fully_matched(&self) -> &Vec<bool> {
- &self.is_fully_matched
+ self.access_plan.fully_matched()
}
/// Prunes the access plan based on the limit and fully contained row groups.
@@ -186,7 +179,7 @@ impl RowGroupAccessPlanFilter {
// Iterate through the currently accessible row groups and try to
// find a set of matching row groups that can satisfy the limit
for &idx in self.access_plan.row_group_indexes().iter() {
- if self.is_fully_matched[idx] {
+ if self.access_plan.is_fully_matched(idx) {
let row_group_row_count = rg_metadata[idx].num_rows() as usize;
fully_matched_row_group_indexes.push(idx);
fully_matched_rows_count += row_group_row_count;
@@ -209,6 +202,7 @@ impl RowGroupAccessPlanFilter {
let mut new_access_plan = ParquetAccessPlan::new_none(rg_metadata.len());
for &idx in &fully_matched_row_group_indexes {
new_access_plan.scan(idx);
+ new_access_plan.mark_fully_matched(idx);
}
self.access_plan = new_access_plan;
}
@@ -397,7 +391,7 @@ impl RowGroupAccessPlanFilter {
// If the inverted predicate *also* prunes this row group (meaning inverted_values[i] is false),
// it implies that *all* rows in this group satisfy the original predicate.
if !inverted_values[i] {
- self.is_fully_matched[original_row_group_idx] = true;
+ self.access_plan.mark_fully_matched(original_row_group_idx);
metrics.row_groups_pruned_statistics.add_fully_matched(1);
}
}
diff --git a/datafusion/physical-expr-common/src/metrics/value.rs b/datafusion/physical-expr-common/src/metrics/value.rs
index 0070f83543..ef0087c20d 100644
--- a/datafusion/physical-expr-common/src/metrics/value.rs
+++ b/datafusion/physical-expr-common/src/metrics/value.rs
@@ -1021,7 +1021,14 @@ impl MetricValue {
Self::SpilledBytes(_) => 11,
Self::SpilledRows(_) => 12,
Self::CurrentMemoryUsage(_) => 13,
- Self::Count { .. } => 14,
+ Self::Count { name, .. } => match name.as_ref() {
+ // This Parquet page-index metric is a plain Count because it
+ // records pages that skipped page-index evaluation, not a
+ // pruned/matched pair. Keep it grouped with the other
+ // page-index pruning metrics in EXPLAIN output.
+ "page_index_pages_skipped_by_fully_matched" => 8,
+ _ => 14,
+ },
Self::Gauge { .. } => 15,
Self::Time { .. } => 16,
Self::Ratio { .. } => 17,
diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
index 179a611d37..befc443303 100644
--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
@@ -104,7 +104,7 @@ Plan with Metrics
03)----ProjectionExec: expr=[id@0 as id, value@1 as v, value@1 + id@0 as name], metrics=[output_rows=10, <slt:ignore>]
04)------FilterExec: value@1 > 3, metrics=[output_rows=10, <slt:ignore>, selectivity=100% (10/10)]
05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, metrics=[output_rows=10, <slt:ignore>]
-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=<slt:ignore>, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18.31% (210/1.15 K)]
+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=<slt:ignore>, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18.31% (210/1.15 K)]
statement ok
set datafusion.explain.analyze_level = dev;
diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.slt
index 1ccb9c92a7..cad36d4fa6 100644
--- a/datafusion/sqllogictest/test_files/explain_analyze.slt
+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt
@@ -247,7 +247,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]
statement ok
reset datafusion.explain.analyze_categories;
@@ -262,7 +262,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
statement ok
reset datafusion.explain.analyze_categories;
@@ -277,7 +277,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
statement ok
reset datafusion.explain.analyze_categories;
diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.slt
index b793425a6e..676306e1f0 100644
--- a/datafusion/sqllogictest/test_files/limit_pruning.slt
+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt
@@ -63,7 +63,55 @@ set datafusion.explain.analyze_level = summary;
query TT
explain analyze select * from tracking_data where species > 'M' AND s >= 50 limit 3;
----
-Plan with Metrics DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], limit=3, file_type=parquet, predicate=species@0 > M AND s@1 >= 50, pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=2 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (171/2.35 K)]
+Plan with Metrics DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], limit=3, file_type=parquet, predicate=species@0 > M AND s@1 >= 50, pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=0 total → 0 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=2 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (171/2.35 K)]
+
+statement ok
+CREATE TABLE fully_matched_limit_source AS VALUES
+ (1),
+ (2),
+ (3),
+ (4),
+ (5),
+ (6),
+ (7),
+ (1),
+ (2);
+
+query I
+COPY (SELECT column1 as a FROM fully_matched_limit_source)
+TO 'test_files/scratch/limit_pruning/fully_matched_limit.parquet'
+STORED AS PARQUET
+OPTIONS (
+ 'format.max_row_group_size' '3'
+);
+----
+9
+
+statement ok
+drop table fully_matched_limit_source;
+
+statement ok
+CREATE EXTERNAL TABLE fully_matched_limit
+STORED AS PARQUET
+LOCATION 'test_files/scratch/limit_pruning/fully_matched_limit.parquet';
+
+# One fully matched row group sits between two filtered row groups.
+# LIMIT must apply across the entire scan, not once per decoder run.
+query TT
+explain analyze select a from fully_matched_limit where a >= 3 limit 4;
+----
+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=4, <slt:ignore>row_groups_pruned_statistics=3 total → 3 matched -> 1 fully matched<slt:ignore>]
+
+query I
+select a from fully_matched_limit where a >= 3 order by a limit 4;
+----
+3
+4
+5
+6
+
+statement ok
+drop table fully_matched_limit;
# limit_pruned_row_groups=0 total → 0 matched
# because of order by, scan needs to preserve sort, so limit pruning is disabled
@@ -72,7 +120,7 @@ explain analyze select * from tracking_data where species > 'M' AND s >= 50 orde
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]
statement ok
drop table tracking_data;
diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.md
index 4102c436b0..9e06acbce4 100644
--- a/docs/source/user-guide/explain-usage.md
+++ b/docs/source/user-guide/explain-usage.md
@@ -228,6 +228,7 @@ When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains
- `output_rows_skew`: output skew score derived from per-partition `output_rows`. `0%` is perfectly balanced, `100%` is maximally skewed, and `N/A` means no output rows were produced.
- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).
- `page_index_pages_pruned`: number of pages evaluated by page index filters. The metric reports both how many pages were considered in total and how many matched (were not pruned).
+- `page_index_pages_skipped_by_fully_matched`: number of pages for which page-index pruning was skipped because row-group statistics proved the containing row group was fully matched. These pages are still scanned.
- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.
- `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched.
- `limit_pruned_row_groups`: number of row groups pruned by the limit.