Apache DataFusion SQL Query Engine
Skip RowFilter and page pruning for fully matched row groups (#21637)
xudong.w3f501f4b9e6b
diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rsindex ca4d097c37..295665055b 100644--- a/datafusion/datasource-parquet/src/access_plan.rs+++ b/datafusion/datasource-parquet/src/access_plan.rs@@ -91,6 +91,12 @@ use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};pub struct ParquetAccessPlan {/// How to access the i-th row grouprow_groups: Vec<RowGroupAccess>,+ /// Whether all rows in the i-th row group are known to match the predicate.+ ///+ /// This is tracked separately from [`RowGroupAccess`] because it describes+ /// whether row-level filter evaluation can be skipped, not which rows should+ /// be read.+ fully_matched: Vec<bool>,}/// Describes how the parquet reader will access a row group@@ -104,6 +110,24 @@ pub enum RowGroupAccess {Selection(RowSelection),}+/// A consecutive set of row groups that share the same row filter requirement.+#[derive(Debug, Clone, PartialEq)]+pub(crate) struct RowGroupRun {+ /// True if this run needs row filter evaluation.+ pub(crate) needs_filter: bool,+ /// The access plan for this run.+ pub(crate) access_plan: ParquetAccessPlan,+}++impl RowGroupRun {+ fn new(needs_filter: bool, access_plan: ParquetAccessPlan) -> Self {+ Self {+ needs_filter,+ access_plan,+ }+ }+}+impl RowGroupAccess {/// Return true if this row group should be scannedpub fn should_scan(&self) -> bool {@@ -119,6 +143,7 @@ impl ParquetAccessPlan {pub fn new_all(row_group_count: usize) -> Self {Self {row_groups: vec![RowGroupAccess::Scan; row_group_count],+ fully_matched: vec![false; row_group_count],}}@@ -126,17 +151,26 @@ impl ParquetAccessPlan {pub fn new_none(row_group_count: usize) -> Self {Self {row_groups: vec![RowGroupAccess::Skip; row_group_count],+ fully_matched: vec![false; row_group_count],}}/// Create a new `ParquetAccessPlan` from the specified [`RowGroupAccess`]espub fn new(row_groups: Vec<RowGroupAccess>) -> Self {- Self { row_groups }+ let row_group_count = row_groups.len();+ Self {+ row_groups,+ fully_matched: vec![false; row_group_count],+ }}/// Set the i-th row group to the specified [`RowGroupAccess`]pub fn set(&mut self, idx: usize, access: RowGroupAccess) {+ let should_scan = access.should_scan();self.row_groups[idx] = access;+ if !should_scan {+ self.fully_matched[idx] = false;+ }}/// skips the i-th row group (should not be scanned)@@ -154,6 +188,32 @@ impl ParquetAccessPlan {self.row_groups[idx].should_scan()}+ /// Marks the i-th row group as fully matched.+ ///+ /// Fully matched row groups are still read according to their+ /// [`RowGroupAccess`], but row-level filter evaluation can be skipped.+ pub(crate) fn mark_fully_matched(&mut self, idx: usize) {+ if self.should_scan(idx) {+ self.fully_matched[idx] = true;+ }+ }++ /// Return true if the i-th row group is fully matched and scanned.+ pub(crate) fn is_fully_matched(&self, idx: usize) -> bool {+ self.should_scan(idx) && self.fully_matched[idx]+ }++ /// Returns the fully matched row group flags.+ pub(crate) fn fully_matched(&self) -> &Vec<bool> {+ &self.fully_matched+ }++ /// Return true if any scanned row group is fully matched.+ fn has_fully_matched(&self) -> bool {+ self.row_group_index_iter()+ .any(|idx| self.is_fully_matched(idx))+ }+/// Set to scan only the [`RowSelection`] in the specified row group.////// Behavior is different depending on the existing access@@ -339,6 +399,54 @@ impl ParquetAccessPlan {self.row_groups}+ /// Split this plan into consecutive row group runs that share the same row+ /// filter requirement.+ pub(crate) fn split_runs(self, needs_filter: bool) -> Vec<RowGroupRun> {+ if !needs_filter || !self.has_fully_matched() {+ return vec![RowGroupRun::new(needs_filter, self)];+ }++ let num_row_groups = self.row_groups.len();+ let row_groups = self.row_groups;+ let fully_matched = self.fully_matched;+ let mut runs: Vec<RowGroupRun> = Vec::new();++ for (idx, (access, fully_matched)) in+ row_groups.into_iter().zip(fully_matched).enumerate()+ {+ if !access.should_scan() {+ continue;+ }++ let row_group_needs_filter = !fully_matched;+ if let Some(run) = runs+ .last_mut()+ .filter(|run| run.needs_filter == row_group_needs_filter)+ {+ run.access_plan.set(idx, access);+ if fully_matched {+ run.access_plan.mark_fully_matched(idx);+ }+ } else {+ let mut run_plan = ParquetAccessPlan::new_none(num_row_groups);+ run_plan.set(idx, access);+ if fully_matched {+ run_plan.mark_fully_matched(idx);+ }+ runs.push(RowGroupRun::new(row_group_needs_filter, run_plan));+ }+ }++ if runs.is_empty() {+ vec![RowGroupRun::new(+ needs_filter,+ ParquetAccessPlan::new_none(num_row_groups),+ )]+ } else {+ runs+ }+ }+/// Prepare this plan and resolve to the final `PreparedAccessPlan`pub(crate) fn prepare(self,diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex 8eb5912b91..262dde024a 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -213,4 +213,28 @@ impl ParquetFileMetrics {predicate_cache_records,}}++ /// Record pages whose page-index pruning was skipped because the containing+ /// row group was fully matched by row-group statistics.+ ///+ /// The counter is only registered when there is a non-zero value. This keeps+ /// [`ParquetFileMetrics::new`] from cloning the filename and metrics set for+ /// files that never use this metric.+ pub(crate) fn add_page_index_pages_skipped_by_fully_matched(+ metrics: &ExecutionPlanMetricsSet,+ partition: usize,+ filename: &str,+ n: usize,+ ) {+ if n == 0 {+ return;+ }++ let count = MetricBuilder::new(metrics)+ .with_new_label("filename", filename.to_string())+ .with_type(MetricType::Summary)+ .with_category(MetricCategory::Rows)+ .counter("page_index_pages_skipped_by_fully_matched", partition);+ count.add(n);+ }}diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex bbbd298687..f46bfc62d0 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -17,6 +17,7 @@//! [`ParquetMorselizer`] state machines for opening Parquet files+use crate::access_plan::PreparedAccessPlan;use crate::page_filter::PagePruningAccessPlanFilter;use crate::row_filter::build_projection_read_plan;use crate::row_group_filter::{BloomFilterStatistics, RowGroupAccessPlanFilter};@@ -30,7 +31,7 @@ use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizeuse datafusion_physical_expr::projection::{ProjectionExprs, Projector};use datafusion_physical_expr::utils::reassign_expr_columns;use datafusion_physical_expr_adapter::replace_columns_with_literals;-use std::collections::HashMap;+use std::collections::{HashMap, VecDeque};use std::fmt;use std::future::Future;use std::mem;@@ -68,7 +69,7 @@ use parquet::DecodeResult;use parquet::arrow::ParquetRecordBatchStreamBuilder;use parquet::arrow::arrow_reader::metrics::ArrowReaderMetrics;use parquet::arrow::arrow_reader::{- ArrowReaderMetadata, ArrowReaderOptions, RowSelectionPolicy,+ ArrowReaderMetadata, ArrowReaderOptions, RowFilter, RowSelectionPolicy,};use parquet::arrow::async_reader::AsyncFileReader;use parquet::arrow::parquet_column;@@ -1075,62 +1076,85 @@ impl RowGroupsPrunedParquetOpen {let file_metadata = Arc::clone(reader_metadata.metadata());let rg_metadata = file_metadata.row_groups();- // Filter pushdown: evaluate predicates during scan- let row_filter = if let Some(predicate) = prepared+ // Filter pushdown: evaluate predicates during scan.+ // Keep the predicate around so we can rebuild RowFilter per decoder run+ // when fully matched row groups split the scan into multiple decoders.+ let pushdown_predicate = prepared.pushdown_filters.then_some(prepared.predicate.clone())- .flatten()- {- let row_filter = row_filter::build_row_filter(- &predicate,- &prepared.physical_file_schema,- file_metadata.as_ref(),- prepared.reorder_predicates,- &prepared.file_metrics,- );+ .flatten();- match row_filter {- Ok(Some(filter)) => Some(filter),- Ok(None) => None,- Err(e) => {- debug!("Ignoring error building row filter for '{predicate:?}': {e}");- None+ let try_build_row_filter =+ |predicate: &Arc<dyn PhysicalExpr>| -> Option<RowFilter> {+ match row_filter::build_row_filter(+ predicate,+ &prepared.physical_file_schema,+ file_metadata.as_ref(),+ prepared.reorder_predicates,+ &prepared.file_metrics,+ ) {+ Ok(Some(filter)) => Some(filter),+ Ok(None) => None,+ Err(e) => {+ debug!(+ "Ignoring error building row filter for '{predicate:?}': {e}"+ );+ None+ }}- }- } else {- None- };+ };++ // Build the first RowFilter eagerly; it will be reused for the first+ // filtered decoder run and rebuilt from pushdown_predicate for any+ // additional filtered runs.+ let mut first_row_filter =+ pushdown_predicate.as_ref().and_then(&try_build_row_filter);+ let has_row_filter = first_row_filter.is_some();// Prune by limit if limit is set and limit order is not sensitiveif let (Some(limit), false) = (prepared.limit, prepared.preserve_order) {row_groups.prune_by_limit(limit, rg_metadata, &prepared.file_metrics);}+ // Build the access plan. Fully matched row groups have all rows+ // satisfying the predicate, so page pruning and row filter evaluation+ // can be skipped for them.+ let mut access_plan = row_groups.build();+// Page index pruning: if all data on individual pages can// be ruled using page metadata, rows from other columns// with that range can be skipped as well.- let mut access_plan = row_groups.build();if prepared.enable_page_index&& !access_plan.is_empty()&& let Some(page_pruning_predicate) = page_pruning_predicate{- access_plan = page_pruning_predicate.prune_plan_with_page_index(- access_plan,- &prepared.physical_file_schema,- reader_metadata.parquet_schema(),- file_metadata.as_ref(),- &prepared.file_metrics,+ let (page_pruned_access_plan, pages_skipped_by_fully_matched) =+ page_pruning_predicate.prune_plan_with_page_index(+ access_plan,+ &prepared.physical_file_schema,+ reader_metadata.parquet_schema(),+ file_metadata.as_ref(),+ &prepared.file_metrics,+ );+ access_plan = page_pruned_access_plan;+ ParquetFileMetrics::add_page_index_pages_skipped_by_fully_matched(+ &prepared.metrics,+ prepared.partition_index,+ &prepared.file_name,+ pages_skipped_by_fully_matched,);}- // Prepare the access plan (extract row groups and row selection)- let mut prepared_plan = access_plan.prepare(rg_metadata)?;-- // Potentially reverse the access plan for performance.- // See `ParquetSource::try_pushdown_sort` for the rationale.- if prepared.reverse_row_groups {- prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;- }+ // Prepare access plans (extract row groups and row selection).+ let prepare_access_plan =+ |plan: ParquetAccessPlan| -> Result<PreparedAccessPlan> {+ let mut prepared_access_plan = plan.prepare(rg_metadata)?;+ if prepared.reverse_row_groups {+ prepared_access_plan =+ prepared_access_plan.reverse(file_metadata.as_ref())?;+ }+ Ok(prepared_access_plan)+ };let arrow_reader_metrics = ArrowReaderMetrics::enabled();let read_plan = build_projection_read_plan(@@ -1139,33 +1163,69 @@ impl RowGroupsPrunedParquetOpen {reader_metadata.parquet_schema(),);- let mut decoder_builder =- ParquetPushDecoderBuilder::new_with_metadata(reader_metadata)- .with_projection(read_plan.projection_mask)+ // Split into consecutive runs of row groups that share the same filter+ // requirement. Fully matched row groups skip the RowFilter; others need it.+ // Reverse the run order for reverse scans so the combined decoder stream+ // preserves the requested global row group order.+ let mut runs = access_plan.split_runs(has_row_filter);+ if prepared.reverse_row_groups {+ runs.reverse();+ }+ let run_count = runs.len();+ let decoder_limit = prepared.limit.filter(|_| run_count == 1);+ let remaining_limit = prepared.limit.filter(|_| run_count > 1);++ // Helper: configure a decoder builder with shared options from+ // the prepared plan.+ let build_decoder = |prepared_access_plan: PreparedAccessPlan,+ metadata: ArrowReaderMetadata|+ -> Result<ParquetPushDecoderBuilder> {+ let mut builder = ParquetPushDecoderBuilder::new_with_metadata(metadata)+ .with_projection(read_plan.projection_mask.clone()).with_batch_size(prepared.batch_size).with_metrics(arrow_reader_metrics.clone());+ if prepared.force_filter_selections {+ builder =+ builder.with_row_selection_policy(RowSelectionPolicy::Selectors);+ }+ if let Some(row_selection) = prepared_access_plan.row_selection {+ builder = builder.with_row_selection(row_selection);+ }+ builder = builder.with_row_groups(prepared_access_plan.row_group_indexes);+ if let Some(limit) = decoder_limit {+ builder = builder.with_limit(limit);+ }+ Ok(builder)+ };- if let Some(row_filter) = row_filter {- decoder_builder = decoder_builder.with_row_filter(row_filter);- }- if prepared.force_filter_selections {- decoder_builder =- decoder_builder.with_row_selection_policy(RowSelectionPolicy::Selectors);- }- if let Some(row_selection) = prepared_plan.row_selection {- decoder_builder = decoder_builder.with_row_selection(row_selection);- }- decoder_builder =- decoder_builder.with_row_groups(prepared_plan.row_group_indexes);- if let Some(limit) = prepared.limit {- decoder_builder = decoder_builder.with_limit(limit);- }- if let Some(max_predicate_cache_size) = prepared.max_predicate_cache_size {- decoder_builder =- decoder_builder.with_max_predicate_cache_size(max_predicate_cache_size);+ // Build a decoder per run.+ let mut decoders = VecDeque::with_capacity(runs.len());+ for run in runs {+ let prepared_access_plan = prepare_access_plan(run.access_plan)?;+ let mut builder =+ build_decoder(prepared_access_plan, reader_metadata.clone())?;+ if run.needs_filter {+ // Reuse pre-built filter for the first filtered run,+ // rebuild from the predicate for subsequent ones.+ let row_filter = first_row_filter.take().or_else(|| {+ pushdown_predicate.as_ref().and_then(&try_build_row_filter)+ });+ if let Some(row_filter) = row_filter {+ builder = builder.with_row_filter(row_filter);+ }+ if let Some(max_predicate_cache_size) = prepared.max_predicate_cache_size+ {+ builder =+ builder.with_max_predicate_cache_size(max_predicate_cache_size);+ }+ }+ decoders.push_back(builder.build()?);}- let decoder = decoder_builder.build()?;+ let decoder = decoders+ .pop_front()+ .expect("at least one decoder must be created");+ let pending_decoders = decoders;let predicate_cache_inner_records =prepared.file_metrics.predicate_cache_inner_records.clone();@@ -1190,6 +1250,8 @@ impl RowGroupsPrunedParquetOpen {let stream = futures::stream::unfold(PushDecoderStreamState {decoder,+ pending_decoders,+ remaining_limit,reader: prepared.async_file_reader,projector,output_schema,@@ -1226,6 +1288,16 @@ impl RowGroupsPrunedParquetOpen {/// fully consumed.struct PushDecoderStreamState {decoder: ParquetPushDecoder,+ /// Additional decoders to process after the current one finishes.+ /// Used when fully matched row groups split the scan into consecutive+ /// runs with different filter configurations, maintaining original order.+ pending_decoders: VecDeque<ParquetPushDecoder>,+ /// Global remaining row limit across all decoder runs.+ ///+ /// Decoder-local limits are only safe for single-run scans. When the scan+ /// is split across multiple decoders, the combined stream limit is enforced+ /// here instead.+ remaining_limit: Option<usize>,reader: Box<dyn AsyncFileReader>,projector: Projector,output_schema: Arc<Schema>,@@ -1252,6 +1324,9 @@ impl PushDecoderStreamState {/// with `unfold`'s ownership across yield points.async fn transition(mut self) -> Option<(Result<RecordBatch>, Self)> {loop {+ if self.remaining_limit == Some(0) {+ return None;+ }match self.decoder.try_decode() {Ok(DecodeResult::NeedsData(ranges)) => {let data = self@@ -1269,6 +1344,18 @@ impl PushDecoderStreamState {}}Ok(DecodeResult::Data(batch)) => {+ let batch = if let Some(remaining_limit) = self.remaining_limit {+ if batch.num_rows() > remaining_limit {+ self.remaining_limit = Some(0);+ batch.slice(0, remaining_limit)+ } else {+ self.remaining_limit =+ Some(remaining_limit - batch.num_rows());+ batch+ }+ } else {+ batch+ };let mut timer = self.baseline_metrics.elapsed_compute().timer();self.copy_arrow_reader_metrics();let result = self.project_batch(&batch);@@ -1278,6 +1365,12 @@ impl PushDecoderStreamState {return Some((result, self));}Ok(DecodeResult::Finished) => {+ // If there are pending decoders (e.g. for consecutive runs+ // with different filter configurations), switch to the next.+ if let Some(next) = self.pending_decoders.pop_front() {+ self.decoder = next;+ continue;+ }return None;}Err(e) => {@@ -1753,6 +1846,12 @@ mod test {self}+ /// Set a row limit.+ fn with_limit(mut self, limit: usize) -> Self {+ self.limit = Some(limit);+ self+ }+/// Set reverse row groups flag.fn with_reverse_row_groups(mut self, enable: bool) -> Self {self.reverse_row_groups = enable;@@ -2714,4 +2813,162 @@ mod test {"without page index all rows are returned");}++ async fn fully_matched_split_test_file(+ store: Arc<dyn ObjectStore>,+ ) -> (SchemaRef, PartitionedFile) {+ use parquet::file::properties::WriterProperties;++ let batch0 =+ record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();+ let batch1 =+ record_batch!(("a", Int32, vec![Some(4), Some(5), Some(6)])).unwrap();+ let batch2 =+ record_batch!(("a", Int32, vec![Some(7), Some(1), Some(2)])).unwrap();++ let props = WriterProperties::builder()+ .set_max_row_group_row_count(Some(3))+ .build();++ let data_len = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ vec![batch0.clone(), batch1, batch2],+ Some(props),+ )+ .await;++ let schema = batch0.schema();+ let file = PartitionedFile::new(+ "test.parquet".to_string(),+ u64::try_from(data_len).unwrap(),+ );+ (schema, file)+ }++ #[tokio::test]+ async fn test_fully_matched_runs_respect_global_limit() {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let (schema, file) = fully_matched_split_test_file(Arc::clone(&store)).await;+ let predicate = logical2physical(&col("a").gt_eq(lit(3)), &schema);++ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_projection_indices(&[0])+ .with_predicate(predicate)+ .with_pushdown_filters(true)+ .with_row_group_stats_pruning(true)+ .with_limit(4)+ .build();++ let values = collect_int32_values(open_file(&opener, file).await.unwrap()).await;+ assert_eq!(values, vec![3, 4, 5, 6]);+ }++ #[tokio::test]+ async fn test_fully_matched_runs_preserve_reverse_order() {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ let (schema, file) = fully_matched_split_test_file(Arc::clone(&store)).await;+ let predicate = logical2physical(&col("a").gt_eq(lit(3)), &schema);++ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_projection_indices(&[0])+ .with_predicate(predicate)+ .with_pushdown_filters(true)+ .with_row_group_stats_pruning(true)+ .with_reverse_row_groups(true)+ .build();++ let values = collect_int32_values(open_file(&opener, file).await.unwrap()).await;+ assert_eq!(values, vec![7, 4, 5, 6, 3]);+ }++ #[test]+ fn test_split_decoder_runs_no_fully_matched() {+ // All row groups need filtering: single run.+ let plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Scan,+ RowGroupAccess::Scan,+ ]);+ let runs = plan.split_runs(true);+ assert_eq!(runs.len(), 1);+ assert!(runs[0].needs_filter);+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0, 1, 2]);+ }++ #[test]+ fn test_split_decoder_runs_all_fully_matched() {+ // All row groups are fully matched: single run, no filter.+ let mut plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Scan,+ RowGroupAccess::Scan,+ ]);+ plan.mark_fully_matched(0);+ plan.mark_fully_matched(1);+ plan.mark_fully_matched(2);++ let runs = plan.split_runs(true);+ assert_eq!(runs.len(), 1);+ assert!(!runs[0].needs_filter);+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0, 1, 2]);+ }++ #[test]+ fn test_split_decoder_runs_mixed() {+ // [F, M, M, F, M] creates 4 runs preserving order.+ let mut plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan, // 0: filtered+ RowGroupAccess::Scan, // 1: matched+ RowGroupAccess::Scan, // 2: matched+ RowGroupAccess::Scan, // 3: filtered+ RowGroupAccess::Scan, // 4: matched+ ]);+ plan.mark_fully_matched(1);+ plan.mark_fully_matched(2);+ plan.mark_fully_matched(4);++ let runs = plan.split_runs(true);+ assert_eq!(runs.len(), 4);++ assert!(runs[0].needs_filter);+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0]);++ assert!(!runs[1].needs_filter);+ assert_eq!(runs[1].access_plan.row_group_indexes(), vec![1, 2]);++ assert!(runs[2].needs_filter);+ assert_eq!(runs[2].access_plan.row_group_indexes(), vec![3]);++ assert!(!runs[3].needs_filter);+ assert_eq!(runs[3].access_plan.row_group_indexes(), vec![4]);+ }++ #[test]+ fn test_split_decoder_runs_with_skipped_groups() {+ // Skipped row groups are excluded from all runs.+ let mut plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan, // 0: filtered+ RowGroupAccess::Skip, // 1: pruned+ RowGroupAccess::Scan, // 2: matched+ RowGroupAccess::Scan, // 3: filtered+ ]);+ plan.mark_fully_matched(2);++ let runs = plan.split_runs(true);+ assert_eq!(runs.len(), 3);++ assert!(runs[0].needs_filter);+ assert_eq!(runs[0].access_plan.row_group_indexes(), vec![0]);++ assert!(!runs[1].needs_filter);+ assert_eq!(runs[1].access_plan.row_group_indexes(), vec![2]);++ assert!(runs[2].needs_filter);+ assert_eq!(runs[2].access_plan.row_group_indexes(), vec![3]);+ }}diff --git a/datafusion/datasource-parquet/src/page_filter.rs b/datafusion/datasource-parquet/src/page_filter.rsindex bc6551c977..fa3e2dd44d 100644--- a/datafusion/datasource-parquet/src/page_filter.rs+++ b/datafusion/datasource-parquet/src/page_filter.rs@@ -160,18 +160,18 @@ impl PagePruningAccessPlanFilter {parquet_schema: &SchemaDescriptor,parquet_metadata: &ParquetMetaData,file_metrics: &ParquetFileMetrics,- ) -> ParquetAccessPlan {+ ) -> (ParquetAccessPlan, usize) {// scoped timer updates on droplet _timer_guard = file_metrics.page_index_eval_time.timer();if self.predicates.is_empty() {- return access_plan;+ return (access_plan, 0);}let page_index_predicates = &self.predicates;let groups = parquet_metadata.row_groups();if groups.is_empty() {- return access_plan;+ return (access_plan, 0);}if parquet_metadata.offset_index().is_none()@@ -182,7 +182,7 @@ impl PagePruningAccessPlanFilter {parquet_metadata.offset_index().is_some(),parquet_metadata.column_index().is_some());- return access_plan;+ return (access_plan, 0);};// track the total number of rows that should be skipped@@ -193,10 +193,22 @@ impl PagePruningAccessPlanFilter {let mut total_pages_skip = 0;// track the total number of pages that should not be skippedlet mut total_pages_select = 0;+ // track pages for which page-index pruning was skipped because the+ // containing row group was already proven fully matched by statistics+ let mut total_pages_skipped_by_fully_matched = 0;// for each row group specified in the access planlet row_group_indexes = access_plan.row_group_indexes();for row_group_index in row_group_indexes {+ // Skip page pruning for fully matched row groups: all rows are+ // known to satisfy the predicate, so page-level pruning is wasted work.+ if access_plan.is_fully_matched(row_group_index) {+ let page_count =+ fully_matched_page_count(row_group_index, parquet_metadata);+ total_pages_skipped_by_fully_matched += page_count;++ continue;+ }// The selection for this particular row grouplet mut overall_selection = None;@@ -211,10 +223,13 @@ impl PagePruningAccessPlanFilter {HashSet::from_iter(0..total_pages_in_group);for predicate in page_index_predicates {- let column = predicate- .required_columns()- .single_column()- .expect("Page pruning requires single column predicates");+ let Some(column) = predicate.required_columns().single_column() else {+ debug!(+ "Ignoring multi-column page pruning predicate: {:?}",+ predicate.predicate_expr()+ );+ continue;+ };let converter = StatisticsConverter::try_new(column.name(),@@ -314,7 +329,7 @@ impl PagePruningAccessPlanFilter {file_metrics.page_index_pages_pruned.add_matched(total_pages_select);- access_plan+ (access_plan, total_pages_skipped_by_fully_matched)}/// Returns the number of filters in the [`PagePruningAccessPlanFilter`]@@ -333,6 +348,19 @@ fn update_selection(}}+/// Returns the number of pages for which page-index pruning is skipped because+/// the containing row group is fully matched by row-group statistics.+fn fully_matched_page_count(+ row_group_index: usize,+ parquet_metadata: &ParquetMetaData,+) -> usize {+ parquet_metadata.offset_index().map_or(0, |offset_index| {+ offset_index[row_group_index]+ .first()+ .map_or(0, |column| column.page_locations.len())+ })+}+/// Returns a [`RowSelection`] for the rows in this row group to scan, in addition to a vec of/// booleans that state if each page was matched (true) or not (false).///diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 9eb9024516..c45e69600f 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -45,20 +45,13 @@ use parquet::{bloom_filter::Sbbf, file::metadata::RowGroupMetaData};pub struct RowGroupAccessPlanFilter {/// which row groups should be accessedaccess_plan: ParquetAccessPlan,- /// Row groups where ALL rows are known to match the pruning predicate- /// (the predicate does not filter any rows)- is_fully_matched: Vec<bool>,}impl RowGroupAccessPlanFilter {/// Create a new `RowGroupPlanBuilder` for pruning out the groups to scan/// based on metadata and statisticspub fn new(access_plan: ParquetAccessPlan) -> Self {- let num_row_groups = access_plan.len();- Self {- access_plan,- is_fully_matched: vec![false; num_row_groups],- }+ Self { access_plan }}/// Return true if there are no row groups@@ -76,14 +69,14 @@ impl RowGroupAccessPlanFilter {self.access_plan.row_group_index_iter()}- /// Returns the inner access plan+ /// Returns the inner access plan.pub fn build(self) -> ParquetAccessPlan {self.access_plan}- /// Returns the is_fully_matched vector+ /// Returns the is_fully_matched vector.pub fn is_fully_matched(&self) -> &Vec<bool> {- &self.is_fully_matched+ self.access_plan.fully_matched()}/// Prunes the access plan based on the limit and fully contained row groups.@@ -186,7 +179,7 @@ impl RowGroupAccessPlanFilter {// Iterate through the currently accessible row groups and try to// find a set of matching row groups that can satisfy the limitfor &idx in self.access_plan.row_group_indexes().iter() {- if self.is_fully_matched[idx] {+ if self.access_plan.is_fully_matched(idx) {let row_group_row_count = rg_metadata[idx].num_rows() as usize;fully_matched_row_group_indexes.push(idx);fully_matched_rows_count += row_group_row_count;@@ -209,6 +202,7 @@ impl RowGroupAccessPlanFilter {let mut new_access_plan = ParquetAccessPlan::new_none(rg_metadata.len());for &idx in &fully_matched_row_group_indexes {new_access_plan.scan(idx);+ new_access_plan.mark_fully_matched(idx);}self.access_plan = new_access_plan;}@@ -397,7 +391,7 @@ impl RowGroupAccessPlanFilter {// If the inverted predicate *also* prunes this row group (meaning inverted_values[i] is false),// it implies that *all* rows in this group satisfy the original predicate.if !inverted_values[i] {- self.is_fully_matched[original_row_group_idx] = true;+ self.access_plan.mark_fully_matched(original_row_group_idx);metrics.row_groups_pruned_statistics.add_fully_matched(1);}}diff --git a/datafusion/physical-expr-common/src/metrics/value.rs b/datafusion/physical-expr-common/src/metrics/value.rsindex 0070f83543..ef0087c20d 100644--- a/datafusion/physical-expr-common/src/metrics/value.rs+++ b/datafusion/physical-expr-common/src/metrics/value.rs@@ -1021,7 +1021,14 @@ impl MetricValue {Self::SpilledBytes(_) => 11,Self::SpilledRows(_) => 12,Self::CurrentMemoryUsage(_) => 13,- Self::Count { .. } => 14,+ Self::Count { name, .. } => match name.as_ref() {+ // This Parquet page-index metric is a plain Count because it+ // records pages that skipped page-index evaluation, not a+ // pruned/matched pair. Keep it grouped with the other+ // page-index pruning metrics in EXPLAIN output.+ "page_index_pages_skipped_by_fully_matched" => 8,+ _ => 14,+ },Self::Gauge { .. } => 15,Self::Time { .. } => 16,Self::Ratio { .. } => 17,diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex 179a611d37..befc443303 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -104,7 +104,7 @@ Plan with Metrics03)----ProjectionExec: expr=[id@0 as id, value@1 as v, value@1 + id@0 as name], metrics=[output_rows=10, <slt:ignore>]04)------FilterExec: value@1 > 3, metrics=[output_rows=10, <slt:ignore>, selectivity=100% (10/10)]05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, metrics=[output_rows=10, <slt:ignore>]-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=<slt:ignore>, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18.31% (210/1.15 K)]+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=<slt:ignore>, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=18.31% (210/1.15 K)]statement okset datafusion.explain.analyze_level = dev;diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.sltindex 1ccb9c92a7..cad36d4fa6 100644--- a/datafusion/sqllogictest/test_files/explain_analyze.slt+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt@@ -247,7 +247,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]statement okreset datafusion.explain.analyze_categories;@@ -262,7 +262,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]statement okreset datafusion.explain.analyze_categories;@@ -277,7 +277,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]statement okreset datafusion.explain.analyze_categories;diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.sltindex b793425a6e..676306e1f0 100644--- a/datafusion/sqllogictest/test_files/limit_pruning.slt+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt@@ -63,7 +63,55 @@ set datafusion.explain.analyze_level = summary;query TTexplain analyze select * from tracking_data where species > 'M' AND s >= 50 limit 3;-----Plan with Metrics DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], limit=3, file_type=parquet, predicate=species@0 > M AND s@1 >= 50, pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=2 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (171/2.35 K)]+Plan with Metrics DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], limit=3, file_type=parquet, predicate=species@0 > M AND s@1 >= 50, pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=0 total → 0 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=2 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (171/2.35 K)]++statement ok+CREATE TABLE fully_matched_limit_source AS VALUES+ (1),+ (2),+ (3),+ (4),+ (5),+ (6),+ (7),+ (1),+ (2);++query I+COPY (SELECT column1 as a FROM fully_matched_limit_source)+TO 'test_files/scratch/limit_pruning/fully_matched_limit.parquet'+STORED AS PARQUET+OPTIONS (+ 'format.max_row_group_size' '3'+);+----+9++statement ok+drop table fully_matched_limit_source;++statement ok+CREATE EXTERNAL TABLE fully_matched_limit+STORED AS PARQUET+LOCATION 'test_files/scratch/limit_pruning/fully_matched_limit.parquet';++# One fully matched row group sits between two filtered row groups.+# LIMIT must apply across the entire scan, not once per decoder run.+query TT+explain analyze select a from fully_matched_limit where a >= 3 limit 4;+----+Plan with Metrics DataSourceExec: <slt:ignore>metrics=[output_rows=4, <slt:ignore>row_groups_pruned_statistics=3 total → 3 matched -> 1 fully matched<slt:ignore>]++query I+select a from fully_matched_limit where a >= 3 order by a limit 4;+----+3+4+5+6++statement ok+drop table fully_matched_limit;# limit_pruned_row_groups=0 total → 0 matched# because of order by, scan needs to preserve sort, so limit pruning is disabled@@ -72,7 +120,7 @@ explain analyze select * from tracking_data where species > 'M' AND s >= 50 orde----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=3 total → 3 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]statement okdrop table tracking_data;diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.mdindex 4102c436b0..9e06acbce4 100644--- a/docs/source/user-guide/explain-usage.md+++ b/docs/source/user-guide/explain-usage.md@@ -228,6 +228,7 @@ When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains- `output_rows_skew`: output skew score derived from per-partition `output_rows`. `0%` is perfectly balanced, `100%` is maximally skewed, and `N/A` means no output rows were produced.- `page_index_rows_pruned`: number of rows evaluated by page index filters. The metric reports both how many rows were considered in total and how many matched (were not pruned).- `page_index_pages_pruned`: number of pages evaluated by page index filters. The metric reports both how many pages were considered in total and how many matched (were not pruned).+- `page_index_pages_skipped_by_fully_matched`: number of pages for which page-index pruning was skipped because row-group statistics proved the containing row group was fully matched. These pages are still scanned.- `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched.- `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched.- `limit_pruned_row_groups`: number of row groups pruned by the limit.