Apache DataFusion SQL Query Engine
refactor: use row-group-local selections for Parquet scans (#25608)
Huaijinb4f813b14fca
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex 4d74a1f282..9ba0f80394 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -310,6 +310,36 @@ async fn two_selections() {}}+#[tokio::test]+async fn mixed_bitmap_and_selector_selections() {+ // RG 0 selects a/c with a bitmap, RG 1 selects f/h with selectors.+ // Each representation must keep coordinates local to its row group.+ TestFull {+ access_plan: Some(ParquetAccessPlan::new(vec![+ RowGroupAccess::Selection(RowSelection::from(+ arrow::buffer::BooleanBuffer::from(vec![true, false, true, false, false]),+ )),+ RowGroupAccess::Selection(select_two_rows()),+ ])),+ row_selection: None,+ expected_rows: 4,+ expected_output: Some(&[+ "+------+------------+",+ "| utf8 | large_utf8 |",+ "+------+------------+",+ "| a | a |",+ "| c | c |",+ "| f | f |",+ "| h | h |",+ "+------+------------+",+ ]),+ predicate: None,+ }+ .run()+ .await+ .unwrap();+}+#[tokio::test]async fn bad_row_groups() {let err = TestFull {diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rsindex b837159cf0..b72612134d 100644--- a/datafusion/datasource-parquet/src/access_plan.rs+++ b/datafusion/datasource-parquet/src/access_plan.rs@@ -15,7 +15,6 @@// specific language governing permissions and limitations// under the License.-use crate::sort::reverse_row_selection;use arrow::datatypes::Schema;use datafusion_common::{Result, assert_eq_or_internal_err, exec_err};use datafusion_physical_expr::expressions::Column;@@ -23,6 +22,7 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;use log::debug;use parquet::arrow::arrow_reader::statistics::StatisticsConverter;use parquet::arrow::arrow_reader::{RowSelection, RowSelector};+use parquet::arrow::push_decoder::RowGroupSelection;use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};/// A selection of rows and row groups within a ParquetFile to decode.@@ -407,7 +407,14 @@ impl ParquetAccessPlan {/// * `Some(selection)` if there are [`RowGroupAccess::Selection`]s////// The returned selection represents which rows to scan across any row- /// row groups which are not skipped.+ /// groups which are not skipped.+ ///+ /// # Deprecated+ ///+ /// DataFusion scans now keep selections local to each row group. For custom+ /// readers, use [`Self::into_inner`] to obtain the [`RowGroupAccess`] entries+ /// and adapt them to the reader's selection API. This compatibility method+ /// retains its existing conversion and length validation behavior.////// # Notes///@@ -422,7 +429,7 @@ impl ParquetAccessPlan {/// # Errors////// Returns an error if any specified row selection does not specify- /// the same number of rows as in it's corresponding `row_group_metadata`.+ /// the same number of rows as in its corresponding `row_group_metadata`.////// # Example: No Selections///@@ -446,12 +453,12 @@ impl ParquetAccessPlan {/// ```text/// RowGroupAccess::Scan (scan all row group 0)/// RowGroupAccess::Skip (skip row group 1)- /// RowGroupAccess::Select (skip 50, scan 50, skip 900) (scan rows 50-100 in row group 2)+ /// RowGroupAccess::Selection (skip 50, scan 50, skip 900) (scan rows 50-100 in row group 2)/// RowGroupAccess::Scan (scan all row group 3)/// ```////// Assuming each row group has 1000 rows, the resulting row selection would- /// be the rows to scan in row group 0, 2 and 4:+ /// be the rows to scan in row group 0, 2 and 3:////// ```text/// RowSelection::Select(1000) (scan all rows in row group 0)@@ -467,6 +474,10 @@ impl ParquetAccessPlan {/// still be `0, 2, 3` .////// [`ArrowReaderBuilder::with_row_selection`]: parquet::arrow::arrow_reader::ArrowReaderBuilder::with_row_selection+ #[deprecated(+ since = "56.0.0",+ note = "Use into_inner() to obtain row-group-local access entries"+ )]pub fn into_overall_row_selection(self,row_group_meta_data: &[RowGroupMetaData],@@ -565,137 +576,90 @@ impl ParquetAccessPlan {self.row_groups}- /// Prepare this plan and resolve to the final `PreparedAccessPlan`+ /// Validate selections and prepare row groups for the decoder without+ /// converting local selections into a single file-level selection.pub(crate) fn prepare(- mut self,+ self,row_group_meta_data: &[RowGroupMetaData],) -> Result<PreparedAccessPlan> {- let row_group_indexes = self.row_group_indexes();- // `fully_matched` is indexed by absolute row-group index; take it out- // before `into_overall_row_selection` consumes `self`.- let fully_matched_by_index = std::mem::take(&mut self.fully_matched);- let row_selection = self.into_overall_row_selection(row_group_meta_data)?;-- let (row_group_indexes, row_selection) =- strip_empty_row_groups(row_group_indexes, row_selection, row_group_meta_data);-- // Carry `fully_matched` flags in the same order as the *surviving*- // `row_group_indexes` so downstream code (per-RG `RowFilter` skip) can- // look them up positionally. Mapping after the strip keeps- // `strip_empty_row_groups` generic (no `fully_matched` parameter).- let fully_matched: Vec<bool> = row_group_indexes+ assert_eq!(row_group_meta_data.len(), self.row_groups.len());+ // Keep the scan policies that previously depended on the presence of+ // an overall selection, even when its partial groups are all empty.+ // Local selections no longer require these restrictions to keep row+ // offsets valid; enabling reordering and runtime pruning with selections+ // is left to a follow-up (#24358).+ let has_row_selection = self+ .row_groups.iter()- .map(|&idx| fully_matched_by_index[idx])- .collect();-- PreparedAccessPlan::new(row_group_indexes, fully_matched, row_selection)- }-}--/// Drop row groups whose post-pruning `RowSelection` selects zero rows, so the-/// prepared plan stays well-formed: `with_row_groups(...)` never names a row-/// group the decoder would immediately skip.-///-/// arrow-rs's push decoder silently advances past an all-skipped row group-/// inside `try_next_reader` without handing back a reader, so keeping such a-/// row group in the plan would leave its row-group list one entry longer than-/// the readers the decoder produces. An empty selection is reachable today via-/// [`ParquetAccessPlan::scan_selection`], which intersects an existing-/// `Selection` with a new one and can leave a row group selecting nothing.-/// A well-formed plan here is also a precondition for re-enabling runtime-/// pruning under a live selection (#24358) and for #23696.-///-/// Walks the flat `RowSelection` once with [`OverallRowSelectionCursor`],-/// rather than a per-row-group [`RowSelection::split_off`] (which reallocates-/// the selector tail on every call, i.e. O(row groups × selectors)), keeping-/// only the row groups that still select at least one row. `row_group_meta_data`-/// is the **full file** metadata, indexed absolutely by row-group index, while-/// `row_selection` covers only the scanned row groups — matching-/// `into_overall_row_selection`, which emits nothing for a skipped row group.-/// When `row_selection` is `None` no row group can be empty and the inputs are-/// returned unchanged.-fn strip_empty_row_groups(- row_group_indexes: Vec<usize>,- row_selection: Option<RowSelection>,- row_group_meta_data: &[RowGroupMetaData],-) -> (Vec<usize>, Option<RowSelection>) {- let Some(selection) = row_selection else {- return (row_group_indexes, None);- };-- let mut cursor = OverallRowSelectionCursor::new(selection);- let mut kept_indexes = Vec::with_capacity(row_group_indexes.len());- let mut kept_selectors: Vec<RowSelector> = Vec::new();-- for &rg_idx in row_group_indexes.iter() {- let rg_row_count = row_group_meta_data[rg_idx].num_rows() as usize;- // Pull this row group's fragments off the shared cursor in a single- // pass (no `split_off` tail reallocation).- let start = kept_selectors.len();- let mut selected = 0usize;- let mut taken = 0usize;- while taken < rg_row_count {- let Some(fragment) = cursor.take(rg_row_count - taken) else {- break;+ .any(|access| matches!(access, RowGroupAccess::Selection(_)));+ let mut row_groups = Vec::with_capacity(self.row_groups.len());+ for (index, (access, fully_matched)) in self+ .row_groups+ .into_iter()+ .zip(self.fully_matched)+ .enumerate()+ {+ let selection = match access {+ RowGroupAccess::Skip => continue,+ RowGroupAccess::Scan => {+ // Preserve strip_empty_row_groups behavior: without an+ // overall selection it returned the scan list unchanged;+ // with one it dropped every group selecting zero rows,+ // including zero-row Scan groups.+ if has_row_selection && row_group_meta_data[index].num_rows() == 0 {+ continue;+ }+ None+ }+ RowGroupAccess::Selection(selection) => {+ let rows_in_selection = selection.total_row_count();+ let row_group_row_count = row_group_meta_data[index].num_rows();+ assert_eq_or_internal_err!(+ rows_in_selection as i64,+ row_group_row_count,+ "Invalid ParquetAccessPlan Selection. Row group {index} has {row_group_row_count} rows \+ but selection only specifies {rows_in_selection} rows. \+ Selection: {selection:?}"+ );+ // Intersections can leave an empty selection: drop the+ // group together with its match status.+ if selection.row_count() == 0 {+ continue;+ }+ Some(selection)+ }};- taken += fragment.row_count;- if !fragment.skip {- selected += fragment.row_count;- }- kept_selectors.push(fragment);- }- if selected > 0 {- kept_indexes.push(rg_idx);- } else {- // Empty row group: arrow-rs would silently skip it. Drop it and its- // fragments so the plan stays 1:1 with the readers.- kept_selectors.truncate(start);+ row_groups.push(PreparedRowGroup {+ selection: RowGroupSelection::new(index, selection),+ fully_matched,+ });}+ Ok(PreparedAccessPlan {+ has_row_selection: has_row_selection && !row_groups.is_empty(),+ row_groups,+ })}+}- let result_selection = if kept_selectors.is_empty() {- None- } else {- Some(RowSelection::from(kept_selectors))- };-- (kept_indexes, result_selection)+/// A row group's selection and static match status travel together when the+/// scan order changes.+#[derive(Debug)]+pub(crate) struct PreparedRowGroup {+ pub(crate) selection: RowGroupSelection,+ /// Whether statistics proved every row passes the predicate.+ pub(crate) fully_matched: bool,}-/// Represents a prepared, fully resolved [`ParquetAccessPlan`]-///-/// The [`RowSelection`] represents the result of applying all pruning such as-/// user provided scans, Row Group statistics, DataPage statistics, and Bloom-/// Filters.-///-/// This plan is what is passed to the parquet reader+/// The final scan order and row-group-local selections passed to the decoder.+#[derive(Debug)]pub(crate) struct PreparedAccessPlan {- /// Row group indexes to read- pub(crate) row_group_indexes: Vec<usize>,- /// Per-RG `fully_matched` flag, positionally aligned with- /// [`Self::row_group_indexes`]. A `true` entry means stats already- /// proved every row of this RG passes the predicate, so the per-row- /// `RowFilter` can be skipped for it.- pub(crate) fully_matched: Vec<bool>,- /// Optional row selection for filtering within row groups- pub(crate) row_selection: Option<RowSelection>,+ pub(crate) row_groups: Vec<PreparedRowGroup>,+ /// Preserve the existing reordering and dynamic-pruning eligibility policy.+ /// This can remain true after an empty selection's group is removed.+ pub(crate) has_row_selection: bool,}impl PreparedAccessPlan {- /// Create a new prepared access plan- fn new(- row_group_indexes: Vec<usize>,- fully_matched: Vec<bool>,- row_selection: Option<RowSelection>,- ) -> Result<Self> {- debug_assert_eq!(row_group_indexes.len(), fully_matched.len());- Ok(Self {- row_group_indexes,- fully_matched,- row_selection,- })- }-/// Reorder row groups by their min statistics for the given sort order.////// This helps TopK queries find optimal values first. Row groups are@@ -714,7 +678,7 @@ impl PreparedAccessPlan {/// dynamic filter converges only as fast as disk order allows.////// Gracefully skips reordering when:- /// - There is a row_selection (too complex to remap)+ /// - There are row selections (preserve the existing scan-order policy)/// - 0 or 1 row groups (nothing to reorder)/// - The leading sort expression is not a simple column reference/// - Statistics are unavailable@@ -724,21 +688,21 @@ impl PreparedAccessPlan {file_metadata: &ParquetMetaData,arrow_schema: &Schema,) -> Result<Self> {- // Skip if row_selection present (too complex to remap)- if self.row_selection.is_some() {+ // Preserve the current policy for scans with page or external selections.+ if self.has_row_selection {debug!("Skipping RG reorder: row_selection present");return Ok(self);}// Nothing to reorder- if self.row_group_indexes.len() <= 1 {+ if self.row_groups.len() <= 1 {return Ok(self);}let rg_metadata: Vec<&RowGroupMetaData> = self- .row_group_indexes+ .row_groups.iter()- .map(|&idx| file_metadata.row_group(idx))+ .map(|rg| file_metadata.row_group(rg.selection.row_group_index())).collect();let leading_descending = sort_order.first().options.descending;@@ -845,41 +809,29 @@ impl PreparedAccessPlan {}};- // Apply the reordering — `fully_matched` must be permuted alongside- // `row_group_indexes` so the two stay positionally aligned for the- // per-RG `RowFilter` skip path.- let original_indexes = self.row_group_indexes.clone();- let original_fully_matched = self.fully_matched.clone();- let order: Vec<usize> = sorted_indices+ // Move each selection together with its match status, without cloning+ // potentially large selection buffers just to reorder row groups.+ let mut original: Vec<_> = self.row_groups.into_iter().map(Some).collect();+ self.row_groups = sorted_indices.values().iter()- .map(|&i| i as usize)+ .map(|&i| original[i as usize].take().expect("unique sort index")).collect();- self.row_group_indexes = order.iter().map(|&i| original_indexes[i]).collect();- self.fully_matched = order.iter().map(|&i| original_fully_matched[i]).collect();-Ok(self)}- /// Reverse the access plan for reverse scanning- pub(crate) fn reverse(mut self, file_metadata: &ParquetMetaData) -> Result<Self> {- // Get the row group indexes before reversing- let row_groups_to_scan = self.row_group_indexes.clone();-- // Reverse the row group indexes (and the parallel `fully_matched`)- self.row_group_indexes = self.row_group_indexes.into_iter().rev().collect();- self.fully_matched = self.fully_matched.into_iter().rev().collect();-- // If we have a row selection, reverse it to match the new row group order- if let Some(row_selection) = self.row_selection {- self.row_selection = Some(reverse_row_selection(- &row_selection,- file_metadata,- &row_groups_to_scan, // Pass the original (non-reversed) row group indexes- )?);- }+ /// Reverse row-group order while preserving local row coordinates.+ pub(crate) fn reverse(mut self) -> Self {+ self.row_groups.reverse();+ self+ }- Ok(self)+ #[cfg(test)]+ pub(crate) fn row_group_indexes(&self) -> Vec<usize> {+ self.row_groups+ .iter()+ .map(|rg| rg.selection.row_group_index())+ .collect()}}@@ -893,122 +845,147 @@ mod test {use std::sync::{Arc, LazyLock};#[test]- fn test_only_scans() {- let access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Scan,- RowGroupAccess::Scan,- RowGroupAccess::Scan,- RowGroupAccess::Scan,- ]);-- let row_group_indexes = access_plan.row_group_indexes();- let row_selection = access_plan- .into_overall_row_selection(&ROW_GROUP_METADATA)- .unwrap();+ #[expect(deprecated)]+ fn test_deprecated_overall_row_selection() {+ for plan in [+ ParquetAccessPlan::new_all(4),+ ParquetAccessPlan::new_none(4),+ ] {+ assert_eq!(+ plan.into_overall_row_selection(&ROW_GROUP_METADATA)+ .unwrap(),+ None+ );+ }- // scan all row groups, no selection- assert_eq!(row_group_indexes, vec![0, 1, 2, 3]);- assert_eq!(row_selection, None);- }+ // Skipped groups do not occupy coordinates in the combined selection.+ // Check both input representations retain the same conversion behavior.+ let selectors =+ RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(20)]);+ let bitmap = RowSelection::from(arrow::buffer::BooleanBuffer::from(+ (0..30).map(|i| i >= 10).collect::<Vec<_>>(),+ ));+ for selection in [selectors, bitmap] {+ let plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Skip,+ RowGroupAccess::Selection(selection),+ RowGroupAccess::Scan,+ ]);+ assert_eq!(+ plan.into_overall_row_selection(&ROW_GROUP_METADATA)+ .unwrap(),+ Some(RowSelection::from(vec![+ RowSelector::select(10),+ RowSelector::skip(10),+ RowSelector::select(60),+ ]))+ );+ }- #[test]- fn test_only_skips() {- let access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Skip,+ let plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Selection(vec![RowSelector::skip(10)].into()),RowGroupAccess::Skip,RowGroupAccess::Skip,RowGroupAccess::Skip,]);-- let row_group_indexes = access_plan.row_group_indexes();- let row_selection = access_plan- .into_overall_row_selection(&ROW_GROUP_METADATA)- .unwrap();-- // skip all row groups, no selection- assert_eq!(row_group_indexes, vec![] as Vec<usize>);- assert_eq!(row_selection, None);+ assert_eq!(+ plan.into_overall_row_selection(&ROW_GROUP_METADATA)+ .unwrap(),+ Some(RowSelection::from(vec![RowSelector::skip(10)]))+ );}+#[test]- fn test_mixed_1() {- let access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Scan,- RowGroupAccess::Selection(- // specifies all 20 rows in row group 1- vec![- RowSelector::select(5),- RowSelector::skip(7),- RowSelector::select(8),- ]- .into(),- ),- RowGroupAccess::Skip,- RowGroupAccess::Skip,- ]);+ #[expect(deprecated)]+ fn test_deprecated_overall_row_selection_validates_length() {+ for rows in [19, 21] {+ let plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Selection(vec![RowSelector::select(rows)].into()),+ RowGroupAccess::Skip,+ RowGroupAccess::Skip,+ ]);+ let err = plan+ .into_overall_row_selection(&ROW_GROUP_METADATA)+ .unwrap_err()+ .to_string();+ assert_contains!(+ err,+ format!(+ "Row group 1 has 20 rows but selection only specifies {rows} rows"+ )+ );+ }+ }- let row_group_indexes = access_plan.row_group_indexes();- let row_selection = access_plan- .into_overall_row_selection(&ROW_GROUP_METADATA)+ #[test]+ fn test_only_scans() {+ let plan = ParquetAccessPlan::new_all(4)+ .prepare(&ROW_GROUP_METADATA).unwrap();-- assert_eq!(row_group_indexes, vec![0, 1]);- assert_eq!(- row_selection,- Some(- vec![- // select the entire first row group- RowSelector::select(10),- // selectors from the second row group- RowSelector::select(5),- RowSelector::skip(7),- RowSelector::select(8)- ]- .into()- )+ assert_eq!(plan.row_group_indexes(), vec![0, 1, 2, 3]);+ assert!(!plan.has_row_selection);+ assert!(+ plan.row_groups+ .iter()+ .all(|rg| rg.selection.selection().is_none()));}#[test]- fn test_mixed_2() {- let access_plan = ParquetAccessPlan::new(vec![+ fn test_only_skips() {+ let plan = ParquetAccessPlan::new_none(4)+ .prepare(&ROW_GROUP_METADATA)+ .unwrap();+ assert!(plan.row_groups.is_empty());+ assert!(!plan.has_row_selection);+ }++ #[test]+ fn test_mixed_selections() {+ let selection = RowSelection::from(vec![+ RowSelector::select(5),+ RowSelector::skip(7),+ RowSelector::select(18),+ ]);+ let plan = ParquetAccessPlan::new(vec![RowGroupAccess::Skip,RowGroupAccess::Scan,- RowGroupAccess::Selection(- // specify all 30 rows in row group 1- vec![- RowSelector::select(5),- RowSelector::skip(7),- RowSelector::select(18),- ]- .into(),- ),+ RowGroupAccess::Selection(selection.clone()),RowGroupAccess::Scan,- ]);-- let row_group_indexes = access_plan.row_group_indexes();- let row_selection = access_plan- .into_overall_row_selection(&ROW_GROUP_METADATA)- .unwrap();-- assert_eq!(row_group_indexes, vec![1, 2, 3]);+ ])+ .prepare(&ROW_GROUP_METADATA)+ .unwrap();+ assert_eq!(plan.row_group_indexes(), vec![1, 2, 3]);+ assert!(plan.has_row_selection);assert_eq!(- row_selection,- Some(- vec![- // select the entire second row group- RowSelector::select(20),- // selectors from the third row group- RowSelector::select(5),- RowSelector::skip(7),- RowSelector::select(18),- // select the entire fourth row group- RowSelector::select(40),- ]- .into()- )+ plan.row_groups[0].selection,+ RowGroupSelection::new(1, None)+ );+ assert_eq!(+ plan.row_groups[1].selection,+ RowGroupSelection::new(2, Some(selection))+ );+ assert_eq!(+ plan.row_groups[2].selection,+ RowGroupSelection::new(3, None));}+ fn scan_plan(indexes: Vec<usize>) -> PreparedAccessPlan {+ PreparedAccessPlan {+ has_row_selection: false,+ row_groups: indexes+ .into_iter()+ .map(|index| PreparedRowGroup {+ selection: RowGroupSelection::new(index, None),+ fully_matched: false,+ })+ .collect(),+ }+ }+#[test]fn test_new_from_overall_row_selection() {let row_selection = RowSelection::from(vec![@@ -1107,7 +1084,7 @@ mod test {let row_group_indexes = access_plan.row_group_indexes();let err = access_plan- .into_overall_row_selection(&ROW_GROUP_METADATA)+ .prepare(&ROW_GROUP_METADATA).unwrap_err().to_string();assert_eq!(row_group_indexes, vec![0, 1, 2, 3]);@@ -1136,7 +1113,7 @@ mod test {let row_group_indexes = access_plan.row_group_indexes();let err = access_plan- .into_overall_row_selection(&ROW_GROUP_METADATA)+ .prepare(&ROW_GROUP_METADATA).unwrap_err().to_string();assert_eq!(row_group_indexes, vec![0, 1, 2, 3]);@@ -1259,7 +1236,8 @@ mod test {#[test]fn reorder_by_statistics_sorts_row_groups_asc_by_min() {let metadata = parquet_metadata_with_int_mins(&[50, 10, 100]);- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();+ let mut plan = scan_plan(vec![0, 1, 2]);+ plan.row_groups[1].fully_matched = true;let result = plan.reorder_by_statistics(@@ -1269,17 +1247,25 @@ mod test {).unwrap();- assert_eq!(result.row_group_indexes, vec![1, 0, 2]);+ assert_eq!(result.row_group_indexes(), vec![1, 0, 2]);+ assert_eq!(+ result+ .row_groups+ .iter()+ .map(|rg| rg.fully_matched)+ .collect::<Vec<_>>(),+ vec![true, false, false]+ );}- /// A `row_selection` is "too complex to remap" through reorder,- /// so the function short-circuits and returns the input untouched.+ /// Preserve the existing scan order when selections are present.#[test]fn reorder_by_statistics_skips_when_row_selection_present() {let metadata = parquet_metadata_with_int_mins(&[50, 10]);let selection = RowSelection::from(vec![RowSelector::select(100)]);- let plan =- PreparedAccessPlan::new(vec![0, 1], vec![false; 2], Some(selection)).unwrap();+ let mut plan = scan_plan(vec![0, 1]);+ plan.row_groups[0].selection = RowGroupSelection::new(0, Some(selection));+ plan.has_row_selection = true;let result = plan.reorder_by_statistics(@@ -1289,14 +1275,14 @@ mod test {).unwrap();- assert_eq!(result.row_group_indexes, vec![0, 1]);+ assert_eq!(result.row_group_indexes(), vec![0, 1]);}/// One row group means nothing to reorder.#[test]fn reorder_by_statistics_skips_when_at_most_one_row_group() {let metadata = parquet_metadata_with_int_mins(&[50]);- let plan = PreparedAccessPlan::new(vec![0], vec![false; 1], None).unwrap();+ let plan = scan_plan(vec![0]);let result = plan.reorder_by_statistics(@@ -1306,7 +1292,7 @@ mod test {).unwrap();- assert_eq!(result.row_group_indexes, vec![0]);+ assert_eq!(result.row_group_indexes(), vec![0]);}/// Non-`Column` sort expressions (e.g. `a + 1`,@@ -1315,7 +1301,7 @@ mod test {#[test]fn reorder_by_statistics_skips_for_non_column_sort_expr() {let metadata = parquet_metadata_with_int_mins(&[50, 10]);- let plan = PreparedAccessPlan::new(vec![0, 1], vec![false; 2], None).unwrap();+ let plan = scan_plan(vec![0, 1]);let arrow_schema = arrow_schema_a_int();let order = LexOrdering::new(vec![PhysicalSortExpr {expr: Arc::new(BinaryExpr::new(@@ -1334,7 +1320,7 @@ mod test {.reorder_by_statistics(&order, &metadata, &arrow_schema).unwrap();- assert_eq!(result.row_group_indexes, vec![0, 1]);+ assert_eq!(result.row_group_indexes(), vec![0, 1]);}/// When the sort column lives outside the file's arrow schema@@ -1344,7 +1330,7 @@ mod test {#[test]fn reorder_by_statistics_skips_when_column_not_in_arrow_schema() {let metadata = parquet_metadata_with_int_mins(&[50, 10]);- let plan = PreparedAccessPlan::new(vec![0, 1], vec![false; 2], None).unwrap();+ let plan = scan_plan(vec![0, 1]);// Arrow schema only has "a"; the sort references "b".let arrow_schema = arrow_schema_a_int();let order = LexOrdering::new(vec![PhysicalSortExpr {@@ -1360,7 +1346,7 @@ mod test {.reorder_by_statistics(&order, &metadata, &arrow_schema).unwrap();- assert_eq!(result.row_group_indexes, vec![0, 1]);+ assert_eq!(result.row_group_indexes(), vec![0, 1]);}// ----------------------------------------------------------------@@ -1447,7 +1433,7 @@ mod test {fn reorder_by_statistics_breaks_leading_ties_with_secondary_column() {let metadata =parquet_metadata_with_two_col_mins(&[(1, 300), (1, 100), (1, 200)]);- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();+ let plan = scan_plan(vec![0, 1, 2]);let order =LexOrdering::new(vec![sort_expr("a", 0, false), sort_expr("b", 1, false)]).unwrap();@@ -1456,7 +1442,7 @@ mod test {.reorder_by_statistics(&order, &metadata, &arrow_schema_ab_int()).unwrap();- assert_eq!(result.row_group_indexes, vec![1, 2, 0]);+ assert_eq!(result.row_group_indexes(), vec![1, 2, 0]);}/// `ORDER BY a ASC, b DESC`: the secondary key's direction is@@ -1466,7 +1452,7 @@ mod test {fn reorder_by_statistics_honors_secondary_direction() {let metadata =parquet_metadata_with_two_col_mins(&[(1, 100), (1, 300), (0, 500)]);- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();+ let plan = scan_plan(vec![0, 1, 2]);let order =LexOrdering::new(vec![sort_expr("a", 0, false), sort_expr("b", 1, true)]).unwrap();@@ -1476,7 +1462,7 @@ mod test {.unwrap();// a=0 first, then the two a=1 groups by b DESC: 300 before 100.- assert_eq!(result.row_group_indexes, vec![2, 1, 0]);+ assert_eq!(result.row_group_indexes(), vec![2, 1, 0]);}/// `ORDER BY a DESC, b DESC` is normalized to ASC lexsort here and@@ -1486,7 +1472,7 @@ mod test {fn reorder_by_statistics_normalizes_desc_desc_for_reverse() {let metadata =parquet_metadata_with_two_col_mins(&[(1, 300), (2, 100), (1, 100)]);- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();+ let plan = scan_plan(vec![0, 1, 2]);let order =LexOrdering::new(vec![sort_expr("a", 0, true), sort_expr("b", 1, true)]).unwrap();@@ -1497,7 +1483,7 @@ mod test {// ASC lexsort of (a, b): (1,100) < (1,300) < (2,100); the later// reverse() produces (2,100), (1,300), (1,100) = (a DESC, b DESC).- assert_eq!(result.row_group_indexes, vec![2, 0, 1]);+ assert_eq!(result.row_group_indexes(), vec![2, 0, 1]);}/// A non-`Column` *secondary* expression stops the stats walk but@@ -1506,7 +1492,7 @@ mod test {fn reorder_by_statistics_keeps_leading_prefix_on_non_column_secondary() {let metadata =parquet_metadata_with_two_col_mins(&[(5, 300), (3, 100), (4, 200)]);- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();+ let plan = scan_plan(vec![0, 1, 2]);let order = LexOrdering::new(vec![sort_expr("a", 0, false),PhysicalSortExpr {@@ -1528,63 +1514,47 @@ mod test {.unwrap();// Ordered by min(a) ASC only: 3, 4, 5.- assert_eq!(result.row_group_indexes, vec![1, 2, 0]);+ assert_eq!(result.row_group_indexes(), vec![1, 2, 0]);}#[test]- fn test_strip_empty_row_groups_drops_only_empties() {- // 4 row groups of [10, 20, 30, 40] rows. RG 1 and RG 3 select nothing- // after pruning, so they must be dropped; RG 0 and RG 2 survive with- // their re-concatenated selections intact.- let selection = RowSelection::from(vec![- RowSelector::select(10), // RG 0: keep all 10- RowSelector::skip(30), // RG 1 (20) fully skipped + RG 2's leading 10- RowSelector::select(20), // RG 2: keep 20- RowSelector::skip(40), // RG 3: skip all 40+ fn test_prepare_drops_empty_selections_and_keeps_match_status() {+ let mut plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(20)])),+ RowGroupAccess::Selection(RowSelection::from(vec![+ RowSelector::skip(10),+ RowSelector::select(20),+ ])),+ RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(40)])),]);-- let (indexes, result) = strip_empty_row_groups(- vec![0, 1, 2, 3],- Some(selection),- &ROW_GROUP_METADATA,+ plan.fully_matched = vec![true, false, false, true];+ let prepared = plan.prepare(&ROW_GROUP_METADATA).unwrap();+ assert_eq!(prepared.row_group_indexes(), vec![0, 2]);+ assert!(prepared.row_groups[0].fully_matched);+ assert!(!prepared.row_groups[1].fully_matched);+ assert_eq!(+ prepared.row_groups[1].selection.selection(),+ Some(&RowSelection::from(vec![+ RowSelector::skip(10),+ RowSelector::select(20)+ ])));-- // RG 1 and RG 3 are dropped; the surviving indexes stay in order.- assert_eq!(indexes, vec![0, 2]);- let result = result.expect("survivors keep a selection");- assert_eq!(result.row_count(), 30); // 10 from RG 0 + 20 from RG 2- assert_eq!(result.skipped_row_count(), 10); // RG 2's leading skip}#[test]- fn test_strip_empty_row_groups_non_contiguous_indexes() {- // Only RG 1 (20 rows) and RG 3 (40 rows) are scanned — RG 0 and RG 2- // are whole-group skips, so the selection covers 20 + 40 = 60 rows.- // This pins down that `strip` indexes `row_group_meta_data`- // *absolutely* (meta[1]=20, meta[3]=40): a relative reading would take- // 10 then 20 rows and misalign the split.- let selection = RowSelection::from(vec![- RowSelector::skip(20), // RG 1: skip all 20 -> dropped- RowSelector::select(40), // RG 3: keep all 40- ]);-- let (indexes, result) =- strip_empty_row_groups(vec![1, 3], Some(selection), &ROW_GROUP_METADATA);-- assert_eq!(indexes, vec![3]);- let result = result.expect("RG 3 survives");- assert_eq!(result.row_count(), 40);- assert_eq!(result.skipped_row_count(), 0);- }-- #[test]- fn test_strip_empty_row_groups_none_selection_unchanged() {- // With no row selection no row group can be empty, so the inputs pass- // through unchanged.- let (indexes, result) =- strip_empty_row_groups(vec![0, 1, 2, 3], None, &ROW_GROUP_METADATA);- assert_eq!(indexes, vec![0, 1, 2, 3]);- assert!(result.is_none());+ fn test_prepare_preserves_selection_policy_after_dropping_empty_group() {+ let prepared = ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(20)])),+ RowGroupAccess::Skip,+ RowGroupAccess::Skip,+ ])+ .prepare(&ROW_GROUP_METADATA)+ .unwrap();+ assert_eq!(prepared.row_group_indexes(), vec![0]);+ assert!(prepared.row_groups[0].selection.selection().is_none());+ assert!(prepared.has_row_selection);}#[test]@@ -1606,7 +1576,7 @@ mod test {1,RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(10)]),);- // RG 2 keeps a genuine partial selection so a flat selection is emitted.+ // RG 2 keeps a genuine partial selection.plan.scan_selection(2,RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(20)]),@@ -1615,6 +1585,6 @@ mod test {let prepared = plan.prepare(&ROW_GROUP_METADATA).expect("prepare");// RG 1 (emptied by the intersection) is stripped; RG 0/2/3 remain.- assert_eq!(prepared.row_group_indexes, vec![0, 2, 3]);+ assert_eq!(prepared.row_group_indexes(), vec![0, 2, 3]);}}diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex de303b11bd..1f4a09a58b 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -1665,8 +1665,8 @@ impl RowGroupsPrunedParquetOpen {//// 2. `reverse`: flip the iteration order for DESC requests, applied// AFTER any reorder so the reversed order is correct whether or- // not reorder changed anything. Also handles `row_selection`- // remapping.+ // not reorder changed anything. Local selections move with their+ // row groups.//// For sorted data: reorder is a no-op, reverse gives perfect DESC.// For unsorted data: reorder fixes the order, reverse flips for DESC.@@ -1685,7 +1685,7 @@ impl RowGroupsPrunedParquetOpen {)?;}if prepared.reverse_row_groups {- prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;+ prepared_plan = prepared_plan.reverse();}Ok(prepared_plan)};@@ -1720,9 +1720,9 @@ impl RowGroupsPrunedParquetOpen {} = {// Build the prepared access plan first — `prepare_access_plan` may// call `reorder_by_statistics` (for `sort_order_for_reorder`) and- // `reverse` (for `reverse_row_groups`), both of which mutate- // `row_group_indexes` to the physical scan order the decoder will- // actually read. We MUST build our `rg_plan` from this reordered+ // `reverse` (for `reverse_row_groups`), both of which arrange row+ // groups in the physical scan order the decoder will read.+ // We MUST build our `rg_plan` from this reordered// list, otherwise our per-RG pruner check would consult the// metadata of a different RG than the decoder is about to yield.let decoder_config = DecoderBuilderConfig {@@ -1734,35 +1734,23 @@ impl RowGroupsPrunedParquetOpen {};let prepared_access_plan = prepare_access_plan(access_plan)?;- // #24355: a row selection (from page-index pruning, or an externally- // supplied `ParquetRowSelection`) is carried by the decoder as one- // flat selection over the concatenation of the remaining row groups.- // The runtime pruner's `into_builder().with_row_groups(...)` rebuild- // drops row groups without slicing that selection to match, so record- // whether a selection is present and disable runtime pruning below- // when it is (mirroring `reorder_by_statistics`, which also bails when- // a row selection is present). The proper fix that keeps pruning- // under a live selection is tracked in- // https://github.com/apache/arrow-rs/issues/10624 /- // https://github.com/apache/datafusion/issues/24358.- let has_row_selection = prepared_access_plan.row_selection.is_some();- // Build `rg_plan` parallel to the decoder's view: the- // `prepared_access_plan` has already had its empty-selection- // row groups stripped, so 1:1 correspondence with the readers- // arrow-rs will hand back is restored. We zip with the- // `fully_matched` flag so the stream can toggle the per-row- // `RowFilter` per RG.- let rg_plan: VecDeque<RgPlanEntry> = prepared_access_plan- .row_group_indexes- .iter()- .copied()- .zip(prepared_access_plan.fully_matched.iter().copied())- .map(|(rg_index, fully_matched)| RgPlanEntry {- rg_index,- fully_matched,- bytes: row_group_bytes(&rg_metadata[rg_index]),- })- .collect();+ let has_row_selection = prepared_access_plan.has_row_selection;+ // Move selections into the decoder, retaining only the metadata+ // needed for pruning, filter toggles, and byte accounting.+ let (selections, rg_plan): (Vec<_>, VecDeque<RgPlanEntry>) =+ prepared_access_plan+ .row_groups+ .into_iter()+ .map(|rg| {+ let rg_index = rg.selection.row_group_index();+ let entry = RgPlanEntry {+ rg_index,+ bytes: row_group_bytes(&rg_metadata[rg_index]),+ fully_matched: rg.fully_matched,+ };+ (rg.selection, entry)+ })+ .unzip();// Decide the initial row filter state based on the first RG to// read. If that RG is `fully_matched` the per-row predicate is@@ -1778,8 +1766,7 @@ impl RowGroupsPrunedParquetOpen {let first_rg_fully_matched = rg_plan.front().is_some_and(|e| e.fully_matched);let row_filter_context = precomputed_context;- let mut builder =- decoder_config.build(prepared_access_plan, reader_metadata.clone());+ let mut builder = decoder_config.build(selections, reader_metadata.clone());let mut filter_installed = false;if let Some(ctx) = row_filter_context.as_ref() {if first_rg_fully_matched {@@ -1857,11 +1844,9 @@ impl RowGroupsPrunedParquetOpen {// via the `DynamicFilterTracker` watch channel (#22460), so detecting// a threshold change is a single atomic load — not a tree walk per// RG check.- // Also disabled when a row selection is live (#24355) — page-index- // pruning is the common source: the pruner rebuilds the decoder via- // `with_row_groups(...)`, which drops row groups without slicing the- // carried selection to match, so pruning under a live selection returns- // wrong results. Decline to prune in that case.+ // Preserve the current policy for scans with selections. Enabling+ // runtime pruning for them is separate from adopting the local API+ // and is tracked in https://github.com/apache/datafusion/issues/24358.let row_group_pruner =match (&prepared.predicate, rg_plan.len() > 1, has_row_selection) {(Some(predicate), true, false)diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rsindex 344e97b1e0..e5c03df486 100644--- a/datafusion/datasource-parquet/src/push_decoder.rs+++ b/datafusion/datasource-parquet/src/push_decoder.rs@@ -29,7 +29,7 @@//! [`RowGroupPruner`] is consulted; row groups it proves unwinnable are//! dropped from the head of `rg_plan` and the decoder is rebuilt via//! [`ParquetPushDecoder::into_builder`] +-//! [`ParquetPushDecoderBuilder::with_row_groups`] so the skipped RGs are+//! [`ParquetPushDecoderBuilder::with_row_group_selections`] so the skipped RGs are//! bypassed entirely — no decode, no row-filter eval.//!//! The opener constructs both halves and hands the state off to@@ -50,7 +50,9 @@ use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ParquetRecordBatchReader, RowFilter, RowSelectionPolicy,};use parquet::arrow::async_reader::AsyncFileReader;-use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder};+use parquet::arrow::push_decoder::{+ ParquetPushDecoder, ParquetPushDecoderBuilder, RowGroupSelection,+};use parquet::file::metadata::ParquetMetaData;use datafusion_common::{DataFusionError, Result, internal_err};@@ -60,7 +62,6 @@ use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Gauge};use datafusion_pruning::{PruningPredicate, PruningPredicateBuilder};use crate::ParquetFileMetrics;-use crate::access_plan::PreparedAccessPlan;use crate::decoder_projection::DecoderProjection;use crate::metrics::{ByteProgress, RowFilterSkippedFullyMatchedMetric};use crate::row_filter::{@@ -83,14 +84,14 @@ pub(crate) struct DecoderBuilderConfig<'a> {}impl DecoderBuilderConfig<'_> {- /// Build a [`ParquetPushDecoderBuilder`] from a prepared access plan.+ /// Build a [`ParquetPushDecoderBuilder`] from row-group-local selections.////// The caller is expected to attach the/// [`RowFilter`] and predicate/// cache size on the returned builder.pub(crate) fn build(&self,- prepared_access_plan: PreparedAccessPlan,+ row_group_selections: Vec<RowGroupSelection>,metadata: ArrowReaderMetadata,) -> ParquetPushDecoderBuilder {let mut builder = ParquetPushDecoderBuilder::new_with_metadata(metadata)@@ -100,10 +101,7 @@ impl DecoderBuilderConfig<'_> {if self.force_filter_selections {builder = builder.with_row_selection_policy(RowSelectionPolicy::Selectors);}- if let Some(row_selection) = prepared_access_plan.row_selection {- builder = builder.with_row_selection(row_selection);- }- builder = builder.with_row_groups(prepared_access_plan.row_group_indexes);+ builder = builder.with_row_group_selections(row_group_selections);if let Some(limit) = self.decoder_limit {builder = builder.with_limit(limit);}@@ -301,7 +299,7 @@ pub(crate) struct PushDecoderStreamState {/// statistics and drop RGs the current threshold proves cannot/// contribute. The decoder is rebuilt via/// [`ParquetPushDecoder::into_builder`] +- /// [`ParquetPushDecoderBuilder::with_row_groups`] so the skipped RGs are+ /// [`ParquetPushDecoderBuilder::with_row_group_selections`] so the skipped RGs are/// bypassed entirely. `None` when the scan has no watching dynamic/// predicate or only one row group remains.pub(crate) row_group_pruner: Option<RowGroupPruner>,@@ -565,6 +563,9 @@ impl PushDecoderStreamState {/// removed every page — which would otherwise leave `rg_plan` trailing the/// decoder by one: a later prune/rebuild would then re-include an/// already-delivered row group (#24352) or toggle the filter for the wrong RG.+ /// Row-group-local selections keep selections and match status aligned when+ /// preparing or reordering the plan, but do not prevent this decoder-side+ /// advancement, so frontier synchronization is still required.fn sync_rg_plan_to_decoder_frontier(&mut self) -> Result<()> {match self.decoder@@ -669,9 +670,18 @@ impl PushDecoderStreamState {}let decoder = self.decoder.take().expect("decoder present");- let new_indices: Vec<usize> = self.rg_plan.iter().map(|e| e.rg_index).collect();let mut builder = decoder.into_builder().map_err(DataFusionError::from)?;- builder = builder.with_row_groups(new_indices);+ // Filter-only rebuilds preserve the decoder's remaining selections.+ // Runtime pruning is disabled for scans with selections, so pruned+ // plans can be rebuilt from row-group indexes alone.+ if pruned_count > 0 {+ let selections = self+ .rg_plan+ .iter()+ .map(|e| RowGroupSelection::new(e.rg_index, None))+ .collect();+ builder = builder.with_row_group_selections(selections);+ }if filter_needs_toggle {let want_filter = desired_filter.expect("filter_needs_toggle ⇒ desired Some");if want_filter {diff --git a/datafusion/datasource-parquet/src/sort.rs b/datafusion/datasource-parquet/src/sort.rsindex e60c257652..4e683d7fcd 100644--- a/datafusion/datasource-parquet/src/sort.rs+++ b/datafusion/datasource-parquet/src/sort.rs@@ -18,109 +18,10 @@//! Sort-related utilities for Parquet scanninguse arrow::datatypes::Schema;-use datafusion_common::{Result, ScalarValue};+use datafusion_common::ScalarValue;use datafusion_datasource::PartitionedFile;use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr_common::sort_expr::LexOrdering;-use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-use parquet::file::metadata::ParquetMetaData;-use std::collections::HashMap;--/// Reverse a row selection to match reversed row group order.-///-/// When scanning row groups in reverse order, we need to adjust the row selection-/// to account for the new ordering. This function:-/// 1. Maps each selection to its corresponding row group-/// 2. Reverses the order of row groups-/// 3. Reconstructs the row selection for the new order-///-/// # Arguments-/// * `row_selection` - Original row selection (only covers row groups that are scanned)-/// * `parquet_metadata` - Metadata containing row group information-/// * `row_groups_to_scan` - Indexes of row groups that will be scanned (in original order)-///-/// # Returns-/// A new `RowSelection` adjusted for reversed row group order-///-/// # Important Notes-/// The input `row_selection` only covers the row groups specified in `row_groups_to_scan`.-/// Row groups that are skipped (not in `row_groups_to_scan`) are not represented in the-/// `row_selection` at all. This function needs `row_groups_to_scan` to correctly map-/// the selection back to the original row groups.-pub fn reverse_row_selection(- row_selection: &RowSelection,- parquet_metadata: &ParquetMetaData,- row_groups_to_scan: &[usize],-) -> Result<RowSelection> {- let rg_metadata = parquet_metadata.row_groups();-- // Build a mapping of row group index to its row range, but ONLY for- // the row groups that are actually being scanned.- //- // IMPORTANT: The row numbers in this mapping are RELATIVE to the scanned row groups,- // not absolute positions in the file.- //- // Example: If row_groups_to_scan = [0, 2, 3] and each has 100 rows:- // RG0: rows 0-99 (relative to scanned data)- // RG2: rows 100-199 (relative to scanned data, NOT 200-299 in file!)- // RG3: rows 200-299 (relative to scanned data, NOT 300-399 in file!)- let mut rg_row_ranges: Vec<(usize, usize, usize)> =- Vec::with_capacity(row_groups_to_scan.len());- let mut current_row = 0;- for &rg_idx in row_groups_to_scan {- let rg = &rg_metadata[rg_idx];- let num_rows = rg.num_rows() as usize;- rg_row_ranges.push((rg_idx, current_row, current_row + num_rows));- current_row += num_rows; // This is relative row number, NOT absolute file position- }-- // Map selections to row groups- let mut rg_selections: HashMap<usize, Vec<RowSelector>> = HashMap::new();-- let mut current_file_row = 0;- for selector in row_selection.iter() {- let selector_end = current_file_row + selector.row_count;-- // Find which row groups this selector spans- for (rg_idx, rg_start, rg_end) in rg_row_ranges.iter() {- if current_file_row < *rg_end && selector_end > *rg_start {- // This selector overlaps with this row group- let overlap_start = current_file_row.max(*rg_start);- let overlap_end = selector_end.min(*rg_end);- let overlap_count = overlap_end - overlap_start;-- if overlap_count > 0 {- let entry = rg_selections.entry(*rg_idx).or_default();- if selector.skip {- entry.push(RowSelector::skip(overlap_count));- } else {- entry.push(RowSelector::select(overlap_count));- }- }- }- }-- current_file_row = selector_end;- }-- // Build new selection for reversed row group order- // Only iterate over the row groups that are being scanned, in reverse order- let mut reversed_selectors = Vec::new();- for &rg_idx in row_groups_to_scan.iter().rev() {- if let Some(selectors) = rg_selections.get(&rg_idx) {- reversed_selectors.extend(selectors.iter().copied());- } else {- // No specific selection for this row group means select all rows in it- if let Some((_, start, end)) =- rg_row_ranges.iter().find(|(idx, _, _)| *idx == rg_idx)- {- reversed_selectors.push(RowSelector::select(end - start));- }- }- }-- Ok(RowSelection::from(reversed_selectors))-}/// Reorder a file list so the most "promising" files are read first,/// matching `PreparedAccessPlan::reorder_by_statistics` at the@@ -283,862 +184,128 @@ mod tests {}#[test]- fn test_prepared_access_plan_reverse_simple() {- // Test: all row groups are scanned, no row selection- let metadata = create_test_metadata(vec![100, 100, 100]);-- let access_plan = ParquetAccessPlan::new_all(3);- let rg_metadata = metadata.row_groups();-- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- // Verify original plan- assert_eq!(prepared_plan.row_group_indexes, vec![0, 1, 2]);-- // No row selection originally due to scanning all rows- assert_eq!(prepared_plan.row_selection, None);-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // Verify row groups are reversed- assert_eq!(reversed_plan.row_group_indexes, vec![2, 1, 0]);-- // If no selection originally, after reversal should still select all rows,- // and the selection should be None- assert_eq!(reversed_plan.row_selection, None);- }-- #[test]- fn test_prepared_access_plan_reverse_with_selection() {- // Test: simple row selection that spans multiple row groups- let metadata = create_test_metadata(vec![100, 100, 100]);-- let mut access_plan = ParquetAccessPlan::new_all(3);-- // Select first 50 rows from first row group, skip rest- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),- );-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(- original_selected, reversed_selected,- "Total selected rows should remain the same"- );- }-- #[test]- fn test_prepared_access_plan_reverse_multi_row_group_selection() {- // Test: row selection spanning multiple row groups- let metadata = create_test_metadata(vec![100, 100, 100]);-- let mut access_plan = ParquetAccessPlan::new_all(3);-- // Create selection that spans RG0 and RG1- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::skip(50), RowSelector::select(50)]),- );- access_plan.scan_selection(- 1,- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),- );-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(original_selected, reversed_selected);- }-- #[test]- fn test_prepared_access_plan_reverse_empty_selection() {- // Test: all rows are skipped. After `strip_empty_row_groups`, every row- // group's selection is empty, so the whole plan strips to nothing.- let metadata = create_test_metadata(vec![100, 100, 100]);-- let mut access_plan = ParquetAccessPlan::new_all(3);- for i in 0..3 {- access_plan- .scan_selection(i, RowSelection::from(vec![RowSelector::skip(100)]));+ fn test_prepared_access_plan_reverse() {+ use RowGroupAccess::{Scan, Selection, Skip};+ let selection = |selectors: Vec<RowSelector>| Selection(selectors.into());+ // Exercise empty, single-group, non-contiguous, unequal-size, and+ // fragmented selections. Reverse group order, never local row order.+ let cases = vec![+ (vec![100, 100, 100], vec![Scan, Scan, Scan], vec![2, 1, 0]),+ (vec![100, 100, 100], vec![Skip, Skip, Skip], vec![]),+ (+ vec![100, 100, 100],+ vec![selection(vec![RowSelector::skip(100)]); 3],+ vec![],+ ),+ (+ vec![100],+ vec![selection(vec![+ RowSelector::select(50),+ RowSelector::skip(50),+ ])],+ vec![0],+ ),+ (+ vec![100, 100, 100],+ vec![+ selection(vec![RowSelector::skip(50), RowSelector::select(50)]),+ selection(vec![RowSelector::select(50), RowSelector::skip(50)]),+ Scan,+ ],+ vec![2, 1, 0],+ ),+ (+ vec![50, 150, 100],+ vec![+ selection(vec![RowSelector::skip(25), RowSelector::select(25)]),+ Scan,+ selection(vec![RowSelector::select(50), RowSelector::skip(50)]),+ ],+ vec![2, 1, 0],+ ),+ (+ vec![100, 100, 100, 100],+ vec![+ Scan,+ Skip,+ selection(vec![RowSelector::select(25), RowSelector::skip(75)]),+ Scan,+ ],+ vec![3, 2, 0],+ ),+ (+ vec![100, 100, 100, 100],+ vec![+ selection(vec![RowSelector::select(30), RowSelector::skip(70)]),+ Skip,+ selection(vec![RowSelector::skip(20), RowSelector::select(80)]),+ Skip,+ ],+ vec![2, 0],+ ),+ (vec![100, 100, 100], vec![Skip, Scan, Skip], vec![1]),+ (+ vec![100, 100, 100],+ vec![+ selection(vec![+ RowSelector::select(30),+ RowSelector::skip(40),+ RowSelector::select(30),+ ]),+ selection(vec![RowSelector::skip(50), RowSelector::select(50)]),+ Scan,+ ],+ vec![2, 1, 0],+ ),+ ];+ for (sizes, accesses, expected_indexes) in cases {+ let metadata = create_test_metadata(sizes);+ let mut plan = ParquetAccessPlan::new(accesses.clone());+ for index in (0..accesses.len()).step_by(2) {+ plan.mark_fully_matched(index);+ }+ let reversed = plan.prepare(metadata.row_groups()).unwrap().reverse();+ assert_eq!(reversed.row_group_indexes(), expected_indexes);+ for rg in reversed.row_groups {+ let index = rg.selection.row_group_index();+ let expected = match &accesses[index] {+ Scan => None,+ Selection(selection) => Some(selection),+ Skip => panic!("skipped group must not be read"),+ };+ assert_eq!(rg.selection.selection(), expected);+ assert_eq!(rg.fully_matched, index % 2 == 0);+ }}-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");- assert!(- prepared_plan.row_group_indexes.is_empty(),- "all-empty selections must be stripped to an empty plan",- );- assert!(prepared_plan.row_selection.is_none());-- // All row groups are empty after pruning, so they are stripped and the- // prepared plan is empty (rather than carrying a selection that skips- // every row).- assert!(prepared_plan.row_group_indexes.is_empty());- assert!(prepared_plan.row_selection.is_none());-- // Reversing an empty plan stays empty.- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");- assert!(reversed_plan.row_group_indexes.is_empty());- assert!(reversed_plan.row_selection.is_none());- }-- #[test]- fn test_prepared_access_plan_reverse_different_row_group_sizes() {- // Test: row groups with different sizes- let metadata = create_test_metadata(vec![50, 150, 100]);-- let mut access_plan = ParquetAccessPlan::new_all(3);-- // Create complex selection pattern- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::skip(25), RowSelector::select(25)]),- );- access_plan.scan_selection(1, RowSelection::from(vec![RowSelector::select(150)]));- access_plan.scan_selection(- 2,- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),- );-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(original_selected, reversed_selected);- }-- #[test]- fn test_prepared_access_plan_reverse_single_row_group() {- // Test: single row group case- let metadata = create_test_metadata(vec![100]);-- let mut access_plan = ParquetAccessPlan::new_all(1);- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),- );-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // With single row group, row_group_indexes should remain [0]- assert_eq!(reversed_plan.row_group_indexes, vec![0]);-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(original_selected, reversed_selected);- assert_eq!(original_selected, 50);- }-- #[test]- fn test_prepared_access_plan_reverse_complex_pattern() {- // Test: complex pattern with multiple select/skip segments- let metadata = create_test_metadata(vec![100, 100, 100]);-- let mut access_plan = ParquetAccessPlan::new_all(3);-- // Complex pattern: select some, skip some, select some more- access_plan.scan_selection(- 0,- RowSelection::from(vec![- RowSelector::select(30),- RowSelector::skip(40),- RowSelector::select(30),- ]),- );- access_plan.scan_selection(- 1,- RowSelection::from(vec![RowSelector::skip(50), RowSelector::select(50)]),- );- access_plan.scan_selection(2, RowSelection::from(vec![RowSelector::select(100)]));-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(original_selected, reversed_selected);- assert_eq!(original_selected, 210); // 30 + 30 + 50 + 100- }-- #[test]- fn test_prepared_access_plan_reverse_with_skipped_row_groups() {- // This is the KEY test case for the bug fix!- // Test scenario where some row groups are completely skipped (not in scan plan)- let metadata = create_test_metadata(vec![100, 100, 100, 100]);-- // Scenario: RG0 (scan all), RG1 (completely skipped), RG2 (partial), RG3 (scan all)- // Only row groups [0, 2, 3] are in the scan plan- let mut access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Scan, // RG0- RowGroupAccess::Skip, // RG1 - NOT in scan plan!- RowGroupAccess::Scan, // RG2- RowGroupAccess::Scan, // RG3- ]);-- // Add row selections for the scanned row groups- // Note: The RowSelection only covers row groups [0, 2, 3] (300 rows total)- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::select(100)]), // RG0: all 100 rows- );- // RG1 is skipped, no selection needed- access_plan.scan_selection(- 2,- RowSelection::from(vec![- RowSelector::select(25), // RG2: first 25 rows- RowSelector::skip(75), // RG2: skip last 75 rows- ]),- );- access_plan.scan_selection(- 3,- RowSelection::from(vec![RowSelector::select(100)]), // RG3: all 100 rows- );-- let rg_metadata = metadata.row_groups();-- // Step 1: Create PreparedAccessPlan- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- // Verify original plan- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2, 3]);- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();- assert_eq!(original_selected, 225); // 100 + 25 + 100-- // Step 2: Reverse the plan (this is the production code path)- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // Verify reversed results- // Row group order should be reversed: [3, 2, 0]- assert_eq!(- reversed_plan.row_group_indexes,- vec![3, 2, 0],- "Row groups should be reversed"- );-- // Verify row selection is also correctly reversed- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(- reversed_selected, 225,- "Total selected rows should remain the same"- );-- // Verify the reversed selection structure- // After reversal, the order becomes: RG3, RG2, RG0- // - RG3: select(100)- // - RG2: select(25), skip(75) (note: internal order preserved, not reversed)- // - RG0: select(100)- //- // After RowSelection::from() merges adjacent selectors of the same type:- // - RG3's select(100) + RG2's select(25) = select(125)- // - RG2's skip(75) remains as skip(75)- // - RG0's select(100) remains as select(100)- let selectors: Vec<_> = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();- assert_eq!(selectors.len(), 3);-- // RG3 (100) + RG2 first part (25) merged into select(125)- assert!(!selectors[0].skip);- assert_eq!(selectors[0].row_count, 125);-- // RG2: skip last 75 rows- assert!(selectors[1].skip);- assert_eq!(selectors[1].row_count, 75);-- // RG0: select all 100 rows- assert!(!selectors[2].skip);- assert_eq!(selectors[2].row_count, 100);}#[test]- fn test_prepared_access_plan_reverse_alternating_row_groups() {- // Test with alternating scan/skip pattern- let metadata = create_test_metadata(vec![100, 100, 100, 100]);-- // Scan RG0 and RG2, skip RG1 and RG3- let mut access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Scan, // RG0- RowGroupAccess::Skip, // RG1- RowGroupAccess::Scan, // RG2- RowGroupAccess::Skip, // RG3+ fn test_prepared_access_plan_preserves_bitmap_selection() {+ let metadata = create_test_metadata(vec![4, 4]);+ let mask = arrow::buffer::BooleanBuffer::from(vec![true, false, true, false]);+ let plan = ParquetAccessPlan::new(vec![+ RowGroupAccess::Selection(RowSelection::from(mask.clone())),+ RowGroupAccess::Selection(RowSelection::from(vec![+ RowSelector::skip(1),+ RowSelector::select(3),+ ])),]);-- access_plan.scan_selection(0, RowSelection::from(vec![RowSelector::select(100)]));- access_plan.scan_selection(2, RowSelection::from(vec![RowSelector::select(100)]));-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- // Original: [0, 2]- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2]);-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // After reverse: [2, 0]- assert_eq!(reversed_plan.row_group_indexes, vec![2, 0]);-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(original_selected, reversed_selected);- assert_eq!(original_selected, 200);- }-- #[test]- fn test_prepared_access_plan_reverse_middle_row_group_only() {- // Test selecting only the middle row group- let metadata = create_test_metadata(vec![100, 100, 100]);-- let mut access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Skip, // RG0- RowGroupAccess::Scan, // RG1- RowGroupAccess::Skip, // RG2- ]);-- access_plan.scan_selection(- 1,- RowSelection::from(vec![RowSelector::select(100)]), // Select all of RG1- );-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- let original_selected: usize = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- // Original: [1]- assert_eq!(prepared_plan.row_group_indexes, vec![1]);-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // After reverse: still [1] (only one row group)- assert_eq!(reversed_plan.row_group_indexes, vec![1]);-- let reversed_selected: usize = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(original_selected, reversed_selected);- assert_eq!(original_selected, 100);- }-- #[test]- fn test_prepared_access_plan_reverse_with_skipped_row_groups_detailed() {- // This is the KEY test case for the bug fix!- // Test scenario where some row groups are completely skipped (not in scan plan)- // This version includes DETAILED verification of the selector distribution- let metadata = create_test_metadata(vec![100, 100, 100, 100]);-- // Scenario: RG0 (scan all), RG1 (completely skipped), RG2 (partial), RG3 (scan all)- // Only row groups [0, 2, 3] are in the scan plan- let mut access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Scan, // RG0- RowGroupAccess::Skip, // RG1 - NOT in scan plan!- RowGroupAccess::Scan, // RG2- RowGroupAccess::Scan, // RG3- ]);-- // Add row selections for the scanned row groups- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::select(100)]), // RG0: all 100 rows- );- // RG1 is skipped, no selection needed- access_plan.scan_selection(- 2,- RowSelection::from(vec![- RowSelector::select(25), // RG2: first 25 rows- RowSelector::skip(75), // RG2: skip last 75 rows- ]),- );- access_plan.scan_selection(- 3,- RowSelection::from(vec![RowSelector::select(100)]), // RG3: all 100 rows- );-- let rg_metadata = metadata.row_groups();-- // Step 1: Create PreparedAccessPlan- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- // Verify original plan in detail- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2, 3]);-- // Detailed verification of original selection- let orig_selectors: Vec<_> = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();-- // Original structure should be:- // RG0: select(100)- // RG2: select(25), skip(75)- // RG3: select(100)- // After merging by RowSelection::from(): select(125), skip(75), select(100)+ let reversed = plan.prepare(metadata.row_groups()).unwrap().reverse();+ assert_eq!(reversed.row_group_indexes(), vec![1, 0]);assert_eq!(- orig_selectors.len(),- 3,- "Original should have 3 selectors after merging"- );- assert!(- !orig_selectors[0].skip && orig_selectors[0].row_count == 125,- "Original: First selector should be select(125) from RG0(100) + RG2(25)"- );- assert!(- orig_selectors[1].skip && orig_selectors[1].row_count == 75,- "Original: Second selector should be skip(75) from RG2"- );- assert!(- !orig_selectors[2].skip && orig_selectors[2].row_count == 100,- "Original: Third selector should be select(100) from RG3"+ reversed.row_groups[1]+ .selection+ .selection()+ .unwrap()+ .as_mask(),+ Some(&mask));-- let original_selected: usize = orig_selectors- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();- assert_eq!(original_selected, 225); // 100 + 25 + 100-- // Step 2: Reverse the plan (this is the production code path)- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // Verify reversed results- // Row group order should be reversed: [3, 2, 0]assert_eq!(- reversed_plan.row_group_indexes,- vec![3, 2, 0],- "Row groups should be reversed"- );-- // Detailed verification of reversed selection- let rev_selectors: Vec<_> = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();-- // After reversal, the order becomes: RG3, RG2, RG0- // - RG3: select(100)- // - RG2: select(25), skip(75) (note: internal order preserved, not reversed)- // - RG0: select(100)- //- // After RowSelection::from() merges adjacent selectors of the same type:- // - RG3's select(100) + RG2's select(25) = select(125)- // - RG2's skip(75) remains as skip(75)- // - RG0's select(100) remains as select(100)-- assert_eq!(- rev_selectors.len(),- 3,- "Reversed should have 3 selectors after merging"- );-- // First selector: RG3 (100) + RG2 first part (25) merged into select(125)- assert!(- !rev_selectors[0].skip && rev_selectors[0].row_count == 125,- "Reversed: First selector should be select(125) from RG3(100) + RG2(25), got skip={} count={}",- rev_selectors[0].skip,- rev_selectors[0].row_count- );-- // Second selector: RG2 skip last 75 rows- assert!(- rev_selectors[1].skip && rev_selectors[1].row_count == 75,- "Reversed: Second selector should be skip(75) from RG2, got skip={} count={}",- rev_selectors[1].skip,- rev_selectors[1].row_count- );-- // Third selector: RG0 select all 100 rows- assert!(- !rev_selectors[2].skip && rev_selectors[2].row_count == 100,- "Reversed: Third selector should be select(100) from RG0, got skip={} count={}",- rev_selectors[2].skip,- rev_selectors[2].row_count- );-- // Verify row selection is also correctly reversed (total count)- let reversed_selected: usize = rev_selectors- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(- reversed_selected, 225,- "Total selected rows should remain the same"- );- }-- #[test]- fn test_prepared_access_plan_reverse_complex_pattern_detailed() {- // Test: complex pattern with detailed verification- let metadata = create_test_metadata(vec![100, 100, 100]);-- let mut access_plan = ParquetAccessPlan::new_all(3);-- // Complex pattern: select some, skip some, select some more- access_plan.scan_selection(- 0,- RowSelection::from(vec![- RowSelector::select(30),- RowSelector::skip(40),- RowSelector::select(30),- ]),- );- access_plan.scan_selection(- 1,- RowSelection::from(vec![RowSelector::skip(50), RowSelector::select(50)]),- );- access_plan.scan_selection(2, RowSelection::from(vec![RowSelector::select(100)]));-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- // Verify original selection structure in detail- let orig_selectors: Vec<_> = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();-- // RG0: select(30), skip(40), select(30)- // RG1: skip(50), select(50)- // RG2: select(100)- // Sequential: sel(30), skip(40), sel(30), skip(50), sel(50), sel(100)- // After merge: sel(30), skip(40), sel(30), skip(50), sel(150)-- let original_selected: usize = orig_selectors- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();- assert_eq!(original_selected, 210); // 30 + 30 + 50 + 100-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // Verify reversed selection structure- let rev_selectors: Vec<_> = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();-- // After reversal: RG2, RG1, RG0- // RG2: select(100)- // RG1: skip(50), select(50)- // RG0: select(30), skip(40), select(30)- // Sequential: sel(100), skip(50), sel(50), sel(30), skip(40), sel(30)- // After merge: sel(100), skip(50), sel(80), skip(40), sel(30)-- let reversed_selected: usize = rev_selectors- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(- reversed_selected, 210,- "Total selected rows should remain the same (30 + 30 + 50 + 100)"- );-- // Verify row group order- assert_eq!(reversed_plan.row_group_indexes, vec![2, 1, 0]);- }-- #[test]- fn test_prepared_access_plan_reverse_alternating_detailed() {- // Test with alternating scan/skip pattern with detailed verification- let metadata = create_test_metadata(vec![100, 100, 100, 100]);-- // Scan RG0 and RG2, skip RG1 and RG3- let mut access_plan = ParquetAccessPlan::new(vec![- RowGroupAccess::Scan, // RG0- RowGroupAccess::Skip, // RG1- RowGroupAccess::Scan, // RG2- RowGroupAccess::Skip, // RG3- ]);-- access_plan.scan_selection(- 0,- RowSelection::from(vec![RowSelector::select(30), RowSelector::skip(70)]),- );- access_plan.scan_selection(- 2,- RowSelection::from(vec![RowSelector::skip(20), RowSelector::select(80)]),- );-- let rg_metadata = metadata.row_groups();- let prepared_plan = access_plan- .prepare(rg_metadata)- .expect("Failed to create PreparedAccessPlan");-- // Original: [0, 2]- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2]);-- // Verify original selection- let orig_selectors: Vec<_> = prepared_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();-- // Original:- // RG0: select(30), skip(70)- // RG2: skip(20), select(80)- // Sequential: sel(30), skip(90), sel(80)- // (RG0's skip(70) + RG2's skip(20) = skip(90))-- let original_selected: usize = orig_selectors- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();- assert_eq!(original_selected, 110); // 30 + 80-- let reversed_plan = prepared_plan- .reverse(&metadata)- .expect("Failed to reverse PreparedAccessPlan");-- // After reverse: [2, 0]- assert_eq!(reversed_plan.row_group_indexes, vec![2, 0]);-- // Verify reversed selection- let rev_selectors: Vec<_> = reversed_plan- .row_selection- .as_ref()- .unwrap()- .iter()- .collect();-- // After reversal: RG2, RG0- // RG2: skip(20), select(80)- // RG0: select(30), skip(70)- // Sequential: skip(20), sel(110), skip(70)- // (RG2's select(80) + RG0's select(30) = select(110))-- let reversed_selected: usize = rev_selectors- .iter()- .filter(|s| !s.skip)- .map(|s| s.row_count)- .sum();-- assert_eq!(reversed_selected, 110); // Should still be 30 + 80-- // Detailed verification of structure- assert_eq!(rev_selectors.len(), 3, "Reversed should have 3 selectors");-- assert!(- rev_selectors[0].skip && rev_selectors[0].row_count == 20,- "First selector should be skip(20) from RG2"- );-- assert!(- !rev_selectors[1].skip && rev_selectors[1].row_count == 110,- "Second selector should be select(110) from RG2(80) + RG0(30)"- );-- assert!(- rev_selectors[2].skip && rev_selectors[2].row_count == 70,- "Third selector should be skip(70) from RG0"+ reversed.row_groups[0].selection.selection(),+ Some(&RowSelection::from(vec![+ RowSelector::skip(1),+ RowSelector::select(3)+ ])));}}diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 3df0e0e7e1..32fc8c9f3d 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -774,7 +774,7 @@ impl FileSource for ParquetSource {// the parquet opener will pause the single decoder at row// group boundaries and consult `RowGroupPruner` to drop// RGs the current threshold proves unwinnable, rebuilding- // the decoder via `into_builder().with_row_groups(...)` to+ // the decoder via `into_builder().with_row_group_selections(...)` to// skip them. The actual pruning count appears as// `row_groups_pruned_dynamic_filter` in EXPLAIN ANALYZE.// We use `contains_dynamic_filter()` (matches both `Watching`diff --git a/datafusion/datasource-parquet/src/statistics_order_tests.rs b/datafusion/datasource-parquet/src/statistics_order_tests.rsindex 8b8e2c9831..93e5f63ad2 100644--- a/datafusion/datasource-parquet/src/statistics_order_tests.rs+++ b/datafusion/datasource-parquet/src/statistics_order_tests.rs@@ -273,18 +273,20 @@ impl TestFile {physical: &Arc<dyn PhysicalExpr>,plan: ParquetAccessPlan,) -> usize {- let mut builder = ParquetRecordBatchReaderBuilder::try_new(self.bytes.clone())- .unwrap()- .with_row_groups(plan.row_group_indexes());- if let Some(selection) = plan- .into_overall_row_selection(self.metadata.row_groups())- .unwrap()- {- builder = builder.with_row_selection(selection);- }- builder- .build()+ plan.prepare(self.metadata.row_groups()).unwrap()+ .row_groups+ .into_iter()+ .flat_map(|rg| {+ let mut builder =+ ParquetRecordBatchReaderBuilder::try_new(self.bytes.clone())+ .unwrap()+ .with_row_groups(vec![rg.selection.row_group_index()]);+ if let Some(selection) = rg.selection.selection() {+ builder = builder.with_row_selection(selection.clone());+ }+ builder.build().unwrap()+ }).map(|batch| {let batch = batch.unwrap();let matches = physicaldiff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex e11024449a..fc5e9d2f03 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -65,6 +65,26 @@ change. Use `FieldMetadata::to_hashmap` where a `HashMap` is still required.[`metadata`]: https://docs.rs/arrow/60.0.0/arrow/datatypes/struct.Metadata.html+### `ParquetAccessPlan::into_overall_row_selection` is deprecated++`ParquetAccessPlan::into_overall_row_selection` is deprecated starting in 56.0.0.+DataFusion now keeps selections local to each row group during Parquet scanning.+The method remains available with its existing conversion and selection-length+validation behavior, so direct callers continue to work but receive a deprecation+warning.++Users who attach a `ParquetAccessPlan` to a `PartitionedFile` for DataFusion to+read need no changes.++For custom readers, use `ParquetAccessPlan::into_inner()` to obtain the+`RowGroupAccess` entries. Their positions are the original file row-group+indexes, and each selection uses coordinates within its own row group. When+migrating away from the deprecated method, callers must handle conversion and+selection-length validation themselves. Parquet's push decoder and async reader+accept `RowGroupSelection` values through `with_row_group_selections`; the+synchronous reader still requires `with_row_groups` and an overall+`with_row_selection`.+### Missing Parquet null counts are treated as unknownDataFusion now preserves an omitted Parquet `null_count` statistic as unknown.