Skip to content

Apache DataFusion SQL Query Engine

refactor: use row-group-local selections for Parquet scans (#25608)

Huaijinb4f813b14fca
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rs
index 4d74a1f282..9ba0f80394 100644
--- a/datafusion/core/tests/parquet/external_access_plan.rs
+++ b/datafusion/core/tests/parquet/external_access_plan.rs
@@ -310,6 +310,36 @@ async fn two_selections() {
}
}
+#[tokio::test]
+async fn mixed_bitmap_and_selector_selections() {
+ // RG 0 selects a/c with a bitmap, RG 1 selects f/h with selectors.
+ // Each representation must keep coordinates local to its row group.
+ TestFull {
+ access_plan: Some(ParquetAccessPlan::new(vec![
+ RowGroupAccess::Selection(RowSelection::from(
+ arrow::buffer::BooleanBuffer::from(vec![true, false, true, false, false]),
+ )),
+ RowGroupAccess::Selection(select_two_rows()),
+ ])),
+ row_selection: None,
+ expected_rows: 4,
+ expected_output: Some(&[
+ "+------+------------+",
+ "| utf8 | large_utf8 |",
+ "+------+------------+",
+ "| a | a |",
+ "| c | c |",
+ "| f | f |",
+ "| h | h |",
+ "+------+------------+",
+ ]),
+ predicate: None,
+ }
+ .run()
+ .await
+ .unwrap();
+}
+
#[tokio::test]
async fn bad_row_groups() {
let err = TestFull {
diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rs
index b837159cf0..b72612134d 100644
--- a/datafusion/datasource-parquet/src/access_plan.rs
+++ b/datafusion/datasource-parquet/src/access_plan.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use crate::sort::reverse_row_selection;
use arrow::datatypes::Schema;
use datafusion_common::{Result, assert_eq_or_internal_err, exec_err};
use datafusion_physical_expr::expressions::Column;
@@ -23,6 +22,7 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;
use log::debug;
use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
+use parquet::arrow::push_decoder::RowGroupSelection;
use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};
/// A selection of rows and row groups within a ParquetFile to decode.
@@ -407,7 +407,14 @@ impl ParquetAccessPlan {
/// * `Some(selection)` if there are [`RowGroupAccess::Selection`]s
///
/// The returned selection represents which rows to scan across any row
- /// row groups which are not skipped.
+ /// groups which are not skipped.
+ ///
+ /// # Deprecated
+ ///
+ /// DataFusion scans now keep selections local to each row group. For custom
+ /// readers, use [`Self::into_inner`] to obtain the [`RowGroupAccess`] entries
+ /// and adapt them to the reader's selection API. This compatibility method
+ /// retains its existing conversion and length validation behavior.
///
/// # Notes
///
@@ -422,7 +429,7 @@ impl ParquetAccessPlan {
/// # Errors
///
/// Returns an error if any specified row selection does not specify
- /// the same number of rows as in it's corresponding `row_group_metadata`.
+ /// the same number of rows as in its corresponding `row_group_metadata`.
///
/// # Example: No Selections
///
@@ -446,12 +453,12 @@ impl ParquetAccessPlan {
/// ```text
/// RowGroupAccess::Scan (scan all row group 0)
/// RowGroupAccess::Skip (skip row group 1)
- /// RowGroupAccess::Select (skip 50, scan 50, skip 900) (scan rows 50-100 in row group 2)
+ /// RowGroupAccess::Selection (skip 50, scan 50, skip 900) (scan rows 50-100 in row group 2)
/// RowGroupAccess::Scan (scan all row group 3)
/// ```
///
/// Assuming each row group has 1000 rows, the resulting row selection would
- /// be the rows to scan in row group 0, 2 and 4:
+ /// be the rows to scan in row group 0, 2 and 3:
///
/// ```text
/// RowSelection::Select(1000) (scan all rows in row group 0)
@@ -467,6 +474,10 @@ impl ParquetAccessPlan {
/// still be `0, 2, 3` .
///
/// [`ArrowReaderBuilder::with_row_selection`]: parquet::arrow::arrow_reader::ArrowReaderBuilder::with_row_selection
+ #[deprecated(
+ since = "56.0.0",
+ note = "Use into_inner() to obtain row-group-local access entries"
+ )]
pub fn into_overall_row_selection(
self,
row_group_meta_data: &[RowGroupMetaData],
@@ -565,137 +576,90 @@ impl ParquetAccessPlan {
self.row_groups
}
- /// Prepare this plan and resolve to the final `PreparedAccessPlan`
+ /// Validate selections and prepare row groups for the decoder without
+ /// converting local selections into a single file-level selection.
pub(crate) fn prepare(
- mut self,
+ self,
row_group_meta_data: &[RowGroupMetaData],
) -> Result<PreparedAccessPlan> {
- let row_group_indexes = self.row_group_indexes();
- // `fully_matched` is indexed by absolute row-group index; take it out
- // before `into_overall_row_selection` consumes `self`.
- let fully_matched_by_index = std::mem::take(&mut self.fully_matched);
- let row_selection = self.into_overall_row_selection(row_group_meta_data)?;
-
- let (row_group_indexes, row_selection) =
- strip_empty_row_groups(row_group_indexes, row_selection, row_group_meta_data);
-
- // Carry `fully_matched` flags in the same order as the *surviving*
- // `row_group_indexes` so downstream code (per-RG `RowFilter` skip) can
- // look them up positionally. Mapping after the strip keeps
- // `strip_empty_row_groups` generic (no `fully_matched` parameter).
- let fully_matched: Vec<bool> = row_group_indexes
+ assert_eq!(row_group_meta_data.len(), self.row_groups.len());
+ // Keep the scan policies that previously depended on the presence of
+ // an overall selection, even when its partial groups are all empty.
+ // Local selections no longer require these restrictions to keep row
+ // offsets valid; enabling reordering and runtime pruning with selections
+ // is left to a follow-up (#24358).
+ let has_row_selection = self
+ .row_groups
.iter()
- .map(|&idx| fully_matched_by_index[idx])
- .collect();
-
- PreparedAccessPlan::new(row_group_indexes, fully_matched, row_selection)
- }
-}
-
-/// Drop row groups whose post-pruning `RowSelection` selects zero rows, so the
-/// prepared plan stays well-formed: `with_row_groups(...)` never names a row
-/// group the decoder would immediately skip.
-///
-/// arrow-rs's push decoder silently advances past an all-skipped row group
-/// inside `try_next_reader` without handing back a reader, so keeping such a
-/// row group in the plan would leave its row-group list one entry longer than
-/// the readers the decoder produces. An empty selection is reachable today via
-/// [`ParquetAccessPlan::scan_selection`], which intersects an existing
-/// `Selection` with a new one and can leave a row group selecting nothing.
-/// A well-formed plan here is also a precondition for re-enabling runtime
-/// pruning under a live selection (#24358) and for #23696.
-///
-/// Walks the flat `RowSelection` once with [`OverallRowSelectionCursor`],
-/// rather than a per-row-group [`RowSelection::split_off`] (which reallocates
-/// the selector tail on every call, i.e. O(row groups × selectors)), keeping
-/// only the row groups that still select at least one row. `row_group_meta_data`
-/// is the **full file** metadata, indexed absolutely by row-group index, while
-/// `row_selection` covers only the scanned row groups — matching
-/// `into_overall_row_selection`, which emits nothing for a skipped row group.
-/// When `row_selection` is `None` no row group can be empty and the inputs are
-/// returned unchanged.
-fn strip_empty_row_groups(
- row_group_indexes: Vec<usize>,
- row_selection: Option<RowSelection>,
- row_group_meta_data: &[RowGroupMetaData],
-) -> (Vec<usize>, Option<RowSelection>) {
- let Some(selection) = row_selection else {
- return (row_group_indexes, None);
- };
-
- let mut cursor = OverallRowSelectionCursor::new(selection);
- let mut kept_indexes = Vec::with_capacity(row_group_indexes.len());
- let mut kept_selectors: Vec<RowSelector> = Vec::new();
-
- for &rg_idx in row_group_indexes.iter() {
- let rg_row_count = row_group_meta_data[rg_idx].num_rows() as usize;
- // Pull this row group's fragments off the shared cursor in a single
- // pass (no `split_off` tail reallocation).
- let start = kept_selectors.len();
- let mut selected = 0usize;
- let mut taken = 0usize;
- while taken < rg_row_count {
- let Some(fragment) = cursor.take(rg_row_count - taken) else {
- break;
+ .any(|access| matches!(access, RowGroupAccess::Selection(_)));
+ let mut row_groups = Vec::with_capacity(self.row_groups.len());
+ for (index, (access, fully_matched)) in self
+ .row_groups
+ .into_iter()
+ .zip(self.fully_matched)
+ .enumerate()
+ {
+ let selection = match access {
+ RowGroupAccess::Skip => continue,
+ RowGroupAccess::Scan => {
+ // Preserve strip_empty_row_groups behavior: without an
+ // overall selection it returned the scan list unchanged;
+ // with one it dropped every group selecting zero rows,
+ // including zero-row Scan groups.
+ if has_row_selection && row_group_meta_data[index].num_rows() == 0 {
+ continue;
+ }
+ None
+ }
+ RowGroupAccess::Selection(selection) => {
+ let rows_in_selection = selection.total_row_count();
+ let row_group_row_count = row_group_meta_data[index].num_rows();
+ assert_eq_or_internal_err!(
+ rows_in_selection as i64,
+ row_group_row_count,
+ "Invalid ParquetAccessPlan Selection. Row group {index} has {row_group_row_count} rows \
+ but selection only specifies {rows_in_selection} rows. \
+ Selection: {selection:?}"
+ );
+ // Intersections can leave an empty selection: drop the
+ // group together with its match status.
+ if selection.row_count() == 0 {
+ continue;
+ }
+ Some(selection)
+ }
};
- taken += fragment.row_count;
- if !fragment.skip {
- selected += fragment.row_count;
- }
- kept_selectors.push(fragment);
- }
- if selected > 0 {
- kept_indexes.push(rg_idx);
- } else {
- // Empty row group: arrow-rs would silently skip it. Drop it and its
- // fragments so the plan stays 1:1 with the readers.
- kept_selectors.truncate(start);
+ row_groups.push(PreparedRowGroup {
+ selection: RowGroupSelection::new(index, selection),
+ fully_matched,
+ });
}
+ Ok(PreparedAccessPlan {
+ has_row_selection: has_row_selection && !row_groups.is_empty(),
+ row_groups,
+ })
}
+}
- let result_selection = if kept_selectors.is_empty() {
- None
- } else {
- Some(RowSelection::from(kept_selectors))
- };
-
- (kept_indexes, result_selection)
+/// A row group's selection and static match status travel together when the
+/// scan order changes.
+#[derive(Debug)]
+pub(crate) struct PreparedRowGroup {
+ pub(crate) selection: RowGroupSelection,
+ /// Whether statistics proved every row passes the predicate.
+ pub(crate) fully_matched: bool,
}
-/// Represents a prepared, fully resolved [`ParquetAccessPlan`]
-///
-/// The [`RowSelection`] represents the result of applying all pruning such as
-/// user provided scans, Row Group statistics, DataPage statistics, and Bloom
-/// Filters.
-///
-/// This plan is what is passed to the parquet reader
+/// The final scan order and row-group-local selections passed to the decoder.
+#[derive(Debug)]
pub(crate) struct PreparedAccessPlan {
- /// Row group indexes to read
- pub(crate) row_group_indexes: Vec<usize>,
- /// Per-RG `fully_matched` flag, positionally aligned with
- /// [`Self::row_group_indexes`]. A `true` entry means stats already
- /// proved every row of this RG passes the predicate, so the per-row
- /// `RowFilter` can be skipped for it.
- pub(crate) fully_matched: Vec<bool>,
- /// Optional row selection for filtering within row groups
- pub(crate) row_selection: Option<RowSelection>,
+ pub(crate) row_groups: Vec<PreparedRowGroup>,
+ /// Preserve the existing reordering and dynamic-pruning eligibility policy.
+ /// This can remain true after an empty selection's group is removed.
+ pub(crate) has_row_selection: bool,
}
impl PreparedAccessPlan {
- /// Create a new prepared access plan
- fn new(
- row_group_indexes: Vec<usize>,
- fully_matched: Vec<bool>,
- row_selection: Option<RowSelection>,
- ) -> Result<Self> {
- debug_assert_eq!(row_group_indexes.len(), fully_matched.len());
- Ok(Self {
- row_group_indexes,
- fully_matched,
- row_selection,
- })
- }
-
/// Reorder row groups by their min statistics for the given sort order.
///
/// This helps TopK queries find optimal values first. Row groups are
@@ -714,7 +678,7 @@ impl PreparedAccessPlan {
/// dynamic filter converges only as fast as disk order allows.
///
/// Gracefully skips reordering when:
- /// - There is a row_selection (too complex to remap)
+ /// - There are row selections (preserve the existing scan-order policy)
/// - 0 or 1 row groups (nothing to reorder)
/// - The leading sort expression is not a simple column reference
/// - Statistics are unavailable
@@ -724,21 +688,21 @@ impl PreparedAccessPlan {
file_metadata: &ParquetMetaData,
arrow_schema: &Schema,
) -> Result<Self> {
- // Skip if row_selection present (too complex to remap)
- if self.row_selection.is_some() {
+ // Preserve the current policy for scans with page or external selections.
+ if self.has_row_selection {
debug!("Skipping RG reorder: row_selection present");
return Ok(self);
}
// Nothing to reorder
- if self.row_group_indexes.len() <= 1 {
+ if self.row_groups.len() <= 1 {
return Ok(self);
}
let rg_metadata: Vec<&RowGroupMetaData> = self
- .row_group_indexes
+ .row_groups
.iter()
- .map(|&idx| file_metadata.row_group(idx))
+ .map(|rg| file_metadata.row_group(rg.selection.row_group_index()))
.collect();
let leading_descending = sort_order.first().options.descending;
@@ -845,41 +809,29 @@ impl PreparedAccessPlan {
}
};
- // Apply the reordering — `fully_matched` must be permuted alongside
- // `row_group_indexes` so the two stay positionally aligned for the
- // per-RG `RowFilter` skip path.
- let original_indexes = self.row_group_indexes.clone();
- let original_fully_matched = self.fully_matched.clone();
- let order: Vec<usize> = sorted_indices
+ // Move each selection together with its match status, without cloning
+ // potentially large selection buffers just to reorder row groups.
+ let mut original: Vec<_> = self.row_groups.into_iter().map(Some).collect();
+ self.row_groups = sorted_indices
.values()
.iter()
- .map(|&i| i as usize)
+ .map(|&i| original[i as usize].take().expect("unique sort index"))
.collect();
- self.row_group_indexes = order.iter().map(|&i| original_indexes[i]).collect();
- self.fully_matched = order.iter().map(|&i| original_fully_matched[i]).collect();
-
Ok(self)
}
- /// Reverse the access plan for reverse scanning
- pub(crate) fn reverse(mut self, file_metadata: &ParquetMetaData) -> Result<Self> {
- // Get the row group indexes before reversing
- let row_groups_to_scan = self.row_group_indexes.clone();
-
- // Reverse the row group indexes (and the parallel `fully_matched`)
- self.row_group_indexes = self.row_group_indexes.into_iter().rev().collect();
- self.fully_matched = self.fully_matched.into_iter().rev().collect();
-
- // If we have a row selection, reverse it to match the new row group order
- if let Some(row_selection) = self.row_selection {
- self.row_selection = Some(reverse_row_selection(
- &row_selection,
- file_metadata,
- &row_groups_to_scan, // Pass the original (non-reversed) row group indexes
- )?);
- }
+ /// Reverse row-group order while preserving local row coordinates.
+ pub(crate) fn reverse(mut self) -> Self {
+ self.row_groups.reverse();
+ self
+ }
- Ok(self)
+ #[cfg(test)]
+ pub(crate) fn row_group_indexes(&self) -> Vec<usize> {
+ self.row_groups
+ .iter()
+ .map(|rg| rg.selection.row_group_index())
+ .collect()
}
}
@@ -893,122 +845,147 @@ mod test {
use std::sync::{Arc, LazyLock};
#[test]
- fn test_only_scans() {
- let access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Scan,
- RowGroupAccess::Scan,
- RowGroupAccess::Scan,
- RowGroupAccess::Scan,
- ]);
-
- let row_group_indexes = access_plan.row_group_indexes();
- let row_selection = access_plan
- .into_overall_row_selection(&ROW_GROUP_METADATA)
- .unwrap();
+ #[expect(deprecated)]
+ fn test_deprecated_overall_row_selection() {
+ for plan in [
+ ParquetAccessPlan::new_all(4),
+ ParquetAccessPlan::new_none(4),
+ ] {
+ assert_eq!(
+ plan.into_overall_row_selection(&ROW_GROUP_METADATA)
+ .unwrap(),
+ None
+ );
+ }
- // scan all row groups, no selection
- assert_eq!(row_group_indexes, vec![0, 1, 2, 3]);
- assert_eq!(row_selection, None);
- }
+ // Skipped groups do not occupy coordinates in the combined selection.
+ // Check both input representations retain the same conversion behavior.
+ let selectors =
+ RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(20)]);
+ let bitmap = RowSelection::from(arrow::buffer::BooleanBuffer::from(
+ (0..30).map(|i| i >= 10).collect::<Vec<_>>(),
+ ));
+ for selection in [selectors, bitmap] {
+ let plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Skip,
+ RowGroupAccess::Selection(selection),
+ RowGroupAccess::Scan,
+ ]);
+ assert_eq!(
+ plan.into_overall_row_selection(&ROW_GROUP_METADATA)
+ .unwrap(),
+ Some(RowSelection::from(vec![
+ RowSelector::select(10),
+ RowSelector::skip(10),
+ RowSelector::select(60),
+ ]))
+ );
+ }
- #[test]
- fn test_only_skips() {
- let access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Skip,
+ let plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Selection(vec![RowSelector::skip(10)].into()),
RowGroupAccess::Skip,
RowGroupAccess::Skip,
RowGroupAccess::Skip,
]);
-
- let row_group_indexes = access_plan.row_group_indexes();
- let row_selection = access_plan
- .into_overall_row_selection(&ROW_GROUP_METADATA)
- .unwrap();
-
- // skip all row groups, no selection
- assert_eq!(row_group_indexes, vec![] as Vec<usize>);
- assert_eq!(row_selection, None);
+ assert_eq!(
+ plan.into_overall_row_selection(&ROW_GROUP_METADATA)
+ .unwrap(),
+ Some(RowSelection::from(vec![RowSelector::skip(10)]))
+ );
}
+
#[test]
- fn test_mixed_1() {
- let access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Scan,
- RowGroupAccess::Selection(
- // specifies all 20 rows in row group 1
- vec![
- RowSelector::select(5),
- RowSelector::skip(7),
- RowSelector::select(8),
- ]
- .into(),
- ),
- RowGroupAccess::Skip,
- RowGroupAccess::Skip,
- ]);
+ #[expect(deprecated)]
+ fn test_deprecated_overall_row_selection_validates_length() {
+ for rows in [19, 21] {
+ let plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Selection(vec![RowSelector::select(rows)].into()),
+ RowGroupAccess::Skip,
+ RowGroupAccess::Skip,
+ ]);
+ let err = plan
+ .into_overall_row_selection(&ROW_GROUP_METADATA)
+ .unwrap_err()
+ .to_string();
+ assert_contains!(
+ err,
+ format!(
+ "Row group 1 has 20 rows but selection only specifies {rows} rows"
+ )
+ );
+ }
+ }
- let row_group_indexes = access_plan.row_group_indexes();
- let row_selection = access_plan
- .into_overall_row_selection(&ROW_GROUP_METADATA)
+ #[test]
+ fn test_only_scans() {
+ let plan = ParquetAccessPlan::new_all(4)
+ .prepare(&ROW_GROUP_METADATA)
.unwrap();
-
- assert_eq!(row_group_indexes, vec![0, 1]);
- assert_eq!(
- row_selection,
- Some(
- vec![
- // select the entire first row group
- RowSelector::select(10),
- // selectors from the second row group
- RowSelector::select(5),
- RowSelector::skip(7),
- RowSelector::select(8)
- ]
- .into()
- )
+ assert_eq!(plan.row_group_indexes(), vec![0, 1, 2, 3]);
+ assert!(!plan.has_row_selection);
+ assert!(
+ plan.row_groups
+ .iter()
+ .all(|rg| rg.selection.selection().is_none())
);
}
#[test]
- fn test_mixed_2() {
- let access_plan = ParquetAccessPlan::new(vec![
+ fn test_only_skips() {
+ let plan = ParquetAccessPlan::new_none(4)
+ .prepare(&ROW_GROUP_METADATA)
+ .unwrap();
+ assert!(plan.row_groups.is_empty());
+ assert!(!plan.has_row_selection);
+ }
+
+ #[test]
+ fn test_mixed_selections() {
+ let selection = RowSelection::from(vec![
+ RowSelector::select(5),
+ RowSelector::skip(7),
+ RowSelector::select(18),
+ ]);
+ let plan = ParquetAccessPlan::new(vec![
RowGroupAccess::Skip,
RowGroupAccess::Scan,
- RowGroupAccess::Selection(
- // specify all 30 rows in row group 1
- vec![
- RowSelector::select(5),
- RowSelector::skip(7),
- RowSelector::select(18),
- ]
- .into(),
- ),
+ RowGroupAccess::Selection(selection.clone()),
RowGroupAccess::Scan,
- ]);
-
- let row_group_indexes = access_plan.row_group_indexes();
- let row_selection = access_plan
- .into_overall_row_selection(&ROW_GROUP_METADATA)
- .unwrap();
-
- assert_eq!(row_group_indexes, vec![1, 2, 3]);
+ ])
+ .prepare(&ROW_GROUP_METADATA)
+ .unwrap();
+ assert_eq!(plan.row_group_indexes(), vec![1, 2, 3]);
+ assert!(plan.has_row_selection);
assert_eq!(
- row_selection,
- Some(
- vec![
- // select the entire second row group
- RowSelector::select(20),
- // selectors from the third row group
- RowSelector::select(5),
- RowSelector::skip(7),
- RowSelector::select(18),
- // select the entire fourth row group
- RowSelector::select(40),
- ]
- .into()
- )
+ plan.row_groups[0].selection,
+ RowGroupSelection::new(1, None)
+ );
+ assert_eq!(
+ plan.row_groups[1].selection,
+ RowGroupSelection::new(2, Some(selection))
+ );
+ assert_eq!(
+ plan.row_groups[2].selection,
+ RowGroupSelection::new(3, None)
);
}
+ fn scan_plan(indexes: Vec<usize>) -> PreparedAccessPlan {
+ PreparedAccessPlan {
+ has_row_selection: false,
+ row_groups: indexes
+ .into_iter()
+ .map(|index| PreparedRowGroup {
+ selection: RowGroupSelection::new(index, None),
+ fully_matched: false,
+ })
+ .collect(),
+ }
+ }
+
#[test]
fn test_new_from_overall_row_selection() {
let row_selection = RowSelection::from(vec![
@@ -1107,7 +1084,7 @@ mod test {
let row_group_indexes = access_plan.row_group_indexes();
let err = access_plan
- .into_overall_row_selection(&ROW_GROUP_METADATA)
+ .prepare(&ROW_GROUP_METADATA)
.unwrap_err()
.to_string();
assert_eq!(row_group_indexes, vec![0, 1, 2, 3]);
@@ -1136,7 +1113,7 @@ mod test {
let row_group_indexes = access_plan.row_group_indexes();
let err = access_plan
- .into_overall_row_selection(&ROW_GROUP_METADATA)
+ .prepare(&ROW_GROUP_METADATA)
.unwrap_err()
.to_string();
assert_eq!(row_group_indexes, vec![0, 1, 2, 3]);
@@ -1259,7 +1236,8 @@ mod test {
#[test]
fn reorder_by_statistics_sorts_row_groups_asc_by_min() {
let metadata = parquet_metadata_with_int_mins(&[50, 10, 100]);
- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();
+ let mut plan = scan_plan(vec![0, 1, 2]);
+ plan.row_groups[1].fully_matched = true;
let result = plan
.reorder_by_statistics(
@@ -1269,17 +1247,25 @@ mod test {
)
.unwrap();
- assert_eq!(result.row_group_indexes, vec![1, 0, 2]);
+ assert_eq!(result.row_group_indexes(), vec![1, 0, 2]);
+ assert_eq!(
+ result
+ .row_groups
+ .iter()
+ .map(|rg| rg.fully_matched)
+ .collect::<Vec<_>>(),
+ vec![true, false, false]
+ );
}
- /// A `row_selection` is "too complex to remap" through reorder,
- /// so the function short-circuits and returns the input untouched.
+ /// Preserve the existing scan order when selections are present.
#[test]
fn reorder_by_statistics_skips_when_row_selection_present() {
let metadata = parquet_metadata_with_int_mins(&[50, 10]);
let selection = RowSelection::from(vec![RowSelector::select(100)]);
- let plan =
- PreparedAccessPlan::new(vec![0, 1], vec![false; 2], Some(selection)).unwrap();
+ let mut plan = scan_plan(vec![0, 1]);
+ plan.row_groups[0].selection = RowGroupSelection::new(0, Some(selection));
+ plan.has_row_selection = true;
let result = plan
.reorder_by_statistics(
@@ -1289,14 +1275,14 @@ mod test {
)
.unwrap();
- assert_eq!(result.row_group_indexes, vec![0, 1]);
+ assert_eq!(result.row_group_indexes(), vec![0, 1]);
}
/// One row group means nothing to reorder.
#[test]
fn reorder_by_statistics_skips_when_at_most_one_row_group() {
let metadata = parquet_metadata_with_int_mins(&[50]);
- let plan = PreparedAccessPlan::new(vec![0], vec![false; 1], None).unwrap();
+ let plan = scan_plan(vec![0]);
let result = plan
.reorder_by_statistics(
@@ -1306,7 +1292,7 @@ mod test {
)
.unwrap();
- assert_eq!(result.row_group_indexes, vec![0]);
+ assert_eq!(result.row_group_indexes(), vec![0]);
}
/// Non-`Column` sort expressions (e.g. `a + 1`,
@@ -1315,7 +1301,7 @@ mod test {
#[test]
fn reorder_by_statistics_skips_for_non_column_sort_expr() {
let metadata = parquet_metadata_with_int_mins(&[50, 10]);
- let plan = PreparedAccessPlan::new(vec![0, 1], vec![false; 2], None).unwrap();
+ let plan = scan_plan(vec![0, 1]);
let arrow_schema = arrow_schema_a_int();
let order = LexOrdering::new(vec![PhysicalSortExpr {
expr: Arc::new(BinaryExpr::new(
@@ -1334,7 +1320,7 @@ mod test {
.reorder_by_statistics(&order, &metadata, &arrow_schema)
.unwrap();
- assert_eq!(result.row_group_indexes, vec![0, 1]);
+ assert_eq!(result.row_group_indexes(), vec![0, 1]);
}
/// When the sort column lives outside the file's arrow schema
@@ -1344,7 +1330,7 @@ mod test {
#[test]
fn reorder_by_statistics_skips_when_column_not_in_arrow_schema() {
let metadata = parquet_metadata_with_int_mins(&[50, 10]);
- let plan = PreparedAccessPlan::new(vec![0, 1], vec![false; 2], None).unwrap();
+ let plan = scan_plan(vec![0, 1]);
// Arrow schema only has "a"; the sort references "b".
let arrow_schema = arrow_schema_a_int();
let order = LexOrdering::new(vec![PhysicalSortExpr {
@@ -1360,7 +1346,7 @@ mod test {
.reorder_by_statistics(&order, &metadata, &arrow_schema)
.unwrap();
- assert_eq!(result.row_group_indexes, vec![0, 1]);
+ assert_eq!(result.row_group_indexes(), vec![0, 1]);
}
// ----------------------------------------------------------------
@@ -1447,7 +1433,7 @@ mod test {
fn reorder_by_statistics_breaks_leading_ties_with_secondary_column() {
let metadata =
parquet_metadata_with_two_col_mins(&[(1, 300), (1, 100), (1, 200)]);
- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();
+ let plan = scan_plan(vec![0, 1, 2]);
let order =
LexOrdering::new(vec![sort_expr("a", 0, false), sort_expr("b", 1, false)])
.unwrap();
@@ -1456,7 +1442,7 @@ mod test {
.reorder_by_statistics(&order, &metadata, &arrow_schema_ab_int())
.unwrap();
- assert_eq!(result.row_group_indexes, vec![1, 2, 0]);
+ assert_eq!(result.row_group_indexes(), vec![1, 2, 0]);
}
/// `ORDER BY a ASC, b DESC`: the secondary key's direction is
@@ -1466,7 +1452,7 @@ mod test {
fn reorder_by_statistics_honors_secondary_direction() {
let metadata =
parquet_metadata_with_two_col_mins(&[(1, 100), (1, 300), (0, 500)]);
- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();
+ let plan = scan_plan(vec![0, 1, 2]);
let order =
LexOrdering::new(vec![sort_expr("a", 0, false), sort_expr("b", 1, true)])
.unwrap();
@@ -1476,7 +1462,7 @@ mod test {
.unwrap();
// a=0 first, then the two a=1 groups by b DESC: 300 before 100.
- assert_eq!(result.row_group_indexes, vec![2, 1, 0]);
+ assert_eq!(result.row_group_indexes(), vec![2, 1, 0]);
}
/// `ORDER BY a DESC, b DESC` is normalized to ASC lexsort here and
@@ -1486,7 +1472,7 @@ mod test {
fn reorder_by_statistics_normalizes_desc_desc_for_reverse() {
let metadata =
parquet_metadata_with_two_col_mins(&[(1, 300), (2, 100), (1, 100)]);
- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();
+ let plan = scan_plan(vec![0, 1, 2]);
let order =
LexOrdering::new(vec![sort_expr("a", 0, true), sort_expr("b", 1, true)])
.unwrap();
@@ -1497,7 +1483,7 @@ mod test {
// ASC lexsort of (a, b): (1,100) < (1,300) < (2,100); the later
// reverse() produces (2,100), (1,300), (1,100) = (a DESC, b DESC).
- assert_eq!(result.row_group_indexes, vec![2, 0, 1]);
+ assert_eq!(result.row_group_indexes(), vec![2, 0, 1]);
}
/// A non-`Column` *secondary* expression stops the stats walk but
@@ -1506,7 +1492,7 @@ mod test {
fn reorder_by_statistics_keeps_leading_prefix_on_non_column_secondary() {
let metadata =
parquet_metadata_with_two_col_mins(&[(5, 300), (3, 100), (4, 200)]);
- let plan = PreparedAccessPlan::new(vec![0, 1, 2], vec![false; 3], None).unwrap();
+ let plan = scan_plan(vec![0, 1, 2]);
let order = LexOrdering::new(vec![
sort_expr("a", 0, false),
PhysicalSortExpr {
@@ -1528,63 +1514,47 @@ mod test {
.unwrap();
// Ordered by min(a) ASC only: 3, 4, 5.
- assert_eq!(result.row_group_indexes, vec![1, 2, 0]);
+ assert_eq!(result.row_group_indexes(), vec![1, 2, 0]);
}
#[test]
- fn test_strip_empty_row_groups_drops_only_empties() {
- // 4 row groups of [10, 20, 30, 40] rows. RG 1 and RG 3 select nothing
- // after pruning, so they must be dropped; RG 0 and RG 2 survive with
- // their re-concatenated selections intact.
- let selection = RowSelection::from(vec![
- RowSelector::select(10), // RG 0: keep all 10
- RowSelector::skip(30), // RG 1 (20) fully skipped + RG 2's leading 10
- RowSelector::select(20), // RG 2: keep 20
- RowSelector::skip(40), // RG 3: skip all 40
+ fn test_prepare_drops_empty_selections_and_keeps_match_status() {
+ let mut plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(20)])),
+ RowGroupAccess::Selection(RowSelection::from(vec![
+ RowSelector::skip(10),
+ RowSelector::select(20),
+ ])),
+ RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(40)])),
]);
-
- let (indexes, result) = strip_empty_row_groups(
- vec![0, 1, 2, 3],
- Some(selection),
- &ROW_GROUP_METADATA,
+ plan.fully_matched = vec![true, false, false, true];
+ let prepared = plan.prepare(&ROW_GROUP_METADATA).unwrap();
+ assert_eq!(prepared.row_group_indexes(), vec![0, 2]);
+ assert!(prepared.row_groups[0].fully_matched);
+ assert!(!prepared.row_groups[1].fully_matched);
+ assert_eq!(
+ prepared.row_groups[1].selection.selection(),
+ Some(&RowSelection::from(vec![
+ RowSelector::skip(10),
+ RowSelector::select(20)
+ ]))
);
-
- // RG 1 and RG 3 are dropped; the surviving indexes stay in order.
- assert_eq!(indexes, vec![0, 2]);
- let result = result.expect("survivors keep a selection");
- assert_eq!(result.row_count(), 30); // 10 from RG 0 + 20 from RG 2
- assert_eq!(result.skipped_row_count(), 10); // RG 2's leading skip
}
#[test]
- fn test_strip_empty_row_groups_non_contiguous_indexes() {
- // Only RG 1 (20 rows) and RG 3 (40 rows) are scanned — RG 0 and RG 2
- // are whole-group skips, so the selection covers 20 + 40 = 60 rows.
- // This pins down that `strip` indexes `row_group_meta_data`
- // *absolutely* (meta[1]=20, meta[3]=40): a relative reading would take
- // 10 then 20 rows and misalign the split.
- let selection = RowSelection::from(vec![
- RowSelector::skip(20), // RG 1: skip all 20 -> dropped
- RowSelector::select(40), // RG 3: keep all 40
- ]);
-
- let (indexes, result) =
- strip_empty_row_groups(vec![1, 3], Some(selection), &ROW_GROUP_METADATA);
-
- assert_eq!(indexes, vec![3]);
- let result = result.expect("RG 3 survives");
- assert_eq!(result.row_count(), 40);
- assert_eq!(result.skipped_row_count(), 0);
- }
-
- #[test]
- fn test_strip_empty_row_groups_none_selection_unchanged() {
- // With no row selection no row group can be empty, so the inputs pass
- // through unchanged.
- let (indexes, result) =
- strip_empty_row_groups(vec![0, 1, 2, 3], None, &ROW_GROUP_METADATA);
- assert_eq!(indexes, vec![0, 1, 2, 3]);
- assert!(result.is_none());
+ fn test_prepare_preserves_selection_policy_after_dropping_empty_group() {
+ let prepared = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(20)])),
+ RowGroupAccess::Skip,
+ RowGroupAccess::Skip,
+ ])
+ .prepare(&ROW_GROUP_METADATA)
+ .unwrap();
+ assert_eq!(prepared.row_group_indexes(), vec![0]);
+ assert!(prepared.row_groups[0].selection.selection().is_none());
+ assert!(prepared.has_row_selection);
}
#[test]
@@ -1606,7 +1576,7 @@ mod test {
1,
RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(10)]),
);
- // RG 2 keeps a genuine partial selection so a flat selection is emitted.
+ // RG 2 keeps a genuine partial selection.
plan.scan_selection(
2,
RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(20)]),
@@ -1615,6 +1585,6 @@ mod test {
let prepared = plan.prepare(&ROW_GROUP_METADATA).expect("prepare");
// RG 1 (emptied by the intersection) is stripped; RG 0/2/3 remain.
- assert_eq!(prepared.row_group_indexes, vec![0, 2, 3]);
+ assert_eq!(prepared.row_group_indexes(), vec![0, 2, 3]);
}
}
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index de303b11bd..1f4a09a58b 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -1665,8 +1665,8 @@ impl RowGroupsPrunedParquetOpen {
//
// 2. `reverse`: flip the iteration order for DESC requests, applied
// AFTER any reorder so the reversed order is correct whether or
- // not reorder changed anything. Also handles `row_selection`
- // remapping.
+ // not reorder changed anything. Local selections move with their
+ // row groups.
//
// For sorted data: reorder is a no-op, reverse gives perfect DESC.
// For unsorted data: reorder fixes the order, reverse flips for DESC.
@@ -1685,7 +1685,7 @@ impl RowGroupsPrunedParquetOpen {
)?;
}
if prepared.reverse_row_groups {
- prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;
+ prepared_plan = prepared_plan.reverse();
}
Ok(prepared_plan)
};
@@ -1720,9 +1720,9 @@ impl RowGroupsPrunedParquetOpen {
} = {
// Build the prepared access plan first — `prepare_access_plan` may
// call `reorder_by_statistics` (for `sort_order_for_reorder`) and
- // `reverse` (for `reverse_row_groups`), both of which mutate
- // `row_group_indexes` to the physical scan order the decoder will
- // actually read. We MUST build our `rg_plan` from this reordered
+ // `reverse` (for `reverse_row_groups`), both of which arrange row
+ // groups in the physical scan order the decoder will read.
+ // We MUST build our `rg_plan` from this reordered
// list, otherwise our per-RG pruner check would consult the
// metadata of a different RG than the decoder is about to yield.
let decoder_config = DecoderBuilderConfig {
@@ -1734,35 +1734,23 @@ impl RowGroupsPrunedParquetOpen {
};
let prepared_access_plan = prepare_access_plan(access_plan)?;
- // #24355: a row selection (from page-index pruning, or an externally
- // supplied `ParquetRowSelection`) is carried by the decoder as one
- // flat selection over the concatenation of the remaining row groups.
- // The runtime pruner's `into_builder().with_row_groups(...)` rebuild
- // drops row groups without slicing that selection to match, so record
- // whether a selection is present and disable runtime pruning below
- // when it is (mirroring `reorder_by_statistics`, which also bails when
- // a row selection is present). The proper fix that keeps pruning
- // under a live selection is tracked in
- // https://github.com/apache/arrow-rs/issues/10624 /
- // https://github.com/apache/datafusion/issues/24358.
- let has_row_selection = prepared_access_plan.row_selection.is_some();
- // Build `rg_plan` parallel to the decoder's view: the
- // `prepared_access_plan` has already had its empty-selection
- // row groups stripped, so 1:1 correspondence with the readers
- // arrow-rs will hand back is restored. We zip with the
- // `fully_matched` flag so the stream can toggle the per-row
- // `RowFilter` per RG.
- let rg_plan: VecDeque<RgPlanEntry> = prepared_access_plan
- .row_group_indexes
- .iter()
- .copied()
- .zip(prepared_access_plan.fully_matched.iter().copied())
- .map(|(rg_index, fully_matched)| RgPlanEntry {
- rg_index,
- fully_matched,
- bytes: row_group_bytes(&rg_metadata[rg_index]),
- })
- .collect();
+ let has_row_selection = prepared_access_plan.has_row_selection;
+ // Move selections into the decoder, retaining only the metadata
+ // needed for pruning, filter toggles, and byte accounting.
+ let (selections, rg_plan): (Vec<_>, VecDeque<RgPlanEntry>) =
+ prepared_access_plan
+ .row_groups
+ .into_iter()
+ .map(|rg| {
+ let rg_index = rg.selection.row_group_index();
+ let entry = RgPlanEntry {
+ rg_index,
+ bytes: row_group_bytes(&rg_metadata[rg_index]),
+ fully_matched: rg.fully_matched,
+ };
+ (rg.selection, entry)
+ })
+ .unzip();
// Decide the initial row filter state based on the first RG to
// read. If that RG is `fully_matched` the per-row predicate is
@@ -1778,8 +1766,7 @@ impl RowGroupsPrunedParquetOpen {
let first_rg_fully_matched = rg_plan.front().is_some_and(|e| e.fully_matched);
let row_filter_context = precomputed_context;
- let mut builder =
- decoder_config.build(prepared_access_plan, reader_metadata.clone());
+ let mut builder = decoder_config.build(selections, reader_metadata.clone());
let mut filter_installed = false;
if let Some(ctx) = row_filter_context.as_ref() {
if first_rg_fully_matched {
@@ -1857,11 +1844,9 @@ impl RowGroupsPrunedParquetOpen {
// via the `DynamicFilterTracker` watch channel (#22460), so detecting
// a threshold change is a single atomic load — not a tree walk per
// RG check.
- // Also disabled when a row selection is live (#24355) — page-index
- // pruning is the common source: the pruner rebuilds the decoder via
- // `with_row_groups(...)`, which drops row groups without slicing the
- // carried selection to match, so pruning under a live selection returns
- // wrong results. Decline to prune in that case.
+ // Preserve the current policy for scans with selections. Enabling
+ // runtime pruning for them is separate from adopting the local API
+ // and is tracked in https://github.com/apache/datafusion/issues/24358.
let row_group_pruner =
match (&prepared.predicate, rg_plan.len() > 1, has_row_selection) {
(Some(predicate), true, false)
diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs
index 344e97b1e0..e5c03df486 100644
--- a/datafusion/datasource-parquet/src/push_decoder.rs
+++ b/datafusion/datasource-parquet/src/push_decoder.rs
@@ -29,7 +29,7 @@
//! [`RowGroupPruner`] is consulted; row groups it proves unwinnable are
//! dropped from the head of `rg_plan` and the decoder is rebuilt via
//! [`ParquetPushDecoder::into_builder`] +
-//! [`ParquetPushDecoderBuilder::with_row_groups`] so the skipped RGs are
+//! [`ParquetPushDecoderBuilder::with_row_group_selections`] so the skipped RGs are
//! bypassed entirely — no decode, no row-filter eval.
//!
//! The opener constructs both halves and hands the state off to
@@ -50,7 +50,9 @@ use parquet::arrow::arrow_reader::{
ArrowReaderMetadata, ParquetRecordBatchReader, RowFilter, RowSelectionPolicy,
};
use parquet::arrow::async_reader::AsyncFileReader;
-use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder};
+use parquet::arrow::push_decoder::{
+ ParquetPushDecoder, ParquetPushDecoderBuilder, RowGroupSelection,
+};
use parquet::file::metadata::ParquetMetaData;
use datafusion_common::{DataFusionError, Result, internal_err};
@@ -60,7 +62,6 @@ use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Gauge};
use datafusion_pruning::{PruningPredicate, PruningPredicateBuilder};
use crate::ParquetFileMetrics;
-use crate::access_plan::PreparedAccessPlan;
use crate::decoder_projection::DecoderProjection;
use crate::metrics::{ByteProgress, RowFilterSkippedFullyMatchedMetric};
use crate::row_filter::{
@@ -83,14 +84,14 @@ pub(crate) struct DecoderBuilderConfig<'a> {
}
impl DecoderBuilderConfig<'_> {
- /// Build a [`ParquetPushDecoderBuilder`] from a prepared access plan.
+ /// Build a [`ParquetPushDecoderBuilder`] from row-group-local selections.
///
/// The caller is expected to attach the
/// [`RowFilter`] and predicate
/// cache size on the returned builder.
pub(crate) fn build(
&self,
- prepared_access_plan: PreparedAccessPlan,
+ row_group_selections: Vec<RowGroupSelection>,
metadata: ArrowReaderMetadata,
) -> ParquetPushDecoderBuilder {
let mut builder = ParquetPushDecoderBuilder::new_with_metadata(metadata)
@@ -100,10 +101,7 @@ impl DecoderBuilderConfig<'_> {
if self.force_filter_selections {
builder = builder.with_row_selection_policy(RowSelectionPolicy::Selectors);
}
- if let Some(row_selection) = prepared_access_plan.row_selection {
- builder = builder.with_row_selection(row_selection);
- }
- builder = builder.with_row_groups(prepared_access_plan.row_group_indexes);
+ builder = builder.with_row_group_selections(row_group_selections);
if let Some(limit) = self.decoder_limit {
builder = builder.with_limit(limit);
}
@@ -301,7 +299,7 @@ pub(crate) struct PushDecoderStreamState {
/// statistics and drop RGs the current threshold proves cannot
/// contribute. The decoder is rebuilt via
/// [`ParquetPushDecoder::into_builder`] +
- /// [`ParquetPushDecoderBuilder::with_row_groups`] so the skipped RGs are
+ /// [`ParquetPushDecoderBuilder::with_row_group_selections`] so the skipped RGs are
/// bypassed entirely. `None` when the scan has no watching dynamic
/// predicate or only one row group remains.
pub(crate) row_group_pruner: Option<RowGroupPruner>,
@@ -565,6 +563,9 @@ impl PushDecoderStreamState {
/// removed every page — which would otherwise leave `rg_plan` trailing the
/// decoder by one: a later prune/rebuild would then re-include an
/// already-delivered row group (#24352) or toggle the filter for the wrong RG.
+ /// Row-group-local selections keep selections and match status aligned when
+ /// preparing or reordering the plan, but do not prevent this decoder-side
+ /// advancement, so frontier synchronization is still required.
fn sync_rg_plan_to_decoder_frontier(&mut self) -> Result<()> {
match self
.decoder
@@ -669,9 +670,18 @@ impl PushDecoderStreamState {
}
let decoder = self.decoder.take().expect("decoder present");
- let new_indices: Vec<usize> = self.rg_plan.iter().map(|e| e.rg_index).collect();
let mut builder = decoder.into_builder().map_err(DataFusionError::from)?;
- builder = builder.with_row_groups(new_indices);
+ // Filter-only rebuilds preserve the decoder's remaining selections.
+ // Runtime pruning is disabled for scans with selections, so pruned
+ // plans can be rebuilt from row-group indexes alone.
+ if pruned_count > 0 {
+ let selections = self
+ .rg_plan
+ .iter()
+ .map(|e| RowGroupSelection::new(e.rg_index, None))
+ .collect();
+ builder = builder.with_row_group_selections(selections);
+ }
if filter_needs_toggle {
let want_filter = desired_filter.expect("filter_needs_toggle ⇒ desired Some");
if want_filter {
diff --git a/datafusion/datasource-parquet/src/sort.rs b/datafusion/datasource-parquet/src/sort.rs
index e60c257652..4e683d7fcd 100644
--- a/datafusion/datasource-parquet/src/sort.rs
+++ b/datafusion/datasource-parquet/src/sort.rs
@@ -18,109 +18,10 @@
//! Sort-related utilities for Parquet scanning
use arrow::datatypes::Schema;
-use datafusion_common::{Result, ScalarValue};
+use datafusion_common::ScalarValue;
use datafusion_datasource::PartitionedFile;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
-use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-use parquet::file::metadata::ParquetMetaData;
-use std::collections::HashMap;
-
-/// Reverse a row selection to match reversed row group order.
-///
-/// When scanning row groups in reverse order, we need to adjust the row selection
-/// to account for the new ordering. This function:
-/// 1. Maps each selection to its corresponding row group
-/// 2. Reverses the order of row groups
-/// 3. Reconstructs the row selection for the new order
-///
-/// # Arguments
-/// * `row_selection` - Original row selection (only covers row groups that are scanned)
-/// * `parquet_metadata` - Metadata containing row group information
-/// * `row_groups_to_scan` - Indexes of row groups that will be scanned (in original order)
-///
-/// # Returns
-/// A new `RowSelection` adjusted for reversed row group order
-///
-/// # Important Notes
-/// The input `row_selection` only covers the row groups specified in `row_groups_to_scan`.
-/// Row groups that are skipped (not in `row_groups_to_scan`) are not represented in the
-/// `row_selection` at all. This function needs `row_groups_to_scan` to correctly map
-/// the selection back to the original row groups.
-pub fn reverse_row_selection(
- row_selection: &RowSelection,
- parquet_metadata: &ParquetMetaData,
- row_groups_to_scan: &[usize],
-) -> Result<RowSelection> {
- let rg_metadata = parquet_metadata.row_groups();
-
- // Build a mapping of row group index to its row range, but ONLY for
- // the row groups that are actually being scanned.
- //
- // IMPORTANT: The row numbers in this mapping are RELATIVE to the scanned row groups,
- // not absolute positions in the file.
- //
- // Example: If row_groups_to_scan = [0, 2, 3] and each has 100 rows:
- // RG0: rows 0-99 (relative to scanned data)
- // RG2: rows 100-199 (relative to scanned data, NOT 200-299 in file!)
- // RG3: rows 200-299 (relative to scanned data, NOT 300-399 in file!)
- let mut rg_row_ranges: Vec<(usize, usize, usize)> =
- Vec::with_capacity(row_groups_to_scan.len());
- let mut current_row = 0;
- for &rg_idx in row_groups_to_scan {
- let rg = &rg_metadata[rg_idx];
- let num_rows = rg.num_rows() as usize;
- rg_row_ranges.push((rg_idx, current_row, current_row + num_rows));
- current_row += num_rows; // This is relative row number, NOT absolute file position
- }
-
- // Map selections to row groups
- let mut rg_selections: HashMap<usize, Vec<RowSelector>> = HashMap::new();
-
- let mut current_file_row = 0;
- for selector in row_selection.iter() {
- let selector_end = current_file_row + selector.row_count;
-
- // Find which row groups this selector spans
- for (rg_idx, rg_start, rg_end) in rg_row_ranges.iter() {
- if current_file_row < *rg_end && selector_end > *rg_start {
- // This selector overlaps with this row group
- let overlap_start = current_file_row.max(*rg_start);
- let overlap_end = selector_end.min(*rg_end);
- let overlap_count = overlap_end - overlap_start;
-
- if overlap_count > 0 {
- let entry = rg_selections.entry(*rg_idx).or_default();
- if selector.skip {
- entry.push(RowSelector::skip(overlap_count));
- } else {
- entry.push(RowSelector::select(overlap_count));
- }
- }
- }
- }
-
- current_file_row = selector_end;
- }
-
- // Build new selection for reversed row group order
- // Only iterate over the row groups that are being scanned, in reverse order
- let mut reversed_selectors = Vec::new();
- for &rg_idx in row_groups_to_scan.iter().rev() {
- if let Some(selectors) = rg_selections.get(&rg_idx) {
- reversed_selectors.extend(selectors.iter().copied());
- } else {
- // No specific selection for this row group means select all rows in it
- if let Some((_, start, end)) =
- rg_row_ranges.iter().find(|(idx, _, _)| *idx == rg_idx)
- {
- reversed_selectors.push(RowSelector::select(end - start));
- }
- }
- }
-
- Ok(RowSelection::from(reversed_selectors))
-}
/// Reorder a file list so the most "promising" files are read first,
/// matching `PreparedAccessPlan::reorder_by_statistics` at the
@@ -283,862 +184,128 @@ mod tests {
}
#[test]
- fn test_prepared_access_plan_reverse_simple() {
- // Test: all row groups are scanned, no row selection
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let access_plan = ParquetAccessPlan::new_all(3);
- let rg_metadata = metadata.row_groups();
-
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- // Verify original plan
- assert_eq!(prepared_plan.row_group_indexes, vec![0, 1, 2]);
-
- // No row selection originally due to scanning all rows
- assert_eq!(prepared_plan.row_selection, None);
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // Verify row groups are reversed
- assert_eq!(reversed_plan.row_group_indexes, vec![2, 1, 0]);
-
- // If no selection originally, after reversal should still select all rows,
- // and the selection should be None
- assert_eq!(reversed_plan.row_selection, None);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_with_selection() {
- // Test: simple row selection that spans multiple row groups
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(3);
-
- // Select first 50 rows from first row group, skip rest
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),
- );
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(
- original_selected, reversed_selected,
- "Total selected rows should remain the same"
- );
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_multi_row_group_selection() {
- // Test: row selection spanning multiple row groups
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(3);
-
- // Create selection that spans RG0 and RG1
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::skip(50), RowSelector::select(50)]),
- );
- access_plan.scan_selection(
- 1,
- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),
- );
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(original_selected, reversed_selected);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_empty_selection() {
- // Test: all rows are skipped. After `strip_empty_row_groups`, every row
- // group's selection is empty, so the whole plan strips to nothing.
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(3);
- for i in 0..3 {
- access_plan
- .scan_selection(i, RowSelection::from(vec![RowSelector::skip(100)]));
+ fn test_prepared_access_plan_reverse() {
+ use RowGroupAccess::{Scan, Selection, Skip};
+ let selection = |selectors: Vec<RowSelector>| Selection(selectors.into());
+ // Exercise empty, single-group, non-contiguous, unequal-size, and
+ // fragmented selections. Reverse group order, never local row order.
+ let cases = vec![
+ (vec![100, 100, 100], vec![Scan, Scan, Scan], vec![2, 1, 0]),
+ (vec![100, 100, 100], vec![Skip, Skip, Skip], vec![]),
+ (
+ vec![100, 100, 100],
+ vec![selection(vec![RowSelector::skip(100)]); 3],
+ vec![],
+ ),
+ (
+ vec![100],
+ vec![selection(vec![
+ RowSelector::select(50),
+ RowSelector::skip(50),
+ ])],
+ vec![0],
+ ),
+ (
+ vec![100, 100, 100],
+ vec![
+ selection(vec![RowSelector::skip(50), RowSelector::select(50)]),
+ selection(vec![RowSelector::select(50), RowSelector::skip(50)]),
+ Scan,
+ ],
+ vec![2, 1, 0],
+ ),
+ (
+ vec![50, 150, 100],
+ vec![
+ selection(vec![RowSelector::skip(25), RowSelector::select(25)]),
+ Scan,
+ selection(vec![RowSelector::select(50), RowSelector::skip(50)]),
+ ],
+ vec![2, 1, 0],
+ ),
+ (
+ vec![100, 100, 100, 100],
+ vec![
+ Scan,
+ Skip,
+ selection(vec![RowSelector::select(25), RowSelector::skip(75)]),
+ Scan,
+ ],
+ vec![3, 2, 0],
+ ),
+ (
+ vec![100, 100, 100, 100],
+ vec![
+ selection(vec![RowSelector::select(30), RowSelector::skip(70)]),
+ Skip,
+ selection(vec![RowSelector::skip(20), RowSelector::select(80)]),
+ Skip,
+ ],
+ vec![2, 0],
+ ),
+ (vec![100, 100, 100], vec![Skip, Scan, Skip], vec![1]),
+ (
+ vec![100, 100, 100],
+ vec![
+ selection(vec![
+ RowSelector::select(30),
+ RowSelector::skip(40),
+ RowSelector::select(30),
+ ]),
+ selection(vec![RowSelector::skip(50), RowSelector::select(50)]),
+ Scan,
+ ],
+ vec![2, 1, 0],
+ ),
+ ];
+ for (sizes, accesses, expected_indexes) in cases {
+ let metadata = create_test_metadata(sizes);
+ let mut plan = ParquetAccessPlan::new(accesses.clone());
+ for index in (0..accesses.len()).step_by(2) {
+ plan.mark_fully_matched(index);
+ }
+ let reversed = plan.prepare(metadata.row_groups()).unwrap().reverse();
+ assert_eq!(reversed.row_group_indexes(), expected_indexes);
+ for rg in reversed.row_groups {
+ let index = rg.selection.row_group_index();
+ let expected = match &accesses[index] {
+ Scan => None,
+ Selection(selection) => Some(selection),
+ Skip => panic!("skipped group must not be read"),
+ };
+ assert_eq!(rg.selection.selection(), expected);
+ assert_eq!(rg.fully_matched, index % 2 == 0);
+ }
}
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
- assert!(
- prepared_plan.row_group_indexes.is_empty(),
- "all-empty selections must be stripped to an empty plan",
- );
- assert!(prepared_plan.row_selection.is_none());
-
- // All row groups are empty after pruning, so they are stripped and the
- // prepared plan is empty (rather than carrying a selection that skips
- // every row).
- assert!(prepared_plan.row_group_indexes.is_empty());
- assert!(prepared_plan.row_selection.is_none());
-
- // Reversing an empty plan stays empty.
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
- assert!(reversed_plan.row_group_indexes.is_empty());
- assert!(reversed_plan.row_selection.is_none());
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_different_row_group_sizes() {
- // Test: row groups with different sizes
- let metadata = create_test_metadata(vec![50, 150, 100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(3);
-
- // Create complex selection pattern
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::skip(25), RowSelector::select(25)]),
- );
- access_plan.scan_selection(1, RowSelection::from(vec![RowSelector::select(150)]));
- access_plan.scan_selection(
- 2,
- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),
- );
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(original_selected, reversed_selected);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_single_row_group() {
- // Test: single row group case
- let metadata = create_test_metadata(vec![100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(1);
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]),
- );
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // With single row group, row_group_indexes should remain [0]
- assert_eq!(reversed_plan.row_group_indexes, vec![0]);
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(original_selected, reversed_selected);
- assert_eq!(original_selected, 50);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_complex_pattern() {
- // Test: complex pattern with multiple select/skip segments
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(3);
-
- // Complex pattern: select some, skip some, select some more
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![
- RowSelector::select(30),
- RowSelector::skip(40),
- RowSelector::select(30),
- ]),
- );
- access_plan.scan_selection(
- 1,
- RowSelection::from(vec![RowSelector::skip(50), RowSelector::select(50)]),
- );
- access_plan.scan_selection(2, RowSelection::from(vec![RowSelector::select(100)]));
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(original_selected, reversed_selected);
- assert_eq!(original_selected, 210); // 30 + 30 + 50 + 100
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_with_skipped_row_groups() {
- // This is the KEY test case for the bug fix!
- // Test scenario where some row groups are completely skipped (not in scan plan)
- let metadata = create_test_metadata(vec![100, 100, 100, 100]);
-
- // Scenario: RG0 (scan all), RG1 (completely skipped), RG2 (partial), RG3 (scan all)
- // Only row groups [0, 2, 3] are in the scan plan
- let mut access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Scan, // RG0
- RowGroupAccess::Skip, // RG1 - NOT in scan plan!
- RowGroupAccess::Scan, // RG2
- RowGroupAccess::Scan, // RG3
- ]);
-
- // Add row selections for the scanned row groups
- // Note: The RowSelection only covers row groups [0, 2, 3] (300 rows total)
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::select(100)]), // RG0: all 100 rows
- );
- // RG1 is skipped, no selection needed
- access_plan.scan_selection(
- 2,
- RowSelection::from(vec![
- RowSelector::select(25), // RG2: first 25 rows
- RowSelector::skip(75), // RG2: skip last 75 rows
- ]),
- );
- access_plan.scan_selection(
- 3,
- RowSelection::from(vec![RowSelector::select(100)]), // RG3: all 100 rows
- );
-
- let rg_metadata = metadata.row_groups();
-
- // Step 1: Create PreparedAccessPlan
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- // Verify original plan
- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2, 3]);
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
- assert_eq!(original_selected, 225); // 100 + 25 + 100
-
- // Step 2: Reverse the plan (this is the production code path)
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // Verify reversed results
- // Row group order should be reversed: [3, 2, 0]
- assert_eq!(
- reversed_plan.row_group_indexes,
- vec![3, 2, 0],
- "Row groups should be reversed"
- );
-
- // Verify row selection is also correctly reversed
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(
- reversed_selected, 225,
- "Total selected rows should remain the same"
- );
-
- // Verify the reversed selection structure
- // After reversal, the order becomes: RG3, RG2, RG0
- // - RG3: select(100)
- // - RG2: select(25), skip(75) (note: internal order preserved, not reversed)
- // - RG0: select(100)
- //
- // After RowSelection::from() merges adjacent selectors of the same type:
- // - RG3's select(100) + RG2's select(25) = select(125)
- // - RG2's skip(75) remains as skip(75)
- // - RG0's select(100) remains as select(100)
- let selectors: Vec<_> = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
- assert_eq!(selectors.len(), 3);
-
- // RG3 (100) + RG2 first part (25) merged into select(125)
- assert!(!selectors[0].skip);
- assert_eq!(selectors[0].row_count, 125);
-
- // RG2: skip last 75 rows
- assert!(selectors[1].skip);
- assert_eq!(selectors[1].row_count, 75);
-
- // RG0: select all 100 rows
- assert!(!selectors[2].skip);
- assert_eq!(selectors[2].row_count, 100);
}
#[test]
- fn test_prepared_access_plan_reverse_alternating_row_groups() {
- // Test with alternating scan/skip pattern
- let metadata = create_test_metadata(vec![100, 100, 100, 100]);
-
- // Scan RG0 and RG2, skip RG1 and RG3
- let mut access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Scan, // RG0
- RowGroupAccess::Skip, // RG1
- RowGroupAccess::Scan, // RG2
- RowGroupAccess::Skip, // RG3
+ fn test_prepared_access_plan_preserves_bitmap_selection() {
+ let metadata = create_test_metadata(vec![4, 4]);
+ let mask = arrow::buffer::BooleanBuffer::from(vec![true, false, true, false]);
+ let plan = ParquetAccessPlan::new(vec![
+ RowGroupAccess::Selection(RowSelection::from(mask.clone())),
+ RowGroupAccess::Selection(RowSelection::from(vec![
+ RowSelector::skip(1),
+ RowSelector::select(3),
+ ])),
]);
-
- access_plan.scan_selection(0, RowSelection::from(vec![RowSelector::select(100)]));
- access_plan.scan_selection(2, RowSelection::from(vec![RowSelector::select(100)]));
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- // Original: [0, 2]
- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2]);
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // After reverse: [2, 0]
- assert_eq!(reversed_plan.row_group_indexes, vec![2, 0]);
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(original_selected, reversed_selected);
- assert_eq!(original_selected, 200);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_middle_row_group_only() {
- // Test selecting only the middle row group
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let mut access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Skip, // RG0
- RowGroupAccess::Scan, // RG1
- RowGroupAccess::Skip, // RG2
- ]);
-
- access_plan.scan_selection(
- 1,
- RowSelection::from(vec![RowSelector::select(100)]), // Select all of RG1
- );
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- let original_selected: usize = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- // Original: [1]
- assert_eq!(prepared_plan.row_group_indexes, vec![1]);
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // After reverse: still [1] (only one row group)
- assert_eq!(reversed_plan.row_group_indexes, vec![1]);
-
- let reversed_selected: usize = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(original_selected, reversed_selected);
- assert_eq!(original_selected, 100);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_with_skipped_row_groups_detailed() {
- // This is the KEY test case for the bug fix!
- // Test scenario where some row groups are completely skipped (not in scan plan)
- // This version includes DETAILED verification of the selector distribution
- let metadata = create_test_metadata(vec![100, 100, 100, 100]);
-
- // Scenario: RG0 (scan all), RG1 (completely skipped), RG2 (partial), RG3 (scan all)
- // Only row groups [0, 2, 3] are in the scan plan
- let mut access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Scan, // RG0
- RowGroupAccess::Skip, // RG1 - NOT in scan plan!
- RowGroupAccess::Scan, // RG2
- RowGroupAccess::Scan, // RG3
- ]);
-
- // Add row selections for the scanned row groups
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::select(100)]), // RG0: all 100 rows
- );
- // RG1 is skipped, no selection needed
- access_plan.scan_selection(
- 2,
- RowSelection::from(vec![
- RowSelector::select(25), // RG2: first 25 rows
- RowSelector::skip(75), // RG2: skip last 75 rows
- ]),
- );
- access_plan.scan_selection(
- 3,
- RowSelection::from(vec![RowSelector::select(100)]), // RG3: all 100 rows
- );
-
- let rg_metadata = metadata.row_groups();
-
- // Step 1: Create PreparedAccessPlan
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- // Verify original plan in detail
- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2, 3]);
-
- // Detailed verification of original selection
- let orig_selectors: Vec<_> = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
-
- // Original structure should be:
- // RG0: select(100)
- // RG2: select(25), skip(75)
- // RG3: select(100)
- // After merging by RowSelection::from(): select(125), skip(75), select(100)
+ let reversed = plan.prepare(metadata.row_groups()).unwrap().reverse();
+ assert_eq!(reversed.row_group_indexes(), vec![1, 0]);
assert_eq!(
- orig_selectors.len(),
- 3,
- "Original should have 3 selectors after merging"
- );
- assert!(
- !orig_selectors[0].skip && orig_selectors[0].row_count == 125,
- "Original: First selector should be select(125) from RG0(100) + RG2(25)"
- );
- assert!(
- orig_selectors[1].skip && orig_selectors[1].row_count == 75,
- "Original: Second selector should be skip(75) from RG2"
- );
- assert!(
- !orig_selectors[2].skip && orig_selectors[2].row_count == 100,
- "Original: Third selector should be select(100) from RG3"
+ reversed.row_groups[1]
+ .selection
+ .selection()
+ .unwrap()
+ .as_mask(),
+ Some(&mask)
);
-
- let original_selected: usize = orig_selectors
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
- assert_eq!(original_selected, 225); // 100 + 25 + 100
-
- // Step 2: Reverse the plan (this is the production code path)
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // Verify reversed results
- // Row group order should be reversed: [3, 2, 0]
assert_eq!(
- reversed_plan.row_group_indexes,
- vec![3, 2, 0],
- "Row groups should be reversed"
- );
-
- // Detailed verification of reversed selection
- let rev_selectors: Vec<_> = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
-
- // After reversal, the order becomes: RG3, RG2, RG0
- // - RG3: select(100)
- // - RG2: select(25), skip(75) (note: internal order preserved, not reversed)
- // - RG0: select(100)
- //
- // After RowSelection::from() merges adjacent selectors of the same type:
- // - RG3's select(100) + RG2's select(25) = select(125)
- // - RG2's skip(75) remains as skip(75)
- // - RG0's select(100) remains as select(100)
-
- assert_eq!(
- rev_selectors.len(),
- 3,
- "Reversed should have 3 selectors after merging"
- );
-
- // First selector: RG3 (100) + RG2 first part (25) merged into select(125)
- assert!(
- !rev_selectors[0].skip && rev_selectors[0].row_count == 125,
- "Reversed: First selector should be select(125) from RG3(100) + RG2(25), got skip={} count={}",
- rev_selectors[0].skip,
- rev_selectors[0].row_count
- );
-
- // Second selector: RG2 skip last 75 rows
- assert!(
- rev_selectors[1].skip && rev_selectors[1].row_count == 75,
- "Reversed: Second selector should be skip(75) from RG2, got skip={} count={}",
- rev_selectors[1].skip,
- rev_selectors[1].row_count
- );
-
- // Third selector: RG0 select all 100 rows
- assert!(
- !rev_selectors[2].skip && rev_selectors[2].row_count == 100,
- "Reversed: Third selector should be select(100) from RG0, got skip={} count={}",
- rev_selectors[2].skip,
- rev_selectors[2].row_count
- );
-
- // Verify row selection is also correctly reversed (total count)
- let reversed_selected: usize = rev_selectors
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(
- reversed_selected, 225,
- "Total selected rows should remain the same"
- );
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_complex_pattern_detailed() {
- // Test: complex pattern with detailed verification
- let metadata = create_test_metadata(vec![100, 100, 100]);
-
- let mut access_plan = ParquetAccessPlan::new_all(3);
-
- // Complex pattern: select some, skip some, select some more
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![
- RowSelector::select(30),
- RowSelector::skip(40),
- RowSelector::select(30),
- ]),
- );
- access_plan.scan_selection(
- 1,
- RowSelection::from(vec![RowSelector::skip(50), RowSelector::select(50)]),
- );
- access_plan.scan_selection(2, RowSelection::from(vec![RowSelector::select(100)]));
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- // Verify original selection structure in detail
- let orig_selectors: Vec<_> = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
-
- // RG0: select(30), skip(40), select(30)
- // RG1: skip(50), select(50)
- // RG2: select(100)
- // Sequential: sel(30), skip(40), sel(30), skip(50), sel(50), sel(100)
- // After merge: sel(30), skip(40), sel(30), skip(50), sel(150)
-
- let original_selected: usize = orig_selectors
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
- assert_eq!(original_selected, 210); // 30 + 30 + 50 + 100
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // Verify reversed selection structure
- let rev_selectors: Vec<_> = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
-
- // After reversal: RG2, RG1, RG0
- // RG2: select(100)
- // RG1: skip(50), select(50)
- // RG0: select(30), skip(40), select(30)
- // Sequential: sel(100), skip(50), sel(50), sel(30), skip(40), sel(30)
- // After merge: sel(100), skip(50), sel(80), skip(40), sel(30)
-
- let reversed_selected: usize = rev_selectors
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(
- reversed_selected, 210,
- "Total selected rows should remain the same (30 + 30 + 50 + 100)"
- );
-
- // Verify row group order
- assert_eq!(reversed_plan.row_group_indexes, vec![2, 1, 0]);
- }
-
- #[test]
- fn test_prepared_access_plan_reverse_alternating_detailed() {
- // Test with alternating scan/skip pattern with detailed verification
- let metadata = create_test_metadata(vec![100, 100, 100, 100]);
-
- // Scan RG0 and RG2, skip RG1 and RG3
- let mut access_plan = ParquetAccessPlan::new(vec![
- RowGroupAccess::Scan, // RG0
- RowGroupAccess::Skip, // RG1
- RowGroupAccess::Scan, // RG2
- RowGroupAccess::Skip, // RG3
- ]);
-
- access_plan.scan_selection(
- 0,
- RowSelection::from(vec![RowSelector::select(30), RowSelector::skip(70)]),
- );
- access_plan.scan_selection(
- 2,
- RowSelection::from(vec![RowSelector::skip(20), RowSelector::select(80)]),
- );
-
- let rg_metadata = metadata.row_groups();
- let prepared_plan = access_plan
- .prepare(rg_metadata)
- .expect("Failed to create PreparedAccessPlan");
-
- // Original: [0, 2]
- assert_eq!(prepared_plan.row_group_indexes, vec![0, 2]);
-
- // Verify original selection
- let orig_selectors: Vec<_> = prepared_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
-
- // Original:
- // RG0: select(30), skip(70)
- // RG2: skip(20), select(80)
- // Sequential: sel(30), skip(90), sel(80)
- // (RG0's skip(70) + RG2's skip(20) = skip(90))
-
- let original_selected: usize = orig_selectors
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
- assert_eq!(original_selected, 110); // 30 + 80
-
- let reversed_plan = prepared_plan
- .reverse(&metadata)
- .expect("Failed to reverse PreparedAccessPlan");
-
- // After reverse: [2, 0]
- assert_eq!(reversed_plan.row_group_indexes, vec![2, 0]);
-
- // Verify reversed selection
- let rev_selectors: Vec<_> = reversed_plan
- .row_selection
- .as_ref()
- .unwrap()
- .iter()
- .collect();
-
- // After reversal: RG2, RG0
- // RG2: skip(20), select(80)
- // RG0: select(30), skip(70)
- // Sequential: skip(20), sel(110), skip(70)
- // (RG2's select(80) + RG0's select(30) = select(110))
-
- let reversed_selected: usize = rev_selectors
- .iter()
- .filter(|s| !s.skip)
- .map(|s| s.row_count)
- .sum();
-
- assert_eq!(reversed_selected, 110); // Should still be 30 + 80
-
- // Detailed verification of structure
- assert_eq!(rev_selectors.len(), 3, "Reversed should have 3 selectors");
-
- assert!(
- rev_selectors[0].skip && rev_selectors[0].row_count == 20,
- "First selector should be skip(20) from RG2"
- );
-
- assert!(
- !rev_selectors[1].skip && rev_selectors[1].row_count == 110,
- "Second selector should be select(110) from RG2(80) + RG0(30)"
- );
-
- assert!(
- rev_selectors[2].skip && rev_selectors[2].row_count == 70,
- "Third selector should be skip(70) from RG0"
+ reversed.row_groups[0].selection.selection(),
+ Some(&RowSelection::from(vec![
+ RowSelector::skip(1),
+ RowSelector::select(3)
+ ]))
);
}
}
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 3df0e0e7e1..32fc8c9f3d 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -774,7 +774,7 @@ impl FileSource for ParquetSource {
// the parquet opener will pause the single decoder at row
// group boundaries and consult `RowGroupPruner` to drop
// RGs the current threshold proves unwinnable, rebuilding
- // the decoder via `into_builder().with_row_groups(...)` to
+ // the decoder via `into_builder().with_row_group_selections(...)` to
// skip them. The actual pruning count appears as
// `row_groups_pruned_dynamic_filter` in EXPLAIN ANALYZE.
// We use `contains_dynamic_filter()` (matches both `Watching`
diff --git a/datafusion/datasource-parquet/src/statistics_order_tests.rs b/datafusion/datasource-parquet/src/statistics_order_tests.rs
index 8b8e2c9831..93e5f63ad2 100644
--- a/datafusion/datasource-parquet/src/statistics_order_tests.rs
+++ b/datafusion/datasource-parquet/src/statistics_order_tests.rs
@@ -273,18 +273,20 @@ impl TestFile {
physical: &Arc<dyn PhysicalExpr>,
plan: ParquetAccessPlan,
) -> usize {
- let mut builder = ParquetRecordBatchReaderBuilder::try_new(self.bytes.clone())
- .unwrap()
- .with_row_groups(plan.row_group_indexes());
- if let Some(selection) = plan
- .into_overall_row_selection(self.metadata.row_groups())
- .unwrap()
- {
- builder = builder.with_row_selection(selection);
- }
- builder
- .build()
+ plan.prepare(self.metadata.row_groups())
.unwrap()
+ .row_groups
+ .into_iter()
+ .flat_map(|rg| {
+ let mut builder =
+ ParquetRecordBatchReaderBuilder::try_new(self.bytes.clone())
+ .unwrap()
+ .with_row_groups(vec![rg.selection.row_group_index()]);
+ if let Some(selection) = rg.selection.selection() {
+ builder = builder.with_row_selection(selection.clone());
+ }
+ builder.build().unwrap()
+ })
.map(|batch| {
let batch = batch.unwrap();
let matches = physical
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index e11024449a..fc5e9d2f03 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -65,6 +65,26 @@ change. Use `FieldMetadata::to_hashmap` where a `HashMap` is still required.
[`metadata`]: https://docs.rs/arrow/60.0.0/arrow/datatypes/struct.Metadata.html
+### `ParquetAccessPlan::into_overall_row_selection` is deprecated
+
+`ParquetAccessPlan::into_overall_row_selection` is deprecated starting in 56.0.0.
+DataFusion now keeps selections local to each row group during Parquet scanning.
+The method remains available with its existing conversion and selection-length
+validation behavior, so direct callers continue to work but receive a deprecation
+warning.
+
+Users who attach a `ParquetAccessPlan` to a `PartitionedFile` for DataFusion to
+read need no changes.
+
+For custom readers, use `ParquetAccessPlan::into_inner()` to obtain the
+`RowGroupAccess` entries. Their positions are the original file row-group
+indexes, and each selection uses coordinates within its own row group. When
+migrating away from the deprecated method, callers must handle conversion and
+selection-length validation themselves. Parquet's push decoder and async reader
+accept `RowGroupSelection` values through `with_row_group_selections`; the
+synchronous reader still requires `with_row_groups` and an overall
+`with_row_selection`.
+
### Missing Parquet null counts are treated as unknown
DataFusion now preserves an omitted Parquet `null_count` statistic as unknown.