Apache DataFusion SQL Query Engine
feat: support file-level parquet row selections (#22940)
Huaijin38269f9c0cf1
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex 31be6fd979..8fd9689ae3 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -29,8 +29,10 @@ use datafusion::common::Result;use datafusion::datasource::listing::PartitionedFile;use datafusion::datasource::physical_plan::ParquetSource;use datafusion::prelude::SessionContext;-use datafusion_common::{DFSchema, assert_contains};-use datafusion_datasource_parquet::{ParquetAccessPlan, RowGroupAccess};+use datafusion_common::{DFSchema, assert_batches_eq, assert_contains};+use datafusion_datasource_parquet::{+ ParquetAccessPlan, ParquetRowSelection, RowGroupAccess,+};use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_expr::{Expr, col, lit};use datafusion_physical_plan::ExecutionPlan;@@ -152,6 +154,94 @@ async fn skip_scan() {}}+#[tokio::test]+async fn row_selection_extension() {+ // The file has 2 row groups of 5 rows each (10 rows total). Attach a+ // file-level `ParquetRowSelection` to the `PartitionedFile` and verify it+ // survives the path from `PartitionedFile` into the parquet opener/reader.++ // select a single row in the first row group+ let parquet_metrics = TestFull {+ access_plan: None,+ row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![+ RowSelector::skip(2),+ RowSelector::select(1),+ RowSelector::skip(7),+ ]))),+ expected_rows: 1,+ expected_output: Some(&[+ "+------+------------+",+ "| utf8 | large_utf8 |",+ "+------+------------+",+ "| c | c |",+ "+------+------------+",+ ]),+ predicate: None,+ }+ .run()+ .await+ .unwrap();++ // only the first row group is read, so some bytes are scanned+ let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();+ assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}",);+}++#[tokio::test]+async fn row_selection_extension_spanning_row_groups() {+ // A selection whose selectors straddle the row group boundary (row 4 is the+ // last row of group 0, rows 5-6 are the first rows of group 1).+ let parquet_metrics = TestFull {+ access_plan: None,+ row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![+ RowSelector::skip(4),+ RowSelector::select(3),+ RowSelector::skip(3),+ ]))),+ expected_rows: 3,+ expected_output: Some(&[+ "+------+------------+",+ "| utf8 | large_utf8 |",+ "+------+------------+",+ "| | |",+ "| e | e |",+ "| f | f |",+ "+------+------------+",+ ]),+ predicate: None,+ }+ .run()+ .await+ .unwrap();++ let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();+ assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}",);+}++#[tokio::test]+async fn bad_row_selection_extension() {+ // selection specifies fewer rows than the file actually contains+ let err = TestFull {+ access_plan: None,+ row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![+ RowSelector::skip(2),+ RowSelector::select(1),+ ]))),+ expected_rows: 10000,+ expected_output: None,+ predicate: None,+ }+ .run()+ .await+ .unwrap_err();+ let err_string = err.to_string();+ assert_contains!(&err_string, "Invalid Parquet RowSelection");+ assert_contains!(+ &err_string,+ "File has 10 rows, but selection specifies 3 rows."+ );+}+#[tokio::test]async fn plan_and_filter() {// show that row group pruning is applied even when an initial plan is supplied@@ -170,7 +260,9 @@ async fn plan_and_filter() {// initiallet parquet_metrics = TestFull {access_plan,+ row_selection: None,expected_rows: 0,+ expected_output: None,predicate: Some(predicate),}.run()@@ -227,7 +319,9 @@ async fn bad_row_groups() {RowGroupAccess::Skip,RowGroupAccess::Scan,])),+ row_selection: None,expected_rows: 0,+ expected_output: None,predicate: None,}.run()@@ -249,8 +343,10 @@ async fn bad_selection() {])),RowGroupAccess::Skip,])),+ row_selection: None,// expects that we hit an error, this should not be runexpected_rows: 10000,+ expected_output: None,predicate: None,}.run()@@ -300,7 +396,9 @@ impl Test {} = self;TestFull {access_plan,+ row_selection: None,expected_rows,+ expected_output: None,predicate: None,}.run()@@ -317,7 +415,9 @@ impl Test {/// 4. Returns the statistics from running the planstruct TestFull {access_plan: Option<ParquetAccessPlan>,+ row_selection: Option<ParquetRowSelection>,expected_rows: usize,+ expected_output: Option<&'static [&'static str]>,predicate: Option<Expr>,}@@ -327,7 +427,9 @@ impl TestFull {let Self {access_plan,+ row_selection,expected_rows,+ expected_output,predicate,} = self;@@ -352,6 +454,11 @@ impl TestFull {partitioned_file = partitioned_file.with_extension(access_plan);}+ // add the file-level row selection, if any, as an extension+ if let Some(row_selection) = row_selection {+ partitioned_file = partitioned_file.with_extension(row_selection);+ }+// Create a DataSourceExec to read the filelet object_store_url = ObjectStoreUrl::local_filesystem();// add the predicate, if requested@@ -380,6 +487,9 @@ impl TestFull {"results: \n{}",pretty_format_batches(&results).unwrap());+ if let Some(expected_output) = expected_output {+ assert_batches_eq!(expected_output, &results);+ }std::fs::remove_file(file_name).unwrap();diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rsindex edbea39948..593ed365cd 100644--- a/datafusion/datasource-parquet/src/access_plan.rs+++ b/datafusion/datasource-parquet/src/access_plan.rs@@ -17,7 +17,7 @@use crate::sort::reverse_row_selection;use arrow::datatypes::Schema;-use datafusion_common::{Result, assert_eq_or_internal_err};+use datafusion_common::{Result, assert_eq_or_internal_err, exec_err};use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr_common::sort_expr::LexOrdering;use log::debug;@@ -104,6 +104,41 @@ pub struct ParquetAccessPlan {fully_matched: Vec<bool>,}+/// A file-level row selection for a parquet scan.+///+/// Attach this type to a [`PartitionedFile`](datafusion_datasource::PartitionedFile)+/// with [`PartitionedFile::with_extension`](datafusion_datasource::PartitionedFile::with_extension)+/// when an external index produces a [`RowSelection`] across the entire parquet+/// file. DataFusion will use parquet metadata to split it into row-group-level+/// access when the file is opened.+#[derive(Debug, Clone, PartialEq)]+pub struct ParquetRowSelection {+ selection: RowSelection,+}++impl ParquetRowSelection {+ /// Create a new file-level parquet row selection.+ pub fn new(selection: RowSelection) -> Self {+ Self { selection }+ }++ /// Return a reference to the underlying [`RowSelection`].+ pub fn selection(&self) -> &RowSelection {+ &self.selection+ }++ /// Convert into the underlying [`RowSelection`].+ pub fn into_inner(self) -> RowSelection {+ self.selection+ }+}++impl From<RowSelection> for ParquetRowSelection {+ fn from(selection: RowSelection) -> Self {+ Self::new(selection)+ }+}+/// Describes how the parquet reader will access a row group#[derive(Debug, Clone, PartialEq)]pub enum RowGroupAccess {@@ -143,6 +178,102 @@ impl RowGroupAccess {}}+/// Single-pass cursor over a file-level [`RowSelection`].+///+/// `take` returns the next selector fragment capped to the requested row count,+/// splitting the current selector when it straddles a row group boundary.+struct OverallRowSelectionCursor {+ selector_iter: std::vec::IntoIter<RowSelector>,+ current: Option<RowSelector>,+}++impl OverallRowSelectionCursor {+ fn new(selection: RowSelection) -> Self {+ let selectors: Vec<RowSelector> = selection.into();+ let mut selector_iter = selectors.into_iter();+ let current = selector_iter.next();+ Self {+ selector_iter,+ current,+ }+ }++ /// Take up to `max_rows` rows from the current selector.+ ///+ /// If the current selector crosses the requested boundary, this returns the+ /// leading fragment and keeps the remaining rows in `self.current` for the+ /// next call.+ #[inline]+ fn take(&mut self, max_rows: usize) -> Option<RowSelector> {+ let sel = self.current?;+ let row_count = sel.row_count.min(max_rows);+ self.current = if row_count < sel.row_count {+ Some(RowSelector {+ row_count: sel.row_count - row_count,+ skip: sel.skip,+ })+ } else {+ self.selector_iter.next()+ };++ Some(RowSelector {+ row_count,+ skip: sel.skip,+ })+ }++ fn remaining_rows(self) -> usize {+ self.current.map_or(0, |s| s.row_count)+ + self.selector_iter.map(|s| s.row_count).sum::<usize>()+ }+}++/// Accumulates the selector fragments that belong to one row group.+struct RowGroupAccessBuilder {+ /// Selector fragments belonging to this row group.+ selectors: Vec<RowSelector>,+ /// Number of selected rows accumulated for this row group.+ selected: usize,+ /// Number of skipped rows accumulated for this row group.+ skipped: usize,+ /// Number of rows still needed to complete this row group.+ remaining: usize,+}++impl RowGroupAccessBuilder {+ fn new(row_group_rows: usize) -> Self {+ Self {+ selectors: Vec::with_capacity(1),+ selected: 0,+ skipped: 0,+ remaining: row_group_rows,+ }+ }++ #[inline]+ fn push(&mut self, selector: RowSelector) {+ self.remaining -= selector.row_count;++ if selector.skip {+ self.skipped += selector.row_count;+ } else {+ self.selected += selector.row_count;+ }++ self.selectors.push(selector);+ }++ fn into_access(self) -> RowGroupAccess {+ if self.selected == 0 {+ RowGroupAccess::Skip+ } else if self.skipped == 0 {+ RowGroupAccess::Scan+ } else {+ RowGroupAccess::Selection(self.selectors.into())+ }+ }+}+impl ParquetAccessPlan {/// Create a new `ParquetAccessPlan` that scans all row groupspub fn new_all(row_group_count: usize) -> Self {@@ -169,6 +300,60 @@ impl ParquetAccessPlan {}}+ /// Create a new `ParquetAccessPlan` from a file-level [`RowSelection`].+ ///+ /// The selection is interpreted across all rows in the file, in row group+ /// order, and is split into row-group level access using `row_group_meta_data`.+ /// Fully skipped row groups become [`RowGroupAccess::Skip`], fully selected+ /// row groups become [`RowGroupAccess::Scan`], and partially selected row+ /// groups become [`RowGroupAccess::Selection`].+ ///+ /// # Errors+ ///+ /// Returns an error if the selection does not specify exactly the same+ /// number of rows as the file metadata.+ pub fn try_new_from_overall_row_selection(+ selection: RowSelection,+ row_group_meta_data: &[RowGroupMetaData],+ ) -> Result<Self> {+ // Keep this as a single pass over the selector stream rather than+ // repeatedly calling `RowSelection::split_off` per row group. The+ // `split_off` version is simpler, but it clones/retains substantially+ // more selector buffer capacity for highly fragmented selections.+ let mut cursor = OverallRowSelectionCursor::new(selection);++ let mut selection_rows = 0usize;+ let mut file_rows = 0usize;++ let mut row_groups = Vec::with_capacity(row_group_meta_data.len());+ for rg_meta in row_group_meta_data {+ let rg_rows = rg_meta.num_rows() as usize;+ file_rows += rg_rows;++ let mut builder = RowGroupAccessBuilder::new(rg_rows);+ while builder.remaining > 0 {+ let Some(selector) = cursor.take(builder.remaining) else {+ break;+ };+ selection_rows += selector.row_count;+ builder.push(selector);+ }++ row_groups.push(builder.into_access());+ }++ selection_rows += cursor.remaining_rows();++ if selection_rows != file_rows {+ return exec_err!(+ "Invalid Parquet RowSelection. File has {file_rows} rows, \+ but selection specifies {selection_rows} rows."+ );+ }++ Ok(Self::new(row_groups))+ }+/// Set the i-th row group to the specified [`RowGroupAccess`]pub fn set(&mut self, idx: usize, access: RowGroupAccess) {let should_scan = access.should_scan();@@ -758,6 +943,90 @@ mod test {);}+ #[test]+ fn test_new_from_overall_row_selection() {+ let row_selection = RowSelection::from(vec![+ RowSelector::select(10),+ RowSelector::skip(25),+ RowSelector::select(10),+ RowSelector::skip(15),+ RowSelector::select(40),+ ]);++ let access_plan = ParquetAccessPlan::try_new_from_overall_row_selection(+ row_selection,+ &ROW_GROUP_METADATA,+ )+ .unwrap();++ assert_eq!(+ access_plan,+ ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Skip,+ RowGroupAccess::Selection(+ vec![+ RowSelector::skip(5),+ RowSelector::select(10),+ RowSelector::skip(15),+ ]+ .into()+ ),+ RowGroupAccess::Scan,+ ])+ );+ }++ #[test]+ fn test_new_from_overall_row_selection_invalid_row_count() {+ let row_selection = RowSelection::from(vec![RowSelector::select(99)]);++ let err = ParquetAccessPlan::try_new_from_overall_row_selection(+ row_selection,+ &ROW_GROUP_METADATA,+ )+ .unwrap_err()+ .to_string();++ assert_contains!(+ err,+ "Invalid Parquet RowSelection. File has 100 rows, but selection specifies 99 rows"+ );+ }++ #[test]+ fn test_new_from_overall_row_selection_boundary_splits() {+ let row_selection = RowSelection::from(vec![+ RowSelector::skip(5),+ RowSelector::select(10),+ RowSelector::skip(20),+ RowSelector::select(25),+ RowSelector::skip(40),+ ]);++ let access_plan = ParquetAccessPlan::try_new_from_overall_row_selection(+ row_selection,+ &ROW_GROUP_METADATA,+ )+ .unwrap();++ assert_eq!(+ access_plan,+ ParquetAccessPlan::new(vec![+ RowGroupAccess::Selection(+ vec![RowSelector::skip(5), RowSelector::select(5)].into()+ ),+ RowGroupAccess::Selection(+ vec![RowSelector::select(5), RowSelector::skip(15)].into()+ ),+ RowGroupAccess::Selection(+ vec![RowSelector::skip(5), RowSelector::select(25)].into()+ ),+ RowGroupAccess::Skip,+ ])+ );+ }+#[test]fn test_invalid_too_few() {let access_plan = ParquetAccessPlan::new(vec![diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex bec0736366..260d6ee471 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -46,7 +46,7 @@ mod test_util;mod virtual_column;mod writer;-pub use access_plan::{ParquetAccessPlan, RowGroupAccess};+pub use access_plan::{ParquetAccessPlan, ParquetRowSelection, RowGroupAccess};pub use file_format::*;pub use metrics::ParquetFileMetrics;pub use page_filter::PagePruningAccessPlanFilter;diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex 5b517663f9..e9f50134d4 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -31,7 +31,7 @@ use crate::row_filter::RowFilterGenerator;use crate::row_group_filter::{BloomFilterStatistics, RowGroupAccessPlanFilter};use crate::{Int96Coercer, ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,- ParquetVirtualColumn, apply_file_schema_type_coercions,+ ParquetRowSelection, ParquetVirtualColumn, apply_file_schema_type_coercions,};use arrow::array::RecordBatch;use arrow::datatypes::DataType;@@ -76,7 +76,7 @@ use parquet::arrow::async_reader::AsyncFileReader;use parquet::arrow::parquet_column;use parquet::basic::Type;use parquet::bloom_filter::Sbbf;-use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};+use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMetaData};/// Morselizer-level state for virtual columns, precomputed once per scan/// partition so each file skips the validator walks, `null_replacements`@@ -1080,7 +1080,7 @@ impl FiltersPreparedParquetOpen {let mut row_groups = RowGroupAccessPlanFilter::new(create_initial_plan(&prepared.file_name,&prepared.extensions,- rg_metadata.len(),+ rg_metadata,)?);// If there is a range restricting what parts of the file to read@@ -1515,29 +1515,44 @@ fn constant_value_from_stats(/// Return the initial [`ParquetAccessPlan`]///-/// If the user has supplied one as an extension, use that-/// otherwise return a plan that scans all row groups+/// If the user has supplied a parquet access extension, use that; otherwise+/// return a plan that scans all row groups.///-/// Returns an error if an invalid `ParquetAccessPlan` is provided+/// Returns an error if an invalid parquet access extension is provided.////// Note: file_name is only used for error messagesfn create_initial_plan(file_name: &str,extensions: &datafusion_datasource::FileExtensions,- row_group_count: usize,+ rg_metadata: &[RowGroupMetaData],) -> Result<ParquetAccessPlan> {- if let Some(access_plan) = extensions.get::<ParquetAccessPlan>() {- let plan_len = access_plan.len();- if plan_len != row_group_count {- return exec_err!(- "Invalid ParquetAccessPlan for {file_name}. Specified {plan_len} row groups, but file has {row_group_count}"- );+ let row_group_count = rg_metadata.len();+ match (+ extensions.get::<ParquetAccessPlan>(),+ extensions.get::<ParquetRowSelection>(),+ ) {+ (Some(_), Some(_)) => exec_err!(+ "Invalid parquet access extensions for {file_name}. \+ Specify either ParquetAccessPlan or ParquetRowSelection, not both"+ ),+ (Some(access_plan), None) => {+ let plan_len = access_plan.len();+ if plan_len != row_group_count {+ return exec_err!(+ "Invalid ParquetAccessPlan for {file_name}. Specified {plan_len} row groups, but file has {row_group_count}"+ );+ }+ Ok(access_plan.clone())+ }+ (None, Some(row_selection)) => {+ ParquetAccessPlan::try_new_from_overall_row_selection(+ row_selection.selection().clone(),+ rg_metadata,+ )}- return Ok(access_plan.clone());+ // default to scanning all row groups+ (None, None) => Ok(ParquetAccessPlan::new_all(row_group_count)),}-- // default to scanning all row groups- Ok(ParquetAccessPlan::new_all(row_group_count))}/// Build a page pruning predicate from an optional predicate expression.@@ -1601,13 +1616,13 @@ async fn load_page_index<T: AsyncFileReader>(mod test {use super::*;use super::{ConstantColumns, ParquetMorselizer, constant_columns_from_stats};- use crate::{DefaultParquetFileReaderFactory, RowGroupAccess};+ use crate::{DefaultParquetFileReaderFactory, ParquetRowSelection, RowGroupAccess};use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion_common::{- ColumnStatistics, ScalarValue, Statistics, internal_err, record_batch,- stats::Precision,+ ColumnStatistics, ScalarValue, Statistics, assert_contains, internal_err,+ record_batch, stats::Precision,};use datafusion_datasource::morsel::{Morsel, Morselizer};use datafusion_datasource::{PartitionedFile, TableSchema, TableSchemaBuilder};@@ -1626,7 +1641,9 @@ mod test {use futures::stream::BoxStream;use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path};use parquet::arrow::ArrowWriter;+ use parquet::file::metadata::ColumnChunkMetaData;use parquet::file::properties::WriterProperties;+ use parquet::schema::types::{SchemaDescPtr, SchemaDescriptor};use std::collections::VecDeque;use std::sync::Arc;@@ -1655,6 +1672,87 @@ mod test {preserve_order: bool,}+ #[test]+ fn create_initial_plan_from_parquet_row_selection_extension() {+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};++ let mut extensions = datafusion_datasource::FileExtensions::new();+ extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![+ RowSelector::select(10),+ RowSelector::skip(20),+ RowSelector::select(30),+ ])));+ let rg_metadata = row_group_metadata(&[10, 20, 30]);++ let access_plan =+ create_initial_plan("test.parquet", &extensions, &rg_metadata).unwrap();++ assert_eq!(+ access_plan,+ ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Skip,+ RowGroupAccess::Scan,+ ])+ );+ }++ #[test]+ fn create_initial_plan_rejects_multiple_access_extensions() {+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};++ let mut extensions = datafusion_datasource::FileExtensions::new();+ extensions.insert(ParquetAccessPlan::new_all(3));+ extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![+ RowSelector::select(60),+ ])));+ let rg_metadata = row_group_metadata(&[10, 20, 30]);++ let err = create_initial_plan("test.parquet", &extensions, &rg_metadata)+ .unwrap_err()+ .to_string();++ assert_contains!(+ err,+ "Specify either ParquetAccessPlan or ParquetRowSelection, not both"+ );+ }++ fn row_group_metadata(row_counts: &[i64]) -> Vec<RowGroupMetaData> {+ let schema_descr = test_schema_descr();++ row_counts+ .iter()+ .map(|num_rows| {+ let column = ColumnChunkMetaData::builder(schema_descr.column(0))+ .set_num_values(*num_rows)+ .build()+ .unwrap();++ RowGroupMetaData::builder(Arc::clone(&schema_descr))+ .set_num_rows(*num_rows)+ .set_column_metadata(vec![column])+ .build()+ .unwrap()+ })+ .collect()+ }++ fn test_schema_descr() -> SchemaDescPtr {+ use parquet::basic::{LogicalType, Type as PhysicalType};+ use parquet::schema::types::Type as SchemaType;++ let field = SchemaType::primitive_type_builder("a", PhysicalType::BYTE_ARRAY)+ .with_logical_type(Some(LogicalType::String))+ .build()+ .unwrap();+ let schema = SchemaType::group_type_builder("schema")+ .with_fields(vec![Arc::new(field)])+ .build()+ .unwrap();+ Arc::new(SchemaDescriptor::new(Arc::new(schema)))+ }+impl ParquetMorselizerBuilder {/// Create a new builder with sensible defaults for tests.fn new() -> Self {diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 840b86dcb8..4c1e1b386d 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -217,6 +217,12 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// used to implement external indexes on top of parquet files and select only/// portions of the files.///+/// If the external index naturally produces a file-level+/// [`RowSelection`](parquet::arrow::arrow_reader::RowSelection), wrap it in+/// [`ParquetRowSelection`](crate::ParquetRowSelection) and provide it as an+/// extension. DataFusion will use the parquet metadata to split the selection+/// into row-group-level access.+////// The `DataSourceExec` will try and reduce any provided `ParquetAccessPlan`/// further based on the contents of `ParquetMetadata` and other settings.///