Skip to content

Apache DataFusion SQL Query Engine

feat: support file-level parquet row selections (#22940)

Huaijin38269f9c0cf1
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rs
index 31be6fd979..8fd9689ae3 100644
--- a/datafusion/core/tests/parquet/external_access_plan.rs
+++ b/datafusion/core/tests/parquet/external_access_plan.rs
@@ -29,8 +29,10 @@ use datafusion::common::Result;
use datafusion::datasource::listing::PartitionedFile;
use datafusion::datasource::physical_plan::ParquetSource;
use datafusion::prelude::SessionContext;
-use datafusion_common::{DFSchema, assert_contains};
-use datafusion_datasource_parquet::{ParquetAccessPlan, RowGroupAccess};
+use datafusion_common::{DFSchema, assert_batches_eq, assert_contains};
+use datafusion_datasource_parquet::{
+ ParquetAccessPlan, ParquetRowSelection, RowGroupAccess,
+};
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_expr::{Expr, col, lit};
use datafusion_physical_plan::ExecutionPlan;
@@ -152,6 +154,94 @@ async fn skip_scan() {
}
}
+#[tokio::test]
+async fn row_selection_extension() {
+ // The file has 2 row groups of 5 rows each (10 rows total). Attach a
+ // file-level `ParquetRowSelection` to the `PartitionedFile` and verify it
+ // survives the path from `PartitionedFile` into the parquet opener/reader.
+
+ // select a single row in the first row group
+ let parquet_metrics = TestFull {
+ access_plan: None,
+ row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![
+ RowSelector::skip(2),
+ RowSelector::select(1),
+ RowSelector::skip(7),
+ ]))),
+ expected_rows: 1,
+ expected_output: Some(&[
+ "+------+------------+",
+ "| utf8 | large_utf8 |",
+ "+------+------------+",
+ "| c | c |",
+ "+------+------------+",
+ ]),
+ predicate: None,
+ }
+ .run()
+ .await
+ .unwrap();
+
+ // only the first row group is read, so some bytes are scanned
+ let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();
+ assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}",);
+}
+
+#[tokio::test]
+async fn row_selection_extension_spanning_row_groups() {
+ // A selection whose selectors straddle the row group boundary (row 4 is the
+ // last row of group 0, rows 5-6 are the first rows of group 1).
+ let parquet_metrics = TestFull {
+ access_plan: None,
+ row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![
+ RowSelector::skip(4),
+ RowSelector::select(3),
+ RowSelector::skip(3),
+ ]))),
+ expected_rows: 3,
+ expected_output: Some(&[
+ "+------+------------+",
+ "| utf8 | large_utf8 |",
+ "+------+------------+",
+ "| | |",
+ "| e | e |",
+ "| f | f |",
+ "+------+------------+",
+ ]),
+ predicate: None,
+ }
+ .run()
+ .await
+ .unwrap();
+
+ let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();
+ assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}",);
+}
+
+#[tokio::test]
+async fn bad_row_selection_extension() {
+ // selection specifies fewer rows than the file actually contains
+ let err = TestFull {
+ access_plan: None,
+ row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![
+ RowSelector::skip(2),
+ RowSelector::select(1),
+ ]))),
+ expected_rows: 10000,
+ expected_output: None,
+ predicate: None,
+ }
+ .run()
+ .await
+ .unwrap_err();
+ let err_string = err.to_string();
+ assert_contains!(&err_string, "Invalid Parquet RowSelection");
+ assert_contains!(
+ &err_string,
+ "File has 10 rows, but selection specifies 3 rows."
+ );
+}
+
#[tokio::test]
async fn plan_and_filter() {
// show that row group pruning is applied even when an initial plan is supplied
@@ -170,7 +260,9 @@ async fn plan_and_filter() {
// initial
let parquet_metrics = TestFull {
access_plan,
+ row_selection: None,
expected_rows: 0,
+ expected_output: None,
predicate: Some(predicate),
}
.run()
@@ -227,7 +319,9 @@ async fn bad_row_groups() {
RowGroupAccess::Skip,
RowGroupAccess::Scan,
])),
+ row_selection: None,
expected_rows: 0,
+ expected_output: None,
predicate: None,
}
.run()
@@ -249,8 +343,10 @@ async fn bad_selection() {
])),
RowGroupAccess::Skip,
])),
+ row_selection: None,
// expects that we hit an error, this should not be run
expected_rows: 10000,
+ expected_output: None,
predicate: None,
}
.run()
@@ -300,7 +396,9 @@ impl Test {
} = self;
TestFull {
access_plan,
+ row_selection: None,
expected_rows,
+ expected_output: None,
predicate: None,
}
.run()
@@ -317,7 +415,9 @@ impl Test {
/// 4. Returns the statistics from running the plan
struct TestFull {
access_plan: Option<ParquetAccessPlan>,
+ row_selection: Option<ParquetRowSelection>,
expected_rows: usize,
+ expected_output: Option<&'static [&'static str]>,
predicate: Option<Expr>,
}
@@ -327,7 +427,9 @@ impl TestFull {
let Self {
access_plan,
+ row_selection,
expected_rows,
+ expected_output,
predicate,
} = self;
@@ -352,6 +454,11 @@ impl TestFull {
partitioned_file = partitioned_file.with_extension(access_plan);
}
+ // add the file-level row selection, if any, as an extension
+ if let Some(row_selection) = row_selection {
+ partitioned_file = partitioned_file.with_extension(row_selection);
+ }
+
// Create a DataSourceExec to read the file
let object_store_url = ObjectStoreUrl::local_filesystem();
// add the predicate, if requested
@@ -380,6 +487,9 @@ impl TestFull {
"results: \n{}",
pretty_format_batches(&results).unwrap()
);
+ if let Some(expected_output) = expected_output {
+ assert_batches_eq!(expected_output, &results);
+ }
std::fs::remove_file(file_name).unwrap();
diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rs
index edbea39948..593ed365cd 100644
--- a/datafusion/datasource-parquet/src/access_plan.rs
+++ b/datafusion/datasource-parquet/src/access_plan.rs
@@ -17,7 +17,7 @@
use crate::sort::reverse_row_selection;
use arrow::datatypes::Schema;
-use datafusion_common::{Result, assert_eq_or_internal_err};
+use datafusion_common::{Result, assert_eq_or_internal_err, exec_err};
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use log::debug;
@@ -104,6 +104,41 @@ pub struct ParquetAccessPlan {
fully_matched: Vec<bool>,
}
+/// A file-level row selection for a parquet scan.
+///
+/// Attach this type to a [`PartitionedFile`](datafusion_datasource::PartitionedFile)
+/// with [`PartitionedFile::with_extension`](datafusion_datasource::PartitionedFile::with_extension)
+/// when an external index produces a [`RowSelection`] across the entire parquet
+/// file. DataFusion will use parquet metadata to split it into row-group-level
+/// access when the file is opened.
+#[derive(Debug, Clone, PartialEq)]
+pub struct ParquetRowSelection {
+ selection: RowSelection,
+}
+
+impl ParquetRowSelection {
+ /// Create a new file-level parquet row selection.
+ pub fn new(selection: RowSelection) -> Self {
+ Self { selection }
+ }
+
+ /// Return a reference to the underlying [`RowSelection`].
+ pub fn selection(&self) -> &RowSelection {
+ &self.selection
+ }
+
+ /// Convert into the underlying [`RowSelection`].
+ pub fn into_inner(self) -> RowSelection {
+ self.selection
+ }
+}
+
+impl From<RowSelection> for ParquetRowSelection {
+ fn from(selection: RowSelection) -> Self {
+ Self::new(selection)
+ }
+}
+
/// Describes how the parquet reader will access a row group
#[derive(Debug, Clone, PartialEq)]
pub enum RowGroupAccess {
@@ -143,6 +178,102 @@ impl RowGroupAccess {
}
}
+/// Single-pass cursor over a file-level [`RowSelection`].
+///
+/// `take` returns the next selector fragment capped to the requested row count,
+/// splitting the current selector when it straddles a row group boundary.
+struct OverallRowSelectionCursor {
+ selector_iter: std::vec::IntoIter<RowSelector>,
+ current: Option<RowSelector>,
+}
+
+impl OverallRowSelectionCursor {
+ fn new(selection: RowSelection) -> Self {
+ let selectors: Vec<RowSelector> = selection.into();
+ let mut selector_iter = selectors.into_iter();
+ let current = selector_iter.next();
+ Self {
+ selector_iter,
+ current,
+ }
+ }
+
+ /// Take up to `max_rows` rows from the current selector.
+ ///
+ /// If the current selector crosses the requested boundary, this returns the
+ /// leading fragment and keeps the remaining rows in `self.current` for the
+ /// next call.
+ #[inline]
+ fn take(&mut self, max_rows: usize) -> Option<RowSelector> {
+ let sel = self.current?;
+ let row_count = sel.row_count.min(max_rows);
+ self.current = if row_count < sel.row_count {
+ Some(RowSelector {
+ row_count: sel.row_count - row_count,
+ skip: sel.skip,
+ })
+ } else {
+ self.selector_iter.next()
+ };
+
+ Some(RowSelector {
+ row_count,
+ skip: sel.skip,
+ })
+ }
+
+ fn remaining_rows(self) -> usize {
+ self.current.map_or(0, |s| s.row_count)
+ + self.selector_iter.map(|s| s.row_count).sum::<usize>()
+ }
+}
+
+/// Accumulates the selector fragments that belong to one row group.
+struct RowGroupAccessBuilder {
+ /// Selector fragments belonging to this row group.
+ selectors: Vec<RowSelector>,
+ /// Number of selected rows accumulated for this row group.
+ selected: usize,
+ /// Number of skipped rows accumulated for this row group.
+ skipped: usize,
+ /// Number of rows still needed to complete this row group.
+ remaining: usize,
+}
+
+impl RowGroupAccessBuilder {
+ fn new(row_group_rows: usize) -> Self {
+ Self {
+ selectors: Vec::with_capacity(1),
+ selected: 0,
+ skipped: 0,
+ remaining: row_group_rows,
+ }
+ }
+
+ #[inline]
+ fn push(&mut self, selector: RowSelector) {
+ self.remaining -= selector.row_count;
+
+ if selector.skip {
+ self.skipped += selector.row_count;
+ } else {
+ self.selected += selector.row_count;
+ }
+
+ self.selectors.push(selector);
+ }
+
+ fn into_access(self) -> RowGroupAccess {
+ if self.selected == 0 {
+ RowGroupAccess::Skip
+ } else if self.skipped == 0 {
+ RowGroupAccess::Scan
+ } else {
+ RowGroupAccess::Selection(self.selectors.into())
+ }
+ }
+}
+
impl ParquetAccessPlan {
/// Create a new `ParquetAccessPlan` that scans all row groups
pub fn new_all(row_group_count: usize) -> Self {
@@ -169,6 +300,60 @@ impl ParquetAccessPlan {
}
}
+ /// Create a new `ParquetAccessPlan` from a file-level [`RowSelection`].
+ ///
+ /// The selection is interpreted across all rows in the file, in row group
+ /// order, and is split into row-group level access using `row_group_meta_data`.
+ /// Fully skipped row groups become [`RowGroupAccess::Skip`], fully selected
+ /// row groups become [`RowGroupAccess::Scan`], and partially selected row
+ /// groups become [`RowGroupAccess::Selection`].
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the selection does not specify exactly the same
+ /// number of rows as the file metadata.
+ pub fn try_new_from_overall_row_selection(
+ selection: RowSelection,
+ row_group_meta_data: &[RowGroupMetaData],
+ ) -> Result<Self> {
+ // Keep this as a single pass over the selector stream rather than
+ // repeatedly calling `RowSelection::split_off` per row group. The
+ // `split_off` version is simpler, but it clones/retains substantially
+ // more selector buffer capacity for highly fragmented selections.
+ let mut cursor = OverallRowSelectionCursor::new(selection);
+
+ let mut selection_rows = 0usize;
+ let mut file_rows = 0usize;
+
+ let mut row_groups = Vec::with_capacity(row_group_meta_data.len());
+ for rg_meta in row_group_meta_data {
+ let rg_rows = rg_meta.num_rows() as usize;
+ file_rows += rg_rows;
+
+ let mut builder = RowGroupAccessBuilder::new(rg_rows);
+ while builder.remaining > 0 {
+ let Some(selector) = cursor.take(builder.remaining) else {
+ break;
+ };
+ selection_rows += selector.row_count;
+ builder.push(selector);
+ }
+
+ row_groups.push(builder.into_access());
+ }
+
+ selection_rows += cursor.remaining_rows();
+
+ if selection_rows != file_rows {
+ return exec_err!(
+ "Invalid Parquet RowSelection. File has {file_rows} rows, \
+ but selection specifies {selection_rows} rows."
+ );
+ }
+
+ Ok(Self::new(row_groups))
+ }
+
/// Set the i-th row group to the specified [`RowGroupAccess`]
pub fn set(&mut self, idx: usize, access: RowGroupAccess) {
let should_scan = access.should_scan();
@@ -758,6 +943,90 @@ mod test {
);
}
+ #[test]
+ fn test_new_from_overall_row_selection() {
+ let row_selection = RowSelection::from(vec![
+ RowSelector::select(10),
+ RowSelector::skip(25),
+ RowSelector::select(10),
+ RowSelector::skip(15),
+ RowSelector::select(40),
+ ]);
+
+ let access_plan = ParquetAccessPlan::try_new_from_overall_row_selection(
+ row_selection,
+ &ROW_GROUP_METADATA,
+ )
+ .unwrap();
+
+ assert_eq!(
+ access_plan,
+ ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Skip,
+ RowGroupAccess::Selection(
+ vec![
+ RowSelector::skip(5),
+ RowSelector::select(10),
+ RowSelector::skip(15),
+ ]
+ .into()
+ ),
+ RowGroupAccess::Scan,
+ ])
+ );
+ }
+
+ #[test]
+ fn test_new_from_overall_row_selection_invalid_row_count() {
+ let row_selection = RowSelection::from(vec![RowSelector::select(99)]);
+
+ let err = ParquetAccessPlan::try_new_from_overall_row_selection(
+ row_selection,
+ &ROW_GROUP_METADATA,
+ )
+ .unwrap_err()
+ .to_string();
+
+ assert_contains!(
+ err,
+ "Invalid Parquet RowSelection. File has 100 rows, but selection specifies 99 rows"
+ );
+ }
+
+ #[test]
+ fn test_new_from_overall_row_selection_boundary_splits() {
+ let row_selection = RowSelection::from(vec![
+ RowSelector::skip(5),
+ RowSelector::select(10),
+ RowSelector::skip(20),
+ RowSelector::select(25),
+ RowSelector::skip(40),
+ ]);
+
+ let access_plan = ParquetAccessPlan::try_new_from_overall_row_selection(
+ row_selection,
+ &ROW_GROUP_METADATA,
+ )
+ .unwrap();
+
+ assert_eq!(
+ access_plan,
+ ParquetAccessPlan::new(vec![
+ RowGroupAccess::Selection(
+ vec![RowSelector::skip(5), RowSelector::select(5)].into()
+ ),
+ RowGroupAccess::Selection(
+ vec![RowSelector::select(5), RowSelector::skip(15)].into()
+ ),
+ RowGroupAccess::Selection(
+ vec![RowSelector::skip(5), RowSelector::select(25)].into()
+ ),
+ RowGroupAccess::Skip,
+ ])
+ );
+ }
+
#[test]
fn test_invalid_too_few() {
let access_plan = ParquetAccessPlan::new(vec![
diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs
index bec0736366..260d6ee471 100644
--- a/datafusion/datasource-parquet/src/mod.rs
+++ b/datafusion/datasource-parquet/src/mod.rs
@@ -46,7 +46,7 @@ mod test_util;
mod virtual_column;
mod writer;
-pub use access_plan::{ParquetAccessPlan, RowGroupAccess};
+pub use access_plan::{ParquetAccessPlan, ParquetRowSelection, RowGroupAccess};
pub use file_format::*;
pub use metrics::ParquetFileMetrics;
pub use page_filter::PagePruningAccessPlanFilter;
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index 5b517663f9..e9f50134d4 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -31,7 +31,7 @@ use crate::row_filter::RowFilterGenerator;
use crate::row_group_filter::{BloomFilterStatistics, RowGroupAccessPlanFilter};
use crate::{
Int96Coercer, ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,
- ParquetVirtualColumn, apply_file_schema_type_coercions,
+ ParquetRowSelection, ParquetVirtualColumn, apply_file_schema_type_coercions,
};
use arrow::array::RecordBatch;
use arrow::datatypes::DataType;
@@ -76,7 +76,7 @@ use parquet::arrow::async_reader::AsyncFileReader;
use parquet::arrow::parquet_column;
use parquet::basic::Type;
use parquet::bloom_filter::Sbbf;
-use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};
+use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMetaData};
/// Morselizer-level state for virtual columns, precomputed once per scan
/// partition so each file skips the validator walks, `null_replacements`
@@ -1080,7 +1080,7 @@ impl FiltersPreparedParquetOpen {
let mut row_groups = RowGroupAccessPlanFilter::new(create_initial_plan(
&prepared.file_name,
&prepared.extensions,
- rg_metadata.len(),
+ rg_metadata,
)?);
// If there is a range restricting what parts of the file to read
@@ -1515,29 +1515,44 @@ fn constant_value_from_stats(
/// Return the initial [`ParquetAccessPlan`]
///
-/// If the user has supplied one as an extension, use that
-/// otherwise return a plan that scans all row groups
+/// If the user has supplied a parquet access extension, use that; otherwise
+/// return a plan that scans all row groups.
///
-/// Returns an error if an invalid `ParquetAccessPlan` is provided
+/// Returns an error if an invalid parquet access extension is provided.
///
/// Note: file_name is only used for error messages
fn create_initial_plan(
file_name: &str,
extensions: &datafusion_datasource::FileExtensions,
- row_group_count: usize,
+ rg_metadata: &[RowGroupMetaData],
) -> Result<ParquetAccessPlan> {
- if let Some(access_plan) = extensions.get::<ParquetAccessPlan>() {
- let plan_len = access_plan.len();
- if plan_len != row_group_count {
- return exec_err!(
- "Invalid ParquetAccessPlan for {file_name}. Specified {plan_len} row groups, but file has {row_group_count}"
- );
+ let row_group_count = rg_metadata.len();
+ match (
+ extensions.get::<ParquetAccessPlan>(),
+ extensions.get::<ParquetRowSelection>(),
+ ) {
+ (Some(_), Some(_)) => exec_err!(
+ "Invalid parquet access extensions for {file_name}. \
+ Specify either ParquetAccessPlan or ParquetRowSelection, not both"
+ ),
+ (Some(access_plan), None) => {
+ let plan_len = access_plan.len();
+ if plan_len != row_group_count {
+ return exec_err!(
+ "Invalid ParquetAccessPlan for {file_name}. Specified {plan_len} row groups, but file has {row_group_count}"
+ );
+ }
+ Ok(access_plan.clone())
+ }
+ (None, Some(row_selection)) => {
+ ParquetAccessPlan::try_new_from_overall_row_selection(
+ row_selection.selection().clone(),
+ rg_metadata,
+ )
}
- return Ok(access_plan.clone());
+ // default to scanning all row groups
+ (None, None) => Ok(ParquetAccessPlan::new_all(row_group_count)),
}
-
- // default to scanning all row groups
- Ok(ParquetAccessPlan::new_all(row_group_count))
}
/// Build a page pruning predicate from an optional predicate expression.
@@ -1601,13 +1616,13 @@ async fn load_page_index<T: AsyncFileReader>(
mod test {
use super::*;
use super::{ConstantColumns, ParquetMorselizer, constant_columns_from_stats};
- use crate::{DefaultParquetFileReaderFactory, RowGroupAccess};
+ use crate::{DefaultParquetFileReaderFactory, ParquetRowSelection, RowGroupAccess};
use arrow::array::RecordBatch;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion_common::{
- ColumnStatistics, ScalarValue, Statistics, internal_err, record_batch,
- stats::Precision,
+ ColumnStatistics, ScalarValue, Statistics, assert_contains, internal_err,
+ record_batch, stats::Precision,
};
use datafusion_datasource::morsel::{Morsel, Morselizer};
use datafusion_datasource::{PartitionedFile, TableSchema, TableSchemaBuilder};
@@ -1626,7 +1641,9 @@ mod test {
use futures::stream::BoxStream;
use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path};
use parquet::arrow::ArrowWriter;
+ use parquet::file::metadata::ColumnChunkMetaData;
use parquet::file::properties::WriterProperties;
+ use parquet::schema::types::{SchemaDescPtr, SchemaDescriptor};
use std::collections::VecDeque;
use std::sync::Arc;
@@ -1655,6 +1672,87 @@ mod test {
preserve_order: bool,
}
+ #[test]
+ fn create_initial_plan_from_parquet_row_selection_extension() {
+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
+
+ let mut extensions = datafusion_datasource::FileExtensions::new();
+ extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![
+ RowSelector::select(10),
+ RowSelector::skip(20),
+ RowSelector::select(30),
+ ])));
+ let rg_metadata = row_group_metadata(&[10, 20, 30]);
+
+ let access_plan =
+ create_initial_plan("test.parquet", &extensions, &rg_metadata).unwrap();
+
+ assert_eq!(
+ access_plan,
+ ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Skip,
+ RowGroupAccess::Scan,
+ ])
+ );
+ }
+
+ #[test]
+ fn create_initial_plan_rejects_multiple_access_extensions() {
+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
+
+ let mut extensions = datafusion_datasource::FileExtensions::new();
+ extensions.insert(ParquetAccessPlan::new_all(3));
+ extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![
+ RowSelector::select(60),
+ ])));
+ let rg_metadata = row_group_metadata(&[10, 20, 30]);
+
+ let err = create_initial_plan("test.parquet", &extensions, &rg_metadata)
+ .unwrap_err()
+ .to_string();
+
+ assert_contains!(
+ err,
+ "Specify either ParquetAccessPlan or ParquetRowSelection, not both"
+ );
+ }
+
+ fn row_group_metadata(row_counts: &[i64]) -> Vec<RowGroupMetaData> {
+ let schema_descr = test_schema_descr();
+
+ row_counts
+ .iter()
+ .map(|num_rows| {
+ let column = ColumnChunkMetaData::builder(schema_descr.column(0))
+ .set_num_values(*num_rows)
+ .build()
+ .unwrap();
+
+ RowGroupMetaData::builder(Arc::clone(&schema_descr))
+ .set_num_rows(*num_rows)
+ .set_column_metadata(vec![column])
+ .build()
+ .unwrap()
+ })
+ .collect()
+ }
+
+ fn test_schema_descr() -> SchemaDescPtr {
+ use parquet::basic::{LogicalType, Type as PhysicalType};
+ use parquet::schema::types::Type as SchemaType;
+
+ let field = SchemaType::primitive_type_builder("a", PhysicalType::BYTE_ARRAY)
+ .with_logical_type(Some(LogicalType::String))
+ .build()
+ .unwrap();
+ let schema = SchemaType::group_type_builder("schema")
+ .with_fields(vec![Arc::new(field)])
+ .build()
+ .unwrap();
+ Arc::new(SchemaDescriptor::new(Arc::new(schema)))
+ }
+
impl ParquetMorselizerBuilder {
/// Create a new builder with sensible defaults for tests.
fn new() -> Self {
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 840b86dcb8..4c1e1b386d 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -217,6 +217,12 @@ use parquet::encryption::decrypt::FileDecryptionProperties;
/// used to implement external indexes on top of parquet files and select only
/// portions of the files.
///
+/// If the external index naturally produces a file-level
+/// [`RowSelection`](parquet::arrow::arrow_reader::RowSelection), wrap it in
+/// [`ParquetRowSelection`](crate::ParquetRowSelection) and provide it as an
+/// extension. DataFusion will use the parquet metadata to split the selection
+/// into row-group-level access.
+///
/// The `DataSourceExec` will try and reduce any provided `ParquetAccessPlan`
/// further based on the contents of `ParquetMetadata` and other settings.
///