Skip to content

Apache DataFusion SQL Query Engine

Refactor parquet datasource into an explicit state machine (#21190)

Andrew Lambf830ee39df83
diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs
index 0e137a706f..9a907f4118 100644
--- a/datafusion/datasource-parquet/src/mod.rs
+++ b/datafusion/datasource-parquet/src/mod.rs
@@ -15,6 +15,10 @@
// specific language governing permissions and limitations
// under the License.
+//! DataFusion Parquet Reader: [`ParquetSource`]
+//!
+//! [`ParquetSource`]: source::ParquetSource
+
// Make sure fast / cheap clones on Arc are explicit:
// https://github.com/apache/datafusion/issues/11143
#![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index 2990f43165..466f631c53 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-//! [`ParquetOpener`] for opening Parquet files
+//! [`ParquetOpener`] state machine for opening Parquet files
use crate::page_filter::PagePruningAccessPlanFilter;
use crate::row_filter::build_projection_read_plan;
@@ -25,18 +25,19 @@ use crate::{
apply_file_schema_type_coercions, coerce_int96_to_resolution, row_filter,
};
use arrow::array::{RecordBatch, RecordBatchOptions};
-use arrow::datatypes::{DataType, Schema};
+use arrow::datatypes::DataType;
use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};
use datafusion_physical_expr::projection::{ProjectionExprs, Projector};
use datafusion_physical_expr::utils::reassign_expr_columns;
use datafusion_physical_expr_adapter::replace_columns_with_literals;
-use parquet::errors::ParquetError;
use std::collections::HashMap;
+use std::future::Future;
+use std::mem;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
-use arrow::datatypes::{SchemaRef, TimeUnit};
+use arrow::datatypes::{Schema, SchemaRef, TimeUnit};
use datafusion_common::encryption::FileDecryptionProperties;
use datafusion_common::stats::Precision;
use datafusion_common::{
@@ -58,7 +59,9 @@ use datafusion_pruning::{FilePruner, PruningPredicate, build_pruning_predicate};
use datafusion_common::config::EncryptionFactoryOptions;
#[cfg(feature = "parquet_encryption")]
use datafusion_execution::parquet_encryption::EncryptionFactory;
-use futures::{Stream, StreamExt, ready};
+use futures::{
+ FutureExt, Stream, StreamExt, future::BoxFuture, ready, stream::BoxStream,
+};
use log::debug;
use parquet::DecodeResult;
use parquet::arrow::ParquetRecordBatchStreamBuilder;
@@ -68,9 +71,14 @@ use parquet::arrow::arrow_reader::{
};
use parquet::arrow::async_reader::AsyncFileReader;
use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder};
+use parquet::errors::ParquetError;
use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};
-/// Implements [`FileOpener`] for a parquet file
+/// Entry point for opening a Parquet file
+///
+/// Reading a Parquet file is a multi-stage process, with multiple CPU-intensive
+/// steps interspersed with I/O steps. The code in this module implements the steps
+/// as an explicit state machine -- see [`ParquetOpenState`] for details.
pub(super) struct ParquetOpener {
/// Execution partition index
pub(crate) partition_index: usize,
@@ -126,15 +134,338 @@ pub(super) struct ParquetOpener {
pub reverse_row_groups: bool,
}
+/// States for [`ParquetOpenFuture`]
+///
+/// These states correspond to the steps required to read and apply various
+/// filter operations.
+///
+/// States whose names beginning with `Load` represent waiting on IO to resolve
+///
+/// ```text
+/// Start
+/// |
+/// v
+/// [LoadEncryption]?
+/// |
+/// v
+/// PruneFile
+/// |
+/// v
+/// LoadMetadata
+/// |
+/// v
+/// PrepareFilters
+/// |
+/// v
+/// LoadPageIndex
+/// |
+/// v
+/// PruneWithStatistics
+/// |
+/// v
+/// PruneWithBloomFilters
+/// |
+/// v
+/// BuildStream
+/// |
+/// v
+/// Done
+/// ```
+///
+/// Note: `LoadEncryption` is only present when the `parquet_encryption` feature is
+/// enabled. All other states are always visited in the order shown above,
+/// though any async state may return `Poll::Pending` and then resume later.
+enum ParquetOpenState {
+ Start {
+ prepared: Box<PreparedParquetOpen>,
+ #[cfg(feature = "parquet_encryption")]
+ encryption_context: Arc<EncryptionContext>,
+ },
+ /// Loading encryption footers
+ #[cfg(feature = "parquet_encryption")]
+ LoadEncryption(BoxFuture<'static, Result<Box<PreparedParquetOpen>>>),
+ /// Try to prune file using only file-level statistics and partition
+ /// values before loading any parquet metadata
+ PruneFile(Box<PreparedParquetOpen>),
+ /// Loading Parquet metadata (in footer)
+ LoadMetadata(BoxFuture<'static, Result<MetadataLoadedParquetOpen>>),
+ /// Specialize any filters for the actual file schema (only known after
+ /// metadata is loaded)
+ PrepareFilters(Box<MetadataLoadedParquetOpen>),
+ /// Loading [Parquet Page Index](https://parquet.apache.org/docs/file-format/pageindex/)
+ LoadPageIndex(BoxFuture<'static, Result<FiltersPreparedParquetOpen>>),
+ /// Pruning Row Groups
+ PruneWithStatistics(Box<FiltersPreparedParquetOpen>),
+ /// Pruning with Bloom Filters
+ ///
+ /// TODO: split state as this currently does both I/O and CPU work
+ PruneWithBloomFilters(BoxFuture<'static, Result<RowGroupsPrunedParquetOpen>>),
+ /// Builds the final reader stream
+ ///
+ /// TODO: split state as this currently does both I/O and CPU work.
+ BuildStream(Box<RowGroupsPrunedParquetOpen>),
+ /// Terminal state: the final opened stream is ready to return.
+ Ready(BoxStream<'static, Result<RecordBatch>>),
+ /// Terminal state: reading complete
+ Done,
+}
+
+struct PreparedParquetOpen {
+ partition_index: usize,
+ partitioned_file: PartitionedFile,
+ file_range: Option<datafusion_datasource::FileRange>,
+ extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,
+ file_name: String,
+ file_metrics: ParquetFileMetrics,
+ baseline_metrics: BaselineMetrics,
+ file_pruner: Option<FilePruner>,
+ metadata_size_hint: Option<usize>,
+ metrics: ExecutionPlanMetricsSet,
+ parquet_file_reader_factory: Arc<dyn ParquetFileReaderFactory>,
+ async_file_reader: Box<dyn AsyncFileReader>,
+ batch_size: usize,
+ logical_file_schema: SchemaRef,
+ physical_file_schema: SchemaRef,
+ output_schema: SchemaRef,
+ projection: ProjectionExprs,
+ predicate: Option<Arc<dyn PhysicalExpr>>,
+ reorder_predicates: bool,
+ pushdown_filters: bool,
+ force_filter_selections: bool,
+ enable_page_index: bool,
+ enable_bloom_filter: bool,
+ enable_row_group_stats_pruning: bool,
+ limit: Option<usize>,
+ coerce_int96: Option<TimeUnit>,
+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+ predicate_creation_errors: Count,
+ max_predicate_cache_size: Option<usize>,
+ reverse_row_groups: bool,
+ preserve_order: bool,
+ #[cfg(feature = "parquet_encryption")]
+ file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
+}
+
+/// State of [`ParquetOpenState`]
+///
+/// Result of loading parquet metadata after file-level pruning is complete.
+struct MetadataLoadedParquetOpen {
+ prepared: PreparedParquetOpen,
+ reader_metadata: ArrowReaderMetadata,
+ options: ArrowReaderOptions,
+}
+
+/// State of [`ParquetOpenState`]
+///
+/// Pruning Predicate and DataPage pruning information
+/// specialized for the files specific schema.
+struct FiltersPreparedParquetOpen {
+ loaded: MetadataLoadedParquetOpen,
+ pruning_predicate: Option<Arc<PruningPredicate>>,
+ page_pruning_predicate: Option<Arc<PagePruningAccessPlanFilter>>,
+}
+
+/// State of [`ParquetOpenState`]
+///
+/// Result of CPU-only row-group pruning before optional bloom-filter I/O.
+struct RowGroupsPrunedParquetOpen {
+ prepared: FiltersPreparedParquetOpen,
+ row_groups: RowGroupAccessPlanFilter,
+}
+
+/// Implements state machine described in [`ParquetOpenState`]
+struct ParquetOpenFuture {
+ state: ParquetOpenState,
+}
+
+impl ParquetOpenFuture {
+ #[cfg(feature = "parquet_encryption")]
+ fn new(prepared: PreparedParquetOpen, encryption_context: EncryptionContext) -> Self {
+ Self {
+ state: ParquetOpenState::Start {
+ prepared: Box::new(prepared),
+ encryption_context: Arc::new(encryption_context),
+ },
+ }
+ }
+
+ #[cfg(not(feature = "parquet_encryption"))]
+ fn new(prepared: PreparedParquetOpen) -> Self {
+ Self {
+ state: ParquetOpenState::Start {
+ prepared: Box::new(prepared),
+ },
+ }
+ }
+}
+
+impl ParquetOpenState {
+ /// Applies one CPU-only state transition.
+ ///
+ /// `Load*` states do not transition here and are returned unchanged so the
+ /// driver loop can poll their inner futures separately.
+ fn transition(self) -> Result<ParquetOpenState> {
+ match self {
+ ParquetOpenState::Start {
+ prepared,
+ #[cfg(feature = "parquet_encryption")]
+ encryption_context,
+ } => {
+ #[cfg(feature = "parquet_encryption")]
+ {
+ let mut prepared = *prepared;
+ let future = async move {
+ let file_location =
+ &prepared.partitioned_file.object_meta.location;
+ prepared.file_decryption_properties = encryption_context
+ .get_file_decryption_properties(file_location)
+ .await?;
+ Ok(Box::new(prepared))
+ }
+ .boxed();
+ Ok(ParquetOpenState::LoadEncryption(future))
+ }
+ #[cfg(not(feature = "parquet_encryption"))]
+ {
+ Ok(ParquetOpenState::PruneFile(prepared))
+ }
+ }
+ #[cfg(feature = "parquet_encryption")]
+ ParquetOpenState::LoadEncryption(future) => {
+ Ok(ParquetOpenState::LoadEncryption(future))
+ }
+ ParquetOpenState::PruneFile(prepared) => {
+ let Some(prepared) = (*prepared).prune_file()? else {
+ return Ok(ParquetOpenState::Done);
+ };
+ Ok(ParquetOpenState::LoadMetadata(prepared.load().boxed()))
+ }
+ ParquetOpenState::LoadMetadata(future) => {
+ Ok(ParquetOpenState::LoadMetadata(future))
+ }
+ ParquetOpenState::PrepareFilters(loaded) => {
+ let prepared_filters = loaded.prepare_filters()?;
+ Ok(ParquetOpenState::LoadPageIndex(
+ prepared_filters.load_page_index().boxed(),
+ ))
+ }
+ ParquetOpenState::LoadPageIndex(future) => {
+ Ok(ParquetOpenState::LoadPageIndex(future))
+ }
+ ParquetOpenState::PruneWithStatistics(prepared) => {
+ let prepared_row_groups = prepared.prune_row_groups()?;
+ Ok(ParquetOpenState::PruneWithBloomFilters(
+ prepared_row_groups.prune_bloom_filters().boxed(),
+ ))
+ }
+ ParquetOpenState::PruneWithBloomFilters(future) => {
+ Ok(ParquetOpenState::PruneWithBloomFilters(future))
+ }
+ ParquetOpenState::BuildStream(prepared) => {
+ Ok(ParquetOpenState::Ready(prepared.build_stream()?))
+ }
+ ParquetOpenState::Ready(stream) => Ok(ParquetOpenState::Ready(stream)),
+ ParquetOpenState::Done => {
+ panic!("ParquetOpenFuture polled after completion");
+ }
+ }
+ }
+}
+
+impl Future for ParquetOpenFuture {
+ type Output = Result<BoxStream<'static, Result<RecordBatch>>>;
+
+ fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ loop {
+ let state = mem::replace(&mut self.state, ParquetOpenState::Done);
+ let mut state = state.transition()?;
+
+ match state {
+ #[cfg(feature = "parquet_encryption")]
+ ParquetOpenState::LoadEncryption(mut future) => {
+ state = match future.poll_unpin(cx) {
+ Poll::Ready(result) => ParquetOpenState::PruneFile(result?),
+ Poll::Pending => {
+ self.state = ParquetOpenState::LoadEncryption(future);
+ return Poll::Pending;
+ }
+ };
+ }
+ ParquetOpenState::LoadMetadata(mut future) => {
+ state = match future.poll_unpin(cx) {
+ Poll::Ready(result) => {
+ ParquetOpenState::PrepareFilters(Box::new(result?))
+ }
+ Poll::Pending => {
+ self.state = ParquetOpenState::LoadMetadata(future);
+ return Poll::Pending;
+ }
+ };
+ }
+ ParquetOpenState::LoadPageIndex(mut future) => {
+ state = match future.poll_unpin(cx) {
+ Poll::Ready(result) => {
+ ParquetOpenState::PruneWithStatistics(Box::new(result?))
+ }
+ Poll::Pending => {
+ self.state = ParquetOpenState::LoadPageIndex(future);
+ return Poll::Pending;
+ }
+ };
+ }
+ ParquetOpenState::PruneWithBloomFilters(mut future) => {
+ state = match future.poll_unpin(cx) {
+ Poll::Ready(result) => {
+ ParquetOpenState::BuildStream(Box::new(result?))
+ }
+ Poll::Pending => {
+ self.state = ParquetOpenState::PruneWithBloomFilters(future);
+ return Poll::Pending;
+ }
+ };
+ }
+ ParquetOpenState::Ready(stream) => {
+ return Poll::Ready(Ok(stream));
+ }
+ ParquetOpenState::Done => {
+ return Poll::Ready(Ok(futures::stream::empty().boxed()));
+ }
+
+ // For all other states, loop again and try to transition
+ // immediately. All states are explicitly listed here to ensure any
+ // new states are handled correctly
+ ParquetOpenState::Start { .. } => {}
+ ParquetOpenState::PruneFile(_) => {}
+ ParquetOpenState::PrepareFilters(_) => {}
+ ParquetOpenState::PruneWithStatistics(_) => {}
+ ParquetOpenState::BuildStream(_) => {}
+ };
+
+ self.state = state;
+ }
+ }
+}
+
impl FileOpener for ParquetOpener {
fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {
- // -----------------------------------
- // Step: prepare configurations, etc.
- // -----------------------------------
+ let prepared = self.prepare_open_file(partitioned_file)?;
+ #[cfg(feature = "parquet_encryption")]
+ let future = ParquetOpenFuture::new(prepared, self.get_encryption_context());
+ #[cfg(not(feature = "parquet_encryption"))]
+ let future = ParquetOpenFuture::new(prepared);
+ Ok(Box::pin(future))
+ }
+}
+
+impl ParquetOpener {
+ /// Perform the CPU-only setup for opening a parquet file.
+ fn prepare_open_file(
+ &self,
+ partitioned_file: PartitionedFile,
+ ) -> Result<PreparedParquetOpen> {
let file_range = partitioned_file.range.clone();
let extensions = partitioned_file.extensions.clone();
- let file_location = partitioned_file.object_meta.location.clone();
- let file_name = file_location.to_string();
+ let file_name = partitioned_file.object_meta.location.to_string();
let file_metrics =
ParquetFileMetrics::new(self.partition_index, &file_name, &self.metrics);
let baseline_metrics = BaselineMetrics::new(&self.metrics, self.partition_index);
@@ -143,7 +474,7 @@ impl FileOpener for ParquetOpener {
.metadata_size_hint
.or(self.metadata_size_hint);
- let mut async_file_reader: Box<dyn AsyncFileReader> =
+ let async_file_reader: Box<dyn AsyncFileReader> =
self.parquet_file_reader_factory.create_reader(
self.partition_index,
partitioned_file.clone(),
@@ -151,8 +482,6 @@ impl FileOpener for ParquetOpener {
&self.metrics,
)?;
- let batch_size = self.batch_size;
-
// Calculate the output schema from the original projection (before literal replacement)
// so we get correct field names from column references
let logical_file_schema = Arc::clone(self.table_schema.file_schema());
@@ -195,7 +524,6 @@ impl FileOpener for ParquetOpener {
&logical_file_schema,
));
- // Apply literal replacements to projection and predicate
let mut projection = self.projection.clone();
let mut predicate = self.predicate.clone();
if !literal_columns.is_empty() {
@@ -207,444 +535,534 @@ impl FileOpener for ParquetOpener {
.transpose()?;
}
- let reorder_predicates = self.reorder_filters;
- let pushdown_filters = self.pushdown_filters;
- let force_filter_selections = self.force_filter_selections;
- let coerce_int96 = self.coerce_int96;
- let enable_bloom_filter = self.enable_bloom_filter;
- let enable_row_group_stats_pruning = self.enable_row_group_stats_pruning;
- let limit = self.limit;
- let parquet_file_reader_factory = Arc::clone(&self.parquet_file_reader_factory);
- let partition_index = self.partition_index;
- let metrics = self.metrics.clone();
-
let predicate_creation_errors = MetricBuilder::new(&self.metrics)
.with_category(MetricCategory::Rows)
.global_counter("num_predicate_creation_errors");
- let expr_adapter_factory = Arc::clone(&self.expr_adapter_factory);
-
- let enable_page_index = self.enable_page_index;
- #[cfg(feature = "parquet_encryption")]
- let encryption_context = self.get_encryption_context();
- let max_predicate_cache_size = self.max_predicate_cache_size;
+ // Apply literal replacements to projection and predicate
+ let file_pruner = predicate
+ .as_ref()
+ .filter(|p| is_dynamic_physical_expr(p) || partitioned_file.has_statistics())
+ .and_then(|p| {
+ FilePruner::try_new(
+ Arc::clone(p),
+ &logical_file_schema,
+ &partitioned_file,
+ predicate_creation_errors.clone(),
+ )
+ });
+
+ Ok(PreparedParquetOpen {
+ partition_index: self.partition_index,
+ partitioned_file,
+ file_range,
+ extensions,
+ file_name,
+ file_metrics,
+ baseline_metrics,
+ file_pruner,
+ metadata_size_hint,
+ metrics: self.metrics.clone(),
+ parquet_file_reader_factory: Arc::clone(&self.parquet_file_reader_factory),
+ async_file_reader,
+ batch_size: self.batch_size,
+ logical_file_schema: Arc::clone(&logical_file_schema),
+ physical_file_schema: logical_file_schema,
+ output_schema,
+ projection,
+ predicate,
+ reorder_predicates: self.reorder_filters,
+ pushdown_filters: self.pushdown_filters,
+ force_filter_selections: self.force_filter_selections,
+ enable_page_index: self.enable_page_index,
+ enable_bloom_filter: self.enable_bloom_filter,
+ enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
+ limit: self.limit,
+ coerce_int96: self.coerce_int96,
+ expr_adapter_factory: Arc::clone(&self.expr_adapter_factory),
+ predicate_creation_errors,
+ max_predicate_cache_size: self.max_predicate_cache_size,
+ reverse_row_groups: self.reverse_row_groups,
+ preserve_order: self.preserve_order,
+ #[cfg(feature = "parquet_encryption")]
+ file_decryption_properties: None,
+ })
+ }
+}
- let reverse_row_groups = self.reverse_row_groups;
- let preserve_order = self.preserve_order;
+impl PreparedParquetOpen {
+ /// Attempt file-level pruning before any metadata is loaded.
+ ///
+ /// Returns `None` if the file can be skipped completely.
+ fn prune_file(mut self) -> Result<Option<Self>> {
+ // Prune this file using the file level statistics and partition values.
+ // Since dynamic filters may have been updated since planning it is possible that we are able
+ // to prune files now that we couldn't prune at planning time.
+ // It is assumed that there is no point in doing pruning here if the predicate is not dynamic,
+ // as it would have been done at planning time.
+ // We'll also check this after every record batch we read,
+ // and if at some point we are able to prove we can prune the file using just the file level statistics
+ // we can end the stream early.
+ //
+ // Make a FilePruner only if there is either
+ // 1. a dynamic expr in the predicate
+ // 2. the file has file-level statistics.
+ //
+ // File-level statistics may prune the file without loading
+ // any row groups or metadata.
+ //
+ // Dynamic filters may prune the file after initial
+ // planning, as the dynamic filter is updated during
+ // execution.
+ //
+ // The case where there is a dynamic filter but no
+ // statistics corresponds to a dynamic filter that
+ // references partition columns. While rare, this is possible
+ // e.g. `select * from table order by partition_col limit
+ // 10` could hit this condition.
+ if let Some(file_pruner) = &mut self.file_pruner
+ && file_pruner.should_prune()?
+ {
+ self.file_metrics
+ .files_ranges_pruned_statistics
+ .add_pruned(1);
+ return Ok(None);
+ }
- Ok(Box::pin(async move {
- #[cfg(feature = "parquet_encryption")]
- let file_decryption_properties = encryption_context
- .get_file_decryption_properties(&file_location)
- .await?;
+ self.file_metrics
+ .files_ranges_pruned_statistics
+ .add_matched(1);
+ Ok(Some(self))
+ }
- // ---------------------------------------------
- // Step: try to prune the current file partition
- // ---------------------------------------------
-
- // Prune this file using the file level statistics and partition values.
- // Since dynamic filters may have been updated since planning it is possible that we are able
- // to prune files now that we couldn't prune at planning time.
- // It is assumed that there is no point in doing pruning here if the predicate is not dynamic,
- // as it would have been done at planning time.
- // We'll also check this after every record batch we read,
- // and if at some point we are able to prove we can prune the file using just the file level statistics
- // we can end the stream early.
- let mut file_pruner = predicate
- .as_ref()
- .filter(|p| {
- // Make a FilePruner only if there is either
- // 1. a dynamic expr in the predicate
- // 2. the file has file-level statistics.
- //
- // File-level statistics may prune the file without loading
- // any row groups or metadata.
- //
- // Dynamic filters may prune the file after initial
- // planning, as the dynamic filter is updated during
- // execution.
- //
- // The case where there is a dynamic filter but no
- // statistics corresponds to a dynamic filter that
- // references partition columns. While rare, this is possible
- // e.g. `select * from table order by partition_col limit
- // 10` could hit this condition.
- is_dynamic_physical_expr(p) || partitioned_file.has_statistics()
- })
- .and_then(|p| {
- FilePruner::try_new(
- Arc::clone(p),
- &logical_file_schema,
- &partitioned_file,
- predicate_creation_errors.clone(),
- )
- });
-
- if let Some(file_pruner) = &mut file_pruner
- && file_pruner.should_prune()?
- {
- // Return an empty stream immediately to skip the work of setting up the actual stream
- file_metrics.files_ranges_pruned_statistics.add_pruned(1);
- return Ok(futures::stream::empty().boxed());
- }
+ /// Load parquet metadata after file-level pruning is complete.
+ async fn load(mut self) -> Result<MetadataLoadedParquetOpen> {
+ // Don't load the page index yet. Since it is not stored inline in
+ // the footer, loading the page index if it is not needed will do
+ // unnecessary I/O. We decide later if it is needed to evaluate the
+ // pruning predicates. Thus default to not requesting it from the
+ // underlying reader.
+ let options =
+ ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Skip);
+ #[cfg(feature = "parquet_encryption")]
+ let mut options = options;
+ #[cfg(feature = "parquet_encryption")]
+ if let Some(fd_val) = &self.file_decryption_properties {
+ options = options.with_file_decryption_properties(Arc::clone(fd_val));
+ }
- file_metrics.files_ranges_pruned_statistics.add_matched(1);
+ let mut metadata_timer = self.file_metrics.metadata_load_time.timer();
+ // Begin by loading the metadata from the underlying reader (note
+ // the returned metadata may actually include page indexes as some
+ // readers may return page indexes even when not requested -- for
+ // example when they are cached)
+ let reader_metadata =
+ ArrowReaderMetadata::load_async(&mut self.async_file_reader, options.clone())
+ .await?;
+ metadata_timer.stop();
+ drop(metadata_timer);
+
+ Ok(MetadataLoadedParquetOpen {
+ prepared: self,
+ reader_metadata,
+ options,
+ })
+ }
+}
- // --------------------------------------------------------
- // Step: fetch Parquet metadata (and optionally page index)
- // --------------------------------------------------------
+impl MetadataLoadedParquetOpen {
+ /// Prepare file-schema coercions and pruning predicates once metadata is
+ /// loaded.
+ fn prepare_filters(self) -> Result<FiltersPreparedParquetOpen> {
+ let MetadataLoadedParquetOpen {
+ mut prepared,
+ mut reader_metadata,
+ mut options,
+ } = self;
+
+ // Note about schemas: we are actually dealing with **3 different schemas** here:
+ // - The table schema as defined by the TableProvider.
+ // This is what the user sees, what they get when they `SELECT * FROM table`, etc.
+ // - The logical file schema: this is the table schema minus any hive partition columns and projections.
+ // This is what the physical file schema is coerced to.
+ // - The physical file schema: this is the schema that the arrow-rs
+ // parquet reader will actually produce.
+ let mut physical_file_schema = Arc::clone(reader_metadata.schema());
+
+ // The schema loaded from the file may not be the same as the
+ // desired schema (for example if we want to instruct the parquet
+ // reader to read strings using Utf8View instead). Update if necessary
+ if let Some(merged) = apply_file_schema_type_coercions(
+ &prepared.logical_file_schema,
+ &physical_file_schema,
+ ) {
+ physical_file_schema = Arc::new(merged);
+ options = options.with_schema(Arc::clone(&physical_file_schema));
+ reader_metadata = ArrowReaderMetadata::try_new(
+ Arc::clone(reader_metadata.metadata()),
+ options.clone(),
+ )?;
+ }
- // Don't load the page index yet. Since it is not stored inline in
- // the footer, loading the page index if it is not needed will do
- // unnecessary I/O. We decide later if it is needed to evaluate the
- // pruning predicates. Thus default to not requesting it from the
- // underlying reader.
- let mut options =
- ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Skip);
- #[cfg(feature = "parquet_encryption")]
- if let Some(fd_val) = file_decryption_properties {
- options = options.with_file_decryption_properties(Arc::clone(&fd_val));
- }
- let mut metadata_timer = file_metrics.metadata_load_time.timer();
-
- // Begin by loading the metadata from the underlying reader (note
- // the returned metadata may actually include page indexes as some
- // readers may return page indexes even when not requested -- for
- // example when they are cached)
- let mut reader_metadata =
- ArrowReaderMetadata::load_async(&mut async_file_reader, options.clone())
- .await?;
-
- // Note about schemas: we are actually dealing with **3 different schemas** here:
- // - The table schema as defined by the TableProvider.
- // This is what the user sees, what they get when they `SELECT * FROM table`, etc.
- // - The logical file schema: this is the table schema minus any hive partition columns and projections.
- // This is what the physical file schema is coerced to.
- // - The physical file schema: this is the schema that the arrow-rs
- // parquet reader will actually produce.
- let mut physical_file_schema = Arc::clone(reader_metadata.schema());
-
- // The schema loaded from the file may not be the same as the
- // desired schema (for example if we want to instruct the parquet
- // reader to read strings using Utf8View instead). Update if necessary
- if let Some(merged) = apply_file_schema_type_coercions(
- &logical_file_schema,
+ if let Some(ref coerce) = prepared.coerce_int96
+ && let Some(merged) = coerce_int96_to_resolution(
+ reader_metadata.parquet_schema(),
&physical_file_schema,
- ) {
- physical_file_schema = Arc::new(merged);
- options = options.with_schema(Arc::clone(&physical_file_schema));
- reader_metadata = ArrowReaderMetadata::try_new(
- Arc::clone(reader_metadata.metadata()),
- options.clone(),
- )?;
- }
+ coerce,
+ )
+ {
+ physical_file_schema = Arc::new(merged);
+ options = options.with_schema(Arc::clone(&physical_file_schema));
+ reader_metadata = ArrowReaderMetadata::try_new(
+ Arc::clone(reader_metadata.metadata()),
+ options.clone(),
+ )?;
+ }
- if let Some(ref coerce) = coerce_int96
- && let Some(merged) = coerce_int96_to_resolution(
- reader_metadata.parquet_schema(),
- &physical_file_schema,
- coerce,
- )
- {
- physical_file_schema = Arc::new(merged);
- options = options.with_schema(Arc::clone(&physical_file_schema));
- reader_metadata = ArrowReaderMetadata::try_new(
- Arc::clone(reader_metadata.metadata()),
- options.clone(),
- )?;
- }
+ // Adapt the projection & filter predicate to the physical file schema.
+ // This evaluates missing columns and inserts any necessary casts.
+ // After rewriting to the file schema, further simplifications may be possible.
+ // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`
+ // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).
+ // Additionally, if any casts were inserted we can move casts from the column to the literal side:
+ // `CAST(col AS INT) = 5` can become `col = CAST(5 AS <col type>)`, which can be evaluated statically.
+ //
+ // When the schemas are identical and there is no predicate, the
+ // rewriter is a no-op: column indices already match (partition
+ // columns are appended after file columns in the table schema),
+ // types are the same, and there are no missing columns. Skip the
+ // tree walk entirely in that case.
+ let needs_rewrite = prepared.predicate.is_some()
+ || prepared.logical_file_schema != physical_file_schema;
+ if needs_rewrite {
+ let rewriter = prepared.expr_adapter_factory.create(
+ Arc::clone(&prepared.logical_file_schema),
+ Arc::clone(&physical_file_schema),
+ )?;
+ let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);
+ prepared.predicate = prepared
+ .predicate
+ .map(|p| simplifier.simplify(rewriter.rewrite(p)?))
+ .transpose()?;
+ prepared.projection = prepared
+ .projection
+ .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;
+ }
+ prepared.physical_file_schema = Arc::clone(&physical_file_schema);
- // Adapt the projection & filter predicate to the physical file schema.
- // This evaluates missing columns and inserts any necessary casts.
- // After rewriting to the file schema, further simplifications may be possible.
- // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`
- // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).
- // Additionally, if any casts were inserted we can move casts from the column to the literal side:
- // `CAST(col AS INT) = 5` can become `col = CAST(5 AS <col type>)`, which can be evaluated statically.
- //
- // When the schemas are identical and there is no predicate, the
- // rewriter is a no-op: column indices already match (partition
- // columns are appended after file columns in the table schema),
- // types are the same, and there are no missing columns. Skip the
- // tree walk entirely in that case.
- let needs_rewrite =
- predicate.is_some() || logical_file_schema != physical_file_schema;
- if needs_rewrite {
- let rewriter = expr_adapter_factory.create(
- Arc::clone(&logical_file_schema),
- Arc::clone(&physical_file_schema),
- )?;
- let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);
- predicate = predicate
- .map(|p| simplifier.simplify(rewriter.rewrite(p)?))
- .transpose()?;
- // Adapt projections to the physical file schema as well
- projection = projection
- .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;
- }
+ // Build predicates for this specific file
+ let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(
+ prepared.predicate.as_ref(),
+ &physical_file_schema,
+ &prepared.predicate_creation_errors,
+ );
- // Build predicates for this specific file
- let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(
- predicate.as_ref(),
- &physical_file_schema,
- &predicate_creation_errors,
- );
+ Ok(FiltersPreparedParquetOpen {
+ loaded: MetadataLoadedParquetOpen {
+ prepared,
+ reader_metadata,
+ options,
+ },
+ pruning_predicate,
+ page_pruning_predicate,
+ })
+ }
+}
- // The page index is not stored inline in the parquet footer so the
- // code above may not have read the page index structures yet. If we
- // need them for reading and they aren't yet loaded, we need to load them now.
- if should_enable_page_index(enable_page_index, &page_pruning_predicate) {
- reader_metadata = load_page_index(
- reader_metadata,
- &mut async_file_reader,
- // Since we're manually loading the page index the option here should not matter but we pass it in for consistency
- options.with_page_index_policy(PageIndexPolicy::Optional),
- )
- .await?;
- }
+impl FiltersPreparedParquetOpen {
+ /// Load the page index if pruning requires it and metadata did not include it.
+ async fn load_page_index(mut self) -> Result<Self> {
+ // The page index is not stored inline in the parquet footer so the
+ // metadata load above may not have read the page index structures yet.
+ // If we need them for reading and they aren't yet loaded, we need to
+ // load them now.
+ if should_enable_page_index(
+ self.loaded.prepared.enable_page_index,
+ &self.page_pruning_predicate,
+ ) {
+ self.loaded.reader_metadata = load_page_index(
+ self.loaded.reader_metadata,
+ &mut self.loaded.prepared.async_file_reader,
+ self.loaded
+ .options
+ .clone()
+ .with_page_index_policy(PageIndexPolicy::Optional),
+ )
+ .await?;
+ }
- metadata_timer.stop();
-
- // ------------------------------------------------------------
- // Step: prune row groups by range, predicate and bloom filter
- // ------------------------------------------------------------
-
- // Determine which row groups to actually read. The idea is to skip
- // as many row groups as possible based on the metadata and query
- let file_metadata = Arc::clone(reader_metadata.metadata());
- let pruning_pred = pruning_predicate.as_ref().map(|p| p.as_ref());
- let rg_metadata = file_metadata.row_groups();
- // track which row groups to actually read
- let access_plan =
- create_initial_plan(&file_name, extensions, rg_metadata.len())?;
- let mut row_groups = RowGroupAccessPlanFilter::new(access_plan);
- // if there is a range restricting what parts of the file to read
- if let Some(range) = file_range.as_ref() {
- row_groups.prune_by_range(rg_metadata, range);
- }
+ Ok(self)
+ }
- // If there is a predicate that can be evaluated against the metadata
- if let Some(pruning_pred) = pruning_pred.as_ref() {
- if enable_row_group_stats_pruning {
- row_groups.prune_by_statistics(
- &physical_file_schema,
- reader_metadata.parquet_schema(),
- rg_metadata,
- pruning_pred,
- &file_metrics,
- );
- } else {
- // Update metrics: statistics unavailable, so all row groups are
- // matched (not pruned)
- file_metrics
- .row_groups_pruned_statistics
- .add_matched(row_groups.remaining_row_group_count());
- }
+ /// Prune row groups using file ranges and parquet metadata.
+ fn prune_row_groups(self) -> Result<RowGroupsPrunedParquetOpen> {
+ let loaded = &self.loaded;
+ let prepared = &loaded.prepared;
+ let file_metadata = Arc::clone(loaded.reader_metadata.metadata());
+ let rg_metadata = file_metadata.row_groups();
+
+ // Determine which row groups to actually read. The idea is to skip
+ // as many row groups as possible based on the metadata and query
+ let mut row_groups = RowGroupAccessPlanFilter::new(create_initial_plan(
+ &prepared.file_name,
+ prepared.extensions.clone(),
+ rg_metadata.len(),
+ )?);
+
+ // If there is a range restricting what parts of the file to read
+ if let Some(range) = prepared.file_range.as_ref() {
+ row_groups.prune_by_range(rg_metadata, range);
+ }
- if enable_bloom_filter && !row_groups.is_empty() {
- // Use the existing reader for bloom filter I/O;
- // replace with a fresh reader for decoding below.
- let bf_reader = std::mem::replace(
- &mut async_file_reader,
- parquet_file_reader_factory.create_reader(
- partition_index,
- partitioned_file.clone(),
- metadata_size_hint,
- &metrics,
- )?,
- );
- let mut bf_builder =
- ParquetRecordBatchStreamBuilder::new_with_metadata(
- bf_reader,
- reader_metadata.clone(),
- );
- row_groups
- .prune_by_bloom_filters(
- &physical_file_schema,
- &mut bf_builder,
- pruning_pred,
- &file_metrics,
- )
- .await;
- } else {
- // Update metrics: bloom filter unavailable, so all row groups are
- // matched (not pruned)
- file_metrics
- .row_groups_pruned_bloom_filter
- .add_matched(row_groups.remaining_row_group_count());
- }
+ // If there is a predicate that can be evaluated against the metadata
+ if let Some(predicate) = self.pruning_predicate.as_ref().map(|p| p.as_ref()) {
+ if prepared.enable_row_group_stats_pruning {
+ row_groups.prune_by_statistics(
+ &prepared.physical_file_schema,
+ loaded.reader_metadata.parquet_schema(),
+ rg_metadata,
+ predicate,
+ &prepared.file_metrics,
+ );
} else {
- // Update metrics: no predicate, so all row groups are matched (not pruned)
- let n_remaining_row_groups = row_groups.remaining_row_group_count();
- file_metrics
+ // Update metrics: statistics unavailable, so all row groups are
+ // matched (not pruned)
+ prepared
+ .file_metrics
.row_groups_pruned_statistics
- .add_matched(n_remaining_row_groups);
- file_metrics
- .row_groups_pruned_bloom_filter
- .add_matched(n_remaining_row_groups);
- }
-
- // Prune by limit if limit is set and limit order is not sensitive
- if let (Some(limit), false) = (limit, preserve_order) {
- row_groups.prune_by_limit(limit, rg_metadata, &file_metrics);
+ .add_matched(row_groups.remaining_row_group_count());
}
- // --------------------------------------------------------
- // Step: prune pages from the kept row groups
- //
- let mut access_plan = row_groups.build();
- // page index pruning: if all data on individual pages can
- // be ruled using page metadata, rows from other columns
- // with that range can be skipped as well
- // --------------------------------------------------------
- if enable_page_index
- && !access_plan.is_empty()
- && let Some(p) = page_pruning_predicate
- {
- access_plan = p.prune_plan_with_page_index(
- access_plan,
- &physical_file_schema,
- reader_metadata.parquet_schema(),
- file_metadata.as_ref(),
- &file_metrics,
- );
+ if !prepared.enable_bloom_filter || row_groups.is_empty() {
+ // Update metrics: bloom filter unavailable, so all row groups are
+ // matched (not pruned)
+ prepared
+ .file_metrics
+ .row_groups_pruned_bloom_filter
+ .add_matched(row_groups.remaining_row_group_count());
}
+ } else {
+ // Update metrics: no predicate, so all row groups are matched (not pruned)
+ let remaining = row_groups.remaining_row_group_count();
+ prepared
+ .file_metrics
+ .row_groups_pruned_statistics
+ .add_matched(remaining);
+ prepared
+ .file_metrics
+ .row_groups_pruned_bloom_filter
+ .add_matched(remaining);
+ }
- // Prepare the access plan (extract row groups and row selection)
- let mut prepared_plan = access_plan.prepare(rg_metadata)?;
-
- // ----------------------------------------------------------
- // Step: potentially reverse the access plan for performance.
- // See `ParquetSource::try_pushdown_sort` for the rationale.
- // ----------------------------------------------------------
- if reverse_row_groups {
- prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;
- }
+ Ok(RowGroupsPrunedParquetOpen {
+ prepared: self,
+ row_groups,
+ })
+ }
+}
- if prepared_plan.row_group_indexes.is_empty() {
- return Ok(futures::stream::empty().boxed());
- }
+impl RowGroupsPrunedParquetOpen {
+ /// Apply bloom filter pruning when it is enabled and a pruning predicate exists.
+ async fn prune_bloom_filters(mut self) -> Result<Self> {
+ if let Some(predicate) =
+ self.prepared.pruning_predicate.as_ref().map(|p| p.as_ref())
+ && self.prepared.loaded.prepared.enable_bloom_filter
+ && !self.row_groups.is_empty()
+ {
+ // Use the existing reader for bloom filter I/O;
+ // replace with a fresh reader for decoding below.
+ let reader_metadata = self.prepared.loaded.reader_metadata.clone();
+ let replacement_reader = {
+ let prepared = &self.prepared.loaded.prepared;
+ prepared.parquet_file_reader_factory.create_reader(
+ prepared.partition_index,
+ prepared.partitioned_file.clone(),
+ prepared.metadata_size_hint,
+ &prepared.metrics,
+ )?
+ };
- // ---------------------------------------------------------
- // Step: construct builder for the final RecordBatch stream
- // ---------------------------------------------------------
-
- let mut builder =
- ParquetPushDecoderBuilder::new_with_metadata(reader_metadata.clone())
- .with_batch_size(batch_size);
-
- // ---------------------------------------------------------------------
- // Step: optionally add row filter to the builder
- //
- // Row filter is used for late materialization in parquet decoding, see
- // `row_filter` for details.
- // ---------------------------------------------------------------------
-
- // Filter pushdown: evaluate predicates during scan
- if let Some(predicate) =
- pushdown_filters.then_some(predicate.as_ref()).flatten()
- {
- let row_filter = row_filter::build_row_filter(
+ let prepared = &mut self.prepared.loaded.prepared;
+ let mut builder = ParquetRecordBatchStreamBuilder::new_with_metadata(
+ mem::replace(&mut prepared.async_file_reader, replacement_reader),
+ reader_metadata,
+ );
+ self.row_groups
+ .prune_by_bloom_filters(
+ &prepared.physical_file_schema,
+ &mut builder,
predicate,
- &physical_file_schema,
- file_metadata.as_ref(),
- reorder_predicates,
- &file_metrics,
- );
-
- match row_filter {
- Ok(Some(filter)) => {
- builder = builder.with_row_filter(filter);
- }
- Ok(None) => {}
- Err(e) => {
- debug!(
- "Ignoring error building row filter for '{predicate:?}': {e}"
- );
- }
- };
- };
- if force_filter_selections {
- builder =
- builder.with_row_selection_policy(RowSelectionPolicy::Selectors);
- }
+ &prepared.file_metrics,
+ )
+ .await;
+ }
- // Apply the prepared plan to the builder
- if let Some(row_selection) = prepared_plan.row_selection {
- builder = builder.with_row_selection(row_selection);
- }
- builder = builder.with_row_groups(prepared_plan.row_group_indexes);
+ Ok(self)
+ }
- if let Some(limit) = limit {
- builder = builder.with_limit(limit)
- }
+ /// Build the final parquet stream once all pruning work is complete.
+ fn build_stream(self) -> Result<BoxStream<'static, Result<RecordBatch>>> {
+ let RowGroupsPrunedParquetOpen {
+ prepared,
+ mut row_groups,
+ } = self;
+ let FiltersPreparedParquetOpen {
+ loaded,
+ pruning_predicate: _,
+ page_pruning_predicate,
+ } = prepared;
+ let MetadataLoadedParquetOpen {
+ prepared,
+ reader_metadata,
+ options: _,
+ } = loaded;
+
+ let file_metadata = Arc::clone(reader_metadata.metadata());
+ let rg_metadata = file_metadata.row_groups();
+
+ // Filter pushdown: evaluate predicates during scan
+ let row_filter = if let Some(predicate) = prepared
+ .pushdown_filters
+ .then_some(prepared.predicate.clone())
+ .flatten()
+ {
+ let row_filter = row_filter::build_row_filter(
+ &predicate,
+ &prepared.physical_file_schema,
+ file_metadata.as_ref(),
+ prepared.reorder_predicates,
+ &prepared.file_metrics,
+ );
- if let Some(max_predicate_cache_size) = max_predicate_cache_size {
- builder = builder.with_max_predicate_cache_size(max_predicate_cache_size);
+ match row_filter {
+ Ok(Some(filter)) => Some(filter),
+ Ok(None) => None,
+ Err(e) => {
+ debug!("Ignoring error building row filter for '{predicate:?}': {e}");
+ None
+ }
}
+ } else {
+ None
+ };
- // metrics from the arrow reader itself
- let arrow_reader_metrics = ArrowReaderMetrics::enabled();
+ // Prune by limit if limit is set and limit order is not sensitive
+ if let (Some(limit), false) = (prepared.limit, prepared.preserve_order) {
+ row_groups.prune_by_limit(limit, rg_metadata, &prepared.file_metrics);
+ }
- let read_plan = build_projection_read_plan(
- projection.expr_iter(),
- &physical_file_schema,
+ // Page index pruning: if all data on individual pages can
+ // be ruled using page metadata, rows from other columns
+ // with that range can be skipped as well.
+ let mut access_plan = row_groups.build();
+ if prepared.enable_page_index
+ && !access_plan.is_empty()
+ && let Some(page_pruning_predicate) = page_pruning_predicate
+ {
+ access_plan = page_pruning_predicate.prune_plan_with_page_index(
+ access_plan,
+ &prepared.physical_file_schema,
reader_metadata.parquet_schema(),
+ file_metadata.as_ref(),
+ &prepared.file_metrics,
);
+ }
+
+ // Prepare the access plan (extract row groups and row selection)
+ let mut prepared_plan = access_plan.prepare(rg_metadata)?;
+
+ // Potentially reverse the access plan for performance.
+ // See `ParquetSource::try_pushdown_sort` for the rationale.
+ if prepared.reverse_row_groups {
+ prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;
+ }
+
+ let arrow_reader_metrics = ArrowReaderMetrics::enabled();
+ let read_plan = build_projection_read_plan(
+ prepared.projection.expr_iter(),
+ &prepared.physical_file_schema,
+ reader_metadata.parquet_schema(),
+ );
- let decoder = builder
+ let mut decoder_builder =
+ ParquetPushDecoderBuilder::new_with_metadata(reader_metadata)
.with_projection(read_plan.projection_mask)
- .with_metrics(arrow_reader_metrics.clone())
- .build()?;
-
- let files_ranges_pruned_statistics =
- file_metrics.files_ranges_pruned_statistics.clone();
- let predicate_cache_inner_records =
- file_metrics.predicate_cache_inner_records.clone();
- let predicate_cache_records = file_metrics.predicate_cache_records.clone();
-
- // Rebase column indices to match the narrowed stream schema.
- // The projection expressions have indices based on physical_file_schema,
- // but the stream only contains the columns selected by the ProjectionMask.
- let stream_schema = read_plan.projected_schema;
- let replace_schema = stream_schema != output_schema;
- let projection = projection
- .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;
- let projector = projection.make_projector(&stream_schema)?;
- let stream = futures::stream::unfold(
- PushDecoderStreamState {
- decoder,
- reader: async_file_reader,
- projector,
- output_schema,
- replace_schema,
- arrow_reader_metrics,
- predicate_cache_inner_records,
- predicate_cache_records,
- baseline_metrics,
- },
- |mut state| async move {
- let result = state.transition().await;
- result.map(|r| (r, state))
- },
+ .with_batch_size(prepared.batch_size)
+ .with_metrics(arrow_reader_metrics.clone());
+
+ if let Some(row_filter) = row_filter {
+ decoder_builder = decoder_builder.with_row_filter(row_filter);
+ }
+ if prepared.force_filter_selections {
+ decoder_builder =
+ decoder_builder.with_row_selection_policy(RowSelectionPolicy::Selectors);
+ }
+ if let Some(row_selection) = prepared_plan.row_selection {
+ decoder_builder = decoder_builder.with_row_selection(row_selection);
+ }
+ decoder_builder =
+ decoder_builder.with_row_groups(prepared_plan.row_group_indexes);
+ if let Some(limit) = prepared.limit {
+ decoder_builder = decoder_builder.with_limit(limit);
+ }
+ if let Some(max_predicate_cache_size) = prepared.max_predicate_cache_size {
+ decoder_builder =
+ decoder_builder.with_max_predicate_cache_size(max_predicate_cache_size);
+ }
+
+ let decoder = decoder_builder.build()?;
+
+ let predicate_cache_inner_records =
+ prepared.file_metrics.predicate_cache_inner_records.clone();
+ let predicate_cache_records =
+ prepared.file_metrics.predicate_cache_records.clone();
+
+ // Check if we need to replace the schema to handle things like differing nullability or metadata.
+ // See note below about file vs. output schema.
+ let stream_schema = read_plan.projected_schema;
+ let replace_schema = stream_schema != prepared.output_schema;
+
+ // Rebase column indices to match the narrowed stream schema.
+ // The projection expressions have indices based on physical_file_schema,
+ // but the stream only contains the columns selected by the ProjectionMask.
+ let projection = prepared
+ .projection
+ .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;
+ let projector = projection.make_projector(&stream_schema)?;
+ let output_schema = Arc::clone(&prepared.output_schema);
+ let files_ranges_pruned_statistics =
+ prepared.file_metrics.files_ranges_pruned_statistics.clone();
+ let stream = futures::stream::unfold(
+ PushDecoderStreamState {
+ decoder,
+ reader: prepared.async_file_reader,
+ projector,
+ output_schema,
+ replace_schema,
+ arrow_reader_metrics,
+ predicate_cache_inner_records,
+ predicate_cache_records,
+ baseline_metrics: prepared.baseline_metrics,
+ },
+ |mut state| async move {
+ let result = state.transition().await;
+ result.map(|r| (r, state))
+ },
+ )
+ .fuse();
+
+ // Wrap the stream so a dynamic filter can stop the file scan early.
+ if let Some(file_pruner) = prepared.file_pruner {
+ let stream = stream.boxed();
+ Ok(EarlyStoppingStream::new(
+ stream,
+ file_pruner,
+ files_ranges_pruned_statistics,
)
- .fuse();
-
- // ----------------------------------------------------------------------
- // Step: wrap the stream so a dynamic filter can stop the file scan early
- // ----------------------------------------------------------------------
- if let Some(file_pruner) = file_pruner {
- let boxed_stream = stream.boxed();
- Ok(EarlyStoppingStream::new(
- boxed_stream,
- file_pruner,
- files_ranges_pruned_statistics,
- )
- .boxed())
- } else {
- Ok(stream.boxed())
- }
- }))
+ .boxed())
+ } else {
+ Ok(stream.boxed())
+ }
}
}
@@ -719,7 +1137,15 @@ impl PushDecoderStreamState {
fn project_batch(&self, batch: &RecordBatch) -> Result<RecordBatch> {
let mut batch = self.projector.project_batch(batch)?;
if self.replace_schema {
- let (_schema, arrays, num_rows) = batch.into_parts();
+ // Ensure the output batch has the expected schema.
+ // This handles things like schema level and field level metadata, which may not be present
+ // in the physical file schema.
+ // It is also possible for nullability to differ; some writers create files with
+ // OPTIONAL fields even when there are no nulls in the data.
+ // In these cases it may make sense for the logical schema to be `NOT NULL`.
+ // RecordBatch::try_new_with_options checks that if the schema is NOT NULL
+ // the array cannot contain nulls, amongst other checks.
+ let (_stream_schema, arrays, num_rows) = batch.into_parts();
let options = RecordBatchOptions::new().with_row_count(Some(num_rows));
batch = RecordBatch::try_new_with_options(
Arc::clone(&self.output_schema),
diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rs
index 4291c9af76..482bf8dced 100644
--- a/datafusion/datasource-parquet/src/reader.rs
+++ b/datafusion/datasource-parquet/src/reader.rs
@@ -37,7 +37,7 @@ use std::fmt::Debug;
use std::ops::Range;
use std::sync::Arc;
-/// Interface for reading parquet files.
+/// Interface for reading Apache Parquet files.
///
/// The combined implementations of [`ParquetFileReaderFactory`] and
/// [`AsyncFileReader`] can be used to provide custom data access operations