Apache DataFusion SQL Query Engine
Refactor parquet datasource into an explicit state machine (#21190)
Andrew Lambf830ee39df83
diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex 0e137a706f..9a907f4118 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -15,6 +15,10 @@// specific language governing permissions and limitations// under the License.+//! DataFusion Parquet Reader: [`ParquetSource`]+//!+//! [`ParquetSource`]: source::ParquetSource+// Make sure fast / cheap clones on Arc are explicit:// https://github.com/apache/datafusion/issues/11143#![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 2990f43165..466f631c53 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-//! [`ParquetOpener`] for opening Parquet files+//! [`ParquetOpener`] state machine for opening Parquet filesuse crate::page_filter::PagePruningAccessPlanFilter;use crate::row_filter::build_projection_read_plan;@@ -25,18 +25,19 @@ use crate::{apply_file_schema_type_coercions, coerce_int96_to_resolution, row_filter,};use arrow::array::{RecordBatch, RecordBatchOptions};-use arrow::datatypes::{DataType, Schema};+use arrow::datatypes::DataType;use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};use datafusion_physical_expr::projection::{ProjectionExprs, Projector};use datafusion_physical_expr::utils::reassign_expr_columns;use datafusion_physical_expr_adapter::replace_columns_with_literals;-use parquet::errors::ParquetError;use std::collections::HashMap;+use std::future::Future;+use std::mem;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};-use arrow::datatypes::{SchemaRef, TimeUnit};+use arrow::datatypes::{Schema, SchemaRef, TimeUnit};use datafusion_common::encryption::FileDecryptionProperties;use datafusion_common::stats::Precision;use datafusion_common::{@@ -58,7 +59,9 @@ use datafusion_pruning::{FilePruner, PruningPredicate, build_pruning_predicate};use datafusion_common::config::EncryptionFactoryOptions;#[cfg(feature = "parquet_encryption")]use datafusion_execution::parquet_encryption::EncryptionFactory;-use futures::{Stream, StreamExt, ready};+use futures::{+ FutureExt, Stream, StreamExt, future::BoxFuture, ready, stream::BoxStream,+};use log::debug;use parquet::DecodeResult;use parquet::arrow::ParquetRecordBatchStreamBuilder;@@ -68,9 +71,14 @@ use parquet::arrow::arrow_reader::{};use parquet::arrow::async_reader::AsyncFileReader;use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder};+use parquet::errors::ParquetError;use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};-/// Implements [`FileOpener`] for a parquet file+/// Entry point for opening a Parquet file+///+/// Reading a Parquet file is a multi-stage process, with multiple CPU-intensive+/// steps interspersed with I/O steps. The code in this module implements the steps+/// as an explicit state machine -- see [`ParquetOpenState`] for details.pub(super) struct ParquetOpener {/// Execution partition indexpub(crate) partition_index: usize,@@ -126,15 +134,338 @@ pub(super) struct ParquetOpener {pub reverse_row_groups: bool,}+/// States for [`ParquetOpenFuture`]+///+/// These states correspond to the steps required to read and apply various+/// filter operations.+///+/// States whose names beginning with `Load` represent waiting on IO to resolve+///+/// ```text+/// Start+/// |+/// v+/// [LoadEncryption]?+/// |+/// v+/// PruneFile+/// |+/// v+/// LoadMetadata+/// |+/// v+/// PrepareFilters+/// |+/// v+/// LoadPageIndex+/// |+/// v+/// PruneWithStatistics+/// |+/// v+/// PruneWithBloomFilters+/// |+/// v+/// BuildStream+/// |+/// v+/// Done+/// ```+///+/// Note: `LoadEncryption` is only present when the `parquet_encryption` feature is+/// enabled. All other states are always visited in the order shown above,+/// though any async state may return `Poll::Pending` and then resume later.+enum ParquetOpenState {+ Start {+ prepared: Box<PreparedParquetOpen>,+ #[cfg(feature = "parquet_encryption")]+ encryption_context: Arc<EncryptionContext>,+ },+ /// Loading encryption footers+ #[cfg(feature = "parquet_encryption")]+ LoadEncryption(BoxFuture<'static, Result<Box<PreparedParquetOpen>>>),+ /// Try to prune file using only file-level statistics and partition+ /// values before loading any parquet metadata+ PruneFile(Box<PreparedParquetOpen>),+ /// Loading Parquet metadata (in footer)+ LoadMetadata(BoxFuture<'static, Result<MetadataLoadedParquetOpen>>),+ /// Specialize any filters for the actual file schema (only known after+ /// metadata is loaded)+ PrepareFilters(Box<MetadataLoadedParquetOpen>),+ /// Loading [Parquet Page Index](https://parquet.apache.org/docs/file-format/pageindex/)+ LoadPageIndex(BoxFuture<'static, Result<FiltersPreparedParquetOpen>>),+ /// Pruning Row Groups+ PruneWithStatistics(Box<FiltersPreparedParquetOpen>),+ /// Pruning with Bloom Filters+ ///+ /// TODO: split state as this currently does both I/O and CPU work+ PruneWithBloomFilters(BoxFuture<'static, Result<RowGroupsPrunedParquetOpen>>),+ /// Builds the final reader stream+ ///+ /// TODO: split state as this currently does both I/O and CPU work.+ BuildStream(Box<RowGroupsPrunedParquetOpen>),+ /// Terminal state: the final opened stream is ready to return.+ Ready(BoxStream<'static, Result<RecordBatch>>),+ /// Terminal state: reading complete+ Done,+}++struct PreparedParquetOpen {+ partition_index: usize,+ partitioned_file: PartitionedFile,+ file_range: Option<datafusion_datasource::FileRange>,+ extensions: Option<Arc<dyn std::any::Any + Send + Sync>>,+ file_name: String,+ file_metrics: ParquetFileMetrics,+ baseline_metrics: BaselineMetrics,+ file_pruner: Option<FilePruner>,+ metadata_size_hint: Option<usize>,+ metrics: ExecutionPlanMetricsSet,+ parquet_file_reader_factory: Arc<dyn ParquetFileReaderFactory>,+ async_file_reader: Box<dyn AsyncFileReader>,+ batch_size: usize,+ logical_file_schema: SchemaRef,+ physical_file_schema: SchemaRef,+ output_schema: SchemaRef,+ projection: ProjectionExprs,+ predicate: Option<Arc<dyn PhysicalExpr>>,+ reorder_predicates: bool,+ pushdown_filters: bool,+ force_filter_selections: bool,+ enable_page_index: bool,+ enable_bloom_filter: bool,+ enable_row_group_stats_pruning: bool,+ limit: Option<usize>,+ coerce_int96: Option<TimeUnit>,+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,+ predicate_creation_errors: Count,+ max_predicate_cache_size: Option<usize>,+ reverse_row_groups: bool,+ preserve_order: bool,+ #[cfg(feature = "parquet_encryption")]+ file_decryption_properties: Option<Arc<FileDecryptionProperties>>,+}++/// State of [`ParquetOpenState`]+///+/// Result of loading parquet metadata after file-level pruning is complete.+struct MetadataLoadedParquetOpen {+ prepared: PreparedParquetOpen,+ reader_metadata: ArrowReaderMetadata,+ options: ArrowReaderOptions,+}++/// State of [`ParquetOpenState`]+///+/// Pruning Predicate and DataPage pruning information+/// specialized for the files specific schema.+struct FiltersPreparedParquetOpen {+ loaded: MetadataLoadedParquetOpen,+ pruning_predicate: Option<Arc<PruningPredicate>>,+ page_pruning_predicate: Option<Arc<PagePruningAccessPlanFilter>>,+}++/// State of [`ParquetOpenState`]+///+/// Result of CPU-only row-group pruning before optional bloom-filter I/O.+struct RowGroupsPrunedParquetOpen {+ prepared: FiltersPreparedParquetOpen,+ row_groups: RowGroupAccessPlanFilter,+}++/// Implements state machine described in [`ParquetOpenState`]+struct ParquetOpenFuture {+ state: ParquetOpenState,+}++impl ParquetOpenFuture {+ #[cfg(feature = "parquet_encryption")]+ fn new(prepared: PreparedParquetOpen, encryption_context: EncryptionContext) -> Self {+ Self {+ state: ParquetOpenState::Start {+ prepared: Box::new(prepared),+ encryption_context: Arc::new(encryption_context),+ },+ }+ }++ #[cfg(not(feature = "parquet_encryption"))]+ fn new(prepared: PreparedParquetOpen) -> Self {+ Self {+ state: ParquetOpenState::Start {+ prepared: Box::new(prepared),+ },+ }+ }+}++impl ParquetOpenState {+ /// Applies one CPU-only state transition.+ ///+ /// `Load*` states do not transition here and are returned unchanged so the+ /// driver loop can poll their inner futures separately.+ fn transition(self) -> Result<ParquetOpenState> {+ match self {+ ParquetOpenState::Start {+ prepared,+ #[cfg(feature = "parquet_encryption")]+ encryption_context,+ } => {+ #[cfg(feature = "parquet_encryption")]+ {+ let mut prepared = *prepared;+ let future = async move {+ let file_location =+ &prepared.partitioned_file.object_meta.location;+ prepared.file_decryption_properties = encryption_context+ .get_file_decryption_properties(file_location)+ .await?;+ Ok(Box::new(prepared))+ }+ .boxed();+ Ok(ParquetOpenState::LoadEncryption(future))+ }+ #[cfg(not(feature = "parquet_encryption"))]+ {+ Ok(ParquetOpenState::PruneFile(prepared))+ }+ }+ #[cfg(feature = "parquet_encryption")]+ ParquetOpenState::LoadEncryption(future) => {+ Ok(ParquetOpenState::LoadEncryption(future))+ }+ ParquetOpenState::PruneFile(prepared) => {+ let Some(prepared) = (*prepared).prune_file()? else {+ return Ok(ParquetOpenState::Done);+ };+ Ok(ParquetOpenState::LoadMetadata(prepared.load().boxed()))+ }+ ParquetOpenState::LoadMetadata(future) => {+ Ok(ParquetOpenState::LoadMetadata(future))+ }+ ParquetOpenState::PrepareFilters(loaded) => {+ let prepared_filters = loaded.prepare_filters()?;+ Ok(ParquetOpenState::LoadPageIndex(+ prepared_filters.load_page_index().boxed(),+ ))+ }+ ParquetOpenState::LoadPageIndex(future) => {+ Ok(ParquetOpenState::LoadPageIndex(future))+ }+ ParquetOpenState::PruneWithStatistics(prepared) => {+ let prepared_row_groups = prepared.prune_row_groups()?;+ Ok(ParquetOpenState::PruneWithBloomFilters(+ prepared_row_groups.prune_bloom_filters().boxed(),+ ))+ }+ ParquetOpenState::PruneWithBloomFilters(future) => {+ Ok(ParquetOpenState::PruneWithBloomFilters(future))+ }+ ParquetOpenState::BuildStream(prepared) => {+ Ok(ParquetOpenState::Ready(prepared.build_stream()?))+ }+ ParquetOpenState::Ready(stream) => Ok(ParquetOpenState::Ready(stream)),+ ParquetOpenState::Done => {+ panic!("ParquetOpenFuture polled after completion");+ }+ }+ }+}++impl Future for ParquetOpenFuture {+ type Output = Result<BoxStream<'static, Result<RecordBatch>>>;++ fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {+ loop {+ let state = mem::replace(&mut self.state, ParquetOpenState::Done);+ let mut state = state.transition()?;++ match state {+ #[cfg(feature = "parquet_encryption")]+ ParquetOpenState::LoadEncryption(mut future) => {+ state = match future.poll_unpin(cx) {+ Poll::Ready(result) => ParquetOpenState::PruneFile(result?),+ Poll::Pending => {+ self.state = ParquetOpenState::LoadEncryption(future);+ return Poll::Pending;+ }+ };+ }+ ParquetOpenState::LoadMetadata(mut future) => {+ state = match future.poll_unpin(cx) {+ Poll::Ready(result) => {+ ParquetOpenState::PrepareFilters(Box::new(result?))+ }+ Poll::Pending => {+ self.state = ParquetOpenState::LoadMetadata(future);+ return Poll::Pending;+ }+ };+ }+ ParquetOpenState::LoadPageIndex(mut future) => {+ state = match future.poll_unpin(cx) {+ Poll::Ready(result) => {+ ParquetOpenState::PruneWithStatistics(Box::new(result?))+ }+ Poll::Pending => {+ self.state = ParquetOpenState::LoadPageIndex(future);+ return Poll::Pending;+ }+ };+ }+ ParquetOpenState::PruneWithBloomFilters(mut future) => {+ state = match future.poll_unpin(cx) {+ Poll::Ready(result) => {+ ParquetOpenState::BuildStream(Box::new(result?))+ }+ Poll::Pending => {+ self.state = ParquetOpenState::PruneWithBloomFilters(future);+ return Poll::Pending;+ }+ };+ }+ ParquetOpenState::Ready(stream) => {+ return Poll::Ready(Ok(stream));+ }+ ParquetOpenState::Done => {+ return Poll::Ready(Ok(futures::stream::empty().boxed()));+ }++ // For all other states, loop again and try to transition+ // immediately. All states are explicitly listed here to ensure any+ // new states are handled correctly+ ParquetOpenState::Start { .. } => {}+ ParquetOpenState::PruneFile(_) => {}+ ParquetOpenState::PrepareFilters(_) => {}+ ParquetOpenState::PruneWithStatistics(_) => {}+ ParquetOpenState::BuildStream(_) => {}+ };++ self.state = state;+ }+ }+}+impl FileOpener for ParquetOpener {fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {- // ------------------------------------ // Step: prepare configurations, etc.- // -----------------------------------+ let prepared = self.prepare_open_file(partitioned_file)?;+ #[cfg(feature = "parquet_encryption")]+ let future = ParquetOpenFuture::new(prepared, self.get_encryption_context());+ #[cfg(not(feature = "parquet_encryption"))]+ let future = ParquetOpenFuture::new(prepared);+ Ok(Box::pin(future))+ }+}++impl ParquetOpener {+ /// Perform the CPU-only setup for opening a parquet file.+ fn prepare_open_file(+ &self,+ partitioned_file: PartitionedFile,+ ) -> Result<PreparedParquetOpen> {let file_range = partitioned_file.range.clone();let extensions = partitioned_file.extensions.clone();- let file_location = partitioned_file.object_meta.location.clone();- let file_name = file_location.to_string();+ let file_name = partitioned_file.object_meta.location.to_string();let file_metrics =ParquetFileMetrics::new(self.partition_index, &file_name, &self.metrics);let baseline_metrics = BaselineMetrics::new(&self.metrics, self.partition_index);@@ -143,7 +474,7 @@ impl FileOpener for ParquetOpener {.metadata_size_hint.or(self.metadata_size_hint);- let mut async_file_reader: Box<dyn AsyncFileReader> =+ let async_file_reader: Box<dyn AsyncFileReader> =self.parquet_file_reader_factory.create_reader(self.partition_index,partitioned_file.clone(),@@ -151,8 +482,6 @@ impl FileOpener for ParquetOpener {&self.metrics,)?;- let batch_size = self.batch_size;-// Calculate the output schema from the original projection (before literal replacement)// so we get correct field names from column referenceslet logical_file_schema = Arc::clone(self.table_schema.file_schema());@@ -195,7 +524,6 @@ impl FileOpener for ParquetOpener {&logical_file_schema,));- // Apply literal replacements to projection and predicatelet mut projection = self.projection.clone();let mut predicate = self.predicate.clone();if !literal_columns.is_empty() {@@ -207,444 +535,534 @@ impl FileOpener for ParquetOpener {.transpose()?;}- let reorder_predicates = self.reorder_filters;- let pushdown_filters = self.pushdown_filters;- let force_filter_selections = self.force_filter_selections;- let coerce_int96 = self.coerce_int96;- let enable_bloom_filter = self.enable_bloom_filter;- let enable_row_group_stats_pruning = self.enable_row_group_stats_pruning;- let limit = self.limit;- let parquet_file_reader_factory = Arc::clone(&self.parquet_file_reader_factory);- let partition_index = self.partition_index;- let metrics = self.metrics.clone();-let predicate_creation_errors = MetricBuilder::new(&self.metrics).with_category(MetricCategory::Rows).global_counter("num_predicate_creation_errors");- let expr_adapter_factory = Arc::clone(&self.expr_adapter_factory);-- let enable_page_index = self.enable_page_index;- #[cfg(feature = "parquet_encryption")]- let encryption_context = self.get_encryption_context();- let max_predicate_cache_size = self.max_predicate_cache_size;+ // Apply literal replacements to projection and predicate+ let file_pruner = predicate+ .as_ref()+ .filter(|p| is_dynamic_physical_expr(p) || partitioned_file.has_statistics())+ .and_then(|p| {+ FilePruner::try_new(+ Arc::clone(p),+ &logical_file_schema,+ &partitioned_file,+ predicate_creation_errors.clone(),+ )+ });++ Ok(PreparedParquetOpen {+ partition_index: self.partition_index,+ partitioned_file,+ file_range,+ extensions,+ file_name,+ file_metrics,+ baseline_metrics,+ file_pruner,+ metadata_size_hint,+ metrics: self.metrics.clone(),+ parquet_file_reader_factory: Arc::clone(&self.parquet_file_reader_factory),+ async_file_reader,+ batch_size: self.batch_size,+ logical_file_schema: Arc::clone(&logical_file_schema),+ physical_file_schema: logical_file_schema,+ output_schema,+ projection,+ predicate,+ reorder_predicates: self.reorder_filters,+ pushdown_filters: self.pushdown_filters,+ force_filter_selections: self.force_filter_selections,+ enable_page_index: self.enable_page_index,+ enable_bloom_filter: self.enable_bloom_filter,+ enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,+ limit: self.limit,+ coerce_int96: self.coerce_int96,+ expr_adapter_factory: Arc::clone(&self.expr_adapter_factory),+ predicate_creation_errors,+ max_predicate_cache_size: self.max_predicate_cache_size,+ reverse_row_groups: self.reverse_row_groups,+ preserve_order: self.preserve_order,+ #[cfg(feature = "parquet_encryption")]+ file_decryption_properties: None,+ })+ }+}- let reverse_row_groups = self.reverse_row_groups;- let preserve_order = self.preserve_order;+impl PreparedParquetOpen {+ /// Attempt file-level pruning before any metadata is loaded.+ ///+ /// Returns `None` if the file can be skipped completely.+ fn prune_file(mut self) -> Result<Option<Self>> {+ // Prune this file using the file level statistics and partition values.+ // Since dynamic filters may have been updated since planning it is possible that we are able+ // to prune files now that we couldn't prune at planning time.+ // It is assumed that there is no point in doing pruning here if the predicate is not dynamic,+ // as it would have been done at planning time.+ // We'll also check this after every record batch we read,+ // and if at some point we are able to prove we can prune the file using just the file level statistics+ // we can end the stream early.+ //+ // Make a FilePruner only if there is either+ // 1. a dynamic expr in the predicate+ // 2. the file has file-level statistics.+ //+ // File-level statistics may prune the file without loading+ // any row groups or metadata.+ //+ // Dynamic filters may prune the file after initial+ // planning, as the dynamic filter is updated during+ // execution.+ //+ // The case where there is a dynamic filter but no+ // statistics corresponds to a dynamic filter that+ // references partition columns. While rare, this is possible+ // e.g. `select * from table order by partition_col limit+ // 10` could hit this condition.+ if let Some(file_pruner) = &mut self.file_pruner+ && file_pruner.should_prune()?+ {+ self.file_metrics+ .files_ranges_pruned_statistics+ .add_pruned(1);+ return Ok(None);+ }- Ok(Box::pin(async move {- #[cfg(feature = "parquet_encryption")]- let file_decryption_properties = encryption_context- .get_file_decryption_properties(&file_location)- .await?;+ self.file_metrics+ .files_ranges_pruned_statistics+ .add_matched(1);+ Ok(Some(self))+ }- // ---------------------------------------------- // Step: try to prune the current file partition- // ----------------------------------------------- // Prune this file using the file level statistics and partition values.- // Since dynamic filters may have been updated since planning it is possible that we are able- // to prune files now that we couldn't prune at planning time.- // It is assumed that there is no point in doing pruning here if the predicate is not dynamic,- // as it would have been done at planning time.- // We'll also check this after every record batch we read,- // and if at some point we are able to prove we can prune the file using just the file level statistics- // we can end the stream early.- let mut file_pruner = predicate- .as_ref()- .filter(|p| {- // Make a FilePruner only if there is either- // 1. a dynamic expr in the predicate- // 2. the file has file-level statistics.- //- // File-level statistics may prune the file without loading- // any row groups or metadata.- //- // Dynamic filters may prune the file after initial- // planning, as the dynamic filter is updated during- // execution.- //- // The case where there is a dynamic filter but no- // statistics corresponds to a dynamic filter that- // references partition columns. While rare, this is possible- // e.g. `select * from table order by partition_col limit- // 10` could hit this condition.- is_dynamic_physical_expr(p) || partitioned_file.has_statistics()- })- .and_then(|p| {- FilePruner::try_new(- Arc::clone(p),- &logical_file_schema,- &partitioned_file,- predicate_creation_errors.clone(),- )- });-- if let Some(file_pruner) = &mut file_pruner- && file_pruner.should_prune()?- {- // Return an empty stream immediately to skip the work of setting up the actual stream- file_metrics.files_ranges_pruned_statistics.add_pruned(1);- return Ok(futures::stream::empty().boxed());- }+ /// Load parquet metadata after file-level pruning is complete.+ async fn load(mut self) -> Result<MetadataLoadedParquetOpen> {+ // Don't load the page index yet. Since it is not stored inline in+ // the footer, loading the page index if it is not needed will do+ // unnecessary I/O. We decide later if it is needed to evaluate the+ // pruning predicates. Thus default to not requesting it from the+ // underlying reader.+ let options =+ ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Skip);+ #[cfg(feature = "parquet_encryption")]+ let mut options = options;+ #[cfg(feature = "parquet_encryption")]+ if let Some(fd_val) = &self.file_decryption_properties {+ options = options.with_file_decryption_properties(Arc::clone(fd_val));+ }- file_metrics.files_ranges_pruned_statistics.add_matched(1);+ let mut metadata_timer = self.file_metrics.metadata_load_time.timer();+ // Begin by loading the metadata from the underlying reader (note+ // the returned metadata may actually include page indexes as some+ // readers may return page indexes even when not requested -- for+ // example when they are cached)+ let reader_metadata =+ ArrowReaderMetadata::load_async(&mut self.async_file_reader, options.clone())+ .await?;+ metadata_timer.stop();+ drop(metadata_timer);++ Ok(MetadataLoadedParquetOpen {+ prepared: self,+ reader_metadata,+ options,+ })+ }+}- // --------------------------------------------------------- // Step: fetch Parquet metadata (and optionally page index)- // --------------------------------------------------------+impl MetadataLoadedParquetOpen {+ /// Prepare file-schema coercions and pruning predicates once metadata is+ /// loaded.+ fn prepare_filters(self) -> Result<FiltersPreparedParquetOpen> {+ let MetadataLoadedParquetOpen {+ mut prepared,+ mut reader_metadata,+ mut options,+ } = self;++ // Note about schemas: we are actually dealing with **3 different schemas** here:+ // - The table schema as defined by the TableProvider.+ // This is what the user sees, what they get when they `SELECT * FROM table`, etc.+ // - The logical file schema: this is the table schema minus any hive partition columns and projections.+ // This is what the physical file schema is coerced to.+ // - The physical file schema: this is the schema that the arrow-rs+ // parquet reader will actually produce.+ let mut physical_file_schema = Arc::clone(reader_metadata.schema());++ // The schema loaded from the file may not be the same as the+ // desired schema (for example if we want to instruct the parquet+ // reader to read strings using Utf8View instead). Update if necessary+ if let Some(merged) = apply_file_schema_type_coercions(+ &prepared.logical_file_schema,+ &physical_file_schema,+ ) {+ physical_file_schema = Arc::new(merged);+ options = options.with_schema(Arc::clone(&physical_file_schema));+ reader_metadata = ArrowReaderMetadata::try_new(+ Arc::clone(reader_metadata.metadata()),+ options.clone(),+ )?;+ }- // Don't load the page index yet. Since it is not stored inline in- // the footer, loading the page index if it is not needed will do- // unnecessary I/O. We decide later if it is needed to evaluate the- // pruning predicates. Thus default to not requesting it from the- // underlying reader.- let mut options =- ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Skip);- #[cfg(feature = "parquet_encryption")]- if let Some(fd_val) = file_decryption_properties {- options = options.with_file_decryption_properties(Arc::clone(&fd_val));- }- let mut metadata_timer = file_metrics.metadata_load_time.timer();-- // Begin by loading the metadata from the underlying reader (note- // the returned metadata may actually include page indexes as some- // readers may return page indexes even when not requested -- for- // example when they are cached)- let mut reader_metadata =- ArrowReaderMetadata::load_async(&mut async_file_reader, options.clone())- .await?;-- // Note about schemas: we are actually dealing with **3 different schemas** here:- // - The table schema as defined by the TableProvider.- // This is what the user sees, what they get when they `SELECT * FROM table`, etc.- // - The logical file schema: this is the table schema minus any hive partition columns and projections.- // This is what the physical file schema is coerced to.- // - The physical file schema: this is the schema that the arrow-rs- // parquet reader will actually produce.- let mut physical_file_schema = Arc::clone(reader_metadata.schema());-- // The schema loaded from the file may not be the same as the- // desired schema (for example if we want to instruct the parquet- // reader to read strings using Utf8View instead). Update if necessary- if let Some(merged) = apply_file_schema_type_coercions(- &logical_file_schema,+ if let Some(ref coerce) = prepared.coerce_int96+ && let Some(merged) = coerce_int96_to_resolution(+ reader_metadata.parquet_schema(),&physical_file_schema,- ) {- physical_file_schema = Arc::new(merged);- options = options.with_schema(Arc::clone(&physical_file_schema));- reader_metadata = ArrowReaderMetadata::try_new(- Arc::clone(reader_metadata.metadata()),- options.clone(),- )?;- }+ coerce,+ )+ {+ physical_file_schema = Arc::new(merged);+ options = options.with_schema(Arc::clone(&physical_file_schema));+ reader_metadata = ArrowReaderMetadata::try_new(+ Arc::clone(reader_metadata.metadata()),+ options.clone(),+ )?;+ }- if let Some(ref coerce) = coerce_int96- && let Some(merged) = coerce_int96_to_resolution(- reader_metadata.parquet_schema(),- &physical_file_schema,- coerce,- )- {- physical_file_schema = Arc::new(merged);- options = options.with_schema(Arc::clone(&physical_file_schema));- reader_metadata = ArrowReaderMetadata::try_new(- Arc::clone(reader_metadata.metadata()),- options.clone(),- )?;- }+ // Adapt the projection & filter predicate to the physical file schema.+ // This evaluates missing columns and inserts any necessary casts.+ // After rewriting to the file schema, further simplifications may be possible.+ // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`+ // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).+ // Additionally, if any casts were inserted we can move casts from the column to the literal side:+ // `CAST(col AS INT) = 5` can become `col = CAST(5 AS <col type>)`, which can be evaluated statically.+ //+ // When the schemas are identical and there is no predicate, the+ // rewriter is a no-op: column indices already match (partition+ // columns are appended after file columns in the table schema),+ // types are the same, and there are no missing columns. Skip the+ // tree walk entirely in that case.+ let needs_rewrite = prepared.predicate.is_some()+ || prepared.logical_file_schema != physical_file_schema;+ if needs_rewrite {+ let rewriter = prepared.expr_adapter_factory.create(+ Arc::clone(&prepared.logical_file_schema),+ Arc::clone(&physical_file_schema),+ )?;+ let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);+ prepared.predicate = prepared+ .predicate+ .map(|p| simplifier.simplify(rewriter.rewrite(p)?))+ .transpose()?;+ prepared.projection = prepared+ .projection+ .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;+ }+ prepared.physical_file_schema = Arc::clone(&physical_file_schema);- // Adapt the projection & filter predicate to the physical file schema.- // This evaluates missing columns and inserts any necessary casts.- // After rewriting to the file schema, further simplifications may be possible.- // For example, if `'a' = col_that_is_missing` becomes `'a' = NULL` that can then be simplified to `FALSE`- // and we can avoid doing any more work on the file (bloom filters, loading the page index, etc.).- // Additionally, if any casts were inserted we can move casts from the column to the literal side:- // `CAST(col AS INT) = 5` can become `col = CAST(5 AS <col type>)`, which can be evaluated statically.- //- // When the schemas are identical and there is no predicate, the- // rewriter is a no-op: column indices already match (partition- // columns are appended after file columns in the table schema),- // types are the same, and there are no missing columns. Skip the- // tree walk entirely in that case.- let needs_rewrite =- predicate.is_some() || logical_file_schema != physical_file_schema;- if needs_rewrite {- let rewriter = expr_adapter_factory.create(- Arc::clone(&logical_file_schema),- Arc::clone(&physical_file_schema),- )?;- let simplifier = PhysicalExprSimplifier::new(&physical_file_schema);- predicate = predicate- .map(|p| simplifier.simplify(rewriter.rewrite(p)?))- .transpose()?;- // Adapt projections to the physical file schema as well- projection = projection- .try_map_exprs(|p| simplifier.simplify(rewriter.rewrite(p)?))?;- }+ // Build predicates for this specific file+ let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(+ prepared.predicate.as_ref(),+ &physical_file_schema,+ &prepared.predicate_creation_errors,+ );- // Build predicates for this specific file- let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates(- predicate.as_ref(),- &physical_file_schema,- &predicate_creation_errors,- );+ Ok(FiltersPreparedParquetOpen {+ loaded: MetadataLoadedParquetOpen {+ prepared,+ reader_metadata,+ options,+ },+ pruning_predicate,+ page_pruning_predicate,+ })+ }+}- // The page index is not stored inline in the parquet footer so the- // code above may not have read the page index structures yet. If we- // need them for reading and they aren't yet loaded, we need to load them now.- if should_enable_page_index(enable_page_index, &page_pruning_predicate) {- reader_metadata = load_page_index(- reader_metadata,- &mut async_file_reader,- // Since we're manually loading the page index the option here should not matter but we pass it in for consistency- options.with_page_index_policy(PageIndexPolicy::Optional),- )- .await?;- }+impl FiltersPreparedParquetOpen {+ /// Load the page index if pruning requires it and metadata did not include it.+ async fn load_page_index(mut self) -> Result<Self> {+ // The page index is not stored inline in the parquet footer so the+ // metadata load above may not have read the page index structures yet.+ // If we need them for reading and they aren't yet loaded, we need to+ // load them now.+ if should_enable_page_index(+ self.loaded.prepared.enable_page_index,+ &self.page_pruning_predicate,+ ) {+ self.loaded.reader_metadata = load_page_index(+ self.loaded.reader_metadata,+ &mut self.loaded.prepared.async_file_reader,+ self.loaded+ .options+ .clone()+ .with_page_index_policy(PageIndexPolicy::Optional),+ )+ .await?;+ }- metadata_timer.stop();-- // ------------------------------------------------------------- // Step: prune row groups by range, predicate and bloom filter- // -------------------------------------------------------------- // Determine which row groups to actually read. The idea is to skip- // as many row groups as possible based on the metadata and query- let file_metadata = Arc::clone(reader_metadata.metadata());- let pruning_pred = pruning_predicate.as_ref().map(|p| p.as_ref());- let rg_metadata = file_metadata.row_groups();- // track which row groups to actually read- let access_plan =- create_initial_plan(&file_name, extensions, rg_metadata.len())?;- let mut row_groups = RowGroupAccessPlanFilter::new(access_plan);- // if there is a range restricting what parts of the file to read- if let Some(range) = file_range.as_ref() {- row_groups.prune_by_range(rg_metadata, range);- }+ Ok(self)+ }- // If there is a predicate that can be evaluated against the metadata- if let Some(pruning_pred) = pruning_pred.as_ref() {- if enable_row_group_stats_pruning {- row_groups.prune_by_statistics(- &physical_file_schema,- reader_metadata.parquet_schema(),- rg_metadata,- pruning_pred,- &file_metrics,- );- } else {- // Update metrics: statistics unavailable, so all row groups are- // matched (not pruned)- file_metrics- .row_groups_pruned_statistics- .add_matched(row_groups.remaining_row_group_count());- }+ /// Prune row groups using file ranges and parquet metadata.+ fn prune_row_groups(self) -> Result<RowGroupsPrunedParquetOpen> {+ let loaded = &self.loaded;+ let prepared = &loaded.prepared;+ let file_metadata = Arc::clone(loaded.reader_metadata.metadata());+ let rg_metadata = file_metadata.row_groups();++ // Determine which row groups to actually read. The idea is to skip+ // as many row groups as possible based on the metadata and query+ let mut row_groups = RowGroupAccessPlanFilter::new(create_initial_plan(+ &prepared.file_name,+ prepared.extensions.clone(),+ rg_metadata.len(),+ )?);++ // If there is a range restricting what parts of the file to read+ if let Some(range) = prepared.file_range.as_ref() {+ row_groups.prune_by_range(rg_metadata, range);+ }- if enable_bloom_filter && !row_groups.is_empty() {- // Use the existing reader for bloom filter I/O;- // replace with a fresh reader for decoding below.- let bf_reader = std::mem::replace(- &mut async_file_reader,- parquet_file_reader_factory.create_reader(- partition_index,- partitioned_file.clone(),- metadata_size_hint,- &metrics,- )?,- );- let mut bf_builder =- ParquetRecordBatchStreamBuilder::new_with_metadata(- bf_reader,- reader_metadata.clone(),- );- row_groups- .prune_by_bloom_filters(- &physical_file_schema,- &mut bf_builder,- pruning_pred,- &file_metrics,- )- .await;- } else {- // Update metrics: bloom filter unavailable, so all row groups are- // matched (not pruned)- file_metrics- .row_groups_pruned_bloom_filter- .add_matched(row_groups.remaining_row_group_count());- }+ // If there is a predicate that can be evaluated against the metadata+ if let Some(predicate) = self.pruning_predicate.as_ref().map(|p| p.as_ref()) {+ if prepared.enable_row_group_stats_pruning {+ row_groups.prune_by_statistics(+ &prepared.physical_file_schema,+ loaded.reader_metadata.parquet_schema(),+ rg_metadata,+ predicate,+ &prepared.file_metrics,+ );} else {- // Update metrics: no predicate, so all row groups are matched (not pruned)- let n_remaining_row_groups = row_groups.remaining_row_group_count();- file_metrics+ // Update metrics: statistics unavailable, so all row groups are+ // matched (not pruned)+ prepared+ .file_metrics.row_groups_pruned_statistics- .add_matched(n_remaining_row_groups);- file_metrics- .row_groups_pruned_bloom_filter- .add_matched(n_remaining_row_groups);- }-- // Prune by limit if limit is set and limit order is not sensitive- if let (Some(limit), false) = (limit, preserve_order) {- row_groups.prune_by_limit(limit, rg_metadata, &file_metrics);+ .add_matched(row_groups.remaining_row_group_count());}- // --------------------------------------------------------- // Step: prune pages from the kept row groups- //- let mut access_plan = row_groups.build();- // page index pruning: if all data on individual pages can- // be ruled using page metadata, rows from other columns- // with that range can be skipped as well- // --------------------------------------------------------- if enable_page_index- && !access_plan.is_empty()- && let Some(p) = page_pruning_predicate- {- access_plan = p.prune_plan_with_page_index(- access_plan,- &physical_file_schema,- reader_metadata.parquet_schema(),- file_metadata.as_ref(),- &file_metrics,- );+ if !prepared.enable_bloom_filter || row_groups.is_empty() {+ // Update metrics: bloom filter unavailable, so all row groups are+ // matched (not pruned)+ prepared+ .file_metrics+ .row_groups_pruned_bloom_filter+ .add_matched(row_groups.remaining_row_group_count());}+ } else {+ // Update metrics: no predicate, so all row groups are matched (not pruned)+ let remaining = row_groups.remaining_row_group_count();+ prepared+ .file_metrics+ .row_groups_pruned_statistics+ .add_matched(remaining);+ prepared+ .file_metrics+ .row_groups_pruned_bloom_filter+ .add_matched(remaining);+ }- // Prepare the access plan (extract row groups and row selection)- let mut prepared_plan = access_plan.prepare(rg_metadata)?;-- // ----------------------------------------------------------- // Step: potentially reverse the access plan for performance.- // See `ParquetSource::try_pushdown_sort` for the rationale.- // ----------------------------------------------------------- if reverse_row_groups {- prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;- }+ Ok(RowGroupsPrunedParquetOpen {+ prepared: self,+ row_groups,+ })+ }+}- if prepared_plan.row_group_indexes.is_empty() {- return Ok(futures::stream::empty().boxed());- }+impl RowGroupsPrunedParquetOpen {+ /// Apply bloom filter pruning when it is enabled and a pruning predicate exists.+ async fn prune_bloom_filters(mut self) -> Result<Self> {+ if let Some(predicate) =+ self.prepared.pruning_predicate.as_ref().map(|p| p.as_ref())+ && self.prepared.loaded.prepared.enable_bloom_filter+ && !self.row_groups.is_empty()+ {+ // Use the existing reader for bloom filter I/O;+ // replace with a fresh reader for decoding below.+ let reader_metadata = self.prepared.loaded.reader_metadata.clone();+ let replacement_reader = {+ let prepared = &self.prepared.loaded.prepared;+ prepared.parquet_file_reader_factory.create_reader(+ prepared.partition_index,+ prepared.partitioned_file.clone(),+ prepared.metadata_size_hint,+ &prepared.metrics,+ )?+ };- // ---------------------------------------------------------- // Step: construct builder for the final RecordBatch stream- // ----------------------------------------------------------- let mut builder =- ParquetPushDecoderBuilder::new_with_metadata(reader_metadata.clone())- .with_batch_size(batch_size);-- // ---------------------------------------------------------------------- // Step: optionally add row filter to the builder- //- // Row filter is used for late materialization in parquet decoding, see- // `row_filter` for details.- // ----------------------------------------------------------------------- // Filter pushdown: evaluate predicates during scan- if let Some(predicate) =- pushdown_filters.then_some(predicate.as_ref()).flatten()- {- let row_filter = row_filter::build_row_filter(+ let prepared = &mut self.prepared.loaded.prepared;+ let mut builder = ParquetRecordBatchStreamBuilder::new_with_metadata(+ mem::replace(&mut prepared.async_file_reader, replacement_reader),+ reader_metadata,+ );+ self.row_groups+ .prune_by_bloom_filters(+ &prepared.physical_file_schema,+ &mut builder,predicate,- &physical_file_schema,- file_metadata.as_ref(),- reorder_predicates,- &file_metrics,- );-- match row_filter {- Ok(Some(filter)) => {- builder = builder.with_row_filter(filter);- }- Ok(None) => {}- Err(e) => {- debug!(- "Ignoring error building row filter for '{predicate:?}': {e}"- );- }- };- };- if force_filter_selections {- builder =- builder.with_row_selection_policy(RowSelectionPolicy::Selectors);- }+ &prepared.file_metrics,+ )+ .await;+ }- // Apply the prepared plan to the builder- if let Some(row_selection) = prepared_plan.row_selection {- builder = builder.with_row_selection(row_selection);- }- builder = builder.with_row_groups(prepared_plan.row_group_indexes);+ Ok(self)+ }- if let Some(limit) = limit {- builder = builder.with_limit(limit)- }+ /// Build the final parquet stream once all pruning work is complete.+ fn build_stream(self) -> Result<BoxStream<'static, Result<RecordBatch>>> {+ let RowGroupsPrunedParquetOpen {+ prepared,+ mut row_groups,+ } = self;+ let FiltersPreparedParquetOpen {+ loaded,+ pruning_predicate: _,+ page_pruning_predicate,+ } = prepared;+ let MetadataLoadedParquetOpen {+ prepared,+ reader_metadata,+ options: _,+ } = loaded;++ let file_metadata = Arc::clone(reader_metadata.metadata());+ let rg_metadata = file_metadata.row_groups();++ // Filter pushdown: evaluate predicates during scan+ let row_filter = if let Some(predicate) = prepared+ .pushdown_filters+ .then_some(prepared.predicate.clone())+ .flatten()+ {+ let row_filter = row_filter::build_row_filter(+ &predicate,+ &prepared.physical_file_schema,+ file_metadata.as_ref(),+ prepared.reorder_predicates,+ &prepared.file_metrics,+ );- if let Some(max_predicate_cache_size) = max_predicate_cache_size {- builder = builder.with_max_predicate_cache_size(max_predicate_cache_size);+ match row_filter {+ Ok(Some(filter)) => Some(filter),+ Ok(None) => None,+ Err(e) => {+ debug!("Ignoring error building row filter for '{predicate:?}': {e}");+ None+ }}+ } else {+ None+ };- // metrics from the arrow reader itself- let arrow_reader_metrics = ArrowReaderMetrics::enabled();+ // Prune by limit if limit is set and limit order is not sensitive+ if let (Some(limit), false) = (prepared.limit, prepared.preserve_order) {+ row_groups.prune_by_limit(limit, rg_metadata, &prepared.file_metrics);+ }- let read_plan = build_projection_read_plan(- projection.expr_iter(),- &physical_file_schema,+ // Page index pruning: if all data on individual pages can+ // be ruled using page metadata, rows from other columns+ // with that range can be skipped as well.+ let mut access_plan = row_groups.build();+ if prepared.enable_page_index+ && !access_plan.is_empty()+ && let Some(page_pruning_predicate) = page_pruning_predicate+ {+ access_plan = page_pruning_predicate.prune_plan_with_page_index(+ access_plan,+ &prepared.physical_file_schema,reader_metadata.parquet_schema(),+ file_metadata.as_ref(),+ &prepared.file_metrics,);+ }++ // Prepare the access plan (extract row groups and row selection)+ let mut prepared_plan = access_plan.prepare(rg_metadata)?;++ // Potentially reverse the access plan for performance.+ // See `ParquetSource::try_pushdown_sort` for the rationale.+ if prepared.reverse_row_groups {+ prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;+ }++ let arrow_reader_metrics = ArrowReaderMetrics::enabled();+ let read_plan = build_projection_read_plan(+ prepared.projection.expr_iter(),+ &prepared.physical_file_schema,+ reader_metadata.parquet_schema(),+ );- let decoder = builder+ let mut decoder_builder =+ ParquetPushDecoderBuilder::new_with_metadata(reader_metadata).with_projection(read_plan.projection_mask)- .with_metrics(arrow_reader_metrics.clone())- .build()?;-- let files_ranges_pruned_statistics =- file_metrics.files_ranges_pruned_statistics.clone();- let predicate_cache_inner_records =- file_metrics.predicate_cache_inner_records.clone();- let predicate_cache_records = file_metrics.predicate_cache_records.clone();-- // Rebase column indices to match the narrowed stream schema.- // The projection expressions have indices based on physical_file_schema,- // but the stream only contains the columns selected by the ProjectionMask.- let stream_schema = read_plan.projected_schema;- let replace_schema = stream_schema != output_schema;- let projection = projection- .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;- let projector = projection.make_projector(&stream_schema)?;- let stream = futures::stream::unfold(- PushDecoderStreamState {- decoder,- reader: async_file_reader,- projector,- output_schema,- replace_schema,- arrow_reader_metrics,- predicate_cache_inner_records,- predicate_cache_records,- baseline_metrics,- },- |mut state| async move {- let result = state.transition().await;- result.map(|r| (r, state))- },+ .with_batch_size(prepared.batch_size)+ .with_metrics(arrow_reader_metrics.clone());++ if let Some(row_filter) = row_filter {+ decoder_builder = decoder_builder.with_row_filter(row_filter);+ }+ if prepared.force_filter_selections {+ decoder_builder =+ decoder_builder.with_row_selection_policy(RowSelectionPolicy::Selectors);+ }+ if let Some(row_selection) = prepared_plan.row_selection {+ decoder_builder = decoder_builder.with_row_selection(row_selection);+ }+ decoder_builder =+ decoder_builder.with_row_groups(prepared_plan.row_group_indexes);+ if let Some(limit) = prepared.limit {+ decoder_builder = decoder_builder.with_limit(limit);+ }+ if let Some(max_predicate_cache_size) = prepared.max_predicate_cache_size {+ decoder_builder =+ decoder_builder.with_max_predicate_cache_size(max_predicate_cache_size);+ }++ let decoder = decoder_builder.build()?;++ let predicate_cache_inner_records =+ prepared.file_metrics.predicate_cache_inner_records.clone();+ let predicate_cache_records =+ prepared.file_metrics.predicate_cache_records.clone();++ // Check if we need to replace the schema to handle things like differing nullability or metadata.+ // See note below about file vs. output schema.+ let stream_schema = read_plan.projected_schema;+ let replace_schema = stream_schema != prepared.output_schema;++ // Rebase column indices to match the narrowed stream schema.+ // The projection expressions have indices based on physical_file_schema,+ // but the stream only contains the columns selected by the ProjectionMask.+ let projection = prepared+ .projection+ .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?;+ let projector = projection.make_projector(&stream_schema)?;+ let output_schema = Arc::clone(&prepared.output_schema);+ let files_ranges_pruned_statistics =+ prepared.file_metrics.files_ranges_pruned_statistics.clone();+ let stream = futures::stream::unfold(+ PushDecoderStreamState {+ decoder,+ reader: prepared.async_file_reader,+ projector,+ output_schema,+ replace_schema,+ arrow_reader_metrics,+ predicate_cache_inner_records,+ predicate_cache_records,+ baseline_metrics: prepared.baseline_metrics,+ },+ |mut state| async move {+ let result = state.transition().await;+ result.map(|r| (r, state))+ },+ )+ .fuse();++ // Wrap the stream so a dynamic filter can stop the file scan early.+ if let Some(file_pruner) = prepared.file_pruner {+ let stream = stream.boxed();+ Ok(EarlyStoppingStream::new(+ stream,+ file_pruner,+ files_ranges_pruned_statistics,)- .fuse();-- // ----------------------------------------------------------------------- // Step: wrap the stream so a dynamic filter can stop the file scan early- // ----------------------------------------------------------------------- if let Some(file_pruner) = file_pruner {- let boxed_stream = stream.boxed();- Ok(EarlyStoppingStream::new(- boxed_stream,- file_pruner,- files_ranges_pruned_statistics,- )- .boxed())- } else {- Ok(stream.boxed())- }- }))+ .boxed())+ } else {+ Ok(stream.boxed())+ }}}@@ -719,7 +1137,15 @@ impl PushDecoderStreamState {fn project_batch(&self, batch: &RecordBatch) -> Result<RecordBatch> {let mut batch = self.projector.project_batch(batch)?;if self.replace_schema {- let (_schema, arrays, num_rows) = batch.into_parts();+ // Ensure the output batch has the expected schema.+ // This handles things like schema level and field level metadata, which may not be present+ // in the physical file schema.+ // It is also possible for nullability to differ; some writers create files with+ // OPTIONAL fields even when there are no nulls in the data.+ // In these cases it may make sense for the logical schema to be `NOT NULL`.+ // RecordBatch::try_new_with_options checks that if the schema is NOT NULL+ // the array cannot contain nulls, amongst other checks.+ let (_stream_schema, arrays, num_rows) = batch.into_parts();let options = RecordBatchOptions::new().with_row_count(Some(num_rows));batch = RecordBatch::try_new_with_options(Arc::clone(&self.output_schema),diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rsindex 4291c9af76..482bf8dced 100644--- a/datafusion/datasource-parquet/src/reader.rs+++ b/datafusion/datasource-parquet/src/reader.rs@@ -37,7 +37,7 @@ use std::fmt::Debug;use std::ops::Range;use std::sync::Arc;-/// Interface for reading parquet files.+/// Interface for reading Apache Parquet files.////// The combined implementations of [`ParquetFileReaderFactory`] and/// [`AsyncFileReader`] can be used to provide custom data access operations