Skip to content

Apache DataFusion SQL Query Engine

refactor(parquet): extract InitialDecoderState from the opener decoder-setup block (#24572)

Qi Zhufc15e25cbb49
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index 2e30fcc430..5e004cf017 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -28,7 +28,8 @@ use crate::decoder_projection::DecoderProjection;
use crate::metrics::ByteProgress;
use crate::page_filter::PagePruningAccessPlanFilter;
use crate::push_decoder::{
- DecoderBuilderConfig, PushDecoderStreamState, RgPlanEntry, RowGroupPruner,
+ DecoderBuilderConfig, InitialDecoderState, PushDecoderStreamState, RgPlanEntry,
+ RowGroupPruner,
};
use crate::row_filter::RowFilterGenerator;
use crate::row_group_filter::{RowGroupAccessPlanFilter, row_group_in_range};
@@ -1458,7 +1459,11 @@ impl RowGroupsPrunedParquetOpen {
prepared.virtual_state.as_deref(),
)?;
- let (decoder, rg_plan, has_row_selection) = {
+ let InitialDecoderState {
+ decoder,
+ rg_plan,
+ has_row_selection,
+ } = {
let pushdown_predicate = prepared
.pushdown_filters
.then_some(prepared.predicate.as_ref())
@@ -1520,7 +1525,11 @@ impl RowGroupsPrunedParquetOpen {
}
}
- (builder.build()?, rg_plan, has_row_selection)
+ InitialDecoderState {
+ decoder: builder.build()?,
+ rg_plan,
+ has_row_selection,
+ }
};
// Track how much of this file range the scan has finished with. Credit
diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs
index e7acefcdd4..8238f13b00 100644
--- a/datafusion/datasource-parquet/src/push_decoder.rs
+++ b/datafusion/datasource-parquet/src/push_decoder.rs
@@ -117,6 +117,21 @@ pub(crate) struct RgPlanEntry {
pub(crate) bytes: u64,
}
+/// The initial per-file decoder state the opener builds and hands off to the
+/// [`PushDecoderStreamState`] stream driver.
+///
+/// Named rather than a bare tuple so the fields carried out of the decoder
+/// setup block stay self-documenting as more are added.
+pub(crate) struct InitialDecoderState {
+ /// The freshly built push decoder for this file.
+ pub(crate) decoder: ParquetPushDecoder,
+ /// The per-row-group plan, in the physical scan order the decoder reads.
+ pub(crate) rg_plan: VecDeque<RgPlanEntry>,
+ /// Whether a row selection is live for this scan. Runtime row-group
+ /// pruning is disabled when it is (see the opener for why).
+ pub(crate) has_row_selection: bool,
+}
+
/// Runtime row-group pruner driven by a dynamic predicate (e.g. the
/// threshold expression a `TopK` operator pushes down).
///