Skip to content

Apache DataFusion SQL Query Engine

refactor: add ParquetOpenerBuilder to reduce test code duplication (#19405)

Shashidhar B Md8e68a404aa7
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index f1ecc86ce8..bea970f144 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -988,6 +988,164 @@ mod test {
use parquet::arrow::ArrowWriter;
use parquet::file::properties::WriterProperties;
+ /// Builder for creating [`ParquetOpener`] instances with sensible defaults for tests.
+ /// This helps reduce code duplication and makes it clear what differs between test cases.
+ struct ParquetOpenerBuilder {
+ store: Option<Arc<dyn ObjectStore>>,
+ table_schema: Option<TableSchema>,
+ partition_index: usize,
+ projection_indices: Option<Vec<usize>>,
+ projection: Option<ProjectionExprs>,
+ batch_size: usize,
+ limit: Option<usize>,
+ predicate: Option<Arc<dyn PhysicalExpr>>,
+ metadata_size_hint: Option<usize>,
+ metrics: ExecutionPlanMetricsSet,
+ pushdown_filters: bool,
+ reorder_filters: bool,
+ force_filter_selections: bool,
+ enable_page_index: bool,
+ enable_bloom_filter: bool,
+ enable_row_group_stats_pruning: bool,
+ coerce_int96: Option<arrow::datatypes::TimeUnit>,
+ max_predicate_cache_size: Option<usize>,
+ reverse_row_groups: bool,
+ }
+
+ impl ParquetOpenerBuilder {
+ /// Create a new builder with sensible defaults for tests.
+ fn new() -> Self {
+ Self {
+ store: None,
+ table_schema: None,
+ partition_index: 0,
+ projection_indices: None,
+ projection: None,
+ batch_size: 1024,
+ limit: None,
+ predicate: None,
+ metadata_size_hint: None,
+ metrics: ExecutionPlanMetricsSet::new(),
+ pushdown_filters: false,
+ reorder_filters: false,
+ force_filter_selections: false,
+ enable_page_index: false,
+ enable_bloom_filter: false,
+ enable_row_group_stats_pruning: false,
+ coerce_int96: None,
+ max_predicate_cache_size: None,
+ reverse_row_groups: false,
+ }
+ }
+
+ /// Set the object store (required for building).
+ fn with_store(mut self, store: Arc<dyn ObjectStore>) -> Self {
+ self.store = Some(store);
+ self
+ }
+
+ /// Create a simple table schema from a file schema (for files without partition columns).
+ fn with_schema(mut self, file_schema: SchemaRef) -> Self {
+ self.table_schema = Some(TableSchema::from_file_schema(file_schema));
+ self
+ }
+
+ /// Set a custom table schema (for files with partition columns).
+ fn with_table_schema(mut self, table_schema: TableSchema) -> Self {
+ self.table_schema = Some(table_schema);
+ self
+ }
+
+ /// Set projection by column indices (convenience method for common case).
+ fn with_projection_indices(mut self, indices: &[usize]) -> Self {
+ self.projection_indices = Some(indices.to_vec());
+ self
+ }
+
+ /// Set the predicate.
+ fn with_predicate(mut self, predicate: Arc<dyn PhysicalExpr>) -> Self {
+ self.predicate = Some(predicate);
+ self
+ }
+
+ /// Enable pushdown filters.
+ fn with_pushdown_filters(mut self, enable: bool) -> Self {
+ self.pushdown_filters = enable;
+ self
+ }
+
+ /// Enable filter reordering.
+ fn with_reorder_filters(mut self, enable: bool) -> Self {
+ self.reorder_filters = enable;
+ self
+ }
+
+ /// Enable row group stats pruning.
+ fn with_row_group_stats_pruning(mut self, enable: bool) -> Self {
+ self.enable_row_group_stats_pruning = enable;
+ self
+ }
+
+ /// Set reverse row groups flag.
+ fn with_reverse_row_groups(mut self, enable: bool) -> Self {
+ self.reverse_row_groups = enable;
+ self
+ }
+
+ /// Build the ParquetOpener instance.
+ ///
+ /// # Panics
+ ///
+ /// Panics if required fields (store, schema/table_schema) are not set.
+ fn build(self) -> ParquetOpener {
+ let store = self
+ .store
+ .expect("ParquetOpenerBuilder: store must be set via with_store()");
+ let table_schema = self.table_schema.expect(
+ "ParquetOpenerBuilder: table_schema must be set via with_schema() or with_table_schema()",
+ );
+ let file_schema = Arc::clone(table_schema.file_schema());
+
+ let projection = if let Some(projection) = self.projection {
+ projection
+ } else if let Some(indices) = self.projection_indices {
+ ProjectionExprs::from_indices(&indices, &file_schema)
+ } else {
+ // Default: project all columns
+ let all_indices: Vec<usize> = (0..file_schema.fields().len()).collect();
+ ProjectionExprs::from_indices(&all_indices, &file_schema)
+ };
+
+ ParquetOpener {
+ partition_index: self.partition_index,
+ projection,
+ batch_size: self.batch_size,
+ limit: self.limit,
+ predicate: self.predicate,
+ table_schema,
+ metadata_size_hint: self.metadata_size_hint,
+ metrics: self.metrics,
+ parquet_file_reader_factory: Arc::new(
+ DefaultParquetFileReaderFactory::new(store),
+ ),
+ pushdown_filters: self.pushdown_filters,
+ reorder_filters: self.reorder_filters,
+ force_filter_selections: self.force_filter_selections,
+ enable_page_index: self.enable_page_index,
+ enable_bloom_filter: self.enable_bloom_filter,
+ enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
+ coerce_int96: self.coerce_int96,
+ #[cfg(feature = "parquet_encryption")]
+ file_decryption_properties: None,
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
+ #[cfg(feature = "parquet_encryption")]
+ encryption_factory: None,
+ max_predicate_cache_size: self.max_predicate_cache_size,
+ reverse_row_groups: self.reverse_row_groups,
+ }
+ }
+ }
+
fn constant_int_stats() -> (Statistics, SchemaRef) {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
@@ -1181,33 +1339,13 @@ mod test {
));
let make_opener = |predicate| {
- ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0, 1], &schema),
- batch_size: 1024,
- limit: None,
- predicate: Some(predicate),
- table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(
- DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
- ),
- pushdown_filters: false, // note that this is false!
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: true,
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: false,
- }
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0, 1])
+ .with_predicate(predicate)
+ .with_row_group_stats_pruning(true)
+ .build()
};
// A filter on "a" should not exclude any rows even if it matches the data
@@ -1249,37 +1387,18 @@ mod test {
Field::new("a", DataType::Int32, false),
]));
+ let table_schema_for_opener = TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ );
let make_opener = |predicate| {
- ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &file_schema),
- batch_size: 1024,
- limit: None,
- predicate: Some(predicate),
- table_schema: TableSchema::new(
- file_schema.clone(),
- vec![Arc::new(Field::new("part", DataType::Int32, false))],
- ),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(
- DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
- ),
- pushdown_filters: false, // note that this is false!
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: true,
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: false,
- }
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema_for_opener.clone())
+ .with_projection_indices(&[0])
+ .with_predicate(predicate)
+ .with_row_group_stats_pruning(true)
+ .build()
};
// Filter should match the partition value
@@ -1337,37 +1456,18 @@ mod test {
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Float32, true),
]));
+ let table_schema_for_opener = TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ );
let make_opener = |predicate| {
- ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &file_schema),
- batch_size: 1024,
- limit: None,
- predicate: Some(predicate),
- table_schema: TableSchema::new(
- file_schema.clone(),
- vec![Arc::new(Field::new("part", DataType::Int32, false))],
- ),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(
- DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
- ),
- pushdown_filters: false, // note that this is false!
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: true,
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: false,
- }
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema_for_opener.clone())
+ .with_projection_indices(&[0])
+ .with_predicate(predicate)
+ .with_row_group_stats_pruning(true)
+ .build()
};
// Filter should match the partition value and file statistics
@@ -1428,37 +1528,19 @@ mod test {
Field::new("a", DataType::Int32, false),
]));
+ let table_schema_for_opener = TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ );
let make_opener = |predicate| {
- ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &file_schema),
- batch_size: 1024,
- limit: None,
- predicate: Some(predicate),
- table_schema: TableSchema::new(
- file_schema.clone(),
- vec![Arc::new(Field::new("part", DataType::Int32, false))],
- ),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(
- DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
- ),
- pushdown_filters: true, // note that this is true!
- reorder_filters: true,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: false, // note that this is false!
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: false,
- }
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema_for_opener.clone())
+ .with_projection_indices(&[0])
+ .with_predicate(predicate)
+ .with_pushdown_filters(true) // note that this is true!
+ .with_reorder_filters(true)
+ .build()
};
// Filter should match the partition value and data value
@@ -1527,37 +1609,17 @@ mod test {
Field::new("part", DataType::Int32, false),
]));
+ let table_schema_for_opener = TableSchema::new(
+ file_schema.clone(),
+ vec![Arc::new(Field::new("part", DataType::Int32, false))],
+ );
let make_opener = |predicate| {
- ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &file_schema),
- batch_size: 1024,
- limit: None,
- predicate: Some(predicate),
- table_schema: TableSchema::new(
- file_schema.clone(),
- vec![Arc::new(Field::new("part", DataType::Int32, false))],
- ),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(
- DefaultParquetFileReaderFactory::new(Arc::clone(&store)),
- ),
- pushdown_filters: false, // note that this is false!
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: false, // note that this is false!
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: false,
- }
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_table_schema(table_schema_for_opener.clone())
+ .with_projection_indices(&[0])
+ .with_predicate(predicate)
+ .build()
};
// This filter could prune based on statistics, but since it's not dynamic it's not applied for pruning
@@ -1633,32 +1695,13 @@ mod test {
u64::try_from(data_len).unwrap(),
);
- let make_opener = |reverse_scan: bool| ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &schema),
- batch_size: 1024,
- limit: None,
- predicate: None,
- table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(
- Arc::clone(&store),
- )),
- pushdown_filters: false,
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: false,
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: reverse_scan,
+ let make_opener = |reverse_scan: bool| {
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_reverse_row_groups(reverse_scan)
+ .build()
};
// Test normal scan (forward)
@@ -1694,32 +1737,13 @@ mod test {
u64::try_from(data_size).unwrap(),
);
- let make_opener = |reverse_scan: bool| ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &schema),
- batch_size: 1024,
- limit: None,
- predicate: None,
- table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(
- Arc::clone(&store),
- )),
- pushdown_filters: false,
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: false,
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: reverse_scan,
+ let make_opener = |reverse_scan: bool| {
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_reverse_row_groups(reverse_scan)
+ .build()
};
// With a single row group, forward and reverse should be the same
@@ -1789,32 +1813,13 @@ mod test {
)
.with_extensions(Arc::new(access_plan));
- let make_opener = |reverse_scan: bool| ParquetOpener {
- partition_index: 0,
- projection: ProjectionExprs::from_indices(&[0], &schema),
- batch_size: 1024,
- limit: None,
- predicate: None,
- table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),
- metadata_size_hint: None,
- metrics: ExecutionPlanMetricsSet::new(),
- parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(
- Arc::clone(&store),
- )),
- pushdown_filters: false,
- reorder_filters: false,
- force_filter_selections: false,
- enable_page_index: false,
- enable_bloom_filter: false,
- enable_row_group_stats_pruning: false,
- coerce_int96: None,
- #[cfg(feature = "parquet_encryption")]
- file_decryption_properties: None,
- expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
- #[cfg(feature = "parquet_encryption")]
- encryption_factory: None,
- max_predicate_cache_size: None,
- reverse_row_groups: reverse_scan,
+ let make_opener = |reverse_scan: bool| {
+ ParquetOpenerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_reverse_row_groups(reverse_scan)
+ .build()
};
// Forward scan: RG0(3,4), RG1(5,6,7,8), RG2(9,10)