Apache DataFusion SQL Query Engine
feat: Add option to assign Parquet row groups to file ranges by midpoint (#25940)
Oleks V1b7f40351dec
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 1e23ec894c..bb8772a2ff 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -25,6 +25,7 @@ use crate::error::{_config_datafusion_err, _config_err};use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};use crate::parquet_config::{DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,+ RowGroupRangeAssignment,};use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};use crate::utils::get_available_parallelism;@@ -1423,6 +1424,12 @@ config_namespace! {/// Defaults to 20.pub max_in_list_size: usize, default = 20+ /// (reading) Which byte range of a split file reads each row group.+ /// `start_offset` picks the range containing the row group's start.+ /// `midpoint` picks the range containing its midpoint, as Spark does,+ /// which spreads large row groups more evenly across ranges.+ pub row_group_range_assignment: RowGroupRangeAssignment, default = RowGroupRangeAssignment::StartOffset+// The following options affect writing to parquet files// and map to parquet::file::properties::WriterPropertiesdiff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rsindex af4554c4ee..cd65e3127f 100644--- a/datafusion/common/src/file_options/parquet_writer.rs+++ b/datafusion/common/src/file_options/parquet_writer.rs@@ -249,6 +249,7 @@ impl ParquetOptions {skip_arrow_metadata: _,max_predicate_cache_size: _,max_in_list_size: _,+ row_group_range_assignment: _,} = self;let mut builder = WriterProperties::builder()@@ -409,6 +410,7 @@ mod tests {enable_page_index: defaults.enable_page_index,pruning: defaults.pruning,max_in_list_size: defaults.max_in_list_size,+ row_group_range_assignment: defaults.row_group_range_assignment,skip_metadata: defaults.skip_metadata,metadata_size_hint: defaults.metadata_size_hint,pushdown_filters: defaults.pushdown_filters,@@ -534,6 +536,8 @@ mod tests {enable_page_index: global_options_defaults.enable_page_index,pruning: global_options_defaults.pruning,max_in_list_size: global_options_defaults.max_in_list_size,+ row_group_range_assignment: global_options_defaults+ .row_group_range_assignment,skip_metadata: global_options_defaults.skip_metadata,metadata_size_hint: global_options_defaults.metadata_size_hint,pushdown_filters: global_options_defaults.pushdown_filters,diff --git a/datafusion/common/src/parquet_config.rs b/datafusion/common/src/parquet_config.rsindex c9710188e0..62570abd6d 100644--- a/datafusion/common/src/parquet_config.rs+++ b/datafusion/common/src/parquet_config.rs@@ -107,6 +107,58 @@ impl From<parquet::file::properties::WriterVersion> for DFParquetWriterVersion {}}+/// Which byte range of a split Parquet file reads each row group+///+/// Each row group maps to one offset, so ranges that cover a file read every+/// row group exactly once.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]+pub enum RowGroupRangeAssignment {+ /// The start of the row group's first column chunk+ #[default]+ StartOffset,+ /// That start plus half of the row group's compressed size, as parquet-java+ /// (and therefore Spark) computes it+ Midpoint,+}++impl FromStr for RowGroupRangeAssignment {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self, Self::Err> {+ match s.to_lowercase().as_str() {+ "start_offset" => Ok(Self::StartOffset),+ "midpoint" => Ok(Self::Midpoint),+ other => Err(DataFusionError::Configuration(format!(+ "Invalid parquet row group range assignment: {other}. Expected one of: start_offset, midpoint"+ ))),+ }+ }+}++impl Display for RowGroupRangeAssignment {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ let s = match self {+ Self::StartOffset => "start_offset",+ Self::Midpoint => "midpoint",+ };+ f.write_str(s)+ }+}++impl RowGroupRangeAssignment {+ /// Parses the protobuf form, where an empty string (options encoded before+ /// this setting existed) means the default+ pub fn from_proto_str(s: &str) -> Result<Self> {+ if s.is_empty() {+ Ok(Self::default())+ } else {+ s.parse()+ }+ }+}++crate::config_field!(RowGroupRangeAssignment, value => RowGroupRangeAssignment::from_str(value)?);+/// Parquet statistics levels supported by the writer////// This enum validates statistics settings at configuration time, ensuring onlydiff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex dc0f9a7be4..c66cec9e8c 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -681,6 +681,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {}),max_row_group_size: global_options.global.max_row_group_size as u64,max_in_list_size: global_options.global.max_in_list_size as u64,+ row_group_range_assignment: global_options.global.row_group_range_assignment.to_string(),created_by: global_options.global.created_by.clone(),column_index_truncate_length_opt: global_options.global.column_index_truncate_length.map(|length| {parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(length as u64)diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rsindex b82bd54839..06e8a86de3 100644--- a/datafusion/datasource-parquet/src/metrics.rs+++ b/datafusion/datasource-parquet/src/metrics.rs@@ -143,8 +143,8 @@ pub struct ParquetFileMetrics {/// The clamp and the final top-up also absorb two small inexactnesses in/// crediting by row group: a file is slightly larger than the sum of its row/// groups (the footer, the page index and any padding belong to no row group),-/// and a row group is assigned to a byte range by the offset of its first page,-/// so a range's row groups do not add up to precisely its length.+/// and each row group is credited whole to the one range that reads it, so a+/// range's row groups do not add up to precisely its length.////// The budget is held as a `usize` because [`Count`] is, so the two cannot/// disagree: on a 32-bit target a range longer than `usize::MAX` saturates once,diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex 6b2e47e134..ccd9e098df 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -53,6 +53,7 @@ use std::sync::Arc;use arrow::datatypes::{FieldRef, Schema, SchemaRef, TimeUnit};#[cfg(feature = "parquet_encryption")]use datafusion_common::encryption::FileDecryptionProperties;+use datafusion_common::parquet_config::RowGroupRangeAssignment;use datafusion_common::stats::Precision;use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion_common::{@@ -299,6 +300,9 @@ pub(super) struct ParquetMorselizer {/// lists skip container-level pruning. Sourced from/// `datafusion.execution.parquet.max_in_list_size`.pub max_in_list_size: usize,+ /// How row groups are assigned to the byte ranges of a split file. Sourced+ /// from `datafusion.execution.parquet.row_group_range_assignment`.+ pub row_group_range_assignment: RowGroupRangeAssignment,/// Whether to read row groups in reverse orderpub reverse_row_groups: bool,/// Optional sort order used to reorder row groups by their min/max statistics.@@ -494,6 +498,7 @@ struct PreparedParquetOpen {predicate_creation_errors: Count,max_predicate_cache_size: Option<usize>,max_in_list_size: usize,+ row_group_range_assignment: RowGroupRangeAssignment,reverse_row_groups: bool,sort_order_for_reorder: Option<LexOrdering>,preserve_order: bool,@@ -995,6 +1000,7 @@ impl ParquetMorselizer {predicate_creation_errors,max_predicate_cache_size: self.max_predicate_cache_size,max_in_list_size: self.max_in_list_size,+ row_group_range_assignment: self.row_group_range_assignment,reverse_row_groups: self.reverse_row_groups,sort_order_for_reorder: self.sort_order_for_reorder.clone(),preserve_order: self.preserve_order,@@ -1269,7 +1275,11 @@ impl FiltersPreparedParquetOpen {// If there is a range restricting what parts of the file to readif let Some(range) = prepared.file_range.as_ref() {- row_groups.prune_by_range(rg_metadata, range);+ row_groups.prune_by_range(+ rg_metadata,+ range,+ prepared.row_group_range_assignment,+ );}// Substituting the columns file statistics proved constant@@ -1819,16 +1829,19 @@ impl RowGroupsPrunedParquetOpen {// that may have just proved most of its work unnecessary.let mut byte_progress = prepared.byte_progress;// Every row group still in `rg_plan` is one this range owns, since the- // plan it was built from had `prune_by_range` applied. The planned row- // groups are therefore a subset of the in-range ones, and subtracting- // leaves exactly those the scan will skip.+ // plan it was built from was pruned by range with the same assignment.+ // The planned row groups are therefore a subset of the in-range ones,+ // and subtracting leaves exactly those the scan will skip.let in_range_bytes: u64 = rg_metadata.iter().filter(|rg_meta| {- prepared- .file_range- .as_ref()- .is_none_or(|range| row_group_in_range(rg_meta, range))+ prepared.file_range.as_ref().is_none_or(|range| {+ row_group_in_range(+ rg_meta,+ range,+ prepared.row_group_range_assignment,+ )+ })}).map(row_group_bytes).sum();@@ -2197,6 +2210,7 @@ mod test {coerce_int96: Option<TimeUnit>,max_predicate_cache_size: Option<usize>,max_in_list_size: usize,+ row_group_range_assignment: RowGroupRangeAssignment,reverse_row_groups: bool,preserve_order: bool,}@@ -2420,6 +2434,7 @@ mod test {coerce_int96: None,max_predicate_cache_size: None,max_in_list_size: MAX_IN_LIST_SIZE,+ row_group_range_assignment: RowGroupRangeAssignment::default(),reverse_row_groups: false,preserve_order: false,}@@ -2531,6 +2546,15 @@ mod test {self}+ /// Set how row groups are assigned to the byte ranges of a split file.+ fn with_row_group_range_assignment(+ mut self,+ assignment: RowGroupRangeAssignment,+ ) -> Self {+ self.row_group_range_assignment = assignment;+ self+ }+/// Build the ParquetMorselizer instance, unwrapping validation errors.////// # Panics@@ -2610,6 +2634,7 @@ mod test {encryption_factory: None,max_predicate_cache_size: self.max_predicate_cache_size,max_in_list_size: self.max_in_list_size,+ row_group_range_assignment: self.row_group_range_assignment,reverse_row_groups: self.reverse_row_groups,sort_order_for_reorder: None,virtual_state,@@ -3028,56 +3053,61 @@ mod test {/// A file split into byte ranges for parallelism: each range credits its/// own size and no more, so the ranges add up to the file exactly rather- /// than each claiming all of it.+ /// than each claiming all of it. The split falls just after row group 1+ /// starts, where the assignments disagree on which range reads it.#[tokio::test]async fn each_range_of_a_split_file_credits_only_its_own_bytes() {let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;let (schema, data_len) = write_three_row_groups(Arc::clone(&store)).await;-- let split = i64::try_from(data_len).unwrap() / 2;+ let file = PartitionedFile::new("test.parquet".to_string(), data_len);+ let metadata = DFParquetMetadata::new(store.as_ref(), &file.object_meta)+ .fetch_metadata()+ .await+ .unwrap();+ let rg1_start = metadata.row_group(1).column(0).byte_range().0;+ let split = i64::try_from(rg1_start).unwrap() + 1;let ranges = [(0, split), (split, i64::try_from(data_len).unwrap())];- let mut total_processed = 0;- let mut total_rows = 0;- for (start, end) in ranges {- let file = PartitionedFile::new_with_range(- "test.parquet".to_string(),- data_len,- start,- end,- );- let expected = file.effective_size();- let metrics = ExecutionPlanMetricsSet::new();-- let morselizer = ParquetMorselizerBuilder::new()- .with_store(Arc::clone(&store))- .with_schema(Arc::clone(&schema))- .with_projection_indices(&[0])- .with_metrics(metrics.clone())- .build();-- let stream = open_file(&morselizer, file).await.unwrap();- assert_eq!(- bytes_processed(&metrics),- 0,- "nothing is pruned here, so range [{start}, {end}) must credit \- nothing at open: every row group it owns is one it will read, \- and the row groups it does not own belong to the other range",- );-- let (_, rows) = count_batches_and_rows(stream).await;-- assert_eq!(- bytes_processed(&metrics),- expected,- "range [{start}, {end}) must credit exactly its own length",- );- total_processed += expected;- total_rows += rows;+ for (assignment, expected) in [+ (RowGroupRangeAssignment::StartOffset, [1..=6, 7..=9]),+ (RowGroupRangeAssignment::Midpoint, [1..=3, 4..=9]),+ ] {+ for ((start, end), expected) in ranges.into_iter().zip(expected) {+ let file = PartitionedFile::new_with_range(+ "test.parquet".to_string(),+ data_len,+ start,+ end,+ );+ let expected_bytes = file.effective_size();+ let metrics = ExecutionPlanMetricsSet::new();+ let morselizer = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_projection_indices(&[0])+ .with_metrics(metrics.clone())+ .with_row_group_range_assignment(assignment)+ .build();++ let stream = open_file(&morselizer, file).await.unwrap();+ assert_eq!(+ bytes_processed(&metrics),+ 0,+ "{assignment} range [{start}, {end}) must credit nothing at \+ open: it reads every row group it owns",+ );+ assert_eq!(+ collect_int32_values(stream).await,+ expected.collect::<Vec<_>>(),+ "{assignment} range [{start}, {end})",+ );+ assert_eq!(+ bytes_processed(&metrics),+ expected_bytes,+ "{assignment} range [{start}, {end}) must credit exactly its own length",+ );+ }}-- assert_eq!(total_rows, 9, "the ranges together must scan every row");- assert_eq!(total_processed, data_len);}/// A file that cannot be opened at all is still a file the scan is donediff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rsindex 626ae46445..fb7e78755d 100644--- a/datafusion/datasource-parquet/src/row_group_filter.rs+++ b/datafusion/datasource-parquet/src/row_group_filter.rs@@ -25,6 +25,7 @@ use crate::pruning::build_inverted_predicate;use arrow::array::{ArrayRef, BooleanArray, UInt64Array};use arrow::compute::nullif;use arrow::datatypes::Schema;+use datafusion_common::parquet_config::RowGroupRangeAssignment;use datafusion_common::pruning::PruningStatistics;use datafusion_common::{Column, Result, ScalarValue};use datafusion_datasource::FileRange;@@ -46,19 +47,32 @@ pub struct RowGroupAccessPlanFilter {access_plan: ParquetAccessPlan,}-/// Returns true if this row group belongs to `range`.+/// Returns true if `assignment` assigns this row group to `range`.///-/// A row group belongs to the range containing its first dictionary/data page,-/// so the ranges a file is split into for parallelism partition its row groups-/// with none shared and none left over.+/// Each row group maps to a single offset, so the ranges a file is split into+/// for parallelism partition its row groups with none shared and none left over.////// Note: don't use the location of metadata/// <https://github.com/apache/datafusion/issues/5995>-pub(crate) fn row_group_in_range(metadata: &RowGroupMetaData, range: &FileRange) -> bool {+pub(crate) fn row_group_in_range(+ metadata: &RowGroupMetaData,+ range: &FileRange,+ assignment: RowGroupRangeAssignment,+) -> bool {let col = metadata.column(0);- let offset = col- .dictionary_page_offset()- .unwrap_or_else(|| col.data_page_offset());+ let data_page_offset = col.data_page_offset();+ let offset = match assignment {+ // The rule from before this option existed. Unlike `Midpoint`, it+ // keeps a dictionary page offset that follows the first data page.+ RowGroupRangeAssignment::StartOffset => {+ col.dictionary_page_offset().unwrap_or(data_page_offset)+ }+ // Like parquet-java, ignore a dictionary page after the first data page+ RowGroupRangeAssignment::Midpoint => col+ .dictionary_page_offset()+ .map_or(data_page_offset, |offset| offset.min(data_page_offset))+ .saturating_add(metadata.compressed_size() / 2),+ };range.contains(offset)}@@ -243,20 +257,26 @@ impl RowGroupAccessPlanFilter {}}- /// Prune remaining row groups to only those within the specified range.+ /// Prune remaining row groups to only those that `assignment` assigns to+ /// the specified range.////// Updates this set to mark row groups that should not be scanned////// # Panics/// if `groups.len() != self.len()`- pub fn prune_by_range(&mut self, groups: &[RowGroupMetaData], range: &FileRange) {+ pub fn prune_by_range(+ &mut self,+ groups: &[RowGroupMetaData],+ range: &FileRange,+ assignment: RowGroupRangeAssignment,+ ) {assert_eq!(groups.len(), self.access_plan.len());for (idx, metadata) in groups.iter().enumerate() {if !self.access_plan.should_scan(idx) {continue;}- if !row_group_in_range(metadata, range) {+ if !row_group_in_range(metadata, range, assignment) {self.access_plan.skip(idx);}}@@ -682,6 +702,104 @@ mod tests {assert_eq!(filter.remaining_row_group_count(), 2);}+ /// A row group with one column chunk of `size` bytes at the given offsets+ fn row_group_at(+ dictionary_page_offset: Option<i64>,+ data_page_offset: i64,+ size: i64,+ ) -> RowGroupMetaData {+ let schema_descr = get_test_schema_descr(vec![PrimitiveTypeField::new(+ "c1",+ PhysicalType::INT32,+ )]);+ let column = ColumnChunkMetaData::builder(schema_descr.column(0))+ .set_dictionary_page_offset(dictionary_page_offset)+ .set_data_page_offset(data_page_offset)+ .set_total_compressed_size(size)+ .build()+ .unwrap();+ RowGroupMetaData::builder(schema_descr)+ .set_column_metadata(vec![column])+ .build()+ .unwrap()+ }++ /// Indexes of the row groups that `assignment` gives to `[start, end)`+ fn row_groups_in_range(+ groups: &[RowGroupMetaData],+ start: i64,+ end: i64,+ assignment: RowGroupRangeAssignment,+ ) -> Vec<usize> {+ let mut filter =+ RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(groups.len()));+ filter.prune_by_range(groups, &FileRange { start, end }, assignment);+ filter.build().row_group_indexes()+ }++ /// Two row groups that both start before a 128MB split, as in+ /// apache/datafusion-comet#3817+ #[test]+ fn prune_by_range_with_large_row_groups() {+ use RowGroupRangeAssignment::{Midpoint, StartOffset};++ const MB: i64 = 1024 * 1024;+ let groups = [+ row_group_at(None, 4, 122 * MB - 4),+ row_group_at(None, 122 * MB, 67 * MB),+ ];+ let (split, end) = (128 * MB, 190 * MB);++ assert_eq!(row_groups_in_range(&groups, 0, split, StartOffset), [0, 1]);+ assert!(row_groups_in_range(&groups, split, end, StartOffset).is_empty());+ assert_eq!(row_groups_in_range(&groups, 0, split, Midpoint), [0]);+ assert_eq!(row_groups_in_range(&groups, split, end, Midpoint), [1]);+ }++ /// Wherever a file is split in two, each row group is read exactly once+ #[test]+ fn prune_by_range_assigns_each_row_group_once() {+ let mut groups = vec![];+ let mut start = 4;+ for size in [100, 5, 300, 40] {+ groups.push(row_group_at(None, start, size));+ start += size;+ }+ let end = start + 60;+ let all: Vec<usize> = (0..groups.len()).collect();++ for assignment in [+ RowGroupRangeAssignment::StartOffset,+ RowGroupRangeAssignment::Midpoint,+ ] {+ for split in 0..=end {+ let mut indexes = row_groups_in_range(&groups, 0, split, assignment);+ indexes.extend(row_groups_in_range(&groups, split, end, assignment));+ assert_eq!(indexes, all, "{assignment} split at {split}");+ }+ }+ }++ /// Like parquet-java, `Midpoint` ignores a dictionary page offset after the+ /// first data page+ #[test]+ fn midpoint_ignores_dictionary_offset_after_first_data_page() {+ let group = row_group_at(Some(1000), 100, 200);+ let range = FileRange { start: 0, end: 500 };++ // Midpoint is 100 + 200 / 2 = 200, start offset is 1000+ assert!(row_group_in_range(+ &group,+ &range,+ RowGroupRangeAssignment::Midpoint+ ));+ assert!(!row_group_in_range(+ &group,+ &range,+ RowGroupRangeAssignment::StartOffset+ ));+ }+#[test]fn row_group_pruning_predicate_simple_expr() {use datafusion_expr::{col, lit};diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 4959a56ce6..789b3c72d0 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -677,6 +677,10 @@ impl FileSource for ParquetSource {encryption_factory: self.get_encryption_factory_with_config(),max_predicate_cache_size: self.max_predicate_cache_size(),max_in_list_size: self.max_in_list_size(),+ row_group_range_assignment: self+ .table_parquet_options+ .global+ .row_group_range_assignment,reverse_row_groups: self.reverse_row_groups,sort_order_for_reorder: self.sort_order_for_reorder.clone(),virtual_state,diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rsindex 86549a96b3..bbe14280a6 100644--- a/datafusion/datasource/src/mod.rs+++ b/datafusion/datasource/src/mod.rs@@ -88,9 +88,11 @@ pub type FileExtensions = datafusion_common::extensions::Extensions;pub type PartitionedFileStream =Pin<Box<dyn Stream<Item = Result<PartitionedFile>> + Send + Sync + 'static>>;-/// Only scan a subset of Row Groups from the Parquet file whose data "midpoint"-/// lies within the [start, end) byte offsets. This option can be used to scan non-overlapping-/// sections of a Parquet file in parallel.+/// A `[start, end)` byte range of a file. Non-overlapping ranges can be scanned+/// in parallel. For Parquet, [`RowGroupRangeAssignment`] picks the row groups a+/// range reads.+///+/// [`RowGroupRangeAssignment`]: datafusion_common::parquet_config::RowGroupRangeAssignment#[derive(Debug, Clone, PartialEq, Hash, Eq, PartialOrd, Ord)]pub struct FileRange {/// Range startdiff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex bdabb0777e..8e189034d4 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -626,6 +626,9 @@ message ParquetOptions {uint64 max_in_list_size = 38;+ // "start_offset" or "midpoint". Empty means "start_offset".+ string row_group_range_assignment = 39;+string created_by = 16;oneof coerce_int96_opt {diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 0bde696c7c..23033c2ba1 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -44,6 +44,7 @@ use datafusion_common::{ParquetColumnOptions, ParquetOptions, TableParquetOptions,},file_options::{csv_writer::CsvWriterOptions, json_writer::JsonWriterOptions},+ parquet_config::RowGroupRangeAssignment,parsers::CompressionTypeVariant,plan_datafusion_err,stats::Precision,@@ -1219,6 +1220,9 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {.transpose()?,max_row_group_size: to_usize(value.max_row_group_size, "max_row_group_size")?,max_in_list_size: to_usize(value.max_in_list_size, "max_in_list_size")?,+ row_group_range_assignment: RowGroupRangeAssignment::from_proto_str(+ &value.row_group_range_assignment,+ )?,created_by: value.created_by.clone(),column_index_truncate_length: value.column_index_truncate_length_opt.as_ref()@@ -1438,7 +1442,9 @@ mod tests {use datafusion_common::config::{MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,};- use datafusion_common::parquet_config::{DFParquetCompression, DFParquetStatistics};+ use datafusion_common::parquet_config::{+ DFParquetCompression, DFParquetStatistics, RowGroupRangeAssignment,+ };#[test]fn constraint_requires_mode() {@@ -1548,6 +1554,24 @@ mod tests {assert_eq!(recovered.coerce_int96_tz, Some("UTC".to_string()));}+ #[test]+ fn test_parquet_options_row_group_range_assignment_round_trip() {+ let opts = ParquetOptions {+ row_group_range_assignment: RowGroupRangeAssignment::Midpoint,+ ..ParquetOptions::default()+ };+ let mut proto: crate::protobuf_common::ParquetOptions =+ (&opts).try_into().expect("to_proto");+ assert_eq!(ParquetOptions::try_from(&proto).unwrap(), opts);++ // Options encoded before the field existed decode with the default+ proto.row_group_range_assignment.clear();+ assert_eq!(+ ParquetOptions::try_from(&proto).unwrap(),+ ParquetOptions::default()+ );+ }+#[test]fn test_parquet_statistics_round_trip() {let opts = ParquetOptions {diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex a1d86c5fe3..d7adf4ac71 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -6491,6 +6491,9 @@ impl serde::Serialize for ParquetOptions {if self.max_in_list_size != 0 {len += 1;}+ if !self.row_group_range_assignment.is_empty() {+ len += 1;+ }if !self.created_by.is_empty() {len += 1;}@@ -6616,6 +6619,9 @@ impl serde::Serialize for ParquetOptions {#[allow(clippy::needless_borrows_for_generic_args)]struct_ser.serialize_field("maxInListSize", ToString::to_string(&self.max_in_list_size).as_str())?;}+ if !self.row_group_range_assignment.is_empty() {+ struct_ser.serialize_field("rowGroupRangeAssignment", &self.row_group_range_assignment)?;+ }if !self.created_by.is_empty() {struct_ser.serialize_field("createdBy", &self.created_by)?;}@@ -6776,6 +6782,8 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {"maxRowGroupSize","max_in_list_size","maxInListSize",+ "row_group_range_assignment",+ "rowGroupRangeAssignment","created_by","createdBy","content_defined_chunking",@@ -6829,6 +6837,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {DataPageRowCountLimit,MaxRowGroupSize,MaxInListSize,+ RowGroupRangeAssignment,CreatedBy,ContentDefinedChunking,MetadataSizeHint,@@ -6886,6 +6895,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {"dataPageRowCountLimit" | "data_page_row_count_limit" => Ok(GeneratedField::DataPageRowCountLimit),"maxRowGroupSize" | "max_row_group_size" => Ok(GeneratedField::MaxRowGroupSize),"maxInListSize" | "max_in_list_size" => Ok(GeneratedField::MaxInListSize),+ "rowGroupRangeAssignment" | "row_group_range_assignment" => Ok(GeneratedField::RowGroupRangeAssignment),"createdBy" | "created_by" => Ok(GeneratedField::CreatedBy),"contentDefinedChunking" | "content_defined_chunking" => Ok(GeneratedField::ContentDefinedChunking),"metadataSizeHint" | "metadata_size_hint" => Ok(GeneratedField::MetadataSizeHint),@@ -6941,6 +6951,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {let mut data_page_row_count_limit__ = None;let mut max_row_group_size__ = None;let mut max_in_list_size__ = None;+ let mut row_group_range_assignment__ = None;let mut created_by__ = None;let mut content_defined_chunking__ = None;let mut metadata_size_hint_opt__ = None;@@ -7100,6 +7111,12 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0);}+ GeneratedField::RowGroupRangeAssignment => {+ if row_group_range_assignment__.is_some() {+ return Err(serde::de::Error::duplicate_field("rowGroupRangeAssignment"));+ }+ row_group_range_assignment__ = Some(map_.next_value()?);+ }GeneratedField::CreatedBy => {if created_by__.is_some() {return Err(serde::de::Error::duplicate_field("createdBy"));@@ -7214,6 +7231,7 @@ impl<'de> serde::Deserialize<'de> for ParquetOptions {data_page_row_count_limit: data_page_row_count_limit__.unwrap_or_default(),max_row_group_size: max_row_group_size__.unwrap_or_default(),max_in_list_size: max_in_list_size__.unwrap_or_default(),+ row_group_range_assignment: row_group_range_assignment__.unwrap_or_default(),created_by: created_by__.unwrap_or_default(),content_defined_chunking: content_defined_chunking__,metadata_size_hint_opt: metadata_size_hint_opt__,diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex f856c1aa39..d045c93f65 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -875,6 +875,9 @@ pub struct ParquetOptions {pub max_row_group_size: u64,#[prost(uint64, tag = "38")]pub max_in_list_size: u64,+ /// "start_offset" or "midpoint". Empty means "start_offset".+ #[prost(string, tag = "39")]+ pub row_group_range_assignment: ::prost::alloc::string::String,#[prost(string, tag = "16")]pub created_by: ::prost::alloc::string::String,#[prost(message, optional, tag = "35")]diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex b8eb46f2c8..dca6606124 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -948,6 +948,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {statistics_enabled_opt: value.statistics_enabled.map(|v| protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v.to_string())),max_row_group_size: value.max_row_group_size as u64,max_in_list_size: value.max_in_list_size as u64,+ row_group_range_assignment: value.row_group_range_assignment.to_string(),created_by: value.created_by.clone(),column_index_truncate_length_opt: value.column_index_truncate_length.map(|v| protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v as u64)),statistics_truncate_length_opt: value.statistics_truncate_length.map(|v| protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v as u64)),diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rsindex fdf3239647..7896f63fa5 100644--- a/datafusion/proto-models/src/from_proto.rs+++ b/datafusion/proto-models/src/from_proto.rs@@ -30,6 +30,7 @@ use datafusion_common::config::{ParquetOptions, TableParquetOptions,};use datafusion_common::display::{PlanType, StringifiedPlan};+use datafusion_common::parquet_config::RowGroupRangeAssignment;use datafusion_common::parsers::{CompressionTypeVariant, CsvQuoteStyle};use datafusion_common::utils::usize_from_wire;use datafusion_common::{@@ -411,6 +412,9 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {.transpose()?,max_row_group_size: to_usize(proto.max_row_group_size, "max_row_group_size")?,max_in_list_size: to_usize(proto.max_in_list_size, "max_in_list_size")?,+ row_group_range_assignment: RowGroupRangeAssignment::from_proto_str(+ &proto.row_group_range_assignment,+ )?,created_by: proto.created_by.clone(),column_index_truncate_length: proto.column_index_truncate_length_optdiff --git a/datafusion/proto-models/src/generated/datafusion_proto_common.rs b/datafusion/proto-models/src/generated/datafusion_proto_common.rsindex f856c1aa39..d045c93f65 100644--- a/datafusion/proto-models/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto-models/src/generated/datafusion_proto_common.rs@@ -875,6 +875,9 @@ pub struct ParquetOptions {pub max_row_group_size: u64,#[prost(uint64, tag = "38")]pub max_in_list_size: u64,+ /// "start_offset" or "midpoint". Empty means "start_offset".+ #[prost(string, tag = "39")]+ pub row_group_range_assignment: ::prost::alloc::string::String,#[prost(string, tag = "16")]pub created_by: ::prost::alloc::string::String,#[prost(message, optional, tag = "35")]diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 7c7a8ef639..e53b7d5a12 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -333,6 +333,10 @@ mod parquet {options.global.writer_version,ParquetOptions::default().writer_version);+ assert_eq!(+ options.global.row_group_range_assignment,+ ParquetOptions::default().row_group_range_assignment+ );}}}diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex a479512c3f..c19b92579e 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -32,7 +32,9 @@ use datafusion::datasource::listing::{use datafusion::execution::options::{ArrowReadOptions, JsonReadOptions};use datafusion::optimizer::Optimizer;use datafusion::optimizer::optimize_unions::OptimizeUnions;-use datafusion_common::parquet_config::DFParquetWriterVersion;+use datafusion_common::parquet_config::{+ DFParquetWriterVersion, RowGroupRangeAssignment,+};use datafusion_common::parsers::CompressionTypeVariant;use datafusion_functions_aggregate::sum::sum_distinct;use prost::Message;@@ -1165,6 +1167,7 @@ async fn roundtrip_logical_plan_copy_to_parquet() -> Result<()> {parquet_format.global.allow_single_file_parallelism = false;parquet_format.global.created_by = "test".to_string();+ parquet_format.global.row_group_range_assignment = RowGroupRangeAssignment::Midpoint;let file_type = format_as_file_type(Arc::new(ParquetFormatFactory::new_with_options(parquet_format.clone()),@@ -1207,6 +1210,10 @@ async fn roundtrip_logical_plan_copy_to_parquet() -> Result<()> {assert_eq!(parquet_config.key_value_metadata, key_value_metadata);assert!(!parquet_config.global.allow_single_file_parallelism);assert_eq!(parquet_config.global.created_by, "test".to_string());+ assert_eq!(+ parquet_config.global.row_group_range_assignment,+ RowGroupRangeAssignment::Midpoint+ );}_ => panic!(),}diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex f6633a165d..a69305c418 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -263,6 +263,7 @@ datafusion.execution.parquet.metadata_size_hint 524288datafusion.execution.parquet.pruning truedatafusion.execution.parquet.pushdown_filters falsedatafusion.execution.parquet.reorder_filters false+datafusion.execution.parquet.row_group_range_assignment start_offsetdatafusion.execution.parquet.schema_force_view_types truedatafusion.execution.parquet.skip_arrow_metadata falsedatafusion.execution.parquet.skip_metadata true@@ -425,6 +426,7 @@ datafusion.execution.parquet.metadata_size_hint 524288 (reading) If specified, tdatafusion.execution.parquet.pruning true (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet filedatafusion.execution.parquet.pushdown_filters false (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization".datafusion.execution.parquet.reorder_filters false (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query+datafusion.execution.parquet.row_group_range_assignment start_offset (reading) Which byte range of a split file reads each row group. `start_offset` picks the range containing the row group's start. `midpoint` picks the range containing its midpoint, as Spark does, which spreads large row groups more evenly across ranges.datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types.datafusion.execution.parquet.skip_arrow_metadata false (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to <https://docs.rs/parquet/53.3.0/parquet/arrow/arrow_writer/struct.ArrowWriterOptions.html#method.with_skip_arrow_metadata>datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadatadiff --git a/datafusion/sqllogictest/test_files/repartition_scan.slt b/datafusion/sqllogictest/test_files/repartition_scan.sltindex fca1223cb0..150d5b1b43 100644--- a/datafusion/sqllogictest/test_files/repartition_scan.slt+++ b/datafusion/sqllogictest/test_files/repartition_scan.slt@@ -147,6 +147,46 @@ DROP TABLE parquet_table;statement okDROP TABLE parquet_table_with_order;+## Either row group range assignment reads each row of a split file once++statement ok+COPY (SELECT v FROM generate_series(1, 4000) AS t(v))+TO 'test_files/scratch/repartition_scan/multi_row_group/data.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '1000');++statement ok+CREATE EXTERNAL TABLE multi_row_group(v bigint)+STORED AS PARQUET+LOCATION 'test_files/scratch/repartition_scan/multi_row_group/';++query IIII+SELECT count(*), sum(v), min(v), max(v) FROM multi_row_group WHERE v <> 0;+----+4000 8002000 1 4000++statement ok+set datafusion.execution.parquet.row_group_range_assignment = 'midpoint';++query IIII+SELECT count(*), sum(v), min(v), max(v) FROM multi_row_group WHERE v <> 0;+----+4000 8002000 1 4000++query error+set datafusion.execution.parquet.row_group_range_assignment = 'middle';+----+DataFusion error: Error setting config datafusion.execution.parquet.row_group_range_assignment+caused by+Invalid or Unsupported Configuration: Invalid parquet row group range assignment: middle. Expected one of: start_offset, midpoint+++statement ok+RESET datafusion.execution.parquet.row_group_range_assignment;++statement ok+DROP TABLE multi_row_group;+###################### CSV testsdiff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 4cc7c30e17..ef27ef0a1e 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -91,6 +91,26 @@ accept `RowGroupSelection` values through `with_row_group_selections`; thesynchronous reader still requires `with_row_groups` and an overall`with_row_selection`.+### `RowGroupAccessPlanFilter::prune_by_range` takes a `RowGroupRangeAssignment`++The new `datafusion.execution.parquet.row_group_range_assignment` option decides+which byte range of a split Parquet file reads each row group. The default,+`start_offset`, keeps the previous behavior. `midpoint` matches Apache Spark.++`RowGroupAccessPlanFilter::prune_by_range` takes the assignment as a new+argument. Pass `RowGroupRangeAssignment::StartOffset` for the previous behavior:++```rust,ignore+// Before+filter.prune_by_range(groups, &range);+// After+filter.prune_by_range(groups, &range, RowGroupRangeAssignment::StartOffset);+```++`ParquetOptions` and its generated protobuf struct gained a+`row_group_range_assignment` field. Exhaustive struct literals must set it or+use `..Default::default()`.+### Missing Parquet null counts are treated as unknownDataFusion now preserves an omitted Parquet `null_count` statistic as unknown.diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex ba8ea52bb0..dd01b93cb9 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -94,6 +94,7 @@ The following configuration settings are available:| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files || datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. || datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. |+| datafusion.execution.parquet.row_group_range_assignment | start_offset | (reading) Which byte range of a split file reads each row group. `start_offset` picks the range containing the row group's start. `midpoint` picks the range containing its midpoint, as Spark does, which spreads large row groups more evenly across ranges. || datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes || datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows || datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" |