Skip to content

Apache DataFusion SQL Query Engine

fix: discard parquet bounds when row group statistics are missing (#25228)

Huaijin9082d6b10c29
diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rs
index 213cac24a8..c62b0de9cd 100644
--- a/datafusion/datasource-parquet/src/metadata.rs
+++ b/datafusion/datasource-parquet/src/metadata.rs
@@ -739,9 +739,9 @@ fn summarize_column_statistics(
) -> Result<()> {
let parquet_index = stats_converter.parquet_column_index();
- if let Some(max_acc) = &mut accumulators.max_accs[logical_schema_index] {
+ if accumulators.max_accs[logical_schema_index].is_some() {
accumulators.is_max_value_exact[logical_schema_index] = summarize_bound(
- max_acc,
+ &mut accumulators.max_accs[logical_schema_index],
&stats_converter.row_group_maxes(row_groups_metadata)?,
parquet_index,
row_groups_metadata,
@@ -750,9 +750,9 @@ fn summarize_column_statistics(
)?;
}
- if let Some(min_acc) = &mut accumulators.min_accs[logical_schema_index] {
+ if accumulators.min_accs[logical_schema_index].is_some() {
accumulators.is_min_value_exact[logical_schema_index] = summarize_bound(
- min_acc,
+ &mut accumulators.min_accs[logical_schema_index],
&stats_converter.row_group_mins(row_groups_metadata)?,
parquet_index,
row_groups_metadata,
@@ -785,13 +785,41 @@ fn summarize_column_statistics(
/// parquet statistics. `row_group_exactness` rebuilds the exactness as a Boolean
/// array and is only called for the rare case where row groups disagree.
fn summarize_bound<A: Accumulator>(
- acc: &mut A,
+ acc: &mut Option<A>,
values: &ArrayRef,
parquet_index: Option<usize>,
row_groups_metadata: &[RowGroupMetaData],
is_exact: impl Fn(&ParquetStatistics) -> bool,
row_group_exactness: impl FnOnce() -> Result<BooleanArray>,
) -> Result<Option<bool>> {
+ // A NULL converted bound can mean missing statistics, not just all-NULL
+ // data. Ignoring it in MIN/MAX would let another row group's exact endpoint
+ // incorrectly establish an exact bound for the whole file. Drop this bound
+ // unless the row group is empty or is proven to contain only NULLs.
+ if values.null_count() > 0
+ && parquet_index.is_some_and(|column_index| {
+ row_groups_metadata
+ .iter()
+ .enumerate()
+ .any(|(index, group)| {
+ if values.is_valid(index) || group.num_rows() == 0 {
+ return false;
+ }
+ let column = group.column(column_index);
+ let all_null = column
+ .statistics()
+ .and_then(|stats| stats.null_count_opt())
+ .is_some_and(|nulls| nulls == column.num_values() as u64);
+ !all_null
+ })
+ })
+ {
+ *acc = None;
+ return Ok(None);
+ }
+ let Some(acc) = acc.as_mut() else {
+ return Ok(None);
+ };
acc.update_batch(&[Arc::clone(values)])?;
Ok(
@@ -1281,7 +1309,7 @@ mod tests {
);
}
- mod ndv_tests {
+ mod statistics_tests {
use super::*;
use arrow::datatypes::Field;
use parquet::basic::Type as PhysicalType;
@@ -1678,6 +1706,108 @@ mod tests {
);
}
+ #[test]
+ fn test_min_max_require_complete_row_group_bounds() {
+ let known =
+ || ParquetStatistics::int32(Some(1), Some(3), None, Some(0), false);
+ let exact = |v| Precision::Exact(ScalarValue::Int32(Some(v)));
+ let cases = [
+ (None, 3, Precision::Absent, Precision::Absent),
+ (
+ Some(ParquetStatistics::int32(
+ None,
+ Some(6),
+ None,
+ Some(0),
+ false,
+ )),
+ 3,
+ Precision::Absent,
+ exact(6),
+ ),
+ (
+ Some(ParquetStatistics::int32(
+ Some(4),
+ None,
+ None,
+ Some(0),
+ false,
+ )),
+ 3,
+ exact(1),
+ Precision::Absent,
+ ),
+ (
+ Some(ParquetStatistics::int32(None, None, None, None, false)),
+ 3,
+ Precision::Absent,
+ Precision::Absent,
+ ),
+ (
+ Some(ParquetStatistics::int32(None, None, None, Some(2), false)),
+ 3,
+ Precision::Absent,
+ Precision::Absent,
+ ),
+ // Empty and proven all-NULL row groups do not contribute extrema.
+ (None, 0, exact(1), exact(3)),
+ (
+ Some(ParquetStatistics::int32(None, None, None, Some(3), false)),
+ 3,
+ exact(1),
+ exact(3),
+ ),
+ (
+ Some(ParquetStatistics::int32(
+ Some(4),
+ Some(6),
+ None,
+ Some(0),
+ false,
+ )),
+ 3,
+ exact(1),
+ exact(6),
+ ),
+ ];
+ for (other, rows, expected_min, expected_max) in cases {
+ for reverse in [false, true] {
+ let schema_descr = create_schema_descr(1);
+ let arrow_schema = create_arrow_schema(1);
+ let mut groups = vec![
+ create_row_group_with_stats(
+ &schema_descr,
+ vec![Some(known())],
+ 3,
+ ),
+ create_row_group_with_stats(
+ &schema_descr,
+ vec![other.clone()],
+ rows,
+ ),
+ ];
+ if reverse {
+ groups.reverse();
+ }
+ let metadata = create_parquet_metadata(schema_descr, groups);
+ let stats = DFParquetMetadata::statistics_from_parquet_metadata(
+ &metadata,
+ &arrow_schema,
+ )
+ .unwrap();
+ assert_eq!(
+ stats.column_statistics[0].min_value, expected_min,
+ "other={other:?}, rows={rows}, reverse={reverse}"
+ );
+ assert_eq!(
+ stats.column_statistics[0].max_value, expected_max,
+ "other={other:?}, rows={rows}, reverse={reverse}"
+ );
+ assert_eq!(stats.num_rows, Precision::Exact((3 + rows) as usize));
+ }
+ }
+ }
+
#[test]
fn test_distinct_count_no_statistics_at_all() {
// No statistics in row group should return Absent for all stats
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index 307a8876ab..87d39e34dc 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -148,6 +148,9 @@ impl TestContext {
let file_name = relative_path.file_name().unwrap().to_str().unwrap();
match file_name {
+ "parquet_missing_bounds.slt" => {
+ register_parquet_missing_bounds(&mut test_ctx).await;
+ }
"cte.slt" => {
info!("Registering strict schema provider for CTE tests");
register_strict_schema_provider(test_ctx.session_ctx());
@@ -415,6 +418,77 @@ pub async fn register_partition_table(test_ctx: &mut TestContext) {
.unwrap();
}
+/// Write row groups with different statistics settings using the public writer API.
+async fn register_parquet_missing_bounds(test_ctx: &mut TestContext) {
+ use datafusion::parquet::column::writer::ColumnWriterImpl;
+ use datafusion::parquet::data_type::{ByteArray, ByteArrayType};
+ use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties};
+ use datafusion::parquet::file::writer::{
+ SerializedFileWriter, SerializedPageWriter, TrackedWrite,
+ };
+ use datafusion::parquet::schema::parser::parse_message_type;
+
+ test_ctx.enable_testdir();
+ let path = test_ctx.testdir_path().join("missing_bounds.parquet");
+ let column_path = test_ctx.testdir_path().join("column.pages");
+ let schema = Arc::new(
+ parse_message_type("message schema { REQUIRED BINARY a (UTF8); }").unwrap(),
+ );
+ let mut writer = SerializedFileWriter::new(
+ File::create(&path).unwrap(),
+ schema,
+ Arc::new(WriterProperties::default()),
+ )
+ .unwrap();
+ let long_value = "z".repeat(8192);
+ for (values, statistics) in [
+ (["a", "b"], EnabledStatistics::Chunk),
+ (
+ [long_value.as_str(), long_value.as_str()],
+ EnabledStatistics::None,
+ ),
+ ] {
+ // parquet-rs truncates long extrema rather than omitting them. Disable
+ // statistics for the second chunk to exercise the missing-bound case
+ // produced naturally by writers such as PyArrow, without editing metadata.
+ let properties = Arc::new(
+ WriterProperties::builder()
+ .set_statistics_enabled(statistics)
+ .build(),
+ );
+ let mut buffer = TrackedWrite::new(File::create(&column_path).unwrap());
+ let mut column = ColumnWriterImpl::<ByteArrayType>::new(
+ writer.schema_descr().column(0),
+ properties,
+ Box::new(SerializedPageWriter::new(&mut buffer)),
+ );
+ let values = values.map(ByteArray::from);
+ column.write_batch(&values, None, None).unwrap();
+ let result = column.close().unwrap();
+ assert_eq!(
+ result.metadata.statistics().is_some(),
+ statistics == EnabledStatistics::Chunk
+ );
+ buffer.into_inner().unwrap();
+ let mut group = writer.next_row_group().unwrap();
+ group
+ .append_column(&File::open(&column_path).unwrap(), result)
+ .unwrap();
+ group.close().unwrap();
+ }
+ writer.close().unwrap();
+ std::fs::remove_file(&column_path).unwrap();
+ test_ctx
+ .ctx
+ .register_parquet(
+ "missing_bounds",
+ path.to_str().unwrap(),
+ ParquetReadOptions::default(),
+ )
+ .await
+ .unwrap();
+}
+
// registers a LOCAL TEMPORARY table.
pub fn register_temp_table(ctx: &SessionContext) {
#[derive(Debug)]
diff --git a/datafusion/sqllogictest/test_files/parquet_missing_bounds.slt b/datafusion/sqllogictest/test_files/parquet_missing_bounds.slt
new file mode 100644
index 0000000000..8b303b4b43
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/parquet_missing_bounds.slt
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The test context writes two row groups using the Rust writer API:
+# ["a", "b"] with statistics, and two 8192-character strings without statistics.
+# This models the missing bounds produced by writers that omit oversized extrema.
+
+# Verify that the long values are present in the actual data.
+query I
+SELECT LENGTH(a) FROM missing_bounds ORDER BY a;
+----
+1
+1
+8192
+8192
+
+# The first group's maximum must not be treated as the entire file's maximum.
+query II
+SELECT LENGTH(MIN(a)), LENGTH(MAX(a)) FROM missing_bounds;
+----
+1 8192
+
+statement ok
+DROP TABLE missing_bounds;