Apache DataFusion SQL Query Engine
fix: discard parquet bounds when row group statistics are missing (#25228)
Huaijin9082d6b10c29
diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rsindex 213cac24a8..c62b0de9cd 100644--- a/datafusion/datasource-parquet/src/metadata.rs+++ b/datafusion/datasource-parquet/src/metadata.rs@@ -739,9 +739,9 @@ fn summarize_column_statistics() -> Result<()> {let parquet_index = stats_converter.parquet_column_index();- if let Some(max_acc) = &mut accumulators.max_accs[logical_schema_index] {+ if accumulators.max_accs[logical_schema_index].is_some() {accumulators.is_max_value_exact[logical_schema_index] = summarize_bound(- max_acc,+ &mut accumulators.max_accs[logical_schema_index],&stats_converter.row_group_maxes(row_groups_metadata)?,parquet_index,row_groups_metadata,@@ -750,9 +750,9 @@ fn summarize_column_statistics()?;}- if let Some(min_acc) = &mut accumulators.min_accs[logical_schema_index] {+ if accumulators.min_accs[logical_schema_index].is_some() {accumulators.is_min_value_exact[logical_schema_index] = summarize_bound(- min_acc,+ &mut accumulators.min_accs[logical_schema_index],&stats_converter.row_group_mins(row_groups_metadata)?,parquet_index,row_groups_metadata,@@ -785,13 +785,41 @@ fn summarize_column_statistics(/// parquet statistics. `row_group_exactness` rebuilds the exactness as a Boolean/// array and is only called for the rare case where row groups disagree.fn summarize_bound<A: Accumulator>(- acc: &mut A,+ acc: &mut Option<A>,values: &ArrayRef,parquet_index: Option<usize>,row_groups_metadata: &[RowGroupMetaData],is_exact: impl Fn(&ParquetStatistics) -> bool,row_group_exactness: impl FnOnce() -> Result<BooleanArray>,) -> Result<Option<bool>> {+ // A NULL converted bound can mean missing statistics, not just all-NULL+ // data. Ignoring it in MIN/MAX would let another row group's exact endpoint+ // incorrectly establish an exact bound for the whole file. Drop this bound+ // unless the row group is empty or is proven to contain only NULLs.+ if values.null_count() > 0+ && parquet_index.is_some_and(|column_index| {+ row_groups_metadata+ .iter()+ .enumerate()+ .any(|(index, group)| {+ if values.is_valid(index) || group.num_rows() == 0 {+ return false;+ }+ let column = group.column(column_index);+ let all_null = column+ .statistics()+ .and_then(|stats| stats.null_count_opt())+ .is_some_and(|nulls| nulls == column.num_values() as u64);+ !all_null+ })+ })+ {+ *acc = None;+ return Ok(None);+ }+ let Some(acc) = acc.as_mut() else {+ return Ok(None);+ };acc.update_batch(&[Arc::clone(values)])?;Ok(@@ -1281,7 +1309,7 @@ mod tests {);}- mod ndv_tests {+ mod statistics_tests {use super::*;use arrow::datatypes::Field;use parquet::basic::Type as PhysicalType;@@ -1678,6 +1706,108 @@ mod tests {);}+ #[test]+ fn test_min_max_require_complete_row_group_bounds() {+ let known =+ || ParquetStatistics::int32(Some(1), Some(3), None, Some(0), false);+ let exact = |v| Precision::Exact(ScalarValue::Int32(Some(v)));+ let cases = [+ (None, 3, Precision::Absent, Precision::Absent),+ (+ Some(ParquetStatistics::int32(+ None,+ Some(6),+ None,+ Some(0),+ false,+ )),+ 3,+ Precision::Absent,+ exact(6),+ ),+ (+ Some(ParquetStatistics::int32(+ Some(4),+ None,+ None,+ Some(0),+ false,+ )),+ 3,+ exact(1),+ Precision::Absent,+ ),+ (+ Some(ParquetStatistics::int32(None, None, None, None, false)),+ 3,+ Precision::Absent,+ Precision::Absent,+ ),+ (+ Some(ParquetStatistics::int32(None, None, None, Some(2), false)),+ 3,+ Precision::Absent,+ Precision::Absent,+ ),+ // Empty and proven all-NULL row groups do not contribute extrema.+ (None, 0, exact(1), exact(3)),+ (+ Some(ParquetStatistics::int32(None, None, None, Some(3), false)),+ 3,+ exact(1),+ exact(3),+ ),+ (+ Some(ParquetStatistics::int32(+ Some(4),+ Some(6),+ None,+ Some(0),+ false,+ )),+ 3,+ exact(1),+ exact(6),+ ),+ ];+ for (other, rows, expected_min, expected_max) in cases {+ for reverse in [false, true] {+ let schema_descr = create_schema_descr(1);+ let arrow_schema = create_arrow_schema(1);+ let mut groups = vec![+ create_row_group_with_stats(+ &schema_descr,+ vec![Some(known())],+ 3,+ ),+ create_row_group_with_stats(+ &schema_descr,+ vec![other.clone()],+ rows,+ ),+ ];+ if reverse {+ groups.reverse();+ }+ let metadata = create_parquet_metadata(schema_descr, groups);+ let stats = DFParquetMetadata::statistics_from_parquet_metadata(+ &metadata,+ &arrow_schema,+ )+ .unwrap();+ assert_eq!(+ stats.column_statistics[0].min_value, expected_min,+ "other={other:?}, rows={rows}, reverse={reverse}"+ );+ assert_eq!(+ stats.column_statistics[0].max_value, expected_max,+ "other={other:?}, rows={rows}, reverse={reverse}"+ );+ assert_eq!(stats.num_rows, Precision::Exact((3 + rows) as usize));+ }+ }+ }+#[test]fn test_distinct_count_no_statistics_at_all() {// No statistics in row group should return Absent for all statsdiff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 307a8876ab..87d39e34dc 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -148,6 +148,9 @@ impl TestContext {let file_name = relative_path.file_name().unwrap().to_str().unwrap();match file_name {+ "parquet_missing_bounds.slt" => {+ register_parquet_missing_bounds(&mut test_ctx).await;+ }"cte.slt" => {info!("Registering strict schema provider for CTE tests");register_strict_schema_provider(test_ctx.session_ctx());@@ -415,6 +418,77 @@ pub async fn register_partition_table(test_ctx: &mut TestContext) {.unwrap();}+/// Write row groups with different statistics settings using the public writer API.+async fn register_parquet_missing_bounds(test_ctx: &mut TestContext) {+ use datafusion::parquet::column::writer::ColumnWriterImpl;+ use datafusion::parquet::data_type::{ByteArray, ByteArrayType};+ use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties};+ use datafusion::parquet::file::writer::{+ SerializedFileWriter, SerializedPageWriter, TrackedWrite,+ };+ use datafusion::parquet::schema::parser::parse_message_type;++ test_ctx.enable_testdir();+ let path = test_ctx.testdir_path().join("missing_bounds.parquet");+ let column_path = test_ctx.testdir_path().join("column.pages");+ let schema = Arc::new(+ parse_message_type("message schema { REQUIRED BINARY a (UTF8); }").unwrap(),+ );+ let mut writer = SerializedFileWriter::new(+ File::create(&path).unwrap(),+ schema,+ Arc::new(WriterProperties::default()),+ )+ .unwrap();+ let long_value = "z".repeat(8192);+ for (values, statistics) in [+ (["a", "b"], EnabledStatistics::Chunk),+ (+ [long_value.as_str(), long_value.as_str()],+ EnabledStatistics::None,+ ),+ ] {+ // parquet-rs truncates long extrema rather than omitting them. Disable+ // statistics for the second chunk to exercise the missing-bound case+ // produced naturally by writers such as PyArrow, without editing metadata.+ let properties = Arc::new(+ WriterProperties::builder()+ .set_statistics_enabled(statistics)+ .build(),+ );+ let mut buffer = TrackedWrite::new(File::create(&column_path).unwrap());+ let mut column = ColumnWriterImpl::<ByteArrayType>::new(+ writer.schema_descr().column(0),+ properties,+ Box::new(SerializedPageWriter::new(&mut buffer)),+ );+ let values = values.map(ByteArray::from);+ column.write_batch(&values, None, None).unwrap();+ let result = column.close().unwrap();+ assert_eq!(+ result.metadata.statistics().is_some(),+ statistics == EnabledStatistics::Chunk+ );+ buffer.into_inner().unwrap();+ let mut group = writer.next_row_group().unwrap();+ group+ .append_column(&File::open(&column_path).unwrap(), result)+ .unwrap();+ group.close().unwrap();+ }+ writer.close().unwrap();+ std::fs::remove_file(&column_path).unwrap();+ test_ctx+ .ctx+ .register_parquet(+ "missing_bounds",+ path.to_str().unwrap(),+ ParquetReadOptions::default(),+ )+ .await+ .unwrap();+}+// registers a LOCAL TEMPORARY table.pub fn register_temp_table(ctx: &SessionContext) {#[derive(Debug)]diff --git a/datafusion/sqllogictest/test_files/parquet_missing_bounds.slt b/datafusion/sqllogictest/test_files/parquet_missing_bounds.sltnew file mode 100644index 0000000000..8b303b4b43--- /dev/null+++ b/datafusion/sqllogictest/test_files/parquet_missing_bounds.slt@@ -0,0 +1,38 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# The test context writes two row groups using the Rust writer API:+# ["a", "b"] with statistics, and two 8192-character strings without statistics.+# This models the missing bounds produced by writers that omit oversized extrema.++# Verify that the long values are present in the actual data.+query I+SELECT LENGTH(a) FROM missing_bounds ORDER BY a;+----+1+1+8192+8192++# The first group's maximum must not be treated as the entire file's maximum.+query II+SELECT LENGTH(MIN(a)), LENGTH(MAX(a)) FROM missing_bounds;+----+1 8192++statement ok+DROP TABLE missing_bounds;