Skip to content

Apache DataFusion SQL Query Engine

fix: preserve missing Parquet null counts (#25242)

Peter Leec14976481ea5
diff --git a/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs b/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs
index 251b7cddd5..4706728b06 100644
--- a/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs
@@ -30,16 +30,190 @@
//! because batch-arrival timing affects how soon the TopK heap fills,
//! and we don't want this test to become flaky.
+use std::io::Write;
use std::sync::Arc;
use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray};
use arrow_schema::{DataType, Field, Schema};
-use datafusion::prelude::SessionConfig;
+use datafusion::physical_plan::collect;
+use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
+use datafusion_common::ScalarValue;
+use parquet::arrow::ArrowWriter;
+use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataWriter};
+use parquet::file::properties::{EnabledStatistics, WriterProperties};
+use parquet::file::statistics::Statistics;
+use parquet::file::writer::TrackedWrite;
+use tempfile::NamedTempFile;
use crate::parquet::Unit::RowGroup;
+use crate::parquet::utils::MetricsFinder;
use crate::parquet::{ContextWithParquet, Scenario};
+/// Keep min/max but omit the middle row group's null count. Negate the
+/// values for DESC so the first group always establishes the winning bound.
+fn file_with_missing_null_count(descending: bool, has_null: bool) -> NamedTempFile {
+ let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, true)]));
+ let props = WriterProperties::builder()
+ .set_statistics_enabled(EnabledStatistics::Chunk)
+ .build();
+ let mut bytes = Vec::new();
+ let mut writer =
+ ArrowWriter::try_new(&mut bytes, schema.clone(), Some(props)).unwrap();
+ for values in [
+ vec![Some(0), Some(1), Some(2)],
+ vec![
+ if has_null { None } else { Some(100) },
+ Some(100),
+ Some(101),
+ ],
+ vec![Some(200), Some(201), Some(202)],
+ ] {
+ let values: Int64Array = values
+ .into_iter()
+ .map(|v| v.map(|v| if descending { -v } else { v }))
+ .collect();
+ let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values)]).unwrap();
+ writer.write(&batch).unwrap();
+ writer.flush().unwrap();
+ }
+ let metadata = writer.close().unwrap();
+ let mut groups = metadata.row_groups().to_vec();
+ if has_null {
+ let column = groups[1].column(0).clone();
+ let Statistics::Int64(stats) = column.statistics().unwrap() else {
+ panic!("expected Int64 statistics");
+ };
+ let stats = Statistics::int64(
+ stats.min_opt().copied(),
+ stats.max_opt().copied(),
+ None,
+ None,
+ false,
+ );
+ let column = column.into_builder().set_statistics(stats).build().unwrap();
+ groups[1] = groups[1]
+ .clone()
+ .into_builder()
+ .set_column_metadata(vec![column])
+ .build()
+ .unwrap();
+ }
+ let metadata = ParquetMetaData::new(metadata.file_metadata().clone(), groups);
+
+ // Replace the footer, leaving the encoded rows and their offsets intact.
+ let footer_len =
+ u32::from_le_bytes(bytes[bytes.len() - 8..bytes.len() - 4].try_into().unwrap());
+ bytes.truncate(bytes.len() - 8 - footer_len as usize);
+ let mut file = tempfile::Builder::new()
+ .suffix(".parquet")
+ .tempfile()
+ .unwrap();
+ let mut output = TrackedWrite::new(file.as_file_mut());
+ output.write_all(&bytes).unwrap();
+ ParquetMetaDataWriter::new_with_tracked(output, &metadata)
+ .finish()
+ .unwrap();
+ file
+}
+
+#[tokio::test]
+async fn missing_null_count_preserves_null_filter_and_count() {
+ let file = file_with_missing_null_count(false, true);
+ let ctx = SessionContext::new();
+ ctx.register_parquet(
+ "t",
+ file.path().to_str().unwrap(),
+ ParquetReadOptions::default(),
+ )
+ .await
+ .unwrap();
+ for (sql, expected) in [
+ ("SELECT COUNT(v) FROM t", 8),
+ ("SELECT COUNT(*) FROM t WHERE v IS NULL", 1),
+ ] {
+ let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap();
+ assert_eq!(
+ ScalarValue::try_from_array(batches[0].column(0), 0).unwrap(),
+ ScalarValue::Int64(Some(expected)),
+ "{sql}",
+ );
+ }
+}
+
+#[tokio::test]
+async fn dynamic_rg_pruning_preserves_missing_null_count() {
+ for descending in [false, true] {
+ for has_null in [false, true] {
+ let file = file_with_missing_null_count(descending, has_null);
+ for nulls_first in [false, true] {
+ for pushdown in [false, true] {
+ let mut config = SessionConfig::new()
+ .with_target_partitions(1)
+ .with_batch_size(1)
+ .with_parquet_page_index_pruning(false);
+ // Keep the live filter on the row-group path: file-level
+ // statistics can otherwise prune the entire remaining file.
+ config.options_mut().execution.collect_statistics = false;
+ config.options_mut().optimizer.enable_sort_pushdown = false;
+ config
+ .options_mut()
+ .optimizer
+ .enable_topk_dynamic_filter_pushdown = pushdown;
+ config.options_mut().execution.parquet.pushdown_filters = false;
+ let ctx = SessionContext::new_with_config(config);
+ ctx.register_parquet(
+ "t",
+ file.path().to_str().unwrap(),
+ ParquetReadOptions::default(),
+ )
+ .await
+ .unwrap();
+ let order = if descending { "DESC" } else { "ASC" };
+ let null_order = if nulls_first { "FIRST" } else { "LAST" };
+ let sql = format!(
+ "SELECT v FROM t ORDER BY v {order} NULLS {null_order} LIMIT 1"
+ );
+ let plan = ctx
+ .sql(&sql)
+ .await
+ .unwrap()
+ .create_physical_plan()
+ .await
+ .unwrap();
+ let batches = collect(plan.clone(), ctx.task_ctx()).await.unwrap();
+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
+ let expected = if has_null && nulls_first {
+ None
+ } else {
+ Some(0)
+ };
+ assert_eq!(
+ ScalarValue::try_from_array(batches[0].column(0), 0).unwrap(),
+ ScalarValue::Int64(expected),
+ "{sql}, has_null={has_null}, pushdown={pushdown}",
+ );
+ let metrics = MetricsFinder::find_metrics(plan.as_ref()).unwrap();
+ let pruned = metrics
+ .sum_by_name("row_groups_pruned_dynamic_filter")
+ .unwrap()
+ .as_usize();
+ if pushdown {
+ assert!(
+ pruned > 0,
+ "runtime pruning must run: {sql}, has_null={has_null}\n{metrics}\n{}",
+ datafusion::physical_plan::displayable(plan.as_ref())
+ .indent(true),
+ );
+ } else {
+ assert_eq!(pruned, 0);
+ }
+ }
+ }
+ }
+ }
+}
+
/// Build five `RecordBatch`es whose `v` column ranges are disjoint:
/// batch `i` carries `v` values `[i*100, (i+1)*100)`. When written with
/// `max_row_group_row_count = 100` each batch lands in its own row group.
diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rs
index c62b0de9cd..4781a309f5 100644
--- a/datafusion/datasource-parquet/src/metadata.rs
+++ b/datafusion/datasource-parquet/src/metadata.rs
@@ -551,6 +551,10 @@ impl<'a> DFParquetMetadata<'a> {
file_metadata.schema_descr(),
) {
Ok(stats_converter) => {
+ // An omitted count must not become an exact zero in
+ // file statistics used for pruning and aggregates.
+ let stats_converter =
+ stats_converter.with_missing_null_counts_as_zero(false);
let parquet_index = stats_converter.parquet_column_index();
if parquet_index.is_some_and(|index| {
has_untrusted_min_max_order(
@@ -1394,6 +1398,43 @@ mod tests {
ParquetMetaData::new(file_meta, row_groups)
}
+ #[test]
+ fn test_statistics_preserve_missing_null_counts() {
+ let schema_descr = create_schema_descr(1);
+ let arrow_schema = create_arrow_schema(1);
+ for (null_counts, expected) in [
+ (vec![None], Precision::Absent),
+ (vec![Some(0), None], Precision::Inexact(0)),
+ (vec![Some(2), None], Precision::Inexact(2)),
+ (vec![Some(0), Some(0)], Precision::Exact(0)),
+ ] {
+ let row_groups = null_counts
+ .into_iter()
+ .map(|null_count| {
+ create_row_group_with_stats(
+ &schema_descr,
+ vec![Some(ParquetStatistics::int32(
+ Some(1),
+ Some(10),
+ None,
+ null_count,
+ false,
+ ))],
+ 10,
+ )
+ })
+ .collect();
+ let metadata =
+ create_parquet_metadata(Arc::clone(&schema_descr), row_groups);
+ let statistics = DFParquetMetadata::statistics_from_parquet_metadata(
+ &metadata,
+ &arrow_schema,
+ )
+ .unwrap();
+ assert_eq!(statistics.column_statistics[0].null_count, expected);
+ }
+ }
+
#[test]
fn test_summarize_null_counts() {
let schema_descr = create_schema_descr(1);
diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs
index 94a0e5049f..77947bd8af 100644
--- a/datafusion/datasource-parquet/src/push_decoder.rs
+++ b/datafusion/datasource-parquet/src/push_decoder.rs
@@ -253,11 +253,6 @@ impl RowGroupPruner {
.map(Vec::as_slice),
row_group_metadatas,
arrow_schema: self.arrow_schema.as_ref(),
- // Match the existing static row-group pruning behavior: when a
- // statistic's null count is missing, treat it as zero. This is
- // sound for runtime pruning because the predicate only needs to
- // prove a row group *cannot* contain matching rows.
- missing_null_counts_as_zero: true,
};
match pp.prune(&stats) {
diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs
index 7f733058e6..ab690e345d 100644
--- a/datafusion/datasource-parquet/src/row_group_filter.rs
+++ b/datafusion/datasource-parquet/src/row_group_filter.rs
@@ -335,11 +335,6 @@ impl RowGroupAccessPlanFilter {
column_orders,
row_group_metadatas,
arrow_schema,
- // Preserve the existing row-group pruning behavior. This path only
- // proves whether matching rows may exist, so it uses the
- // StatisticsConverter default for older parquet-rs files where a
- // missing null count can mean there are zero nulls.
- missing_null_counts_as_zero: true,
};
// try to prune the row groups in a single call
@@ -407,11 +402,6 @@ impl RowGroupAccessPlanFilter {
.map(|&i| &groups[i])
.collect::<Vec<_>>(),
arrow_schema,
- // Fully matched row groups require a stronger proof: every row
- // must pass the predicate. Missing null counts are unknown here;
- // treating them as zero can incorrectly mark nullable row groups as
- // fully matched and make limit pruning unsound.
- missing_null_counts_as_zero: false,
};
let Ok(inverted_values) = inverted_predicate.prune(&inverted_pruning_stats)
@@ -495,7 +485,6 @@ pub(crate) struct RowGroupPruningStatistics<'a> {
pub(crate) column_orders: Option<&'a [ColumnOrder]>,
pub(crate) row_group_metadatas: Vec<&'a RowGroupMetaData>,
pub(crate) arrow_schema: &'a Schema,
- pub(crate) missing_null_counts_as_zero: bool,
}
impl<'a> RowGroupPruningStatistics<'a> {
@@ -510,7 +499,9 @@ impl<'a> RowGroupPruningStatistics<'a> {
self.arrow_schema,
self.parquet_schema,
)?
- .with_missing_null_counts_as_zero(self.missing_null_counts_as_zero))
+ // Missing counts cannot rule out nulls, either when pruning groups or
+ // when proving that every row matches the predicate.
+ .with_missing_null_counts_as_zero(false))
}
fn min_max_statistics_converter(
@@ -783,6 +774,48 @@ mod tests {
assert_eq!(row_groups.is_fully_matched(), &vec![false, true, false]);
}
+ #[test]
+ fn row_group_pruning_predicate_missing_null_count() {
+ let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, true)]));
+ let schema_descr = get_test_schema_descr(vec![PrimitiveTypeField::new(
+ "c1",
+ PhysicalType::INT32,
+ )]);
+ let groups = [None, Some(0), Some(1)].map(|null_count| {
+ get_row_group_meta_data(
+ &schema_descr,
+ vec![ParquetStatistics::int32(
+ Some(100),
+ Some(101),
+ None,
+ null_count,
+ false,
+ )],
+ )
+ });
+
+ for (expr, expected) in [
+ (col("c1").is_null(), vec![0, 2]),
+ (col("c1").is_null().or(col("c1").lt(lit(0))), vec![0, 2]),
+ (col("c1").lt(lit(0)), vec![]),
+ ] {
+ let predicate = build_test_pruning_predicate(
+ logical2physical(&expr, &schema),
+ Arc::clone(&schema),
+ );
+ let mut filter =
+ RowGroupAccessPlanFilter::new(ParquetAccessPlan::new_all(groups.len()));
+ filter.prune_by_statistics(
+ &schema,
+ &schema_descr,
+ &groups,
+ &predicate,
+ &parquet_file_metrics(),
+ );
+ assert_eq!(filter.build().row_group_indexes(), expected, "{expr}");
+ }
+ }
+
#[test]
fn prune_by_limit_preserves_row_selection() {
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32);
diff --git a/datafusion/datasource-parquet/src/statistics_order_tests.rs b/datafusion/datasource-parquet/src/statistics_order_tests.rs
index e25c2c0da5..2156a92dcd 100644
--- a/datafusion/datasource-parquet/src/statistics_order_tests.rs
+++ b/datafusion/datasource-parquet/src/statistics_order_tests.rs
@@ -520,7 +520,6 @@ fn byte_array_order_guard_follows_parquet_type_not_arrow_representation() {
column_orders: metadata.column_orders().map(Vec::as_slice),
row_group_metadatas: file.metadata.row_groups().iter().collect(),
arrow_schema: &schema,
- missing_null_counts_as_zero: true,
};
for values in [stats.min_values(&column), stats.max_values(&column)] {
let values = values.unwrap();
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 8de2abc5a0..7658ce60ed 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -25,6 +25,43 @@
in this section pertains to features and changes that have already been merged
to the main branch and are awaiting release in this version.
+### Missing Parquet null counts are treated as unknown
+
+DataFusion now preserves an omitted Parquet `null_count` statistic as unknown.
+Previously, it assumed zero nulls, which could discard matching NULL rows in
+`IS NULL` filters and `ORDER BY ... NULLS FIRST LIMIT` queries, or produce an
+incorrectly high `COUNT(column)` from metadata.
+
+Queries over affected files may read or sort more data because these optimizations
+require a known null count:
+
+- Pruning row groups for `IS NULL` filters.
+- Computing `COUNT(column)` using only file metadata.
+- Pruning row groups with a dynamic `NULLS FIRST` TopK filter.
+- Eliminating a sort over nullable columns in sorted, non-overlapping files.
+ Such queries may now retain a full `SortExec`, even when the data has no NULLs.
+
+parquet-rs versions before 53.1.0 omitted zero null counts. This includes files
+produced with the older parquet-rs dependency used by DataFusion releases before
+42.1.0. [Arrow #6490](https://github.com/apache/arrow-rs/pull/6490) changed the
+writer to record known zero counts. Files with explicit counts retain their
+existing behavior.
+
+To identify column chunks with bounds but no null count, run this query in the
+DataFusion CLI:
+
+```sql
+SELECT row_group_id, path_in_schema
+FROM parquet_metadata('data.parquet')
+WHERE (stats_min IS NOT NULL OR stats_max IS NOT NULL)
+ AND stats_null_count IS NULL;
+```
+
+Rewriting affected files with a current DataFusion version, for example using
+`COPY ... TO` with Parquet statistics enabled, records the missing counts and
+restores optimizations that depend on them. Preserve the required data ordering
+and ordering metadata when rewriting sorted files.
+
### `ForeignSession::create_physical_plan` is unsupported
`ForeignSession::create_physical_plan` no longer forwards to the library that