Skip to content

Apache DataFusion SQL Query Engine

fix: preserve uncertainty in singleton filter statistics (#25863)

Kumar Ujjawalb9298572aeee
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index f71d67d04e..dcaae294b6 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -1254,9 +1254,9 @@ fn scale_byte_size_at_rows(
/// Returns the NDV for a column constrained to one non-null value (e.g.
/// `column = literal` or a singleton interval), derived from the filtered row
-/// estimate: zero rows means zero distinct values, a known positive row count
-/// means exactly one, and an unknown row count means an inexact one (the column
-/// could still be empty).
+/// estimate: zero rows means zero distinct values, an exact positive row count
+/// means exactly one, and an estimated or unknown row count means an inexact
+/// one (the column could still be empty).
///
/// The caller is responsible for proving the singleton domain.
fn distinct_count_for_singleton_domain(
@@ -1264,10 +1264,10 @@ fn distinct_count_for_singleton_domain(
) -> Precision<usize> {
match filtered_num_rows {
Precision::Exact(0) | Precision::Inexact(0) => filtered_num_rows,
- // The row count is unknown, so the column could still be empty (zero
- // distinct values); report an inexact one rather than overstating it.
- Precision::Absent => Precision::Inexact(1),
- _ => Precision::Exact(1),
+ Precision::Exact(_) => Precision::Exact(1),
+ // The row count is not known exactly, so the column could still be
+ // empty (zero distinct values); report an inexact one.
+ Precision::Inexact(_) | Precision::Absent => Precision::Inexact(1),
}
}
@@ -2990,7 +2990,7 @@ mod tests {
Operator::Eq,
Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"utf8view equality",
@@ -3006,7 +3006,7 @@ mod tests {
"hello".to_string(),
)))),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"largeutf8 equality",
@@ -3022,7 +3022,7 @@ mod tests {
"hello".to_string(),
)))),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"utf8 reversed (literal = column)",
@@ -3036,7 +3036,7 @@ mod tests {
Operator::Eq,
Arc::new(Column::new("name", 0)),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"OR is not collapsed to NDV=1, but NDV is capped at filtered rows",
@@ -3093,7 +3093,7 @@ mod tests {
Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),
)),
)),
- vec![Precision::Exact(1), Precision::Exact(1)],
+ vec![Precision::Inexact(1), Precision::Inexact(1)],
),
(
"numeric equality with min/max bounds (interval analysis path)",
@@ -3109,7 +3109,7 @@ mod tests {
Operator::Eq,
Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"timestamp equality",
@@ -3130,7 +3130,7 @@ mod tests {
None,
))),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"contradictory numeric equality (infeasible)",
@@ -3166,7 +3166,7 @@ mod tests {
Operator::Eq,
Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))),
)),
- vec![Precision::Exact(1)],
+ vec![Precision::Inexact(1)],
),
(
"contradictory utf8 equality (infeasible)",
@@ -3231,7 +3231,7 @@ mod tests {
Arc::new(Literal::new(ScalarValue::Int32(Some(2)))),
)),
)),
- vec![Precision::Exact(1), Precision::Exact(1)],
+ vec![Precision::Inexact(1), Precision::Inexact(1)],
),
];
@@ -3529,7 +3529,7 @@ mod tests {
// Equality predicates collapse NDV and reject nulls for their columns.
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
assert_eq!(
statistics.column_statistics[0].null_count,
@@ -3544,7 +3544,7 @@ mod tests {
);
assert_eq!(
statistics.column_statistics[2].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
assert_eq!(
statistics.column_statistics[2].null_count,
@@ -3582,7 +3582,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
Ok(())
}
@@ -3616,7 +3616,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
Ok(())
}
@@ -3650,7 +3650,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
Ok(())
}
@@ -3684,7 +3684,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
Ok(())
}
@@ -3719,7 +3719,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
Ok(())
}
@@ -3766,7 +3766,7 @@ mod tests {
StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
- Precision::Exact(1)
+ Precision::Inexact(1)
);
Ok(())
}
@@ -4174,6 +4174,39 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn test_filter_statistics_singleton_precision() -> Result<()> {
+ use Precision::{Exact, Inexact};
+
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+ // The bounds include 5, but no row contains it.
+ let values = Int32Array::from_iter_values(
+ (0..1000).map(|i| if i % 2 == 0 { 1 } else { 100 }),
+ );
+ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(values)])?;
+ let input = test::TestMemoryExec::try_new_exec(
+ &[vec![batch]],
+ Arc::clone(&schema),
+ None,
+ )?;
+ let mut input_stats = StatisticsContext::new()
+ .compute(input.as_ref(), &StatisticsArgs::new())?
+ .as_ref()
+ .clone();
+ // Supply the bounds a file scan could report for these rows.
+ input_stats.column_statistics[0].min_value = Exact(ScalarValue::Int32(Some(1)));
+ input_stats.column_statistics[0].max_value = Exact(ScalarValue::Int32(Some(100)));
+ let predicate = binary(col("a", &schema)?, Operator::Eq, lit(5i32), &schema)?;
+ let filter = FilterExecBuilder::new(predicate, input).build()?;
+ let stats = filter
+ .statistics_from_inputs(&[Arc::new(input_stats)], &StatisticsArgs::new())?;
+ let batches =
+ collect(filter.execute(0, Arc::new(TaskContext::default()))?).await?;
+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 0);
+ assert_eq!(stats.column_statistics[0].distinct_count, Inexact(1));
+ Ok(())
+ }
+
#[tokio::test]
async fn test_filter_statistics_is_not_null_rejects_nulls() -> Result<()> {
let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt
index 71de383370..2a8d174751 100644
--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt
@@ -59,7 +59,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
@@ -84,7 +84,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
@@ -152,7 +152,7 @@ query TT
EXPLAIN SELECT i8 FROM typed_table WHERE i8 = 2;
----
physical_plan
-01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
+01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i8], file_type=parquet, predicate=i8@0 = 2, pruning_predicate=i8_null_count@2 != row_count@3 AND i8_min@0 <= 2 AND 2 <= i8_max@1, required_guarantees=[i8 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]
@@ -161,7 +161,7 @@ query TT
EXPLAIN SELECT i64 FROM typed_table WHERE i64 = 2;
----
physical_plan
-01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(8))]]
+01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i64], file_type=parquet, predicate=i64@1 = 2, pruning_predicate=i64_null_count@2 != row_count@3 AND i64_min@0 <= 2 AND 2 <= i64_max@1, required_guarantees=[i64 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
@@ -170,7 +170,7 @@ query TT
EXPLAIN SELECT f32 FROM typed_table WHERE f32 = 2.5;
----
physical_plan
-01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
+01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f32], file_type=parquet, predicate=CAST(f32@2 AS Float64) = 2.5, pruning_predicate=f32_null_count@2 != row_count@3 AND CAST(f32_min@0 AS Float64) <= 2.5 AND 2.5 <= CAST(f32_max@1 AS Float64), required_guarantees=[], statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]
@@ -179,7 +179,7 @@ query TT
EXPLAIN SELECT f64 FROM typed_table WHERE 2.5 = f64;
----
physical_plan
-01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
+01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f64], file_type=parquet, predicate=f64@3 = 2.5, pruning_predicate=f64_null_count@2 != row_count@3 AND f64_min@0 <= 2.5 AND 2.5 <= f64_max@1, required_guarantees=[f64 in (2.5)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]