Apache DataFusion SQL Query Engine
Use shared statistics merge for union stats (#21430)
Kumar Ujjawald09a9195e711
diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rsindex 254e437b69..29b9c36c0a 100644--- a/datafusion/common/src/stats.rs+++ b/datafusion/common/src/stats.rs@@ -372,6 +372,27 @@ pub struct Statistics {pub column_statistics: Vec<ColumnStatistics>,}+/// Fallback to use when NDV overlap can not be estimated from column bounds.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]+pub enum NdvFallback {+ /// Use the larger input NDV. This is the conservative default for+ /// related fragments such as files from the same table.+ #[default]+ Max,+ /// Sum the input NDVs. This is a conservative upper bound for+ /// independent inputs such as `UNION ALL`.+ Sum,+}++impl NdvFallback {+ fn merge(self, left: usize, right: usize) -> usize {+ match self {+ Self::Max => usize::max(left, right),+ Self::Sum => left.saturating_add(right),+ }+ }+}+impl Default for Statistics {/// Returns a new [`Statistics`] instance with all fields set to unknown/// and no columns.@@ -630,6 +651,9 @@ impl Statistics {/// The method assumes that all statistics are for the same schema./// If not, maybe you can call `SchemaMapper::map_column_statistics` to make them consistent.///+ /// This method uses [`NdvFallback::Max`] when `distinct_count` overlap+ /// can not be estimated from column bounds.+ ////// Returns an error if the statistics do not match the specified schemas.////// # Example@@ -670,6 +694,19 @@ impl Statistics {/// Precision::Exact(ScalarValue::Int64(Some(1500))));/// ```pub fn try_merge_iter<'a, I>(items: I, schema: &Schema) -> Result<Statistics>+ where+ I: IntoIterator<Item = &'a Statistics>,+ {+ Self::try_merge_iter_with_ndv_fallback(items, schema, NdvFallback::Max)+ }++ /// Same as [`Statistics::try_merge_iter`], but lets callers choose the+ /// fallback used when `distinct_count` overlap can not be estimated.+ pub fn try_merge_iter_with_ndv_fallback<'a, I>(+ items: I,+ schema: &Schema,+ ndv_fallback: NdvFallback,+ ) -> Result<Statistics>whereI: IntoIterator<Item = &'a Statistics>,{@@ -717,7 +754,7 @@ impl Statistics {) {(Some(&l), Some(&r)) => Precision::Inexact(estimate_ndv_with_overlap(col_stats, item_cs, l, r)- .unwrap_or_else(|| usize::max(l, r)),+ .unwrap_or_else(|| ndv_fallback.merge(l, r)),),_ => Precision::Absent,};@@ -1465,6 +1502,44 @@ mod tests {}}+ fn make_single_i64_ndv_stats(+ distinct_count: Precision<usize>,+ min_value: Option<i64>,+ max_value: Option<i64>,+ ) -> Statistics {+ let to_precision = |value| Precision::Exact(ScalarValue::Int64(Some(value)));++ Statistics::default()+ .with_num_rows(Precision::Exact(10))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(distinct_count)+ .with_min_value(+ min_value.map(to_precision).unwrap_or(Precision::Absent),+ )+ .with_max_value(+ max_value.map(to_precision).unwrap_or(Precision::Absent),+ ),+ )+ }++ fn merge_single_i64_ndv_distinct_count(+ left: Statistics,+ right: Statistics,+ ndv_fallback: NdvFallback,+ ) -> Precision<usize> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int64, true)]);++ Statistics::try_merge_iter_with_ndv_fallback(+ [&left, &right],+ &schema,+ ndv_fallback,+ )+ .unwrap()+ .column_statistics[0]+ .distinct_count+ }+#[test]fn test_try_merge() {// Create a schema with two columns@@ -1815,7 +1890,7 @@ mod tests {let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);let merged = Statistics::try_merge_iter([&stats1, &stats2], &schema).unwrap();- // No min/max -> fallback to max(5, 8)+ // No min/max -> default fallback is maxassert_eq!(merged.column_statistics[0].distinct_count,Precision::Inexact(8)@@ -1851,13 +1926,55 @@ mod tests {let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);let merged = Statistics::try_merge_iter([&stats1, &stats2], &schema).unwrap();- // distance() unsupported for strings -> fallback to max+ // distance() unsupported for strings -> default fallback is maxassert_eq!(merged.column_statistics[0].distinct_count,Precision::Inexact(8));}+ #[test]+ fn test_try_merge_ndv_non_numeric_types_sum_fallback() {+ let stats1 = Statistics::default()+ .with_num_rows(Precision::Exact(10))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_min_value(Precision::Exact(ScalarValue::Utf8(Some(+ "aaa".to_string(),+ ))))+ .with_max_value(Precision::Exact(ScalarValue::Utf8(Some(+ "zzz".to_string(),+ ))))+ .with_distinct_count(Precision::Exact(5)),+ );+ let stats2 = Statistics::default()+ .with_num_rows(Precision::Exact(10))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_min_value(Precision::Exact(ScalarValue::Utf8(Some(+ "bbb".to_string(),+ ))))+ .with_max_value(Precision::Exact(ScalarValue::Utf8(Some(+ "yyy".to_string(),+ ))))+ .with_distinct_count(Precision::Exact(8)),+ );++ let schema = Schema::new(vec![Field::new("a", DataType::Utf8, true)]);+ let merged = Statistics::try_merge_iter_with_ndv_fallback(+ [&stats1, &stats2],+ &schema,+ NdvFallback::Sum,+ )+ .unwrap();++ // distance() unsupported for strings -> sum fallback is caller-selected+ assert_eq!(+ merged.column_statistics[0].distinct_count,+ Precision::Inexact(13)+ );+ }+#[test]fn test_try_merge_ndv_constant_columns() {// Same constant: [5,5]+[5,5] -> max@@ -1910,6 +2027,183 @@ mod tests {);}+ #[test]+ fn test_try_merge_ndv_original_union_edge_cases() {+ struct NdvTestCase {+ name: &'static str,+ left_ndv: Precision<usize>,+ left_min: Option<i64>,+ left_max: Option<i64>,+ right_ndv: Precision<usize>,+ right_min: Option<i64>,+ right_max: Option<i64>,+ expected: Precision<usize>,+ }++ let cases = vec![+ NdvTestCase {+ name: "disjoint ranges",+ left_ndv: Precision::Exact(5),+ left_min: Some(0),+ left_max: Some(10),+ right_ndv: Precision::Exact(3),+ right_min: Some(20),+ right_max: Some(30),+ expected: Precision::Inexact(8),+ },+ NdvTestCase {+ name: "identical ranges",+ left_ndv: Precision::Exact(10),+ left_min: Some(0),+ left_max: Some(100),+ right_ndv: Precision::Exact(8),+ right_min: Some(0),+ right_max: Some(100),+ expected: Precision::Inexact(10),+ },+ NdvTestCase {+ name: "partial overlap",+ left_ndv: Precision::Exact(100),+ left_min: Some(0),+ left_max: Some(100),+ right_ndv: Precision::Exact(50),+ right_min: Some(50),+ right_max: Some(150),+ expected: Precision::Inexact(125),+ },+ NdvTestCase {+ name: "right contained in left",+ left_ndv: Precision::Exact(100),+ left_min: Some(0),+ left_max: Some(100),+ right_ndv: Precision::Exact(50),+ right_min: Some(25),+ right_max: Some(75),+ expected: Precision::Inexact(100),+ },+ NdvTestCase {+ name: "same constant value",+ left_ndv: Precision::Exact(1),+ left_min: Some(5),+ left_max: Some(5),+ right_ndv: Precision::Exact(1),+ right_min: Some(5),+ right_max: Some(5),+ expected: Precision::Inexact(1),+ },+ NdvTestCase {+ name: "different constant values",+ left_ndv: Precision::Exact(1),+ left_min: Some(5),+ left_max: Some(5),+ right_ndv: Precision::Exact(1),+ right_min: Some(10),+ right_max: Some(10),+ expected: Precision::Inexact(2),+ },+ NdvTestCase {+ name: "left constant within right range",+ left_ndv: Precision::Exact(1),+ left_min: Some(5),+ left_max: Some(5),+ right_ndv: Precision::Exact(10),+ right_min: Some(0),+ right_max: Some(10),+ expected: Precision::Inexact(10),+ },+ NdvTestCase {+ name: "left constant outside right range",+ left_ndv: Precision::Exact(1),+ left_min: Some(20),+ left_max: Some(20),+ right_ndv: Precision::Exact(10),+ right_min: Some(0),+ right_max: Some(10),+ expected: Precision::Inexact(11),+ },+ NdvTestCase {+ name: "right constant within left range",+ left_ndv: Precision::Exact(10),+ left_min: Some(0),+ left_max: Some(10),+ right_ndv: Precision::Exact(1),+ right_min: Some(5),+ right_max: Some(5),+ expected: Precision::Inexact(10),+ },+ NdvTestCase {+ name: "right constant outside left range",+ left_ndv: Precision::Exact(10),+ left_min: Some(0),+ left_max: Some(10),+ right_ndv: Precision::Exact(1),+ right_min: Some(20),+ right_max: Some(20),+ expected: Precision::Inexact(11),+ },+ NdvTestCase {+ name: "missing bounds exact plus exact",+ left_ndv: Precision::Exact(10),+ left_min: None,+ left_max: None,+ right_ndv: Precision::Exact(5),+ right_min: None,+ right_max: None,+ expected: Precision::Inexact(15),+ },+ NdvTestCase {+ name: "missing bounds exact plus inexact",+ left_ndv: Precision::Exact(10),+ left_min: None,+ left_max: None,+ right_ndv: Precision::Inexact(5),+ right_min: None,+ right_max: None,+ expected: Precision::Inexact(15),+ },+ NdvTestCase {+ name: "missing bounds inexact plus inexact",+ left_ndv: Precision::Inexact(7),+ left_min: None,+ left_max: None,+ right_ndv: Precision::Inexact(3),+ right_min: None,+ right_max: None,+ expected: Precision::Inexact(10),+ },+ NdvTestCase {+ name: "exact plus absent",+ left_ndv: Precision::Exact(10),+ left_min: None,+ left_max: None,+ right_ndv: Precision::Absent,+ right_min: None,+ right_max: None,+ expected: Precision::Absent,+ },+ NdvTestCase {+ name: "inexact plus absent",+ left_ndv: Precision::Inexact(4),+ left_min: None,+ left_max: None,+ right_ndv: Precision::Absent,+ right_min: None,+ right_max: None,+ expected: Precision::Absent,+ },+ ];++ for case in cases {+ let actual = merge_single_i64_ndv_distinct_count(+ make_single_i64_ndv_stats(case.left_ndv, case.left_min, case.left_max),+ make_single_i64_ndv_stats(case.right_ndv, case.right_min, case.right_max),+ NdvFallback::Sum,+ );++ assert_eq!(actual, case.expected, "case {} failed", case.name);+ }+ }+#[test]fn test_with_fetch_basic_preservation() {// Test that column statistics and byte size are preserved (as inexact) when applying fetchdiff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rsindex e7fef49539..032f4cec0b 100644--- a/datafusion/datasource/src/statistics.rs+++ b/datafusion/datasource/src/statistics.rs@@ -29,7 +29,7 @@ use arrow::array::RecordBatch;use arrow::compute::SortColumn;use arrow::datatypes::SchemaRef;use arrow::row::{Row, Rows};-use datafusion_common::stats::Precision;+use datafusion_common::stats::{NdvFallback, Precision};use datafusion_common::{DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,};@@ -479,7 +479,11 @@ pub fn compute_file_group_statistics(let stats = file.statistics.as_ref()?;Some(stats.as_ref())});- let statistics = Statistics::try_merge_iter(file_group_stats, &file_schema)?;+ let statistics = Statistics::try_merge_iter_with_ndv_fallback(+ file_group_stats,+ &file_schema,+ NdvFallback::Max,+ )?;Ok(file_group.with_statistics(Arc::new(statistics)))}@@ -524,8 +528,11 @@ pub fn compute_all_files_statistics(.iter().filter_map(|file_group| file_group.file_statistics(None));- let mut statistics =- Statistics::try_merge_iter(file_groups_statistics, &table_schema)?;+ let mut statistics = Statistics::try_merge_iter_with_ndv_fallback(+ file_groups_statistics,+ &table_schema,+ NdvFallback::Max,+ )?;if inexact_stats {statistics = statistics.to_inexact()@@ -546,6 +553,7 @@ pub fn add_row_stats(mod tests {use super::*;use crate::PartitionedFile;+ use crate::file_groups::FileGroup;use arrow::datatypes::{DataType, Field, Schema};use futures::stream;@@ -597,6 +605,24 @@ mod tests {}}+ fn utf8_file_stats(ndv: usize, min: &str, max: &str) -> Statistics {+ Statistics {+ num_rows: Precision::Exact(1),+ total_byte_size: Precision::Exact(16),+ column_statistics: vec![ColumnStatistics {+ null_count: Precision::Exact(0),+ max_value: Precision::Exact(ScalarValue::Utf8(Some(max.to_string()))),+ min_value: Precision::Exact(ScalarValue::Utf8(Some(min.to_string()))),+ sum_value: Precision::Absent,+ distinct_count: Precision::Exact(ndv),+ byte_size: Precision::Exact(16),+ }],+ }+ }++ fn file_with_stats(path: &str, stats: Statistics) -> PartitionedFile {+ PartitionedFile::new(path, 1).with_statistics(Arc::new(stats))+ }#[tokio::test]#[expect(deprecated)]async fn test_get_statistics_with_limit_casts_first_file_sum_to_sum_type()@@ -795,4 +821,73 @@ mod tests {Precision::Inexact(128));}++ #[test]+ fn test_compute_file_group_statistics_uses_max_ndv_fallback() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Utf8, true)]));+ let file_group = FileGroup::new(vec![+ file_with_stats("f1.parquet", utf8_file_stats(5, "a", "x")),+ file_with_stats("f2.parquet", utf8_file_stats(8, "b", "z")),+ ]);++ let file_group =+ compute_file_group_statistics(file_group, Arc::clone(&schema), true)?;+ let stats = file_group.file_statistics(None).unwrap();++ assert_eq!(+ stats.column_statistics[0].distinct_count,+ Precision::Inexact(8)+ );+ assert_eq!(+ stats.column_statistics[0].min_value,+ Precision::Exact(ScalarValue::Utf8(Some("a".to_string())))+ );+ assert_eq!(+ stats.column_statistics[0].max_value,+ Precision::Exact(ScalarValue::Utf8(Some("z".to_string())))+ );++ Ok(())+ }++ #[test]+ fn test_compute_all_files_statistics_uses_max_ndv_fallback() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Utf8, true)]));+ let file_groups = vec![+ FileGroup::new(vec![+ file_with_stats("f1.parquet", utf8_file_stats(5, "a", "x")),+ file_with_stats("f2.parquet", utf8_file_stats(8, "b", "z")),+ ]),+ FileGroup::new(vec![+ file_with_stats("f3.parquet", utf8_file_stats(3, "c", "w")),+ file_with_stats("f4.parquet", utf8_file_stats(6, "d", "y")),+ ]),+ ];++ let (file_groups, stats) =+ compute_all_files_statistics(file_groups, schema, true, false)?;++ assert_eq!(+ file_groups[0]+ .file_statistics(None)+ .unwrap()+ .column_statistics[0]+ .distinct_count,+ Precision::Inexact(8)+ );+ assert_eq!(+ file_groups[1]+ .file_statistics(None)+ .unwrap()+ .column_statistics[0]+ .distinct_count,+ Precision::Inexact(6)+ );+ assert_eq!(+ stats.column_statistics[0].distinct_count,+ Precision::Inexact(8)+ );++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 20295b7e6f..ec9ea376e0 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -27,9 +27,8 @@ use std::sync::Arc;use std::task::{Context, Poll};use super::{- ColumnStatistics, DisplayAs, DisplayFormatType, ExecutionPlan,- ExecutionPlanProperties, Partitioning, PlanProperties, RecordBatchStream,- SendableRecordBatchStream, Statistics,+ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,+ PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics,metrics::{ExecutionPlanMetricsSet, MetricsSet},};use crate::check_if_same_properties;@@ -49,7 +48,7 @@ use crate::stream::ObservedStream;use arrow::datatypes::{Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;-use datafusion_common::stats::{Precision, estimate_ndv_with_overlap};+use datafusion_common::stats::NdvFallback;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_or_internal_err, exec_err, internal_datafusion_err,@@ -342,23 +341,12 @@ impl ExecutionPlan for UnionExec {// If we get here, the partition index is out of boundsOk(Arc::new(Statistics::new_unknown(&self.schema())))} else {- // Collect statistics from all inputs- let stats = self- .inputs- .iter()- .map(|input_exec| {- input_exec- .partition_statistics(None)- .map(Arc::unwrap_or_clone)- })- .collect::<Result<Vec<_>>>()?;-- Ok(Arc::new(- stats- .into_iter()- .reduce(stats_union)- .unwrap_or_else(|| Statistics::new_unknown(&self.schema())),- ))+ let schema = self.schema();+ Ok(Arc::new(merge_input_statistics(+ &self.inputs,+ None,+ schema.as_ref(),+ )?))}}@@ -669,21 +657,12 @@ impl ExecutionPlan for InterleaveExec {}fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {- let stats = self- .inputs- .iter()- .map(|stat| {- stat.partition_statistics(partition)- .map(Arc::unwrap_or_clone)- })- .collect::<Result<Vec<_>>>()?;-- Ok(Arc::new(- stats- .into_iter()- .reduce(stats_union)- .unwrap_or_else(|| Statistics::new_unknown(&self.schema())),- ))+ let schema = self.schema();+ Ok(Arc::new(merge_input_statistics(+ &self.inputs,+ partition,+ schema.as_ref(),+ )?))}fn benefits_from_input_partitioning(&self) -> Vec<bool> {@@ -842,69 +821,35 @@ impl Stream for CombinedRecordBatchStream {}}-fn col_stats_union(- mut left: ColumnStatistics,- right: &ColumnStatistics,-) -> ColumnStatistics {- left.distinct_count = union_distinct_count(&left, right);- left.min_value = left.min_value.min(&right.min_value);- left.max_value = left.max_value.max(&right.max_value);- left.sum_value = left.sum_value.add_for_sum(&right.sum_value);- left.null_count = left.null_count.add(&right.null_count);-- left-}--fn union_distinct_count(- left: &ColumnStatistics,- right: &ColumnStatistics,-) -> Precision<usize> {- let (ndv_left, ndv_right) = match (- left.distinct_count.get_value(),- right.distinct_count.get_value(),- ) {- (Some(&l), Some(&r)) => (l, r),- _ => return Precision::Absent,- };-- // Even with exact inputs, the union NDV depends on how- // many distinct values are shared between the left and right.- // We can only estimate this via range overlap. Thus both paths- // below return `Inexact`.- if let Some(ndv) = estimate_ndv_with_overlap(left, right, ndv_left, ndv_right) {- return Precision::Inexact(ndv);- }-- Precision::Inexact(ndv_left + ndv_right)-}+fn merge_input_statistics(+ inputs: &[Arc<dyn ExecutionPlan>],+ partition: Option<usize>,+ schema: &Schema,+) -> Result<Statistics> {+ let stats = inputs+ .iter()+ .map(|input| {+ input+ .partition_statistics(partition)+ .map(Arc::unwrap_or_clone)+ })+ .collect::<Result<Vec<_>>>()?;-fn stats_union(mut left: Statistics, right: Statistics) -> Statistics {- let Statistics {- num_rows: right_num_rows,- total_byte_size: right_total_bytes,- column_statistics: right_column_statistics,- ..- } = right;- left.num_rows = left.num_rows.add(&right_num_rows);- left.total_byte_size = left.total_byte_size.add(&right_total_bytes);- left.column_statistics = left- .column_statistics- .into_iter()- .zip(right_column_statistics.iter())- .map(|(a, b)| col_stats_union(a, b))- .collect::<Vec<_>>();- left+ Statistics::try_merge_iter_with_ndv_fallback(stats.iter(), schema, NdvFallback::Sum)}#[cfg(test)]mod tests {use super::*;use crate::collect;+ use crate::repartition::RepartitionExec;+ use crate::test::exec::StatisticsExec;use crate::test::{self, TestMemoryExec};use arrow::compute::SortOptions;use arrow::datatypes::DataType;- use datafusion_common::ScalarValue;+ use datafusion_common::stats::Precision;+ use datafusion_common::{ColumnStatistics, ScalarValue};use datafusion_physical_expr::equivalence::convert_to_orderings;use datafusion_physical_expr::expressions::col;@@ -959,294 +904,204 @@ mod tests {Ok(())}- #[tokio::test]- async fn test_stats_union() {- let left = Statistics {- num_rows: Precision::Exact(5),- total_byte_size: Precision::Exact(23),- column_statistics: vec![- ColumnStatistics {- distinct_count: Precision::Exact(5),- max_value: Precision::Exact(ScalarValue::Int64(Some(21))),- min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),- sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),- null_count: Precision::Exact(0),- byte_size: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Exact(1),- max_value: Precision::Exact(ScalarValue::from("x")),- min_value: Precision::Exact(ScalarValue::from("a")),- sum_value: Precision::Absent,- null_count: Precision::Exact(3),- byte_size: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Exact(ScalarValue::Float32(Some(1.1))),- min_value: Precision::Exact(ScalarValue::Float32(Some(0.1))),- sum_value: Precision::Exact(ScalarValue::Float32(Some(42.0))),- null_count: Precision::Absent,- byte_size: Precision::Absent,- },- ],- };+ fn stats_merge_inputs() -> (SchemaRef, Statistics, Statistics, Statistics) {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, true)]));++ let left = Statistics::default()+ .with_num_rows(Precision::Exact(5))+ .with_total_byte_size(Precision::Exact(23))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Exact(5))+ .with_min_value(Precision::Exact(ScalarValue::UInt32(Some(1))))+ .with_max_value(Precision::Exact(ScalarValue::UInt32(Some(21))))+ .with_sum_value(Precision::Exact(ScalarValue::UInt32(Some(42))))+ .with_null_count(Precision::Exact(0))+ .with_byte_size(Precision::Exact(40)),+ );- let right = Statistics {- num_rows: Precision::Exact(7),- total_byte_size: Precision::Exact(29),- column_statistics: vec![- ColumnStatistics {- distinct_count: Precision::Exact(3),- max_value: Precision::Exact(ScalarValue::Int64(Some(34))),- min_value: Precision::Exact(ScalarValue::Int64(Some(1))),- sum_value: Precision::Exact(ScalarValue::Int64(Some(42))),- null_count: Precision::Exact(1),- byte_size: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Exact(ScalarValue::from("c")),- min_value: Precision::Exact(ScalarValue::from("b")),- sum_value: Precision::Absent,- null_count: Precision::Absent,- byte_size: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Absent,- min_value: Precision::Absent,- sum_value: Precision::Absent,- null_count: Precision::Absent,- byte_size: Precision::Absent,- },- ],- };+ let right = Statistics::default()+ .with_num_rows(Precision::Exact(7))+ .with_total_byte_size(Precision::Exact(29))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Exact(3))+ .with_min_value(Precision::Exact(ScalarValue::UInt32(Some(22))))+ .with_max_value(Precision::Exact(ScalarValue::UInt32(Some(34))))+ .with_sum_value(Precision::Exact(ScalarValue::UInt32(Some(8))))+ .with_null_count(Precision::Exact(1))+ .with_byte_size(Precision::Exact(60)),+ );- let result = stats_union(left, right);- let expected = Statistics {- num_rows: Precision::Exact(12),- total_byte_size: Precision::Exact(52),- column_statistics: vec![- ColumnStatistics {- distinct_count: Precision::Inexact(6),- max_value: Precision::Exact(ScalarValue::Int64(Some(34))),- min_value: Precision::Exact(ScalarValue::Int64(Some(-4))),- sum_value: Precision::Exact(ScalarValue::Int64(Some(84))),- null_count: Precision::Exact(1),- byte_size: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Exact(ScalarValue::from("x")),- min_value: Precision::Exact(ScalarValue::from("a")),- sum_value: Precision::Absent,- null_count: Precision::Absent,- byte_size: Precision::Absent,- },- ColumnStatistics {- distinct_count: Precision::Absent,- max_value: Precision::Absent,- min_value: Precision::Absent,- sum_value: Precision::Absent,- null_count: Precision::Absent,- byte_size: Precision::Absent,- },- ],- };+ let expected = Statistics::default()+ .with_num_rows(Precision::Exact(12))+ .with_total_byte_size(Precision::Exact(52))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Inexact(8))+ .with_min_value(Precision::Exact(ScalarValue::UInt32(Some(1))))+ .with_max_value(Precision::Exact(ScalarValue::UInt32(Some(34))))+ .with_sum_value(Precision::Exact(ScalarValue::UInt64(Some(50))))+ .with_null_count(Precision::Exact(1))+ .with_byte_size(Precision::Exact(100)),+ );++ (schema, left, right, expected)+ }- assert_eq!(result, expected);+ fn stats_merge_multicolumn_inputs() -> (SchemaRef, Statistics, Statistics, Statistics)+ {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, true),+ Field::new("b", DataType::Utf8, true),+ Field::new("c", DataType::Float32, true),+ ]));++ let left = Statistics::default()+ .with_num_rows(Precision::Exact(5))+ .with_total_byte_size(Precision::Exact(23))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Exact(5))+ .with_min_value(Precision::Exact(ScalarValue::Int64(Some(-4))))+ .with_max_value(Precision::Exact(ScalarValue::Int64(Some(21))))+ .with_sum_value(Precision::Exact(ScalarValue::Int64(Some(42))))+ .with_null_count(Precision::Exact(0)),+ )+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Exact(2))+ .with_min_value(Precision::Exact(ScalarValue::from("a")))+ .with_max_value(Precision::Exact(ScalarValue::from("x")))+ .with_null_count(Precision::Exact(3)),+ )+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_max_value(Precision::Exact(ScalarValue::Float32(Some(1.1))))+ .with_min_value(Precision::Exact(ScalarValue::Float32(Some(0.1))))+ .with_sum_value(Precision::Exact(ScalarValue::Float32(Some(42.0)))),+ );++ let right = Statistics::default()+ .with_num_rows(Precision::Exact(7))+ .with_total_byte_size(Precision::Exact(29))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Exact(3))+ .with_min_value(Precision::Exact(ScalarValue::Int64(Some(1))))+ .with_max_value(Precision::Exact(ScalarValue::Int64(Some(34))))+ .with_sum_value(Precision::Exact(ScalarValue::Int64(Some(42))))+ .with_null_count(Precision::Exact(1)),+ )+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Exact(3))+ .with_min_value(Precision::Exact(ScalarValue::from("b")))+ .with_max_value(Precision::Exact(ScalarValue::from("z"))),+ )+ .add_column_statistics(ColumnStatistics::new_unknown());++ let expected = Statistics::default()+ .with_num_rows(Precision::Exact(12))+ .with_total_byte_size(Precision::Exact(52))+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Inexact(6))+ .with_min_value(Precision::Exact(ScalarValue::Int64(Some(-4))))+ .with_max_value(Precision::Exact(ScalarValue::Int64(Some(34))))+ .with_sum_value(Precision::Exact(ScalarValue::Int64(Some(84))))+ .with_null_count(Precision::Exact(1)),+ )+ .add_column_statistics(+ ColumnStatistics::new_unknown()+ .with_distinct_count(Precision::Inexact(5))+ .with_min_value(Precision::Exact(ScalarValue::from("a")))+ .with_max_value(Precision::Exact(ScalarValue::from("z"))),+ )+ .add_column_statistics(ColumnStatistics::new_unknown());++ (schema, left, right, expected)}#[test]- fn test_union_distinct_count() {- // (left_ndv, left_min, left_max, right_ndv, right_min, right_max, expected)- type NdvTestCase = (- Precision<usize>,- Option<i64>,- Option<i64>,- Precision<usize>,- Option<i64>,- Option<i64>,- Precision<usize>,- );- let cases: Vec<NdvTestCase> = vec![- // disjoint ranges: NDV = 5 + 3- (- Precision::Exact(5),- Some(0),- Some(10),- Precision::Exact(3),- Some(20),- Some(30),- Precision::Inexact(8),- ),- // identical ranges: intersection = max(10, 8) = 10- (- Precision::Exact(10),- Some(0),- Some(100),- Precision::Exact(8),- Some(0),- Some(100),- Precision::Inexact(10),- ),- // partial overlap: 50 + 50 + 25 = 125- (- Precision::Exact(100),- Some(0),- Some(100),- Precision::Exact(50),- Some(50),- Some(150),- Precision::Inexact(125),- ),- // right contained in left: 50 + 50 + 0 = 100- (- Precision::Exact(100),- Some(0),- Some(100),- Precision::Exact(50),- Some(25),- Some(75),- Precision::Inexact(100),- ),- // both constant, same value- (- Precision::Exact(1),- Some(5),- Some(5),- Precision::Exact(1),- Some(5),- Some(5),- Precision::Inexact(1),- ),- // both constant, different values- (- Precision::Exact(1),- Some(5),- Some(5),- Precision::Exact(1),- Some(10),- Some(10),- Precision::Inexact(2),- ),- // left constant within right range- (- Precision::Exact(1),- Some(5),- Some(5),- Precision::Exact(10),- Some(0),- Some(10),- Precision::Inexact(10),- ),- // left constant outside right range- (- Precision::Exact(1),- Some(20),- Some(20),- Precision::Exact(10),- Some(0),- Some(10),- Precision::Inexact(11),- ),- // right constant within left range- (- Precision::Exact(10),- Some(0),- Some(10),- Precision::Exact(1),- Some(5),- Some(5),- Precision::Inexact(10),- ),- // right constant outside left range- (- Precision::Exact(10),- Some(0),- Some(10),- Precision::Exact(1),- Some(20),- Some(20),- Precision::Inexact(11),- ),- // missing min/max falls back to sum (exact + exact)- (- Precision::Exact(10),- None,- None,- Precision::Exact(5),- None,- None,- Precision::Inexact(15),- ),- // missing min/max falls back to sum (exact + inexact)- (- Precision::Exact(10),- None,- None,- Precision::Inexact(5),- None,- None,- Precision::Inexact(15),- ),- // missing min/max falls back to sum (inexact + inexact)- (- Precision::Inexact(7),- None,- None,- Precision::Inexact(3),- None,- None,- Precision::Inexact(10),- ),- // one side absent- (- Precision::Exact(10),- None,- None,- Precision::Absent,- None,- None,- Precision::Absent,- ),- // one side absent (inexact + absent)- (- Precision::Inexact(4),- None,- None,- Precision::Absent,- None,- None,- Precision::Absent,- ),- ];+ fn test_union_partition_statistics_uses_shared_statistics_merge() -> Result<()> {+ let (schema, left, right, expected) = stats_merge_inputs();- for (- i,- (left_ndv, left_min, left_max, right_ndv, right_min, right_max, expected),- ) in cases.into_iter().enumerate()- {- let to_sv = |v| Precision::Exact(ScalarValue::Int64(Some(v)));- let left = ColumnStatistics {- distinct_count: left_ndv,- min_value: left_min.map(to_sv).unwrap_or(Precision::Absent),- max_value: left_max.map(to_sv).unwrap_or(Precision::Absent),- ..Default::default()- };- let right = ColumnStatistics {- distinct_count: right_ndv,- min_value: right_min.map(to_sv).unwrap_or(Precision::Absent),- max_value: right_max.map(to_sv).unwrap_or(Precision::Absent),- ..Default::default()- };- assert_eq!(- union_distinct_count(&left, &right),- expected,- "case {i} failed"- );- }+ let left: Arc<dyn ExecutionPlan> =+ Arc::new(StatisticsExec::new(left, schema.as_ref().clone()));+ let right: Arc<dyn ExecutionPlan> =+ Arc::new(StatisticsExec::new(right, schema.as_ref().clone()));++ let union = UnionExec::try_new(vec![left, right])?;+ let stats = union.partition_statistics(None)?;++ assert_eq!(stats.as_ref(), &expected);+ Ok(())+ }++ #[test]+ fn test_union_partition_statistics_uses_shared_statistics_merge_multicolumn()+ -> Result<()> {+ let (schema, left, right, expected) = stats_merge_multicolumn_inputs();++ let left: Arc<dyn ExecutionPlan> =+ Arc::new(StatisticsExec::new(left, schema.as_ref().clone()));+ let right: Arc<dyn ExecutionPlan> =+ Arc::new(StatisticsExec::new(right, schema.as_ref().clone()));++ let union = UnionExec::try_new(vec![left, right])?;+ let stats = union.partition_statistics(None)?;++ assert_eq!(stats.as_ref(), &expected);+ Ok(())+ }++ #[test]+ fn test_interleave_partition_statistics_uses_shared_statistics_merge() -> Result<()> {+ let (schema, left, right, expected) = stats_merge_inputs();+ let hash_expr = vec![col("a", schema.as_ref())?];++ let left: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(+ Arc::new(StatisticsExec::new(left, schema.as_ref().clone())),+ Partitioning::Hash(hash_expr.clone(), 2),+ )?);+ let right: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(+ Arc::new(StatisticsExec::new(right, schema.as_ref().clone())),+ Partitioning::Hash(hash_expr, 2),+ )?);++ let interleave = InterleaveExec::try_new(vec![left, right])?;+ let stats = interleave.partition_statistics(None)?;++ assert_eq!(stats.as_ref(), &expected);+ Ok(())+ }++ #[test]+ fn test_interleave_partition_statistics_for_partition_uses_shared_statistics_merge()+ -> Result<()> {+ let (schema, left, right, _) = stats_merge_inputs();+ let hash_expr = vec![col("a", schema.as_ref())?];++ let left: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(+ Arc::new(StatisticsExec::new(left, schema.as_ref().clone())),+ Partitioning::Hash(hash_expr.clone(), 2),+ )?);+ let right: Arc<dyn ExecutionPlan> = Arc::new(RepartitionExec::try_new(+ Arc::new(StatisticsExec::new(right, schema.as_ref().clone())),+ Partitioning::Hash(hash_expr, 2),+ )?);++ let interleave = InterleaveExec::try_new(vec![left, right])?;+ let stats = interleave.partition_statistics(Some(0))?;++ let expected = Statistics::default()+ .with_num_rows(Precision::Inexact(5))+ .with_total_byte_size(Precision::Inexact(25))+ .add_column_statistics(ColumnStatistics::new_unknown());++ assert_eq!(stats.as_ref(), &expected);+ Ok(())}#[tokio::test]