Apache DataFusion SQL Query Engine
feat: Add `partition_stats()` for `EmptyExec` (#20203)
Jonathan Chenaa9520e6e260
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex b33305c23e..5d168c0641 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -1294,4 +1294,64 @@ mod test {Ok(())}++ #[tokio::test]+ async fn test_statistics_by_partition_of_empty_exec() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("name", DataType::Utf8, true),+ ]));++ // Try to test with single partition+ let empty_single = Arc::new(EmptyExec::new(Arc::clone(&schema)));++ let stats = empty_single.partition_statistics(Some(0))?;+ assert_eq!(stats.num_rows, Precision::Exact(0));+ assert_eq!(stats.total_byte_size, Precision::Exact(0));+ assert_eq!(stats.column_statistics.len(), 2);++ for col_stat in &stats.column_statistics {+ assert_eq!(col_stat.null_count, Precision::Exact(0));+ assert_eq!(col_stat.distinct_count, Precision::Exact(0));+ assert_eq!(col_stat.byte_size, Precision::Exact(0));+ assert_eq!(col_stat.min_value, Precision::<ScalarValue>::Absent);+ assert_eq!(col_stat.max_value, Precision::<ScalarValue>::Absent);+ assert_eq!(col_stat.sum_value, Precision::<ScalarValue>::Absent);+ assert_eq!(col_stat.byte_size, Precision::Exact(0));+ }++ let overall_stats = empty_single.partition_statistics(None)?;+ assert_eq!(stats, overall_stats);++ validate_statistics_with_data(empty_single, vec![ExpectedStatistics::Empty], 0)+ .await?;++ // Test with multiple partitions+ let empty_multi: Arc<dyn ExecutionPlan> =+ Arc::new(EmptyExec::new(Arc::clone(&schema)).with_partitions(3));++ let statistics = (0..empty_multi.output_partitioning().partition_count())+ .map(|idx| empty_multi.partition_statistics(Some(idx)))+ .collect::<Result<Vec<_>>>()?;++ assert_eq!(statistics.len(), 3);++ for stat in &statistics {+ assert_eq!(stat.num_rows, Precision::Exact(0));+ assert_eq!(stat.total_byte_size, Precision::Exact(0));+ }++ validate_statistics_with_data(+ empty_multi,+ vec![+ ExpectedStatistics::Empty,+ ExpectedStatistics::Empty,+ ExpectedStatistics::Empty,+ ],+ 0,+ )+ .await?;++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rsindex fcfbcfa3e8..43f12469db 100644--- a/datafusion/physical-plan/src/empty.rs+++ b/datafusion/physical-plan/src/empty.rs@@ -21,7 +21,7 @@ use std::any::Any;use std::sync::Arc;use crate::memory::MemoryStream;-use crate::{DisplayAs, PlanProperties, SendableRecordBatchStream, Statistics, common};+use crate::{DisplayAs, PlanProperties, SendableRecordBatchStream, Statistics};use crate::{DisplayFormatType, ExecutionPlan, Partitioning,execution_plan::{Boundedness, EmissionType},@@ -29,7 +29,8 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;-use datafusion_common::{Result, assert_or_internal_err};+use datafusion_common::stats::Precision;+use datafusion_common::{ColumnStatistics, Result, ScalarValue, assert_or_internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;@@ -169,20 +170,31 @@ impl ExecutionPlan for EmptyExec {);}- let batch = self- .data()- .expect("Create empty RecordBatch should not fail");- Ok(common::compute_record_batch_statistics(- &[batch],- &self.schema,- None,- ))+ // Build explicit stats: exact zero rows and bytes, with explicit known column stats+ let mut stats = Statistics::default()+ .with_num_rows(Precision::Exact(0))+ .with_total_byte_size(Precision::Exact(0));++ // Add explicit column stats for each field in schema+ for _ in self.schema.fields() {+ stats = stats.add_column_statistics(ColumnStatistics {+ null_count: Precision::Exact(0),+ distinct_count: Precision::Exact(0),+ min_value: Precision::<ScalarValue>::Absent,+ max_value: Precision::<ScalarValue>::Absent,+ sum_value: Precision::<ScalarValue>::Absent,+ byte_size: Precision::Exact(0),+ });+ }++ Ok(stats)}}#[cfg(test)]mod tests {use super::*;+ use crate::common;use crate::test;use crate::with_new_children_if_necessary;