Skip to content

Apache DataFusion SQL Query Engine

feat: Add `partition_stats()` for `EmptyExec` (#20203)

Jonathan Chenaa9520e6e260
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
index b33305c23e..5d168c0641 100644
--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs
+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
@@ -1294,4 +1294,64 @@ mod test {
Ok(())
}
+
+ #[tokio::test]
+ async fn test_statistics_by_partition_of_empty_exec() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ ]));
+
+ // Try to test with single partition
+ let empty_single = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+
+ let stats = empty_single.partition_statistics(Some(0))?;
+ assert_eq!(stats.num_rows, Precision::Exact(0));
+ assert_eq!(stats.total_byte_size, Precision::Exact(0));
+ assert_eq!(stats.column_statistics.len(), 2);
+
+ for col_stat in &stats.column_statistics {
+ assert_eq!(col_stat.null_count, Precision::Exact(0));
+ assert_eq!(col_stat.distinct_count, Precision::Exact(0));
+ assert_eq!(col_stat.byte_size, Precision::Exact(0));
+ assert_eq!(col_stat.min_value, Precision::<ScalarValue>::Absent);
+ assert_eq!(col_stat.max_value, Precision::<ScalarValue>::Absent);
+ assert_eq!(col_stat.sum_value, Precision::<ScalarValue>::Absent);
+ assert_eq!(col_stat.byte_size, Precision::Exact(0));
+ }
+
+ let overall_stats = empty_single.partition_statistics(None)?;
+ assert_eq!(stats, overall_stats);
+
+ validate_statistics_with_data(empty_single, vec![ExpectedStatistics::Empty], 0)
+ .await?;
+
+ // Test with multiple partitions
+ let empty_multi: Arc<dyn ExecutionPlan> =
+ Arc::new(EmptyExec::new(Arc::clone(&schema)).with_partitions(3));
+
+ let statistics = (0..empty_multi.output_partitioning().partition_count())
+ .map(|idx| empty_multi.partition_statistics(Some(idx)))
+ .collect::<Result<Vec<_>>>()?;
+
+ assert_eq!(statistics.len(), 3);
+
+ for stat in &statistics {
+ assert_eq!(stat.num_rows, Precision::Exact(0));
+ assert_eq!(stat.total_byte_size, Precision::Exact(0));
+ }
+
+ validate_statistics_with_data(
+ empty_multi,
+ vec![
+ ExpectedStatistics::Empty,
+ ExpectedStatistics::Empty,
+ ExpectedStatistics::Empty,
+ ],
+ 0,
+ )
+ .await?;
+
+ Ok(())
+ }
}
diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs
index fcfbcfa3e8..43f12469db 100644
--- a/datafusion/physical-plan/src/empty.rs
+++ b/datafusion/physical-plan/src/empty.rs
@@ -21,7 +21,7 @@ use std::any::Any;
use std::sync::Arc;
use crate::memory::MemoryStream;
-use crate::{DisplayAs, PlanProperties, SendableRecordBatchStream, Statistics, common};
+use crate::{DisplayAs, PlanProperties, SendableRecordBatchStream, Statistics};
use crate::{
DisplayFormatType, ExecutionPlan, Partitioning,
execution_plan::{Boundedness, EmissionType},
@@ -29,7 +29,8 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
-use datafusion_common::{Result, assert_or_internal_err};
+use datafusion_common::stats::Precision;
+use datafusion_common::{ColumnStatistics, Result, ScalarValue, assert_or_internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -169,20 +170,31 @@ impl ExecutionPlan for EmptyExec {
);
}
- let batch = self
- .data()
- .expect("Create empty RecordBatch should not fail");
- Ok(common::compute_record_batch_statistics(
- &[batch],
- &self.schema,
- None,
- ))
+ // Build explicit stats: exact zero rows and bytes, with explicit known column stats
+ let mut stats = Statistics::default()
+ .with_num_rows(Precision::Exact(0))
+ .with_total_byte_size(Precision::Exact(0));
+
+ // Add explicit column stats for each field in schema
+ for _ in self.schema.fields() {
+ stats = stats.add_column_statistics(ColumnStatistics {
+ null_count: Precision::Exact(0),
+ distinct_count: Precision::Exact(0),
+ min_value: Precision::<ScalarValue>::Absent,
+ max_value: Precision::<ScalarValue>::Absent,
+ sum_value: Precision::<ScalarValue>::Absent,
+ byte_size: Precision::Exact(0),
+ });
+ }
+
+ Ok(stats)
}
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::common;
use crate::test;
use crate::with_new_children_if_necessary;