Skip to content

Apache DataFusion SQL Query Engine

docs(physical-plan): add doctest example for StatisticsContext::compute (#25689)

Alessandro Solimando04e6b3780f9a
diff --git a/datafusion/physical-plan/src/statistics.rs b/datafusion/physical-plan/src/statistics.rs
index c1a8ad29bc..18d9d0f28b 100644
--- a/datafusion/physical-plan/src/statistics.rs
+++ b/datafusion/physical-plan/src/statistics.rs
@@ -170,6 +170,38 @@ impl StatisticsContext {
///
/// With no providers registered this is the plain built-in walk: only the
/// `statistics` cache is touched, so it carries no extension overhead.
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// # use arrow::datatypes::{DataType, Field, Schema};
+ /// # use datafusion_common::Statistics;
+ /// # use datafusion_common::stats::Precision;
+ /// # use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
+ /// # use datafusion_physical_plan::test::exec::StatisticsExec;
+ ///
+ /// let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ /// let overall_stats =
+ /// Statistics::new_unknown(&schema).with_num_rows(Precision::Exact(100));
+ /// let partition_stats = vec![
+ /// Statistics::new_unknown(&schema).with_num_rows(Precision::Exact(60)),
+ /// Statistics::new_unknown(&schema).with_num_rows(Precision::Exact(40)),
+ /// ];
+ /// let plan = StatisticsExec::new(overall_stats, schema)
+ /// .with_partition_statistics(partition_stats);
+ ///
+ /// let context = StatisticsContext::new();
+ ///
+ /// // Statistics for the whole plan (all partitions combined).
+ /// let overall = context.compute(&plan, &StatisticsArgs::new())?;
+ /// assert_eq!(overall.num_rows, Precision::Exact(100));
+ ///
+ /// // Statistics for a single partition.
+ /// let args = StatisticsArgs::new().with_partition(Some(0));
+ /// let per_partition = context.compute(&plan, &args)?;
+ /// assert_eq!(per_partition.num_rows, Precision::Exact(60));
+ /// # Ok::<(), datafusion_common::DataFusionError>(())
+ /// ```
pub fn compute(
&self,
plan: &dyn ExecutionPlan,
diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rs
index f9517469d5..dee87011b9 100644
--- a/datafusion/physical-plan/src/test/exec.rs
+++ b/datafusion/physical-plan/src/test/exec.rs
@@ -654,6 +654,10 @@ impl ExecutionPlan for ErrorExec {
#[derive(Debug, Clone)]
pub struct StatisticsExec {
stats: Statistics,
+ /// Per-partition statistics. `None` means they are not modeled: a
+ /// request for a specific partition then returns
+ /// [`Statistics::new_unknown`].
+ partition_statistics: Option<Vec<Statistics>>,
schema: Arc<Schema>,
cache: Arc<PlanProperties>,
}
@@ -664,19 +668,44 @@ impl StatisticsExec {
schema.fields().len(),
"if defined, the column statistics vector length should be the number of fields"
);
- let cache = Self::compute_properties(Arc::new(schema.clone()));
+ let cache = Self::compute_properties(Arc::new(schema.clone()), 2);
Self {
stats,
+ partition_statistics: None,
schema: Arc::new(schema),
cache: Arc::new(cache),
}
}
+ /// Sets the per-partition statistics, in partition order.
+ ///
+ /// The plan's partition count becomes `partition_statistics.len()`. Each
+ /// entry's column statistics length must match the schema, like the
+ /// overall statistics passed to [`Self::new`].
+ pub fn with_partition_statistics(
+ mut self,
+ partition_statistics: Vec<Statistics>,
+ ) -> Self {
+ for stats in &partition_statistics {
+ assert_eq!(
+ stats.column_statistics.len(),
+ self.schema.fields().len(),
+ "if defined, the column statistics vector length should be the number of fields"
+ );
+ }
+ self.cache = Arc::new(Self::compute_properties(
+ Arc::clone(&self.schema),
+ partition_statistics.len(),
+ ));
+ self.partition_statistics = Some(partition_statistics);
+ self
+ }
+
/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.
- fn compute_properties(schema: SchemaRef) -> PlanProperties {
+ fn compute_properties(schema: SchemaRef, partition_count: usize) -> PlanProperties {
PlanProperties::new(
EquivalenceProperties::new(schema),
- Partitioning::UnknownPartitioning(2),
+ Partitioning::UnknownPartitioning(partition_count),
EmissionType::Incremental,
Boundedness::Bounded,
)
@@ -757,10 +786,20 @@ impl ExecutionPlan for StatisticsExec {
_input_stats: &[Arc<Statistics>],
args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
- Ok(Arc::new(if args.partition().is_some() {
- Statistics::new_unknown(&self.schema)
- } else {
- self.stats.clone()
+ Ok(Arc::new(match args.partition() {
+ Some(idx) => match &self.partition_statistics {
+ Some(partition_statistics) => match partition_statistics.get(idx) {
+ Some(stats) => stats.clone(),
+ None => {
+ return internal_err!(
+ "Invalid partition index: {idx}, the partition count is {}",
+ partition_statistics.len()
+ );
+ }
+ },
+ None => Statistics::new_unknown(&self.schema),
+ },
+ None => self.stats.clone(),
}))
}
}