Apache DataFusion SQL Query Engine
docs(physical-plan): add doctest example for StatisticsContext::compute (#25689)
Alessandro Solimando04e6b3780f9a
diff --git a/datafusion/physical-plan/src/statistics.rs b/datafusion/physical-plan/src/statistics.rsindex c1a8ad29bc..18d9d0f28b 100644--- a/datafusion/physical-plan/src/statistics.rs+++ b/datafusion/physical-plan/src/statistics.rs@@ -170,6 +170,38 @@ impl StatisticsContext {////// With no providers registered this is the plain built-in walk: only the/// `statistics` cache is touched, so it carries no extension overhead.+ ///+ /// # Example+ ///+ /// ```+ /// # use arrow::datatypes::{DataType, Field, Schema};+ /// # use datafusion_common::Statistics;+ /// # use datafusion_common::stats::Precision;+ /// # use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};+ /// # use datafusion_physical_plan::test::exec::StatisticsExec;+ ///+ /// let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ /// let overall_stats =+ /// Statistics::new_unknown(&schema).with_num_rows(Precision::Exact(100));+ /// let partition_stats = vec![+ /// Statistics::new_unknown(&schema).with_num_rows(Precision::Exact(60)),+ /// Statistics::new_unknown(&schema).with_num_rows(Precision::Exact(40)),+ /// ];+ /// let plan = StatisticsExec::new(overall_stats, schema)+ /// .with_partition_statistics(partition_stats);+ ///+ /// let context = StatisticsContext::new();+ ///+ /// // Statistics for the whole plan (all partitions combined).+ /// let overall = context.compute(&plan, &StatisticsArgs::new())?;+ /// assert_eq!(overall.num_rows, Precision::Exact(100));+ ///+ /// // Statistics for a single partition.+ /// let args = StatisticsArgs::new().with_partition(Some(0));+ /// let per_partition = context.compute(&plan, &args)?;+ /// assert_eq!(per_partition.num_rows, Precision::Exact(60));+ /// # Ok::<(), datafusion_common::DataFusionError>(())+ /// ```pub fn compute(&self,plan: &dyn ExecutionPlan,diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rsindex f9517469d5..dee87011b9 100644--- a/datafusion/physical-plan/src/test/exec.rs+++ b/datafusion/physical-plan/src/test/exec.rs@@ -654,6 +654,10 @@ impl ExecutionPlan for ErrorExec {#[derive(Debug, Clone)]pub struct StatisticsExec {stats: Statistics,+ /// Per-partition statistics. `None` means they are not modeled: a+ /// request for a specific partition then returns+ /// [`Statistics::new_unknown`].+ partition_statistics: Option<Vec<Statistics>>,schema: Arc<Schema>,cache: Arc<PlanProperties>,}@@ -664,19 +668,44 @@ impl StatisticsExec {schema.fields().len(),"if defined, the column statistics vector length should be the number of fields");- let cache = Self::compute_properties(Arc::new(schema.clone()));+ let cache = Self::compute_properties(Arc::new(schema.clone()), 2);Self {stats,+ partition_statistics: None,schema: Arc::new(schema),cache: Arc::new(cache),}}+ /// Sets the per-partition statistics, in partition order.+ ///+ /// The plan's partition count becomes `partition_statistics.len()`. Each+ /// entry's column statistics length must match the schema, like the+ /// overall statistics passed to [`Self::new`].+ pub fn with_partition_statistics(+ mut self,+ partition_statistics: Vec<Statistics>,+ ) -> Self {+ for stats in &partition_statistics {+ assert_eq!(+ stats.column_statistics.len(),+ self.schema.fields().len(),+ "if defined, the column statistics vector length should be the number of fields"+ );+ }+ self.cache = Arc::new(Self::compute_properties(+ Arc::clone(&self.schema),+ partition_statistics.len(),+ ));+ self.partition_statistics = Some(partition_statistics);+ self+ }+/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.- fn compute_properties(schema: SchemaRef) -> PlanProperties {+ fn compute_properties(schema: SchemaRef, partition_count: usize) -> PlanProperties {PlanProperties::new(EquivalenceProperties::new(schema),- Partitioning::UnknownPartitioning(2),+ Partitioning::UnknownPartitioning(partition_count),EmissionType::Incremental,Boundedness::Bounded,)@@ -757,10 +786,20 @@ impl ExecutionPlan for StatisticsExec {_input_stats: &[Arc<Statistics>],args: &StatisticsArgs,) -> Result<Arc<Statistics>> {- Ok(Arc::new(if args.partition().is_some() {- Statistics::new_unknown(&self.schema)- } else {- self.stats.clone()+ Ok(Arc::new(match args.partition() {+ Some(idx) => match &self.partition_statistics {+ Some(partition_statistics) => match partition_statistics.get(idx) {+ Some(stats) => stats.clone(),+ None => {+ return internal_err!(+ "Invalid partition index: {idx}, the partition count is {}",+ partition_statistics.len()+ );+ }+ },+ None => Statistics::new_unknown(&self.schema),+ },+ None => self.stats.clone(),}))}}