Skip to content

Apache DataFusion SQL Query Engine

refactor(physical-plan): externalize statistics traversal into StatisticsContext (#23051)

Alessandro Solimandoff774c6c3ce3
diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rs
index 6df1113e47..2c696d92d7 100644
--- a/datafusion-examples/examples/relation_planner/table_sample.rs
+++ b/datafusion-examples/examples/relation_planner/table_sample.rs
@@ -108,7 +108,8 @@ use datafusion::{
},
physical_expr::EquivalenceProperties,
physical_plan::{
- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, StatisticsArgs,
+ ChildStats, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
+ StatisticsArgs,
metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RecordOutput},
},
physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner},
@@ -722,10 +723,16 @@ impl ExecutionPlan for SampleExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let mut stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let mut stats = input_stats[0].as_ref().clone();
let ratio = self.upper_bound - self.lower_bound;
// Scale statistics by sampling ratio (inexact due to randomness)
diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs
index d9254bc8cf..651a15d776 100644
--- a/datafusion/core/src/datasource/file_format/csv.rs
+++ b/datafusion/core/src/datasource/file_format/csv.rs
@@ -45,7 +45,7 @@ mod tests {
use datafusion_datasource::file_format::FileFormat;
use datafusion_datasource::write::BatchSerializer;
use datafusion_expr::{col, lit};
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::{ExecutionPlan, collect};
use arrow::array::{
@@ -217,11 +217,14 @@ mod tests {
// test metadata
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?.num_rows,
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
+ .num_rows,
Precision::Absent
);
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent
);
diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rs
index 1de0ec2e77..1f6f27242e 100644
--- a/datafusion/core/src/datasource/file_format/json.rs
+++ b/datafusion/core/src/datasource/file_format/json.rs
@@ -36,7 +36,7 @@ mod tests {
BatchDeserializer, DecoderDeserializer, DeserializerOutput,
};
use datafusion_datasource::file_format::FileFormat;
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::{ExecutionPlan, collect};
use arrow::compute::concat_batches;
@@ -119,11 +119,14 @@ mod tests {
// test metadata
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?.num_rows,
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
+ .num_rows,
Precision::Absent
);
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent
);
diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs
index 5f7fc2eebf..0f5db4a057 100644
--- a/datafusion/core/src/datasource/file_format/parquet.rs
+++ b/datafusion/core/src/datasource/file_format/parquet.rs
@@ -141,7 +141,7 @@ mod tests {
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_execution::runtime_env::RuntimeEnv;
use datafusion_expr::dml::InsertOp;
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
use datafusion_physical_plan::{ExecutionPlan, collect};
@@ -716,12 +716,15 @@ mod tests {
// test metadata
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?.num_rows,
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
+ .num_rows,
Precision::Exact(8)
);
// TODO correct byte size: https://github.com/apache/datafusion/issues/14936
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent,
);
@@ -766,11 +769,14 @@ mod tests {
// note: even if the limit is set, the executor rounds up to the batch size
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?.num_rows,
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
+ .num_rows,
Precision::Exact(8)
);
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent,
);
diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs
index d9cbd5bace..db2623c24d 100644
--- a/datafusion/core/src/datasource/listing/table.rs
+++ b/datafusion/core/src/datasource/listing/table.rs
@@ -145,7 +145,7 @@ mod tests {
use datafusion_physical_expr::expressions::{Column, binary};
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_plan::empty::EmptyExec;
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::{
ExecutionPlanProperties, Partitioning, RangePartitioning, SplitPoint, collect,
};
@@ -266,11 +266,14 @@ mod tests {
// test metadata
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?.num_rows,
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
+ .num_rows,
Precision::Exact(8)
);
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent,
);
@@ -1612,16 +1615,16 @@ mod tests {
let exec_default = table_default.scan(&state, None, &[], None).await?;
assert_eq!(
- exec_default
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec_default.as_ref(), &StatisticsArgs::new())?
.num_rows,
Precision::Exact(8)
);
// TODO correct byte size: https://github.com/apache/datafusion/issues/14936
assert_eq!(
- exec_default
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec_default.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent
);
@@ -1638,14 +1641,14 @@ mod tests {
let exec_disabled = table_disabled.scan(&state, None, &[], None).await?;
assert_eq!(
- exec_disabled
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec_disabled.as_ref(), &StatisticsArgs::new())?
.num_rows,
Precision::Absent
);
assert_eq!(
- exec_disabled
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec_disabled.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent
);
@@ -1662,15 +1665,15 @@ mod tests {
let exec_enabled = table_enabled.scan(&state, None, &[], None).await?;
assert_eq!(
- exec_enabled
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec_enabled.as_ref(), &StatisticsArgs::new())?
.num_rows,
Precision::Exact(8)
);
// TODO correct byte size: https://github.com/apache/datafusion/issues/14936
assert_eq!(
- exec_enabled
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(exec_enabled.as_ref(), &StatisticsArgs::new())?
.total_byte_size,
Precision::Absent,
);
diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rs
index 0b0df57e5a..c70722cb2f 100644
--- a/datafusion/core/tests/custom_sources_cases/mod.rs
+++ b/datafusion/core/tests/custom_sources_cases/mod.rs
@@ -179,7 +179,11 @@ impl ExecutionPlan for CustomExecutionPlan {
Ok(Box::pin(TestCustomRecordBatchStream { nb_batch: 1 }))
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
return Ok(Arc::new(Statistics::new_unknown(&self.schema())));
}
diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rs
index 1ea2b202b1..d289b5c348 100644
--- a/datafusion/core/tests/custom_sources_cases/statistics.rs
+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs
@@ -35,8 +35,8 @@ use datafusion::{
use datafusion_catalog::Session;
use datafusion_common::{project_schema, stats::Precision};
use datafusion_physical_expr::EquivalenceProperties;
-use datafusion_physical_plan::StatisticsArgs;
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
+use datafusion_physical_plan::{StatisticsArgs, StatisticsContext};
use async_trait::async_trait;
@@ -174,7 +174,11 @@ impl ExecutionPlan for StatisticsValidation {
unimplemented!("This plan only serves for testing statistics")
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
} else {
@@ -233,7 +237,8 @@ async fn sql_basic() -> Result<()> {
// the statistics should be those of the source
assert_eq!(
stats,
- *physical_plan.statistics_with_args(&StatisticsArgs::new())?
+ *StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?
);
Ok(())
@@ -250,7 +255,8 @@ async fn sql_filter() -> Result<()> {
.unwrap();
let physical_plan = df.create_physical_plan().await.unwrap();
- let stats = physical_plan.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?;
assert_eq!(stats.num_rows, Precision::Inexact(7));
Ok(())
@@ -265,7 +271,8 @@ async fn sql_limit() -> Result<()> {
let physical_plan = df.create_physical_plan().await.unwrap();
// when the limit is smaller than the original number of lines we mark the statistics as inexact
// and cap NDV at the new row count
- let limit_stats = physical_plan.statistics_with_args(&StatisticsArgs::new())?;
+ let limit_stats = StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?;
assert_eq!(limit_stats.num_rows, Precision::Exact(5));
// c1: NDV=2 stays at 2 (already below limit of 5)
assert_eq!(
@@ -286,7 +293,8 @@ async fn sql_limit() -> Result<()> {
// when the limit is larger than the original number of lines, statistics remain unchanged
assert_eq!(
stats,
- *physical_plan.statistics_with_args(&StatisticsArgs::new())?
+ *StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?
);
Ok(())
@@ -304,7 +312,8 @@ async fn sql_window() -> Result<()> {
let physical_plan = df.create_physical_plan().await.unwrap();
- let result = physical_plan.statistics_with_args(&StatisticsArgs::new())?;
+ let result = StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?;
assert_eq!(stats.num_rows, result.num_rows);
let col_stats = &result.column_statistics;
diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rs
index e0eed40283..f6d733ec69 100644
--- a/datafusion/core/tests/parquet/file_statistics.rs
+++ b/datafusion/core/tests/parquet/file_statistics.rs
@@ -45,7 +45,7 @@ use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::filter::FilterExec;
-use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use tempfile::tempdir;
#[tokio::test]
@@ -65,7 +65,8 @@ async fn check_stats_precision_with_filter_pushdown() {
// Scan without filter, stats are exact
let exec = table.scan(&state, None, &[], None).await.unwrap();
assert_eq!(
- exec.statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec.as_ref(), &StatisticsArgs::new())
.unwrap()
.num_rows,
Precision::Exact(8),
@@ -99,8 +100,8 @@ async fn check_stats_precision_with_filter_pushdown() {
);
// Scan with filter pushdown, stats are inexact
assert_eq!(
- optimized_exec
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(optimized_exec.as_ref(), &StatisticsArgs::new())
.unwrap()
.num_rows,
Precision::Inexact(8),
@@ -135,15 +136,15 @@ async fn load_table_stats_with_session_level_cache() {
let exec1 = table1.scan(&state1, None, &[], None).await.unwrap();
assert_eq!(
- exec1
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec1.as_ref(), &StatisticsArgs::new())
.unwrap()
.num_rows,
Precision::Exact(8)
);
assert_eq!(
- exec1
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec1.as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
// Byte size is absent because we cannot estimate the output size
@@ -157,15 +158,15 @@ async fn load_table_stats_with_session_level_cache() {
assert_eq!(get_static_cache_size(&state2), 0);
let exec2 = table2.scan(&state2, None, &[], None).await.unwrap();
assert_eq!(
- exec2
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec2.as_ref(), &StatisticsArgs::new())
.unwrap()
.num_rows,
Precision::Exact(8)
);
assert_eq!(
- exec2
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec2.as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
// Absent because the data contains variable length columns
@@ -178,15 +179,15 @@ async fn load_table_stats_with_session_level_cache() {
assert_eq!(get_static_cache_size(&state1), 1);
let exec3 = table1.scan(&state1, None, &[], None).await.unwrap();
assert_eq!(
- exec3
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec3.as_ref(), &StatisticsArgs::new())
.unwrap()
.num_rows,
Precision::Exact(8)
);
assert_eq!(
- exec3
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(exec3.as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
// Absent because the data contains variable length columns
@@ -252,7 +253,9 @@ async fn anonymous_parquet_stats_cache_with_explicit_wider_schema() {
.await
.unwrap();
- let stats = plan.statistics_with_args(&StatisticsArgs::new()).unwrap();
+ let stats = StatisticsContext::new()
+ .compute(plan.as_ref(), &StatisticsArgs::new())
+ .unwrap();
assert_eq!(stats.column_statistics.len(), 2);
assert_eq!(stats.column_statistics[1].null_count, Precision::Exact(1));
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs
index 80e0a3f23e..2db9f18f31 100644
--- a/datafusion/core/tests/physical_optimizer/join_selection.rs
+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs
@@ -45,6 +45,7 @@ use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, Partitio
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, StatisticsArgs,
+ StatisticsContext,
execution_plan::{Boundedness, EmissionType},
};
@@ -248,17 +249,15 @@ async fn test_join_with_swap() {
.expect("The type of the plan should not be changed");
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -296,17 +295,15 @@ async fn test_left_join_no_swap() {
.expect("The type of the plan should not be changed");
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -347,17 +344,15 @@ async fn test_join_with_swap_semi() {
assert_eq!(swapped_join.schema().fields().len(), 1);
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -400,17 +395,15 @@ async fn test_join_with_swap_mark() {
assert_eq!(swapped_join.schema().fields().len(), 2);
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -528,17 +521,15 @@ async fn test_join_no_swap() {
.expect("The type of the plan should not be changed");
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -603,17 +594,15 @@ async fn test_nl_join_with_swap(join_type: JoinType) {
);
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -676,17 +665,15 @@ async fn test_nl_join_with_swap_no_proj(join_type: JoinType) {
);
assert_eq!(
- swapped_join
- .left()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.left().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(8192)
);
assert_eq!(
- swapped_join
- .right()
- .statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new()
+ .compute(swapped_join.right().as_ref(), &StatisticsArgs::new())
.unwrap()
.total_byte_size,
Precision::Inexact(2097152)
@@ -1152,7 +1139,11 @@ impl ExecutionPlan for StatisticsExec {
unimplemented!("This plan only serves for testing statistics")
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
Ok(Arc::new(if args.partition().is_some() {
Statistics::new_unknown(&self.schema)
} else {
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
index 6a79c668bd..6cabcdb710 100644
--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs
+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
@@ -55,7 +55,7 @@ mod test {
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort::SortExec;
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
use datafusion_physical_plan::windows::{WindowAggExec, create_window_expr};
use datafusion_physical_plan::{
@@ -240,7 +240,8 @@ mod test {
let scan = create_scan_exec_with_statistics(None, Some(2)).await;
let statistics = (0..scan.output_partitioning().partition_count())
.map(|idx| {
- scan.statistics_with_args(
+ StatisticsContext::new().compute(
+ scan.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -288,7 +289,8 @@ mod test {
Arc::new(ProjectionExec::try_new(exprs, scan)?);
let statistics = (0..projection.output_partitioning().partition_count())
.map(|idx| {
- projection.statistics_with_args(
+ StatisticsContext::new().compute(
+ projection.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -324,7 +326,8 @@ mod test {
let sort_exec: Arc<dyn ExecutionPlan> = Arc::new(sort);
let statistics = (0..sort_exec.output_partitioning().partition_count())
.map(|idx| {
- sort_exec.statistics_with_args(
+ StatisticsContext::new().compute(
+ sort_exec.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -367,7 +370,8 @@ mod test {
);
let statistics = (0..sort_exec.output_partitioning().partition_count())
.map(|idx| {
- sort_exec.statistics_with_args(
+ StatisticsContext::new().compute(
+ sort_exec.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -397,7 +401,8 @@ mod test {
)?;
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, scan)?);
- let full_statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let full_statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
let expected_full_statistic = Statistics {
num_rows: Precision::Inexact(0),
total_byte_size: Precision::Inexact(0),
@@ -424,7 +429,8 @@ mod test {
let statistics = (0..filter.output_partitioning().partition_count())
.map(|idx| {
- filter.statistics_with_args(
+ StatisticsContext::new().compute(
+ filter.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -464,7 +470,8 @@ mod test {
UnionExec::try_new(vec![scan.clone(), scan])?;
let statistics = (0..union_exec.output_partitioning().partition_count())
.map(|idx| {
- union_exec.statistics_with_args(
+ StatisticsContext::new().compute(
+ union_exec.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -531,7 +538,8 @@ mod test {
// Verify the result of partition statistics
let stats = (0..interleave.output_partitioning().partition_count())
.map(|idx| {
- interleave.statistics_with_args(
+ StatisticsContext::new().compute(
+ interleave.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -581,7 +589,8 @@ mod test {
Arc::new(CrossJoinExec::new(left_scan, right_scan));
let statistics = (0..cross_join.output_partitioning().partition_count())
.map(|idx| {
- cross_join.statistics_with_args(
+ StatisticsContext::new().compute(
+ cross_join.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -691,8 +700,8 @@ mod test {
// Test partition_statistics(None) - returns overall statistics
// For RightSemi join, output columns come from right side only
- let full_statistics =
- nested_loop_join.statistics_with_args(&StatisticsArgs::new())?;
+ let full_statistics = StatisticsContext::new()
+ .compute(nested_loop_join.as_ref(), &StatisticsArgs::new())?;
// With empty join columns, estimate_join_statistics returns Inexact row count
// based on the outer side (right side for RightSemi)
let expected_full_statistics = create_partition_statistics(
@@ -728,7 +737,8 @@ mod test {
let statistics = (0..nested_loop_join.output_partitioning().partition_count())
.map(|idx| {
- nested_loop_join.statistics_with_args(
+ StatisticsContext::new().compute(
+ nested_loop_join.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -762,7 +772,8 @@ mod test {
);
let statistics = (0..coalesce_partitions.output_partitioning().partition_count())
.map(|idx| {
- coalesce_partitions.statistics_with_args(
+ StatisticsContext::new().compute(
+ coalesce_partitions.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -783,7 +794,8 @@ mod test {
Arc::new(LocalLimitExec::new(scan.clone(), 1));
let statistics = (0..local_limit.output_partitioning().partition_count())
.map(|idx| {
- local_limit.statistics_with_args(
+ StatisticsContext::new().compute(
+ local_limit.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -814,7 +826,8 @@ mod test {
Arc::new(GlobalLimitExec::new(scan.clone(), 0, Some(2)));
let statistics = (0..global_limit.output_partitioning().partition_count())
.map(|idx| {
- global_limit.statistics_with_args(
+ StatisticsContext::new().compute(
+ global_limit.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -876,8 +889,10 @@ mod test {
@"AggregateExec: mode=Partial, gby=[id@0 as id, 1 + id@0 as expr], aggr=[COUNT(c)]"
);
- let p0_statistics = aggregate_exec_partial
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?;
+ let p0_statistics = StatisticsContext::new().compute(
+ aggregate_exec_partial.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )?;
// Aggregate doesn't propagate num_rows and ColumnStatistics byte_size from input
let expected_p0_statistics = Statistics {
@@ -916,8 +931,10 @@ mod test {
],
};
- let p1_statistics = aggregate_exec_partial
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?;
+ let p1_statistics = StatisticsContext::new().compute(
+ aggregate_exec_partial.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(1)),
+ )?;
assert_eq!(*p1_statistics, expected_p1_statistics);
validate_statistics_with_data(
@@ -939,12 +956,16 @@ mod test {
aggregate_exec_partial.schema(),
)?);
- let p0_statistics = agg_final
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?;
+ let p0_statistics = StatisticsContext::new().compute(
+ agg_final.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )?;
assert_eq!(*p0_statistics, expected_p0_statistics);
- let p1_statistics = agg_final
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?;
+ let p1_statistics = StatisticsContext::new().compute(
+ agg_final.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(1)),
+ )?;
assert_eq!(*p1_statistics, expected_p1_statistics);
validate_statistics_with_data(
@@ -991,13 +1012,17 @@ mod test {
assert_eq!(
empty_stat,
- *agg_partial
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?
+ *StatisticsContext::new().compute(
+ agg_partial.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0))
+ )?
);
assert_eq!(
empty_stat,
- *agg_partial
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?
+ *StatisticsContext::new().compute(
+ agg_partial.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(1))
+ )?
);
validate_statistics_with_data(
agg_partial.clone(),
@@ -1026,13 +1051,17 @@ mod test {
assert_eq!(
empty_stat,
- *agg_final
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?
+ *StatisticsContext::new().compute(
+ agg_final.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0))
+ )?
);
assert_eq!(
empty_stat,
- *agg_final
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?
+ *StatisticsContext::new().compute(
+ agg_final.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(1))
+ )?
);
validate_statistics_with_data(
@@ -1059,13 +1088,17 @@ mod test {
};
assert_eq!(
expect_partial_stat,
- *agg_partial
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?
+ *StatisticsContext::new().compute(
+ agg_partial.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0))
+ )?
);
assert_eq!(
expect_partial_stat,
- *agg_partial
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?
+ *StatisticsContext::new().compute(
+ agg_partial.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(1))
+ )?
);
let expect_partial_overall_stat = Statistics {
@@ -1075,7 +1108,8 @@ mod test {
};
assert_eq!(
expect_partial_overall_stat,
- *agg_partial.statistics_with_args(&StatisticsArgs::new())?
+ *StatisticsContext::new()
+ .compute(agg_partial.as_ref(), &StatisticsArgs::new())?
);
// Verify that the partial aggregate emits one accumulator-state row per
@@ -1110,8 +1144,10 @@ mod test {
assert_eq!(
expect_stat,
- *agg_final
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?
+ *StatisticsContext::new().compute(
+ agg_final.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0))
+ )?
);
// Verify that the aggregate final result has exactly one partition with one row
@@ -1140,8 +1176,10 @@ mod test {
let mut all_batches = vec![];
for (i, partition_stream) in partitions.into_iter().enumerate() {
let batches: Vec<RecordBatch> = partition_stream.try_collect().await?;
- let actual = plan
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(i)))?;
+ let actual = StatisticsContext::new().compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(i)),
+ )?;
let expected = compute_record_batch_statistics(
std::slice::from_ref(&batches),
&schema,
@@ -1151,7 +1189,8 @@ mod test {
all_batches.push(batches);
}
- let actual = plan.statistics_with_args(&StatisticsArgs::new())?;
+ let actual =
+ StatisticsContext::new().compute(plan.as_ref(), &StatisticsArgs::new())?;
let expected = compute_record_batch_statistics(&all_batches, &schema, None);
assert_eq!(*actual, expected);
@@ -1169,7 +1208,8 @@ mod test {
let statistics = (0..repartition.partitioning().partition_count())
.map(|idx| {
- repartition.statistics_with_args(
+ StatisticsContext::new().compute(
+ repartition.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -1223,14 +1263,16 @@ mod test {
Partitioning::RoundRobinBatch(2),
)?);
- let result = repartition
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(2)));
+ let result = StatisticsContext::new().compute(
+ repartition.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(2)),
+ );
assert!(result.is_err());
let error = result.unwrap_err();
assert!(
error
.to_string()
- .contains("RepartitionExec invalid partition 2 (expected less than 2)")
+ .contains("Invalid partition index: 2, the partition count is 2")
);
let partitions = execute_stream_partitioned(
@@ -1253,9 +1295,19 @@ mod test {
Partitioning::RoundRobinBatch(0),
)?);
- let result = repartition
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?;
- assert_eq!(*result, Statistics::new_unknown(&scan_schema));
+ // Requesting a specific partition of a zero-partition plan is out of
+ // range, so the context rejects it.
+ let result = StatisticsContext::new().compute(
+ repartition.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ );
+ assert!(result.is_err());
+ assert!(
+ result
+ .unwrap_err()
+ .to_string()
+ .contains("Invalid partition index: 0, the partition count is 0")
+ );
// Verify that the result has exactly 0 partitions
let partitions = execute_stream_partitioned(
@@ -1282,7 +1334,8 @@ mod test {
// Verify the result of partition statistics of repartition
let stats = (0..repartition.partitioning().partition_count())
.map(|idx| {
- repartition.statistics_with_args(
+ StatisticsContext::new().compute(
+ repartition.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -1344,7 +1397,8 @@ mod test {
// Verify partition statistics are properly propagated (not unknown)
let statistics = (0..window_agg.output_partitioning().partition_count())
.map(|idx| {
- window_agg.statistics_with_args(
+ StatisticsContext::new().compute(
+ window_agg.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -1433,8 +1487,10 @@ mod test {
// Try to test with single partition
let empty_single = Arc::new(EmptyExec::new(Arc::clone(&schema)));
- let stats = empty_single
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?;
+ let stats = StatisticsContext::new().compute(
+ empty_single.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )?;
assert_eq!(stats.num_rows, Precision::Exact(0));
assert_eq!(stats.total_byte_size, Precision::Exact(0));
assert_eq!(stats.column_statistics.len(), 2);
@@ -1449,7 +1505,8 @@ mod test {
assert_eq!(col_stat.byte_size, Precision::Exact(0));
}
- let overall_stats = empty_single.statistics_with_args(&StatisticsArgs::new())?;
+ let overall_stats = StatisticsContext::new()
+ .compute(empty_single.as_ref(), &StatisticsArgs::new())?;
assert_eq!(stats, overall_stats);
validate_statistics_with_data(empty_single, vec![ExpectedStatistics::Empty], 0)
@@ -1461,7 +1518,8 @@ mod test {
let statistics = (0..empty_multi.output_partitioning().partition_count())
.map(|idx| {
- empty_multi.statistics_with_args(
+ StatisticsContext::new().compute(
+ empty_multi.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -1525,7 +1583,8 @@ mod test {
// Test partition statistics for CollectLeft mode
let statistics = (0..collect_left_join.output_partitioning().partition_count())
.map(|idx| {
- collect_left_join.statistics_with_args(
+ StatisticsContext::new().compute(
+ collect_left_join.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -1605,7 +1664,8 @@ mod test {
// Test partition statistics for Partitioned mode
let statistics = (0..partitioned_join.output_partitioning().partition_count())
.map(|idx| {
- partitioned_join.statistics_with_args(
+ StatisticsContext::new().compute(
+ partitioned_join.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
@@ -1683,7 +1743,8 @@ mod test {
// Test partition statistics for Auto mode
let statistics = (0..auto_join.output_partitioning().partition_count())
.map(|idx| {
- auto_join.statistics_with_args(
+ StatisticsContext::new().compute(
+ auto_join.as_ref(),
&StatisticsArgs::new().with_partition(Some(idx)),
)
})
diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs
index d43a4a4cb9..915bd9a05f 100644
--- a/datafusion/core/tests/physical_optimizer/test_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs
@@ -1002,7 +1002,11 @@ impl ExecutionPlan for TestScan {
internal_err!("TestScan is for testing optimizer only, not for execution")
}
- fn statistics_with_args(&self, _args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
}
diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs
index de6349d129..82a15eb401 100644
--- a/datafusion/core/tests/sql/path_partition.rs
+++ b/datafusion/core/tests/sql/path_partition.rs
@@ -38,7 +38,7 @@ use datafusion_common::ScalarValue;
use datafusion_common::stats::Precision;
use datafusion_common::test_util::batches_to_sort_string;
use datafusion_execution::config::SessionConfig;
-use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use async_trait::async_trait;
use bytes::Bytes;
@@ -462,8 +462,8 @@ async fn parquet_statistics() -> Result<()> {
let schema = physical_plan.schema();
assert_eq!(schema.fields().len(), 4);
- let stat_cols = physical_plan
- .statistics_with_args(&StatisticsArgs::new())?
+ let stat_cols = StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?
.column_statistics
.clone();
assert_eq!(stat_cols.len(), 4);
@@ -489,8 +489,8 @@ async fn parquet_statistics() -> Result<()> {
let schema = physical_plan.schema();
assert_eq!(schema.fields().len(), 2);
- let stat_cols = physical_plan
- .statistics_with_args(&StatisticsArgs::new())?
+ let stat_cols = StatisticsContext::new()
+ .compute(physical_plan.as_ref(), &StatisticsArgs::new())?
.column_statistics
.clone();
assert_eq!(stat_cols.len(), 2);
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 1336ee69cd..91caabeee6 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -2394,7 +2394,7 @@ mod tests {
// of just the projected ones.
use crate::source::DataSourceExec;
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
// Create a schema with 4 columns
let schema = Arc::new(Schema::new(vec![
@@ -2448,8 +2448,11 @@ mod tests {
let exec = DataSourceExec::from_data_source(config);
// Get statistics for partition 0
- let partition_stats = exec
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))
+ let partition_stats = StatisticsContext::new()
+ .compute(
+ exec.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
.unwrap();
// Verify that only 2 columns are in the statistics (the projected ones)
diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rs
index a4e30d7f0b..255dd76cbd 100644
--- a/datafusion/datasource/src/memory.rs
+++ b/datafusion/datasource/src/memory.rs
@@ -853,7 +853,7 @@ mod tests {
use datafusion_common::stats::{ColumnStatistics, Precision};
use datafusion_physical_expr::PhysicalSortExpr;
use datafusion_physical_plan::expressions::lit;
- use datafusion_physical_plan::statistics::StatisticsArgs;
+ use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::ExecutionPlan;
@@ -986,7 +986,7 @@ mod tests {
let values = MemorySourceConfig::try_new_as_values(schema, data)?;
assert_eq!(
- *values.statistics_with_args(&StatisticsArgs::new())?,
+ *StatisticsContext::new().compute(values.as_ref(), &StatisticsArgs::new())?,
Statistics {
num_rows: Precision::Exact(rows),
total_byte_size: Precision::Exact(8), // not important
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index 9eb92e7e35..c280470bb0 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -437,7 +437,11 @@ impl ExecutionPlan for DataSourceExec {
Some(metrics)
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
self.data_source.partition_statistics(args.partition())
}
diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rs
index 738f87fd61..087a351b69 100644
--- a/datafusion/ffi/src/execution_plan.rs
+++ b/datafusion/ffi/src/execution_plan.rs
@@ -25,6 +25,7 @@ use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr_common::metrics::MetricsSet;
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, StatisticsArgs,
+ StatisticsContext,
};
use stabby::string::String as SString;
use stabby::vec::Vec as SVec;
@@ -209,8 +210,11 @@ unsafe extern "C" fn partition_statistics_fn_wrapper(
partition: FFI_Option<usize>,
) -> FFI_Result<SVec<u8>> {
let partition: Option<usize> = partition.into();
- plan.inner()
- .statistics_with_args(&StatisticsArgs::new().with_partition(partition))
+ StatisticsContext::new()
+ .compute(
+ plan.inner().as_ref(),
+ &StatisticsArgs::new().with_partition(partition),
+ )
.map(|stats| SVec::from(serialize_statistics(stats.as_ref()).as_slice()))
.into()
}
@@ -556,8 +560,9 @@ pub mod tests {
self.metrics.clone()
}
- fn statistics_with_args(
+ fn statistics_from_inputs(
&self,
+ _input_stats: &[Arc<Statistics>],
_args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
Ok(Arc::new(self.statistics.clone().unwrap_or_else(|| {
@@ -745,15 +750,17 @@ pub mod tests {
/// Same round trip as
/// [`test_ffi_execution_plan_partition_statistics_round_trip`], but queried
- /// through the **new** `statistics_with_args` entry point.
+ /// through the **new** `StatisticsContext::compute` entry point.
#[test]
- fn test_ffi_execution_plan_statistics_with_args_round_trip() -> Result<()> {
+ fn test_ffi_execution_plan_statistics_context_round_trip() -> Result<()> {
let (schema, original_stats) = stats_round_trip_fixture();
// A plan without explicit statistics reports new_unknown.
let bare = export_empty_exec_over_ffi(&schema, None)?;
assert_eq!(
- bare.statistics_with_args(&StatisticsArgs::new())?.as_ref(),
+ StatisticsContext::new()
+ .compute(bare.as_ref(), &StatisticsArgs::new())?
+ .as_ref(),
&Statistics::new_unknown(&schema)
);
@@ -761,14 +768,17 @@ pub mod tests {
let with_stats =
export_empty_exec_over_ffi(&schema, Some(original_stats.clone()))?;
assert_eq!(
- with_stats
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(with_stats.as_ref(), &StatisticsArgs::new())?
.as_ref(),
&original_stats
);
assert_eq!(
- with_stats
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(1)))?
+ StatisticsContext::new()
+ .compute(
+ with_stats.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(1)),
+ )?
.as_ref(),
&original_stats
);
diff --git a/datafusion/physical-optimizer/src/aggregate_statistics.rs b/datafusion/physical-optimizer/src/aggregate_statistics.rs
index b83f4ed730..43b1abb4b6 100644
--- a/datafusion/physical-optimizer/src/aggregate_statistics.rs
+++ b/datafusion/physical-optimizer/src/aggregate_statistics.rs
@@ -25,7 +25,7 @@ use datafusion_physical_plan::aggregates::{
};
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
-use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::udaf::{
AggregateFunctionExpr, StatisticsArgs as PlanStatisticsArgs,
};
@@ -58,9 +58,8 @@ impl PhysicalOptimizerRule for AggregateStatistics {
let partial_agg_exec = partial_agg_exec
.downcast_ref::<AggregateExec>()
.expect("take_optimizable() ensures that this is a AggregateExec");
- let stats = partial_agg_exec
- .input()
- .statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new()
+ .compute(partial_agg_exec.input().as_ref(), &StatisticsArgs::new())?;
let mut projections = vec![];
for expr in partial_agg_exec.aggr_expr() {
let field = expr.field();
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs
index 3cf79619cd..55769fce01 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs
@@ -62,7 +62,7 @@ use datafusion_physical_plan::joins::{
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
-use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::tree_node::PlanContext;
use datafusion_physical_plan::union::{InterleaveExec, UnionExec, can_interleave};
use datafusion_physical_plan::windows::WindowAggExec;
@@ -1003,8 +1003,8 @@ fn get_repartition_requirement_status(
{
// Decide whether adding a round robin is beneficial depending on
// the statistical information we have on the number of rows:
- let roundrobin_beneficial_stats = match child
- .statistics_with_args(&StatisticsArgs::new())?
+ let roundrobin_beneficial_stats = match StatisticsContext::new()
+ .compute(child.as_ref(), &StatisticsArgs::new())?
.num_rows
{
Precision::Exact(n_rows) => n_rows > batch_size,
diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rs
index 82294825b6..42736f8205 100644
--- a/datafusion/physical-optimizer/src/join_selection.rs
+++ b/datafusion/physical-optimizer/src/join_selection.rs
@@ -40,7 +40,7 @@ use datafusion_physical_plan::joins::{
StreamJoinPartitionMode, SymmetricHashJoinExec,
};
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
-use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
use std::sync::Arc;
@@ -66,7 +66,7 @@ fn get_stats(
reg.compute(plan)
.map(|s| Arc::<Statistics>::clone(s.base_arc()))
} else {
- plan.statistics_with_args(&StatisticsArgs::new())
+ StatisticsContext::new().compute(plan, &StatisticsArgs::new())
}
}
diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rs
index 224084d576..01a288f7f1 100644
--- a/datafusion/physical-optimizer/src/limit_pushdown.rs
+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs
@@ -76,7 +76,7 @@ use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
-use datafusion_physical_plan::statistics::StatisticsArgs;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
/// This rule inspects [`ExecutionPlan`]'s and pushes down the fetch limit from
/// the parent to the child if applicable.
@@ -352,8 +352,8 @@ fn limit_eliminable_exact_num_rows(
}
if matches!(
- current
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(current.as_ref(), &StatisticsArgs::new())?
.num_rows,
Precision::Exact(0)
) {
diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rs
index c6f5f87622..40ac5643de 100644
--- a/datafusion/physical-optimizer/src/output_requirements.rs
+++ b/datafusion/physical-optimizer/src/output_requirements.rs
@@ -32,7 +32,6 @@ use datafusion_common::{Result, Statistics, internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::Distribution;
use datafusion_physical_expr_common::sort_expr::OrderingRequirements;
-use datafusion_physical_plan::StatisticsArgs;
use datafusion_physical_plan::execution_plan::Boundedness;
use datafusion_physical_plan::projection::{
ProjectionExec, make_with_child, update_expr, update_ordering_requirement,
@@ -40,8 +39,8 @@ use datafusion_physical_plan::projection::{
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
use datafusion_physical_plan::{
- DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
- SendableRecordBatchStream,
+ ChildStats, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties,
+ PlanProperties, SendableRecordBatchStream, StatisticsArgs,
};
/// This rule either adds or removes [`OutputRequirements`]s to/from the physical
@@ -251,8 +250,16 @@ impl ExecutionPlan for OutputRequirementExec {
unreachable!();
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- args.compute_child_statistics(&self.input, args.partition())
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::clone(&input_stats[0]))
}
#[expect(
diff --git a/datafusion/physical-plan/benches/compute_statistics.rs b/datafusion/physical-plan/benches/compute_statistics.rs
index 04b5612563..56a518c952 100644
--- a/datafusion/physical-plan/benches/compute_statistics.rs
+++ b/datafusion/physical-plan/benches/compute_statistics.rs
@@ -47,6 +47,7 @@ use datafusion_physical_plan::joins::CrossJoinExec;
use datafusion_physical_plan::statistics::StatisticsArgs;
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType, Partitioning, SendableRecordBatchStream,
+ StatisticsContext,
};
/// Minimal leaf node for benchmarking
@@ -111,7 +112,11 @@ impl ExecutionPlan for BenchLeaf {
unimplemented!()
}
- fn statistics_with_args(&self, _args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
}
}
@@ -175,10 +180,10 @@ fn build_mixed_chain(groups: usize) -> Arc<dyn ExecutionPlan> {
}
/// Recursive walk without a shared cross-node cache, simulating pre-cache behavior.
-/// Each operator's internal `compute_child_statistics` call triggers a fresh
-/// subtree walk, resulting in O(n^2) total node visits for a chain of depth n.
+/// Each node is computed with a fresh `StatisticsContext`, so every call triggers a
+/// fresh subtree walk, resulting in O(n^2) total node visits for a chain of depth n.
///
-/// Note: each `compute_child_statistics` re-walk still benefits from its own
+/// Note: each `StatisticsContext::compute` re-walk still benefits from its own
/// ephemeral cache; only the cross-node sharing is removed.
fn compute_statistics_without_shared_cache(
plan: &dyn ExecutionPlan,
@@ -188,7 +193,7 @@ fn compute_statistics_without_shared_cache(
compute_statistics_without_shared_cache(child.as_ref(), None)?;
}
let args = StatisticsArgs::new().with_partition(partition);
- plan.statistics_with_args(&args)
+ StatisticsContext::new().compute(plan, &args)
}
fn bench_compute_statistics(c: &mut Criterion) {
@@ -198,7 +203,11 @@ fn bench_compute_statistics(c: &mut Criterion) {
for depth in [10, 20, 50] {
let plan = build_coalesce_chain(depth);
group.bench_with_input(BenchmarkId::new("cached", depth), &plan, |b, plan| {
- b.iter(|| plan.statistics_with_args(&StatisticsArgs::new()).unwrap());
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(plan.as_ref(), &StatisticsArgs::new())
+ .unwrap()
+ });
});
group.bench_with_input(
BenchmarkId::new("no_shared_cache", depth),
@@ -215,7 +224,7 @@ fn bench_compute_statistics(c: &mut Criterion) {
// --- Cross-join tree (balanced binary plan) ---
// Binary trees arise from multi-way joins (e.g. physical_many_self_joins
// in sql_planner.rs, see #19795). CrossJoinExec calls
- // compute_child_statistics for per-partition stats, re-walking the left
+ // StatisticsContext::compute for per-partition stats, re-walking the left
// subtree at each node. The gap between cached/uncached is smaller than
// the linear chain because only the left child triggers a re-walk.
let mut group = c.benchmark_group("compute_statistics_cross_join_tree");
@@ -225,7 +234,11 @@ fn bench_compute_statistics(c: &mut Criterion) {
let label = format!("depth={depth}_leaves={}", 1usize << depth);
group.bench_with_input(BenchmarkId::new("cached", &label), &plan, |b, plan| {
b.iter(|| {
- plan.statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))
+ StatisticsContext::new()
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
.unwrap()
});
});
@@ -254,10 +267,12 @@ fn bench_compute_statistics(c: &mut Criterion) {
&plan,
|b, plan| {
b.iter(|| {
- plan.statistics_with_args(
- &StatisticsArgs::new().with_partition(Some(0)),
- )
- .unwrap()
+ StatisticsContext::new()
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
+ .unwrap()
});
},
);
@@ -265,7 +280,11 @@ fn bench_compute_statistics(c: &mut Criterion) {
BenchmarkId::new("cached_overall", depth),
&plan,
|b, plan| {
- b.iter(|| plan.statistics_with_args(&StatisticsArgs::new()).unwrap());
+ b.iter(|| {
+ StatisticsContext::new()
+ .compute(plan.as_ref(), &StatisticsArgs::new())
+ .unwrap()
+ });
},
);
group.bench_with_input(
@@ -290,7 +309,11 @@ fn bench_compute_statistics(c: &mut Criterion) {
let depth = groups * 3; // 2 filters + 1 coalesce per group
group.bench_with_input(BenchmarkId::new("cached", depth), &plan, |b, plan| {
b.iter(|| {
- plan.statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))
+ StatisticsContext::new()
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
.unwrap()
});
});
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 48254edc6a..7ae481e96c 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -37,7 +37,7 @@ use crate::filter_pushdown::{
FilterPushdownPropagation, PushedDownPredicate,
};
use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements,
InputOrderMode, SendableRecordBatchStream, Statistics, check_if_same_properties,
@@ -1890,9 +1890,16 @@ impl ExecutionPlan for AggregateExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let child_statistics =
- args.compute_child_statistics(&self.input, args.partition())?;
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let child_statistics = Arc::clone(&input_stats[0]);
Ok(Arc::new(
self.statistics_inner(&child_statistics, args.partition())?,
))
@@ -2544,7 +2551,7 @@ mod tests {
use crate::execution_plan::Boundedness;
use crate::expressions::col;
use crate::metrics::MetricValue;
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test::TestMemoryExec;
use crate::test::assert_is_pending;
use crate::test::exec::{
@@ -2961,7 +2968,8 @@ mod tests {
)?);
// Verify statistics are preserved proportionally through aggregation
- let final_stats = merged_aggregate.statistics_with_args(&StatisticsArgs::new())?;
+ let final_stats = StatisticsContext::new()
+ .compute(merged_aggregate.as_ref(), &StatisticsArgs::new())?;
assert!(final_stats.total_byte_size.get_value().is_some());
let task_ctx = if spill {
@@ -3096,7 +3104,11 @@ mod tests {
Ok(Box::pin(stream))
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
return Ok(Arc::new(Statistics::new_unknown(self.schema().as_ref())));
}
@@ -5374,7 +5386,7 @@ mod tests {
PhysicalGroupBy::default(),
None,
)?;
- let stats = agg.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new().compute(&agg, &StatisticsArgs::new())?;
assert_eq!(stats.total_byte_size, Precision::Absent);
let zero_row_stats = Statistics {
@@ -5391,7 +5403,8 @@ mod tests {
PhysicalGroupBy::default(),
None,
)?;
- let stats_zero = agg_zero.statistics_with_args(&StatisticsArgs::new())?;
+ let stats_zero =
+ StatisticsContext::new().compute(&agg_zero, &StatisticsArgs::new())?;
assert_eq!(stats_zero.total_byte_size, Precision::Absent);
let single_input =
@@ -5412,7 +5425,7 @@ mod tests {
1
);
let single_stats_zero =
- single_agg_zero.statistics_with_args(&StatisticsArgs::new())?;
+ StatisticsContext::new().compute(&single_agg_zero, &StatisticsArgs::new())?;
assert_eq!(single_stats_zero.num_rows, Precision::Exact(1));
Ok(())
@@ -5785,7 +5798,7 @@ mod tests {
let agg =
build_test_aggregate(&schema, input_stats, group_by, case.limit_options)?;
- let stats = agg.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new().compute(&agg, &StatisticsArgs::new())?;
assert_eq!(
stats.num_rows, case.expected_num_rows,
"FAILED: '{}' — expected {:?}, got {:?}",
@@ -5824,7 +5837,7 @@ mod tests {
None,
)?;
- let stats = agg.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new().compute(&agg, &StatisticsArgs::new())?;
assert_eq!(
stats.column_statistics[0].distinct_count,
Precision::Exact(100),
@@ -5878,7 +5891,7 @@ mod tests {
let agg = build_test_aggregate(&schema, input_stats, grouping_set, None)?;
- let stats = agg.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new().compute(&agg, &StatisticsArgs::new())?;
// Per-set NDV: (a,NULL)=100, (NULL,b)=50, (a,b)=100*50=5000
// Total = 100 + 50 + 5000 = 5150
assert_eq!(
@@ -5908,8 +5921,8 @@ mod tests {
Arc::clone(&schema),
)?;
assert_eq!(
- single_agg
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(&single_agg, &StatisticsArgs::new())?
.num_rows,
Precision::Exact(2)
);
@@ -5936,9 +5949,10 @@ mod tests {
let task_ctx = Arc::new(TaskContext::default());
for partition in 0..2 {
assert_eq!(
- partial_agg
- .statistics_with_args(
- &StatisticsArgs::new().with_partition(Some(partition))
+ StatisticsContext::new()
+ .compute(
+ partial_agg.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(partition)),
)?
.num_rows,
Precision::Exact(2)
@@ -5949,8 +5963,8 @@ mod tests {
}
assert_eq!(
- partial_agg
- .statistics_with_args(&StatisticsArgs::new())?
+ StatisticsContext::new()
+ .compute(partial_agg.as_ref(), &StatisticsArgs::new())?
.num_rows,
Precision::Exact(4)
);
@@ -5995,7 +6009,7 @@ mod tests {
PhysicalGroupBy::new_single(vec![(expr_a_plus_b, "a+b".to_string())]);
let agg = build_test_aggregate(&schema, input_stats, group_by, None)?;
- let stats = agg.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new().compute(&agg, &StatisticsArgs::new())?;
assert_eq!(
stats.num_rows,
Precision::Inexact(1_000_000),
diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rs
index 4e88daae73..6f83e37196 100644
--- a/datafusion/physical-plan/src/buffer.rs
+++ b/datafusion/physical-plan/src/buffer.rs
@@ -24,7 +24,7 @@ use crate::filter_pushdown::{
FilterPushdownPropagation,
};
use crate::projection::ProjectionExec;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::RecordBatchStreamAdapter;
use crate::{
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SortOrderPushdownResult,
@@ -238,8 +238,16 @@ impl ExecutionPlan for BufferExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- args.compute_child_statistics(&self.input, args.partition())
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::clone(&input_stats[0]))
}
fn supports_limit_pushdown(&self) -> bool {
diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs
index fc0fae6cc3..667a8a3ce0 100644
--- a/datafusion/physical-plan/src/coalesce_batches.rs
+++ b/datafusion/physical-plan/src/coalesce_batches.rs
@@ -24,7 +24,7 @@ use std::task::{Context, Poll};
use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, Statistics};
use crate::projection::ProjectionExec;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::EmptyRecordBatchStream;
use crate::{
DisplayFormatType, ExecutionPlan, RecordBatchStream, SendableRecordBatchStream,
@@ -216,10 +216,16 @@ impl ExecutionPlan for CoalesceBatchesExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let stats = input_stats[0].as_ref().clone();
Ok(Arc::new(stats.with_fetch(self.fetch, 0, 1)?))
}
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index a858b1cd1b..1c6f95c53b 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -30,7 +30,7 @@ use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};
use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};
use crate::projection::{ProjectionExec, make_with_child};
use crate::sort_pushdown::SortOrderPushdownResult;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{DisplayFormatType, ExecutionPlan, Partitioning, check_if_same_properties};
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
@@ -232,9 +232,16 @@ impl ExecutionPlan for CoalescePartitionsExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let stats =
- Arc::unwrap_or_clone(args.compute_child_statistics(&self.input, None)?);
+ fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(None)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let stats = input_stats[0].as_ref().clone();
Ok(Arc::new(stats.with_fetch(self.fetch, 0, 1)?))
}
diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs
index 46141b7e7a..94e4fdca2b 100644
--- a/datafusion/physical-plan/src/coop.rs
+++ b/datafusion/physical-plan/src/coop.rs
@@ -84,7 +84,7 @@ use crate::filter_pushdown::{
FilterPushdownPropagation,
};
use crate::projection::ProjectionExec;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream,
SendableRecordBatchStream, SortOrderPushdownResult, check_if_same_properties,
@@ -299,8 +299,16 @@ impl ExecutionPlan for CooperativeExec {
Ok(make_cooperative(child_stream))
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- args.compute_child_statistics(&self.input, args.partition())
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::clone(&input_stats[0]))
}
fn supports_limit_pushdown(&self) -> bool {
diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs
index 2c1d30eaab..6a4d09057b 100644
--- a/datafusion/physical-plan/src/display.rs
+++ b/datafusion/physical-plan/src/display.rs
@@ -32,7 +32,7 @@ use datafusion_physical_expr::LexOrdering;
use crate::metrics::{MetricCategory, MetricType, MetricValue};
use crate::render_tree::RenderTree;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{StatisticsArgs, StatisticsContext};
use super::{ExecutionPlan, ExecutionPlanVisitor, accept};
@@ -581,8 +581,8 @@ impl ExecutionPlanVisitor for IndentVisitor<'_, '_> {
}
}
if self.show_statistics {
- let stats = plan
- .statistics_with_args(&StatisticsArgs::default())
+ let stats = StatisticsContext::new()
+ .compute(plan, &StatisticsArgs::new())
.map_err(|_e| fmt::Error)?;
write!(self.f, ", statistics=[{stats}]")?;
}
@@ -679,8 +679,8 @@ impl ExecutionPlanVisitor for GraphvizVisitor<'_, '_> {
};
let statistics = if self.show_statistics {
- let stats = plan
- .statistics_with_args(&StatisticsArgs::new())
+ let stats = StatisticsContext::new()
+ .compute(plan, &StatisticsArgs::new())
.map_err(|_e| fmt::Error)?;
format!("statistics=[{stats}]")
} else {
@@ -1504,7 +1504,11 @@ mod tests {
todo!()
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
return Ok(Arc::new(Statistics::new_unknown(self.schema().as_ref())));
}
diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs
index a8f4af5b3d..44a6f444dc 100644
--- a/datafusion/physical-plan/src/empty.rs
+++ b/datafusion/physical-plan/src/empty.rs
@@ -152,7 +152,11 @@ impl ExecutionPlan for EmptyExec {
)?))
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if let Some(partition) = args.partition() {
assert_or_internal_err!(
partition < self.partitions,
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index 5f92ff7659..e58acee4ce 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -48,7 +48,7 @@ use crate::metrics::MetricsSet;
use crate::projection::ProjectionExec;
use crate::repartition::RepartitionExec;
use crate::sorts::sort_preserving_merge::SortPreservingMergeExec;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::RecordBatchStreamAdapter;
use arrow::array::{Array, RecordBatch};
@@ -538,10 +538,10 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
/// Returns statistics for a specific partition of this `ExecutionPlan` node.
///
- /// Deprecated: use [`Self::statistics_with_args`] instead,
- /// which accepts a [`StatisticsArgs`] carrying pre-computed child
- /// statistics.
- #[deprecated(since = "55.0.0", note = "Use statistics_with_args instead")]
+ /// Deprecated: use [`StatisticsContext::compute`] instead.
+ ///
+ /// [`StatisticsContext::compute`]: crate::statistics::StatisticsContext::compute
+ #[deprecated(since = "55.0.0", note = "Use StatisticsContext::compute instead")]
fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
if let Some(idx) = partition {
// Validate partition index
@@ -556,21 +556,47 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
Ok(Arc::new(Statistics::new_unknown(&self.schema())))
}
- /// Returns statistics for a specific partition of this `ExecutionPlan` node.
+ /// Returns statistics for a specific partition of this `ExecutionPlan` node,
+ /// given pre-computed child statistics.
+ ///
/// If statistics are not available, should return [`Statistics::new_unknown`]
/// (the default), not an error.
- /// If `partition` is `None`, it returns statistics for all partitions.
+ /// If `args.partition()` is `None`, it returns statistics for all partitions.
///
- /// [`StatisticsArgs`] carries the partition index and a shared cache.
- /// Create one with [`StatisticsArgs::new`] and pass it to this method.
+ /// Implementations should not call [`StatisticsContext::compute`] from within
+ /// this method; child statistics are provided via `input_stats`.
///
- /// [`StatisticsArgs`]: crate::statistics::StatisticsArgs
- /// [`StatisticsArgs::new`]: crate::statistics::StatisticsArgs::new
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ /// Use [`StatisticsContext::compute`] to initiate a full plan-tree walk.
+ ///
+ /// [`StatisticsContext::compute`]: crate::statistics::StatisticsContext::compute
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
#[expect(deprecated)]
self.partition_statistics(args.partition())
}
+ /// Returns, per child, which statistics the [`StatisticsContext`] should resolve
+ /// before calling [`Self::statistics_from_inputs`].
+ ///
+ /// One entry per child (same order as [`Self::children`]): [`ChildStats::At`]
+ /// requests the child's statistics at a partition (`None` = overall);
+ /// [`ChildStats::Skip`] omits a child whose statistics this node does not need
+ /// (a `Statistics::new_unknown` placeholder fills its `input_stats` slot).
+ ///
+ /// The default skips every child, so a node that derives nothing from its
+ /// children (for example one that only overrides the deprecated
+ /// [`Self::partition_statistics`]) triggers no child traversal. A node that reads
+ /// `input_stats` in [`Self::statistics_from_inputs`] must override this to declare
+ /// the children it uses.
+ ///
+ /// [`StatisticsContext`]: crate::statistics::StatisticsContext
+ fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {
+ self.children().iter().map(|_| ChildStats::Skip).collect()
+ }
+
/// Returns `true` if a limit can be safely pushed down through this
/// `ExecutionPlan` node.
///
@@ -1727,8 +1753,9 @@ mod tests {
unimplemented!()
}
- fn statistics_with_args(
+ fn statistics_from_inputs(
&self,
+ _input_stats: &[Arc<Statistics>],
_args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
unimplemented!()
@@ -1789,8 +1816,9 @@ mod tests {
unimplemented!()
}
- fn statistics_with_args(
+ fn statistics_from_inputs(
&self,
+ _input_stats: &[Arc<Statistics>],
_args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
unimplemented!()
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 9c09ff6f4f..ad40a3fb5f 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -42,7 +42,7 @@ use crate::projection::{
EmbeddedProjection, ProjectionExec, ProjectionExpr, make_with_child,
try_embed_projection, update_expr,
};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs, StatisticsContext};
use crate::stream::EmptyRecordBatchStream;
use crate::{
DisplayFormatType, ExecutionPlan,
@@ -422,7 +422,10 @@ impl FilterExec {
let schema = input.schema();
let stats = Self::statistics_helper(
&schema,
- Arc::unwrap_or_clone(input.statistics_with_args(&StatisticsArgs::new())?),
+ Arc::unwrap_or_clone(
+ StatisticsContext::new()
+ .compute(input.as_ref(), &StatisticsArgs::new())?,
+ ),
predicate,
default_selectivity,
)?;
@@ -589,12 +592,18 @@ impl ExecutionPlan for FilterExec {
Some(self.metrics.clone_inner())
}
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
/// The output statistics of a filtering operation can be estimated if the
/// predicate's selectivity value can be determined for the incoming data.
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let input_stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let input_stats = input_stats[0].as_ref().clone();
let stats = Self::statistics_helper(
&self.input.schema(),
input_stats,
@@ -1268,7 +1277,7 @@ mod tests {
use super::*;
use crate::empty::EmptyExec;
use crate::expressions::*;
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test;
use crate::test::exec::StatisticsExec;
use arrow::datatypes::{Field, Schema, UnionFields, UnionMode};
@@ -1345,7 +1354,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(25));
assert_eq!(
statistics.total_byte_size,
@@ -1397,7 +1407,8 @@ mod tests {
sub_filter,
)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(16));
assert_eq!(
statistics.column_statistics,
@@ -1459,7 +1470,8 @@ mod tests {
binary(col("a", &schema)?, Operator::GtEq, lit(10i32), &schema)?,
b_gt_5,
)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
// On a uniform distribution, only fifteen rows will satisfy the
// filter that 'a' proposed (a >= 10 AND a <= 25) (15/100) and only
// 5 rows will satisfy the filter that 'b' proposed (b > 45) (5/50).
@@ -1509,7 +1521,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Absent);
Ok(())
@@ -1582,7 +1595,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
// 0.5 (from a) * 0.333333... (from b) * 0.798387... (from c) ≈ 0.1330...
// num_rows after ceil => 133.0... => 134
// total_byte_size after ceil => 532.0... => 533
@@ -1680,8 +1694,8 @@ mod tests {
// The filter predicate passes all (non-null) entries, so min/max/NDV
// are unchanged. `a < 200` and `1 <= b` are null-rejecting, though, so
// both columns lose any nulls regardless of selectivity.
- let mut expected = input
- .statistics_with_args(&StatisticsArgs::new())?
+ let mut expected = StatisticsContext::new()
+ .compute(input.as_ref(), &StatisticsArgs::new())?
.column_statistics
.clone();
for col in &mut expected {
@@ -1689,7 +1703,8 @@ mod tests {
}
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(1000));
assert_eq!(statistics.total_byte_size, Precision::Inexact(4000));
@@ -1742,7 +1757,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(0));
assert_eq!(statistics.total_byte_size, Precision::Inexact(0));
@@ -1829,7 +1845,8 @@ mod tests {
Arc::new(FilterExec::try_new(outer_predicate, inner_filter)?);
// Should succeed without error
- let statistics = outer_filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics = StatisticsContext::new()
+ .compute(outer_filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(0));
Ok(())
@@ -1868,7 +1885,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(490));
assert_eq!(statistics.total_byte_size, Precision::Inexact(1960));
@@ -1922,7 +1940,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let filter_statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let filter_statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
let expected_filter_statistics = Statistics {
num_rows: Precision::Absent,
@@ -1959,7 +1978,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let filter_statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let filter_statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
// First column is "a", and it is a column with only one value after the filter.
assert!(filter_statistics.column_statistics[0].is_singleton());
@@ -2006,11 +2026,13 @@ mod tests {
Arc::new(Literal::new(ScalarValue::Decimal128(Some(10), 10, 10))),
));
let filter = FilterExec::try_new(predicate, input)?;
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(&filter, &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(200));
assert_eq!(statistics.total_byte_size, Precision::Inexact(800));
let filter = filter.with_default_selectivity(40)?;
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(&filter, &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(400));
assert_eq!(statistics.total_byte_size, Precision::Inexact(1600));
Ok(())
@@ -2045,7 +2067,9 @@ mod tests {
Arc::new(EmptyExec::new(Arc::clone(&schema))),
)?;
- exec.statistics_with_args(&StatisticsArgs::new()).unwrap();
+ StatisticsContext::new()
+ .compute(&exec, &StatisticsArgs::new())
+ .unwrap();
Ok(())
}
@@ -2201,8 +2225,10 @@ mod tests {
assert_eq!(filter1.projection(), filter2.projection());
// Verify statistics are the same
- let stats1 = filter1.statistics_with_args(&StatisticsArgs::new())?;
- let stats2 = filter2.statistics_with_args(&StatisticsArgs::new())?;
+ let stats1 =
+ StatisticsContext::new().compute(&filter1, &StatisticsArgs::new())?;
+ let stats2 =
+ StatisticsContext::new().compute(&filter2, &StatisticsArgs::new())?;
assert_eq!(stats1.num_rows, stats2.num_rows);
assert_eq!(stats1.total_byte_size, stats2.total_byte_size);
@@ -2255,7 +2281,8 @@ mod tests {
.unwrap()
.build()?;
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(&filter, &StatisticsArgs::new())?;
// Verify statistics reflect both filtering and projection
assert!(matches!(statistics.num_rows, Precision::Inexact(_)));
@@ -2486,7 +2513,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
let col_b_stats = &statistics.column_statistics[1];
assert_eq!(col_b_stats.min_value, Precision::Absent);
assert_eq!(col_b_stats.max_value, Precision::Absent);
@@ -2773,7 +2801,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics = StatisticsContext::new()
+ .compute(filter.as_ref(), &StatisticsArgs::new())?;
for (i, expected) in expected_ndvs.iter().enumerate() {
assert_eq!(
@@ -2834,7 +2863,8 @@ mod tests {
let input = Arc::new(StatisticsExec::new(input_stats, schema));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics = StatisticsContext::new()
+ .compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.num_rows,
@@ -2911,7 +2941,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
// Equality predicates collapse NDV and reject nulls for their columns.
assert_eq!(
statistics.column_statistics[0].distinct_count,
@@ -2964,7 +2995,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
@@ -2997,7 +3029,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
@@ -3030,7 +3063,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
@@ -3063,7 +3097,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
@@ -3097,7 +3132,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
@@ -3143,7 +3179,8 @@ mod tests {
));
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
@@ -3445,7 +3482,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
// Filter estimates ~10 rows (selectivity = 10/100)
assert_eq!(statistics.num_rows, Precision::Inexact(10));
let ndv = &statistics.column_statistics[0].distinct_count;
@@ -3491,7 +3529,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(20));
assert_eq!(
statistics.column_statistics[0].null_count,
@@ -3534,7 +3573,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(20));
assert_eq!(
statistics.column_statistics[0].null_count,
@@ -3575,7 +3615,8 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics = filter.statistics_with_args(&StatisticsArgs::new())?;
+ let statistics =
+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
assert_eq!(statistics.num_rows, Precision::Inexact(20));
assert_eq!(
statistics.column_statistics[0].null_count,
diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs
index d7ff07d00f..99c6800659 100644
--- a/datafusion/physical-plan/src/joins/cross_join.rs
+++ b/datafusion/physical-plan/src/joins/cross_join.rs
@@ -31,7 +31,7 @@ use crate::projection::{
ProjectionExec, join_allows_pushdown, join_table_borders, new_join_children,
physical_to_column_exprs,
};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::EmptyRecordBatchStream;
use crate::{
ColumnStatistics, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,
@@ -376,14 +376,19 @@ impl ExecutionPlan for CrossJoinExec {
}
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- // Left side is always broadcast, so it always needs overall stats
- let left_stats =
- Arc::unwrap_or_clone(args.compute_child_statistics(&self.left, None)?);
- // Right side is partitioned, so it needs per-partition stats
- let right_stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.right, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ // Left side is always broadcast, so it always needs overall stats.
+ // Right side is partitioned, so it needs per-partition stats.
+ vec![ChildStats::At(None), ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let left_stats = input_stats[0].as_ref().clone();
+ let right_stats = input_stats[1].as_ref().clone();
Ok(Arc::new(stats_cartesian_product(left_stats, right_stats)))
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 56e7132dc4..c5a64da1ea 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -52,7 +52,7 @@ use crate::projection::{
try_pushdown_through_join,
};
use crate::repartition::REPARTITION_RANDOM_STATE;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,
InputDistributionRequirements, Partitioning, PlanProperties,
@@ -1509,72 +1509,43 @@ impl ExecutionPlan for HashJoinExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let stats = match (args.partition(), self.mode) {
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ match (partition, self.mode) {
// Left side is broadcast, so it always needs overall stats
// Right side is partitioned, so it needs per-partition stats
(Some(_), PartitionMode::CollectLeft) => {
- let left_stats = args.compute_child_statistics(&self.left, None)?;
- let right_stats =
- args.compute_child_statistics(&self.right, args.partition())?;
-
- estimate_join_statistics(
- Arc::unwrap_or_clone(left_stats),
- Arc::unwrap_or_clone(right_stats),
- &self.on,
- self.null_equality,
- &self.join_type,
- &self.join_schema,
- )?
+ vec![ChildStats::At(None), ChildStats::At(partition)]
}
-
// For Partitioned mode, both sides are hash-partitioned symmetrically,
// so each output partition uses the matching partition from both sides.
(Some(_), PartitionMode::Partitioned) => {
- let left_stats =
- args.compute_child_statistics(&self.left, args.partition())?;
- let right_stats =
- args.compute_child_statistics(&self.right, args.partition())?;
-
- estimate_join_statistics(
- Arc::unwrap_or_clone(left_stats),
- Arc::unwrap_or_clone(right_stats),
- &self.on,
- self.null_equality,
- &self.join_type,
- &self.join_schema,
- )?
+ vec![ChildStats::At(partition), ChildStats::At(partition)]
}
-
// Overall stats requested, look up overall child stats.
- (None, _) => {
- let left_stats = args.compute_child_statistics(&self.left, None)?;
- let right_stats = args.compute_child_statistics(&self.right, None)?;
- estimate_join_statistics(
- Arc::unwrap_or_clone(left_stats),
- Arc::unwrap_or_clone(right_stats),
- &self.on,
- self.null_equality,
- &self.join_type,
- &self.join_schema,
- )?
- }
-
+ (None, _) => vec![ChildStats::At(None), ChildStats::At(None)],
// Auto mode hasn't decided partitioning yet, so it needs
// overall stats from both sides.
(Some(_), PartitionMode::Auto) => {
- let left_stats = args.compute_child_statistics(&self.left, None)?;
- let right_stats = args.compute_child_statistics(&self.right, None)?;
- estimate_join_statistics(
- Arc::unwrap_or_clone(left_stats),
- Arc::unwrap_or_clone(right_stats),
- &self.on,
- self.null_equality,
- &self.join_type,
- &self.join_schema,
- )?
+ vec![ChildStats::At(None), ChildStats::At(None)]
}
- };
+ }
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let left_stats = Arc::clone(&input_stats[0]);
+ let right_stats = Arc::clone(&input_stats[1]);
+ let stats = estimate_join_statistics(
+ Arc::unwrap_or_clone(left_stats),
+ Arc::unwrap_or_clone(right_stats),
+ &self.on,
+ self.null_equality,
+ &self.join_type,
+ &self.join_schema,
+ )?;
// Project statistics if there is a projection
let stats = stats.project(self.projection.as_ref());
// Apply fetch limit to statistics
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index 2d1a3ae62d..d32dd69923 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -42,7 +42,7 @@ use crate::projection::{
EmbeddedProjection, JoinData, ProjectionExec, try_embed_projection,
try_pushdown_through_join,
};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
PlanProperties, RecordBatchStream, SendableRecordBatchStream,
@@ -694,7 +694,17 @@ impl ExecutionPlan for NestedLoopJoinExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ // Left side is always broadcast, so it always needs overall stats.
+ // Right side is partitioned, so it needs per-partition stats.
+ vec![ChildStats::At(None), ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
// NestedLoopJoinExec is designed for joins without equijoin keys in the
// ON clause (e.g., `t1 JOIN t2 ON (t1.v1 + t2.v1) % 2 = 0`). Any join
// predicates are stored in `self.filter`, but `estimate_join_statistics`
@@ -704,13 +714,8 @@ impl ExecutionPlan for NestedLoopJoinExec {
// unknown row counts.
let join_columns = Vec::new();
- // Left side is always broadcast, so it always needs overall stats
- let left_stats =
- Arc::unwrap_or_clone(args.compute_child_statistics(&self.left, None)?);
- // Right side is partitioned, so it needs per-partition stats
- let right_stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.right, args.partition())?,
- );
+ let left_stats = input_stats[0].as_ref().clone();
+ let right_stats = input_stats[1].as_ref().clone();
let stats = estimate_join_statistics(
left_stats,
@@ -3066,7 +3071,7 @@ fn build_unmatched_batch(
#[cfg(test)]
pub(crate) mod tests {
use super::*;
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test::{TestMemoryExec, assert_join_metrics};
use crate::{
common, expressions::Column, repartition::RepartitionExec, test::build_table_i32,
@@ -3446,7 +3451,8 @@ pub(crate) mod tests {
&JoinType::Left,
Some(vec![1, 2]),
)?;
- let stats = nested_loop_join.statistics_with_args(&StatisticsArgs::new())?;
+ let stats = StatisticsContext::new()
+ .compute(&nested_loop_join, &StatisticsArgs::new())?;
assert_eq!(
nested_loop_join.schema().fields().len(),
stats.column_statistics.len(),
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index 82d9c900e8..46c37696ec 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -38,7 +38,7 @@ use crate::projection::{
physical_to_column_exprs, update_join_on,
};
use crate::spill::spill_manager::SpillManager;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
PlanProperties, SendableRecordBatchStream, Statistics, check_if_same_properties,
@@ -568,7 +568,15 @@ impl ExecutionPlan for SortMergeJoinExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition), ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
// SortMergeJoinExec uses symmetric hash partitioning where both left and right
// inputs are hash-partitioned on the join keys. This means partition `i` of the
// left input is joined with partition `i` of the right input.
@@ -576,12 +584,8 @@ impl ExecutionPlan for SortMergeJoinExec {
// TODO stats: it is not possible in general to know the output size of joins
// There are some special cases though, for example:
// - `A LEFT JOIN B ON A.col=B.col` with `COUNT_DISTINCT(B.col)=COUNT(B.col)`
- let left_stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.left, args.partition())?,
- );
- let right_stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.right, args.partition())?,
- );
+ let left_stats = input_stats[0].as_ref().clone();
+ let right_stats = input_stats[1].as_ref().clone();
Ok(Arc::new(estimate_join_statistics(
left_stats,
right_stats,
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index ccd9c155f1..64dadcb123 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -3388,7 +3388,7 @@ async fn test_left_outer_join_filtered_mask() -> Result<()> {
#[test]
fn test_partition_statistics() -> Result<()> {
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_common::stats::Precision;
let left = build_table(
@@ -3425,7 +3425,8 @@ fn test_partition_statistics() -> Result<()> {
// Test aggregate statistics (partition = None)
// Should return meaningful statistics computed from both inputs
- let stats = join_exec.statistics_with_args(&StatisticsArgs::new())?;
+ let stats =
+ StatisticsContext::new().compute(&join_exec, &StatisticsArgs::new())?;
assert_eq!(
stats.column_statistics.len(),
expected_cols,
@@ -3443,8 +3444,8 @@ fn test_partition_statistics() -> Result<()> {
// Since the child TestMemoryExec returns unknown stats for specific partitions,
// the join output will also have Absent num_rows. This is expected behavior
// as the statistics depend on what the children can provide.
- let partition_stats = join_exec
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?;
+ let partition_stats = StatisticsContext::new()
+ .compute(&join_exec, &StatisticsArgs::new().with_partition(Some(0)))?;
assert_eq!(
partition_stats.column_statistics.len(),
expected_cols,
diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs
index 8f40dde22a..0ab232bc10 100644
--- a/datafusion/physical-plan/src/lib.rs
+++ b/datafusion/physical-plan/src/lib.rs
@@ -52,7 +52,7 @@ pub use crate::execution_plan::{
pub use crate::metrics::Metric;
pub use crate::ordering::InputOrderMode;
pub use crate::sort_pushdown::SortOrderPushdownResult;
-pub use crate::statistics::StatisticsArgs;
+pub use crate::statistics::{ChildStats, StatisticsArgs, StatisticsContext};
pub use crate::stream::EmptyRecordBatchStream;
pub use crate::topk::TopK;
pub use crate::visitor::{ExecutionPlanVisitor, accept, visit_execution_plan};
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index 3327098040..2f63ac05e8 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -27,7 +27,7 @@ use super::{
SendableRecordBatchStream, Statistics,
};
use crate::execution_plan::{Boundedness, CardinalityEffect};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayFormatType, Distribution, ExecutionPlan, Partitioning,
check_if_same_properties,
@@ -224,10 +224,16 @@ impl ExecutionPlan for GlobalLimitExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let stats = input_stats[0].as_ref().clone();
Ok(Arc::new(stats.with_fetch(self.fetch, self.skip, 1)?))
}
@@ -389,10 +395,16 @@ impl ExecutionPlan for LocalLimitExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let stats = input_stats[0].as_ref().clone();
Ok(Arc::new(stats.with_fetch(Some(self.fetch), 0, 1)?))
}
@@ -539,7 +551,7 @@ mod tests {
use super::*;
use crate::coalesce_partitions::CoalescePartitionsExec;
use crate::common::collect;
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test;
use crate::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};
@@ -822,8 +834,8 @@ mod tests {
let offset =
GlobalLimitExec::new(Arc::new(CoalescePartitionsExec::new(csv)), skip, fetch);
- Ok(offset
- .statistics_with_args(&StatisticsArgs::new())?
+ Ok(StatisticsContext::new()
+ .compute(&offset, &StatisticsArgs::new())?
.num_rows)
}
@@ -864,8 +876,8 @@ mod tests {
fetch,
);
- Ok(offset
- .statistics_with_args(&StatisticsArgs::new())?
+ Ok(StatisticsContext::new()
+ .compute(&offset, &StatisticsArgs::new())?
.num_rows)
}
@@ -879,8 +891,8 @@ mod tests {
let offset = LocalLimitExec::new(csv, fetch);
- Ok(offset
- .statistics_with_args(&StatisticsArgs::new())?
+ Ok(StatisticsContext::new()
+ .compute(&offset, &StatisticsArgs::new())?
.num_rows)
}
diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rs
index 990bb4a682..142768fcf4 100644
--- a/datafusion/physical-plan/src/operator_statistics/mod.rs
+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs
@@ -94,7 +94,7 @@ use datafusion_common::stats::Precision;
use datafusion_common::{Result, Statistics};
use crate::ExecutionPlan;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{StatisticsArgs, StatisticsContext};
// ============================================================================
// ExtendedStatistics: Statistics with type-safe extensions
@@ -267,7 +267,7 @@ impl StatisticsProvider for DefaultStatisticsProvider {
plan: &dyn ExecutionPlan,
_child_stats: &[ExtendedStatistics],
) -> Result<StatisticsResult> {
- let base = plan.statistics_with_args(&StatisticsArgs::new())?;
+ let base = StatisticsContext::new().compute(plan, &StatisticsArgs::new())?;
Ok(StatisticsResult::Computed(ExtendedStatistics::new_arc(
base,
)))
@@ -359,7 +359,7 @@ impl StatisticsRegistry {
pub fn compute(&self, plan: &dyn ExecutionPlan) -> Result<ExtendedStatistics> {
// Fast path: no providers registered, skip the walk entirely
if self.providers.is_empty() {
- let base = plan.statistics_with_args(&StatisticsArgs::new())?;
+ let base = StatisticsContext::new().compute(plan, &StatisticsArgs::new())?;
return Ok(ExtendedStatistics::new_arc(base));
}
@@ -383,7 +383,7 @@ impl StatisticsRegistry {
}
}
// Fallback: use plan's built-in stats
- let base = plan.statistics_with_args(&StatisticsArgs::new())?;
+ let base = StatisticsContext::new().compute(plan, &StatisticsArgs::new())?;
Ok(ExtendedStatistics::new_arc(base))
}
@@ -506,8 +506,9 @@ fn computed_with_row_count(
plan: &dyn ExecutionPlan,
num_rows: Precision<usize>,
) -> Result<StatisticsResult> {
- let mut base =
- Arc::unwrap_or_clone(plan.statistics_with_args(&StatisticsArgs::new())?);
+ let mut base = Arc::unwrap_or_clone(
+ StatisticsContext::new().compute(plan, &StatisticsArgs::new())?,
+ );
rescale_byte_size(&mut base, num_rows);
Ok(StatisticsResult::Computed(ExtendedStatistics::new(base)))
}
@@ -1124,8 +1125,9 @@ mod tests {
unimplemented!()
}
- fn statistics_with_args(
+ fn statistics_from_inputs(
&self,
+ _input_stats: &[Arc<Statistics>],
_args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
Ok(Arc::new(self.stats.clone()))
diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs
index 64b192d58d..20d267331b 100644
--- a/datafusion/physical-plan/src/placeholder_row.rs
+++ b/datafusion/physical-plan/src/placeholder_row.rs
@@ -165,7 +165,11 @@ impl ExecutionPlan for PlaceholderRowExec {
Ok(Box::pin(cooperative(ms)))
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
let batches = self
.data()
.expect("Create single row placeholder RecordBatch should not fail");
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index 18f9e8d938..6ea02fa4fd 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -33,7 +33,7 @@ use crate::filter_pushdown::{
FilterPushdownPropagation, FilterRemapper, PushedDownPredicate,
};
use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn, JoinOnRef};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{DisplayFormatType, ExecutionPlan, PhysicalExpr, check_if_same_properties};
use std::collections::HashMap;
use std::pin::Pin;
@@ -349,10 +349,16 @@ impl ExecutionPlan for ProjectionExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let input_stats = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let input_stats = input_stats[0].as_ref().clone();
let output_schema = self.schema();
Ok(Arc::new(
self.projector
@@ -1186,7 +1192,7 @@ mod tests {
use crate::common::collect;
use crate::filter_pushdown::PushedDown;
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test;
use crate::test::exec::StatisticsExec;
@@ -1377,8 +1383,8 @@ mod tests {
let projection = ProjectionExec::try_new(exprs, input).unwrap();
- let stats = projection
- .statistics_with_args(&StatisticsArgs::new())
+ let stats = StatisticsContext::new()
+ .compute(&projection, &StatisticsArgs::new())
.unwrap();
assert_eq!(stats.num_rows, Precision::Exact(10));
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index a07d110e66..6587157421 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -40,7 +40,7 @@ use crate::projection::{ProjectionExec, all_columns, make_with_child, update_exp
use crate::sorts::streaming_merge::StreamingMergeBuilder;
use crate::spill::spill_manager::SpillManager;
use crate::spill::spill_pool::{self, SpillPoolWriter};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::{EmptyRecordBatchStream, RecordBatchStreamAdapter};
use crate::{
DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, Statistics,
@@ -1479,22 +1479,27 @@ impl ExecutionPlan for RepartitionExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- if let Some(partition) = args.partition() {
- let partition_count = self.partitioning().partition_count();
- if partition_count == 0 {
- return Ok(Arc::new(Statistics::new_unknown(&self.schema())));
- }
+ fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(None)]
+ }
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ if args.partition().is_some() {
+ let partition_count = self.partitioning().partition_count();
+ // `StatisticsContext::compute` validates the partition index against
+ // this same count before calling, so it is non-zero here; guard
+ // defensively against a direct call so the division below cannot
+ // divide by zero
assert_or_internal_err!(
- partition < partition_count,
- "RepartitionExec invalid partition {} (expected less than {})",
- partition,
- partition_count
+ partition_count > 0,
+ "RepartitionExec statistics requested for a partition but the partition count is 0"
);
- let mut stats =
- Arc::unwrap_or_clone(args.compute_child_statistics(&self.input, None)?);
+ let mut stats = input_stats[0].as_ref().clone();
// Distribute statistics across partitions
stats.num_rows = stats
@@ -1517,7 +1522,7 @@ impl ExecutionPlan for RepartitionExec {
Ok(Arc::new(stats))
} else {
- args.compute_child_statistics(&self.input, None)
+ Ok(Arc::clone(&input_stats[0]))
}
}
diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rs
index dd44d09c38..74de1f11bf 100644
--- a/datafusion/physical-plan/src/scalar_subquery.rs
+++ b/datafusion/physical-plan/src/scalar_subquery.rs
@@ -33,7 +33,7 @@ use datafusion_expr::execution_props::{ScalarSubqueryResults, SubqueryIndex};
use crate::execution_plan::{CardinalityEffect, ExecutionPlan, PlanProperties};
use crate::joins::utils::{OnceAsync, OnceFut};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::RecordBatchStreamAdapter;
use crate::{DisplayAs, DisplayFormatType, SendableRecordBatchStream};
@@ -236,8 +236,19 @@ impl ExecutionPlan for ScalarSubqueryExec {
vec![false; self.subqueries.len() + 1]
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- args.compute_child_statistics(&self.input, args.partition())
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ // Only `self.input` (child 0) is used; the subqueries are skipped.
+ let mut requests = vec![ChildStats::Skip; 1 + self.subqueries.len()];
+ requests[0] = ChildStats::At(partition);
+ requests
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::clone(&input_stats[0]))
}
fn cardinality_effect(&self) -> CardinalityEffect {
diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs
index 916cf1bcbb..3a4ddd2fbe 100644
--- a/datafusion/physical-plan/src/sorts/partial_sort.rs
+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs
@@ -58,7 +58,7 @@ use std::task::{Context, Poll};
use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
use crate::sorts::sort::sort_batch;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::EmptyRecordBatchStream;
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
@@ -369,8 +369,16 @@ impl ExecutionPlan for PartialSortExec {
Some(self.metrics_set.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- args.compute_child_statistics(&self.input, args.partition())
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::clone(&input_stats[0]))
}
}
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index df6ff37888..685a5e4bbf 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -45,7 +45,7 @@ use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};
use crate::spill::get_record_batch_memory_size;
use crate::spill::in_progress_spill_file::InProgressSpillFile;
use crate::spill::spill_manager::{GetSlicedSize, SpillManager};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::ReservationStream;
use crate::stream::{ObservedStream, RecordBatchStreamAdapter};
use crate::topk::TopK;
@@ -1411,14 +1411,21 @@ impl ExecutionPlan for SortExec {
Some(self.metrics_set.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let partition = if self.preserve_partitioning() {
- args.partition()
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ let child_partition = if self.preserve_partitioning() {
+ partition
} else {
None
};
- let child_stats = args.compute_child_statistics(&self.input, partition)?;
- let stats = Arc::unwrap_or_clone(child_stats);
+ vec![ChildStats::At(child_partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let stats = input_stats[0].as_ref().clone();
Ok(Arc::new(stats.with_fetch(self.fetch, 0, 1)?))
}
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index b6625885eb..693f9789b2 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -24,7 +24,7 @@ use crate::limit::LimitStream;
use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
use crate::projection::{ProjectionExec, make_with_child, update_ordering};
use crate::sorts::streaming_merge::StreamingMergeBuilder;
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
Partitioning, PlanProperties, SendableRecordBatchStream, Statistics,
@@ -387,8 +387,16 @@ impl ExecutionPlan for SortPreservingMergeExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- args.compute_child_statistics(&self.input, None)
+ fn child_stats_requests(&self, _partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(None)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ Ok(Arc::clone(&input_stats[0]))
}
fn supports_limit_pushdown(&self) -> bool {
diff --git a/datafusion/physical-plan/src/statistics.rs b/datafusion/physical-plan/src/statistics.rs
index 5ed5558e28..9246d7d9f5 100644
--- a/datafusion/physical-plan/src/statistics.rs
+++ b/datafusion/physical-plan/src/statistics.rs
@@ -18,10 +18,12 @@
//! Statistics computation for physical plans.
//!
//! [`StatisticsArgs`] provides external context to
-//! [`ExecutionPlan::statistics_with_args`].
+//! [`ExecutionPlan::statistics_from_inputs`].
use crate::ExecutionPlan;
-use datafusion_common::{Result, Statistics, assert_or_internal_err};
+use datafusion_common::{
+ Result, Statistics, assert_eq_or_internal_err, assert_or_internal_err,
+};
use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::Rc;
@@ -30,7 +32,7 @@ use std::sync::Arc;
/// Per-call memoization cache for statistics computation.
///
/// Keyed by `(plan node pointer address, partition)`. Shared across
-/// a single statistics walk via [`StatisticsArgs`].
+/// a single statistics walk via [`StatisticsContext`].
///
/// The pointer-based key is safe within a single synchronous walk:
/// all `Arc<dyn ExecutionPlan>` nodes are held by the plan tree for
@@ -65,18 +67,16 @@ impl StatsCache {
}
}
-/// Arguments passed to [`ExecutionPlan::statistics_with_args`] carrying
+/// Arguments passed to [`ExecutionPlan::statistics_from_inputs`] carrying
/// external information that operators can use when computing their
/// statistics.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
pub struct StatisticsArgs {
partition: Option<usize>,
- /// Shared memoization cache for the current statistics walk.
- cache: Rc<RefCell<StatsCache>>,
}
impl StatisticsArgs {
- /// Creates new statistics arguments with a fresh cache.
+ /// Creates new statistics arguments.
///
/// By default the partition is set to `None` (statistics should be computed
/// for the entire plan).
@@ -89,18 +89,8 @@ impl StatisticsArgs {
/// * `None` means statistics should be computed for the entire plan.
/// * `Some(idx)` means statistics should be computed for the specified
/// partition index.
- ///
- /// Changing the partition starts a new statistics walk, so the
- /// memoization cache is reset to avoid reusing entries computed for a
- /// different partition.
pub fn set_partition(&mut self, partition: Option<usize>) {
- if self.partition != partition {
- self.partition = partition;
- // Drop the previous walk's cache: its entries are keyed by raw
- // plan pointer and the prior partition, so they must not leak
- // into the new walk.
- self.cache = Rc::new(RefCell::new(StatsCache::default()));
- }
+ self.partition = partition;
}
/// Builder Style API for [`Self::set_partition`]
@@ -113,15 +103,60 @@ impl StatisticsArgs {
pub fn partition(&self) -> Option<usize> {
self.partition
}
+}
+
+/// Directive returned by [`ExecutionPlan::child_stats_requests`] describing
+/// how the [`StatisticsContext`] should obtain each child's statistics.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ChildStats {
+ /// Compute the child's statistics at this partition (`None` = overall).
+ At(Option<usize>),
+ /// Skip this child; the parent does not need its statistics. A placeholder
+ /// [`Statistics::new_unknown`] is supplied in its slot.
+ Skip,
+}
+
+/// Owns the bottom-up traversal and per-walk memoization cache for statistics
+/// computation. Call [`StatisticsContext::compute`] to walk a plan tree.
+pub struct StatisticsContext {
+ cache: Rc<RefCell<StatsCache>>,
+}
+
+impl Default for StatisticsContext {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl StatisticsContext {
+ /// Creates a context with an empty cache.
+ pub fn new() -> Self {
+ Self {
+ cache: Rc::new(RefCell::new(StatsCache::default())),
+ }
+ }
- /// Computes statistics for a child plan, using the shared cache
- /// to avoid redundant subtree walks.
- pub fn compute_child_statistics(
+ /// Clears the memoization cache.
+ ///
+ /// The cache is keyed by raw plan-node pointers, which are only stable
+ /// while the current plan tree is alive. Reset between optimizer passes
+ /// (which rewrite the plan) when reusing one context across them, so stale
+ /// pointer keys cannot collide.
+ pub fn reset_cache(&self) {
+ self.cache.borrow_mut().0.clear();
+ }
+
+ /// Computes statistics for `plan`, resolving children first and passing
+ /// the results to [`ExecutionPlan::statistics_from_inputs`].
+ ///
+ /// When `args.partition()` is `Some(idx)`, `idx` is validated against the
+ /// plan's partition count.
+ pub fn compute(
&self,
- plan: impl AsRef<dyn ExecutionPlan>,
- partition: Option<usize>,
+ plan: &dyn ExecutionPlan,
+ args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
- let plan = plan.as_ref();
+ let partition = args.partition();
if let Some(idx) = partition {
let partition_count = plan.properties().partitioning.partition_count();
@@ -137,12 +172,30 @@ impl StatisticsArgs {
return Ok(Arc::clone(cached));
}
- let child_args = StatisticsArgs {
- partition,
- cache: Rc::clone(&self.cache),
- };
- let result = plan.statistics_with_args(&child_args)?;
+ let children = plan.children();
+ let requests = plan.child_stats_requests(partition);
+ assert_eq_or_internal_err!(
+ requests.len(),
+ children.len(),
+ "{} child_stats_requests returned {} entries for {} children",
+ plan.name(),
+ requests.len(),
+ children.len()
+ );
+ let child_stats = children
+ .iter()
+ .zip(requests)
+ .map(|(child, directive)| match directive {
+ ChildStats::At(p) => {
+ self.compute(child.as_ref(), &StatisticsArgs::new().with_partition(p))
+ }
+ ChildStats::Skip => {
+ Ok(Arc::new(Statistics::new_unknown(child.schema().as_ref())))
+ }
+ })
+ .collect::<Result<Vec<_>>>()?;
+ let result = plan.statistics_from_inputs(&child_stats, args)?;
self.cache
.borrow_mut()
.insert(plan, partition, Arc::clone(&result));
@@ -183,47 +236,39 @@ mod tests {
let leaf = make_stats_leaf(100);
let plan: Arc<dyn ExecutionPlan> = Arc::new(CoalescePartitionsExec::new(leaf));
- let args = StatisticsArgs::new().with_partition(Some(0));
- let stats = plan.statistics_with_args(&args).unwrap();
+ let ctx = StatisticsContext::new();
+ let stats = ctx
+ .compute(
+ plan.as_ref(),
+ &StatisticsArgs::new().with_partition(Some(0)),
+ )
+ .unwrap();
assert_eq!(stats.num_rows, Precision::Exact(100));
- let args_none = StatisticsArgs::new();
- let stats_none = plan.statistics_with_args(&args_none).unwrap();
+ let stats_none = ctx.compute(plan.as_ref(), &StatisticsArgs::new()).unwrap();
assert_eq!(stats_none.num_rows, Precision::Exact(100));
}
#[test]
- fn changing_partition_resets_cache() {
- let leaf = make_stats_leaf(100);
+ fn context_caches_within_walk() {
+ let leaf = make_stats_leaf(42);
+ let ctx = StatisticsContext::new();
+ let args = StatisticsArgs::new();
- // Populate the memoization cache for an initial walk.
- let mut args = StatisticsArgs::new();
- let _ = args
- .compute_child_statistics(Arc::clone(&leaf), Some(0))
- .unwrap();
- assert!(
- !args.cache.borrow().0.is_empty(),
- "cache should be populated after a statistics walk"
- );
+ let s1 = ctx.compute(leaf.as_ref(), &args).unwrap();
+ assert!(!ctx.cache.borrow().0.is_empty());
- // Changing the partition starts a new walk and must reset the cache
- // so stale, pointer-keyed entries cannot leak across walks.
- args.set_partition(Some(1));
- assert!(
- args.cache.borrow().0.is_empty(),
- "cache should be cleared when the partition changes"
- );
+ let s2 = ctx.compute(leaf.as_ref(), &args).unwrap();
+ assert!(Arc::ptr_eq(&s1, &s2));
+ }
- // Setting the partition to its current value is a no-op and retains
- // the cache (avoids needlessly discarding work mid-walk).
- let _ = args
- .compute_child_statistics(Arc::clone(&leaf), Some(0))
- .unwrap();
- assert!(!args.cache.borrow().0.is_empty());
- args.set_partition(Some(1));
- assert!(
- !args.cache.borrow().0.is_empty(),
- "cache should be retained when the partition is unchanged"
- );
+ #[test]
+ fn reset_cache_clears_entries() {
+ let leaf = make_stats_leaf(10);
+ let ctx = StatisticsContext::new();
+ let _ = ctx.compute(leaf.as_ref(), &StatisticsArgs::new()).unwrap();
+ assert!(!ctx.cache.borrow().0.is_empty());
+ ctx.reset_cache();
+ assert!(ctx.cache.borrow().0.is_empty());
}
}
diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rs
index 44aacfa87a..e8c775a786 100644
--- a/datafusion/physical-plan/src/test.rs
+++ b/datafusion/physical-plan/src/test.rs
@@ -165,7 +165,11 @@ impl ExecutionPlan for TestMemoryExec {
unimplemented!()
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
} else {
diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rs
index 2bd19ccbeb..b92008c6b2 100644
--- a/datafusion/physical-plan/src/test/exec.rs
+++ b/datafusion/physical-plan/src/test/exec.rs
@@ -249,7 +249,11 @@ impl ExecutionPlan for MockExec {
}
// Panics if one of the batches is an error
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
return Ok(Arc::new(Statistics::new_unknown(&self.schema)));
}
@@ -474,7 +478,11 @@ impl ExecutionPlan for BarrierExec {
Ok(builder.build())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if args.partition().is_some() {
return Ok(Arc::new(Statistics::new_unknown(&self.schema)));
}
@@ -654,7 +662,11 @@ impl ExecutionPlan for StatisticsExec {
unimplemented!("This plan only serves for testing statistics")
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
Ok(Arc::new(if args.partition().is_some() {
Statistics::new_unknown(&self.schema)
} else {
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index 2f6d75eac6..01ebb21ad1 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -43,7 +43,7 @@ use crate::filter_pushdown::{
};
use crate::metrics::BaselineMetrics;
use crate::projection::{ProjectionExec, make_with_child};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::ObservedStream;
use arrow::datatypes::{Field, Schema, SchemaRef};
@@ -145,6 +145,20 @@ impl UnionExec {
&self.inputs
}
+ /// Maps a global output partition index to the `(input index, local
+ /// partition index)` of the input that owns it, or `None` if out of range.
+ fn owning_input(&self, partition: usize) -> Option<(usize, usize)> {
+ let mut remaining = partition;
+ for (i, input) in self.inputs.iter().enumerate() {
+ let count = input.output_partitioning().partition_count();
+ if remaining < count {
+ return Some((i, remaining));
+ }
+ remaining -= count;
+ }
+ None
+ }
+
/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.
fn compute_properties(
inputs: &[Arc<dyn ExecutionPlan>],
@@ -299,30 +313,41 @@ impl ExecutionPlan for UnionExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ if let Some(partition_idx) = partition {
+ // For a specific partition, compute stats only for the input that
+ // owns it; the other inputs are not needed and are skipped.
+ let targeted = self.owning_input(partition_idx);
+ self.inputs
+ .iter()
+ .enumerate()
+ .map(|(i, _)| match targeted {
+ Some((target_i, target_partition)) if i == target_i => {
+ ChildStats::At(Some(target_partition))
+ }
+ _ => ChildStats::Skip,
+ })
+ .collect()
+ } else {
+ vec![ChildStats::At(None); self.inputs.len()]
+ }
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
if let Some(partition_idx) = args.partition() {
// For a specific partition, find which input it belongs to
- let mut remaining_idx = partition_idx;
- for (i, input) in self.inputs.iter().enumerate() {
- let input_partition_count = input.output_partitioning().partition_count();
- if remaining_idx < input_partition_count {
- // This partition belongs to this input - compute stats
- // for the specific child at the specific partition
- let child = &self.inputs[i];
- return args.compute_child_statistics(child, Some(remaining_idx));
- }
- remaining_idx -= input_partition_count;
+ if let Some((target_i, _)) = self.owning_input(partition_idx) {
+ // This partition belongs to this input - return its stats
+ return Ok(Arc::clone(&input_stats[target_i]));
}
// If we get here, the partition index is out of bounds
Ok(Arc::new(Statistics::new_unknown(&self.schema())))
} else {
- // Collect overall stats for each input from the cache
- let stats = self
- .inputs
- .iter()
- .map(|input| args.compute_child_statistics(input, None))
- .collect::<Result<Vec<_>>>()?;
- let stats_refs = stats.iter().map(|s| s.as_ref()).collect::<Vec<_>>();
+ let stats_refs = input_stats.iter().map(|s| s.as_ref()).collect::<Vec<_>>();
Ok(Arc::new(Statistics::try_merge_iter_with_ndv_fallback(
stats_refs,
@@ -631,15 +656,19 @@ impl ExecutionPlan for InterleaveExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let stats = self
- .inputs
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition); self.inputs.len()]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let stats = input_stats
.iter()
- .map(|input| {
- args.compute_child_statistics(input, args.partition())
- .map(Arc::unwrap_or_clone)
- })
- .collect::<Result<Vec<_>>>()?;
+ .map(|s| s.as_ref().clone())
+ .collect::<Vec<_>>();
Ok(Arc::new(Statistics::try_merge_iter_with_ndv_fallback(
stats.iter(),
@@ -809,7 +838,7 @@ mod tests {
use super::*;
use crate::collect;
use crate::repartition::RepartitionExec;
- use crate::statistics::StatisticsArgs;
+ use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test::exec::StatisticsExec;
use crate::test::{self, TestMemoryExec};
@@ -1000,7 +1029,8 @@ mod tests {
Arc::new(StatisticsExec::new(right, schema.as_ref().clone()));
let union = UnionExec::try_new(vec![left, right])?;
- let stats = union.statistics_with_args(&StatisticsArgs::new())?;
+ let stats =
+ StatisticsContext::new().compute(union.as_ref(), &StatisticsArgs::new())?;
assert_eq!(stats.as_ref(), &expected);
Ok(())
@@ -1017,7 +1047,8 @@ mod tests {
Arc::new(StatisticsExec::new(right, schema.as_ref().clone()));
let union = UnionExec::try_new(vec![left, right])?;
- let stats = union.statistics_with_args(&StatisticsArgs::new())?;
+ let stats =
+ StatisticsContext::new().compute(union.as_ref(), &StatisticsArgs::new())?;
assert_eq!(stats.as_ref(), &expected);
Ok(())
@@ -1038,7 +1069,8 @@ mod tests {
)?);
let interleave = InterleaveExec::try_new(vec![left, right])?;
- let stats = interleave.statistics_with_args(&StatisticsArgs::new())?;
+ let stats =
+ StatisticsContext::new().compute(&interleave, &StatisticsArgs::new())?;
assert_eq!(stats.as_ref(), &expected);
Ok(())
@@ -1060,8 +1092,8 @@ mod tests {
)?);
let interleave = InterleaveExec::try_new(vec![left, right])?;
- let stats = interleave
- .statistics_with_args(&StatisticsArgs::new().with_partition(Some(0)))?;
+ let stats = StatisticsContext::new()
+ .compute(&interleave, &StatisticsArgs::new().with_partition(Some(0)))?;
let expected = Statistics::default()
.with_num_rows(Precision::Inexact(5))
diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
index cc3d70a1ae..bb475edbfb 100644
--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
@@ -28,7 +28,7 @@ use std::task::{Context, Poll};
use super::utils::create_schema;
use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::EmptyRecordBatchStream;
use crate::windows::{
calc_requirements, get_ordered_partition_by_indices, get_partition_by_sort_exprs,
@@ -382,10 +382,16 @@ impl ExecutionPlan for BoundedWindowAggExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let input_stat = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let input_stat = input_stats[0].as_ref().clone();
Ok(Arc::new(self.statistics_helper(input_stat)?))
}
diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs
index f1b78ef5c1..bae7cfcfd8 100644
--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs
@@ -24,7 +24,7 @@ use std::task::{Context, Poll};
use super::utils::create_schema;
use crate::execution_plan::{CardinalityEffect, EmissionType};
use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
-use crate::statistics::StatisticsArgs;
+use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::EmptyRecordBatchStream;
use crate::windows::{
calc_requirements, get_ordered_partition_by_indices, get_partition_by_sort_exprs,
@@ -285,10 +285,16 @@ impl ExecutionPlan for WindowAggExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, args: &StatisticsArgs) -> Result<Arc<Statistics>> {
- let input_stat = Arc::unwrap_or_clone(
- args.compute_child_statistics(&self.input, args.partition())?,
- );
+ fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
+ }
+
+ fn statistics_from_inputs(
+ &self,
+ input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
+ let input_stat = input_stats[0].as_ref().clone();
let win_cols = self.window_expr.len();
let input_cols = self.input.schema().fields().len();
// TODO stats: some windowing function will maintain invariants such as min, max...
diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs
index 9bf167aa73..c92face1e5 100644
--- a/datafusion/physical-plan/src/work_table.rs
+++ b/datafusion/physical-plan/src/work_table.rs
@@ -228,7 +228,11 @@ impl ExecutionPlan for WorkTableExec {
Some(self.metrics.clone_inner())
}
- fn statistics_with_args(&self, _args: &StatisticsArgs) -> Result<Arc<Statistics>> {
+ fn statistics_from_inputs(
+ &self,
+ _input_stats: &[Arc<Statistics>],
+ _args: &StatisticsArgs,
+ ) -> Result<Arc<Statistics>> {
Ok(Arc::new(Statistics::new_unknown(&self.schema())))
}
diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md
index 181e0e0b7f..f7d2745549 100644
--- a/docs/source/library-user-guide/upgrading/55.0.0.md
+++ b/docs/source/library-user-guide/upgrading/55.0.0.md
@@ -295,37 +295,42 @@ as a supertrait:
+ pub trait QueryPlanner: Any + Debug
```
-### `ExecutionPlan::partition_statistics` deprecated in favor of `statistics_with_args`
+### `ExecutionPlan::partition_statistics` deprecated in favor of `statistics_from_inputs`
-`ExecutionPlan::partition_statistics` is deprecated. A new method
-`statistics_with_args` accepts a `StatisticsArgs` parameter that carries
-the partition index and a shared cache for memoized child statistics lookups.
+`ExecutionPlan::partition_statistics` is deprecated. Statistics computation is
+now split into two parts:
+
+- `StatisticsContext` owns the bottom-up plan-tree traversal and a per-walk
+ cache of memoized child statistics. Call `StatisticsContext::compute` to
+ obtain statistics for a plan.
+- `ExecutionPlan::statistics_from_inputs` computes a node's statistics from its
+ children's already-resolved statistics, which the context passes in. The node
+ does not traverse the tree itself.
Existing implementations of `partition_statistics` continue to work unchanged.
-The default `statistics_with_args` delegates to the deprecated method, so no
+The default `statistics_from_inputs` delegates to the deprecated method, so no
migration is required until the deprecated method is removed.
-> **Warning:** The delegation is **one-way**: the default `statistics_with_args`
+> **Warning:** The delegation is **one-way**: the default `statistics_from_inputs`
> calls `partition_statistics`, but the default `partition_statistics` does
-> **not** call `statistics_with_args` — it returns `Statistics::new_unknown`.
-> Nodes that override only `statistics_with_args` will silently return
+> **not** call `statistics_from_inputs` — it returns `Statistics::new_unknown`.
+> Nodes that override only `statistics_from_inputs` will silently return
> `Statistics::new_unknown` to any caller still using the deprecated
> `partition_statistics`.
**Who is affected:**
- Users who implement custom `ExecutionPlan` nodes (recommended to migrate)
-- Users who call `partition_statistics` directly (recommended to switch to `statistics_with_args`)
+- Users who call `partition_statistics` directly (recommended to switch to `StatisticsContext::compute`)
**Migration guide:**
-For **implementations**, override `statistics_with_args` instead of
-`partition_statistics`. Leaf nodes that do not have children can ignore
-the args.
-
-Child statistics are looked up via `args.compute_child_statistics(child, partition)`.
-Use `args.partition()` for partition-preserving operators, or `None` for
-partition-merging operators that always need overall stats:
+For **implementations**, override `statistics_from_inputs` instead of
+`partition_statistics`, plus `child_stats_requests` to declare which children to
+resolve. Child statistics then arrive pre-computed in `input_stats` (one entry per
+child, in `children()` order), so the node only expresses its local propagation
+logic. Leaf nodes, and nodes that derive their statistics without reading children,
+need neither override (the default `child_stats_requests` skips every child).
```rust,ignore
// Before:
@@ -334,36 +339,39 @@ fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistic
// ... transform child_stats ...
}
-// After (partition-preserving):
-fn statistics_with_args(
- &self,
- args: &StatisticsArgs,
-) -> Result<Arc<Statistics>> {
- let child_stats = args.compute_child_statistics(&self.input, args.partition())?;
- // ... transform child_stats ...
+// After: declare the child to resolve, then compute from its statistics.
+fn child_stats_requests(&self, partition: Option<usize>) -> Vec<ChildStats> {
+ vec![ChildStats::At(partition)]
}
-// After (partition-merging):
-fn statistics_with_args(
+fn statistics_from_inputs(
&self,
+ input_stats: &[Arc<Statistics>],
args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
- let child_stats = args.compute_child_statistics(&self.input, None)?;
+ let child_stats = Arc::clone(&input_stats[0]);
// ... transform child_stats ...
}
```
-For **callers**, create a `StatisticsArgs` and call `statistics_with_args`
-directly. The cache is created automatically:
+> **Important:** the default `child_stats_requests` skips every child, so a node that
+> reads `input_stats` must override it to declare the children it uses, or those slots
+> are filled with `Statistics::new_unknown` placeholders. Request a child with
+> `ChildStats::At(partition)` (`None` = overall) and omit one with `ChildStats::Skip`.
+> For example, a partition-merging operator requests `ChildStats::At(None)`, and a
+> broadcast join requests its build side at `None`.
+
+For **callers**, walk a plan through `StatisticsContext::compute`. The cache is
+created with the context:
```rust,ignore
-use datafusion_physical_plan::StatisticsArgs;
+use datafusion_physical_plan::{StatisticsArgs, StatisticsContext};
// Before:
let stats = plan.partition_statistics(None)?;
// After:
-let stats = plan.statistics_with_args(&StatisticsArgs::new())?;
+let stats = StatisticsContext::new().compute(plan.as_ref(), &StatisticsArgs::new())?;
```
### `DdlStatement::CreateExternalTable` and `CreateFunction` are now boxed