Skip to content

Apache DataFusion SQL Query Engine

Implement `partition_statistics` API for `NestedLoopJoinExec` (#19468)

Kumar Ujjawaldb7b8cc4dbc6
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
index 3f079b88d5..ba53d079e3 100644
--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs
+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs
@@ -46,7 +46,7 @@ mod test {
use datafusion_physical_plan::common::compute_record_batch_statistics;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::filter::FilterExec;
- use datafusion_physical_plan::joins::CrossJoinExec;
+ use datafusion_physical_plan::joins::{CrossJoinExec, NestedLoopJoinExec};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
@@ -638,6 +638,81 @@ mod test {
Ok(())
}
+ #[tokio::test]
+ async fn test_statistic_by_partition_of_nested_loop_join() -> Result<()> {
+ use datafusion_expr::JoinType;
+
+ let left_scan = create_scan_exec_with_statistics(None, Some(2)).await;
+ let left_scan_coalesced: Arc<dyn ExecutionPlan> =
+ Arc::new(CoalescePartitionsExec::new(left_scan));
+
+ let right_scan = create_scan_exec_with_statistics(None, Some(2)).await;
+
+ let nested_loop_join: Arc<dyn ExecutionPlan> =
+ Arc::new(NestedLoopJoinExec::try_new(
+ left_scan_coalesced,
+ right_scan,
+ None,
+ &JoinType::RightSemi,
+ None,
+ )?);
+
+ // Test partition_statistics(None) - returns overall statistics
+ // For RightSemi join, output columns come from right side only
+ let full_statistics = nested_loop_join.partition_statistics(None)?;
+ // With empty join columns, estimate_join_statistics returns Inexact row count
+ // based on the outer side (right side for RightSemi)
+ let mut expected_full_statistics = create_partition_statistics(
+ 4,
+ 32,
+ 1,
+ 4,
+ Some((DATE_2025_03_01, DATE_2025_03_04)),
+ );
+ expected_full_statistics.num_rows = Precision::Inexact(4);
+ expected_full_statistics.total_byte_size = Precision::Absent;
+ assert_eq!(full_statistics, expected_full_statistics);
+
+ // Test partition_statistics(Some(idx)) - returns partition-specific statistics
+ // Partition 1: ids [3,4], dates [2025-03-01, 2025-03-02]
+ let mut expected_statistic_partition_1 = create_partition_statistics(
+ 2,
+ 16,
+ 3,
+ 4,
+ Some((DATE_2025_03_01, DATE_2025_03_02)),
+ );
+ expected_statistic_partition_1.num_rows = Precision::Inexact(2);
+ expected_statistic_partition_1.total_byte_size = Precision::Absent;
+
+ // Partition 2: ids [1,2], dates [2025-03-03, 2025-03-04]
+ let mut expected_statistic_partition_2 = create_partition_statistics(
+ 2,
+ 16,
+ 1,
+ 2,
+ Some((DATE_2025_03_03, DATE_2025_03_04)),
+ );
+ expected_statistic_partition_2.num_rows = Precision::Inexact(2);
+ expected_statistic_partition_2.total_byte_size = Precision::Absent;
+
+ let statistics = (0..nested_loop_join.output_partitioning().partition_count())
+ .map(|idx| nested_loop_join.partition_statistics(Some(idx)))
+ .collect::<Result<Vec<_>>>()?;
+ assert_eq!(statistics.len(), 2);
+ assert_eq!(statistics[0], expected_statistic_partition_1);
+ assert_eq!(statistics[1], expected_statistic_partition_2);
+
+ // Check the statistics_by_partition with real results
+ let expected_stats = vec![
+ ExpectedStatistics::NonEmpty(3, 4, 2),
+ ExpectedStatistics::NonEmpty(1, 2, 2),
+ ];
+ validate_statistics_with_data(nested_loop_join, expected_stats, 0).await?;
+
+ Ok(())
+ }
+
#[tokio::test]
async fn test_statistic_by_partition_of_coalesce_batches() -> Result<()> {
let scan = create_scan_exec_with_statistics(None, Some(2)).await;
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index 76dca72391..44637321a7 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -550,17 +550,34 @@ impl ExecutionPlan for NestedLoopJoinExec {
}
fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
- if partition.is_some() {
- return Ok(Statistics::new_unknown(&self.schema()));
- }
+ // NestedLoopJoinExec is designed for joins without equijoin keys in the
+ // ON clause (e.g., `t1 JOIN t2 ON (t1.v1 + t2.v1) % 2 = 0`). Any join
+ // predicates are stored in `self.filter`, but `estimate_join_statistics`
+ // currently doesn't support selectivity estimation for such arbitrary
+ // filter expressions. We pass an empty join column list, which means
+ // the cardinality estimation cannot use column statistics and returns
+ // unknown row counts.
let join_columns = Vec::new();
- estimate_join_statistics(
- self.left.partition_statistics(None)?,
- self.right.partition_statistics(None)?,
+
+ // Left side is always a single partition (Distribution::SinglePartition),
+ // so we always request overall stats with `None`. Right side can have
+ // multiple partitions, so we forward the partition parameter to get
+ // partition-specific statistics when requested.
+ let left_stats = self.left.partition_statistics(None)?;
+ let right_stats = match partition {
+ Some(partition) => self.right.partition_statistics(Some(partition))?,
+ None => self.right.partition_statistics(None)?,
+ };
+
+ let stats = estimate_join_statistics(
+ left_stats,
+ right_stats,
&join_columns,
&self.join_type,
- &self.schema(),
- )
+ &self.join_schema,
+ )?;
+
+ Ok(stats.project(self.projection.as_ref()))
}
/// Tries to push `projection` down through `nested_loop_join`. If possible, performs the
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index e0cdda6d5a..a9243fe04e 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -411,7 +411,35 @@ struct PartialJoinStatistics {
pub column_statistics: Vec<ColumnStatistics>,
}
-/// Estimate the statistics for the given join's output.
+/// Estimates the output statistics for a join operation based on input statistics.
+///
+/// # Statistics Propagation
+///
+/// This function estimates join output statistics using the following approach:
+/// - **Row count estimation**: Uses the `on` parameter (equijoin keys) to estimate
+/// output cardinality via [`estimate_join_cardinality`]. The estimation is based on
+/// column-level statistics (distinct counts, min/max values) of the join keys.
+/// - **Column statistics**: Combines column statistics from both inputs. For join types
+/// that preserve all columns (Inner, Left, Right, Full), statistics from both sides
+/// are concatenated. For semi/anti joins, only the relevant side's statistics are kept.
+/// - **Byte size**: Always returns `Precision::Absent` as join output size is difficult
+/// to estimate without knowing the actual data.
+///
+/// # The `on` Parameter
+///
+/// The `on` parameter represents equijoin keys (e.g., `t1.id = t2.id`). When `on` is
+/// empty (as in NestedLoopJoinExec which handles non-equijoin predicates), the
+/// cardinality estimation cannot compute selectivity from join keys, and this function
+/// returns unknown statistics (`num_rows: Precision::Absent`).
+///
+/// # Limitations
+///
+/// - Does not account for selectivity of arbitrary join filter expressions
+/// (e.g., `(t1.v1 + t2.v1) % 2 = 0`). Such filters, common in NestedLoopJoinExec,
+/// are not factored into the cardinality estimation.
+/// - Column statistics for the output are simply combined from inputs without
+/// adjusting for join selectivity (acknowledged in the code as needing
+/// "filter selectivity analysis").
pub(crate) fn estimate_join_statistics(
left_stats: Statistics,
right_stats: Statistics,