Apache DataFusion SQL Query Engine
fix: correct cross join byte size statistics (#22700)
Neil Conway766096a8aa34
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex d06e506abf..181b7de7d9 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -556,10 +556,10 @@ mod test {// Check that we have 2 partitionsassert_eq!(statistics.len(), 2);// Cross join output schema: [left.id, left.date, right.id]- // Cross join doesn't propagate Column's byte_sizelet expected_statistic_partition_1 = Statistics {num_rows: Precision::Exact(8),- total_byte_size: Precision::Exact(512),+ total_byte_size: Precision::Exact(96),+ // Cross join doesn't propagate Column's byte_sizecolumn_statistics: vec![// column 0: left.id (Int32, file column from t1)ColumnStatistics {@@ -593,7 +593,7 @@ mod test {};let expected_statistic_partition_2 = Statistics {num_rows: Precision::Exact(8),- total_byte_size: Precision::Exact(512),+ total_byte_size: Precision::Exact(96),column_statistics: vec![// column 0: left.id (Int32, file column from t1)ColumnStatistics {diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 6661d2782b..45b34692ab 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -430,13 +430,14 @@ fn stats_cartesian_product(let left_row_count = left_stats.num_rows;let right_row_count = right_stats.num_rows;- // calculate global stats+ // Calculate global statslet num_rows = left_row_count.multiply(&right_row_count);- // the result size is two times a*b because you have the columns of both left and right- let total_byte_size = left_stats- .total_byte_size- .multiply(&right_stats.total_byte_size)- .multiply(&Precision::Exact(2));++ // Each output row includes every left and right column, so the left side is+ // repeated once per right row and the right side once per left row.+ let left_byte_size = left_stats.total_byte_size.multiply(&right_row_count);+ let right_byte_size = right_stats.total_byte_size.multiply(&left_row_count);+ let total_byte_size = left_byte_size.add(&right_byte_size);let left_col_stats = left_stats.column_statistics;let right_col_stats = right_stats.column_statistics;@@ -494,7 +495,7 @@ fn stats_cartesian_product(}}-/// A stream that issues [RecordBatch]es as they arrive from the right of the join.+/// A stream that issues [RecordBatch]es as they arrive from the right of the join.struct CrossJoinStream<T> {/// Input schemaschema: Arc<Schema>,@@ -755,7 +756,9 @@ mod tests {let expected = Statistics {num_rows: Precision::Exact(left_row_count * right_row_count),- total_byte_size: Precision::Exact(2 * left_bytes * right_bytes),+ total_byte_size: Precision::Exact(+ left_bytes * right_row_count + right_bytes * left_row_count,+ ),column_statistics: vec![ColumnStatistics {distinct_count: Precision::Exact(5),