Apache DataFusion SQL Query Engine
fix: preserve byte-size statistics in AggregateExec (#18885)
Tamar14f34f641e6c
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex fb6dc3bcba..0c6ccf1b07 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -3325,30 +3325,33 @@ async fn test_count_wildcard_on_where_scalar_subquery() -> Result<()> {assert_snapshot!(pretty_format_batches(&sql_results).unwrap(),@r"- +---------------+-------------------------------------------------------------------------------------------------------------------------+- | plan_type | plan |- +---------------+-------------------------------------------------------------------------------------------------------------------------+- | logical_plan | Projection: t1.a, t1.b |- | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |- | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |- | | Left Join: t1.a = __scalar_sq_1.a |- | | TableScan: t1 projection=[a, b] |- | | SubqueryAlias: __scalar_sq_1 |- | | Projection: count(Int64(1)) AS count(*), t2.a, Boolean(true) AS __always_true |- | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1))]] |- | | TableScan: t2 projection=[a] |- | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |- | | CoalesceBatchesExec: target_batch_size=8192 |- | | HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@1)], projection=[a@0, b@1, count(*)@2, __always_true@4] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | ProjectionExec: expr=[count(Int64(1))@1 as count(*), a@0 as a, true as __always_true] |- | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(Int64(1))] |- | | CoalesceBatchesExec: target_batch_size=8192 |- | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |- | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(Int64(1))] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | |- +---------------+-------------------------------------------------------------------------------------------------------------------------++ +---------------+------------------------------------------------------------------------------------------------------------------------------++ | plan_type | plan |+ +---------------+------------------------------------------------------------------------------------------------------------------------------++ | logical_plan | Projection: t1.a, t1.b |+ | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |+ | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |+ | | Left Join: t1.a = __scalar_sq_1.a |+ | | TableScan: t1 projection=[a, b] |+ | | SubqueryAlias: __scalar_sq_1 |+ | | Projection: count(Int64(1)) AS count(*), t2.a, Boolean(true) AS __always_true |+ | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1))]] |+ | | TableScan: t2 projection=[a] |+ | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |+ | | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |+ | | ProjectionExec: expr=[a@2 as a, b@3 as b, count(*)@0 as count(*), __always_true@1 as __always_true] |+ | | CoalesceBatchesExec: target_batch_size=8192 |+ | | HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@1, a@0)], projection=[count(*)@0, __always_true@2, a@3, b@4] |+ | | CoalescePartitionsExec |+ | | ProjectionExec: expr=[count(Int64(1))@1 as count(*), a@0 as a, true as __always_true] |+ | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(Int64(1))] |+ | | CoalesceBatchesExec: target_batch_size=8192 |+ | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |+ | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(Int64(1))] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | |+ +---------------+------------------------------------------------------------------------------------------------------------------------------+");@@ -3380,30 +3383,33 @@ async fn test_count_wildcard_on_where_scalar_subquery() -> Result<()> {assert_snapshot!(pretty_format_batches(&df_results).unwrap(),@r"- +---------------+-------------------------------------------------------------------------------------------------------------------------+- | plan_type | plan |- +---------------+-------------------------------------------------------------------------------------------------------------------------+- | logical_plan | Projection: t1.a, t1.b |- | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |- | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |- | | Left Join: t1.a = __scalar_sq_1.a |- | | TableScan: t1 projection=[a, b] |- | | SubqueryAlias: __scalar_sq_1 |- | | Projection: count(*), t2.a, Boolean(true) AS __always_true |- | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1)) AS count(*)]] |- | | TableScan: t2 projection=[a] |- | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |- | | CoalesceBatchesExec: target_batch_size=8192 |- | | HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@1)], projection=[a@0, b@1, count(*)@2, __always_true@4] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | ProjectionExec: expr=[count(*)@1 as count(*), a@0 as a, true as __always_true] |- | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(*)] |- | | CoalesceBatchesExec: target_batch_size=8192 |- | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |- | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(*)] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | |- +---------------+-------------------------------------------------------------------------------------------------------------------------++ +---------------+------------------------------------------------------------------------------------------------------------------------------++ | plan_type | plan |+ +---------------+------------------------------------------------------------------------------------------------------------------------------++ | logical_plan | Projection: t1.a, t1.b |+ | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |+ | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |+ | | Left Join: t1.a = __scalar_sq_1.a |+ | | TableScan: t1 projection=[a, b] |+ | | SubqueryAlias: __scalar_sq_1 |+ | | Projection: count(*), t2.a, Boolean(true) AS __always_true |+ | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1)) AS count(*)]] |+ | | TableScan: t2 projection=[a] |+ | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |+ | | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |+ | | ProjectionExec: expr=[a@2 as a, b@3 as b, count(*)@0 as count(*), __always_true@1 as __always_true] |+ | | CoalesceBatchesExec: target_batch_size=8192 |+ | | HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@1, a@0)], projection=[count(*)@0, __always_true@2, a@3, b@4] |+ | | CoalescePartitionsExec |+ | | ProjectionExec: expr=[count(*)@1 as count(*), a@0 as a, true as __always_true] |+ | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(*)] |+ | | CoalesceBatchesExec: target_batch_size=8192 |+ | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |+ | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(*)] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | |+ +---------------+------------------------------------------------------------------------------------------------------------------------------+");diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex 7034b71fd5..7045cb8ea1 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -627,7 +627,7 @@ mod test {let expected_p0_statistics = Statistics {num_rows: Precision::Inexact(2),- total_byte_size: Precision::Absent,+ total_byte_size: Precision::Inexact(110),column_statistics: vec![ColumnStatistics {null_count: Precision::Absent,@@ -645,7 +645,7 @@ mod test {let expected_p1_statistics = Statistics {num_rows: Precision::Inexact(2),- total_byte_size: Precision::Absent,+ total_byte_size: Precision::Inexact(110),column_statistics: vec![ColumnStatistics {null_count: Precision::Absent,diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 6bf59fd3d3..f175fd4cdb 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -792,10 +792,13 @@ impl AggregateExec {AggregateMode::Final | AggregateMode::FinalPartitionedif self.group_by.expr.is_empty() =>{+ let total_byte_size =+ Self::calculate_scaled_byte_size(child_statistics, 1);+Ok(Statistics {num_rows: Precision::Exact(1),column_statistics,- total_byte_size: Precision::Absent,+ total_byte_size,})}_ => {@@ -815,14 +818,48 @@ impl AggregateExec {} else {Precision::Absent};++ let total_byte_size = num_rows+ .get_value()+ .and_then(|&output_rows| {+ Self::calculate_scaled_byte_size(child_statistics, output_rows)+ .get_value()+ .map(|&bytes| Precision::Inexact(bytes))+ })+ .unwrap_or(Precision::Absent);+Ok(Statistics {num_rows,column_statistics,- total_byte_size: Precision::Absent,+ total_byte_size,})}}}++ /// Calculate scaled byte size based on row count ratio.+ /// Returns `Precision::Absent` if input statistics are insufficient.+ /// Returns `Precision::Inexact` with the scaled value otherwise.+ ///+ /// This is a simple heuristic that assumes uniform row sizes.+ #[inline]+ fn calculate_scaled_byte_size(+ input_stats: &Statistics,+ target_row_count: usize,+ ) -> Precision<usize> {+ match (+ input_stats.num_rows.get_value(),+ input_stats.total_byte_size.get_value(),+ ) {+ (Some(&input_rows), Some(&input_bytes)) if input_rows > 0 => {+ let bytes_per_row = input_bytes as f64 / input_rows as f64;+ let scaled_bytes =+ (bytes_per_row * target_row_count as f64).ceil() as usize;+ Precision::Inexact(scaled_bytes)+ }+ _ => Precision::Absent,+ }+ }}impl DisplayAs for AggregateExec {@@ -1921,6 +1958,10 @@ mod tests {input_schema,)?);+ // Verify statistics are preserved proportionally through aggregation+ let final_stats = merged_aggregate.partition_statistics(None)?;+ assert!(final_stats.total_byte_size.get_value().is_some());+let task_ctx = if spill {// enlarge memory limit to let the final aggregation finishnew_spill_ctx(2, 2600)@@ -3146,4 +3187,77 @@ mod tests {run_test_with_spill_pool_if_necessary(20_000, false).await?;Ok(())}++ #[tokio::test]+ async fn test_aggregate_statistics_edge_cases() -> Result<()> {+ use crate::test::exec::StatisticsExec;+ use datafusion_common::ColumnStatistics;++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Float64, false),+ ]));++ // Test 1: Absent statistics remain absent+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows: Precision::Exact(100),+ total_byte_size: Precision::Absent,+ column_statistics: vec![+ ColumnStatistics::new_unknown(),+ ColumnStatistics::new_unknown(),+ ],+ },+ (*schema).clone(),+ )) as Arc<dyn ExecutionPlan>;++ let agg = Arc::new(AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::default(),+ vec![Arc::new(+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("COUNT(a)")+ .build()?,+ )],+ vec![None],+ input,+ Arc::clone(&schema),+ )?);++ let stats = agg.partition_statistics(None)?;+ assert_eq!(stats.total_byte_size, Precision::Absent);++ // Test 2: Zero rows returns Absent (can't estimate output size from zero input)+ let input_zero = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows: Precision::Exact(0),+ total_byte_size: Precision::Exact(0),+ column_statistics: vec![+ ColumnStatistics::new_unknown(),+ ColumnStatistics::new_unknown(),+ ],+ },+ (*schema).clone(),+ )) as Arc<dyn ExecutionPlan>;++ let agg_zero = Arc::new(AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::default(),+ vec![Arc::new(+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("COUNT(a)")+ .build()?,+ )],+ vec![None],+ input_zero,+ Arc::clone(&schema),+ )?);++ let stats_zero = agg_zero.partition_statistics(None)?;+ assert_eq!(stats_zero.total_byte_size, Precision::Absent);++ Ok(())+ }}diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.sltindex 9e63f79f45..c20598239c 100644--- a/datafusion/sqllogictest/test_files/union.slt+++ b/datafusion/sqllogictest/test_files/union.slt@@ -307,17 +307,17 @@ logical_planphysical_plan01)UnionExec02)--CoalesceBatchesExec: target_batch_size=2-03)----HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, CAST(t2.id AS Int32)@2), (name@1, name@1)], NullsEqual: true+03)----HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(CAST(t2.id AS Int32)@2, id@0), (name@1, name@1)], NullsEqual: true04)------CoalescePartitionsExec-05)--------AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[]-06)----------CoalesceBatchesExec: target_batch_size=2-07)------------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4-08)--------------AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[]-09)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-10)------------------DataSourceExec: partitions=1, partition_sizes=[1]-11)------ProjectionExec: expr=[id@0 as id, name@1 as name, CAST(id@0 AS Int32) as CAST(t2.id AS Int32)]-12)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-13)----------DataSourceExec: partitions=1, partition_sizes=[1]+05)--------ProjectionExec: expr=[id@0 as id, name@1 as name, CAST(id@0 AS Int32) as CAST(t2.id AS Int32)]+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------DataSourceExec: partitions=1, partition_sizes=[1]+08)------AggregateExec: mode=FinalPartitioned, gby=[id@0 as id, name@1 as name], aggr=[]+09)--------CoalesceBatchesExec: target_batch_size=2+10)----------RepartitionExec: partitioning=Hash([id@0, name@1], 4), input_partitions=4+11)------------AggregateExec: mode=Partial, gby=[id@0 as id, name@1 as name], aggr=[]+12)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+13)----------------DataSourceExec: partitions=1, partition_sizes=[1]14)--ProjectionExec: expr=[CAST(id@0 AS Int32) as id, name@1 as name]15)----CoalesceBatchesExec: target_batch_size=216)------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(CAST(t2.id AS Int32)@2, id@0), (name@1, name@1)], projection=[id@0, name@1], NullsEqual: true