Apache DataFusion SQL Query Engine
fix: account for hash-join row-index storage (#25508)
Chao Sun43f567a976ca
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 7b9e701119..9858a4c06c 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -2797,13 +2797,27 @@ fn new_join_hashmap(if num_rows > u32::MAX as usize {let estimated_hashtable_size =- estimate_memory_size::<(u64, u64)>(num_rows, fixed_size_u64)?;+ estimate_memory_size::<(u64, u64)>(num_rows, fixed_size_u64)?+ // Each build row also owns an index in the duplicate-key chain.+ .checked_add(num_rows * size_of::<u64>())+ .ok_or_else(|| {+ datafusion_common::exec_datafusion_err!(+ "Hash join table size overflow"+ )+ })?;reservation.try_grow(estimated_hashtable_size)?;metrics.build_mem_used.add(estimated_hashtable_size);Ok(Box::new(JoinHashMapU64::with_capacity(num_rows)))} else {let estimated_hashtable_size =- estimate_memory_size::<(u32, u64)>(num_rows, fixed_size_u32)?;+ estimate_memory_size::<(u32, u64)>(num_rows, fixed_size_u32)?+ // Each build row also owns an index in the duplicate-key chain.+ .checked_add(num_rows * size_of::<u32>())+ .ok_or_else(|| {+ datafusion_common::exec_datafusion_err!(+ "Hash join table size overflow"+ )+ })?;reservation.try_grow(estimated_hashtable_size)?;metrics.build_mem_used.add(estimated_hashtable_size);Ok(Box::new(JoinHashMapU32::with_capacity(num_rows)))@@ -3218,6 +3232,38 @@ mod tests {use rstest::*;use rstest_reuse::*;+ #[test]+ fn hash_map_admits_row_indices_before_allocation() -> Result<()> {+ use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool};++ let rows = 1024;+ let buckets =+ estimate_memory_size::<(u32, u64)>(rows, size_of::<JoinHashMapU32>())?;+ let bytes = buckets + rows * size_of::<u32>();+ for (limit, succeeds) in [(bytes - 1, false), (bytes, true)] {+ let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(limit));+ let mut reservation = MemoryConsumer::new("row indices").register(&pool);+ let metrics = BuildProbeJoinMetrics::new(0, &ExecutionPlanMetricsSet::new());+ let result = new_join_hashmap(rows, &mut reservation, &metrics);+ if succeeds {+ let map = result?;+ assert_eq!(reservation.size(), bytes);+ assert_eq!(metrics.build_mem_used.value(), bytes);+ drop(map);+ } else {+ assert!(matches!(+ result,+ Err(datafusion_common::DataFusionError::ResourcesExhausted(_))+ ));+ assert_eq!(reservation.size(), 0);+ assert_eq!(metrics.build_mem_used.value(), 0);+ }+ drop(reservation);+ assert_eq!(pool.reserved(), 0);+ }+ Ok(())+ }+#[derive(Debug)]struct PartitionedTestExec {cache: Arc<PlanProperties>,