Skip to content

Apache DataFusion SQL Query Engine

fix: account for hash-join row-index storage (#25508)

Chao Sun43f567a976ca
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 7b9e701119..9858a4c06c 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -2797,13 +2797,27 @@ fn new_join_hashmap(
if num_rows > u32::MAX as usize {
let estimated_hashtable_size =
- estimate_memory_size::<(u64, u64)>(num_rows, fixed_size_u64)?;
+ estimate_memory_size::<(u64, u64)>(num_rows, fixed_size_u64)?
+ // Each build row also owns an index in the duplicate-key chain.
+ .checked_add(num_rows * size_of::<u64>())
+ .ok_or_else(|| {
+ datafusion_common::exec_datafusion_err!(
+ "Hash join table size overflow"
+ )
+ })?;
reservation.try_grow(estimated_hashtable_size)?;
metrics.build_mem_used.add(estimated_hashtable_size);
Ok(Box::new(JoinHashMapU64::with_capacity(num_rows)))
} else {
let estimated_hashtable_size =
- estimate_memory_size::<(u32, u64)>(num_rows, fixed_size_u32)?;
+ estimate_memory_size::<(u32, u64)>(num_rows, fixed_size_u32)?
+ // Each build row also owns an index in the duplicate-key chain.
+ .checked_add(num_rows * size_of::<u32>())
+ .ok_or_else(|| {
+ datafusion_common::exec_datafusion_err!(
+ "Hash join table size overflow"
+ )
+ })?;
reservation.try_grow(estimated_hashtable_size)?;
metrics.build_mem_used.add(estimated_hashtable_size);
Ok(Box::new(JoinHashMapU32::with_capacity(num_rows)))
@@ -3218,6 +3232,38 @@ mod tests {
use rstest::*;
use rstest_reuse::*;
+ #[test]
+ fn hash_map_admits_row_indices_before_allocation() -> Result<()> {
+ use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool};
+
+ let rows = 1024;
+ let buckets =
+ estimate_memory_size::<(u32, u64)>(rows, size_of::<JoinHashMapU32>())?;
+ let bytes = buckets + rows * size_of::<u32>();
+ for (limit, succeeds) in [(bytes - 1, false), (bytes, true)] {
+ let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(limit));
+ let mut reservation = MemoryConsumer::new("row indices").register(&pool);
+ let metrics = BuildProbeJoinMetrics::new(0, &ExecutionPlanMetricsSet::new());
+ let result = new_join_hashmap(rows, &mut reservation, &metrics);
+ if succeeds {
+ let map = result?;
+ assert_eq!(reservation.size(), bytes);
+ assert_eq!(metrics.build_mem_used.value(), bytes);
+ drop(map);
+ } else {
+ assert!(matches!(
+ result,
+ Err(datafusion_common::DataFusionError::ResourcesExhausted(_))
+ ));
+ assert_eq!(reservation.size(), 0);
+ assert_eq!(metrics.build_mem_used.value(), 0);
+ }
+ drop(reservation);
+ assert_eq!(pool.reserved(), 0);
+ }
+ Ok(())
+ }
+
#[derive(Debug)]
struct PartitionedTestExec {
cache: Arc<PlanProperties>,