Skip to content

Apache DataFusion SQL Query Engine

refactor: retain hash join build batches after allocation failure (#26182)

Peter Lee4e84f71c49eb
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 806cff9773..eca97556d0 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -3121,9 +3121,10 @@ fn estimate_concat_allocation(array: &dyn Array) -> Result<usize> {
/// `inputs_reserved` is the memory `reservation` already holds for `batches`.
/// The copy is reserved before it is made, and once `batches` are dropped the
/// reservation is trimmed to the memory retained by the returned batch.
+/// On success, `batches` contains that batch; errors retain all input rows.
fn concat_build_batches(
schema: &SchemaRef,
- batches: Vec<RecordBatch>,
+ batches: &mut Vec<RecordBatch>,
reverse: bool,
inputs_reserved: usize,
reservation: &mut MemoryReservation,
@@ -3132,7 +3133,7 @@ fn concat_build_batches(
// Concatenating a single batch is zero-copy
let copy_size = if batches.len() > 1 {
let mut copy_size = 0;
- for batch in &batches {
+ for batch in batches.iter() {
for array in batch.columns() {
copy_size += estimate_concat_allocation(array.as_ref())?;
}
@@ -3149,7 +3150,8 @@ fn concat_build_batches(
} else {
concat_batches(schema, batches.iter())?
};
- drop(batches);
+ batches.clear();
+ batches.push(batch.clone());
// The inputs are gone: only hold on to what the concatenated batch retains,
// which includes any buffers it still shares with the inputs.
@@ -3222,11 +3224,6 @@ async fn collect_left_input(
let schema = left_stream.schema();
let prepared = mode == BuildMode::Prepared;
- // The extra scope maps + null bitmap are only built for correlated
- // null-aware joins (see `NullAwareMode`).
- let with_null_aware_row_state = null_aware.is_some_and(NullAwareMode::is_correlated);
- let null_aware_value_keys = null_aware.map_or(1, NullAwareMode::value_keys);
-
let is_phj_candidate = is_perfect_hash_join_candidate(&on_left, &schema)?;
let mut state = BuildSideState::try_new(
@@ -3266,7 +3263,7 @@ async fn collect_left_input(
// Bind the reservation first so error paths release allocations before their charge.
let BuildSideState {
- mut reservation,
+ reservation,
mut batches,
num_rows,
metrics,
@@ -3285,7 +3282,7 @@ async fn collect_left_input(
};
// Compute bounds
- let mut bounds = match bounds_accumulators {
+ let bounds = match bounds_accumulators {
Some(accumulators) if num_rows > 0 => {
let bounds = accumulators
.into_iter()
@@ -3296,30 +3293,89 @@ async fn collect_left_input(
_ => None,
};
+ build_in_memory(
+ &random_state,
+ &schema,
+ &mut batches,
+ num_rows,
+ max_batch_rows,
+ input_bytes,
+ copy_bytes,
+ &on_left,
+ &metrics,
+ reservation,
+ bounds,
+ with_visited_indices_bitmap,
+ probe_threads_count,
+ should_compute_dynamic_filters,
+ &config,
+ null_equality,
+ null_aware,
+ &array_map_created_count,
+ prepared,
+ )
+}
+
+/// Builds the hash table (or perfect-hash [`ArrayMap`]) and the bitmaps over
+/// the collected build `batches`, the second half of [`collect_left_input`].
+///
+/// `reservation` already covers `batches`; the structures built here grow it
+/// further. On error the reservation is dropped, releasing everything, while
+/// `batches` keeps every build row (possibly concatenated into one batch) for
+/// callers that need to recover from a failed build.
+#[expect(clippy::too_many_arguments)]
+fn build_in_memory(
+ random_state: &RandomState,
+ schema: &SchemaRef,
+ batches: &mut Vec<RecordBatch>,
+ num_rows: usize,
+ max_batch_rows: usize,
+ input_bytes: usize,
+ copy_bytes: usize,
+ on_left: &[PhysicalExprRef],
+ metrics: &BuildProbeJoinMetrics,
+ mut reservation: MemoryReservation,
+ mut bounds: Option<PartitionBounds>,
+ with_visited_indices_bitmap: bool,
+ probe_threads_count: usize,
+ should_compute_dynamic_filters: bool,
+ config: &ConfigOptions,
+ null_equality: NullEquality,
+ null_aware: Option<NullAwareMode>,
+ array_map_created_count: &Count,
+ prepared: bool,
+) -> Result<JoinLeftData> {
+ // The extra scope maps + null bitmap are only built for correlated
+ // null-aware joins (see `NullAwareMode`).
+ let with_null_aware_row_state = null_aware.is_some_and(NullAwareMode::is_correlated);
+ let null_aware_value_keys = null_aware.map_or(1, NullAwareMode::value_keys);
+
+ let is_phj_candidate = is_perfect_hash_join_candidate(on_left, schema)?;
+
let (join_hash_map, batch, left_values) = if let Some((min_val, max_val)) =
array_map_key_range(
bounds.as_ref(),
- &schema,
- &batches,
- &on_left,
+ schema,
+ batches,
+ on_left,
&mut reservation,
config.execution.perfect_hash_join_small_build_threshold,
config.execution.perfect_hash_join_min_key_density,
null_equality,
)? {
let batch = if prepared {
- concat_batches(&schema, batches.iter())?
+ concat_batches(schema, batches.iter())?
} else {
concat_build_batches(
- &schema,
- std::mem::take(&mut batches),
+ schema,
+ batches,
false,
input_bytes,
&mut reservation,
- &metrics,
+ metrics,
)?
};
- let left_values = evaluate_expressions_to_arrays(&on_left, &batch)?;
+ let left_values = evaluate_expressions_to_arrays(on_left, &batch)?;
let array_map = ArrayMap::try_new(&left_values[0], min_val, max_val)?;
array_map_created_count.add(1);
@@ -3332,7 +3388,7 @@ async fn collect_left_input(
// Use `u32` indices for the JoinHashMap when num_rows ≤ u32::MAX, otherwise use the
// `u64` indice variant
// Arc is used instead of Box to allow sharing with SharedBuildAccumulator for hash map pushdown
- let mut hashmap = new_join_hashmap(num_rows, &mut reservation, &metrics)?;
+ let mut hashmap = new_join_hashmap(num_rows, &mut reservation, metrics)?;
let scratch_reservation = reservation.new_empty();
if prepared {
@@ -3351,11 +3407,11 @@ async fn collect_left_input(
hashes_buffer.clear();
hashes_buffer.resize(batch.num_rows(), 0);
update_hash(
- &on_left,
+ on_left,
batch,
&mut *hashmap,
offset,
- &random_state,
+ random_state,
&mut hashes_buffer,
0,
true,
@@ -3366,19 +3422,19 @@ async fn collect_left_input(
// Merge all batches into a single batch, so we can directly index into the arrays
let batch = if prepared {
- concat_batches(&schema, batches.iter().rev())?
+ concat_batches(schema, batches.iter().rev())?
} else {
concat_build_batches(
- &schema,
- std::mem::take(&mut batches),
+ schema,
+ batches,
true,
input_bytes,
&mut reservation,
- &metrics,
+ metrics,
)?
};
- let left_values = evaluate_expressions_to_arrays(&on_left, &batch)?;
+ let left_values = evaluate_expressions_to_arrays(on_left, &batch)?;
(Map::HashMap(hashmap), batch, left_values)
};
@@ -3435,7 +3491,7 @@ async fn collect_left_input(
// Scope-only NULL marking uses a HashMap (the primary join map may
// use ArrayMap for full-key matches, but scope keys have arbitrary
// shape).
- let mut scope_map = new_join_hashmap(num_rows, &mut reservation, &metrics)?;
+ let mut scope_map = new_join_hashmap(num_rows, &mut reservation, metrics)?;
let mut hashes_buffer = vec![0; batch.num_rows()];
update_hash(
@@ -3443,7 +3499,7 @@ async fn collect_left_input(
&batch,
&mut *scope_map,
0,
- &random_state,
+ random_state,
&mut hashes_buffer,
0,
true,
@@ -3479,9 +3535,9 @@ async fn collect_left_input(
None
} else {
let null_rows = build_indices.len();
- let mut map = new_join_hashmap(null_rows, &mut reservation, &metrics)?;
+ let mut map = new_join_hashmap(null_rows, &mut reservation, metrics)?;
let mut hashes_buffer = vec![0; null_rows];
- create_hashes(&scope_values, &random_state, &mut hashes_buffer)?;
+ create_hashes(&scope_values, random_state, &mut hashes_buffer)?;
map.update_from_iter(Box::new(hashes_buffer.iter().enumerate().rev()), 0);
Some(map)
};
@@ -3593,7 +3649,7 @@ async fn collect_left_input(
}
if prepared {
- drop(batches);
+ batches.clear();
// Prepared keys are direct columns. IN-list arrays share these batch
// buffers, including through multi-key StructArray children.
let retained = RecordBatchMemoryCounter::new().count_batch(&batch);
@@ -8236,7 +8292,7 @@ mod tests {
};
let batch = concat_build_batches(
&schema,
- batches.clone(),
+ &mut batches.clone(),
reverse,
inputs_reserved,
&mut reservation,
@@ -8249,7 +8305,7 @@ mod tests {
#[test]
fn concat_build_batches_reserves_copy() -> Result<()> {
- let batches = concat_test_batches(4, 1000);
+ let mut batches = concat_test_batches(4, 1000);
let schema = batches[0].schema();
let metrics = BuildProbeJoinMetrics::new(0, &ExecutionPlanMetricsSet::new());
let inputs: usize = batches.iter().map(get_record_batch_memory_size).sum();
@@ -8258,9 +8314,10 @@ mod tests {
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(inputs * 5 / 4));
let (mut reservation, inputs_reserved) = reserve_inputs(&batches, &pool)?;
assert_eq!(inputs_reserved, inputs);
+ let original = batches.clone();
let err = concat_build_batches(
&schema,
- batches.clone(),
+ &mut batches,
false,
inputs_reserved,
&mut reservation,
@@ -8268,6 +8325,8 @@ mod tests {
)
.unwrap_err();
assert_contains!(err.to_string(), "Resources exhausted");
+ assert_eq!(batches, original);
+ drop(original);
drop(reservation);
// With room for the copy, the reservation ends up at what is retained
@@ -8275,7 +8334,7 @@ mod tests {
let (mut reservation, inputs_reserved) = reserve_inputs(&batches, &pool)?;
let batch = concat_build_batches(
&schema,
- batches,
+ &mut batches,
false,
inputs_reserved,
&mut reservation,
@@ -8283,13 +8342,14 @@ mod tests {
)?;
assert_eq!(reservation.size(), get_record_batch_memory_size(&batch));
assert_eq!(pool.reserved(), reservation.size());
+ assert_eq!(batches, vec![batch]);
Ok(())
}
/// Concatenating a single batch is zero-copy, so nothing more is reserved.
#[test]
fn concat_build_batches_single_batch_not_reserved_twice() -> Result<()> {
- let batches = concat_test_batches(1, 1000);
+ let mut batches = concat_test_batches(1, 1000);
let schema = batches[0].schema();
let metrics = BuildProbeJoinMetrics::new(0, &ExecutionPlanMetricsSet::new());
let inputs = get_record_batch_memory_size(&batches[0]);
@@ -8298,7 +8358,7 @@ mod tests {
let (mut reservation, inputs_reserved) = reserve_inputs(&batches, &pool)?;
let batch = concat_build_batches(
&schema,
- batches,
+ &mut batches,
true,
inputs_reserved,
&mut reservation,
@@ -8316,7 +8376,7 @@ mod tests {
) -> Result<()> {
let batches = concat_test_batches(4, 1000);
let column = batches[0].schema().index_of(column)?;
- let batches = batches
+ let mut batches = batches
.into_iter()
.map(|batch| batch.project(&[column]))
.collect::<Result<Vec<_>, _>>()?;
@@ -8331,7 +8391,7 @@ mod tests {
let (mut reservation, inputs_reserved) = reserve_inputs(&batches, &pool)?;
let batch = concat_build_batches(
&schema,
- batches,
+ &mut batches,
false,
inputs_reserved,
&mut reservation,
@@ -11443,4 +11503,117 @@ mod tests {
assert!(join.set_dynamic_filter(df).is_err());
Ok(())
}
+ fn fallback_inputs() -> Result<(Arc<dyn ExecutionPlan>, Arc<dyn ExecutionPlan>)> {
+ let side = |offset: i32| {
+ let batches = (0..32)
+ .map(|i| {
+ let ids: Vec<i32> = (i * 16..(i + 1) * 16).collect();
+ let keys: Vec<i32> = ids.iter().map(|id| id % 47 + offset).collect();
+ build_table_i32(("id", &ids), ("key", &keys), ("value", &ids))
+ })
+ .collect::<Vec<_>>();
+ TestMemoryExec::try_new_exec(
+ std::slice::from_ref(&batches),
+ batches[0].schema(),
+ None,
+ )
+ .map(|exec| exec as Arc<dyn ExecutionPlan>)
+ };
+ Ok((side(0)?, side(5)?))
+ }
+
+ fn fallback_context(limit: Option<usize>) -> Arc<TaskContext> {
+ let mut runtime = RuntimeEnvBuilder::new();
+ if let Some(limit) = limit {
+ runtime = runtime.with_memory_limit(limit, 1.0);
+ }
+ Arc::new(
+ TaskContext::default()
+ .with_session_config(
+ SessionConfig::default()
+ .with_batch_size(16)
+ .with_sort_spill_reservation_bytes(0),
+ )
+ .with_runtime(runtime.build_arc().unwrap()),
+ )
+ }
+
+ fn fallback_join(
+ left: Arc<dyn ExecutionPlan>,
+ right: Arc<dyn ExecutionPlan>,
+ join_type: JoinType,
+ ) -> Result<HashJoinExec> {
+ HashJoinExecBuilder::new(
+ left,
+ right,
+ vec![(
+ Arc::new(Column::new("key", 1)),
+ Arc::new(Column::new("key", 1)),
+ )],
+ join_type,
+ )
+ .with_partition_mode(PartitionMode::Partitioned)
+ .build()
+ }
+
+ #[tokio::test]
+ async fn partitioned_join_memory_limit() -> Result<()> {
+ let (left, right) = fallback_inputs()?;
+ let reference =
+ fallback_join(Arc::clone(&left), Arc::clone(&right), JoinType::Inner)?;
+ let expected =
+ common::collect(reference.execute(0, fallback_context(None))?).await?;
+ assert!(!expected.is_empty());
+ let join = fallback_join(left, right, JoinType::Inner)?;
+ let error = common::collect(join.execute(0, fallback_context(Some(4 * 1024)))?)
+ .await
+ .unwrap_err();
+ assert_contains!(error.to_string(), "Resources exhausted");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn sort_merge_fallback_preserves_promised_ordering() -> Result<()> {
+ let (left, right) = fallback_inputs()?;
+ let right = Arc::new(crate::sorts::sort::SortExec::new(
+ [PhysicalSortExpr::new_default(Arc::new(Column::new(
+ "id", 0,
+ )))]
+ .into(),
+ right,
+ ));
+ let join = fallback_join(left, right, JoinType::Inner)?;
+ assert!(join.properties().output_ordering().is_some());
+ let error = common::collect(join.execute(0, fallback_context(Some(4096)))?)
+ .await
+ .unwrap_err();
+ assert_contains!(error.to_string(), "Resources exhausted");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn memory_limited_join_keeps_streaming_inputs() -> Result<()> {
+ let (left, right) = fallback_inputs()?;
+ let batches = common::collect(right.execute(0, fallback_context(None))?).await?;
+ let right: Arc<dyn ExecutionPlan> =
+ Arc::new(crate::streaming::StreamingTableExec::try_new(
+ right.schema(),
+ vec![Arc::new(
+ crate::test::TestPartitionStream::new_with_batches(batches),
+ )],
+ None,
+ [],
+ true,
+ None,
+ )?);
+ // A hash join can stream this probe. A fallback must not turn it into a
+ // sort that waits forever for an unbounded source to finish.
+ assert!(right.boundedness().is_unbounded());
+ let join = fallback_join(left, right, JoinType::Inner)?;
+ let error = common::collect(join.execute(0, fallback_context(Some(4096)))?)
+ .await
+ .unwrap_err();
+ assert_contains!(error.to_string(), "Resources exhausted");
+ Ok(())
+ }
}