Skip to content

Apache DataFusion SQL Query Engine

Account for transformer-retained batches in symmetric hash join (#24364)

kosiewd1fc4b334a20
diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs
index b842c7e21f..2927db444c 100644
--- a/datafusion/common/src/utils/memory.rs
+++ b/datafusion/common/src/utils/memory.rs
@@ -19,9 +19,12 @@
use crate::error::_exec_datafusion_err;
use crate::{HashSet, Result};
-use arrow::array::types::{ByteArrayType, ByteViewType, RunEndIndexType};
+use arrow::array::types::{
+ ByteArrayType, ByteViewType, Int8Type, Int16Type, Int32Type, Int64Type,
+ RunEndIndexType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+};
use arrow::array::{
- Array, AsArray, GenericByteArray, GenericByteViewArray, GenericListArray,
+ Array, ArrayRef, AsArray, GenericByteArray, GenericByteViewArray, GenericListArray,
GenericListViewArray, RunArray,
};
use arrow::buffer::Buffer;
@@ -30,6 +33,7 @@ use arrow::downcast_primitive_array;
use arrow::record_batch::RecordBatch;
use std::mem::size_of;
use std::num::NonZero;
+use std::sync::Arc;
/// Maximum number of distinct buffer IDs retained inline before promotion to
/// a [`HashSet`]. Sixteen keeps small buffer sets allocation-free while
@@ -166,7 +170,9 @@ pub struct RecordBatchMemoryCounter {
/// Start addresses of `Buffer`s that have already been counted (instead of
/// actual used data region's pointer represented by current `Array`)
counted_buffers: BufferIdSet,
- /// Total memory of all unique buffers counted so far
+ /// Array objects already counted by [`Self::count_batch_with_array_overhead`]
+ counted_arrays: HashSet<usize>,
+ /// Total memory of all counted allocations
memory_usage: usize,
}
@@ -187,7 +193,28 @@ impl RecordBatchMemoryCounter {
self.memory_usage - previous_memory_usage
}
- /// Total memory of the unique buffers of all batches counted so far.
+ /// Counts unique buffers and Array objects retained by `batch`.
+ ///
+ /// This is useful for accounting a sequence of batches at an operator
+ /// boundary. It counts buffers once, including buffers shared by multiple
+ /// batches. Array-object overhead is deduplicated recursively when a shared
+ /// child has the same `ArrayRef` identity; Arrow constructors that rebuild
+ /// child array objects may conservatively count their object overhead again.
+ pub fn count_batch_with_array_overhead(&mut self, batch: &RecordBatch) -> usize {
+ let mut total_size = self.count_batch(batch);
+ let mut array_overhead = 0;
+
+ for array in batch.columns() {
+ array_overhead +=
+ count_unique_array_object_memory_size(array, &mut self.counted_arrays);
+ }
+
+ total_size += array_overhead;
+ self.memory_usage += array_overhead;
+ total_size
+ }
+
+ /// Total memory of all counted allocations.
pub fn memory_usage(&self) -> usize {
self.memory_usage
}
@@ -274,21 +301,9 @@ impl RecordBatchMemoryCounter {
self.count_array_memory_size(array.entries());
}
DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() {
- DataType::Int16 => {
- self.count_run_array_memory_size::<arrow::datatypes::Int16Type>(
- array,
- );
- }
- DataType::Int32 => {
- self.count_run_array_memory_size::<arrow::datatypes::Int32Type>(
- array,
- );
- }
- DataType::Int64 => {
- self.count_run_array_memory_size::<arrow::datatypes::Int64Type>(
- array,
- );
- }
+ DataType::Int16 => self.count_run_array_memory_size::<Int16Type>(array),
+ DataType::Int32 => self.count_run_array_memory_size::<Int32Type>(array),
+ DataType::Int64 => self.count_run_array_memory_size::<Int64Type>(array),
// Arrow only permits Int16, Int32, and Int64 run-end indexes. A
// custom Array implementation may still expose malformed data;
// retain correct accounting for it without panicking.
@@ -361,6 +376,86 @@ impl RecordBatchMemoryCounter {
}
}
+/// Counts the unique Array object memory retained by `array` and its children.
+fn count_unique_array_object_memory_size(
+ array: &ArrayRef,
+ counted_arrays: &mut HashSet<usize>,
+) -> usize {
+ let array_ptr = Arc::as_ptr(array) as *const () as usize;
+ if !counted_arrays.insert(array_ptr) {
+ return 0;
+ }
+
+ let children = array_children(array);
+ let children_overhead: usize = children
+ .iter()
+ .map(|child| child.get_array_memory_size() - child.get_buffer_memory_size())
+ .sum();
+ let own_overhead = array.get_array_memory_size()
+ - array.get_buffer_memory_size()
+ - children_overhead;
+
+ own_overhead
+ + children
+ .into_iter()
+ .map(|child| count_unique_array_object_memory_size(child, counted_arrays))
+ .sum::<usize>()
+}
+
+/// Returns the `ArrayRef` children whose object allocations may be shared.
+fn array_children(array: &ArrayRef) -> Vec<&ArrayRef> {
+ match array.data_type() {
+ DataType::Struct(_) => array.as_struct().columns().iter().collect(),
+ DataType::List(_) => vec![array.as_list::<i32>().values()],
+ DataType::LargeList(_) => vec![array.as_list::<i64>().values()],
+ DataType::ListView(_) => vec![array.as_list_view::<i32>().values()],
+ DataType::LargeListView(_) => vec![array.as_list_view::<i64>().values()],
+ DataType::FixedSizeList(_, _) => vec![array.as_fixed_size_list().values()],
+ DataType::Map(_, _) => {
+ let map = array.as_map();
+ vec![map.keys(), map.values()]
+ }
+ DataType::Union(_, _) => {
+ let union = array.as_union();
+ union
+ .fields()
+ .iter()
+ .map(|(type_id, _)| union.child(type_id))
+ .collect()
+ }
+ DataType::Dictionary(key_type, _) => match key_type.as_ref() {
+ DataType::Int8 => vec![array.as_dictionary::<Int8Type>().values()],
+ DataType::Int16 => vec![array.as_dictionary::<Int16Type>().values()],
+ DataType::Int32 => vec![array.as_dictionary::<Int32Type>().values()],
+ DataType::Int64 => vec![array.as_dictionary::<Int64Type>().values()],
+ DataType::UInt8 => vec![array.as_dictionary::<UInt8Type>().values()],
+ DataType::UInt16 => vec![array.as_dictionary::<UInt16Type>().values()],
+ DataType::UInt32 => vec![array.as_dictionary::<UInt32Type>().values()],
+ DataType::UInt64 => vec![array.as_dictionary::<UInt64Type>().values()],
+ _ => unreachable!("invalid dictionary key type: {key_type}"),
+ },
+ DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() {
+ DataType::Int16 => array
+ .as_any()
+ .downcast_ref::<RunArray<Int16Type>>()
+ .map(|array| vec![array.values()])
+ .unwrap_or_default(),
+ DataType::Int32 => array
+ .as_any()
+ .downcast_ref::<RunArray<Int32Type>>()
+ .map(|array| vec![array.values()])
+ .unwrap_or_default(),
+ DataType::Int64 => array
+ .as_any()
+ .downcast_ref::<RunArray<Int64Type>>()
+ .map(|array| vec![array.values()])
+ .unwrap_or_default(),
+ _ => vec![],
+ },
+ _ => vec![],
+ }
+}
+
/// Tracks a small number of buffers inline, avoiding a heap allocation for
/// typical batches, and promotes to a hash set when more buffers are seen.
#[derive(Debug)]
@@ -442,12 +537,14 @@ mod tests {
mod record_batch_tests {
use super::*;
use arrow::array::{
- ArrayData, ArrayRef, BinaryViewArray, Float64Array, Int16Array, Int32Array,
- Int64Array, LargeListViewArray, ListArray, ListViewArray, RunArray, StringArray,
- StringViewArray, new_null_array,
+ ArrayData, ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int16Array,
+ Int32Array, Int64Array, LargeListViewArray, ListArray, ListViewArray, MapArray,
+ RunArray, StringArray, StringViewArray, StructArray, UnionArray, new_null_array,
};
+ use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{
- DataType, Field, Int16Type, Int32Type, Int64Type, Schema, UnionFields, UnionMode,
+ DataType, Field, Fields, Int16Type, Int32Type, Int64Type, Schema, UnionFields,
+ UnionMode,
};
use std::sync::Arc;
@@ -569,6 +666,249 @@ mod record_batch_tests {
assert_eq!(size_origin, size_sliced);
}
+ #[test]
+ fn test_record_batch_memory_counter_array_overhead_shared_across_batches() {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("ints", DataType::Int32, false),
+ Field::new("floats", DataType::Float64, false),
+ ]));
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])),
+ Arc::new(Float64Array::from(vec![1., 2., 3., 4., 5., 6.])),
+ ],
+ )
+ .unwrap();
+
+ let mut counter = RecordBatchMemoryCounter::new();
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&batch),
+ batch.get_array_memory_size()
+ );
+ assert_eq!(counter.count_batch_with_array_overhead(&batch), 0);
+ assert_eq!(counter.memory_usage(), batch.get_array_memory_size());
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_deduplicates_shared_nested_array_overhead() {
+ let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let fields =
+ Fields::from(vec![Arc::new(Field::new("value", DataType::Int32, false))]);
+ let first = Arc::new(StructArray::new(
+ fields.clone(),
+ vec![Arc::clone(&shared_child)],
+ None,
+ )) as _;
+ let second = Arc::new(StructArray::new(
+ fields,
+ vec![Arc::clone(&shared_child)],
+ None,
+ )) as _;
+ let batch =
+ RecordBatch::try_from_iter(vec![("first", first), ("second", second)])
+ .unwrap();
+
+ let mut counter = RecordBatchMemoryCounter::new();
+ counter.count_batch_with_array_overhead(&batch);
+
+ assert_eq!(
+ counter.memory_usage(),
+ batch.get_array_memory_size() - shared_child.get_array_memory_size()
+ );
+ }
+
+ fn assert_recursive_shared_child_memory(
+ name: &str,
+ first: ArrayRef,
+ second: ArrayRef,
+ shared_memory: usize,
+ ) {
+ let first_memory = first.get_array_memory_size();
+ let second_memory = second.get_array_memory_size();
+ let first_batch = RecordBatch::try_from_iter(vec![(name, first)]).unwrap();
+ let second_batch = RecordBatch::try_from_iter(vec![(name, second)]).unwrap();
+ let mut counter = RecordBatchMemoryCounter::new();
+
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&first_batch),
+ first_memory,
+ "{name}: first batch"
+ );
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&second_batch),
+ second_memory - shared_memory,
+ "{name}: shared child"
+ );
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_deduplicates_shared_list_child() {
+ let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let list_field = Arc::new(Field::new_list_field(DataType::Int32, false));
+
+ let first = Arc::new(ListArray::new(
+ Arc::clone(&list_field),
+ OffsetBuffer::new(vec![0, 3].into()),
+ Arc::clone(&shared_child),
+ None,
+ ));
+ let second = Arc::new(ListArray::new(
+ list_field,
+ OffsetBuffer::new(vec![0, 3].into()),
+ Arc::clone(&shared_child),
+ None,
+ ));
+
+ assert_recursive_shared_child_memory(
+ "list",
+ first,
+ second,
+ shared_child.get_array_memory_size(),
+ );
+ }
+
+ fn map_with_shared_children(
+ fields: &Fields,
+ shared_key: &ArrayRef,
+ shared_value: &ArrayRef,
+ ) -> ArrayRef {
+ Arc::new(
+ MapArray::try_new(
+ Arc::new(Field::new(
+ "entries",
+ DataType::Struct(fields.clone()),
+ false,
+ )),
+ OffsetBuffer::new(vec![0, 3].into()),
+ StructArray::new(
+ fields.clone(),
+ vec![Arc::clone(shared_key), Arc::clone(shared_value)],
+ None,
+ ),
+ None,
+ false,
+ )
+ .unwrap(),
+ )
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_deduplicates_shared_map_children() {
+ let shared_key: ArrayRef = Arc::new(Int32Array::from(vec![4, 5, 6]));
+ let shared_value: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let fields = Fields::from(vec![
+ Arc::new(Field::new("key", DataType::Int32, false)),
+ Arc::new(Field::new("value", DataType::Int32, false)),
+ ]);
+
+ let first = map_with_shared_children(&fields, &shared_key, &shared_value);
+ let second = map_with_shared_children(&fields, &shared_key, &shared_value);
+
+ assert_recursive_shared_child_memory(
+ "map",
+ first,
+ second,
+ shared_key.get_array_memory_size() + shared_value.get_array_memory_size(),
+ );
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_deduplicates_union_slice_child_overhead() {
+ let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let fields: UnionFields =
+ std::iter::once((0, Arc::new(Field::new("value", DataType::Int32, false))))
+ .collect();
+ let first = Arc::new(
+ UnionArray::try_new(
+ fields,
+ vec![0, 0, 0].into(),
+ Some(vec![0, 1, 2].into()),
+ vec![Arc::clone(&shared_child)],
+ )
+ .unwrap(),
+ ) as ArrayRef;
+ let second = Arc::new(first.as_union().slice(1, 2)) as ArrayRef;
+ let first_batch =
+ RecordBatch::try_from_iter(vec![("union", Arc::clone(&first))]).unwrap();
+ let second_batch =
+ RecordBatch::try_from_iter(vec![("union", Arc::clone(&second))]).unwrap();
+ let child_overhead =
+ shared_child.get_array_memory_size() - shared_child.get_buffer_memory_size();
+ let second_parent_overhead = second.get_array_memory_size()
+ - second.get_buffer_memory_size()
+ - child_overhead;
+ let mut counter = RecordBatchMemoryCounter::new();
+
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&first_batch),
+ first.get_array_memory_size()
+ );
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&second_batch),
+ second_parent_overhead
+ );
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_deduplicates_run_end_slice_child_overhead() {
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let first = Arc::new(
+ RunArray::<Int32Type>::try_new(
+ &Int32Array::from(vec![1, 2, 3]),
+ values.as_ref(),
+ )
+ .unwrap(),
+ ) as ArrayRef;
+ let second = Arc::new(
+ first
+ .as_any()
+ .downcast_ref::<RunArray<Int32Type>>()
+ .unwrap()
+ .slice(1, 2),
+ ) as ArrayRef;
+ let first_batch =
+ RecordBatch::try_from_iter(vec![("run", Arc::clone(&first))]).unwrap();
+ let second_batch =
+ RecordBatch::try_from_iter(vec![("run", Arc::clone(&second))]).unwrap();
+ let child_overhead =
+ values.get_array_memory_size() - values.get_buffer_memory_size();
+ let second_parent_overhead = second.get_array_memory_size()
+ - second.get_buffer_memory_size()
+ - child_overhead;
+ let mut counter = RecordBatchMemoryCounter::new();
+
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&first_batch),
+ first.get_array_memory_size()
+ );
+ assert_eq!(
+ counter.count_batch_with_array_overhead(&second_batch),
+ second_parent_overhead
+ );
+ }
+
+ #[test]
+ fn test_record_batch_memory_counter_deduplicates_shared_dictionary_child() {
+ let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let make_dictionary = || {
+ Arc::new(
+ DictionaryArray::<Int32Type>::try_new(
+ Int32Array::from(vec![0, 1, 2]),
+ Arc::clone(&shared_child),
+ )
+ .unwrap(),
+ ) as ArrayRef
+ };
+
+ assert_recursive_shared_child_memory(
+ "dictionary",
+ make_dictionary(),
+ make_dictionary(),
+ shared_child.get_array_memory_size(),
+ );
+ }
+
#[test]
fn test_record_batch_memory_counter_buffer_shared_across_batches() {
let schema = Arc::new(Schema::new(vec![Field::new(
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index e6dad48e25..c9b9975d75 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -68,6 +68,7 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::bisect;
+use datafusion_common::utils::memory::RecordBatchMemoryCounter;
use datafusion_common::{
HashSet, JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err,
plan_err,
@@ -1482,6 +1483,10 @@ impl OneSideHashJoiner {
size += size_of_val(&self.deleted_offset);
size
}
+
+ fn size_without_input_buffer(&self) -> usize {
+ self.size() - self.input_buffer.get_array_memory_size()
+ }
pub fn new(
build_side: JoinSide,
on: Vec<PhysicalExprRef>,
@@ -1693,11 +1698,14 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {
}
StatefulStreamResult::Ready(Some(batch)) => {
self.batch_transformer.set_batch(batch);
+ self.update_reservation()?;
}
_ => {}
}
}
Some((batch, _)) => {
+ // The transformer released this batch before it is emitted.
+ self.update_reservation()?;
return self
.metrics
.baseline_metrics
@@ -1949,13 +1957,23 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {
self.state.clone()
}
+ /// Returns the memory retained by this stream at its reservation boundary.
+ ///
+ /// Input buffers and a transformer-held output batch can share Arrow buffers,
+ /// so count them as one sequence rather than summing individual batch sizes.
fn size(&self) -> usize {
- let mut size = 0;
+ let mut batch_memory_counter = RecordBatchMemoryCounter::new();
+ batch_memory_counter.count_batch_with_array_overhead(&self.left.input_buffer);
+ batch_memory_counter.count_batch_with_array_overhead(&self.right.input_buffer);
+ self.batch_transformer
+ .count_memory(&mut batch_memory_counter);
+
+ let mut size = batch_memory_counter.memory_usage();
size += size_of_val(&self.schema);
size += size_of_val(&self.filter);
size += size_of_val(&self.join_type);
- size += self.left.size();
- size += self.right.size();
+ size += self.left.size_without_input_buffer();
+ size += self.right.size_without_input_buffer();
size += size_of_val(&self.column_indices);
size += self.graph.as_ref().map(|g| g.size()).unwrap_or(0);
size += size_of_val(&self.left_sorted_filter_expr);
@@ -1966,6 +1984,14 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {
size
}
+ /// Resizes the stream reservation to match all memory retained by the stream.
+ fn update_reservation(&mut self) -> Result<()> {
+ let capacity = self.size();
+ self.reservation.try_resize(capacity)?;
+ self.metrics.stream_memory_usage.set(capacity);
+ Ok(())
+ }
+
/// Performs a join operation for the specified `probe_side` (either left or right).
/// This function:
/// 1. Determines which side is the probe and which is the build side.
@@ -2063,9 +2089,7 @@ impl<T: BatchTransformer> SymmetricHashJoinStream<T> {
// Combine results:
let result = combine_two_batches(&self.schema, equal_result, anti_result)?;
- let capacity = self.size();
- self.metrics.stream_memory_usage.set(capacity);
- self.reservation.try_resize(capacity)?;
+ self.update_reservation()?;
Ok(result)
}
}
@@ -2110,11 +2134,16 @@ mod tests {
join_expr_tests_fixture_temporal, partitioned_hash_join_with_filter,
partitioned_sym_join_with_filter, split_record_batches,
};
-
+ use crate::test::TestMemoryExec;
+ use arrow::array::{ArrayRef, Int32Array, StructArray};
use arrow::compute::SortOptions;
- use arrow::datatypes::{DataType, Field, IntervalUnit, TimeUnit};
- use datafusion_common::ScalarValue;
+ use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit};
+ use datafusion_common::{DataFusionError, ScalarValue};
use datafusion_execution::config::SessionConfig;
+ use datafusion_execution::memory_pool::{
+ MemoryLimit, MemoryPool, MemoryReservation, UnboundedMemoryPool,
+ };
+ use datafusion_execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
use datafusion_expr::Operator;
use datafusion_physical_expr::expressions::{Column, binary, col, lit};
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
@@ -2130,6 +2159,414 @@ mod tests {
static TABLE_CACHE: LazyLock<Mutex<HashMap<TableKey, TableValue>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
+ fn create_stream<T: BatchTransformer>(
+ batch_transformer: T,
+ input_schema: SchemaRef,
+ ) -> SymmetricHashJoinStream<T> {
+ let context = TaskContext::default();
+ create_stream_with_context(batch_transformer, input_schema, &context)
+ }
+
+ fn create_stream_with_context<T: BatchTransformer>(
+ batch_transformer: T,
+ input_schema: SchemaRef,
+ context: &TaskContext,
+ ) -> SymmetricHashJoinStream<T> {
+ let metrics = ExecutionPlanMetricsSet::new();
+ SymmetricHashJoinStream {
+ left_stream: Box::pin(EmptyRecordBatchStream::new(Arc::clone(&input_schema))),
+ right_stream: Box::pin(EmptyRecordBatchStream::new(Arc::clone(
+ &input_schema,
+ ))),
+ schema: Arc::clone(&input_schema),
+ filter: None,
+ join_type: JoinType::Inner,
+ left: OneSideHashJoiner::new(
+ JoinSide::Left,
+ vec![],
+ Arc::clone(&input_schema),
+ ),
+ right: OneSideHashJoiner::new(JoinSide::Right, vec![], input_schema),
+ column_indices: vec![],
+ graph: None,
+ left_sorted_filter_expr: None,
+ right_sorted_filter_expr: None,
+ random_state: RandomState::default(),
+ null_equality: NullEquality::NullEqualsNothing,
+ metrics: StreamJoinMetrics::new(0, &metrics),
+ reservation: Arc::new(
+ MemoryConsumer::new("SymmetricHashJoinStream[test]")
+ .register(context.memory_pool()),
+ ),
+ state: SHJStreamState::PullRight,
+ batch_transformer,
+ }
+ }
+
+ fn assert_stream_accounts_for_transformer<T: BatchTransformer>(batch_transformer: T) {
+ let batch = RecordBatch::try_from_iter(vec![(
+ "a",
+ Arc::new(Int32Array::from_iter_values(0..10)) as _,
+ )])
+ .unwrap();
+ let expected_size = batch.get_array_memory_size();
+ let mut stream = create_stream(batch_transformer, batch.schema());
+
+ let empty_size = stream.size();
+ stream.batch_transformer.set_batch(batch.clone());
+ stream.update_reservation().unwrap();
+ assert_eq!(stream.size() - empty_size, expected_size);
+ assert_eq!(stream.reservation.size(), stream.size());
+
+ while stream.batch_transformer.next().is_some() {}
+ stream.update_reservation().unwrap();
+ assert_eq!(stream.reservation.size(), empty_size);
+
+ stream.left.input_buffer = batch;
+ let size_with_shared_batch = stream.size();
+ stream
+ .batch_transformer
+ .set_batch(stream.left.input_buffer.clone());
+ assert_eq!(stream.size(), size_with_shared_batch);
+ }
+
+ #[test]
+ fn stream_accounts_for_transformer_batches_once() {
+ assert_stream_accounts_for_transformer(NoopBatchTransformer::new());
+ assert_stream_accounts_for_transformer(BatchSplitter::new(3));
+ }
+
+ #[tokio::test]
+ async fn symmetric_hash_join_retains_reservation_until_dropped() -> Result<()> {
+ let pool = Arc::new(RecordingMemoryPool::default());
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::clone(&pool) as Arc<dyn MemoryPool>)
+ .build_arc()?;
+ let context = TaskContext::default().with_runtime(runtime);
+ let schema = Arc::new(Schema::empty());
+ let mut stream =
+ create_stream_with_context(NoopBatchTransformer::new(), schema, &context);
+ stream.update_reservation()?;
+ let reserved_size = stream.reservation.size();
+ assert_ne!(reserved_size, 0);
+
+ // Empty input streams drive the normal exhaustion/finalization path:
+ // PullRight -> RightExhausted -> BothExhausted(false) -> final_result.
+ assert!(stream.next().await.is_none());
+ assert!(matches!(
+ stream.state(),
+ SHJStreamState::BothExhausted { final_result: true }
+ ));
+ assert_eq!(stream.reservation.size(), reserved_size);
+ assert_eq!(stream.metrics.stream_memory_usage.value(), reserved_size);
+
+ drop(stream);
+ assert_eq!(pool.reserved(), 0);
+ Ok(())
+ }
+
+ fn assert_stream_deduplicates_nested_transformer_batch<T: BatchTransformer>(
+ batch_transformer: T,
+ ) {
+ let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let fields =
+ Fields::from(vec![Arc::new(Field::new("value", DataType::Int32, false))]);
+ let left_batch = RecordBatch::try_from_iter(vec![(
+ "nested",
+ Arc::new(StructArray::new(
+ fields.clone(),
+ vec![Arc::clone(&shared_child)],
+ None,
+ )) as ArrayRef,
+ )])
+ .unwrap();
+ let transformer_batch = RecordBatch::try_from_iter(vec![(
+ "nested",
+ Arc::new(StructArray::new(
+ fields,
+ vec![Arc::clone(&shared_child)],
+ None,
+ )) as ArrayRef,
+ )])
+ .unwrap();
+ let mut stream = create_stream(batch_transformer, left_batch.schema());
+ stream.left.input_buffer = left_batch;
+ let size_without_transformer = stream.size();
+
+ stream
+ .batch_transformer
+ .set_batch(transformer_batch.clone());
+
+ assert_eq!(
+ stream.size() - size_without_transformer,
+ transformer_batch.get_array_memory_size()
+ - shared_child.get_array_memory_size()
+ );
+ }
+
+ #[test]
+ fn stream_deduplicates_nested_transformer_batches() {
+ assert_stream_deduplicates_nested_transformer_batch(NoopBatchTransformer::new());
+ assert_stream_deduplicates_nested_transformer_batch(BatchSplitter::new(3));
+ }
+
+ fn assert_transformer_reservation_exhausts_pool<T: BatchTransformer>(
+ batch_transformer: T,
+ ) -> Result<()> {
+ let batch = RecordBatch::try_from_iter(vec![(
+ "a",
+ Arc::new(Int32Array::from_iter_values(0..10)) as _,
+ )])?;
+ let empty_size =
+ create_stream(NoopBatchTransformer::new(), batch.schema()).size();
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(empty_size + batch.get_array_memory_size() - 1, 1.0)
+ .build_arc()?;
+ let context = TaskContext::default().with_runtime(runtime);
+ let mut stream =
+ create_stream_with_context(batch_transformer, batch.schema(), &context);
+
+ stream.update_reservation()?;
+ let reserved_size = stream.reservation.size();
+ stream.batch_transformer.set_batch(batch);
+ let error = stream.update_reservation().unwrap_err();
+ assert!(
+ matches!(error.find_root(), DataFusionError::ResourcesExhausted(_)),
+ "expected a memory-pool error, got: {error}"
+ );
+ assert_eq!(stream.metrics.stream_memory_usage.value(), reserved_size);
+ Ok(())
+ }
+
+ #[test]
+ fn transformer_reservation_exhausts_pool() -> Result<()> {
+ assert_transformer_reservation_exhausts_pool(NoopBatchTransformer::new())?;
+ assert_transformer_reservation_exhausts_pool(BatchSplitter::new(3))?;
+ Ok(())
+ }
+
+ #[derive(Debug, Default)]
+ struct RecordingMemoryPool {
+ inner: UnboundedMemoryPool,
+ symmetric_join_changes: Mutex<Vec<isize>>,
+ }
+
+ impl fmt::Display for RecordingMemoryPool {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Display::fmt(&self.inner, f)
+ }
+ }
+
+ impl RecordingMemoryPool {
+ fn record(&self, reservation: &MemoryReservation, change: isize) {
+ if reservation
+ .consumer()
+ .name()
+ .starts_with("SymmetricHashJoinStream")
+ {
+ self.symmetric_join_changes
+ .lock()
+ .expect("recording pool mutex is not poisoned")
+ .push(change);
+ }
+ }
+
+ fn change_deltas(&self) -> Vec<isize> {
+ self.symmetric_join_changes
+ .lock()
+ .expect("recording pool mutex is not poisoned")
+ .clone()
+ }
+ }
+
+ impl MemoryPool for RecordingMemoryPool {
+ fn name(&self) -> &str {
+ self.inner.name()
+ }
+
+ fn grow(&self, reservation: &MemoryReservation, additional: usize) {
+ self.inner.grow(reservation, additional);
+ self.record(reservation, additional as isize);
+ }
+
+ fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
+ self.inner.shrink(reservation, shrink);
+ self.record(reservation, -(shrink as isize));
+ }
+
+ fn try_grow(
+ &self,
+ reservation: &MemoryReservation,
+ additional: usize,
+ ) -> Result<()> {
+ self.inner.try_grow(reservation, additional)?;
+ self.record(reservation, additional as isize);
+ Ok(())
+ }
+
+ fn reserved(&self) -> usize {
+ self.inner.reserved()
+ }
+
+ fn memory_limit(&self) -> MemoryLimit {
+ self.inner.memory_limit()
+ }
+ }
+
+ fn transformer_lifecycle_test_join() -> Result<SymmetricHashJoinExec> {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from_iter_values(0..10))],
+ )?;
+ let left = TestMemoryExec::try_new_exec(
+ &[vec![batch.clone()]],
+ Arc::clone(&schema),
+ None,
+ )?;
+ let right =
+ TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?;
+ let on = vec![(col("id", &schema)?, col("id", &schema)?)];
+ SymmetricHashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &JoinType::Inner,
+ NullEquality::NullEqualsNothing,
+ None,
+ None,
+ StreamJoinPartitionMode::Partitioned,
+ )
+ }
+
+ fn transformer_lifecycle_test_context(
+ runtime: Arc<RuntimeEnv>,
+ enforce_batch_size_in_joins: bool,
+ ) -> Arc<TaskContext> {
+ Arc::new(
+ TaskContext::default()
+ .with_session_config(
+ SessionConfig::new()
+ .with_batch_size(3)
+ .with_enforce_batch_size_in_joins(enforce_batch_size_in_joins),
+ )
+ .with_runtime(runtime),
+ )
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn symmetric_hash_join_updates_reservation_while_transforming_output(
+ #[values(false, true)] enforce_batch_size_in_joins: bool,
+ ) -> Result<()> {
+ let pool = Arc::new(RecordingMemoryPool::default());
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::clone(&pool) as Arc<dyn MemoryPool>)
+ .build_arc()?;
+ let mut stream = transformer_lifecycle_test_join()?.execute(
+ 0,
+ transformer_lifecycle_test_context(runtime, enforce_batch_size_in_joins),
+ )?;
+
+ let first_batch = stream.next().await.transpose()?.unwrap();
+ let retained_batch_size = first_batch.get_array_memory_size() as isize;
+ let changes_after_first_batch = pool.change_deltas();
+ assert!(
+ changes_after_first_batch.contains(&retained_batch_size),
+ "expected transformer retain reservation update: {changes_after_first_batch:?}"
+ );
+ if enforce_batch_size_in_joins {
+ assert!(
+ !changes_after_first_batch.contains(&-retained_batch_size),
+ "splitter must retain the output batch after a non-final slice: {changes_after_first_batch:?}"
+ );
+ } else {
+ assert!(
+ changes_after_first_batch
+ .windows(2)
+ .any(|changes| changes == [retained_batch_size, -retained_batch_size]),
+ "expected Noop transformer release after emission: {changes_after_first_batch:?}"
+ );
+ }
+
+ let remaining_batches = crate::common::collect(stream).await?;
+ let changes_after_completion = pool.change_deltas();
+ if enforce_batch_size_in_joins {
+ assert!(
+ changes_after_completion.contains(&-retained_batch_size),
+ "expected splitter release after final slice: {changes_after_completion:?}"
+ );
+ }
+ let output_rows = first_batch.num_rows()
+ + remaining_batches
+ .iter()
+ .map(RecordBatch::num_rows)
+ .sum::<usize>();
+ assert_eq!(output_rows, 10);
+ Ok(())
+ }
+
+ #[rstest]
+ #[tokio::test]
+ async fn symmetric_hash_join_transformer_retention_exhausts_bounded_pool(
+ #[values(false, true)] enforce_batch_size_in_joins: bool,
+ ) -> Result<()> {
+ let calibration_pool = Arc::new(RecordingMemoryPool::default());
+ let calibration_runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::clone(&calibration_pool) as Arc<dyn MemoryPool>)
+ .build_arc()?;
+ let mut calibration_stream = transformer_lifecycle_test_join()?.execute(
+ 0,
+ transformer_lifecycle_test_context(
+ calibration_runtime,
+ enforce_batch_size_in_joins,
+ ),
+ )?;
+
+ let first_batch = calibration_stream.next().await.transpose()?.unwrap();
+ let retained_size = first_batch.get_array_memory_size() as isize;
+ let changes = calibration_pool.change_deltas();
+ let retain_index = changes
+ .len()
+ .checked_sub(if enforce_batch_size_in_joins { 1 } else { 2 })
+ .expect("transformer retain must update the stream reservation");
+ assert_eq!(
+ changes[retain_index], retained_size,
+ "expected transformer retain reservation update: {changes:?}"
+ );
+ if !enforce_batch_size_in_joins {
+ assert_eq!(
+ changes.last(),
+ Some(&-retained_size),
+ "Noop must release the retained batch before emitting it: {changes:?}"
+ );
+ }
+ let pre_retain_size: isize = changes[..retain_index].iter().sum();
+ assert!(
+ pre_retain_size > 0,
+ "expected a positive reservation before retention: {changes:?}"
+ );
+
+ // This limit is one byte below the independently measured retained
+ // output batch added to the reservation immediately before it.
+ // Without retain-side accounting, the first batch is emitted.
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit((pre_retain_size + retained_size) as usize - 1, 1.0)
+ .build_arc()?;
+ let mut stream = transformer_lifecycle_test_join()?.execute(
+ 0,
+ transformer_lifecycle_test_context(runtime, enforce_batch_size_in_joins),
+ )?;
+ let error = stream.next().await.transpose().unwrap_err();
+ assert!(
+ matches!(error.find_root(), DataFusionError::ResourcesExhausted(_)),
+ "expected transformer retention to exhaust the pool, got: {error}"
+ );
+ Ok(())
+ }
+
fn get_or_create_table(
cardinality: (i32, i32),
batch_size: usize,
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index fe19123cb2..5e8f3c5de9 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -64,6 +64,7 @@ use datafusion_common::cast::as_boolean_array;
use datafusion_common::hash_utils::RandomState;
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::stats::Precision;
+use datafusion_common::utils::memory::RecordBatchMemoryCounter;
use datafusion_common::utils::normalize_float_zero;
use datafusion_common::{
DataFusionError, JoinSide, JoinType, NullEquality, Result, SharedResult,
@@ -1956,6 +1957,21 @@ pub(crate) trait BatchTransformer: Debug + Clone {
/// Returns `None` if all batches have been produced.
/// The boolean flag indicates whether the batch is the last one.
fn next(&mut self) -> Option<(RecordBatch, bool)>;
+
+ /// Adds all memory retained by this transformer to `counter`.
+ ///
+ /// The stream shares this counter with its other retained batches, so
+ /// implementations must let it deduplicate shared Arrow buffers and array objects.
+ fn count_memory(&self, counter: &mut RecordBatchMemoryCounter);
+}
+
+fn count_retained_batch_memory(
+ batch: &Option<RecordBatch>,
+ counter: &mut RecordBatchMemoryCounter,
+) {
+ if let Some(batch) = batch {
+ counter.count_batch_with_array_overhead(batch);
+ }
}
#[derive(Debug, Clone)]
@@ -1979,6 +1995,10 @@ impl BatchTransformer for NoopBatchTransformer {
fn next(&mut self) -> Option<(RecordBatch, bool)> {
self.batch.take().map(|batch| (batch, true))
}
+
+ fn count_memory(&self, counter: &mut RecordBatchMemoryCounter) {
+ count_retained_batch_memory(&self.batch, counter);
+ }
}
#[derive(Debug, Clone)]
@@ -2027,6 +2047,10 @@ impl BatchTransformer for BatchSplitter {
Some((sliced_batch, last))
}
+
+ fn count_memory(&self, counter: &mut RecordBatchMemoryCounter) {
+ count_retained_batch_memory(&self.batch, counter);
+ }
}
/// When the order of the join inputs are changed, the output order of columns