Apache DataFusion SQL Query Engine
refactor: build the hash join hash map from the concatenated build batch (#26793)
dustind3ae55ecc425
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 3da890acb8..ffa77f5589 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -31,7 +31,8 @@ use crate::joins::key_range_bitmap::KeyRangeBitmap;use crate::joins::probe_completion::{Completion, ProbeCompletion};use crate::joins::utils::{OnceAsync, OnceFut, asymmetric_join_output_partitioning, emits_unmatched_left_rows,- is_existence_join, reorder_output_after_swap, swap_join_projection, update_hash,+ is_existence_join, reorder_output_after_swap, swap_join_projection,+ update_hash_from_values,};use crate::joins::{JoinOn, JoinOnRef, PartitionMode, ReservedMap, SharedBitmapBuilder};use crate::metrics::{Count, MetricBuilder, MetricCategory};@@ -58,7 +59,7 @@ use crate::{metrics::{ExecutionPlanMetricsSet, MetricsSet},};-use arrow::array::{Array, ArrayRef, BooleanBufferBuilder};+use arrow::array::{Array, ArrayData, ArrayRef, BooleanBufferBuilder};use arrow::compute::concat_batches;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;@@ -103,12 +104,62 @@ pub(crate) const HASH_JOIN_SEED: SeededRandomState =const ARRAY_MAP_CREATED_COUNT_METRIC_NAME: &str = "array_map_created_count";+/// The length of the longest array under `data` that `create_hashes` hashes+/// in full whatever slice of `data` it gets: everything beneath a dictionary+/// or a list view, at any depth.+fn unsliced_hash_len(data: &ArrayData) -> usize {+ let own = match data.data_type() {+ DataType::Dictionary(_, _)+ | DataType::ListView(_)+ | DataType::LargeListView(_) => subtree_max_len(&data.child_data()[0]),+ _ => 0,+ };+ data.child_data()+ .iter()+ .map(unsliced_hash_len)+ .fold(own, usize::max)+}++/// A build slice is this many times longer than the shared values beneath a+/// dictionary or list view that it rehashes, so rehashing them adds at most+/// about 1/16 to the hashing work.+const SHARED_VALUES_SLICE_FACTOR: usize = 16;++/// The number of rows in each slice the hash map is filled from.+///+/// A slice is never shorter than the largest build batch or the longest+/// shared values under any key, which keeps hashing linear in the build rows.+/// The overhead ratio is per key column, so the longest key counts, not the sum.+/// Beyond that, a slice is [`SHARED_VALUES_SLICE_FACTOR`] times the shared+/// values, and never longer than the build side.+fn hash_chunk_rows(+ left_values: &[ArrayRef],+ max_batch_rows: usize,+ num_rows: usize,+) -> usize {+ let shared = left_values+ .iter()+ .map(|values| unsliced_hash_len(&values.to_data()))+ .max()+ .unwrap_or(0);+ max_batch_rows+ .max(shared.saturating_mul(SHARED_VALUES_SLICE_FACTOR))+ .min(num_rows)+}++/// The longest array anywhere in `data`, including `data` itself.+fn subtree_max_len(data: &ArrayData) -> usize {+ data.child_data()+ .iter()+ .map(subtree_max_len)+ .fold(data.len(), usize::max)+}+/// Decides whether the build side should be joined with an [`ArrayMap`]/// (perfect hash join), returning the `(min, max)` key range to build it over.////// On `Some`, the memory of the [`ArrayMap`] has already been added to-/// `reservation`; the caller concatenates the build batches and creates the map-/// with [`ArrayMap::try_new`].+/// `reservation`; the caller creates the map with [`ArrayMap::try_new`].#[expect(clippy::too_many_arguments)]fn array_map_key_range(bounds: Option<&PartitionBounds>,@@ -680,40 +731,38 @@ impl From<&HashJoinExec> for HashJoinExecBuilder {////// When using the standard `JoinHashMap`, hash join uses LIFO data structure as a hash table,/// and in order to retain original build-side input order while obtaining data during probe phase,-/// hash table is updated by iterating batch sequence in reverse order, which keeps rows-/// with smaller indices "on the top" of hash table. The concatenated build-side batch keeps-/// the build side's input order, so each batch is inserted with the offset of its first row-/// in that concatenated batch.+/// hash table is updated by iterating the build-side rows in reverse order, which keeps rows+/// with smaller indices "on the top" of hash table. The build-side batches are first+/// concatenated in their input order. The hash table is then filled from slices of the+/// concatenated batch, as long as the largest build-side batch. When a key has values+/// under a dictionary or list view, which every slice hashes in full, slices grow to 16+/// times the length of those values. Filling starts from the last slice, and each slice+/// is inserted with the offset of its first row in the concatenated batch.///-/// Example of build phase for 3 record batches:+/// Example of build phase for 3 record batches, filled from slices of 3 rows:///////// ```text///-/// Original build-side data Inserting build-side values into hashmap Concatenated build-side batch-/// ┌───────────────────────────┐-/// hashmap.insert(row-hash, row-idx + offset) │ idx │-/// ┌───────┐ │ ┌───────┐ │-/// │ Row 1 │ 1) update_hash for batch 3 with offset 5 │ │ Row 1 │ 0 │-/// Batch 1 │ │ - hashmap.insert(Row 7, idx 6) │ Batch 1 │ │ │-/// │ Row 2 │ - hashmap.insert(Row 6, idx 5) │ │ Row 2 │ 1 │-/// └───────┘ │ └───────┘ │-/// │ │-/// ┌───────┐ │ ┌───────┐ │-/// │ Row 3 │ 2) update_hash for batch 2 with offset 2 │ │ Row 3 │ 2 │-/// │ │ - hashmap.insert(Row 5, idx 4) │ │ │ │-/// Batch 2 │ Row 4 │ - hashmap.insert(Row 4, idx 3) │ Batch 2 │ Row 4 │ 3 │-/// │ │ - hashmap.insert(Row 3, idx 2) │ │ │ │-/// │ Row 5 │ │ │ Row 5 │ 4 │-/// └───────┘ │ └───────┘ │-/// │ │-/// ┌───────┐ │ ┌───────┐ │-/// │ Row 6 │ 3) update_hash for batch 1 with offset 0 │ │ Row 6 │ 5 │-/// Batch 3 │ │ - hashmap.insert(Row 2, idx 1) │ Batch 3 │ │ │-/// │ Row 7 │ - hashmap.insert(Row 1, idx 0) │ │ Row 7 │ 6 │-/// └───────┘ │ └───────┘ │-/// │ │-/// └───────────────────────────┘+/// Original build-side data Concatenated build-side batch Inserting build-side values into hashmap+/// ┌──────────────────────────┐+/// │ idx │ hashmap.insert(row-hash, row-idx + offset)+/// ┌───────┐ │ ┌───────┐ │+/// Batch 1 │ Row 1 │ │ Slice 1 │ Row 1 │ 0 │ 1) hash slice 3 at offset 6+/// │ Row 2 │ │ │ Row 2 │ 1 │ - hashmap.insert(Row 7, idx 6)+/// └───────┘ │ │ Row 3 │ 2 │+/// │ └───────┘ │ 2) hash slice 2 at offset 3+/// ┌───────┐ │ │ - hashmap.insert(Row 6, idx 5)+/// Batch 2 │ Row 3 │ │ ┌───────┐ │ - hashmap.insert(Row 5, idx 4)+/// │ Row 4 │ │ Slice 2 │ Row 4 │ 3 │ - hashmap.insert(Row 4, idx 3)+/// │ Row 5 │ │ │ Row 5 │ 4 │+/// └───────┘ │ │ Row 6 │ 5 │ 3) hash slice 1 at offset 0+/// │ └───────┘ │ - hashmap.insert(Row 3, idx 2)+/// ┌───────┐ │ │ - hashmap.insert(Row 2, idx 1)+/// Batch 3 │ Row 6 │ │ ┌───────┐ │ - hashmap.insert(Row 1, idx 0)+/// │ Row 7 │ │ Slice 3 │ Row 7 │ 6 │+/// └───────┘ │ └───────┘ │+/// └──────────────────────────┘/// ```////// 2. the **probe phase** where the tuples of the probe side are streamed@@ -3208,29 +3257,32 @@ fn build_in_memory(// Bound before the map so error paths release the map before its charge.let mut map_reservation = reservation.new_empty();- let (join_hash_map, batch, left_values) = if let Some((min_val, max_val)) =- array_map_key_range(- bounds.as_ref(),- schema,- batches,- on_left,- &mut map_reservation,- config.execution.perfect_hash_join_small_build_threshold,- config.execution.perfect_hash_join_min_key_density,- null_equality,- )? {- let batch = if prepared {- concat_batches(schema, batches.iter())?- } else {- concat_build_batches(schema, batches, input_bytes, &mut reservation, metrics)?- };- let left_values = evaluate_expressions_to_arrays(on_left, &batch)?;+ // Merge all batches into a single batch, so we can directly index into the+ // arrays. Outside prepared mode the input batches are dropped here, before+ // the map is allocated.+ let batch = if prepared {+ concat_batches(schema, batches.iter())?+ } else {+ concat_build_batches(schema, batches, input_bytes, &mut reservation, metrics)?+ };+ let left_values = evaluate_expressions_to_arrays(on_left, &batch)?;++ let join_hash_map = if let Some((min_val, max_val)) = array_map_key_range(+ bounds.as_ref(),+ schema,+ std::slice::from_ref(&batch),+ on_left,+ &mut map_reservation,+ config.execution.perfect_hash_join_small_build_threshold,+ config.execution.perfect_hash_join_min_key_density,+ null_equality,+ )? {let array_map = ArrayMap::try_new(&left_values[0], min_val, max_val)?;array_map_created_count.add(1);metrics.build_mem_used.add(array_map.size());- (Map::ArrayMap(array_map), batch, left_values)+ Map::ArrayMap(array_map)} else {// Estimation of memory size, required for hashtable, prior to allocation.// Final result can be verified using `RawTable.allocation_info()`@@ -3239,28 +3291,42 @@ fn build_in_memory(// Arc is used instead of Box to allow sharing with SharedBuildAccumulator for hash map pushdownlet mut hashmap = new_join_hashmap(num_rows, &mut map_reservation, metrics)?;+ // Every slice shares everything beneath a dictionary or list view,+ // at any depth of a key, which `create_hashes` hashes on each call:+ // slices `SHARED_VALUES_SLICE_FACTOR` times longer than those values+ // (never shorter than the largest batch) keep that cost small.+ let chunk_rows = hash_chunk_rows(&left_values, max_batch_rows, num_rows);+let scratch_reservation = reservation.new_empty();if prepared {scratch_reservation.try_grow(prepared::prepared_scratch_bytes(- max_batch_rows,+ chunk_rows,on_left.len(),null_equality,)?)?;+ } else if chunk_rows > max_batch_rows {+ // A hashes buffer longer than a batch grows with the build side+ scratch_reservation.try_grow(chunk_rows * size_of::<u64>())?;}// The maximum is known: avoid geometric growth and its excess capacity.- let mut hashes_buffer = Vec::with_capacity(max_batch_rows);- let mut offset = num_rows;-- // Update the hashmap starting from the last batch, so rows with smaller- // indices end up on top of the chains. Each batch is indexed at its- // position in the forward concatenation below.- for batch in batches.iter().rev() {- offset -= batch.num_rows();+ let mut hashes_buffer = Vec::with_capacity(chunk_rows);+ // `chunk_rows` is 0 when the build side is empty+ let chunk_rows = chunk_rows.max(1);++ // Update the hashmap from slices of the key arrays `left_values` of at+ // most `chunk_rows` rows, starting from the last one, so rows with+ // smaller indices end up on top of the chains. Each slice is indexed at+ // its offset in `batch`.+ for offset in (0..num_rows).step_by(chunk_rows).rev() {+ let len = chunk_rows.min(num_rows - offset);+ let chunk: Vec<ArrayRef> = left_values+ .iter()+ .map(|values| values.slice(offset, len))+ .collect();hashes_buffer.clear();- hashes_buffer.resize(batch.num_rows(), 0);- update_hash(- on_left,- batch,+ hashes_buffer.resize(len, 0);+ update_hash_from_values(+ &chunk,&mut *hashmap,offset,random_state,@@ -3271,16 +3337,7 @@ fn build_in_memory()?;}- // Merge all batches into a single batch, so we can directly index into the arrays- let batch = if prepared {- concat_batches(schema, batches.iter())?- } else {- concat_build_batches(schema, batches, input_bytes, &mut reservation, metrics)?- };-- let left_values = evaluate_expressions_to_arrays(on_left, &batch)?;-- (Map::HashMap(hashmap), batch, left_values)+ Map::HashMap(hashmap)};// Rewrite float `-0.0` to `+0.0` once here, so the per probe batch key@@ -3499,10 +3556,11 @@ mod tests {use crate::joins::join_hash_map::ProbeOffset::AtRow;use arrow::array::{Array, ArrayRef, AsArray, BinaryViewArray, Date32Array, DictionaryArray,- Float32Array, Float64Array, Int32Array, Int64Array, StringArray, StringViewArray,- StructArray, UInt32Array, UInt64Array,+ FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array,+ ListArray, ListViewArray, StringArray, StringViewArray, StructArray, UInt32Array,+ UInt64Array,};- use arrow::buffer::NullBuffer;+ use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};use arrow::datatypes::{DataType, Field, Float32Type, Float64Type, Int32Type};use datafusion_common::hash_utils::create_hashes;use datafusion_common::stats::Precision;@@ -5256,6 +5314,349 @@ mod tests {Ok(())}+ /// Build batches of uneven sizes, with keys repeated across batches and+ /// NULL keys, must join as if the build side were a single batch, whatever+ /// the slices the hash map is filled from.+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn join_keeps_build_order_across_uneven_batches(+ batch_size: usize,+ use_perfect_hash_join_as_possible: bool,+ ) -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", DataType::Int32, true),+ ]));+ let batch = |a: Vec<i32>, b: Vec<Option<i32>>| {+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(a)), Arc::new(Int32Array::from(b))],+ )+ };+ let batches = vec![+ batch(vec![1, 2], vec![Some(5), None])?,+ batch(vec![3, 4, 5], vec![Some(4), Some(5), None])?,+ batch(vec![6, 7], vec![Some(5), Some(8)])?,+ ];+ let right = build_table_two_cols(+ ("a2", &vec![Some(10), Some(20), Some(30), Some(40)]),+ ("b2", &vec![Some(5), None, Some(4), Some(9)]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &schema)?) as _,+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,+ )];++ let inner: &[&str] = &[+ "+----+----+----+----+",+ "| a1 | b1 | a2 | b2 |",+ "+----+----+----+----+",+ "| 1 | 5 | 10 | 5 |",+ "| 4 | 5 | 10 | 5 |",+ "| 6 | 5 | 10 | 5 |",+ "| 3 | 4 | 30 | 4 |",+ "+----+----+----+----+",+ ];+ let inner_eq: &[&str] = &[+ "+----+----+----+----+",+ "| a1 | b1 | a2 | b2 |",+ "+----+----+----+----+",+ "| 1 | 5 | 10 | 5 |",+ "| 4 | 5 | 10 | 5 |",+ "| 6 | 5 | 10 | 5 |",+ "| 2 | | 20 | |",+ "| 5 | | 20 | |",+ "| 3 | 4 | 30 | 4 |",+ "+----+----+----+----+",+ ];+ let anti: &[&str] = &[+ "+----+----+",+ "| a1 | b1 |",+ "+----+----+",+ "| 2 | |",+ "| 5 | |",+ "| 7 | 8 |",+ "+----+----+",+ ];+ let anti_eq: &[&str] = &[+ "+----+----+",+ "| a1 | b1 |",+ "+----+----+",+ "| 7 | 8 |",+ "+----+----+",+ ];+ for (join_type, null_equality, expected) in [+ (JoinType::Inner, NullEquality::NullEqualsNothing, inner),+ (JoinType::Inner, NullEquality::NullEqualsNull, inner_eq),+ (JoinType::LeftAnti, NullEquality::NullEqualsNothing, anti),+ (JoinType::LeftAnti, NullEquality::NullEqualsNull, anti_eq),+ ] {+ let task_ctx =+ prepare_task_ctx(batch_size, use_perfect_hash_join_as_possible);+ let left = TestMemoryExec::try_new_exec(+ std::slice::from_ref(&batches),+ Arc::clone(&schema),+ None,+ )?;+ let (_, output, metrics) = join_collect(+ left,+ Arc::clone(&right),+ on.clone(),+ &join_type,+ null_equality,+ task_ctx,+ )+ .await?;++ assert_batches_eq!(expected, &output);+ // NULL build keys rule out the array map when NULLs match+ assert_phj_used(+ &metrics,+ use_perfect_hash_join_as_possible+ && null_equality == NullEquality::NullEqualsNothing,+ );+ }++ Ok(())+ }++ /// Only what a slice cannot cut counts, which is everything below a+ /// dictionary or a list view, also below a struct or a list, but not the+ /// child of a plain list or fixed size list.+ #[test]+ fn unsliced_hash_len_counts_shared_values_at_any_depth() -> Result<()> {+ let dict: ArrayRef = Arc::new(DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0, 1]),+ Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])),+ )?);+ let in_struct: ArrayRef = Arc::new(StructArray::from(vec![(+ Arc::new(Field::new("d", dict.data_type().clone(), false)),+ Arc::clone(&dict),+ )]));+ let list_view: ArrayRef = Arc::new(ListViewArray::new(+ Arc::new(Field::new_list_field(DataType::Int32, false)),+ ScalarBuffer::from(vec![0, 1]),+ ScalarBuffer::from(vec![1, 1]),+ Arc::new(Int32Array::from(vec![1, 2, 3, 4])),+ None,+ ));+ let list: ArrayRef =+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ Some(vec![Some(4), Some(5), Some(6), Some(7)]),+ ]));+ let fixed_size_list: ArrayRef = Arc::new(FixedSizeListArray::new(+ Arc::new(Field::new_list_field(DataType::Int32, false)),+ 2,+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])),+ None,+ ));+ let list_of_dict: ArrayRef = Arc::new(ListArray::new(+ Arc::new(Field::new_list_field(dict.data_type().clone(), false)),+ OffsetBuffer::from_lengths([1, 1]),+ Arc::clone(&dict),+ None,+ ));+ let dict_of_list: ArrayRef = Arc::new(DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0, 1]),+ Arc::clone(&list),+ )?);+ for (array, expected) in [+ (dict.slice(1, 1), 5),+ (in_struct.slice(1, 1), 5),+ (list_view.slice(0, 1), 4),+ (list.slice(0, 1), 0),+ (fixed_size_list.slice(1, 1), 0),+ (list_of_dict.slice(1, 1), 5),+ (dict_of_list.slice(1, 1), 7),+ ] {+ assert_eq!(+ unsliced_hash_len(&array.to_data()),+ expected,+ "{}",+ array.data_type()+ );+ }+ Ok(())+ }++ /// A key with shared values gets slices `SHARED_VALUES_SLICE_FACTOR` times+ /// longer than those values, never shorter than the largest batch and+ /// never longer than the build side.+ #[test]+ fn hash_chunk_rows_amortizes_shared_values() -> Result<()> {+ fn dict(values: i32) -> Result<ArrayRef> {+ Ok(Arc::new(DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0]),+ Arc::new(Int32Array::from_iter_values(0..values)),+ )?))+ }+ let plain: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));+ let cases = [+ (+ vec![dict(8192)?],+ 8192,+ 2_097_152,+ SHARED_VALUES_SLICE_FACTOR * 8192,+ ),+ (vec![Arc::clone(&plain)], 8192, 2_097_152, 8192),+ // The largest batch is longer than 16 times the dictionary+ (vec![dict(100)?], 8192, 2_097_152, 8192),+ (vec![Arc::clone(&plain), dict(8192)?], 8192, 50_000, 50_000),+ (vec![dict(100_000)?], 8192, 4_194_304, 1_600_000),+ // The build side caps the slice+ (vec![dict(2_097_152)?], 8192, 4_194_304, 4_194_304),+ // A batch longer than 16 times the values is the slice length+ (vec![dict(1000)?], 2_097_152, 4_194_304, 2_097_152),+ (vec![], 8192, 0, 0),+ (vec![dict(8)?], 8192, 0, 0),+ ];+ for (case, (keys, max_batch_rows, num_rows, expected)) in+ cases.into_iter().enumerate()+ {+ assert_eq!(+ hash_chunk_rows(&keys, max_batch_rows, num_rows),+ expected,+ "case {case}"+ );+ }+ Ok(())+ }++ /// Build batches sharing one key dictionary fill the hash map from slices+ /// longer than the batches, which must join as if the build side were a+ /// single batch.+ #[tokio::test]+ async fn join_keeps_build_order_with_dictionary_keys() -> Result<()> {+ let dict_type =+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));+ let schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", dict_type.clone(), true),+ ]));+ // Concatenation keeps a dictionary shared by every batch as is, so the+ // 45 build rows, in 3-row batches over 2 dictionary values, are filled+ // from slices of 32 rows: [0, 32) and [32, 45). The first boundary is+ // inside the batch [30, 33), and "x" has matches in both slices.+ let values: ArrayRef = Arc::new(StringArray::from(vec!["x", "y"]));+ // Every fifth row has a null key, the others alternate "x" and "y"+ let key = |a1: i32| (a1 % 5 != 0).then_some((a1 + 1) % 2);+ let batches = (1..=45)+ .step_by(3)+ .map(|first| {+ let a1: Vec<i32> = (first..first + 3).collect();+ let b1 = DictionaryArray::<Int32Type>::try_new(+ a1.iter().map(|&a1| key(a1)).collect(),+ Arc::clone(&values),+ )?;+ Ok(RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(a1)), Arc::new(b1)],+ )?)+ })+ .collect::<Result<Vec<_>>>()?;+ assert_eq!(+ hash_chunk_rows(&[Arc::clone(batches[0].column(1))], 3, 45),+ 32+ );+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("a2", DataType::Int32, false),+ Field::new("b2", dict_type, true),+ ]));+ let right = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![10])),+ Arc::new(DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0]),+ Arc::new(StringArray::from(vec!["x"])),+ )?),+ ],+ )?;+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &schema)?) as _,+ Arc::new(Column::new_with_schema("b2", &right_schema)?) as _,+ )];+ let left = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;+ let right = TestMemoryExec::try_new_exec(&[vec![right]], right_schema, None)?;+ let (_, output, _) = join_collect(+ left,+ right,+ on,+ &JoinType::Inner,+ NullEquality::NullEqualsNothing,+ prepare_task_ctx(8192, false),+ )+ .await?;++ assert_snapshot!(batches_to_string(&output), @r"+ +----+----+----+----++ | a1 | b1 | a2 | b2 |+ +----+----+----+----++ | 1 | x | 10 | x |+ | 3 | x | 10 | x |+ | 7 | x | 10 | x |+ | 9 | x | 10 | x |+ | 11 | x | 10 | x |+ | 13 | x | 10 | x |+ | 17 | x | 10 | x |+ | 19 | x | 10 | x |+ | 21 | x | 10 | x |+ | 23 | x | 10 | x |+ | 27 | x | 10 | x |+ | 29 | x | 10 | x |+ | 31 | x | 10 | x |+ | 33 | x | 10 | x |+ | 37 | x | 10 | x |+ | 39 | x | 10 | x |+ | 41 | x | 10 | x |+ | 43 | x | 10 | x |+ +----+----+----+----++ ");+ Ok(())+ }++ /// A build side without any batch has no rows to fill the hash map from.+ #[tokio::test]+ async fn join_build_side_without_batches() -> Result<()> {+ let task_ctx = prepare_task_ctx(8192, false);+ let schema =+ build_table_i32(("a1", &vec![]), ("b1", &vec![]), ("c1", &vec![])).schema();+ let left = TestMemoryExec::try_new_exec(&[vec![]], schema, None)?;+ let right = build_table(+ ("a2", &vec![10, 20]),+ ("b1", &vec![1, 2]),+ ("c2", &vec![70, 80]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,+ )];++ let (columns, batches, metrics) = join_collect(+ left,+ right,+ on,+ &JoinType::Right,+ NullEquality::NullEqualsNothing,+ task_ctx,+ )+ .await?;++ assert_eq!(columns, vec!["a1", "b1", "c1", "a2", "b1", "c2"]);+ assert_snapshot!(batches_to_string(&batches), @r"+ +----+----+----+----+----+----++ | a1 | b1 | c1 | a2 | b1 | c2 |+ +----+----+----+----+----+----++ | | | | 10 | 1 | 70 |+ | | | | 20 | 2 | 80 |+ +----+----+----+----+----+----++ ");+ assert_phj_used(&metrics, false);++ Ok(())+ }+/// `probe_hit_rate` and `avg_fanout` must count each probe row once, even/// when a probe batch is processed in several chunks. Every probe row matches/// all 3 build rows, so any `batch_size` below 9 splits the probe batch into@@ -8392,6 +8793,7 @@ mod tests {/// The build side is concatenated into a single batch, and that copy must/// be visible to the memory pool while the input batches are still alive.+ /// The map is reserved only after the input batches are dropped.#[rstest]#[tokio::test]async fn join_build_concat_is_reserved(@@ -8416,16 +8818,33 @@ mod tests {Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,)];- // Reserved on top of the inputs and their copy+ // Reserved on top of the copy, once the inputs are droppedlet build_rows = batches.len() * num_rows as usize;- let map = if use_perfect_hash_join_as_possible {- ArrayMap::estimate_memory_size(0, num_rows as u64 - 1, build_rows)+ let cases = if use_perfect_hash_join_as_possible {+ let map = ArrayMap::estimate_memory_size(0, num_rows as u64 - 1, build_rows);+ assert!(map < inputs / 2);+ vec![+ // Room for the inputs plus the map, not for the inputs plus their copy+ (inputs * 7 / 4, false),+ // The map fits next to the copy+ (inputs * 2 + map / 2, true),+ ]} else {- estimate_memory_size::<(u32, u64)>(build_rows, size_of::<JoinHashMapU32>())?- + build_rows * size_of::<u32>()+ let map = estimate_memory_size::<(u32, u64)>(+ build_rows,+ size_of::<JoinHashMapU32>(),+ )? + build_rows * size_of::<u32>();+ assert!(map > inputs * 2);+ vec![+ // The copy fits next to the inputs, the map does not fit next+ // to the copy+ (map + inputs / 2, false),+ // The map fits next to the copy, not next to the inputs too+ (map + inputs * 3 / 2, true),+ ]};- for (limit, fits) in [(map + inputs * 3 / 2, false), (map + inputs * 3, true)] {+ for (limit, fits) in cases {let left = TestMemoryExec::try_new_exec(std::slice::from_ref(&batches),Arc::clone(&left_schema),@@ -8554,6 +8973,101 @@ mod tests {Ok(())}+ /// Slices longer than a build batch need a hashes buffer that grows with+ /// the build side, which must be reserved.+ #[tokio::test]+ async fn join_build_hashes_buffer_is_reserved() -> Result<()> {+ let dict_type =+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int32));+ let schema = Arc::new(Schema::new(vec![Field::new("b1", dict_type, false)]));+ // 8 batches of 1024 rows share 512 dictionary values, so the whole+ // build side is hashed as one slice of 8192 rows+ let values: ArrayRef = Arc::new(Int32Array::from_iter_values(0..512));+ let batches = (0..8)+ .map(|_| {+ let keys = Int32Array::from_iter_values((0..1024).map(|i| i % 512));+ let b1 =+ DictionaryArray::<Int32Type>::try_new(keys, Arc::clone(&values))?;+ Ok(RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(b1)],+ )?)+ })+ .collect::<Result<Vec<_>>>()?;+ let build_rows = 8 * 1024;+ assert_eq!(+ hash_chunk_rows(&[Arc::clone(batches[0].column(0))], 1024, build_rows),+ build_rows+ );+ let right = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(vec![0]),+ Arc::clone(&values),+ )?)],+ )?;+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &schema)?) as _,+ Arc::new(Column::new_with_schema("b1", &schema)?) as _,+ )];++ // The concatenated copy and the map are held while the map is filled+ let copy = get_record_batch_memory_size(&concat_batches(&schema, &batches)?);+ let map =+ estimate_memory_size::<(u32, u64)>(build_rows, size_of::<JoinHashMapU32>())?+ + build_rows * size_of::<u32>();+ let hashes = build_rows * size_of::<u64>();++ for (limit, fits) in [+ // Room for the copy and the map, not for the hashes too+ (copy + map + hashes / 2, false),+ (copy + map + hashes * 3 / 2, true),+ ] {+ let left = TestMemoryExec::try_new_exec(+ std::slice::from_ref(&batches),+ Arc::clone(&schema),+ None,+ )?;+ let right = TestMemoryExec::try_new_exec(+ &[vec![right.clone()]],+ Arc::clone(&schema),+ None,+ )?;+ let join = HashJoinExec::try_new(+ left,+ right,+ on.clone(),+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?;++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(limit, 1.0)+ .build_arc()?;+ let task_ctx = prepare_task_ctx(8192, false);+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(task_ctx.session_config().clone())+ .with_runtime(runtime),+ );++ let result = common::collect(join.execute(0, task_ctx)?).await;+ if fits {+ assert_eq!(result?.iter().map(|b| b.num_rows()).sum::<usize>(), 16);+ } else {+ assert_contains!(+ result.unwrap_err().to_string(),+ "Resources exhausted: Additional allocation failed for HashJoinInput"+ );+ }+ }+ Ok(())+ }+fn build_table_struct(struct_name: &str,field_name_and_values: (&str, &Vec<Option<i32>>),diff --git a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rsindex d20e1431e5..70bdfdd098 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs@@ -233,14 +233,15 @@ pub(super) fn prepared_copy_bytes(batches: &[RecordBatch]) -> Result<usize> {.ok_or_else(overflow)}-/// Bound hashes and flat-key null masks for the largest input batch.+/// Bound hashes and flat-key null masks for the largest slice the hash map is+/// filled from.pub(super) fn prepared_scratch_bytes(- max_batch_rows: usize,+ chunk_rows: usize,num_keys: usize,null_equality: NullEquality,) -> Result<usize> {let overflow = || exec_datafusion_err!("Prepared hash-join scratch size overflow");- let hash_bytes = max_batch_rows+ let hash_bytes = chunk_rows.checked_mul(size_of::<u64>()).ok_or_else(overflow)?;if null_equality == NullEquality::NullEqualsNull {@@ -248,7 +249,7 @@ pub(super) fn prepared_scratch_bytes(}// Allow one logical null mask per key plus the combined mask.let masks = num_keys.checked_add(1).ok_or_else(overflow)?;- let mask_bytes = max_batch_rows+ let mask_bytes = chunk_rows.div_ceil(8).checked_add(64).and_then(|bytes| bytes.checked_mul(masks))