Apache DataFusion SQL Query Engine
refactor: replace MapOffset tuple with ProbeOffset enum (#26203)
Yu-Chuan Hung2aab9731ca02
diff --git a/datafusion/physical-plan/src/joins/array_map.rs b/datafusion/physical-plan/src/joins/array_map.rsindex 1a1cd8d329..84e1540a86 100644--- a/datafusion/physical-plan/src/joins/array_map.rs+++ b/datafusion/physical-plan/src/joins/array_map.rs@@ -19,8 +19,9 @@ use arrow_schema::DataType;use num_traits::AsPrimitive;use std::mem::size_of;-use crate::joins::MapOffset;use crate::joins::chain::traverse_chain;+use crate::joins::join_hash_map::ProbeOffset;+use crate::joins::join_hash_map::ProbeOffset::{AtRow, MidChain};use arrow::array::{Array, ArrayRef, AsArray, BooleanArray};use arrow::buffer::BooleanBuffer;use arrow::datatypes::ArrowNumericType;@@ -248,10 +249,10 @@ impl ArrayMap {&self,prob_side_keys: &[ArrayRef],limit: usize,- current_offset: MapOffset,+ current_offset: ProbeOffset,probe_indices: &mut Vec<u32>,build_indices: &mut Vec<u64>,- ) -> Result<Option<MapOffset>> {+ ) -> Result<Option<ProbeOffset>> {if prob_side_keys.len() != 1 {return internal_err!("ArrayMap expects 1 join key, but got {}",@@ -287,10 +288,10 @@ impl ArrayMap {&self,array: &ArrayRef,limit: usize,- current_offset: MapOffset,+ current_offset: ProbeOffset,probe_indices: &mut Vec<u32>,build_indices: &mut Vec<u64>,- ) -> Result<Option<MapOffset>>+ ) -> Result<Option<ProbeOffset>>whereT::Native: Copy + AsPrimitive<u64>,{@@ -302,9 +303,17 @@ impl ArrayMap {let have_null = arr.null_count() > 0;if self.next.is_empty() {- for prob_idx in current_offset.0..arr.len() {+ debug_assert!(+ matches!(current_offset, AtRow { .. }),+ "since the next offset is empty, mid-chain is not possible"+ );+ let rows = match current_offset {+ AtRow { row } => row,+ MidChain { row, .. } => row,+ };+ for prob_idx in rows..arr.len() {if build_indices.len() == limit {- return Ok(Some((prob_idx, None)));+ return Ok(Some(AtRow { row: prob_idx }));}// short circuit@@ -322,14 +331,14 @@ impl ArrayMap {} else {let mut remaining_output = limit;let to_skip = match current_offset {- // None `initial_next_idx` indicates that `initial_idx` processing hasn't been started- (idx, None) => idx,- // Zero `initial_next_idx` indicates that `initial_idx` has been processed during- // previous iteration, and it should be skipped- (idx, Some(0)) => idx + 1,- // Otherwise, process remaining `initial_idx` matches by traversing `next_chain`,+ // `AtRow` indicates that `idx` processing hasn't been started+ AtRow { row: idx } => idx,+ // Otherwise, process remaining `idx` matches by traversing `next_chain`,// to start with the next index- (idx, Some(next_idx)) => {+ MidChain {+ row: idx,+ next: next_idx,+ } => {let is_last = idx == arr.len() - 1;if let Some(next_offset) = traverse_chain(&self.next,@@ -348,7 +357,7 @@ impl ArrayMap {for prob_side_idx in to_skip..arr.len() {if remaining_output == 0 {- return Ok(Some((prob_side_idx, None)));+ return Ok(Some(AtRow { row: prob_side_idx }));}if have_null && arr.is_null(prob_side_idx) {@@ -433,7 +442,7 @@ mod tests {let mut prob_idx = Vec::new();let mut build_idx = Vec::new();- let mut next = Some((0, None));+ let mut next = Some(AtRow { row: 0 });let mut results = vec![];while let Some(o) = next {@@ -448,8 +457,8 @@ mod tests {}let expected = vec![- (vec![0], vec![0], Some((0, Some(2)))),- (vec![0], vec![1], Some((0, Some(0)))),+ (vec![0], vec![0], Some(MidChain { row: 0, next: 2 })),+ (vec![0], vec![1], Some(AtRow { row: 1 })),(vec![1], vec![2], None),];assert_eq!(results, expected);@@ -467,13 +476,13 @@ mod tests {let next = map.get_matched_indices_with_limit_offset(&probe,1,- (0, None),+ AtRow { row: 0 },&mut p_idx,&mut b_idx,)?;assert_eq!(p_idx, vec![1]);assert_eq!(b_idx, vec![0]);- assert_eq!(next, Some((2, None)));+ assert_eq!(next, Some(AtRow { row: 2 }));// Find 2, endlet next = map.get_matched_indices_with_limit_offset(@@ -504,14 +513,14 @@ mod tests {let result_offset = array_map.get_matched_indices_with_limit_offset(&prob_side_keys,3,- (0, None),+ AtRow { row: 0 },&mut prob_indices,&mut build_indices,)?;assert_eq!(prob_indices, vec![1, 1, 3]);assert_eq!(build_indices, vec![0, 1, 0]);- assert_eq!(result_offset, Some((3, Some(2))));+ assert_eq!(result_offset, Some(MidChain { row: 3, next: 2 }));Ok(())}@@ -542,7 +551,7 @@ mod tests {let next = map.get_matched_indices_with_limit_offset(&probe,10,- (0, None),+ AtRow { row: 0 },&mut matched_probe_indices,&mut matched_build_indices,)?;@@ -579,7 +588,7 @@ mod tests {let result_offset = array_map.get_matched_indices_with_limit_offset(&prob_side_keys,10, // A batch size larger than number of probes- (0, None),+ AtRow { row: 0 },&mut prob_indices,&mut build_indices,)?;diff --git a/datafusion/physical-plan/src/joins/chain.rs b/datafusion/physical-plan/src/joins/chain.rsindex 846b7505d6..6f7cdca51d 100644--- a/datafusion/physical-plan/src/joins/chain.rs+++ b/datafusion/physical-plan/src/joins/chain.rs@@ -20,7 +20,8 @@ use std::ops::Sub;use arrow::datatypes::ArrowNativeType;-use crate::joins::MapOffset;+use crate::joins::join_hash_map::ProbeOffset;+use crate::joins::join_hash_map::ProbeOffset::{AtRow, MidChain};/// Traverses the chain of matching indices, collecting results up to the remaining limit./// Returns `Some(offset)` if the limit was reached and there are more results to process,@@ -34,7 +35,7 @@ pub(crate) fn traverse_chain<T>(input_indices: &mut Vec<u32>,match_indices: &mut Vec<u64>,is_last_input: bool,-) -> Option<MapOffset>+) -> Option<ProbeOffset>whereT: Copy + TryFrom<usize> + PartialOrd + Into<u64> + Sub<Output = T>,<T as TryFrom<usize>>::Error: Debug,@@ -56,8 +57,15 @@ wherereturn if is_last_input && next == zero {// Finished processing the last input rowNone+ } else if next == zero {+ // `prob_idx` has been processed during this iteration, so the+ // next call should start with the next input row+ Some(AtRow { row: prob_idx + 1 })} else {- Some((prob_idx, Some(next.into())))+ Some(MidChain {+ row: prob_idx,+ next: next.into(),+ })};}if next == zero {diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex de39c16386..5c68018bda 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -3982,6 +3982,7 @@ mod tests {test::exec::MockExec,};+ use crate::joins::join_hash_map::ProbeOffset::AtRow;use arrow::array::{Array, ArrayRef, AsArray, BinaryViewArray, Date32Array, DictionaryArray,Float32Array, Float64Array, Int32Array, Int64Array, StringArray, StringViewArray,@@ -7403,7 +7404,7 @@ mod tests {&hashes_buffer,None,8192,- (0, None),+ AtRow { row: 0 },&mut probe_indices_buffer,&mut build_indices_buffer,&mut None,@@ -7466,7 +7467,7 @@ mod tests {&hashes_buffer,None,8192,- (0, None),+ AtRow { row: 0 },&mut probe_indices_buffer,&mut build_indices_buffer,&mut None,diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex cc911e14ff..a0fb90ad21 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -25,7 +25,6 @@ use std::task::Poll;use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};use crate::joins::Map;-use crate::joins::MapOffset;use crate::joins::PartitionMode;use crate::joins::hash_join::exec::{JoinLeftData, NullAwareMode, null_value_key_mask};use crate::joins::hash_join::probe_completion::ProbeSideSummary;@@ -33,6 +32,8 @@ use crate::joins::hash_join::shared_bounds::{PartitionBounds, PartitionBuildData, SharedBuildAccumulator,};use crate::joins::integer_prefilter::PrefilterState;+use crate::joins::join_hash_map::ProbeOffset;+use crate::joins::join_hash_map::ProbeOffset::AtRow;use crate::joins::utils::{JoinKeyComparator, OnceFut, equal_rows_arr_with_normalized_left, matchable_join_keys,};@@ -194,7 +195,7 @@ pub(super) struct ProcessProbeBatchState {/// Original validity combined with integer membership, for lookup only.lookup_keys: Option<NullBuffer>,/// Starting offset for JoinHashMap lookups- offset: MapOffset,+ offset: ProbeOffset,/// Max joined probe-side index from current batchjoined_probe_idx: Option<usize>,/// Max probe-side index with a join-key match from current batch, before@@ -208,7 +209,7 @@ pub(super) struct ProcessProbeBatchState {}impl ProcessProbeBatchState {- fn advance(&mut self, offset: MapOffset, joined_probe_idx: Option<usize>) {+ fn advance(&mut self, offset: ProbeOffset, joined_probe_idx: Option<usize>) {self.offset = offset;if joined_probe_idx.is_some() {self.joined_probe_idx = joined_probe_idx;@@ -231,6 +232,15 @@ impl ProcessProbeBatchState {}count}++ /// Returns whether the next lookup is the first chunk of this probe batch.+ ///+ /// A probe batch starts at `AtRow { row: 0 }`, and a lookup never returns+ /// that offset (see [`ProbeOffset`]), so this holds only before the first+ /// lookup.+ fn is_first_chunk(&self) -> bool {+ self.offset == AtRow { row: 0 }+ }}/// Container for HashJoinStreamState::EmitUnmatchedBuildRows related data@@ -492,11 +502,11 @@ pub(super) fn lookup_join_hashmap(hashes_buffer: &[u64],valid_keys: Option<&NullBuffer>,limit: usize,- offset: MapOffset,+ offset: ProbeOffset,probe_indices_buffer: &mut Vec<u32>,build_indices_buffer: &mut Vec<u64>,key_comparator: &mut Option<JoinKeyComparator>,-) -> Result<(UInt64Array, UInt32Array, Option<MapOffset>)> {+) -> Result<(UInt64Array, UInt32Array, Option<ProbeOffset>)> {let next_offset = build_hashmap.get_matched_indices_with_limit_offset(hashes_buffer,valid_keys,@@ -875,7 +885,7 @@ impl HashJoinStream {values: keys_values,valid_keys,lookup_keys,- offset: (0, None),+ offset: AtRow { row: 0 },joined_probe_idx: None,matched_probe_idx: None,key_comparator: None,@@ -897,8 +907,8 @@ impl HashJoinStream {let build_side = self.build_side.try_as_ready_mut()?;// A probe batch may be processed in several chunks; count its rows once,- // on the first chunked lookup (offset == (0, None)).- if state.offset == (0, None) {+ // on the first chunked lookup (offset == AtRow {row: 0}).+ if state.is_first_chunk() {self.join_metrics.probe_hit_rate.add_total(state.batch.num_rows());@@ -915,11 +925,11 @@ impl HashJoinStream {// For correlated null-aware joins, record this batch's UNKNOWN// candidates once, before the first chunked lookup- // (offset == (0, None)).+ // (offset == AtRow {row: 0}).//// Must precede the empty-build-map return below: an all-NULL-key// build side has an empty full-key map but still needs UNKNOWN marks.- if mode.is_correlated() && state.offset == (0, None) {+ if mode.is_correlated() && state.is_first_chunk() {mark_null_candidates_for_probe_batch(build_side,state,@@ -1347,7 +1357,7 @@ fn null_aware_skip_probe_batch(correlated: false, ..} => {let is_anti = matches!(mode, NullAwareMode::LeftAnti { .. });- if state.offset == (0, None) {+ if state.is_first_chunk() {// `on[0]` is the `NOT IN` value key for both modes. A probe// batch may resume lookup many times, but its summary is// invariant and only needs to be computed once.@@ -1729,7 +1739,7 @@ fn for_each_scope_match(build_indices_buffer: &mut Vec<u64>,mut f: impl FnMut(UInt64Array, UInt32Array) -> Result<()>,) -> Result<()> {- let mut offset = (0, None);+ let mut offset = AtRow { row: 0 };let mut key_comparator = None;loop {let (build_indices, probe_indices, next_offset) = lookup_join_hashmap(diff --git a/datafusion/physical-plan/src/joins/join_hash_map.rs b/datafusion/physical-plan/src/joins/join_hash_map.rsindex 3f58756fc3..7ddc381bf0 100644--- a/datafusion/physical-plan/src/joins/join_hash_map.rs+++ b/datafusion/physical-plan/src/joins/join_hash_map.rs@@ -28,6 +28,27 @@ use arrow::datatypes::ArrowNativeType;use hashbrown::HashTable;use hashbrown::hash_table::Entry::{Occupied, Vacant};+/// Resume point of a chunked lookup, see+/// [`JoinHashMapType::get_matched_indices_with_limit_offset`].+///+/// A lookup emits at most `limit` matches. If the limit is reached before+/// the probe batch is exhausted, it returns the position to resume from;+/// once the batch is exhausted it returns `None`.+///+/// Each position has a single encoding: once a probe row's chain has been+/// fully traversed, the next offset is the following row, `AtRow { row: row + 1 }`.+/// Given `limit > 0`, a returned offset is therefore never `AtRow { row: 0 }`,+/// which marks the first lookup of a probe batch.+#[derive(Debug, PartialEq, Eq, Clone, Copy)]+pub enum ProbeOffset {+ /// Resume at probe row `row`, from the head of its chain.+ AtRow { row: usize },+ /// Resume in the middle of probe row `row`'s chain, where `next` is the+ /// 1-based index of the next build row in the chain. `next` is never `0`,+ /// the end-of-chain marker.+ MidChain { row: usize, next: u64 },+}+/// Maps a `u64` hash value based on the build side ["on" values] to a list of indices with this key's value.////// By allocating a `HashMap` with capacity for *at least* the number of rows for entries at the build side,@@ -131,10 +152,10 @@ pub trait JoinHashMapType: Send + Sync {hash_values: &[u64],valid_keys: Option<&NullBuffer>,limit: usize,- offset: MapOffset,+ offset: ProbeOffset,input_indices: &mut Vec<u32>,match_indices: &mut Vec<u64>,- ) -> Option<MapOffset>;+ ) -> Option<ProbeOffset>;/// Returns a BooleanArray indicating which of the provided hashes exist in the map.fn contain_hashes(&self, hash_values: &[u64]) -> BooleanArray;@@ -202,10 +223,10 @@ impl JoinHashMapType for JoinHashMapU32 {hash_values: &[u64],valid_keys: Option<&NullBuffer>,limit: usize,- offset: MapOffset,+ offset: ProbeOffset,input_indices: &mut Vec<u32>,match_indices: &mut Vec<u64>,- ) -> Option<MapOffset> {+ ) -> Option<ProbeOffset> {get_matched_indices_with_limit_offset::<u32>(&self.map,&self.next,@@ -288,10 +309,10 @@ impl JoinHashMapType for JoinHashMapU64 {hash_values: &[u64],valid_keys: Option<&NullBuffer>,limit: usize,- offset: MapOffset,+ offset: ProbeOffset,input_indices: &mut Vec<u32>,match_indices: &mut Vec<u64>,- ) -> Option<MapOffset> {+ ) -> Option<ProbeOffset> {get_matched_indices_with_limit_offset::<u64>(&self.map,&self.next,@@ -318,8 +339,8 @@ impl JoinHashMapType for JoinHashMapU64 {}}-use crate::joins::MapOffset;use crate::joins::chain::traverse_chain;+use crate::joins::join_hash_map::ProbeOffset::{AtRow, MidChain};/// Inserts the rows of `iter` into `map` and `next`. Returns `true` if any row/// was chained onto an existing hash value.@@ -418,10 +439,10 @@ pub fn get_matched_indices_with_limit_offset<T>(hash_values: &[u64],valid_keys: Option<&NullBuffer>,limit: usize,- offset: MapOffset,+ offset: ProbeOffset,input_indices: &mut Vec<u32>,match_indices: &mut Vec<u64>,-) -> Option<MapOffset>+) -> Option<ProbeOffset>whereT: Copy + TryFrom<usize> + PartialOrd + Into<u64> + Sub<Output = T>,<T as TryFrom<usize>>::Error: Debug,@@ -435,7 +456,14 @@ where// Check if hashmap consists of unique values// If so, we can skip the chain traversalif unique {- let start = offset.0;+ debug_assert!(+ matches!(offset, AtRow { .. }),+ "unique keys cannot have mid-chain"+ );+ let start = match offset {+ AtRow { row } => row,+ MidChain { row, .. } => row,+ };let end = (start + limit).min(hash_values.len());for (i, &hash) in hash_values[start..end].iter().enumerate() {// NULL keys cannot match any build row@@ -450,7 +478,7 @@ wherereturn if end == hash_values.len() {None} else {- Some((end, None))+ Some(AtRow { row: end })};}@@ -458,14 +486,14 @@ where// Calculate initial `hash_values` index before iteratinglet to_skip = match offset {- // None `initial_next_idx` indicates that `initial_idx` processing hasn't been started- (idx, None) => idx,- // Zero `initial_next_idx` indicates that `initial_idx` has been processed during- // previous iteration, and it should be skipped- (idx, Some(0)) => idx + 1,- // Otherwise, process remaining `initial_idx` matches by traversing `next_chain`,+ // `AtRow` indicates that `idx` processing hasn't been started+ AtRow { row: idx } => idx,+ // Otherwise, process remaining `idx` matches by traversing `next_chain`,// to start with the next index- (idx, Some(next_idx)) => {+ MidChain {+ row: idx,+ next: next_idx,+ } => {let next_idx: T = T::usize_as(next_idx as usize);let is_last = idx == hash_values.len() - 1;if let Some(next_offset) = traverse_chain(@@ -555,7 +583,7 @@ mod tests {&probe_hashes,Some(&valid_keys),8192,- (0, None),+ AtRow { row: 0 },&mut input_indices,&mut match_indices,);@@ -586,7 +614,7 @@ mod tests {&probe_hashes,Some(&valid_keys),8192,- (0, None),+ AtRow { row: 0 },&mut input_indices,&mut match_indices,);@@ -608,34 +636,36 @@ mod tests {0,);- let probe_hashes = vec![10, 20, 30];+ // Probe row 1 has no match.+ let probe_hashes = vec![10, 40, 20, 30];let mut input_indices = vec![];let mut match_indices = vec![];let next_offset = hash_map.get_matched_indices_with_limit_offset(&probe_hashes,None,2,- (0, None),+ AtRow { row: 0 },&mut input_indices,&mut match_indices,);- // The fast path limits by probe rows and resumes at probe row 2; the- // chain walk would stop after two matches at `(1, Some(0))`.- assert_eq!(next_offset, Some((2, None)));- assert_eq!(input_indices, vec![0, 1]);- assert_eq!(match_indices, vec![0, 2]);+ // The fast path limits by probe rows and resumes at probe row 2 after a+ // single match; the chain walk limits by matches and would also emit+ // probe row 2's match, resuming at probe row 3.+ assert_eq!(next_offset, Some(AtRow { row: 2 }));+ assert_eq!(input_indices, vec![0]);+ assert_eq!(match_indices, vec![0]);let next_offset = hash_map.get_matched_indices_with_limit_offset(&probe_hashes,None,2,- (2, None),+ AtRow { row: 2 },&mut input_indices,&mut match_indices,);assert_eq!(next_offset, None);- assert_eq!(input_indices, vec![2]);- assert_eq!(match_indices, vec![3]);+ assert_eq!(input_indices, vec![2, 3]);+ assert_eq!(match_indices, vec![2, 3]);}#[test]@@ -652,7 +682,7 @@ mod tests {&[10],None,8192,- (0, None),+ AtRow { row: 0 },&mut input_indices,&mut match_indices,);@@ -679,7 +709,7 @@ mod tests {let mut input_indices = vec![];let mut match_indices = vec![];let mut matched = vec![];- let mut offset = (0, None);+ let mut offset = AtRow { row: 0 };for _ in 0..3 {let next_offset = hash_map.get_matched_indices_with_limit_offset(&[10],diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex 78bbd21b84..38d31d16c3 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -107,8 +107,6 @@ impl Map {}}-pub(crate) type MapOffset = (usize, Option<u64>);-#[cfg(test)]pub mod test_utils;diff --git a/datafusion/physical-plan/src/joins/stream_join_utils.rs b/datafusion/physical-plan/src/joins/stream_join_utils.rsindex f74c3671d0..90ce0d87a1 100644--- a/datafusion/physical-plan/src/joins/stream_join_utils.rs+++ b/datafusion/physical-plan/src/joins/stream_join_utils.rs@@ -23,7 +23,7 @@ use std::collections::{HashMap, VecDeque};use std::mem::size_of;use std::sync::Arc;-use crate::joins::MapOffset;+use crate::joins::join_hash_map::ProbeOffset;use crate::joins::join_hash_map::{contain_hashes, get_matched_indices, get_matched_indices_with_limit_offset,update_from_iter,@@ -87,10 +87,10 @@ impl JoinHashMapType for PruningJoinHashMap {hash_values: &[u64],valid_keys: Option<&NullBuffer>,limit: usize,- offset: MapOffset,+ offset: ProbeOffset,input_indices: &mut Vec<u32>,match_indices: &mut Vec<u64>,- ) -> Option<MapOffset> {+ ) -> Option<ProbeOffset> {// Flatten the dequelet next: Vec<u64> = self.next.iter().copied().collect();get_matched_indices_with_limit_offset::<u64>(diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex c64d46e308..ef62dcf2f7 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -45,6 +45,7 @@ pub use super::join_filter::JoinFilter;/// [API policy]: https://datafusion.apache.org/contributor-guide/api-health.html#internal-public-apis#[doc(hidden)]pub use super::join_hash_map::JoinHashMapType;+pub use super::join_hash_map::ProbeOffset;pub use crate::joins::{JoinOn, JoinOnRef};use arrow::array::{@@ -2774,6 +2775,7 @@ mod tests {use datafusion_common::{ScalarValue, SplitPoint, arrow_datafusion_err, arrow_err};use datafusion_physical_expr::PhysicalSortExpr;+ use crate::joins::join_hash_map::ProbeOffset::AtRow;use rstest::rstest;fn assert_u32_values(array: &UInt32Array, expected: &[u32]) {@@ -2826,7 +2828,7 @@ mod tests {hashes_buffer,None,8192,- (0, None),+ AtRow { row: 0 },&mut input_indices,&mut match_indices,);@@ -2907,7 +2909,7 @@ mod tests {)?;let (mut input_indices, mut match_indices, mut pairs) =(vec![], vec![], vec![]);- let mut offset = Some((0, None));+ let mut offset = Some(AtRow { row: 0 });while let Some(current) = offset {offset = map.get_matched_indices_with_limit_offset(&probe_hashes,