Apache DataFusion SQL Query Engine
refactor: use `ProbeCompletion` in `NestedLoopJoinExec` (#26717)
Yu-Chuan Hung5ed258f6043b
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 135c1c4482..dd584d44f4 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -33,7 +33,6 @@ use crate::joins::Map;use crate::joins::array_map::ArrayMap;use crate::joins::hash_join::inlist_builder::build_struct_inlist_values;use crate::joins::hash_join::not_in::{NotIn, NotInBuild};-use crate::joins::hash_join::probe_completion::{ProbeCompletion, ProbeSideSummary};use crate::joins::hash_join::shared_bounds::{ColumnBounds, PartitionBounds, PushdownStrategy, SharedBuildAccumulator,};@@ -42,6 +41,7 @@ use crate::joins::hash_join::stream::{};use crate::joins::join_hash_map::{JoinHashMapU32, JoinHashMapU64};use crate::joins::key_range_bitmap::KeyRangeBitmap;+use crate::joins::probe_completion::{Completion, ProbeCompletion};use crate::joins::utils::{OnceAsync, OnceFut, asymmetric_join_output_partitioning, emits_unmatched_left_rows,is_existence_join, reorder_output_after_swap, swap_join_projection, update_hash,@@ -299,19 +299,21 @@ impl JoinLeftData {/// Whether some probe partition has already recorded a NULL join key.////// Only an early-exit hint for the probe phase: it may lag behind sibling- /// partitions. Final-stage decisions must use the [`ProbeSideSummary`]+ /// partitions. Final-stage decisions must use the+ /// [`ProbeSideSummary`](crate::joins::probe_completion::ProbeSideSummary)/// returned by [`Self::report_probe_completed`] instead.pub(super) fn probe_side_has_null_hint(&self) -> bool {self.probe_completion.saw_null_key_hint()}- /// Marks this probe partition as finished, returning `Some` for the last- /// one together with what every partition saw.+ /// Marks this probe partition as finished, returning+ /// [`Completion::Emit`] for the last one together with what every+ /// partition saw.////// The summary is reachable only through this call, which is what stops/// the final stage from reading the shared flags before its own/// decrement. See [`ProbeCompletion`] for why that ordering matters.- pub(super) fn report_probe_completed(&self) -> Option<ProbeSideSummary> {+ pub(super) fn report_probe_completed(&self) -> Completion {self.probe_completion.report_completed()}}diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rsindex 9bfd7ecbf6..bcb257fcff 100644--- a/datafusion/physical-plan/src/joins/hash_join/mod.rs+++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs@@ -31,6 +31,5 @@ mod exec;mod inlist_builder;mod not_in;mod partitioned_hash_eval;-mod probe_completion;mod shared_bounds;mod stream;diff --git a/datafusion/physical-plan/src/joins/hash_join/not_in.rs b/datafusion/physical-plan/src/joins/hash_join/not_in.rsindex eaebd75387..268160b421 100644--- a/datafusion/physical-plan/src/joins/hash_join/not_in.rs+++ b/datafusion/physical-plan/src/joins/hash_join/not_in.rs@@ -54,10 +54,10 @@ use datafusion_physical_expr_common::datum::apply_cmp;use crate::joins::PartitionMode;use crate::joins::SharedBitmapBuilder;use crate::joins::hash_join::exec::{JoinLeftData, new_join_hashmap};-use crate::joins::hash_join::probe_completion::ProbeSideSummary;use crate::joins::hash_join::stream::lookup_join_hashmap;use crate::joins::join_hash_map::JoinHashMapType;use crate::joins::join_hash_map::ProbeOffset::AtRow;+use crate::joins::probe_completion::ProbeSideSummary;use crate::joins::utils::{BuildProbeJoinMetrics, JoinFilter, apply_join_filter_to_indices, update_hash,};diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 2d317fbc6c..631fcaaffd 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -28,12 +28,12 @@ use crate::joins::Map;use crate::joins::PartitionMode;use crate::joins::hash_join::exec::JoinLeftData;use crate::joins::hash_join::not_in::NotInProbe;-use crate::joins::hash_join::probe_completion::ProbeSideSummary;use crate::joins::hash_join::shared_bounds::{PartitionBounds, PartitionBuildData, SharedBuildAccumulator,};use crate::joins::join_hash_map::ProbeOffset;use crate::joins::join_hash_map::ProbeOffset::AtRow;+use crate::joins::probe_completion::{Completion, ProbeSideSummary};use crate::joins::utils::{JoinKeyComparator, OnceFut, equal_rows_arr_with_normalized_left, matchable_join_keys,};@@ -1077,7 +1077,9 @@ impl HashJoinStream {// and only it receives the shared probe-side summary (see// `JoinLeftData::report_probe_completed` for why the flags are not// readable any other way here).- let Some(probe_summary) = build_side.left_data.report_probe_completed() else {+ let Completion::Emit(probe_summary) =+ build_side.left_data.report_probe_completed()+ else {timer.done();self.state = HashJoinStreamState::Completed;return Ok(StatefulStreamResult::Continue);diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex 259e49fd62..c9d362fd00 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -47,6 +47,7 @@ mod hash_join;mod logical_batch;mod nested_loop_join;mod piecewise_merge_join;+mod probe_completion;#[cfg(feature = "proto")]mod proto;mod sort_merge_join;diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex ea1127c0b0..fecbd887de 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -20,7 +20,6 @@use std::fmt::Formatter;use std::ops::{BitOr, ControlFlow};use std::sync::Arc;-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};use std::task::Poll;use super::utils::{@@ -34,6 +33,7 @@ use crate::filter_pushdown::{};use crate::joins::SharedBitmapBuilder;use crate::joins::logical_batch::{BatchRow, LogicalBatch, RowIndices};+use crate::joins::probe_completion::{Completion, ProbeCompletion};use crate::joins::utils::{BuildProbeJoinMetrics, ColumnIndex, JoinFilter, OnceAsync, OnceFut,boolean_mask_from_filter, build_join_schema, check_join_is_valid,@@ -1158,8 +1158,6 @@ pub(crate) struct JoinLeftData {batch: LogicalBatch,/// Shared bitmap builder for visited left indicesbitmap: SharedBitmapBuilder,- /// Counter of running probe-threads, potentially able to update `bitmap`- probe_threads_counter: AtomicUsize,/// Memory reservation for tracking batch and bitmap/// Cleared on `JoinLeftData` drop/// reservation is cleared on Drop@@ -1171,13 +1169,11 @@ impl JoinLeftData {pub(crate) fn new(batch: LogicalBatch,bitmap: SharedBitmapBuilder,- probe_threads_counter: AtomicUsize,reservation: MemoryReservation,) -> Self {Self {batch,bitmap,- probe_threads_counter,reservation,}}@@ -1189,12 +1185,6 @@ impl JoinLeftData {pub(crate) fn bitmap(&self) -> &SharedBitmapBuilder {&self.bitmap}-- /// Decrements counter of running threads, and returns `true`- /// if caller is the last running thread- pub(crate) fn report_probe_completed(&self) -> bool {- self.probe_threads_counter.fetch_sub(1, Ordering::Relaxed) == 1- }}/// Asynchronously collect the left input in a single pass over the stream.@@ -1301,12 +1291,14 @@ async fn collect_left_input(BooleanBufferBuilder::new(0)};- Ok(LeftLoad::InMemory(Arc::new(JoinLeftData::new(- buffered_build_batch,- Mutex::new(visited_left_side),- AtomicUsize::new(probe_threads_count),- reservation,- ))))+ Ok(LeftLoad::InMemory {+ left_data: Arc::new(JoinLeftData::new(+ buffered_build_batch,+ Mutex::new(visited_left_side),+ reservation,+ )),+ probe_completion: Arc::new(ProbeCompletion::new(probe_threads_count)),+ })}/// A left side with no rows needs no spill file, so it stays on the in-memory path.@@ -1331,12 +1323,14 @@ fn left_load_from_spill(LeftLoad::Spilled(left_spill)}// No rows means no bitmap either, whatever the join type.- None => LeftLoad::InMemory(Arc::new(JoinLeftData::new(- LogicalBatch::new_empty(schema),- Mutex::new(BooleanBufferBuilder::new(0)),- AtomicUsize::new(probe_threads_count),- reservation,- ))),+ None => LeftLoad::InMemory {+ left_data: Arc::new(JoinLeftData::new(+ LogicalBatch::new_empty(schema),+ Mutex::new(BooleanBufferBuilder::new(0)),+ reservation,+ )),+ probe_completion: Arc::new(ProbeCompletion::new(probe_threads_count)),+ },}}@@ -1414,7 +1408,7 @@ enum NLJState {/// Entered exactly once per stream, when it has finished probing the whole/// left side: the probe (right) side is exhausted or, in memory-limited/// mode, every left chunk has been probed. This state owns the single- /// `report_probe_completed` call that decrements the shared probe-threads+ /// probe-completion report that decrements the shared probe-threads/// counter, and records in `is_unmatched_left_emitter` whether this stream/// is the one responsible for emitting unmatched-left rows. Splitting this/// decision out of `EmitLeftUnmatched` makes "decrement exactly once" a@@ -1432,7 +1426,12 @@ enum NLJState {/// Outcome of the single pass over the left (build) input.pub(crate) enum LeftLoad {/// The left side fit the memory budget and is buffered as one batch.- InMemory(Arc<JoinLeftData>),+ InMemory {+ /// The buffered left side+ left_data: Arc<JoinLeftData>,+ /// Partitions that are still probing `left_data`+ probe_completion: Arc<ProbeCompletion>,+ },/// The budget ran out, so the left side was spilled during that same pass. Every partition/// shares this handle, and the chunks that are read back from the file.Spilled(Arc<LeftSpillData>),@@ -1448,10 +1447,10 @@ struct SpilledLeftFile {/// The spilled left side, shared by every output partition.///-/// This is the memory-limited counterpart of [`JoinLeftData`]: the rows live in-/// a spill file instead of memory, but the visited bitmap and the probe-threads-/// counter cover the whole left side and are shared by all partitions in the-/// same way.+/// This is the memory-limited counterpart of [`LeftLoad::InMemory`]: the rows+/// live in a spill file instead of memory, but the visited bitmap and the+/// probe-threads counter cover the whole left side and are shared by all+/// partitions in the same way.////// The rows come back one chunk at a time (see [`LeftChunkBarrier`]), while the/// bitmap spans all of them: bits are addressed by the row's position in the@@ -1473,12 +1472,9 @@ pub(crate) struct LeftSpillData {/// Visited bitmap over every row of `spill_file`. Empty when the join type/// does not need it.visited: SharedBitmapBuilder,- /// Counter of partitions that have neither finished probing every chunk- /// nor gone away- probe_threads_counter: AtomicUsize,- /// Set when a partition went away before it finished probing. The final- /// left rows are then not emitted, as on the in-memory path.- incomplete: AtomicBool,+ /// Partitions that have neither finished probing every chunk nor gone+ /// away, and whether any went away before it finished+ probe_completion: ProbeCompletion,/// Memory reservation for `visited`reservation: MemoryReservation,}@@ -1513,8 +1509,7 @@ impl LeftSpillData {num_rows,reader: Arc::new(Mutex::new(None)),visited: Mutex::new(visited),- probe_threads_counter: AtomicUsize::new(probe_threads_count),- incomplete: AtomicBool::new(false),+ probe_completion: ProbeCompletion::new(probe_threads_count),reservation,}}@@ -1563,15 +1558,15 @@ impl LeftSpillData {/// running thread and no partition went away unfinished, it is the emitter/// and gets the complete visited bitmap, which it must account for.fn report_probe_completed(&self) -> Option<BooleanBuffer> {- if self.probe_threads_counter.fetch_sub(1, Ordering::AcqRel) != 1 {- return None;- }- let visited = self.take_visited();- if self.incomplete.load(Ordering::Acquire) {+ match self.probe_completion.report_completed() {+ Completion::NotLast => None,+ Completion::Emit(_) => Some(self.take_visited()),// Nobody will emit, so nobody needs the bitmap- return None;+ Completion::Abandoned => {+ drop(self.take_visited());+ None+ }}- Some(visited)}/// `count` partitions went away before they finished probing.@@ -1581,15 +1576,7 @@ impl LeftSpillData {/// the last to finish or go away, while the plan may live on. But the final/// left rows are then not emitted.fn depart_unfinished(&self, count: usize) {- if count == 0 {- return;- }- self.incomplete.store(true, Ordering::Release);- if self- .probe_threads_counter- .fetch_sub(count, Ordering::AcqRel)- == count- {+ if self.probe_completion.abandon(count) {drop(self.take_visited());}}@@ -2186,6 +2173,10 @@ pub(crate) struct NestedLoopJoinStream {// -----------------/// The current buffered left data to joinbuffered_left_data: Option<Arc<JoinLeftData>>,+ /// Probe partitions still running over the in-memory left side, shared by+ /// all of them. `None` on the spill path, which reports on+ /// [`LeftSpillData`] instead.+ in_memory_probe_completion: Option<Arc<ProbeCompletion>>,/// Index into the left buffered batch. Used in `ProbeRight` stateleft_probe_idx: usize,/// Index into the left buffered batch. Used in `EmitLeftUnmatched` state@@ -2209,8 +2200,9 @@ pub(crate) struct NestedLoopJoinStream {/// Whether this stream is the one responsible for emitting unmatched-left/// rows. Set in the [`NLJState::ProbeEnd`] state, which is entered exactly- /// once per stream and owns the single `report_probe_completed` call (on- /// [`JoinLeftData`], or on [`LeftSpillData`] in memory-limited mode): the+ /// once per stream and owns the single probe-completion report (on+ /// [`LeftLoad::InMemory`]'s [`ProbeCompletion`], or on [`LeftSpillData`] in+ /// memory-limited mode): the/// stream that drives the/// shared probe-threads counter to zero (the last to finish probing) becomes/// the emitter. Because the decrement happens once in `ProbeEnd` rather than@@ -2545,6 +2537,7 @@ impl NestedLoopJoinStream {column_indices,left_data,metrics,+ in_memory_probe_completion: None,buffered_left_data: None,output_buffer: Box::new(BatchCoalescer::new(schema, batch_size)),batch_size,@@ -2660,7 +2653,12 @@ impl NestedLoopJoinStream {// Standard path: use OnceFutmatch self.left_data.get_shared(cx) {Poll::Ready(Ok(load)) => match load.as_ref() {- LeftLoad::InMemory(left_data) => {+ LeftLoad::InMemory {+ left_data,+ probe_completion,+ } => {+ self.in_memory_probe_completion =+ Some(Arc::clone(probe_completion));self.buffered_left_data = Some(Arc::clone(left_data));self.left_exhausted = true;self.state = NLJState::FetchingRight;@@ -2749,13 +2747,11 @@ impl NestedLoopJoinStream {};// This `JoinLeftData` is private to the partition: it shares the- // chunk's rows but has its own bitmap, so its probe-threads counter is- // not used. Probe completion is reported once for the whole left side,- // on `LeftSpillData`.+ // chunk's rows but has its own bitmap. Probe completion is reported+ // once for the whole left side, on `LeftSpillData`.self.buffered_left_data = Some(Arc::new(JoinLeftData::new(batch,Mutex::new(visited_left_side),- AtomicUsize::new(1),active.chunk_reservation.take(),)));active.current_chunk = Some(chunk);@@ -3006,7 +3002,7 @@ impl NestedLoopJoinStream {////// Entered exactly once per stream, when it has probed the whole left side./// This is the single place that decrements the shared probe-threads counter- /// via `report_probe_completed`: the stream that drives the+ /// via [`ProbeCompletion::report_completed`]: the stream that drives the/// counter to zero (the last to finish probing) is the one responsible for/// emitting unmatched-left rows, recorded in `is_unmatched_left_emitter`.///@@ -3039,8 +3035,13 @@ impl NestedLoopJoinStream {None => false,}} else {- match self.get_left_data() {- Ok(left_data) => left_data.report_probe_completed(),+ match self.get_in_memory_probe_completion() {+ Ok(probe_completion) => match probe_completion.report_completed() {+ Completion::NotLast => false,+ // Nothing abandons the in-memory left side+ Completion::Abandoned => false,+ Completion::Emit(_) => true,+ },Err(e) => return ControlFlow::Break(Poll::Ready(Some(Err(e)))),}};@@ -3067,6 +3068,7 @@ impl NestedLoopJoinStream {Ok(false) => match self.output_buffer.finish_buffered_batch() {Ok(()) => {self.buffered_left_data = None;+ self.in_memory_probe_completion = None;self.state = NLJState::Done;ControlFlow::Continue(())}@@ -3744,6 +3746,12 @@ impl NestedLoopJoinStream {.ok_or_else(|| internal_datafusion_err!("LeftData should be available"))}+ fn get_in_memory_probe_completion(&self) -> Result<&Arc<ProbeCompletion>> {+ self.in_memory_probe_completion.as_ref().ok_or_else(|| {+ internal_datafusion_err!("In-memory probe completion should be available")+ })+ }+/// Flush the `output_buffer` if there are batches ready to output/// None if no result batch ready.fn maybe_flush_ready_batch(&mut self) -> Option<Poll<Option<Result<RecordBatch>>>> {@@ -4261,6 +4269,7 @@ fn build_unmatched_batch(#[cfg(test)]pub(crate) mod tests {use std::pin::Pin;+ use std::sync::atomic::{AtomicUsize, Ordering};use std::time::Duration;use super::*;@@ -5304,7 +5313,6 @@ pub(crate) mod tests {let left_data = JoinLeftData::new(left.into(),Mutex::new(BooleanBufferBuilder::new(0)),- AtomicUsize::new(1),MemoryConsumer::new("test").register(ctx.memory_pool()),);let mut stream = NestedLoopJoinStream::new(@@ -6776,10 +6784,6 @@ pub(crate) mod tests {.as_ref().expect("the left side spilled"),);- assert!(- left_spill.probe_threads_counter.load(Ordering::Acquire) >= 1,- "the replaying partition has not reported"- );drop(streams.remove(replaying));let survivor = streams.pop().unwrap();@@ -6788,13 +6792,17 @@ pub(crate) mod tests {.await.expect("the remaining partition stalled")?;// The survivor emits its own unmatched right rows, but the final left- // rows are not emitted- assert_eq!(count_final_left_rows(JoinType::Full, &batches), 0);- assert!(left_spill.incomplete.load(Ordering::Acquire));- assert_eq!(left_spill.probe_threads_counter.load(Ordering::Acquire), 0);+ // rows are not emitted. Had the replaying partition already reported,+ // they would be.+ assert_eq!(+ count_final_left_rows(JoinType::Full, &batches),+ 0,+ "the replaying partition must not have reported before it was dropped"+ );drop(left_spill);- // Nothing stays reserved, although the plan is still alive+ // Nothing stays reserved, although the plan is still alive. The bitmap+ // is only released once every partition has finished or gone away.assert_eq!(task_ctx.memory_pool().reserved(), 0);Ok(())}diff --git a/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs b/datafusion/physical-plan/src/joins/probe_completion.rssimilarity index 67%rename from datafusion/physical-plan/src/joins/hash_join/probe_completion.rsrename to datafusion/physical-plan/src/joins/probe_completion.rsindex 11fb72d352..bf30ac9365 100644--- a/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs+++ b/datafusion/physical-plan/src/joins/probe_completion.rs@@ -15,18 +15,23 @@// specific language governing permissions and limitations// under the License.-//! Completion protocol shared by the probe partitions of a hash join.+//! Completion protocol shared by the probe partitions of a join.//!-//! The probe partitions of one `HashJoinExec` run concurrently over a shared-//! build side. Two facts have to cross partition boundaries:+//! The probe partitions of one `HashJoinExec` or `NestedLoopJoinExec` run+//! concurrently over a shared build side. Some facts have to cross partition+//! boundaries://!//! - which partition emits the build-side rows in the final stage, namely the-//! last one to finish, and+//! last one to finish,+//! - whether any partition stopped before it finished, in which case its+//! matches are missing and nobody may emit, and//! - what the partitions collectively saw, which the null-aware `NOT IN` rules-//! need: did any partition see a row, and did any see a NULL join key.+//! of the hash join need: did any partition see a row, and did any see a+//! NULL join key.//!-//! [`ProbeCompletion`] owns both, because reading the second without ordering-//! it against the first produced wrong results. A partition that read the NULL+//! [`ProbeCompletion`] owns all of them, because reading the facts without+//! ordering them against the choice of the last partition produced wrong+//! results. A partition that read the NULL//! flag *before* decrementing the counter could observe `false`, have a//! sibling record a NULL and finish, then become the last partition itself and//! emit build rows that `NOT IN` must suppress.@@ -64,6 +69,17 @@ pub(super) struct ProbeSideSummary {pub(super) has_null: bool,}+/// Describes the state of a probe partition after it has finished its work.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum Completion {+ /// Not the last partition to finish.+ NotLast,+ /// Last partition to finish and emit a summary.+ Emit(ProbeSideSummary),+ /// The last partition with some previous partition abandoned.+ Abandoned,+}+/// Defines [`ProbeCompletion`] over the given atomic types.////// [loom] can only explore its own atomics, so the protocol is written once@@ -82,7 +98,7 @@ macro_rules! define_probe_completion {type Flag = $flag;/// Tracks how many probe partitions are still running, together with- /// the null-aware facts they contribute.+ /// the facts they contribute.////// See the [module docs](self) for the invariant this upholds.#[derive(Debug)]@@ -93,6 +109,8 @@ macro_rules! define_probe_completion {saw_row: Flag,/// Set once any probe partition has seen a NULL join key.saw_null_key: Flag,+ /// Set once any probe partition has stopped before it finished.+ abandoned: Flag,}impl ProbeCompletion {@@ -102,6 +120,7 @@ macro_rules! define_probe_completion {running: Counter::new(probe_threads),saw_row: Flag::new(false),saw_null_key: Flag::new(false),+ abandoned: Flag::new(false),}}@@ -129,20 +148,45 @@ macro_rules! define_probe_completion {self.saw_null_key.load(Ordering::Relaxed)}- /// Marks the calling partition finished, returning `Some` only for- /// the last one, together with what all partitions saw.+ /// Marks the calling partition finished. Only the last one gets+ /// more than [`Completion::NotLast`]: [`Completion::Emit`] with+ /// what all partitions saw, or [`Completion::Abandoned`] if some+ /// partition stopped early (see [`Self::abandon`]).////// The `AcqRel` decrement publishes this partition's own stores/// and acquires those of the partitions that finished earlier, so/// the summary handed to the last caller is complete.- pub(super) fn report_completed(&self) -> Option<ProbeSideSummary> {+ pub(super) fn report_completed(&self) -> Completion {// `fetch_sub` returns the value from *before* the decrement,// so `1` means this call is the one that took it to zero.let was_last = self.running.fetch_sub(1, Ordering::AcqRel) == 1;- was_last.then(|| ProbeSideSummary {- non_empty: self.saw_row.load(Ordering::Relaxed),- has_null: self.saw_null_key.load(Ordering::Relaxed),- })+ match (was_last, self.abandoned.load(Ordering::Relaxed)) {+ (false, _) => Completion::NotLast,+ (true, false) => {+ let summary = ProbeSideSummary {+ non_empty: self.saw_row.load(Ordering::Relaxed),+ has_null: self.saw_null_key.load(Ordering::Relaxed),+ };+ Completion::Emit(summary)+ }+ (true, true) => Completion::Abandoned,+ }+ }++ /// Marks `count` probe partitions as having stopped before they finished.+ ///+ /// They still count down, so the partitions that remain are not left+ /// waiting. But their matches are missing, so the build-side rows must not+ /// be emitted: the last partition is not handed a summary.+ ///+ /// `Relaxed` suffices for the flag, as in [`Self::record_batch`]: the+ /// `AcqRel` decrement publishes it.+ pub(super) fn abandon(&self, count: usize) -> bool {+ if count == 0 {+ return false;+ }+ self.abandoned.store(true, Ordering::Relaxed);+ self.running.fetch_sub(count, Ordering::AcqRel) == count}}};@@ -166,14 +210,14 @@ mod tests {let completion = ProbeCompletion::new(3);completion.record_batch(true, false);- assert_eq!(completion.report_completed(), None);+ assert_eq!(completion.report_completed(), Completion::NotLast);completion.record_batch(true, true);- assert_eq!(completion.report_completed(), None);+ assert_eq!(completion.report_completed(), Completion::NotLast);assert_eq!(completion.report_completed(),- Some(ProbeSideSummary {+ Completion::Emit(ProbeSideSummary {non_empty: true,has_null: true,})@@ -188,7 +232,7 @@ mod tests {assert_eq!(completion.report_completed(),- Some(ProbeSideSummary {+ Completion::Emit(ProbeSideSummary {non_empty: false,has_null: false,})@@ -208,6 +252,22 @@ mod tests {completion.record_batch(true, true);assert!(completion.saw_null_key_hint());}++ /// Abandoning no partitions changes nothing. A caller may report a count of+ /// zero, for example when no partition went away before the left side+ /// spilled, and that must not suppress the build-side rows.+ #[test]+ fn abandoning_no_partitions_still_emits() {+ let completion = ProbeCompletion::new(1);+ assert!(!completion.abandon(0));+ assert_eq!(+ completion.report_completed(),+ Completion::Emit(ProbeSideSummary {+ non_empty: false,+ has_null: false,+ })+ );+ }}/// Model-checked concurrency tests for the protocol.@@ -215,7 +275,7 @@ mod tests {/// These instantiate the protocol over loom's atomics (see the/// [module docs](self)) and let loom enumerate the interleavings and store/// visibility the memory model allows. They fail if the decrement in-/// `report_completed` is weakened to `Relaxed`.+/// `report_completed` or `abandon` is weakened to `Relaxed`.////// Compiled only under `--cfg datafusion_loom`, which is also what pulls the `loom`/// dev-dependency into the build (see `datafusion-physical-plan`'s@@ -226,6 +286,7 @@ mod tests {/// ```#[cfg(all(test, datafusion_loom))]mod loom_tests {+ use super::Completion;use super::ProbeSideSummary;use loom::sync::Arc;use std::sync::atomic::Ordering;@@ -258,9 +319,15 @@ mod loom_tests {let recorded = recorder.join().unwrap();let summary = match (recorded, observer) {- (Some(summary), None) | (None, Some(summary)) => summary,- (Some(_), Some(_)) => panic!("two partitions both finished last"),- (None, None) => panic!("no partition finished last"),+ (Completion::Emit(summary), Completion::NotLast)+ | (Completion::NotLast, Completion::Emit(summary)) => summary,+ (Completion::Emit(_), Completion::Emit(_)) => {+ panic!("two partitions both finished last")+ }+ (Completion::NotLast, Completion::NotLast) => {+ panic!("no partition finished last")+ }+ other => panic!("unexpected outcome: {other:?}"),};assert!(@@ -295,7 +362,11 @@ mod loom_tests {let here = completion.report_completed();let there = other.join().unwrap();- let summary = here.or(there).expect("some partition must finish last");+ let summary = match (here, there) {+ (Completion::Emit(summary), Completion::NotLast)+ | (Completion::NotLast, Completion::Emit(summary)) => summary,+ other => panic!("some partition must finish last: {other:?}"),+ };assert_eq!(summary,ProbeSideSummary {@@ -316,10 +387,12 @@ mod loom_tests {let other = {let completion = Arc::clone(&completion);- loom::thread::spawn(move || completion.report_completed().is_some())+ loom::thread::spawn(move || {+ completion.report_completed() != Completion::NotLast+ })};- let here = completion.report_completed().is_some();+ let here = completion.report_completed() != Completion::NotLast;let there = other.join().unwrap();assert!(here ^ there, "exactly one partition must finish last");@@ -347,4 +420,30 @@ mod loom_tests {assert!(!here && !there, "the hint reported an unrecorded NULL");});}++ /// A partition that stops early still counts down, so exactly one+ /// partition finishes last. But its matches are missing, so no partition+ /// may emit, whichever of the two finishes last.+ #[test]+ fn an_abandoned_partition_counts_down_but_suppresses_the_emit() {+ loom::model(|| {+ let completion = Arc::new(ProbeCompletion::new(2));++ let quitter = {+ let completion = Arc::clone(&completion);+ loom::thread::spawn(move || completion.abandon(1))+ };++ let here = completion.report_completed();+ let quitter_was_last = quitter.join().unwrap();++ match (here, quitter_was_last) {+ // This partition finished last and saw the abandon+ (Completion::Abandoned, false) => {}+ // The quitter finished last and releases the shared state+ (Completion::NotLast, true) => {}+ other => panic!("unexpected outcome: {other:?}"),+ }+ });+ }}