Skip to content

Apache DataFusion SQL Query Engine

refactor: use `ProbeCompletion` in `NestedLoopJoinExec` (#26717)

Yu-Chuan Hung5ed258f6043b
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 135c1c4482..dd584d44f4 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -33,7 +33,6 @@ use crate::joins::Map;
use crate::joins::array_map::ArrayMap;
use crate::joins::hash_join::inlist_builder::build_struct_inlist_values;
use crate::joins::hash_join::not_in::{NotIn, NotInBuild};
-use crate::joins::hash_join::probe_completion::{ProbeCompletion, ProbeSideSummary};
use crate::joins::hash_join::shared_bounds::{
ColumnBounds, PartitionBounds, PushdownStrategy, SharedBuildAccumulator,
};
@@ -42,6 +41,7 @@ use crate::joins::hash_join::stream::{
};
use crate::joins::join_hash_map::{JoinHashMapU32, JoinHashMapU64};
use crate::joins::key_range_bitmap::KeyRangeBitmap;
+use crate::joins::probe_completion::{Completion, ProbeCompletion};
use crate::joins::utils::{
OnceAsync, OnceFut, asymmetric_join_output_partitioning, emits_unmatched_left_rows,
is_existence_join, reorder_output_after_swap, swap_join_projection, update_hash,
@@ -299,19 +299,21 @@ impl JoinLeftData {
/// Whether some probe partition has already recorded a NULL join key.
///
/// Only an early-exit hint for the probe phase: it may lag behind sibling
- /// partitions. Final-stage decisions must use the [`ProbeSideSummary`]
+ /// partitions. Final-stage decisions must use the
+ /// [`ProbeSideSummary`](crate::joins::probe_completion::ProbeSideSummary)
/// returned by [`Self::report_probe_completed`] instead.
pub(super) fn probe_side_has_null_hint(&self) -> bool {
self.probe_completion.saw_null_key_hint()
}
- /// Marks this probe partition as finished, returning `Some` for the last
- /// one together with what every partition saw.
+ /// Marks this probe partition as finished, returning
+ /// [`Completion::Emit`] for the last one together with what every
+ /// partition saw.
///
/// The summary is reachable only through this call, which is what stops
/// the final stage from reading the shared flags before its own
/// decrement. See [`ProbeCompletion`] for why that ordering matters.
- pub(super) fn report_probe_completed(&self) -> Option<ProbeSideSummary> {
+ pub(super) fn report_probe_completed(&self) -> Completion {
self.probe_completion.report_completed()
}
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rs
index 9bfd7ecbf6..bcb257fcff 100644
--- a/datafusion/physical-plan/src/joins/hash_join/mod.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs
@@ -31,6 +31,5 @@ mod exec;
mod inlist_builder;
mod not_in;
mod partitioned_hash_eval;
-mod probe_completion;
mod shared_bounds;
mod stream;
diff --git a/datafusion/physical-plan/src/joins/hash_join/not_in.rs b/datafusion/physical-plan/src/joins/hash_join/not_in.rs
index eaebd75387..268160b421 100644
--- a/datafusion/physical-plan/src/joins/hash_join/not_in.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/not_in.rs
@@ -54,10 +54,10 @@ use datafusion_physical_expr_common::datum::apply_cmp;
use crate::joins::PartitionMode;
use crate::joins::SharedBitmapBuilder;
use crate::joins::hash_join::exec::{JoinLeftData, new_join_hashmap};
-use crate::joins::hash_join::probe_completion::ProbeSideSummary;
use crate::joins::hash_join::stream::lookup_join_hashmap;
use crate::joins::join_hash_map::JoinHashMapType;
use crate::joins::join_hash_map::ProbeOffset::AtRow;
+use crate::joins::probe_completion::ProbeSideSummary;
use crate::joins::utils::{
BuildProbeJoinMetrics, JoinFilter, apply_join_filter_to_indices, update_hash,
};
diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs
index 2d317fbc6c..631fcaaffd 100644
--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs
@@ -28,12 +28,12 @@ use crate::joins::Map;
use crate::joins::PartitionMode;
use crate::joins::hash_join::exec::JoinLeftData;
use crate::joins::hash_join::not_in::NotInProbe;
-use crate::joins::hash_join::probe_completion::ProbeSideSummary;
use crate::joins::hash_join::shared_bounds::{
PartitionBounds, PartitionBuildData, SharedBuildAccumulator,
};
use crate::joins::join_hash_map::ProbeOffset;
use crate::joins::join_hash_map::ProbeOffset::AtRow;
+use crate::joins::probe_completion::{Completion, ProbeSideSummary};
use crate::joins::utils::{
JoinKeyComparator, OnceFut, equal_rows_arr_with_normalized_left, matchable_join_keys,
};
@@ -1077,7 +1077,9 @@ impl HashJoinStream {
// and only it receives the shared probe-side summary (see
// `JoinLeftData::report_probe_completed` for why the flags are not
// readable any other way here).
- let Some(probe_summary) = build_side.left_data.report_probe_completed() else {
+ let Completion::Emit(probe_summary) =
+ build_side.left_data.report_probe_completed()
+ else {
timer.done();
self.state = HashJoinStreamState::Completed;
return Ok(StatefulStreamResult::Continue);
diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rs
index 259e49fd62..c9d362fd00 100644
--- a/datafusion/physical-plan/src/joins/mod.rs
+++ b/datafusion/physical-plan/src/joins/mod.rs
@@ -47,6 +47,7 @@ mod hash_join;
mod logical_batch;
mod nested_loop_join;
mod piecewise_merge_join;
+mod probe_completion;
#[cfg(feature = "proto")]
mod proto;
mod sort_merge_join;
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index ea1127c0b0..fecbd887de 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -20,7 +20,6 @@
use std::fmt::Formatter;
use std::ops::{BitOr, ControlFlow};
use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::task::Poll;
use super::utils::{
@@ -34,6 +33,7 @@ use crate::filter_pushdown::{
};
use crate::joins::SharedBitmapBuilder;
use crate::joins::logical_batch::{BatchRow, LogicalBatch, RowIndices};
+use crate::joins::probe_completion::{Completion, ProbeCompletion};
use crate::joins::utils::{
BuildProbeJoinMetrics, ColumnIndex, JoinFilter, OnceAsync, OnceFut,
boolean_mask_from_filter, build_join_schema, check_join_is_valid,
@@ -1158,8 +1158,6 @@ pub(crate) struct JoinLeftData {
batch: LogicalBatch,
/// Shared bitmap builder for visited left indices
bitmap: SharedBitmapBuilder,
- /// Counter of running probe-threads, potentially able to update `bitmap`
- probe_threads_counter: AtomicUsize,
/// Memory reservation for tracking batch and bitmap
/// Cleared on `JoinLeftData` drop
/// reservation is cleared on Drop
@@ -1171,13 +1169,11 @@ impl JoinLeftData {
pub(crate) fn new(
batch: LogicalBatch,
bitmap: SharedBitmapBuilder,
- probe_threads_counter: AtomicUsize,
reservation: MemoryReservation,
) -> Self {
Self {
batch,
bitmap,
- probe_threads_counter,
reservation,
}
}
@@ -1189,12 +1185,6 @@ impl JoinLeftData {
pub(crate) fn bitmap(&self) -> &SharedBitmapBuilder {
&self.bitmap
}
-
- /// Decrements counter of running threads, and returns `true`
- /// if caller is the last running thread
- pub(crate) fn report_probe_completed(&self) -> bool {
- self.probe_threads_counter.fetch_sub(1, Ordering::Relaxed) == 1
- }
}
/// Asynchronously collect the left input in a single pass over the stream.
@@ -1301,12 +1291,14 @@ async fn collect_left_input(
BooleanBufferBuilder::new(0)
};
- Ok(LeftLoad::InMemory(Arc::new(JoinLeftData::new(
- buffered_build_batch,
- Mutex::new(visited_left_side),
- AtomicUsize::new(probe_threads_count),
- reservation,
- ))))
+ Ok(LeftLoad::InMemory {
+ left_data: Arc::new(JoinLeftData::new(
+ buffered_build_batch,
+ Mutex::new(visited_left_side),
+ reservation,
+ )),
+ probe_completion: Arc::new(ProbeCompletion::new(probe_threads_count)),
+ })
}
/// A left side with no rows needs no spill file, so it stays on the in-memory path.
@@ -1331,12 +1323,14 @@ fn left_load_from_spill(
LeftLoad::Spilled(left_spill)
}
// No rows means no bitmap either, whatever the join type.
- None => LeftLoad::InMemory(Arc::new(JoinLeftData::new(
- LogicalBatch::new_empty(schema),
- Mutex::new(BooleanBufferBuilder::new(0)),
- AtomicUsize::new(probe_threads_count),
- reservation,
- ))),
+ None => LeftLoad::InMemory {
+ left_data: Arc::new(JoinLeftData::new(
+ LogicalBatch::new_empty(schema),
+ Mutex::new(BooleanBufferBuilder::new(0)),
+ reservation,
+ )),
+ probe_completion: Arc::new(ProbeCompletion::new(probe_threads_count)),
+ },
}
}
@@ -1414,7 +1408,7 @@ enum NLJState {
/// Entered exactly once per stream, when it has finished probing the whole
/// left side: the probe (right) side is exhausted or, in memory-limited
/// mode, every left chunk has been probed. This state owns the single
- /// `report_probe_completed` call that decrements the shared probe-threads
+ /// probe-completion report that decrements the shared probe-threads
/// counter, and records in `is_unmatched_left_emitter` whether this stream
/// is the one responsible for emitting unmatched-left rows. Splitting this
/// decision out of `EmitLeftUnmatched` makes "decrement exactly once" a
@@ -1432,7 +1426,12 @@ enum NLJState {
/// Outcome of the single pass over the left (build) input.
pub(crate) enum LeftLoad {
/// The left side fit the memory budget and is buffered as one batch.
- InMemory(Arc<JoinLeftData>),
+ InMemory {
+ /// The buffered left side
+ left_data: Arc<JoinLeftData>,
+ /// Partitions that are still probing `left_data`
+ probe_completion: Arc<ProbeCompletion>,
+ },
/// The budget ran out, so the left side was spilled during that same pass. Every partition
/// shares this handle, and the chunks that are read back from the file.
Spilled(Arc<LeftSpillData>),
@@ -1448,10 +1447,10 @@ struct SpilledLeftFile {
/// The spilled left side, shared by every output partition.
///
-/// This is the memory-limited counterpart of [`JoinLeftData`]: the rows live in
-/// a spill file instead of memory, but the visited bitmap and the probe-threads
-/// counter cover the whole left side and are shared by all partitions in the
-/// same way.
+/// This is the memory-limited counterpart of [`LeftLoad::InMemory`]: the rows
+/// live in a spill file instead of memory, but the visited bitmap and the
+/// probe-threads counter cover the whole left side and are shared by all
+/// partitions in the same way.
///
/// The rows come back one chunk at a time (see [`LeftChunkBarrier`]), while the
/// bitmap spans all of them: bits are addressed by the row's position in the
@@ -1473,12 +1472,9 @@ pub(crate) struct LeftSpillData {
/// Visited bitmap over every row of `spill_file`. Empty when the join type
/// does not need it.
visited: SharedBitmapBuilder,
- /// Counter of partitions that have neither finished probing every chunk
- /// nor gone away
- probe_threads_counter: AtomicUsize,
- /// Set when a partition went away before it finished probing. The final
- /// left rows are then not emitted, as on the in-memory path.
- incomplete: AtomicBool,
+ /// Partitions that have neither finished probing every chunk nor gone
+ /// away, and whether any went away before it finished
+ probe_completion: ProbeCompletion,
/// Memory reservation for `visited`
reservation: MemoryReservation,
}
@@ -1513,8 +1509,7 @@ impl LeftSpillData {
num_rows,
reader: Arc::new(Mutex::new(None)),
visited: Mutex::new(visited),
- probe_threads_counter: AtomicUsize::new(probe_threads_count),
- incomplete: AtomicBool::new(false),
+ probe_completion: ProbeCompletion::new(probe_threads_count),
reservation,
}
}
@@ -1563,15 +1558,15 @@ impl LeftSpillData {
/// running thread and no partition went away unfinished, it is the emitter
/// and gets the complete visited bitmap, which it must account for.
fn report_probe_completed(&self) -> Option<BooleanBuffer> {
- if self.probe_threads_counter.fetch_sub(1, Ordering::AcqRel) != 1 {
- return None;
- }
- let visited = self.take_visited();
- if self.incomplete.load(Ordering::Acquire) {
+ match self.probe_completion.report_completed() {
+ Completion::NotLast => None,
+ Completion::Emit(_) => Some(self.take_visited()),
// Nobody will emit, so nobody needs the bitmap
- return None;
+ Completion::Abandoned => {
+ drop(self.take_visited());
+ None
+ }
}
- Some(visited)
}
/// `count` partitions went away before they finished probing.
@@ -1581,15 +1576,7 @@ impl LeftSpillData {
/// the last to finish or go away, while the plan may live on. But the final
/// left rows are then not emitted.
fn depart_unfinished(&self, count: usize) {
- if count == 0 {
- return;
- }
- self.incomplete.store(true, Ordering::Release);
- if self
- .probe_threads_counter
- .fetch_sub(count, Ordering::AcqRel)
- == count
- {
+ if self.probe_completion.abandon(count) {
drop(self.take_visited());
}
}
@@ -2186,6 +2173,10 @@ pub(crate) struct NestedLoopJoinStream {
// -----------------
/// The current buffered left data to join
buffered_left_data: Option<Arc<JoinLeftData>>,
+ /// Probe partitions still running over the in-memory left side, shared by
+ /// all of them. `None` on the spill path, which reports on
+ /// [`LeftSpillData`] instead.
+ in_memory_probe_completion: Option<Arc<ProbeCompletion>>,
/// Index into the left buffered batch. Used in `ProbeRight` state
left_probe_idx: usize,
/// Index into the left buffered batch. Used in `EmitLeftUnmatched` state
@@ -2209,8 +2200,9 @@ pub(crate) struct NestedLoopJoinStream {
/// Whether this stream is the one responsible for emitting unmatched-left
/// rows. Set in the [`NLJState::ProbeEnd`] state, which is entered exactly
- /// once per stream and owns the single `report_probe_completed` call (on
- /// [`JoinLeftData`], or on [`LeftSpillData`] in memory-limited mode): the
+ /// once per stream and owns the single probe-completion report (on
+ /// [`LeftLoad::InMemory`]'s [`ProbeCompletion`], or on [`LeftSpillData`] in
+ /// memory-limited mode): the
/// stream that drives the
/// shared probe-threads counter to zero (the last to finish probing) becomes
/// the emitter. Because the decrement happens once in `ProbeEnd` rather than
@@ -2545,6 +2537,7 @@ impl NestedLoopJoinStream {
column_indices,
left_data,
metrics,
+ in_memory_probe_completion: None,
buffered_left_data: None,
output_buffer: Box::new(BatchCoalescer::new(schema, batch_size)),
batch_size,
@@ -2660,7 +2653,12 @@ impl NestedLoopJoinStream {
// Standard path: use OnceFut
match self.left_data.get_shared(cx) {
Poll::Ready(Ok(load)) => match load.as_ref() {
- LeftLoad::InMemory(left_data) => {
+ LeftLoad::InMemory {
+ left_data,
+ probe_completion,
+ } => {
+ self.in_memory_probe_completion =
+ Some(Arc::clone(probe_completion));
self.buffered_left_data = Some(Arc::clone(left_data));
self.left_exhausted = true;
self.state = NLJState::FetchingRight;
@@ -2749,13 +2747,11 @@ impl NestedLoopJoinStream {
};
// This `JoinLeftData` is private to the partition: it shares the
- // chunk's rows but has its own bitmap, so its probe-threads counter is
- // not used. Probe completion is reported once for the whole left side,
- // on `LeftSpillData`.
+ // chunk's rows but has its own bitmap. Probe completion is reported
+ // once for the whole left side, on `LeftSpillData`.
self.buffered_left_data = Some(Arc::new(JoinLeftData::new(
batch,
Mutex::new(visited_left_side),
- AtomicUsize::new(1),
active.chunk_reservation.take(),
)));
active.current_chunk = Some(chunk);
@@ -3006,7 +3002,7 @@ impl NestedLoopJoinStream {
///
/// Entered exactly once per stream, when it has probed the whole left side.
/// This is the single place that decrements the shared probe-threads counter
- /// via `report_probe_completed`: the stream that drives the
+ /// via [`ProbeCompletion::report_completed`]: the stream that drives the
/// counter to zero (the last to finish probing) is the one responsible for
/// emitting unmatched-left rows, recorded in `is_unmatched_left_emitter`.
///
@@ -3039,8 +3035,13 @@ impl NestedLoopJoinStream {
None => false,
}
} else {
- match self.get_left_data() {
- Ok(left_data) => left_data.report_probe_completed(),
+ match self.get_in_memory_probe_completion() {
+ Ok(probe_completion) => match probe_completion.report_completed() {
+ Completion::NotLast => false,
+ // Nothing abandons the in-memory left side
+ Completion::Abandoned => false,
+ Completion::Emit(_) => true,
+ },
Err(e) => return ControlFlow::Break(Poll::Ready(Some(Err(e)))),
}
};
@@ -3067,6 +3068,7 @@ impl NestedLoopJoinStream {
Ok(false) => match self.output_buffer.finish_buffered_batch() {
Ok(()) => {
self.buffered_left_data = None;
+ self.in_memory_probe_completion = None;
self.state = NLJState::Done;
ControlFlow::Continue(())
}
@@ -3744,6 +3746,12 @@ impl NestedLoopJoinStream {
.ok_or_else(|| internal_datafusion_err!("LeftData should be available"))
}
+ fn get_in_memory_probe_completion(&self) -> Result<&Arc<ProbeCompletion>> {
+ self.in_memory_probe_completion.as_ref().ok_or_else(|| {
+ internal_datafusion_err!("In-memory probe completion should be available")
+ })
+ }
+
/// Flush the `output_buffer` if there are batches ready to output
/// None if no result batch ready.
fn maybe_flush_ready_batch(&mut self) -> Option<Poll<Option<Result<RecordBatch>>>> {
@@ -4261,6 +4269,7 @@ fn build_unmatched_batch(
#[cfg(test)]
pub(crate) mod tests {
use std::pin::Pin;
+ use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use super::*;
@@ -5304,7 +5313,6 @@ pub(crate) mod tests {
let left_data = JoinLeftData::new(
left.into(),
Mutex::new(BooleanBufferBuilder::new(0)),
- AtomicUsize::new(1),
MemoryConsumer::new("test").register(ctx.memory_pool()),
);
let mut stream = NestedLoopJoinStream::new(
@@ -6776,10 +6784,6 @@ pub(crate) mod tests {
.as_ref()
.expect("the left side spilled"),
);
- assert!(
- left_spill.probe_threads_counter.load(Ordering::Acquire) >= 1,
- "the replaying partition has not reported"
- );
drop(streams.remove(replaying));
let survivor = streams.pop().unwrap();
@@ -6788,13 +6792,17 @@ pub(crate) mod tests {
.await
.expect("the remaining partition stalled")?;
// The survivor emits its own unmatched right rows, but the final left
- // rows are not emitted
- assert_eq!(count_final_left_rows(JoinType::Full, &batches), 0);
- assert!(left_spill.incomplete.load(Ordering::Acquire));
- assert_eq!(left_spill.probe_threads_counter.load(Ordering::Acquire), 0);
+ // rows are not emitted. Had the replaying partition already reported,
+ // they would be.
+ assert_eq!(
+ count_final_left_rows(JoinType::Full, &batches),
+ 0,
+ "the replaying partition must not have reported before it was dropped"
+ );
drop(left_spill);
- // Nothing stays reserved, although the plan is still alive
+ // Nothing stays reserved, although the plan is still alive. The bitmap
+ // is only released once every partition has finished or gone away.
assert_eq!(task_ctx.memory_pool().reserved(), 0);
Ok(())
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs b/datafusion/physical-plan/src/joins/probe_completion.rs
similarity index 67%
rename from datafusion/physical-plan/src/joins/hash_join/probe_completion.rs
rename to datafusion/physical-plan/src/joins/probe_completion.rs
index 11fb72d352..bf30ac9365 100644
--- a/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs
+++ b/datafusion/physical-plan/src/joins/probe_completion.rs
@@ -15,18 +15,23 @@
// specific language governing permissions and limitations
// under the License.
-//! Completion protocol shared by the probe partitions of a hash join.
+//! Completion protocol shared by the probe partitions of a join.
//!
-//! The probe partitions of one `HashJoinExec` run concurrently over a shared
-//! build side. Two facts have to cross partition boundaries:
+//! The probe partitions of one `HashJoinExec` or `NestedLoopJoinExec` run
+//! concurrently over a shared build side. Some facts have to cross partition
+//! boundaries:
//!
//! - which partition emits the build-side rows in the final stage, namely the
-//! last one to finish, and
+//! last one to finish,
+//! - whether any partition stopped before it finished, in which case its
+//! matches are missing and nobody may emit, and
//! - what the partitions collectively saw, which the null-aware `NOT IN` rules
-//! need: did any partition see a row, and did any see a NULL join key.
+//! of the hash join need: did any partition see a row, and did any see a
+//! NULL join key.
//!
-//! [`ProbeCompletion`] owns both, because reading the second without ordering
-//! it against the first produced wrong results. A partition that read the NULL
+//! [`ProbeCompletion`] owns all of them, because reading the facts without
+//! ordering them against the choice of the last partition produced wrong
+//! results. A partition that read the NULL
//! flag *before* decrementing the counter could observe `false`, have a
//! sibling record a NULL and finish, then become the last partition itself and
//! emit build rows that `NOT IN` must suppress.
@@ -64,6 +69,17 @@ pub(super) struct ProbeSideSummary {
pub(super) has_null: bool,
}
+/// Describes the state of a probe partition after it has finished its work.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum Completion {
+ /// Not the last partition to finish.
+ NotLast,
+ /// Last partition to finish and emit a summary.
+ Emit(ProbeSideSummary),
+ /// The last partition with some previous partition abandoned.
+ Abandoned,
+}
+
/// Defines [`ProbeCompletion`] over the given atomic types.
///
/// [loom] can only explore its own atomics, so the protocol is written once
@@ -82,7 +98,7 @@ macro_rules! define_probe_completion {
type Flag = $flag;
/// Tracks how many probe partitions are still running, together with
- /// the null-aware facts they contribute.
+ /// the facts they contribute.
///
/// See the [module docs](self) for the invariant this upholds.
#[derive(Debug)]
@@ -93,6 +109,8 @@ macro_rules! define_probe_completion {
saw_row: Flag,
/// Set once any probe partition has seen a NULL join key.
saw_null_key: Flag,
+ /// Set once any probe partition has stopped before it finished.
+ abandoned: Flag,
}
impl ProbeCompletion {
@@ -102,6 +120,7 @@ macro_rules! define_probe_completion {
running: Counter::new(probe_threads),
saw_row: Flag::new(false),
saw_null_key: Flag::new(false),
+ abandoned: Flag::new(false),
}
}
@@ -129,20 +148,45 @@ macro_rules! define_probe_completion {
self.saw_null_key.load(Ordering::Relaxed)
}
- /// Marks the calling partition finished, returning `Some` only for
- /// the last one, together with what all partitions saw.
+ /// Marks the calling partition finished. Only the last one gets
+ /// more than [`Completion::NotLast`]: [`Completion::Emit`] with
+ /// what all partitions saw, or [`Completion::Abandoned`] if some
+ /// partition stopped early (see [`Self::abandon`]).
///
/// The `AcqRel` decrement publishes this partition's own stores
/// and acquires those of the partitions that finished earlier, so
/// the summary handed to the last caller is complete.
- pub(super) fn report_completed(&self) -> Option<ProbeSideSummary> {
+ pub(super) fn report_completed(&self) -> Completion {
// `fetch_sub` returns the value from *before* the decrement,
// so `1` means this call is the one that took it to zero.
let was_last = self.running.fetch_sub(1, Ordering::AcqRel) == 1;
- was_last.then(|| ProbeSideSummary {
- non_empty: self.saw_row.load(Ordering::Relaxed),
- has_null: self.saw_null_key.load(Ordering::Relaxed),
- })
+ match (was_last, self.abandoned.load(Ordering::Relaxed)) {
+ (false, _) => Completion::NotLast,
+ (true, false) => {
+ let summary = ProbeSideSummary {
+ non_empty: self.saw_row.load(Ordering::Relaxed),
+ has_null: self.saw_null_key.load(Ordering::Relaxed),
+ };
+ Completion::Emit(summary)
+ }
+ (true, true) => Completion::Abandoned,
+ }
+ }
+
+ /// Marks `count` probe partitions as having stopped before they finished.
+ ///
+ /// They still count down, so the partitions that remain are not left
+ /// waiting. But their matches are missing, so the build-side rows must not
+ /// be emitted: the last partition is not handed a summary.
+ ///
+ /// `Relaxed` suffices for the flag, as in [`Self::record_batch`]: the
+ /// `AcqRel` decrement publishes it.
+ pub(super) fn abandon(&self, count: usize) -> bool {
+ if count == 0 {
+ return false;
+ }
+ self.abandoned.store(true, Ordering::Relaxed);
+ self.running.fetch_sub(count, Ordering::AcqRel) == count
}
}
};
@@ -166,14 +210,14 @@ mod tests {
let completion = ProbeCompletion::new(3);
completion.record_batch(true, false);
- assert_eq!(completion.report_completed(), None);
+ assert_eq!(completion.report_completed(), Completion::NotLast);
completion.record_batch(true, true);
- assert_eq!(completion.report_completed(), None);
+ assert_eq!(completion.report_completed(), Completion::NotLast);
assert_eq!(
completion.report_completed(),
- Some(ProbeSideSummary {
+ Completion::Emit(ProbeSideSummary {
non_empty: true,
has_null: true,
})
@@ -188,7 +232,7 @@ mod tests {
assert_eq!(
completion.report_completed(),
- Some(ProbeSideSummary {
+ Completion::Emit(ProbeSideSummary {
non_empty: false,
has_null: false,
})
@@ -208,6 +252,22 @@ mod tests {
completion.record_batch(true, true);
assert!(completion.saw_null_key_hint());
}
+
+ /// Abandoning no partitions changes nothing. A caller may report a count of
+ /// zero, for example when no partition went away before the left side
+ /// spilled, and that must not suppress the build-side rows.
+ #[test]
+ fn abandoning_no_partitions_still_emits() {
+ let completion = ProbeCompletion::new(1);
+ assert!(!completion.abandon(0));
+ assert_eq!(
+ completion.report_completed(),
+ Completion::Emit(ProbeSideSummary {
+ non_empty: false,
+ has_null: false,
+ })
+ );
+ }
}
/// Model-checked concurrency tests for the protocol.
@@ -215,7 +275,7 @@ mod tests {
/// These instantiate the protocol over loom's atomics (see the
/// [module docs](self)) and let loom enumerate the interleavings and store
/// visibility the memory model allows. They fail if the decrement in
-/// `report_completed` is weakened to `Relaxed`.
+/// `report_completed` or `abandon` is weakened to `Relaxed`.
///
/// Compiled only under `--cfg datafusion_loom`, which is also what pulls the `loom`
/// dev-dependency into the build (see `datafusion-physical-plan`'s
@@ -226,6 +286,7 @@ mod tests {
/// ```
#[cfg(all(test, datafusion_loom))]
mod loom_tests {
+ use super::Completion;
use super::ProbeSideSummary;
use loom::sync::Arc;
use std::sync::atomic::Ordering;
@@ -258,9 +319,15 @@ mod loom_tests {
let recorded = recorder.join().unwrap();
let summary = match (recorded, observer) {
- (Some(summary), None) | (None, Some(summary)) => summary,
- (Some(_), Some(_)) => panic!("two partitions both finished last"),
- (None, None) => panic!("no partition finished last"),
+ (Completion::Emit(summary), Completion::NotLast)
+ | (Completion::NotLast, Completion::Emit(summary)) => summary,
+ (Completion::Emit(_), Completion::Emit(_)) => {
+ panic!("two partitions both finished last")
+ }
+ (Completion::NotLast, Completion::NotLast) => {
+ panic!("no partition finished last")
+ }
+ other => panic!("unexpected outcome: {other:?}"),
};
assert!(
@@ -295,7 +362,11 @@ mod loom_tests {
let here = completion.report_completed();
let there = other.join().unwrap();
- let summary = here.or(there).expect("some partition must finish last");
+ let summary = match (here, there) {
+ (Completion::Emit(summary), Completion::NotLast)
+ | (Completion::NotLast, Completion::Emit(summary)) => summary,
+ other => panic!("some partition must finish last: {other:?}"),
+ };
assert_eq!(
summary,
ProbeSideSummary {
@@ -316,10 +387,12 @@ mod loom_tests {
let other = {
let completion = Arc::clone(&completion);
- loom::thread::spawn(move || completion.report_completed().is_some())
+ loom::thread::spawn(move || {
+ completion.report_completed() != Completion::NotLast
+ })
};
- let here = completion.report_completed().is_some();
+ let here = completion.report_completed() != Completion::NotLast;
let there = other.join().unwrap();
assert!(here ^ there, "exactly one partition must finish last");
@@ -347,4 +420,30 @@ mod loom_tests {
assert!(!here && !there, "the hint reported an unrecorded NULL");
});
}
+
+ /// A partition that stops early still counts down, so exactly one
+ /// partition finishes last. But its matches are missing, so no partition
+ /// may emit, whichever of the two finishes last.
+ #[test]
+ fn an_abandoned_partition_counts_down_but_suppresses_the_emit() {
+ loom::model(|| {
+ let completion = Arc::new(ProbeCompletion::new(2));
+
+ let quitter = {
+ let completion = Arc::clone(&completion);
+ loom::thread::spawn(move || completion.abandon(1))
+ };
+
+ let here = completion.report_completed();
+ let quitter_was_last = quitter.join().unwrap();
+
+ match (here, quitter_was_last) {
+ // This partition finished last and saw the abandon
+ (Completion::Abandoned, false) => {}
+ // The quitter finished last and releases the shared state
+ (Completion::NotLast, true) => {}
+ other => panic!("unexpected outcome: {other:?}"),
+ }
+ });
+ }
}