Apache DataFusion SQL Query Engine
fix: release semi/anti/mark sort-merge join reservations before the final output (#25986)
Jay Zhan95a1961cc7ec
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex a36137b3db..d8be37ba15 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -467,13 +467,22 @@ impl BitwiseSortMergeJoinStream {/// Clear inner key group state after processing. Does not resize the/// reservation — the next key group will resize when buffering, or- /// the stream's Drop will free it. This avoids unnecessary memory+ /// [`Self::release_inner`] will free it. This avoids unnecessary memory/// pool interactions (see apache/datafusion#20729).fn clear_inner_key_group(&mut self) {self.inner_key_buffer.clear();self.inner_buffer_size = 0;}+ /// Drops the inner input and frees the inner key group reservation once no outer row can+ /// match any more, so that memory is back in the pool before the remaining output is+ /// emitted rather than when the stream is dropped.+ fn release_inner(&mut self) {+ self.clear_inner_key_group();+ self.reservation.free();+ self.inner = Box::pin(EmptyRecordBatchStream::new(self.inner.schema()));+ }+/// Fetch the next outer batch. Returns true if a batch was loaded.async fn next_outer_batch(&mut self) -> Result<bool> {loop {@@ -1050,12 +1059,17 @@ impl BitwiseSortMergeJoinStream {&mut self,emitter: &mut TryEmitter<RecordBatch, DataFusionError>,) -> Result<()> {+ // The remaining outer rows can no longer match. Anti and mark joins still emit+ // them; a semi join is finished without reading them and drops the outer input too.+ let is_semi = matches!(self.join_type, JoinType::LeftSemi | JoinType::RightSemi);+ self.release_inner();+ if is_semi {+ self.outer = Box::pin(EmptyRecordBatchStream::new(self.outer.schema()));+ }loop {self.emit_outer_batch()?;self.emit_completed_batches(emitter).await;- // The remaining outer rows can no longer match. Anti and mark joins still emit- // them; a semi join is finished without reading them.- if matches!(self.join_type, JoinType::LeftSemi | JoinType::RightSemi) {+ if is_semi {break;}if !self.next_outer_batch().await? {@@ -1116,6 +1130,10 @@ impl BitwiseSortMergeJoinStream {}}+ // The outer input is exhausted or dropped by now; when it ran out first, the inner+ // input and the last inner key group are still held.+ self.release_inner();+// Flush whatever is still buffered in the coalescer.self.coalescer.finish_buffered_batch()?;self.emit_completed_batches(emitter).await;diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex 7ca8adaa2a..53f44ba014 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -728,6 +728,224 @@ async fn join_chain_reuses_memory_released_by_completed_child() -> Result<()> {Ok(())}+/// A 2 MB pool with spilling disabled, so a reservation a join fails to release shows up in+/// `reserved()` or as an allocation failure rather than as a spill.+fn small_pool_no_spill_ctx() -> Result<Arc<TaskContext>> {+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(2_000_000, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled),+ )+ .build_arc()?;+ Ok(Arc::new(TaskContext::default().with_runtime(runtime)))+}++/// Drains `stream`, returning its row count and the pool's reservation while each output+/// batch is in hand, before the stream ends and is dropped.+async fn rows_and_reserved_per_batch(+ mut stream: SendableRecordBatchStream,+ ctx: &TaskContext,+) -> Result<(usize, Vec<usize>)> {+ let mut rows = 0;+ let mut reserved = vec![];+ while let Some(batch) = stream.next().await {+ rows += batch?.num_rows();+ reserved.push(ctx.memory_pool().reserved());+ }+ Ok((rows, reserved))+}++/// A semi/anti/mark join whose filter (`c1 < c2`) buffers a large inner key group (32 x 4096+/// rows with key 2) and rejects all of it. The outer row with key 1 passes against the one+/// inner row with key 1, so every join type has output with `a1 = 1`. Without+/// `inner_runs_out_first` the outer side ends inside the large group. With it, 3 x 4096+/// outer rows with key 3 and `a1 = 2` follow, which the join drains after the inner side+/// ends, emitting a full output batch before the final one.+fn bitwise_large_inner_key_group_join(+ inner_runs_out_first: bool,+ join_type: JoinType,+) -> Result<SortMergeJoinExec> {+ let mut outer = vec![build_table_i32(+ ("a1", &vec![1, 1]),+ ("b1", &vec![1, 2]),+ ("c1", &vec![0, 9]),+ )];+ if inner_runs_out_first {+ outer.extend((0..3).map(|_| {+ build_table_i32(+ ("a1", &vec![2; 4096]),+ ("b1", &vec![3; 4096]),+ ("c1", &vec![9; 4096]),+ )+ }));+ }+ let left = build_table_from_batches(outer);+ let right = build_table_from_batches(+ std::iter::once(build_table_i32(+ ("a2", &vec![1]),+ ("b1", &vec![1]),+ ("c2", &vec![1]),+ ))+ .chain((0..32).map(|_| {+ build_table_i32(+ ("a2", &vec![2; 4096]),+ ("b1", &vec![2; 4096]),+ ("c2", &vec![2; 4096]),+ )+ }))+ .collect(),+ );+ let on = on_b1(&left, &right)?;+ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());+ join_with_filter(+ left,+ right,+ on,+ filter,+ join_type,+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ )+}++/// Output rows with `a1 = 1`: semi keeps the outer row with key 1, anti the one with key 2,+/// mark both.+fn bitwise_large_inner_key_group_a1_rows(join_type: JoinType) -> usize {+ if join_type == LeftMark { 2 } else { 1 }+}++/// Once either input runs out, the join must give the last inner key group's reservation back+/// before it emits anything else, not when the stream is dropped.+#[tokio::test]+async fn bitwise_join_releases_inner_key_group_reservation_before_output() -> Result<()> {+ for inner_runs_out_first in [false, true] {+ for join_type in [LeftSemi, LeftAnti, LeftMark] {+ let join =+ bitwise_large_inner_key_group_join(inner_runs_out_first, join_type)?;+ let ctx = small_pool_no_spill_ctx()?;+ let stream = join.execute(0, Arc::clone(&ctx))?;+ let (rows, reserved) = rows_and_reserved_per_batch(stream, &ctx).await?;+ let drained = if inner_runs_out_first && join_type != LeftSemi {+ 3 * 4096+ } else {+ 0+ };+ let case =+ format!("{join_type:?}, inner_runs_out_first={inner_runs_out_first}");+ assert_eq!(+ rows,+ bitwise_large_inner_key_group_a1_rows(join_type) + drained,+ "{case}"+ );+ assert!(+ reserved.iter().all(|&r| r == 0),+ "{case}: reservation held while emitting output: {reserved:?}"+ );+ }+ }+ Ok(())+}++/// The parent join consumes the child's output while the child stream is still alive, with+/// spilling disabled, so an inner key group reservation the child failed to release surfaces+/// as an allocation failure in the parent.+#[tokio::test]+async fn bitwise_join_chain_reuses_memory_released_by_completed_child() -> Result<()> {+ for inner_runs_out_first in [false, true] {+ for join_type in [LeftSemi, LeftAnti, LeftMark] {+ let child: Arc<dyn ExecutionPlan> = Arc::new(+ bitwise_large_inner_key_group_join(inner_runs_out_first, join_type)?,+ );+ let parent_right = build_table_from_batches(+ (0..10)+ .map(|_| {+ build_table_i32(+ ("a3", &vec![1; 4096]),+ ("b3", &vec![1; 4096]),+ ("c3", &vec![1; 4096]),+ )+ })+ .collect(),+ );+ let on = vec![(+ Arc::new(Column::new("a1", 0)) as _,+ Arc::new(Column::new("a3", 0)) as _,+ )];+ let parent = join(child, parent_right, on, Inner)?;+ let case =+ format!("{join_type:?}, inner_runs_out_first={inner_runs_out_first}");+ let batches = common::collect(parent.execute(0, small_pool_no_spill_ctx()?)?)+ .await+ .map_err(|e| e.context(case.clone()))?;+ assert_eq!(+ batches.iter().map(|b| b.num_rows()).sum::<usize>(),+ bitwise_large_inner_key_group_a1_rows(join_type) * 40_960,+ "{case}"+ );+ }+ }+ Ok(())+}++/// A sorted input that sorts into two batches: key 1 first, then 16,383 rows with key 5.+fn sorted_key_1_then_key_5(names: [&str; 3]) -> Result<Arc<dyn ExecutionPlan>> {+ let mut keys = vec![5; 16_384];+ keys[16_383] = 1;+ let input = build_table((names[0], &keys), (names[1], &keys), (names[2], &keys));+ let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(+ Column::new_with_schema("b1", &input.schema())?,+ ))])+ .unwrap();+ Ok(Arc::new(SortExec::new(ordering, input)))+}++/// A sorted input keeps its unread sorted batches reserved until it is read to the end or+/// dropped. Once the join cannot use an input's remaining rows, it must drop that input+/// before its final batch: the outer input of a semi join whose inner side ran out, and the+/// inner input of any join whose outer side ran out.+#[tokio::test]+async fn bitwise_join_releases_unread_input_before_final_batch() -> Result<()> {+ let one_row = build_table(("a2", &vec![1]), ("b1", &vec![1]), ("c2", &vec![1]));+ let two_rows = build_table(+ ("a1", &vec![0, 1]),+ ("b1", &vec![0, 1]),+ ("c1", &vec![0, 1]),+ );+ let cases = [+ (+ "inner runs out first",+ sorted_key_1_then_key_5(["a1", "b1", "c1"])?,+ one_row,+ [1, 16_383, 16_384],+ ),+ (+ "outer runs out first",+ two_rows,+ sorted_key_1_then_key_5(["a2", "b1", "c2"])?,+ [1, 1, 2],+ ),+ ];+ for (shape, left, right, semi_anti_mark_rows) in cases {+ for (join_type, expected_rows) in [LeftSemi, LeftAnti, LeftMark]+ .into_iter()+ .zip(semi_anti_mark_rows)+ {+ let on = on_b1(&left, &right)?;+ let join = join(Arc::clone(&left), Arc::clone(&right), on, join_type)?;+ let ctx = small_pool_no_spill_ctx()?;+ let stream = join.execute(0, Arc::clone(&ctx))?;+ let (rows, reserved) = rows_and_reserved_per_batch(stream, &ctx).await?;+ assert_eq!(rows, expected_rows, "{join_type:?}, {shape}");+ assert_eq!(+ reserved.last(),+ Some(&0),+ "{join_type:?}, {shape}: unread input held at the final batch"+ );+ }+ }+ Ok(())+}+#[tokio::test]async fn join_inner_one() -> Result<()> {let left = build_table(