Apache DataFusion SQL Query Engine
fix: drain completed sort-merge join output before awaiting input (#24685)
Chao Sun4ea997529194
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 1b90f24b96..4ee963beb7 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -821,6 +821,7 @@ impl BitwiseSortMergeJoinStream {async fn process_filtered_match_loop(&mut self,spill: Option<Arc<dyn SpillFile>>,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,) -> Result<()> {loop {self.process_key_match_with_filter(spill.as_ref()).await?;@@ -836,6 +837,7 @@ impl BitwiseSortMergeJoinStream {slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1);self.emit_outer_batch()?;+ self.emit_completed_batches(emitter).await;if !self.next_outer_batch().await? {break;@@ -856,7 +858,10 @@ impl BitwiseSortMergeJoinStream {/// Mark the outer key group as matched. If the outer key group continues/// into subsequent outer batches, keep marking there too.- async fn process_unfiltered_match_loop(&mut self) -> Result<()> {+ async fn process_unfiltered_match_loop(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {loop {self.mark_outer_key_group_matched()?;@@ -871,6 +876,7 @@ impl BitwiseSortMergeJoinStream {slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1);self.emit_outer_batch()?;+ self.emit_completed_batches(emitter).await;if !self.next_outer_batch().await? {return Ok(());@@ -888,18 +894,21 @@ impl BitwiseSortMergeJoinStream {/// Keys at both cursors are equal: determine which outer rows in the key/// group have a match. Both key groups may span batch boundaries.- async fn process_key_match(&mut self) -> Result<()> {+ async fn process_key_match(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {if self.filter.is_some() {// Buffer the inner key group so each inner row can be evaluated// against the outer key group, OR-ing filter results into the// matched bitset.let spill = self.buffer_inner_key_group().await?;- self.process_filtered_match_loop(spill).await+ self.process_filtered_match_loop(spill, emitter).await} else {// Without a filter, key equality alone means every outer row in// the group matches; the inner rows themselves are not needed.self.advance_inner_past_key_group().await?;- self.process_unfiltered_match_loop().await+ self.process_unfiltered_match_loop(emitter).await}}@@ -1037,10 +1046,16 @@ impl BitwiseSortMergeJoinStream {/// current outer batch and all remaining ones with their current matched/// bits (semi drops unmatched rows, anti emits them, mark emits them/// with mark=false).- async fn drain_outer(&mut self) -> Result<()> {- self.emit_outer_batch()?;- while self.next_outer_batch().await? {+ async fn drain_outer(+ &mut self,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ loop {self.emit_outer_batch()?;+ self.emit_completed_batches(emitter).await;+ if !self.next_outer_batch().await? {+ break;+ }}Ok(())}@@ -1070,7 +1085,7 @@ impl BitwiseSortMergeJoinStream {// helpers are only entered at batch boundaries.while self.has_current_outer_row() || self.advance_outer_row(emitter).await? {if !(self.has_current_inner_row() || self.advance_inner_row().await?) {- self.drain_outer().await?;+ self.drain_outer(emitter).await?;break;}@@ -1086,7 +1101,7 @@ impl BitwiseSortMergeJoinStream {}Ordering::Equal => {if !self.try_process_key_match()? {- self.process_key_match().await?;+ self.process_key_match(emitter).await?;}}}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex a0c72dc8a6..bab4850860 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -4440,6 +4440,109 @@ async fn collect_stream(stream: SendableRecordBatchStream) -> Result<Vec<RecordBcommon::collect(stream).await}+#[rstest::rstest]+#[case::empty_inner(true, false)]+#[case::matching_key(false, false)]+#[case::matching_key_with_filter(false, true)]+#[tokio::test]+async fn bitwise_emits_completed_batches_before_pending_outer(+ #[values(LeftMark, RightMark)] join_type: JoinType,+ #[case] empty_inner: bool,+ #[case] with_filter: bool,+) -> Result<()> {+ let outer_batches = vec![+ build_table_i32(+ ("id", &vec![0, 1]),+ ("key", &vec![1, 1]),+ ("value", &vec![10, 20]),+ ),+ build_table_i32(+ ("id", &vec![2, 3]),+ ("key", &vec![1, 1]),+ ("value", &vec![10, 20]),+ ),+ ];+ let inner_batch = if empty_inner {+ RecordBatch::new_empty(outer_batches[0].schema())+ } else {+ build_table_i32(("id", &vec![0]), ("key", &vec![1]), ("value", &vec![10]))+ };+ let outer = Box::pin(PendingStream::new(outer_batches.clone(), vec![false, true]));+ let inner = Box::pin(PendingStream::new(vec![inner_batch], vec![false]));+ let filter = with_filter.then(|| {+ JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("left_value", 0)),+ Operator::Eq,+ Arc::new(Column::new("right_value", 1)),+ )),+ vec![+ ColumnIndex {+ index: 2,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 2,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ Field::new("left_value", DataType::Int32, false),+ Field::new("right_value", DataType::Int32, false),+ ])),+ )+ });+ let mut fields = outer.schema().fields().to_vec();+ fields.push(Arc::new(Field::new("mark", DataType::Boolean, false)));+ let schema = Arc::new(Schema::new(fields));+ let metrics = ExecutionPlanMetricsSet::new();+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(inner.schema(), &metrics);+ let mut stream = BitwiseSortMergeJoinStream::try_new(+ Arc::clone(&schema),+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ vec![Arc::new(Column::new("key", 1))],+ vec![Arc::new(Column::new("key", 1))],+ filter,+ join_type,+ 2,+ 0,+ &metrics,+ reservation,+ spill_manager,+ runtime_env,+ )?;++ // The first outer batch already fills an output batch. Do not wait for the next+ // outer batch, even when draining unmatched rows or continuing the same key group.+ let first = match futures::poll!(stream.next()) {+ Poll::Ready(Some(batch)) => batch?,+ other => panic!("Completed output must precede pending outer input: {other:?}"),+ };+ let mut output = vec![first];+ output.extend(collect_stream(stream).await?);++ let expected = outer_batches+ .iter()+ .map(|batch| {+ let mut columns = batch.columns().to_vec();+ columns.push(Arc::new(BooleanArray::from(vec![+ !empty_inner,+ !empty_inner && !with_filter,+ ])));+ RecordBatch::try_new(Arc::clone(&schema), columns)+ })+ .collect::<std::result::Result<Vec<_>, _>>()?;+ assert_eq!(+ arrow::compute::concat_batches(&schema, &output)?,+ arrow::compute::concat_batches(&schema, &expected)?,+ );+ Ok(())+}+// ==================== join_time metric tests ====================//// These verify that `join_time` measures only the join's own work: waiting