Apache DataFusion SQL Query Engine
fix: preserve no-filter SMJ matches across pending outer batches (#23049)
Neil Conway67425b4b8baf
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 99aef6ed82..815cc370ca 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -1177,12 +1177,13 @@ impl BitwiseSortMergeJoinStream {self.emit_outer_batch()?;self.pending_boundary =Some(PendingBoundary::NoFilter { saved_keys });+ // Clear stale batch before polling+ self.outer_batch = None;match ready!(self.poll_next_outer_batch(cx)) {Err(e) => return Poll::Ready(Err(e)),Ok(false) => {self.pending_boundary = None;- self.outer_batch = None;break;}Ok(true) => {diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex 338c5111d2..bc8b15472a 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -4342,6 +4342,105 @@ async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {Ok(())}+/// Verifies no-filter semi/anti joins when a matching outer key group spans+/// multiple batches and the next outer batch is temporarily unavailable.+///+/// The outer input has an unmatched prefix row followed by a matching key+/// group that continues in the next batch. Both rows with key=1 should be+/// treated as matched. Returning `Pending` before the second batch forces+/// `poll_join` to return and later resume from its top-level state, rather+/// than continuing the same in-progress boundary loop.+#[tokio::test]+async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c1", DataType::Int32, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("a2", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ]));++ // Key=0 is unmatched. Key=1 matches inner and spans the batch boundary.+ let outer_batch1 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![0, 1])),+ Arc::new(Int32Array::from(vec![0, 1])),+ Arc::new(Int32Array::from(vec![0, 10])),+ ],+ )?;+ let outer_batch2 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![2])),+ Arc::new(Int32Array::from(vec![1])), // same key+ Arc::new(Int32Array::from(vec![20])),+ ],+ )?;++ // Key=1 matches two outer rows. Key=2 keeps the inner input non-exhausted.+ let inner_batch = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![100, 200])),+ Arc::new(Int32Array::from(vec![1, 2])),+ Arc::new(Int32Array::from(vec![50, 60])),+ ],+ )?;++ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];++ for (join_type, expected_a1) in [(LeftSemi, vec![1, 2]), (LeftAnti, vec![0])] {+ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(+ vec![outer_batch1.clone(), outer_batch2.clone()],+ vec![false, true], // Pending before 2nd outer batch+ ));+ let inner: SendableRecordBatchStream =+ Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));++ let metrics = ExecutionPlanMetricsSet::new();+ let inner_schema = inner.schema();+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(inner_schema, &metrics);+ let stream = BitwiseSortMergeJoinStream::try_new(+ Arc::clone(&left_schema),+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ on_outer.clone(),+ on_inner.clone(),+ None, // no filter+ join_type,+ 8192,+ 0,+ &metrics,+ reservation,+ spill_manager,+ runtime_env,+ )?;++ let batches = collect_stream(stream).await?;+ let actual_a1 = batches+ .iter()+ .flat_map(|batch| {+ let values = batch+ .column(0)+ .as_any()+ .downcast_ref::<Int32Array>()+ .unwrap();+ (0..batch.num_rows()).map(|row| values.value(row))+ })+ .collect::<Vec<_>>();+ assert_eq!(actual_a1, expected_a1, "{join_type:?}");+ }+ Ok(())+}+/// Tests the filtered boundary Pending re-entry: outer key group spans/// batches with a filter, and poll_next_outer_batch returns Pending.///