Skip to content

Apache DataFusion SQL Query Engine

perf: sort-merge join (SMJ) batch deferred filtering and move mark joins to bitwise stream. Near-unique LEFT and FULL SMJ 20-50x faster (#21184)

Matt Butrovich0be5982a913e
diff --git a/benchmarks/src/smj.rs b/benchmarks/src/smj.rs
index 1829b77a3d..3d173b7116 100644
--- a/benchmarks/src/smj.rs
+++ b/benchmarks/src/smj.rs
@@ -39,7 +39,7 @@ use futures::StreamExt;
#[derive(Debug, Args, Clone)]
#[command(verbatim_doc_comment)]
pub struct RunOpt {
- /// Query number (between 1 and 23). If not specified, runs all queries
+ /// Query number (between 1 and 26). If not specified, runs all queries
#[arg(short, long)]
query: Option<usize>,
@@ -456,6 +456,72 @@ const SMJ_QUERIES: &[&str] = &[
ON t1_sorted.key = t2_sorted.key
AND t1_sorted.data + t2_sorted.data < 10000000
"#,
+ // Q24: LEFT MARK 1M x 10M | 1:10 | 1%
+ r#"
+ WITH t1_sorted AS (
+ SELECT value % 100000 as key, value as data
+ FROM range(1000000)
+ ORDER BY key, data
+ ),
+ t2_sorted AS (
+ SELECT value % 100000 as key, value as data
+ FROM range(10000000)
+ ORDER BY key, data
+ )
+ SELECT t1_sorted.key, t1_sorted.data
+ FROM t1_sorted
+ WHERE t1_sorted.data < 0
+ OR EXISTS (
+ SELECT 1 FROM t2_sorted
+ WHERE t2_sorted.key = t1_sorted.key
+ AND t2_sorted.data <> t1_sorted.data
+ AND t2_sorted.data % 100 = 0
+ )
+ "#,
+ // Q25: LEFT MARK 1M x 10M | 1:10 | 50%
+ r#"
+ WITH t1_sorted AS (
+ SELECT value % 100000 as key, value as data
+ FROM range(1000000)
+ ORDER BY key, data
+ ),
+ t2_sorted AS (
+ SELECT value % 100000 as key, value as data
+ FROM range(10000000)
+ ORDER BY key, data
+ )
+ SELECT t1_sorted.key, t1_sorted.data
+ FROM t1_sorted
+ WHERE t1_sorted.data < 0
+ OR EXISTS (
+ SELECT 1 FROM t2_sorted
+ WHERE t2_sorted.key = t1_sorted.key
+ AND t2_sorted.data <> t1_sorted.data
+ AND t2_sorted.data % 2 = 0
+ )
+ "#,
+ // Q26: LEFT MARK 1M x 10M | 1:10 | 90%
+ r#"
+ WITH t1_sorted AS (
+ SELECT value % 100000 as key, value as data
+ FROM range(1000000)
+ ORDER BY key, data
+ ),
+ t2_sorted AS (
+ SELECT value % 100000 as key, value as data
+ FROM range(10000000)
+ ORDER BY key, data
+ )
+ SELECT t1_sorted.key, t1_sorted.data
+ FROM t1_sorted
+ WHERE t1_sorted.data < 0
+ OR EXISTS (
+ SELECT 1 FROM t2_sorted
+ WHERE t2_sorted.key = t1_sorted.key
+ AND t2_sorted.data <> t1_sorted.data
+ AND t2_sorted.data % 10 <> 0
+ )
+ "#,
];
impl RunOpt {
@@ -489,7 +555,10 @@ impl RunOpt {
let sql = SMJ_QUERIES[query_index];
benchmark_run.start_new_case(&format!("Query {query_id}"));
- let query_run = self.benchmark_query(sql, &query_id.to_string(), &ctx).await;
+ let expect_mark = query_id >= 24;
+ let query_run = self
+ .benchmark_query(sql, &query_id.to_string(), expect_mark, &ctx)
+ .await;
match query_run {
Ok(query_results) => {
for iter in query_results {
@@ -513,6 +582,7 @@ impl RunOpt {
&self,
sql: &str,
query_name: &str,
+ expect_mark: bool,
ctx: &SessionContext,
) -> Result<Vec<QueryResult>> {
let mut query_results = vec![];
@@ -528,6 +598,12 @@ impl RunOpt {
));
}
+ if expect_mark && !plan_string.contains("LeftMark") {
+ return Err(exec_datafusion_err!(
+ "Query {query_name} expected LeftMark join. Physical plan: {plan_string}"
+ ));
+ }
+
for i in 0..self.common.iterations {
let start = Instant::now();
diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rs
index 669b98e39f..f570335e92 100644
--- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs
@@ -38,6 +38,9 @@ use datafusion::physical_plan::joins::{
};
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::{NullEquality, ScalarValue};
+use datafusion_execution::TaskContext;
+use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
+use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_physical_expr::PhysicalExprRef;
use datafusion_physical_expr::expressions::Literal;
@@ -1125,6 +1128,138 @@ impl JoinFuzzTestCase {
}
}
+/// Fuzz test: compare SMJ (with spilling) against HJ (no spill) for filtered
+/// outer joins under memory pressure. This exercises the deferred filtering +
+/// spill read-back path that unit tests can't easily cover with random data.
+#[tokio::test]
+async fn test_filtered_join_spill_fuzz() {
+ let join_types = [JoinType::Left, JoinType::Right, JoinType::Full];
+
+ let runtime_spill = RuntimeEnvBuilder::new()
+ .with_memory_limit(4096, 1.0)
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()
+ .unwrap();
+
+ for join_type in &join_types {
+ for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
+ let input1 = make_staggered_batches_i32(1000, left_extra);
+ let input2 = make_staggered_batches_i32(1000, right_extra);
+
+ let schema1 = input1[0].schema();
+ let schema2 = input2[0].schema();
+ let filter = col_lt_col_filter(schema1.clone(), schema2.clone());
+
+ let on = vec![
+ (
+ Arc::new(Column::new_with_schema("a", &schema1).unwrap()) as _,
+ Arc::new(Column::new_with_schema("a", &schema2).unwrap()) as _,
+ ),
+ (
+ Arc::new(Column::new_with_schema("b", &schema1).unwrap()) as _,
+ Arc::new(Column::new_with_schema("b", &schema2).unwrap()) as _,
+ ),
+ ];
+
+ for batch_size in [2, 49, 100] {
+ let session_config = SessionConfig::new().with_batch_size(batch_size);
+
+ // HJ baseline (no memory limit)
+ let left_hj = MemorySourceConfig::try_new_exec(
+ std::slice::from_ref(&input1),
+ schema1.clone(),
+ None,
+ )
+ .unwrap();
+ let right_hj = MemorySourceConfig::try_new_exec(
+ std::slice::from_ref(&input2),
+ schema2.clone(),
+ None,
+ )
+ .unwrap();
+ let hj = Arc::new(
+ HashJoinExec::try_new(
+ left_hj,
+ right_hj,
+ on.clone(),
+ Some(filter.clone()),
+ join_type,
+ None,
+ PartitionMode::Partitioned,
+ NullEquality::NullEqualsNothing,
+ false,
+ )
+ .unwrap(),
+ );
+ let ctx_hj = SessionContext::new_with_config(session_config.clone());
+ let hj_collected = collect(hj, ctx_hj.task_ctx()).await.unwrap();
+
+ // SMJ with spilling
+ let left_smj = MemorySourceConfig::try_new_exec(
+ std::slice::from_ref(&input1),
+ schema1.clone(),
+ None,
+ )
+ .unwrap();
+ let right_smj = MemorySourceConfig::try_new_exec(
+ std::slice::from_ref(&input2),
+ schema2.clone(),
+ None,
+ )
+ .unwrap();
+ let smj = Arc::new(
+ SortMergeJoinExec::try_new(
+ left_smj,
+ right_smj,
+ on.clone(),
+ Some(filter.clone()),
+ *join_type,
+ vec![SortOptions::default(); on.len()],
+ NullEquality::NullEqualsNothing,
+ )
+ .unwrap(),
+ );
+ let task_ctx_spill = Arc::new(
+ TaskContext::default()
+ .with_session_config(session_config)
+ .with_runtime(Arc::clone(&runtime_spill)),
+ );
+ let smj_collected = collect(smj, task_ctx_spill).await.unwrap();
+
+ let hj_rows: usize = hj_collected.iter().map(|b| b.num_rows()).sum();
+ let smj_rows: usize = smj_collected.iter().map(|b| b.num_rows()).sum();
+
+ assert_eq!(
+ hj_rows, smj_rows,
+ "Row count mismatch for {join_type:?} batch_size={batch_size} \
+ left_extra={left_extra} right_extra={right_extra}: \
+ HJ={hj_rows} SMJ={smj_rows}"
+ );
+
+ if hj_rows > 0 {
+ let hj_fmt =
+ pretty_format_batches(&hj_collected).unwrap().to_string();
+ let smj_fmt =
+ pretty_format_batches(&smj_collected).unwrap().to_string();
+
+ let mut hj_sorted: Vec<&str> = hj_fmt.trim().lines().collect();
+ hj_sorted.sort_unstable();
+ let mut smj_sorted: Vec<&str> = smj_fmt.trim().lines().collect();
+ smj_sorted.sort_unstable();
+
+ assert_eq!(
+ hj_sorted, smj_sorted,
+ "Content mismatch for {join_type:?} batch_size={batch_size} \
+ left_extra={left_extra} right_extra={right_extra}"
+ );
+ }
+ }
+ }
+ }
+}
+
/// Return randomly sized record batches with:
/// two sorted int32 columns 'a', 'b' ranged from 0..99 as join columns
/// two random int32 columns 'x', 'y' as other columns
diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rs
index 0a198b81a9..2cdfa1e6ac 100644
--- a/datafusion/physical-plan/src/joins/mod.rs
+++ b/datafusion/physical-plan/src/joins/mod.rs
@@ -34,7 +34,6 @@ mod cross_join;
mod hash_join;
mod nested_loop_join;
mod piecewise_merge_join;
-pub(crate) mod semi_anti_sort_merge_join;
mod sort_merge_join;
mod stream_join_utils;
mod symmetric_hash_join;
diff --git a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rs b/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rs
deleted file mode 100644
index 5c5a2727b6..0000000000
--- a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Specialized Sort Merge Join stream for Semi/Anti joins.
-//!
-//! Used internally by `SortMergeJoinExec` for semi/anti join types.
-
-pub(crate) mod stream;
-
-#[cfg(test)]
-mod tests;
diff --git a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rs
deleted file mode 100644
index 92dc78503a..0000000000
--- a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rs
+++ /dev/null
@@ -1,801 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::pin::Pin;
-use std::sync::Arc;
-use std::task::{Context, Poll};
-
-use super::stream::SemiAntiSortMergeJoinStream;
-use crate::ExecutionPlan;
-use crate::RecordBatchStream;
-use crate::common;
-use crate::expressions::Column;
-use crate::joins::SortMergeJoinExec;
-use crate::joins::utils::{ColumnIndex, JoinFilter};
-use crate::metrics::ExecutionPlanMetricsSet;
-use crate::metrics::{MetricBuilder, SpillMetrics};
-use crate::spill::spill_manager::SpillManager;
-use crate::test::TestMemoryExec;
-
-use arrow::array::{Int32Array, RecordBatch};
-use arrow::compute::SortOptions;
-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion_common::JoinSide;
-use datafusion_common::JoinType::*;
-use datafusion_common::{NullEquality, Result};
-use datafusion_execution::memory_pool::MemoryConsumer;
-use datafusion_execution::{SendableRecordBatchStream, TaskContext};
-use datafusion_expr::Operator;
-use datafusion_physical_expr::expressions::BinaryExpr;
-use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;
-use futures::Stream;
-
-/// Create test memory/spill resources for stream-level tests.
-fn test_stream_resources(
- inner_schema: SchemaRef,
- metrics: &ExecutionPlanMetricsSet,
-) -> (
- datafusion_execution::memory_pool::MemoryReservation,
- crate::metrics::Gauge,
- SpillManager,
- Arc<datafusion_execution::runtime_env::RuntimeEnv>,
-) {
- let ctx = TaskContext::default();
- let runtime_env = ctx.runtime_env();
- let reservation = MemoryConsumer::new("test").register(ctx.memory_pool());
- let peak_mem_used = MetricBuilder::new(metrics).gauge("peak_mem_used", 0);
- let spill_manager = SpillManager::new(
- Arc::clone(&runtime_env),
- SpillMetrics::new(metrics, 0),
- inner_schema,
- );
- (reservation, peak_mem_used, spill_manager, runtime_env)
-}
-
-fn build_table(
- a: (&str, &Vec<i32>),
- b: (&str, &Vec<i32>),
- c: (&str, &Vec<i32>),
-) -> Arc<dyn ExecutionPlan> {
- let schema = Arc::new(Schema::new(vec![
- Field::new(a.0, DataType::Int32, false),
- Field::new(b.0, DataType::Int32, false),
- Field::new(c.0, DataType::Int32, false),
- ]));
- let batch = RecordBatch::try_new(
- Arc::clone(&schema),
- vec![
- Arc::new(Int32Array::from(a.1.clone())),
- Arc::new(Int32Array::from(b.1.clone())),
- Arc::new(Int32Array::from(c.1.clone())),
- ],
- )
- .unwrap();
- TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap()
-}
-
-// ── Async re-entry tests using PendingStream ──────────────────────────────
-// reproduce bugs that only manifest when async input streams return
-// Poll::Pending at specific points.
-
-/// A RecordBatch stream that yields Poll::Pending once before delivering
-/// each batch at a specified index. This simulates the behavior of
-/// repartitioned tokio::sync::mpsc channels where data isn't immediately
-/// available.
-struct PendingStream {
- batches: Vec<RecordBatch>,
- index: usize,
- /// If pending_before[i] is true, yield Pending once before delivering
- /// the batch at index i.
- pending_before: Vec<bool>,
- /// True if we've already yielded Pending for the current index.
- yielded_pending: bool,
- schema: SchemaRef,
-}
-
-impl PendingStream {
- fn new(batches: Vec<RecordBatch>, pending_before: Vec<bool>) -> Self {
- assert_eq!(batches.len(), pending_before.len());
- let schema = batches[0].schema();
- Self {
- batches,
- index: 0,
- pending_before,
- yielded_pending: false,
- schema,
- }
- }
-}
-
-impl Stream for PendingStream {
- type Item = Result<RecordBatch>;
-
- fn poll_next(
- mut self: Pin<&mut Self>,
- cx: &mut Context<'_>,
- ) -> Poll<Option<Self::Item>> {
- if self.index >= self.batches.len() {
- return Poll::Ready(None);
- }
- if self.pending_before[self.index] && !self.yielded_pending {
- self.yielded_pending = true;
- cx.waker().wake_by_ref();
- return Poll::Pending;
- }
- self.yielded_pending = false;
- let batch = self.batches[self.index].clone();
- self.index += 1;
- Poll::Ready(Some(Ok(batch)))
- }
-}
-
-impl RecordBatchStream for PendingStream {
- fn schema(&self) -> SchemaRef {
- Arc::clone(&self.schema)
- }
-}
-
-/// Helper: collect all output from a SemiAntiSortMergeJoinStream.
-async fn collect_stream(stream: SemiAntiSortMergeJoinStream) -> Result<Vec<RecordBatch>> {
- common::collect(Box::pin(stream)).await
-}
-
-/// Reproduces the buffer_inner_key_group re-entry bug:
-///
-/// When buffer_inner_key_group buffers inner rows across batch boundaries
-/// and poll_next_inner_batch returns Pending mid-way, the ready! macro
-/// exits poll_join. On re-entry, the merge-scan reaches Equal again and
-/// calls buffer_inner_key_group a second time — which starts with
-/// clear(), destroying the partially collected inner rows. Previously
-/// consumed batches are gone, so re-buffering misses them.
-///
-/// Setup:
-/// - Inner: 3 single-row batches, all with key=1, filter values c2=[10, 20, 30]
-/// - Outer: 1 row, key=1, filter value c1=10
-/// - Filter: c1 == c2 (only first inner row c2=10 matches)
-/// - Pending injected before 3rd inner batch
-///
-/// Without the bug: outer row emitted (match via c2=10)
-/// With the bug: outer row missing (c2=10 batch lost on re-entry)
-#[tokio::test]
-async fn filter_buffer_pending_loses_inner_rows() -> Result<()> {
- let left_schema = Arc::new(Schema::new(vec![
- Field::new("a1", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c1", DataType::Int32, false),
- ]));
- let right_schema = Arc::new(Schema::new(vec![
- Field::new("a2", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ]));
-
- // Outer: 1 row, key=1, c1=10
- let outer_batch = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![1])), // join key
- Arc::new(Int32Array::from(vec![10])), // filter value
- ],
- )?;
-
- // Inner: 3 single-row batches, key=1, c2=[10, 20, 30]
- let inner_batch1 = RecordBatch::try_new(
- Arc::clone(&right_schema),
- vec![
- Arc::new(Int32Array::from(vec![100])),
- Arc::new(Int32Array::from(vec![1])), // join key
- Arc::new(Int32Array::from(vec![10])), // matches filter
- ],
- )?;
- let inner_batch2 = RecordBatch::try_new(
- Arc::clone(&right_schema),
- vec![
- Arc::new(Int32Array::from(vec![200])),
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![20])), // doesn't match
- ],
- )?;
- let inner_batch3 = RecordBatch::try_new(
- Arc::clone(&right_schema),
- vec![
- Arc::new(Int32Array::from(vec![300])),
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![30])), // doesn't match
- ],
- )?;
-
- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
- vec![outer_batch],
- vec![false], // outer delivers immediately
- ));
- let inner: SendableRecordBatchStream = Box::pin(PendingStream::new(
- vec![inner_batch1, inner_batch2, inner_batch3],
- vec![false, false, true], // Pending before 3rd batch
- ));
-
- // Filter: c1 == c2
- let filter = JoinFilter::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("c1", 0)),
- Operator::Eq,
- Arc::new(Column::new("c2", 1)),
- )),
- vec![
- ColumnIndex {
- index: 2,
- side: JoinSide::Left,
- },
- ColumnIndex {
- index: 2,
- side: JoinSide::Right,
- },
- ],
- Arc::new(Schema::new(vec![
- Field::new("c1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ])),
- );
-
- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
-
- let metrics = ExecutionPlanMetricsSet::new();
- let inner_schema = inner.schema();
- let (reservation, peak_mem_used, spill_manager, runtime_env) =
- test_stream_resources(inner_schema, &metrics);
- let stream = SemiAntiSortMergeJoinStream::try_new(
- left_schema, // output schema = outer schema for semi
- vec![SortOptions::default()],
- NullEquality::NullEqualsNothing,
- outer,
- inner,
- on_outer,
- on_inner,
- Some(filter),
- LeftSemi,
- 8192,
- 0,
- &metrics,
- reservation,
- peak_mem_used,
- spill_manager,
- runtime_env,
- )?;
-
- let batches = collect_stream(stream).await?;
- let total: usize = batches.iter().map(|b| b.num_rows()).sum();
- assert_eq!(
- total, 1,
- "LeftSemi with filter: outer row should be emitted because \
- inner row c2=10 matches filter c1==c2. Got {total} rows."
- );
- Ok(())
-}
-
-/// Reproduces the no-filter boundary Pending re-entry bug:
-///
-/// When an outer key group spans a batch boundary, the no-filter path
-/// emits the current batch, then polls for the next outer batch. If
-/// poll returns Pending, poll_join exits. On re-entry, without the
-/// PendingBoundary fix, the new batch is processed fresh by the
-/// merge-scan. Since inner already advanced past this key, the outer
-/// rows with the matching key are skipped via Ordering::Less.
-///
-/// Setup:
-/// - Outer: 2 single-row batches, both with key=1 (key group spans boundary)
-/// - Inner: 1 row with key=1
-/// - Pending injected on outer before 2nd batch
-///
-/// Without fix: only first outer row emitted (second lost on re-entry)
-/// With fix: both outer rows emitted
-#[tokio::test]
-async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {
- let left_schema = Arc::new(Schema::new(vec![
- Field::new("a1", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c1", DataType::Int32, false),
- ]));
- let right_schema = Arc::new(Schema::new(vec![
- Field::new("a2", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ]));
-
- // Outer: 2 single-row batches, both key=1
- let outer_batch1 = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![10])),
- ],
- )?;
- let outer_batch2 = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![2])),
- Arc::new(Int32Array::from(vec![1])), // same key
- Arc::new(Int32Array::from(vec![20])),
- ],
- )?;
-
- // Inner: 1 row, key=1
- let inner_batch = RecordBatch::try_new(
- Arc::clone(&right_schema),
- vec![
- Arc::new(Int32Array::from(vec![100])),
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![50])),
- ],
- )?;
-
- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
- vec![outer_batch1, outer_batch2],
- vec![false, true], // Pending before 2nd outer batch
- ));
- let inner: SendableRecordBatchStream =
- Box::pin(PendingStream::new(vec![inner_batch], vec![false]));
-
- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
-
- let metrics = ExecutionPlanMetricsSet::new();
- let inner_schema = inner.schema();
- let (reservation, peak_mem_used, spill_manager, runtime_env) =
- test_stream_resources(inner_schema, &metrics);
- let stream = SemiAntiSortMergeJoinStream::try_new(
- left_schema,
- vec![SortOptions::default()],
- NullEquality::NullEqualsNothing,
- outer,
- inner,
- on_outer,
- on_inner,
- None, // no filter
- LeftSemi,
- 8192,
- 0,
- &metrics,
- reservation,
- peak_mem_used,
- spill_manager,
- runtime_env,
- )?;
-
- let batches = collect_stream(stream).await?;
- let total: usize = batches.iter().map(|b| b.num_rows()).sum();
- assert_eq!(
- total, 2,
- "LeftSemi no filter: both outer rows (key=1) should be emitted \
- because inner has key=1. Got {total} rows."
- );
- Ok(())
-}
-
-/// Tests the filtered boundary Pending re-entry: outer key group spans
-/// batches with a filter, and poll_next_outer_batch returns Pending.
-///
-/// Setup:
-/// - Outer: 2 single-row batches, both key=1, c1=[10, 20]
-/// - Inner: 1 row, key=1, c2=10
-/// - Filter: c1 == c2 (first outer row matches, second doesn't)
-/// - Pending before 2nd outer batch
-///
-/// Expected: 1 row (only the first outer row c1=10 passes the filter)
-#[tokio::test]
-async fn filtered_boundary_pending_outer_rows() -> Result<()> {
- let left_schema = Arc::new(Schema::new(vec![
- Field::new("a1", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c1", DataType::Int32, false),
- ]));
- let right_schema = Arc::new(Schema::new(vec![
- Field::new("a2", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ]));
-
- let outer_batch1 = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![10])), // matches filter
- ],
- )?;
- let outer_batch2 = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![2])),
- Arc::new(Int32Array::from(vec![1])), // same key
- Arc::new(Int32Array::from(vec![20])), // doesn't match
- ],
- )?;
-
- let inner_batch = RecordBatch::try_new(
- Arc::clone(&right_schema),
- vec![
- Arc::new(Int32Array::from(vec![100])),
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![10])),
- ],
- )?;
-
- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
- vec![outer_batch1, outer_batch2],
- vec![false, true], // Pending before 2nd outer batch
- ));
- let inner: SendableRecordBatchStream =
- Box::pin(PendingStream::new(vec![inner_batch], vec![false]));
-
- let filter = JoinFilter::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("c1", 0)),
- Operator::Eq,
- Arc::new(Column::new("c2", 1)),
- )),
- vec![
- ColumnIndex {
- index: 2,
- side: JoinSide::Left,
- },
- ColumnIndex {
- index: 2,
- side: JoinSide::Right,
- },
- ],
- Arc::new(Schema::new(vec![
- Field::new("c1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ])),
- );
-
- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
-
- let metrics = ExecutionPlanMetricsSet::new();
- let inner_schema = inner.schema();
- let (reservation, peak_mem_used, spill_manager, runtime_env) =
- test_stream_resources(inner_schema, &metrics);
- let stream = SemiAntiSortMergeJoinStream::try_new(
- left_schema,
- vec![SortOptions::default()],
- NullEquality::NullEqualsNothing,
- outer,
- inner,
- on_outer,
- on_inner,
- Some(filter),
- LeftSemi,
- 8192,
- 0,
- &metrics,
- reservation,
- peak_mem_used,
- spill_manager,
- runtime_env,
- )?;
-
- let batches = collect_stream(stream).await?;
- let total: usize = batches.iter().map(|b| b.num_rows()).sum();
- assert_eq!(
- total, 1,
- "LeftSemi filtered boundary: only first outer row (c1=10) matches \
- filter c1==c2. Got {total} rows."
- );
- Ok(())
-}
-
-// ==================== SPILL TESTS ====================
-
-/// Exercises inner key group spilling under memory pressure.
-///
-/// Uses a tiny memory limit (100 bytes) with disk spilling enabled. Since our
-/// operator only buffers inner rows when a filter is present, this test includes
-/// a filter (c1 < c2, always true). Verifies:
-/// 1. Spill metrics are recorded (spill_count, spilled_bytes, spilled_rows > 0)
-/// 2. Results match a non-spilled run
-#[tokio::test]
-async fn spill_with_filter() -> Result<()> {
- use datafusion_execution::config::SessionConfig;
- use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
- use datafusion_execution::runtime_env::RuntimeEnvBuilder;
-
- let left = build_table(
- ("a1", &vec![1, 2, 3, 4, 5, 6]),
- ("b1", &vec![1, 2, 3, 4, 5, 6]),
- ("c1", &vec![4, 5, 6, 7, 8, 9]),
- );
- let right = build_table(
- ("a2", &vec![10, 20, 30, 40, 50]),
- ("b1", &vec![1, 3, 4, 6, 8]),
- ("c2", &vec![50, 60, 70, 80, 90]),
- );
- let on = vec![(
- Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
- Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
- )];
- let sort_options = vec![SortOptions::default(); on.len()];
-
- // c1 < c2 is always true for matching keys
- let filter = JoinFilter::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("c1", 0)),
- Operator::Lt,
- Arc::new(Column::new("c2", 1)),
- )),
- vec![
- ColumnIndex {
- index: 2,
- side: JoinSide::Left,
- },
- ColumnIndex {
- index: 2,
- side: JoinSide::Right,
- },
- ],
- Arc::new(Schema::new(vec![
- Field::new("c1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ])),
- );
-
- let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(100, 1.0)
- .with_disk_manager_builder(
- DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
- )
- .build_arc()?;
-
- for batch_size in [1, 50] {
- let session_config = SessionConfig::default().with_batch_size(batch_size);
-
- for join_type in [LeftSemi, LeftAnti, RightSemi, RightAnti] {
- let task_ctx = Arc::new(
- TaskContext::default()
- .with_session_config(session_config.clone())
- .with_runtime(Arc::clone(&runtime)),
- );
-
- let join = SortMergeJoinExec::try_new(
- Arc::clone(&left),
- Arc::clone(&right),
- on.clone(),
- Some(filter.clone()),
- join_type,
- sort_options.clone(),
- NullEquality::NullEqualsNothing,
- )?;
- let stream = join.execute(0, task_ctx)?;
- let spilled_result = common::collect(stream).await.unwrap();
-
- assert!(
- join.metrics().is_some(),
- "metrics missing for {join_type:?}"
- );
- let metrics = join.metrics().unwrap();
- assert!(
- metrics.spill_count().unwrap() > 0,
- "expected spill_count > 0 for {join_type:?}, batch_size={batch_size}"
- );
- assert!(
- metrics.spilled_bytes().unwrap() > 0,
- "expected spilled_bytes > 0 for {join_type:?}, batch_size={batch_size}"
- );
- assert!(
- metrics.spilled_rows().unwrap() > 0,
- "expected spilled_rows > 0 for {join_type:?}, batch_size={batch_size}"
- );
-
- // Run without spilling and compare results
- let task_ctx_no_spill = Arc::new(
- TaskContext::default().with_session_config(session_config.clone()),
- );
- let join_no_spill = SortMergeJoinExec::try_new(
- Arc::clone(&left),
- Arc::clone(&right),
- on.clone(),
- Some(filter.clone()),
- join_type,
- sort_options.clone(),
- NullEquality::NullEqualsNothing,
- )?;
- let stream = join_no_spill.execute(0, task_ctx_no_spill)?;
- let no_spill_result = common::collect(stream).await.unwrap();
-
- let no_spill_metrics = join_no_spill.metrics().unwrap();
- assert_eq!(
- no_spill_metrics.spill_count(),
- Some(0),
- "unexpected spill for {join_type:?} without memory limit"
- );
-
- assert_eq!(
- spilled_result, no_spill_result,
- "spilled vs non-spilled results differ for {join_type:?}, batch_size={batch_size}"
- );
- }
- }
-
- Ok(())
-}
-
-/// Reproduces a bug where `resume_boundary` for the Filtered pending case
-/// only checks `inner_key_buffer.is_empty()` but ignores `inner_key_spill`.
-/// After spilling, the in-memory buffer is cleared while the spill file
-/// holds the data. If the outer key group spans a batch boundary, the
-/// second outer batch's rows are never evaluated against the inner group.
-///
-/// Setup:
-/// - Outer: 2 single-row batches, both key=1, c1=[10, 10]
-/// - Inner: 1 batch with many rows all key=1 (enough to trigger spill)
-/// - Filter: c1 == c2 (matches when c2=10)
-/// - Memory limit: tiny (100 bytes) to force spilling
-/// - Pending before 2nd outer batch to trigger boundary re-entry
-///
-/// Expected: both outer rows match (semi=2 rows, anti=0 rows)
-/// Bug: second outer row is skipped because resume_boundary sees empty
-/// inner_key_buffer and skips re-evaluation.
-#[tokio::test]
-async fn spill_filtered_boundary_loses_outer_rows() -> Result<()> {
- use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
- use datafusion_execution::runtime_env::RuntimeEnvBuilder;
-
- let left_schema = Arc::new(Schema::new(vec![
- Field::new("a1", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c1", DataType::Int32, false),
- ]));
- let right_schema = Arc::new(Schema::new(vec![
- Field::new("a2", DataType::Int32, false),
- Field::new("b1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ]));
-
- // Two single-row outer batches with the same key — key group spans boundary
- let outer_batch1 = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![1])),
- Arc::new(Int32Array::from(vec![1])), // key=1
- Arc::new(Int32Array::from(vec![10])), // matches filter
- ],
- )?;
- let outer_batch2 = RecordBatch::try_new(
- Arc::clone(&left_schema),
- vec![
- Arc::new(Int32Array::from(vec![2])),
- Arc::new(Int32Array::from(vec![1])), // same key=1
- Arc::new(Int32Array::from(vec![10])), // also matches filter
- ],
- )?;
-
- // Inner: many rows with key=1 to force spilling, followed by key=2.
- // c2=10 so the filter c1==c2 passes for both outer rows.
- // The key=2 row ensures the inner cursor advances past the key group
- // (buffer_inner_key_group returns Ok(false) instead of Ok(true)).
- let n_inner = 200;
- let mut inner_a = vec![100; n_inner];
- inner_a.push(101);
- let mut inner_b = vec![1; n_inner];
- inner_b.push(2); // different key — forces inner cursor past key=1
- let mut inner_c = vec![10; n_inner];
- inner_c.push(10);
- let inner_batch = RecordBatch::try_new(
- Arc::clone(&right_schema),
- vec![
- Arc::new(Int32Array::from(inner_a)),
- Arc::new(Int32Array::from(inner_b)),
- Arc::new(Int32Array::from(inner_c)),
- ],
- )?;
-
- // Filter: c1 == c2
- let filter = JoinFilter::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("c1", 0)),
- Operator::Eq,
- Arc::new(Column::new("c2", 1)),
- )),
- vec![
- ColumnIndex {
- index: 2,
- side: JoinSide::Left,
- },
- ColumnIndex {
- index: 2,
- side: JoinSide::Right,
- },
- ],
- Arc::new(Schema::new(vec![
- Field::new("c1", DataType::Int32, false),
- Field::new("c2", DataType::Int32, false),
- ])),
- );
-
- let runtime = RuntimeEnvBuilder::new()
- .with_memory_limit(100, 1.0)
- .with_disk_manager_builder(
- DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
- )
- .build_arc()?;
-
- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
-
- for join_type in [LeftSemi, LeftAnti] {
- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
- vec![outer_batch1.clone(), outer_batch2.clone()],
- vec![false, true], // Pending before 2nd outer batch
- ));
- let inner: SendableRecordBatchStream =
- Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));
-
- let metrics = ExecutionPlanMetricsSet::new();
- let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);
- let peak_mem_used = MetricBuilder::new(&metrics).gauge("peak_mem_used", 0);
- let spill_manager = SpillManager::new(
- Arc::clone(&runtime),
- SpillMetrics::new(&metrics, 0),
- Arc::clone(&right_schema),
- );
-
- let stream = SemiAntiSortMergeJoinStream::try_new(
- Arc::clone(&left_schema),
- vec![SortOptions::default()],
- NullEquality::NullEqualsNothing,
- outer,
- inner,
- on_outer.clone(),
- on_inner.clone(),
- Some(filter.clone()),
- join_type,
- 8192,
- 0,
- &metrics,
- reservation,
- peak_mem_used,
- spill_manager,
- Arc::clone(&runtime),
- )?;
-
- let batches = collect_stream(stream).await?;
- let total: usize = batches.iter().map(|b| b.num_rows()).sum();
-
- match join_type {
- LeftSemi => {
- assert_eq!(
- total, 2,
- "LeftSemi spill+boundary: both outer rows match filter, \
- expected 2 rows, got {total}"
- );
- }
- LeftAnti => {
- assert_eq!(
- total, 0,
- "LeftAnti spill+boundary: both outer rows match filter, \
- expected 0 rows, got {total}"
- );
- }
- _ => unreachable!(),
- }
- }
-
- Ok(())
-}
diff --git a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
similarity index 93%
rename from datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/stream.rs
rename to datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
index 40e2022d41..2f7c9acb9d 100644
--- a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
@@ -15,14 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-//! Sort-merge join stream specialized for semi/anti joins.
+//! Sort-merge join stream specialized for semi/anti/mark joins.
//!
//! Instantiated by [`SortMergeJoinExec`](crate::joins::sort_merge_join::SortMergeJoinExec)
-//! when the join type is `LeftSemi`, `LeftAnti`, `RightSemi`, or `RightAnti`.
+//! when the join type is `LeftSemi`, `LeftAnti`, `RightSemi`, `RightAnti`,
+//! `LeftMark`, or `RightMark`.
//!
//! # Motivation
//!
-//! The general-purpose `SortMergeJoinStream`
+//! The general-purpose `MaterializingSortMergeJoinStream`
//! handles semi/anti joins by materializing `(outer, inner)` row pairs,
//! applying a filter, then using a "corrected filter mask" to deduplicate.
//! Semi/anti joins only need a boolean per outer row (does a match exist?),
@@ -36,7 +37,8 @@
//!
//! For `Left*` join types, left is outer and right is inner.
//! For `Right*` join types, right is outer and left is inner.
-//! The output schema always equals the outer side's schema.
+//! The output schema always equals the outer side's schema (for semi/anti)
+//! or the outer side's schema plus a boolean mark column (for mark joins).
//!
//! # Algorithm
//!
@@ -75,6 +77,7 @@
//! On emit:
//! Semi → filter_record_batch(outer_batch, &matched)
//! Anti → filter_record_batch(outer_batch, &NOT(matched))
+//! Mark → outer_batch + matched as boolean column
//! ```
//!
//! ## Batch boundaries
@@ -245,14 +248,18 @@ enum PendingBoundary {
Filtered { saved_keys: Vec<ArrayRef> },
}
-pub(crate) struct SemiAntiSortMergeJoinStream {
- // Decomposed from JoinType to avoid matching on 4 variants in hot paths.
- // true for semi (emit matched), false for anti (emit unmatched).
- is_semi: bool,
+/// Sort-Merge join stream for Semi/Anti/Mark joins.
+///
+/// Named "bitwise" because it tracks outer-row matches via a per-batch
+/// boolean bitset (`BooleanBufferBuilder`) rather than materializing
+/// `(outer, inner)` row pairs. Filter results are OR'd into the bitset
+/// in `u64` chunks, and emitting applies the bitset directly.
+pub(crate) struct BitwiseSortMergeJoinStream {
+ join_type: JoinType,
// Input streams — in the nested-loop model that sort-merge join
// implements, "outer" is the driving loop and "inner" is probed for
- // matches. The existing SortMergeJoinStream calls these "streamed"
+ // matches. The existing MaterializingSortMergeJoinStream calls these "streamed"
// and "buffered" respectively. For Left* joins, outer=left; for
// Right* joins, outer=right. Output schema equals the outer side.
outer: SendableRecordBatchStream,
@@ -330,7 +337,7 @@ pub(crate) struct SemiAntiSortMergeJoinStream {
batch_emitted: bool,
}
-impl SemiAntiSortMergeJoinStream {
+impl BitwiseSortMergeJoinStream {
#[expect(clippy::too_many_arguments)]
pub fn try_new(
schema: SchemaRef,
@@ -346,21 +353,35 @@ impl SemiAntiSortMergeJoinStream {
partition: usize,
metrics: &ExecutionPlanMetricsSet,
reservation: MemoryReservation,
- peak_mem_used: Gauge,
spill_manager: SpillManager,
runtime_env: Arc<datafusion_execution::runtime_env::RuntimeEnv>,
) -> Result<Self> {
- let is_semi = matches!(join_type, JoinType::LeftSemi | JoinType::RightSemi);
- let outer_is_left = matches!(join_type, JoinType::LeftSemi | JoinType::LeftAnti);
+ debug_assert!(
+ matches!(
+ join_type,
+ JoinType::LeftSemi
+ | JoinType::RightSemi
+ | JoinType::LeftAnti
+ | JoinType::RightAnti
+ | JoinType::LeftMark
+ | JoinType::RightMark
+ ),
+ "BitwiseSortMergeJoinStream does not handle {join_type:?}"
+ );
+ let outer_is_left = matches!(
+ join_type,
+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark
+ );
let join_time = MetricBuilder::new(metrics).subset_time("join_time", partition);
let input_batches =
MetricBuilder::new(metrics).counter("input_batches", partition);
let input_rows = MetricBuilder::new(metrics).counter("input_rows", partition);
let baseline_metrics = BaselineMetrics::new(metrics, partition);
+ let peak_mem_used = MetricBuilder::new(metrics).gauge("peak_mem_used", partition);
Ok(Self {
- is_semi,
+ join_type,
outer,
inner,
outer_batch: None,
@@ -492,17 +513,38 @@ impl SemiAntiSortMergeJoinStream {
// finish() converts the bit-packed builder directly to a
// BooleanBuffer — no iteration or repacking needed.
- let selection = BooleanArray::new(self.matched.finish(), None);
-
- let selection = if self.is_semi {
- selection
- } else {
- not(&selection)?
- };
-
- let filtered = filter_record_batch(batch, &selection)?;
- if filtered.num_rows() > 0 {
- self.coalescer.push_batch(filtered)?;
+ let matched_buf = self.matched.finish();
+
+ match self.join_type {
+ JoinType::LeftMark | JoinType::RightMark => {
+ // Mark joins emit ALL outer rows with a boolean match column appended.
+ debug_assert_eq!(
+ self.schema.fields().len(),
+ batch.num_columns() + 1,
+ "Mark join output schema should be outer schema + 1 mark column"
+ );
+ let mark_col = Arc::new(BooleanArray::new(matched_buf, None)) as ArrayRef;
+ let mut columns = Vec::with_capacity(batch.num_columns() + 1);
+ columns.extend_from_slice(batch.columns());
+ columns.push(mark_col);
+ let output = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
+ self.coalescer.push_batch(output)?;
+ }
+ JoinType::LeftSemi | JoinType::RightSemi => {
+ let selection = BooleanArray::new(matched_buf, None);
+ let filtered = filter_record_batch(batch, &selection)?;
+ if filtered.num_rows() > 0 {
+ self.coalescer.push_batch(filtered)?;
+ }
+ }
+ JoinType::LeftAnti | JoinType::RightAnti => {
+ let selection = not(&BooleanArray::new(matched_buf, None))?;
+ let filtered = filter_record_batch(batch, &selection)?;
+ if filtered.num_rows() > 0 {
+ self.coalescer.push_batch(filtered)?;
+ }
+ }
+ _ => unreachable!(),
}
Ok(())
}
@@ -1184,7 +1226,7 @@ fn keys_match(
/// Evaluate the join filter for one inner row against a slice of outer rows.
///
-/// Free function (not a method on SemiAntiSortMergeJoinStream) so that Rust
+/// Free function (not a method on BitwiseSortMergeJoinStream) so that Rust
/// can split the struct borrow in process_key_match_with_filter: the caller
/// holds &mut self.matched and &self.inner_key_buffer simultaneously, which
/// is impossible if this borrows all of &self.
@@ -1257,7 +1299,7 @@ fn evaluate_filter_for_inner_row(
}
}
-impl Stream for SemiAntiSortMergeJoinStream {
+impl Stream for BitwiseSortMergeJoinStream {
type Item = Result<RecordBatch>;
fn poll_next(
@@ -1269,7 +1311,7 @@ impl Stream for SemiAntiSortMergeJoinStream {
}
}
-impl RecordBatchStream for SemiAntiSortMergeJoinStream {
+impl RecordBatchStream for BitwiseSortMergeJoinStream {
fn schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index d5d1df4baf..e2c169e558 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -23,17 +23,17 @@ use std::any::Any;
use std::fmt::Formatter;
use std::sync::Arc;
+use super::bitwise_stream::BitwiseSortMergeJoinStream;
+use super::materializing_stream::MaterializingSortMergeJoinStream;
+use super::metrics::SortMergeJoinMetrics;
use crate::execution_plan::{EmissionType, boundedness_from_children};
use crate::expressions::PhysicalSortExpr;
-use crate::joins::semi_anti_sort_merge_join::stream::SemiAntiSortMergeJoinStream;
-use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;
-use crate::joins::sort_merge_join::stream::SortMergeJoinStream;
use crate::joins::utils::{
JoinFilter, JoinOn, JoinOnRef, build_join_schema, check_join_is_valid,
estimate_join_statistics, reorder_output_after_swap,
symmetric_join_output_partitioning,
};
-use crate::metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet, SpillMetrics};
+use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet, SpillMetrics};
use crate::projection::{
ProjectionExec, join_allows_pushdown, join_table_borders, new_join_children,
physical_to_column_exprs, update_join_on,
@@ -337,6 +337,8 @@ impl SortMergeJoinExec {
| JoinType::RightSemi
| JoinType::LeftAnti
| JoinType::RightAnti
+ | JoinType::LeftMark
+ | JoinType::RightMark
) {
Ok(Arc::new(new_join))
} else {
@@ -527,6 +529,12 @@ impl ExecutionPlan for SortMergeJoinExec {
let batch_size = context.session_config().batch_size();
let reservation = MemoryConsumer::new(format!("SMJStream[{partition}]"))
.register(context.memory_pool());
+ let spill_manager = SpillManager::new(
+ context.runtime_env(),
+ SpillMetrics::new(&self.metrics, partition),
+ buffered.schema(),
+ )
+ .with_compression_type(context.session_config().spill_compression());
if matches!(
self.join_type,
@@ -534,17 +542,10 @@ impl ExecutionPlan for SortMergeJoinExec {
| JoinType::LeftAnti
| JoinType::RightSemi
| JoinType::RightAnti
+ | JoinType::LeftMark
+ | JoinType::RightMark
) {
- let peak_mem_used =
- MetricBuilder::new(&self.metrics).gauge("peak_mem_used", partition);
- let spill_manager = SpillManager::new(
- context.runtime_env(),
- SpillMetrics::new(&self.metrics, partition),
- buffered.schema(),
- )
- .with_compression_type(context.session_config().spill_compression());
-
- Ok(Box::pin(SemiAntiSortMergeJoinStream::try_new(
+ Ok(Box::pin(BitwiseSortMergeJoinStream::try_new(
Arc::clone(&self.schema),
self.sort_options.clone(),
self.null_equality,
@@ -558,13 +559,11 @@ impl ExecutionPlan for SortMergeJoinExec {
partition,
&self.metrics,
reservation,
- peak_mem_used,
spill_manager,
context.runtime_env(),
)?))
} else {
- Ok(Box::pin(SortMergeJoinStream::try_new(
- context.session_config().spill_compression(),
+ Ok(Box::pin(MaterializingSortMergeJoinStream::try_new(
Arc::clone(&self.schema),
self.sort_options.clone(),
self.null_equality,
@@ -577,6 +576,7 @@ impl ExecutionPlan for SortMergeJoinExec {
batch_size,
SortMergeJoinMetrics::new(partition, &self.metrics),
reservation,
+ spill_manager,
context.runtime_env(),
)?))
}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs b/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs
index d598442b65..3a5f0d0f05 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs
@@ -19,7 +19,7 @@
//!
//! This module encapsulates the complexity of join filter evaluation, including:
//! - Immediate filtering for INNER joins
-//! - Deferred filtering for outer/semi/anti/mark joins
+//! - Deferred filtering for outer joins
//! - Metadata tracking for grouping output rows by input row
//! - Correcting filter masks to handle multiple matches per input row
@@ -27,9 +27,10 @@ use std::sync::Arc;
use arrow::array::{
Array, ArrayBuilder, ArrayRef, BooleanArray, BooleanBuilder, RecordBatch,
- UInt64Array, UInt64Builder,
+ RecordBatchOptions, UInt64Array, UInt64Builder, new_null_array,
};
-use arrow::compute::{self, concat_batches, filter_record_batch};
+use arrow::compute::kernels::zip::zip;
+use arrow::compute::{self, filter_record_batch};
use arrow::datatypes::SchemaRef;
use datafusion_common::{JoinSide, JoinType, Result};
@@ -38,9 +39,9 @@ use crate::joins::utils::JoinFilter;
/// Metadata for tracking filter results during deferred filtering
///
/// When a join filter is present and we need to ensure each input row produces
-/// at least one output (outer joins) or exactly one output (semi joins), we can't
-/// filter immediately. Instead, we accumulate all joined rows with metadata,
-/// then post-process to determine which rows to output.
+/// at least one output (outer joins), we can't filter immediately. Instead,
+/// we accumulate all joined rows with metadata, then post-process to determine
+/// which rows to output.
#[derive(Debug)]
pub struct FilterMetadata {
/// Did each output row pass the join filter?
@@ -96,23 +97,10 @@ impl FilterMetadata {
"row_indices and filter_mask must have same length"
);
- for i in 0..row_indices.len() {
- if filter_mask.is_null(i) {
- self.filter_mask.append_null();
- } else if filter_mask.value(i) {
- self.filter_mask.append_value(true);
- } else {
- self.filter_mask.append_value(false);
- }
-
- if row_indices.is_null(i) {
- self.row_indices.append_null();
- } else {
- self.row_indices.append_value(row_indices.value(i));
- }
-
- self.batch_ids.push(batch_id);
- }
+ self.filter_mask.extend(filter_mask);
+ self.row_indices.extend(row_indices);
+ self.batch_ids
+ .resize(self.batch_ids.len() + row_indices.len(), batch_id);
}
/// Verify that metadata arrays are aligned (same length)
@@ -149,24 +137,12 @@ impl Default for FilterMetadata {
/// Deferred filtering is required when:
/// - A filter exists AND
/// - The join type requires ensuring each input row produces at least one output
-/// (or exactly one for semi joins)
pub fn needs_deferred_filtering(
filter: &Option<JoinFilter>,
join_type: JoinType,
) -> bool {
filter.is_some()
- && matches!(
- join_type,
- JoinType::Left
- | JoinType::LeftSemi
- | JoinType::LeftMark
- | JoinType::Right
- | JoinType::RightSemi
- | JoinType::RightMark
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::Full
- )
+ && matches!(join_type, JoinType::Left | JoinType::Right | JoinType::Full)
}
/// Gets the arrays which join filters are applied on
@@ -253,9 +229,6 @@ fn last_index_for_row(
/// This function groups them by input row and applies join-type-specific logic:
///
/// - **Outer joins**: Keep first matching row, convert rest to nulls, add null-joined for unmatched
-/// - **Semi joins**: Keep first matching row, discard rest
-/// - **Anti joins**: Keep row only if NO matches passed filter
-/// - **Mark joins**: Like semi but first match only
///
/// # Arguments
/// * `join_type` - The type of join being performed
@@ -267,7 +240,7 @@ fn last_index_for_row(
/// # Returns
/// Corrected mask indicating which rows to include in final output:
/// - `true`: Include this row
-/// - `false`: Convert to null-joined row (outer joins) or include as unmatched (anti joins)
+/// - `false`: Convert to null-joined row (outer joins)
/// - `null`: Discard this row
pub fn get_corrected_filter_mask(
join_type: JoinType,
@@ -282,314 +255,134 @@ pub fn get_corrected_filter_mask(
let mut seen_true = false;
match join_type {
- JoinType::Left | JoinType::Right => {
- // For outer joins: Keep first matching row per input row,
- // convert rest to nulls, add null-joined rows for unmatched
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- if filter_mask.value(i) {
- seen_true = true;
- corrected_mask.append_value(true);
- } else if seen_true || !filter_mask.value(i) && !last_index {
- corrected_mask.append_null(); // to be ignored and not set to output
- } else {
- corrected_mask.append_value(false); // to be converted to null joined row
- }
-
- if last_index {
- seen_true = false;
- }
- }
-
- // Generate null joined rows for records which have no matching join key
- corrected_mask.append_n(expected_size - corrected_mask.len(), false);
- Some(corrected_mask.finish())
- }
- JoinType::LeftMark | JoinType::RightMark => {
- // For mark joins: Like outer but only keep first match, mark with boolean
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- if filter_mask.value(i) && !seen_true {
- seen_true = true;
- corrected_mask.append_value(true);
- } else if seen_true || !filter_mask.value(i) && !last_index {
- corrected_mask.append_null(); // to be ignored and not set to output
- } else {
- corrected_mask.append_value(false); // to be converted to null joined row
- }
-
- if last_index {
- seen_true = false;
- }
- }
-
- // Generate null joined rows for records which have no matching join key
- corrected_mask.append_n(expected_size - corrected_mask.len(), false);
- Some(corrected_mask.finish())
- }
- JoinType::LeftSemi | JoinType::RightSemi => {
- // For semi joins: Keep only first matching row per input row, discard rest
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- if filter_mask.value(i) && !seen_true {
- seen_true = true;
- corrected_mask.append_value(true);
- } else {
- corrected_mask.append_null(); // to be ignored and not set to output
- }
-
- if last_index {
- seen_true = false;
- }
- }
-
- Some(corrected_mask.finish())
- }
- JoinType::LeftAnti | JoinType::RightAnti => {
- // For anti joins: Keep row only if NO matches passed the filter
+ JoinType::Left | JoinType::Right | JoinType::Full => {
+ // For each input row group: keep first filter-passing row,
+ // discard (null) remaining matches, null-join if none passed.
+ // Null metadata entries are already-null-joined rows that
+ // flow through unchanged to preserve output ordering.
for i in 0..row_indices_length {
let last_index =
last_index_for_row(i, row_indices, batch_ids, row_indices_length);
-
- if filter_mask.value(i) {
- seen_true = true;
- }
-
- if last_index {
- if !seen_true {
- corrected_mask.append_value(true);
- } else {
- corrected_mask.append_null();
- }
-
- seen_true = false;
- } else {
- corrected_mask.append_null();
- }
- }
- // Generate null joined rows for records which have no matching join key,
- // for LeftAnti non-matched considered as true
- corrected_mask.append_n(expected_size - corrected_mask.len(), true);
- Some(corrected_mask.finish())
- }
- JoinType::Full => {
- // For full joins: Similar to outer but handle both sides
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
-
if filter_mask.is_null(i) {
- // null joined
corrected_mask.append_value(true);
} else if filter_mask.value(i) {
seen_true = true;
corrected_mask.append_value(true);
} else if seen_true || !filter_mask.value(i) && !last_index {
- corrected_mask.append_null(); // to be ignored and not set to output
+ corrected_mask.append_null();
} else {
- corrected_mask.append_value(false); // to be converted to null joined row
+ corrected_mask.append_value(false);
}
if last_index {
seen_true = false;
}
}
- // Generate null joined rows for records which have no matching join key
+
corrected_mask.append_n(expected_size - corrected_mask.len(), false);
Some(corrected_mask.finish())
}
- JoinType::Inner => {
- // Inner joins don't need deferred filtering
- None
+ JoinType::LeftMark
+ | JoinType::RightMark
+ | JoinType::LeftSemi
+ | JoinType::RightSemi
+ | JoinType::LeftAnti
+ | JoinType::RightAnti => {
+ unreachable!("Semi/anti/mark joins are handled by BitwiseSortMergeJoinStream")
}
+ JoinType::Inner => None,
}
}
/// Applies corrected filter mask to record batch based on join type
///
-/// Different join types require different handling of filtered results:
-/// - Outer joins: Add null-joined rows for false mask values
-/// - Semi/Anti joins: May need projection to remove right columns
-/// - Full joins: Add null-joined rows for both sides
+/// The corrected mask has three possible values per row:
+/// - `true`: Keep the row as-is (matched and passed filter)
+/// - `false`: Convert to null-joined row (all filter matches failed for this input row)
+/// - `null`: Discard the row entirely (duplicate match for an already-output input row)
+///
+/// This function preserves input row ordering by processing each row in place
+/// rather than separating matched/unmatched rows.
pub fn filter_record_batch_by_join_type(
record_batch: &RecordBatch,
corrected_mask: &BooleanArray,
join_type: JoinType,
schema: &SchemaRef,
- streamed_schema: &SchemaRef,
buffered_schema: &SchemaRef,
) -> Result<RecordBatch> {
- let filtered_record_batch = filter_record_batch(record_batch, corrected_mask)?;
-
match join_type {
- JoinType::Left | JoinType::LeftMark => {
- // For left joins, add null-joined rows where mask is false
- let null_mask = compute::not(corrected_mask)?;
- let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;
+ JoinType::Left | JoinType::Right | JoinType::Full => {
+ if record_batch.num_rows() == 0 {
+ return Ok(record_batch.clone());
+ }
+
+ // Discard null-masked rows (keep true + false only)
+ let keep_mask = compute::is_not_null(corrected_mask)?;
+ let kept_batch = filter_record_batch(record_batch, &keep_mask)?;
- if null_joined_batch.num_rows() == 0 {
- return Ok(filtered_record_batch);
+ if kept_batch.num_rows() == 0 {
+ return Ok(kept_batch);
}
- // Create null columns for right side
- let null_joined_streamed_batch = create_null_joined_batch(
- &null_joined_batch,
- buffered_schema,
- JoinSide::Left,
- join_type,
- schema,
- )?;
-
- Ok(concat_batches(
- schema,
- &[filtered_record_batch, null_joined_streamed_batch],
- )?)
- }
- JoinType::LeftSemi
- | JoinType::LeftAnti
- | JoinType::RightSemi
- | JoinType::RightAnti => {
- // For semi/anti joins, project to only include the outer side columns
- // Both Left and Right semi/anti use streamed_schema.len() because:
- // - For Left: columns are [left, right], so we take first streamed_schema.len()
- // - For Right: columns are [right, left], and streamed side is right, so we take first streamed_schema.len()
- let output_column_indices: Vec<usize> =
- (0..streamed_schema.fields().len()).collect();
- Ok(filtered_record_batch.project(&output_column_indices)?)
- }
- JoinType::Right | JoinType::RightMark => {
- // For right joins, add null-joined rows where mask is false
- let null_mask = compute::not(corrected_mask)?;
- let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;
+ let kept_corrected = compute::filter(corrected_mask, &keep_mask)?;
+ let kept_corrected = kept_corrected
+ .as_any()
+ .downcast_ref::<BooleanArray>()
+ .unwrap();
- if null_joined_batch.num_rows() == 0 {
- return Ok(filtered_record_batch);
+ // All rows passed the filter — no null-joining needed
+ if kept_corrected.true_count() == kept_corrected.len() {
+ return Ok(kept_batch);
}
- // Create null columns for left side (buffered side for RIGHT join)
- let null_joined_buffered_batch = create_null_joined_batch(
- &null_joined_batch,
- buffered_schema, // Pass buffered (left) schema to create nulls for it
- JoinSide::Right,
- join_type,
- schema,
- )?;
-
- Ok(concat_batches(
- schema,
- &[filtered_record_batch, null_joined_buffered_batch],
- )?)
- }
- JoinType::Full => {
- // For full joins, add null-joined rows for both sides
- let joined_filter_not_matched_mask = compute::not(corrected_mask)?;
- let joined_filter_not_matched_batch =
- filter_record_batch(record_batch, &joined_filter_not_matched_mask)?;
-
- if joined_filter_not_matched_batch.num_rows() == 0 {
- return Ok(filtered_record_batch);
+ // For false entries: replace the non-preserved side with nulls.
+ // This preserves row ordering unlike filter+concat.
+ let (null_side_start, null_side_len) = match join_type {
+ JoinType::Left => {
+ // Left join: null out right (buffered) columns
+ let left_cols =
+ schema.fields().len() - buffered_schema.fields().len();
+ (left_cols, buffered_schema.fields().len())
+ }
+ JoinType::Right => {
+ // Right join: null out left (buffered) columns
+ (0, buffered_schema.fields().len())
+ }
+ JoinType::Full => {
+ // Full join: null out buffered columns for streamed rows
+ // that matched but failed the filter. Unmatched buffered
+ // rows are null-joined on the streamed side separately
+ // when the buffered batch is drained.
+ let left_cols =
+ schema.fields().len() - buffered_schema.fields().len();
+ (left_cols, buffered_schema.fields().len())
+ }
+ _ => unreachable!(),
+ };
+
+ let num_rows = kept_batch.num_rows();
+ let mut columns: Vec<ArrayRef> = kept_batch.columns().to_vec();
+
+ for col in columns.iter_mut().skip(null_side_start).take(null_side_len) {
+ let null_array = new_null_array(col.data_type(), num_rows);
+ *col = zip(kept_corrected, &*col, &null_array)?;
}
- // Create null-joined batches for both sides
- let left_null_joined_batch = create_null_joined_batch(
- &joined_filter_not_matched_batch,
- buffered_schema,
- JoinSide::Left,
- join_type,
- schema,
- )?;
-
- Ok(concat_batches(
- schema,
- &[filtered_record_batch, left_null_joined_batch],
+ let options = RecordBatchOptions::new().with_row_count(Some(num_rows));
+ Ok(RecordBatch::try_new_with_options(
+ Arc::clone(schema),
+ columns,
+ &options,
)?)
}
- JoinType::Inner => Ok(filtered_record_batch),
+ JoinType::LeftSemi
+ | JoinType::LeftAnti
+ | JoinType::RightSemi
+ | JoinType::RightAnti
+ | JoinType::LeftMark
+ | JoinType::RightMark => unreachable!(
+ "Semi/anti/mark joins are handled by SemiAntiMarkSortMergeJoinStream"
+ ),
+ JoinType::Inner => Ok(filter_record_batch(record_batch, corrected_mask)?),
}
}
-
-/// Creates a batch with null columns for the non-joined side
-///
-/// Note: The input `batch` is assumed to be a fully-joined batch that already contains
-/// columns from both sides. We need to extract the data side columns and replace the
-/// null side columns with actual nulls.
-fn create_null_joined_batch(
- batch: &RecordBatch,
- null_schema: &SchemaRef,
- join_side: JoinSide,
- join_type: JoinType,
- output_schema: &SchemaRef,
-) -> Result<RecordBatch> {
- let num_rows = batch.num_rows();
-
- // The input batch is a fully-joined batch [left_cols..., right_cols...]
- // We need to extract the appropriate side and replace the other with nulls (or mark column)
- let columns = match (join_side, join_type) {
- (JoinSide::Left, JoinType::LeftMark) => {
- // For LEFT mark: output is [left_cols..., mark_col]
- // Batch is [left_cols..., right_cols...], extract left from beginning
- // Number of left columns = output columns - 1 (mark column)
- let left_col_count = output_schema.fields().len() - 1;
- let mut result: Vec<ArrayRef> = batch.columns()[..left_col_count].to_vec();
- result.push(Arc::new(BooleanArray::from(vec![false; num_rows])) as ArrayRef);
- result
- }
- (JoinSide::Right, JoinType::RightMark) => {
- // For RIGHT mark: output is [right_cols..., mark_col]
- // For RIGHT joins, batch is [right_cols..., left_cols...] (right comes first!)
- // Extract right columns from the beginning
- let right_col_count = output_schema.fields().len() - 1; // -1 for mark column
- let mut result: Vec<ArrayRef> = batch.columns()[..right_col_count].to_vec();
- result.push(Arc::new(BooleanArray::from(vec![false; num_rows])) as ArrayRef);
- result
- }
- (JoinSide::Left, _) => {
- // For LEFT join: output is [left_cols..., right_cols...]
- // Extract left columns, then add null right columns
- let null_columns: Vec<ArrayRef> = null_schema
- .fields()
- .iter()
- .map(|field| arrow::array::new_null_array(field.data_type(), num_rows))
- .collect();
- let left_col_count = output_schema.fields().len() - null_columns.len();
- let mut result: Vec<ArrayRef> = batch.columns()[..left_col_count].to_vec();
- result.extend(null_columns);
- result
- }
- (JoinSide::Right, _) => {
- // For RIGHT join: batch is [left_cols..., right_cols...] (same as schema)
- // We want: [null_left..., actual_right...]
- // Extract left columns from beginning, replace with nulls, keep right columns
- let null_columns: Vec<ArrayRef> = null_schema
- .fields()
- .iter()
- .map(|field| arrow::array::new_null_array(field.data_type(), num_rows))
- .collect();
- let left_col_count = null_columns.len();
- let mut result = null_columns;
- // Extract right columns starting after left columns
- result.extend_from_slice(&batch.columns()[left_col_count..]);
- result
- }
- (JoinSide::None, _) => {
- // This should not happen in normal join operations
- unreachable!(
- "JoinSide::None should not be used in null-joined batch creation"
- )
- }
- };
-
- // Create the batch - don't validate nullability since outer joins can have
- // null values in columns that were originally non-nullable
- use arrow::array::RecordBatchOptions;
- let mut options = RecordBatchOptions::new();
- options = options.with_row_count(Some(num_rows));
- Ok(RecordBatch::try_new_with_options(
- Arc::clone(output_schema),
- columns,
- &options,
- )?)
-}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
similarity index 80%
rename from datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
rename to datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
index 4dcbe1f647..c387e05390 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
@@ -45,14 +45,14 @@ use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream};
use arrow::array::{types::UInt64Type, *};
use arrow::compute::{
- self, BatchCoalescer, SortOptions, concat_batches, filter_record_batch, is_not_null,
+ self, BatchCoalescer, SortOptions, concat_batches, filter_record_batch, interleave,
take, take_arrays,
};
use arrow::datatypes::{DataType, SchemaRef, TimeUnit};
use arrow::ipc::reader::StreamReader;
-use datafusion_common::config::SpillCompression;
+use datafusion_common::cast::as_uint64_array;
use datafusion_common::{
- HashSet, JoinType, NullEquality, Result, exec_err, internal_err, not_impl_err,
+ JoinType, NullEquality, Result, exec_err, internal_err, not_impl_err,
};
use datafusion_execution::disk_manager::RefCountedTempFile;
use datafusion_execution::memory_pool::MemoryReservation;
@@ -132,10 +132,6 @@ pub(super) struct StreamedBatch {
pub num_output_rows: usize,
/// Index of currently scanned batch from buffered data
pub buffered_batch_idx: Option<usize>,
- /// Indices that found a match for the given join filter
- /// Used for semi joins to keep track the streaming index which got a join filter match
- /// and already emitted to the output.
- pub join_filter_matched_idxs: HashSet<u64>,
}
impl StreamedBatch {
@@ -148,7 +144,6 @@ impl StreamedBatch {
output_indices: vec![],
num_output_rows: 0,
buffered_batch_idx: None,
- join_filter_matched_idxs: HashSet::new(),
}
}
@@ -160,7 +155,6 @@ impl StreamedBatch {
output_indices: vec![],
num_output_rows: 0,
buffered_batch_idx: None,
- join_filter_matched_idxs: HashSet::new(),
}
}
@@ -282,9 +276,12 @@ pub(super) enum BufferedBatchState {
Spilled(RefCountedTempFile),
}
-/// Sort-Merge join stream that consumes streamed and buffered data streams
-/// and produces joined output stream.
-pub(super) struct SortMergeJoinStream {
+/// Sort-Merge join stream for Inner/Left/Right/Full joins.
+///
+/// Named "materializing" because it builds explicit `(streamed, buffered)` row
+/// pairs in [`JoinedRecordBatches`] to produce output columns from both sides
+/// of the join.
+pub(super) struct MaterializingSortMergeJoinStream {
// ========================================================================
// PROPERTIES:
// These fields are initialized at the start and remain constant throughout
@@ -428,16 +425,19 @@ impl JoinedRecordBatches {
}
}
- /// Pushes a batch with null metadata (Full join null-joined rows only)
+ /// Pushes a batch with null metadata (rows that need no filter correction)
///
- /// These buffered rows had NO matching streamed rows. Since we can't group
- /// by input row (no input row exists), we use null metadata as a sentinel.
+ /// Used for: (1) Full join buffered rows with no streamed match, and
+ /// (2) outer join streamed rows with no buffered match. These rows are
+ /// already in final form but must flow through the deferred filtering
+ /// pipeline to preserve output ordering. Null metadata causes
+ /// get_corrected_filter_mask() to pass them through unchanged.
///
/// Maintains invariant: N rows → N metadata entries (nulls)
fn push_batch_with_null_metadata(&mut self, batch: RecordBatch, join_type: JoinType) {
debug_assert!(
- join_type == JoinType::Full,
- "push_batch_with_null_metadata should only be called for Full joins"
+ matches!(join_type, JoinType::Left | JoinType::Right | JoinType::Full),
+ "push_batch_with_null_metadata should only be called for deferred-filtered joins"
);
let num_rows = batch.num_rows();
@@ -450,7 +450,7 @@ impl JoinedRecordBatches {
.expect("Failed to push batch to BatchCoalescer");
}
- /// Pushes a batch with filter metadata (filtered outer/semi/anti/mark joins)
+ /// Pushes a batch with filter metadata (filtered outer joins)
///
/// Deferred filtering: An input row may join with multiple buffered rows, but we
/// don't know yet if all matches failed the filter. We track metadata so
@@ -468,19 +468,8 @@ impl JoinedRecordBatches {
join_type: JoinType,
) {
debug_assert!(
- matches!(
- join_type,
- JoinType::Left
- | JoinType::LeftSemi
- | JoinType::LeftMark
- | JoinType::Right
- | JoinType::RightSemi
- | JoinType::RightMark
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::Full
- ),
- "push_batch_with_filter_metadata should only be called for outer/semi/anti/mark joins that need deferred filtering"
+ matches!(join_type, JoinType::Left | JoinType::Right | JoinType::Full),
+ "push_batch_with_filter_metadata should only be called for outer joins that need deferred filtering"
);
debug_assert_eq!(
@@ -506,7 +495,7 @@ impl JoinedRecordBatches {
/// No deferred filtering needed. Either every join match is output (Inner),
/// or null-joined rows are handled separately. No need to track which input
/// row produced which output row.
- fn push_batch_without_metadata(&mut self, batch: RecordBatch, _join_type: JoinType) {
+ fn push_batch_without_metadata(&mut self, batch: RecordBatch) {
self.joined_batches
.push_batch(batch)
.expect("Failed to push batch to BatchCoalescer");
@@ -519,13 +508,13 @@ impl JoinedRecordBatches {
self.debug_assert_empty_consistency();
}
}
-impl RecordBatchStream for SortMergeJoinStream {
+impl RecordBatchStream for MaterializingSortMergeJoinStream {
fn schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
}
}
-impl Stream for SortMergeJoinStream {
+impl Stream for MaterializingSortMergeJoinStream {
type Item = Result<RecordBatch>;
fn poll_next(
@@ -547,15 +536,35 @@ impl Stream for SortMergeJoinStream {
match self.current_ordering {
Ordering::Less | Ordering::Equal => {
if !streamed_exhausted {
+ // Batch deferred filtering: process_filtered_batches()
+ // only when >= batch_size rows have accumulated.
+ // Without this gate, unique keys cause per-row pipeline
+ // execution (concat + correct_mask + filter_by_type),
+ // which dominates runtime.
+ //
+ // Accumulated rows are bounded to ~2*batch_size:
+ // one batch_size worth from freeze_dequeuing_buffered()
+ // (when an input batch is fully consumed), plus up to
+ // batch_size pairs accumulating toward the next freeze.
+ // This does not reintroduce the unbounded buffering
+ // fixed by PR #20482. Exhausted state flushes remainder.
if needs_deferred_filtering(
&self.filter,
self.join_type,
) {
- match self.process_filtered_batches()? {
- Poll::Ready(Some(batch)) => {
- return Poll::Ready(Some(Ok(batch)));
+ let accumulated = self.num_unfrozen_pairs()
+ + self
+ .joined_record_batches
+ .filter_metadata
+ .filter_mask
+ .len();
+ if accumulated >= self.batch_size {
+ match self.process_filtered_batches()? {
+ Poll::Ready(Some(batch)) => {
+ return Poll::Ready(Some(Ok(batch)));
+ }
+ Poll::Ready(None) | Poll::Pending => {}
}
- Poll::Ready(None) | Poll::Pending => {}
}
}
@@ -734,11 +743,9 @@ impl Stream for SortMergeJoinStream {
}
}
-impl SortMergeJoinStream {
+impl MaterializingSortMergeJoinStream {
#[expect(clippy::too_many_arguments)]
pub fn try_new(
- // Configured via `datafusion.execution.spill_compression`.
- spill_compression: SpillCompression,
schema: SchemaRef,
sort_options: Vec<SortOptions>,
null_equality: NullEquality,
@@ -751,16 +758,19 @@ impl SortMergeJoinStream {
batch_size: usize,
join_metrics: SortMergeJoinMetrics,
reservation: MemoryReservation,
+ spill_manager: SpillManager,
runtime_env: Arc<RuntimeEnv>,
) -> Result<Self> {
let streamed_schema = streamed.schema();
let buffered_schema = buffered.schema();
- let spill_manager = SpillManager::new(
- Arc::clone(&runtime_env),
- join_metrics.spill_metrics().clone(),
- Arc::clone(&buffered_schema),
- )
- .with_compression_type(spill_compression);
+ debug_assert!(
+ matches!(
+ join_type,
+ JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full
+ ),
+ "MaterializingSortMergeJoinStream does not handle {join_type:?}; \
+ semi/anti/mark joins use BitwiseSortMergeJoinStream"
+ );
Ok(Self {
state: SortMergeJoinState::Init,
sort_options,
@@ -1053,68 +1063,20 @@ impl SortMergeJoinStream {
let mut join_streamed = false;
// Whether to join buffered rows
let mut join_buffered = false;
- // For Mark join we store a dummy id to indicate the row has a match
- let mut mark_row_as_match = false;
// determine whether we need to join streamed/buffered rows
match self.current_ordering {
Ordering::Less => {
if matches!(
self.join_type,
- JoinType::Left
- | JoinType::Right
- | JoinType::Full
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::LeftMark
- | JoinType::RightMark
+ JoinType::Left | JoinType::Right | JoinType::Full
) {
join_streamed = !self.streamed_joined;
}
}
Ordering::Equal => {
- if matches!(
- self.join_type,
- JoinType::LeftSemi
- | JoinType::LeftMark
- | JoinType::RightSemi
- | JoinType::RightMark
- ) {
- mark_row_as_match = matches!(
- self.join_type,
- JoinType::LeftMark | JoinType::RightMark
- );
- // if the join filter is specified then its needed to output the streamed index
- // only if it has not been emitted before
- // the `join_filter_matched_idxs` keeps track on if streamed index has a successful
- // filter match and prevents the same index to go into output more than once
- if self.filter.is_some() {
- join_streamed = !self
- .streamed_batch
- .join_filter_matched_idxs
- .contains(&(self.streamed_batch.idx as u64))
- && !self.streamed_joined;
- // if the join filter specified there can be references to buffered columns
- // so buffered columns are needed to access them
- join_buffered = join_streamed;
- } else {
- join_streamed = !self.streamed_joined;
- }
- }
- if matches!(
- self.join_type,
- JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full
- ) {
- join_streamed = true;
- join_buffered = true;
- };
-
- if matches!(self.join_type, JoinType::LeftAnti | JoinType::RightAnti)
- && self.filter.is_some()
- {
- join_streamed = !self.streamed_joined;
- join_buffered = join_streamed;
- }
+ join_streamed = true;
+ join_buffered = true;
}
Ordering::Greater => {
if self.join_type == JoinType::Full {
@@ -1162,12 +1124,9 @@ impl SortMergeJoinStream {
} else {
Some(self.buffered_data.scanning_batch_idx)
};
- // For Mark join we store a dummy id to indicate the row has a match
- let scanning_idx = mark_row_as_match.then_some(0);
-
self.streamed_batch.append_output_pair(
scanning_batch_idx,
- scanning_idx,
+ None,
self.batch_size,
);
self.buffered_data.scanning_finish();
@@ -1267,209 +1226,294 @@ impl SortMergeJoinStream {
// Produces and stages record batch for all output indices found
// for current streamed batch and clears staged output indices.
+ //
+ // Null-joined chunks (no buffered match) are pushed immediately.
+ // Matched chunks are collected and processed together in
+ // freeze_streamed_matched() to amortize filter evaluation overhead.
fn freeze_streamed(&mut self) -> Result<()> {
+ let mut matched_chunks: Vec<(usize, UInt64Array, UInt64Array)> = Vec::new();
+ let mut total_matched_rows: usize = 0;
+
for chunk in self.streamed_batch.output_indices.iter_mut() {
- // The row indices of joined streamed batch
let left_indices = chunk.streamed_indices.finish();
-
if left_indices.is_empty() {
continue;
}
+ let right_indices: UInt64Array = chunk.buffered_indices.finish();
- let mut left_columns = if let Some(range) = is_contiguous_range(&left_indices)
- {
- // When indices form a contiguous range (common for the streamed
- // side which advances sequentially), use zero-copy slice instead
- // of the O(n) take kernel.
- self.streamed_batch
- .batch
- .slice(range.start, range.len())
- .columns()
- .to_vec()
- } else {
- take_arrays(self.streamed_batch.batch.columns(), &left_indices, None)?
- };
+ if chunk.buffered_batch_idx.is_none() {
+ let left_columns =
+ materialize_left_columns(&self.streamed_batch.batch, &left_indices)?;
+ let right_columns =
+ create_unmatched_columns(&self.buffered_schema, left_indices.len());
- // The row indices of joined buffered batch
- let right_indices: UInt64Array = chunk.buffered_indices.finish();
- let mut right_columns =
- if matches!(self.join_type, JoinType::LeftMark | JoinType::RightMark) {
- vec![Arc::new(is_not_null(&right_indices)?) as ArrayRef]
- } else if matches!(
- self.join_type,
- JoinType::LeftSemi
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::RightSemi
- ) {
- vec![]
- } else if let Some(buffered_idx) = chunk.buffered_batch_idx {
- fetch_right_columns_by_idxs(
- &self.buffered_data,
- buffered_idx,
- &right_indices,
- )?
+ let columns = if self.join_type != JoinType::Right {
+ [left_columns, right_columns].concat()
} else {
- // If buffered batch none, meaning it is null joined batch.
- // We need to create null arrays for buffered columns to join with streamed rows.
- create_unmatched_columns(
- self.join_type,
- &self.buffered_schema,
- right_indices.len(),
- )
+ [right_columns, left_columns].concat()
};
+ let batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
- // Prepare the columns we apply join filter on later.
- // Only for joined rows between streamed and buffered.
- let filter_columns = if let Some(buffered_batch_idx) =
- chunk.buffered_batch_idx
- {
- if self.join_type != JoinType::Right {
- if matches!(
- self.join_type,
- JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark
- ) {
- let right_cols = fetch_right_columns_by_idxs(
- &self.buffered_data,
- buffered_batch_idx,
- &right_indices,
- )?;
-
- get_filter_columns(&self.filter, &left_columns, &right_cols)
- } else if matches!(
- self.join_type,
- JoinType::RightAnti | JoinType::RightSemi | JoinType::RightMark
- ) {
- let right_cols = fetch_right_columns_by_idxs(
- &self.buffered_data,
- buffered_batch_idx,
- &right_indices,
- )?;
-
- get_filter_columns(&self.filter, &right_cols, &left_columns)
- } else {
- get_filter_columns(&self.filter, &left_columns, &right_columns)
- }
+ // Null-joined rows (no buffered match) need no filter correction,
+ // but must flow through the same pipeline as matched rows to
+ // preserve output ordering. Use null metadata as a sentinel so
+ // get_corrected_filter_mask() passes them through unchanged.
+ if needs_deferred_filtering(&self.filter, self.join_type) {
+ self.joined_record_batches
+ .push_batch_with_null_metadata(batch, self.join_type);
} else {
- get_filter_columns(&self.filter, &right_columns, &left_columns)
+ self.joined_record_batches
+ .push_batch_without_metadata(batch);
}
- } else {
- // This chunk is totally for null joined rows (outer join), we don't need to apply join filter.
- // Any join filter applied only on either streamed or buffered side will be pushed already.
- vec![]
- };
+ continue;
+ }
- let columns = if self.join_type != JoinType::Right {
- left_columns.extend(right_columns);
- left_columns
- } else {
- right_columns.extend(left_columns);
- right_columns
- };
+ total_matched_rows += left_indices.len();
+ matched_chunks.push((
+ chunk.buffered_batch_idx.unwrap(),
+ left_indices,
+ right_indices,
+ ));
+ }
+
+ if !matched_chunks.is_empty() {
+ self.freeze_streamed_matched(&matched_chunks, total_matched_rows)?;
+ }
+
+ self.streamed_batch.output_indices.clear();
+ self.streamed_batch.num_output_rows = 0;
+ Ok(())
+ }
+
+ /// Materializes columns, evaluates the join filter, and pushes output
+ /// for all matched chunks in a single batch. This avoids per-chunk
+ /// RecordBatch construction and filter evaluation, which dominates
+ /// cost when keys are near-unique (1 row per chunk).
+ fn freeze_streamed_matched(
+ &mut self,
+ matched_chunks: &[(usize, UInt64Array, UInt64Array)],
+ total_matched_rows: usize,
+ ) -> Result<()> {
+ debug_assert!(
+ !matched_chunks.is_empty(),
+ "caller guards this with an is_empty check before calling"
+ );
+ debug_assert!(
+ matched_chunks.iter().all(|(idx, left, right)| {
+ left.len() == right.len() && *idx < self.buffered_data.batches.len()
+ }),
+ "left/right indices are built in pairs from the same streamed×buffered cross, \
+ and batch_idx comes from iterating buffered_data.batches"
+ );
+ debug_assert_eq!(
+ matched_chunks
+ .iter()
+ .map(|(_, l, _)| l.len())
+ .sum::<usize>(),
+ total_matched_rows,
+ "total_matched_rows is accumulated from the same chunks in freeze_streamed"
+ );
- let output_batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
- // Apply join filter if any
- if !filter_columns.is_empty() {
- if let Some(f) = &self.filter {
- // Construct batch with only filter columns
- let filter_batch =
- RecordBatch::try_new(Arc::clone(f.schema()), filter_columns)?;
-
- let filter_result = f
- .expression()
- .evaluate(&filter_batch)?
- .into_array(filter_batch.num_rows())?;
-
- // The boolean selection mask of the join filter result
- let pre_mask =
- datafusion_common::cast::as_boolean_array(&filter_result)?;
-
- // If there are nulls in join filter result, exclude them from selecting
- // the rows to output.
- let mask = if pre_mask.null_count() > 0 {
- compute::prep_null_mask_filter(
- datafusion_common::cast::as_boolean_array(&filter_result)?,
- )
+ let combined_left_indices = if matched_chunks.len() == 1 {
+ matched_chunks[0].1.clone()
+ } else {
+ let refs: Vec<&dyn Array> =
+ matched_chunks.iter().map(|c| &c.1 as &dyn Array).collect();
+ as_uint64_array(&compute::concat(&refs)?)?.clone()
+ };
+
+ let left_columns =
+ materialize_left_columns(&self.streamed_batch.batch, &combined_left_indices)?;
+
+ let right_columns =
+ self.materialize_right_columns(matched_chunks, total_matched_rows)?;
+
+ let filter_columns = if self.join_type == JoinType::Right {
+ get_filter_columns(&self.filter, &right_columns, &left_columns)
+ } else {
+ get_filter_columns(&self.filter, &left_columns, &right_columns)
+ };
+
+ let columns = if self.join_type != JoinType::Right {
+ [left_columns, right_columns].concat()
+ } else {
+ [right_columns, left_columns].concat()
+ };
+ let output_batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
+
+ if !filter_columns.is_empty() {
+ if let Some(f) = &self.filter {
+ let filter_batch =
+ RecordBatch::try_new(Arc::clone(f.schema()), filter_columns)?;
+ let filter_result = f
+ .expression()
+ .evaluate(&filter_batch)?
+ .into_array(filter_batch.num_rows())?;
+
+ let pre_mask = datafusion_common::cast::as_boolean_array(&filter_result)?;
+
+ let mask = if pre_mask.null_count() > 0 {
+ compute::prep_null_mask_filter(pre_mask)
+ } else {
+ pre_mask.clone()
+ };
+
+ if needs_deferred_filtering(&self.filter, self.join_type) {
+ // Full join uses pre_mask (preserving nulls) for
+ // get_corrected_filter_mask; other outer joins use mask.
+ let mask_to_use = if self.join_type != JoinType::Full {
+ &mask
} else {
- pre_mask.clone()
+ pre_mask
};
- // Push the filtered batch which contains rows passing join filter to the output
- // For outer/semi/anti/mark joins with deferred filtering, push the unfiltered batch with metadata
- // For INNER joins, filter immediately and push without metadata
- let needs_deferred_filtering = matches!(
+ self.joined_record_batches.push_batch_with_filter_metadata(
+ output_batch,
+ &combined_left_indices,
+ mask_to_use,
+ self.streamed_batch_counter.load(Relaxed),
self.join_type,
- JoinType::Left
- | JoinType::LeftSemi
- | JoinType::Right
- | JoinType::RightSemi
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::LeftMark
- | JoinType::RightMark
- | JoinType::Full
);
+ } else {
+ let filtered_batch = filter_record_batch(&output_batch, &mask)?;
+ self.joined_record_batches
+ .push_batch_without_metadata(filtered_batch);
+ }
- if needs_deferred_filtering {
- // Outer/semi/anti/mark joins: push unfiltered batch with metadata for deferred filtering
- let mask_to_use = if self.join_type != JoinType::Full {
- &mask
- } else {
- pre_mask
- };
-
- self.joined_record_batches.push_batch_with_filter_metadata(
- output_batch,
- &left_indices,
- mask_to_use,
- self.streamed_batch_counter.load(Relaxed),
- self.join_type,
- );
- } else {
- // INNER joins: filter immediately and push without metadata
- let filtered_batch = filter_record_batch(&output_batch, &mask)?;
- self.joined_record_batches
- .push_batch_without_metadata(filtered_batch, self.join_type);
- }
+ // Track which buffered rows had all filter matches fail,
+ // so full join can emit them as null-joined later.
+ if self.join_type == JoinType::Full {
+ let mut offset = 0usize;
+ for (batch_idx, _left, right) in matched_chunks {
+ let chunk_len = right.len();
+ let buffered_batch = &mut self.buffered_data.batches[*batch_idx];
- // For outer joins, we need to push the null joined rows to the output if
- // all joined rows are failed on the join filter.
- // I.e., if all rows joined from a streamed row are failed with the join filter,
- // we need to join it with nulls as buffered side.
- if self.join_type == JoinType::Full {
- let buffered_batch = &mut self.buffered_data.batches
- [chunk.buffered_batch_idx.unwrap()];
-
- for i in 0..pre_mask.len() {
- // If the buffered row is not joined with streamed side,
- // skip it.
- if right_indices.is_null(i) {
+ for i in 0..chunk_len {
+ if right.is_null(i) {
continue;
}
-
- let buffered_index = right_indices.value(i);
-
+ let buffered_index = right.value(i);
buffered_batch.join_filter_not_matched_map.insert(
buffered_index,
*buffered_batch
.join_filter_not_matched_map
.get(&buffered_index)
.unwrap_or(&true)
- && !pre_mask.value(i),
+ && !pre_mask.value(offset + i),
);
}
+ offset += chunk_len;
}
+ debug_assert_eq!(
+ offset, total_matched_rows,
+ "offset must advance through every chunk exactly once"
+ );
}
+ }
+ } else {
+ self.joined_record_batches
+ .push_batch_without_metadata(output_batch);
+ }
+
+ Ok(())
+ }
+
+ /// Materializes right-side columns across all matched chunks.
+ ///
+ /// When chunks reference a single buffered batch, indices are concatenated
+ /// for a single fetch. When multiple batches are involved, `interleave`
+ /// gathers columns across sources. A null-row sentinel at source index 0
+ /// handles null right indices (unmatched streamed rows).
+ fn materialize_right_columns(
+ &self,
+ matched_chunks: &[(usize, UInt64Array, UInt64Array)],
+ total_matched_rows: usize,
+ ) -> Result<Vec<ArrayRef>> {
+ let first_batch_idx = matched_chunks[0].0;
+ let single_source = matched_chunks.iter().all(|c| c.0 == first_batch_idx);
+
+ if single_source {
+ let combined_right_indices = if matched_chunks.len() == 1 {
+ matched_chunks[0].2.clone()
} else {
- self.joined_record_batches
- .push_batch_without_metadata(output_batch, self.join_type);
+ let refs: Vec<&dyn Array> =
+ matched_chunks.iter().map(|c| &c.2 as &dyn Array).collect();
+ as_uint64_array(&compute::concat(&refs)?)?.clone()
+ };
+ return fetch_right_columns_by_idxs(
+ &self.buffered_data,
+ first_batch_idx,
+ &combined_right_indices,
+ );
+ }
+
+ // Multiple source batches: map each buffered_batch_idx to a
+ // contiguous source index, reserving source 0 for a null sentinel.
+ let mut batch_idx_to_source: HashMap<usize, usize> = HashMap::new();
+ let mut source_batches: Vec<usize> = Vec::new();
+ for (batch_idx, _, _) in matched_chunks {
+ batch_idx_to_source.entry(*batch_idx).or_insert_with(|| {
+ let idx = source_batches.len() + 1;
+ source_batches.push(*batch_idx);
+ idx
+ });
+ }
+
+ let mut interleave_indices: Vec<(usize, usize)> =
+ Vec::with_capacity(total_matched_rows);
+ for (batch_idx, _, right) in matched_chunks {
+ let source = batch_idx_to_source[batch_idx];
+ for i in 0..right.len() {
+ if right.is_null(i) {
+ interleave_indices.push((0, 0));
+ } else {
+ interleave_indices.push((source, right.value(i) as usize));
+ }
}
}
- self.streamed_batch.output_indices.clear();
- self.streamed_batch.num_output_rows = 0;
+ let num_right_cols = self.buffered_schema.fields().len();
+ let mut right_columns = Vec::with_capacity(num_right_cols);
- Ok(())
+ // Read each source batch once (spilled batches require disk I/O).
+ let source_data: Vec<Option<RecordBatch>> = source_batches
+ .iter()
+ .map(|&idx| {
+ let bb = &self.buffered_data.batches[idx];
+ match &bb.batch {
+ BufferedBatchState::InMemory(batch) => Some(batch.clone()),
+ BufferedBatchState::Spilled(spill_file) => {
+ let file = BufReader::new(File::open(spill_file.path()).ok()?);
+ let reader = StreamReader::try_new(file, None).ok()?;
+ reader.into_iter().next()?.ok()
+ }
+ }
+ })
+ .collect();
+
+ for col_idx in 0..num_right_cols {
+ let dtype = self.buffered_schema.field(col_idx).data_type();
+ let null_array = new_null_array(dtype, 1);
+
+ let mut source_arrays: Vec<&dyn Array> =
+ Vec::with_capacity(source_batches.len() + 1);
+ source_arrays.push(null_array.as_ref());
+
+ for data in &source_data {
+ match data {
+ Some(batch) => source_arrays.push(batch.column(col_idx).as_ref()),
+ None => {
+ return internal_err!(
+ "Failed to read spilled buffered batch during interleave"
+ );
+ }
+ }
+ }
+
+ right_columns.push(interleave(&source_arrays, &interleave_indices)?);
+ }
+
+ Ok(right_columns)
}
fn filter_joined_batch(&mut self) -> Result<RecordBatch> {
@@ -1556,7 +1600,6 @@ impl SortMergeJoinStream {
corrected_mask,
self.join_type,
&self.schema,
- &self.streamed_schema,
&self.buffered_schema,
)?;
@@ -1567,22 +1610,26 @@ impl SortMergeJoinStream {
}
}
-fn create_unmatched_columns(
- join_type: JoinType,
- schema: &SchemaRef,
- size: usize,
-) -> Vec<ArrayRef> {
- if matches!(join_type, JoinType::LeftMark | JoinType::RightMark) {
- vec![Arc::new(BooleanArray::from(vec![false; size])) as ArrayRef]
+/// Materialize left (streamed) columns using slice or take.
+fn materialize_left_columns(
+ batch: &RecordBatch,
+ indices: &UInt64Array,
+) -> Result<Vec<ArrayRef>> {
+ if let Some(range) = is_contiguous_range(indices) {
+ Ok(batch.slice(range.start, range.len()).columns().to_vec())
} else {
- schema
- .fields()
- .iter()
- .map(|f| new_null_array(f.data_type(), size))
- .collect::<Vec<_>>()
+ Ok(take_arrays(batch.columns(), indices, None)?)
}
}
+fn create_unmatched_columns(schema: &SchemaRef, size: usize) -> Vec<ArrayRef> {
+ schema
+ .fields()
+ .iter()
+ .map(|f| new_null_array(f.data_type(), size))
+ .collect::<Vec<_>>()
+}
+
fn produce_buffered_null_batch(
schema: &SchemaRef,
streamed_schema: &SchemaRef,
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs b/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs
index cf76b2ba8c..62efb77f87 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs
@@ -19,7 +19,7 @@
use crate::metrics::{
BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder,
- MetricCategory, SpillMetrics, Time,
+ MetricCategory, Time,
};
/// Metrics for SortMergeJoinExec
@@ -35,8 +35,6 @@ pub(super) struct SortMergeJoinMetrics {
/// Peak memory used for buffered data.
/// Calculated as sum of peak memory values across partitions
peak_mem_used: Gauge,
- /// Metrics related to spilling
- spill_metrics: SpillMetrics,
}
impl SortMergeJoinMetrics {
@@ -51,7 +49,6 @@ impl SortMergeJoinMetrics {
let peak_mem_used = MetricBuilder::new(metrics)
.with_category(MetricCategory::Bytes)
.gauge("peak_mem_used", partition);
- let spill_metrics = SpillMetrics::new(metrics, partition);
let baseline_metrics = BaselineMetrics::new(metrics, partition);
@@ -61,7 +58,6 @@ impl SortMergeJoinMetrics {
input_rows,
baseline_metrics,
peak_mem_used,
- spill_metrics,
}
}
@@ -84,8 +80,4 @@ impl SortMergeJoinMetrics {
pub fn peak_mem_used(&self) -> Gauge {
self.peak_mem_used.clone()
}
-
- pub fn spill_metrics(&self) -> SpillMetrics {
- self.spill_metrics.clone()
- }
}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs b/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs
index 06290ec4d0..2fdb0924e7 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs
@@ -19,10 +19,11 @@
pub use exec::SortMergeJoinExec;
+pub(crate) mod bitwise_stream;
mod exec;
mod filter;
+pub(crate) mod materializing_stream;
mod metrics;
-mod stream;
#[cfg(test)]
mod tests;
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index a1043a9d8b..5824c2328d 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -24,15 +24,22 @@
//!
//! Add relevant tests under the specified sections.
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use super::bitwise_stream::BitwiseSortMergeJoinStream;
use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn};
use crate::joins::{HashJoinExec, PartitionMode, SortMergeJoinExec};
+use crate::metrics::{ExecutionPlanMetricsSet, SpillMetrics};
+use crate::spill::spill_manager::SpillManager;
use crate::test::TestMemoryExec;
use crate::test::exec::BarrierExec;
use crate::test::{build_table_i32, build_table_i32_two_cols};
-use crate::{ExecutionPlan, common};
+use crate::{ExecutionPlan, RecordBatchStream, common};
use crate::{
expressions::Column, joins::sort_merge_join::filter::get_corrected_filter_mask,
- joins::sort_merge_join::stream::JoinedRecordBatches,
+ joins::sort_merge_join::materializing_stream::JoinedRecordBatches,
};
use arrow::array::{
BinaryArray, BooleanArray, Date32Array, Date64Array, FixedSizeBinaryArray,
@@ -53,15 +60,15 @@ use datafusion_common::{
use datafusion_common_runtime::JoinSet;
use datafusion_execution::config::SessionConfig;
use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
+use datafusion_execution::memory_pool::MemoryConsumer;
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_expr::Operator;
use datafusion_physical_expr::expressions::BinaryExpr;
-use futures::StreamExt;
-use insta::{allow_duplicates, assert_snapshot};
+use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;
+use futures::{Stream, StreamExt};
+use insta::assert_snapshot;
use itertools::Itertools;
-use std::sync::Arc;
-use std::task::Poll;
fn build_table(
a: (&str, &Vec<i32>),
@@ -1278,7 +1285,7 @@ async fn join_right_anti_output_two_batches() -> Result<()> {
let (_, batches) =
join_collect_batch_size_equals_two(left, right, on, LeftAnti).await?;
- // SemiAntiSortMergeJoinStream uses a coalescer, so batch boundaries differ
+ // BitwiseSortMergeJoinStream uses a coalescer, so batch boundaries differ
// from the old stream. Only assert data correctness.
let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(total_rows, 3);
@@ -1567,7 +1574,7 @@ async fn join_right_semi_output_two_batches() -> Result<()> {
"| 2 | 5 | 8 |",
"+----+----+----+",
];
- // SemiAntiSortMergeJoinStream uses a coalescer, so batch boundaries differ
+ // BitwiseSortMergeJoinStream uses a coalescer, so batch boundaries differ
// from the old stream. Only assert data correctness.
let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(total_rows, 3);
@@ -2061,9 +2068,9 @@ async fn overallocation_single_batch_no_spill() -> Result<()> {
let sort_options = vec![SortOptions::default(); on.len()];
let join_types = vec![
- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks
- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.
- Inner, Left, Right, Full, LeftMark, RightMark,
+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks
+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.
+ Inner, Left, Right, Full,
];
// Disable DiskManager to prevent spilling
@@ -2144,9 +2151,9 @@ async fn overallocation_multi_batch_no_spill() -> Result<()> {
let sort_options = vec![SortOptions::default(); on.len()];
let join_types = vec![
- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks
- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.
- Inner, Left, Right, Full, LeftMark, RightMark,
+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks
+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.
+ Inner, Left, Right, Full,
];
// Disable DiskManager to prevent spilling
@@ -2206,9 +2213,9 @@ async fn overallocation_single_batch_spill() -> Result<()> {
let sort_options = vec![SortOptions::default(); on.len()];
let join_types = [
- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks
- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.
- Inner, Left, Right, Full, LeftMark, RightMark,
+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks
+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.
+ Inner, Left, Right, Full,
];
// Enable DiskManager to allow spilling
@@ -2312,9 +2319,9 @@ async fn overallocation_multi_batch_spill() -> Result<()> {
let sort_options = vec![SortOptions::default(); on.len()];
let join_types = [
- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks
- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.
- Inner, Left, Right, Full, LeftMark, RightMark,
+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks
+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.
+ Inner, Left, Right, Full,
];
// Enable DiskManager to allow spilling
@@ -2377,6 +2384,321 @@ async fn overallocation_multi_batch_spill() -> Result<()> {
Ok(())
}
+/// Build a c1 < c2 filter on the third column of each side.
+fn build_c1_lt_c2_filter(left_schema: &Schema, right_schema: &Schema) -> JoinFilter {
+ JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("c1", 0)),
+ Operator::Lt,
+ Arc::new(Column::new("c2", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ left_schema
+ .field_with_name("c1")
+ .unwrap()
+ .clone()
+ .with_nullable(true),
+ right_schema
+ .field_with_name("c2")
+ .unwrap()
+ .clone()
+ .with_nullable(true),
+ ])),
+ )
+}
+
+#[tokio::test]
+async fn spill_with_filter_deferred() -> Result<()> {
+ let left = build_table(
+ ("a1", &vec![0, 1, 2, 3, 4, 5]),
+ ("b1", &vec![1, 2, 3, 4, 5, 6]),
+ ("c1", &vec![4, 5, 6, 7, 8, 9]),
+ );
+ let right = build_table(
+ ("a2", &vec![0, 10, 20, 30, 40]),
+ ("b2", &vec![1, 3, 4, 6, 8]),
+ ("c2", &vec![50, 60, 70, 80, 90]),
+ );
+ let on = vec![(
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,
+ )];
+ let sort_options = vec![SortOptions::default(); on.len()];
+
+ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());
+
+ // Deferred filtering join types handled by the main MaterializingSortMergeJoinStream
+ let join_types = [Left, Right, Full];
+
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(100, 1.0)
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()?;
+
+ for batch_size in [1, 50] {
+ let session_config = SessionConfig::default().with_batch_size(batch_size);
+
+ for join_type in &join_types {
+ // Run with spilling
+ let task_ctx = Arc::new(
+ TaskContext::default()
+ .with_session_config(session_config.clone())
+ .with_runtime(Arc::clone(&runtime)),
+ );
+ let join = join_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ *join_type,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join.execute(0, task_ctx)?;
+ let spilled_result = common::collect(stream).await.unwrap();
+
+ assert!(join.metrics().is_some());
+ assert!(
+ join.metrics().unwrap().spill_count().unwrap() > 0,
+ "Expected spilling for {join_type:?} batch_size={batch_size}"
+ );
+
+ // Run without spilling
+ let task_ctx_no_spill = Arc::new(
+ TaskContext::default().with_session_config(session_config.clone()),
+ );
+ let join_no_spill = join_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ *join_type,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;
+ let no_spill_result = common::collect(stream).await.unwrap();
+
+ let spilled_str = batches_to_sort_string(&spilled_result);
+ let no_spill_str = batches_to_sort_string(&no_spill_result);
+ assert_eq!(
+ spilled_str, no_spill_str,
+ "Spill vs no-spill mismatch for {join_type:?} batch_size={batch_size}"
+ );
+ }
+ }
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn spill_with_filter_multi_batch() -> Result<()> {
+ let left_batch_1 = build_table_i32(
+ ("a1", &vec![0, 1]),
+ ("b1", &vec![1, 1]),
+ ("c1", &vec![4, 5]),
+ );
+ let left_batch_2 = build_table_i32(
+ ("a1", &vec![2, 3]),
+ ("b1", &vec![1, 1]),
+ ("c1", &vec![6, 7]),
+ );
+ let left_batch_3 = build_table_i32(
+ ("a1", &vec![4, 5]),
+ ("b1", &vec![1, 1]),
+ ("c1", &vec![8, 9]),
+ );
+ let right_batch_1 = build_table_i32(
+ ("a2", &vec![0, 10]),
+ ("b2", &vec![1, 1]),
+ ("c2", &vec![50, 60]),
+ );
+ let right_batch_2 = build_table_i32(
+ ("a2", &vec![20, 30]),
+ ("b2", &vec![1, 1]),
+ ("c2", &vec![70, 80]),
+ );
+ let right_batch_3 =
+ build_table_i32(("a2", &vec![40]), ("b2", &vec![1]), ("c2", &vec![90]));
+ let left = build_table_from_batches(vec![left_batch_1, left_batch_2, left_batch_3]);
+ let right =
+ build_table_from_batches(vec![right_batch_1, right_batch_2, right_batch_3]);
+ let on = vec![(
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,
+ )];
+ let sort_options = vec![SortOptions::default(); on.len()];
+
+ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());
+
+ let join_types = [Left, Right, Full];
+
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(500, 1.0)
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()?;
+
+ for batch_size in [1, 50] {
+ let session_config = SessionConfig::default().with_batch_size(batch_size);
+
+ for join_type in &join_types {
+ // Run with spilling
+ let task_ctx = Arc::new(
+ TaskContext::default()
+ .with_session_config(session_config.clone())
+ .with_runtime(Arc::clone(&runtime)),
+ );
+ let join = join_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ *join_type,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join.execute(0, task_ctx)?;
+ let spilled_result = common::collect(stream).await.unwrap();
+
+ assert!(join.metrics().is_some());
+ assert!(
+ join.metrics().unwrap().spill_count().unwrap() > 0,
+ "Expected spilling for {join_type:?} batch_size={batch_size}"
+ );
+
+ // Run without spilling
+ let task_ctx_no_spill = Arc::new(
+ TaskContext::default().with_session_config(session_config.clone()),
+ );
+ let join_no_spill = join_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ *join_type,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;
+ let no_spill_result = common::collect(stream).await.unwrap();
+
+ let spilled_str = batches_to_sort_string(&spilled_result);
+ let no_spill_str = batches_to_sort_string(&no_spill_result);
+ assert_eq!(
+ spilled_str, no_spill_str,
+ "Spill vs no-spill mismatch for {join_type:?} batch_size={batch_size}"
+ );
+ }
+ }
+
+ Ok(())
+}
+
+/// FULL join where all buffered rows match on key but fail the filter.
+/// Verifies produce_buffered_not_matched emits null-joined rows under spill.
+#[tokio::test]
+async fn spill_full_join_filter_not_matched() -> Result<()> {
+ // c1 values (100..105) are always > c2 values (1..5), so c1 < c2 always fails
+ let left = build_table(
+ ("a1", &vec![0, 1, 2, 3, 4]),
+ ("b1", &vec![1, 1, 1, 1, 1]),
+ ("c1", &vec![100, 101, 102, 103, 104]),
+ );
+ let right = build_table(
+ ("a2", &vec![10, 20, 30, 40, 50]),
+ ("b2", &vec![1, 1, 1, 1, 1]),
+ ("c2", &vec![1, 2, 3, 4, 5]),
+ );
+ let on = vec![(
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,
+ )];
+ let sort_options = vec![SortOptions::default(); on.len()];
+
+ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());
+
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(100, 1.0)
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()?;
+
+ for batch_size in [1, 50] {
+ let session_config = SessionConfig::default().with_batch_size(batch_size);
+
+ // Run with spilling
+ let task_ctx = Arc::new(
+ TaskContext::default()
+ .with_session_config(session_config.clone())
+ .with_runtime(Arc::clone(&runtime)),
+ );
+ let join = join_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ Full,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join.execute(0, task_ctx)?;
+ let spilled_result = common::collect(stream).await.unwrap();
+
+ assert!(
+ join.metrics().unwrap().spill_count().unwrap() > 0,
+ "Expected spilling for FULL batch_size={batch_size}"
+ );
+
+ // Run without spilling
+ let task_ctx_no_spill =
+ Arc::new(TaskContext::default().with_session_config(session_config.clone()));
+ let join_no_spill = join_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ Full,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;
+ let no_spill_result = common::collect(stream).await.unwrap();
+
+ // All filter evaluations fail, so FULL join should produce:
+ // - 5 rows with left columns + null right columns (unmatched left)
+ // - 5 rows with null left columns + right columns (unmatched right)
+ let total_rows: usize = no_spill_result.iter().map(|b| b.num_rows()).sum();
+ assert_eq!(
+ total_rows, 10,
+ "FULL join with all-failing filter should produce 10 rows, got {total_rows}"
+ );
+
+ let spilled_str = batches_to_sort_string(&spilled_result);
+ let no_spill_str = batches_to_sort_string(&no_spill_result);
+ assert_eq!(
+ spilled_str, no_spill_str,
+ "Spill vs no-spill mismatch for FULL join batch_size={batch_size}"
+ );
+ }
+
+ Ok(())
+}
+
fn build_joined_record_batches() -> Result<JoinedRecordBatches> {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, true),
@@ -2721,394 +3043,46 @@ async fn test_left_outer_join_filtered_mask() -> Result<()> {
Ok(())
}
-#[tokio::test]
-async fn test_semi_join_filtered_mask() -> Result<()> {
- for join_type in [LeftSemi, RightSemi] {
- let mut joined_batches = build_joined_record_batches()?;
- let schema = joined_batches.joined_batches.schema();
+#[test]
+fn test_partition_statistics() -> Result<()> {
+ use crate::ExecutionPlan;
+ use datafusion_common::stats::Precision;
- let output = joined_batches.concat_batches(&schema)?;
- let out_mask = joined_batches.filter_metadata.filter_mask.finish();
- let out_indices = joined_batches.filter_metadata.row_indices.finish();
+ let left = build_table(
+ ("a1", &vec![1, 2, 3]),
+ ("b1", &vec![4, 5, 5]),
+ ("c1", &vec![7, 8, 9]),
+ );
+ let right = build_table(
+ ("a2", &vec![10, 20, 30]),
+ ("b1", &vec![4, 5, 6]),
+ ("c2", &vec![70, 80, 90]),
+ );
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0]),
- &[0usize],
- &BooleanArray::from(vec![true]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![true])
- );
+ let on = vec![(
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
+ )];
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0]),
- &[0usize],
- &BooleanArray::from(vec![false]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![None])
- );
+ // Test different join types to ensure partition_statistics works correctly for all
+ let join_types = vec![
+ (Inner, 6), // left cols + right cols
+ (Left, 6), // left cols + right cols
+ (Right, 6), // left cols + right cols
+ (Full, 6), // left cols + right cols
+ (LeftSemi, 3), // only left cols
+ (LeftAnti, 3), // only left cols
+ (RightSemi, 3), // only right cols
+ (RightAnti, 3), // only right cols
+ ];
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0]),
- &[0usize; 2],
- &BooleanArray::from(vec![true, true]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![Some(true), None])
- );
+ for (join_type, expected_cols) in join_types {
+ let join_exec =
+ join(Arc::clone(&left), Arc::clone(&right), on.clone(), join_type)?;
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![true, true, true]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![Some(true), None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![true, false, true]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![Some(true), None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![false, false, true]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, Some(true),])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![false, true, true]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![None, Some(true), None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![false, false, false]),
- output.num_rows()
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, None])
- );
-
- let corrected_mask = get_corrected_filter_mask(
- join_type,
- &out_indices,
- &joined_batches.filter_metadata.batch_ids,
- &out_mask,
- output.num_rows(),
- )
- .unwrap();
-
- assert_eq!(
- corrected_mask,
- BooleanArray::from(vec![
- Some(true),
- None,
- Some(true),
- None,
- Some(true),
- None,
- None,
- None
- ])
- );
-
- let filtered_rb = filter_record_batch(&output, &corrected_mask)?;
-
- assert_batches_eq!(
- &[
- "+---+----+---+----+",
- "| a | b | x | y |",
- "+---+----+---+----+",
- "| 1 | 10 | 1 | 11 |",
- "| 1 | 11 | 1 | 12 |",
- "| 1 | 12 | 1 | 13 |",
- "+---+----+---+----+",
- ],
- &[filtered_rb]
- );
-
- // output null rows
- let null_mask = arrow::compute::not(&corrected_mask)?;
- assert_eq!(
- null_mask,
- BooleanArray::from(vec![
- Some(false),
- None,
- Some(false),
- None,
- Some(false),
- None,
- None,
- None
- ])
- );
-
- let null_joined_batch = filter_record_batch(&output, &null_mask)?;
-
- assert_batches_eq!(
- &[
- "+---+---+---+---+",
- "| a | b | x | y |",
- "+---+---+---+---+",
- "+---+---+---+---+",
- ],
- &[null_joined_batch]
- );
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn test_anti_join_filtered_mask() -> Result<()> {
- for join_type in [LeftAnti, RightAnti] {
- let mut joined_batches = build_joined_record_batches()?;
- let schema = joined_batches.joined_batches.schema();
-
- let output = joined_batches.concat_batches(&schema)?;
- let out_mask = joined_batches.filter_metadata.filter_mask.finish();
- let out_indices = joined_batches.filter_metadata.row_indices.finish();
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0]),
- &[0usize],
- &BooleanArray::from(vec![true]),
- 1
- )
- .unwrap(),
- BooleanArray::from(vec![None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0]),
- &[0usize],
- &BooleanArray::from(vec![false]),
- 1
- )
- .unwrap(),
- BooleanArray::from(vec![Some(true)])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0]),
- &[0usize; 2],
- &BooleanArray::from(vec![true, true]),
- 2
- )
- .unwrap(),
- BooleanArray::from(vec![None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![true, true, true]),
- 3
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![true, false, true]),
- 3
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![false, false, true]),
- 3
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![false, true, true]),
- 3
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, None])
- );
-
- assert_eq!(
- get_corrected_filter_mask(
- join_type,
- &UInt64Array::from(vec![0, 0, 0]),
- &[0usize; 3],
- &BooleanArray::from(vec![false, false, false]),
- 3
- )
- .unwrap(),
- BooleanArray::from(vec![None, None, Some(true)])
- );
-
- let corrected_mask = get_corrected_filter_mask(
- join_type,
- &out_indices,
- &joined_batches.filter_metadata.batch_ids,
- &out_mask,
- output.num_rows(),
- )
- .unwrap();
-
- assert_eq!(
- corrected_mask,
- BooleanArray::from(vec![
- None,
- None,
- None,
- None,
- None,
- Some(true),
- None,
- Some(true)
- ])
- );
-
- let filtered_rb = filter_record_batch(&output, &corrected_mask)?;
-
- allow_duplicates! {
- assert_snapshot!(batches_to_string(&[filtered_rb]), @r"
- +---+----+---+----+
- | a | b | x | y |
- +---+----+---+----+
- | 1 | 13 | 1 | 12 |
- | 1 | 14 | 1 | 11 |
- +---+----+---+----+
- ");
- }
-
- // output null rows
- let null_mask = arrow::compute::not(&corrected_mask)?;
- assert_eq!(
- null_mask,
- BooleanArray::from(vec![
- None,
- None,
- None,
- None,
- None,
- Some(false),
- None,
- Some(false),
- ])
- );
-
- let null_joined_batch = filter_record_batch(&output, &null_mask)?;
-
- allow_duplicates! {
- assert_snapshot!(batches_to_string(&[null_joined_batch]), @r"
- +---+---+---+---+
- | a | b | x | y |
- +---+---+---+---+
- +---+---+---+---+
- ");
- }
- }
-
- Ok(())
-}
-
-#[test]
-fn test_partition_statistics() -> Result<()> {
- use crate::ExecutionPlan;
- use datafusion_common::stats::Precision;
-
- let left = build_table(
- ("a1", &vec![1, 2, 3]),
- ("b1", &vec![4, 5, 5]),
- ("c1", &vec![7, 8, 9]),
- );
- let right = build_table(
- ("a2", &vec![10, 20, 30]),
- ("b1", &vec![4, 5, 6]),
- ("c2", &vec![70, 80, 90]),
- );
-
- let on = vec![(
- Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
- Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
- )];
-
- // Test different join types to ensure partition_statistics works correctly for all
- let join_types = vec![
- (Inner, 6), // left cols + right cols
- (Left, 6), // left cols + right cols
- (Right, 6), // left cols + right cols
- (Full, 6), // left cols + right cols
- (LeftSemi, 3), // only left cols
- (LeftAnti, 3), // only left cols
- (RightSemi, 3), // only right cols
- (RightAnti, 3), // only right cols
- ];
-
- for (join_type, expected_cols) in join_types {
- let join_exec =
- join(Arc::clone(&left), Arc::clone(&right), on.clone(), join_type)?;
-
- // Test aggregate statistics (partition = None)
- // Should return meaningful statistics computed from both inputs
- let stats = join_exec.partition_statistics(None)?;
+ // Test aggregate statistics (partition = None)
+ // Should return meaningful statistics computed from both inputs
+ let stats = join_exec.partition_statistics(None)?;
assert_eq!(
stats.column_statistics.len(),
expected_cols,
@@ -3557,7 +3531,878 @@ async fn consume_stream_until_finish_barrier_reached(
Ok((output_batched, after_finish_barrier_reached))
}
+/// Exercises the multi-source interleave path in `materialize_right_columns`.
+///
+/// When the right (buffered) side is split into many small batches with unique
+/// keys, a single `freeze_streamed()` call references multiple `BufferedBatch`es.
+/// This forces the `interleave` kernel instead of the single-source `take` path.
+/// Without this test, the interleave path has zero coverage from unit tests
+/// (fuzz tests use ~100 unique keys across 1000 rows, so all keys fit in one
+/// buffered batch).
+#[tokio::test]
+async fn join_filtered_with_multiple_buffered_batches() -> Result<()> {
+ let left_schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new("val_l", DataType::Int32, false),
+ ]));
+ let right_schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new("val_r", DataType::Int32, false),
+ ]));
+
+ // Left: single batch, keys 1..=6
+ let left_batch = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])),
+ Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])),
+ ],
+ )?;
+ let left = build_table_from_batches(vec![left_batch]);
+
+ // Right: one row per batch so each key lives in a separate BufferedBatch
+ let right_batches: Vec<RecordBatch> = (1..=6)
+ .map(|k| {
+ RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![k])),
+ Arc::new(Int32Array::from(vec![k * 100])),
+ ],
+ )
+ .unwrap()
+ })
+ .collect();
+ let right = build_table_from_batches(right_batches);
+
+ let on: JoinOn = vec![(
+ Arc::new(Column::new_with_schema("key", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("key", &right.schema())?) as _,
+ )];
+
+ // Filter: val_l + val_r < 350 — passes for keys 1-3, fails for 4-6
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("val_l", 0)),
+ Operator::Plus,
+ Arc::new(Column::new("val_r", 1)),
+ )),
+ Operator::Lt,
+ Arc::new(datafusion_physical_expr::expressions::Literal::new(
+ datafusion_common::ScalarValue::Int32(Some(350)),
+ )),
+ )),
+ vec![
+ ColumnIndex {
+ index: 1,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 1,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ Field::new("val_l", DataType::Int32, true),
+ Field::new("val_r", DataType::Int32, true),
+ ])),
+ );
+
+ // Inner: only rows passing the filter
+ let (_, batches) = join_collect_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ Inner,
+ )
+ .await?;
+ let result = batches_to_sort_string(&batches);
+ assert_snapshot!(result, @r"
+ +-----+-------+-----+-------+
+ | key | val_l | key | val_r |
+ +-----+-------+-----+-------+
+ | 1 | 10 | 1 | 100 |
+ | 2 | 20 | 2 | 200 |
+ | 3 | 30 | 3 | 300 |
+ +-----+-------+-----+-------+
+ ");
+
+ // Left: unmatched left rows get null right columns
+ let (_, batches) = join_collect_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ Left,
+ )
+ .await?;
+ let result = batches_to_sort_string(&batches);
+ assert_snapshot!(result, @r"
+ +-----+-------+-----+-------+
+ | key | val_l | key | val_r |
+ +-----+-------+-----+-------+
+ | 1 | 10 | 1 | 100 |
+ | 2 | 20 | 2 | 200 |
+ | 3 | 30 | 3 | 300 |
+ | 4 | 40 | | |
+ | 5 | 50 | | |
+ | 6 | 60 | | |
+ +-----+-------+-----+-------+
+ ");
+
+ // Full: unmatched rows on both sides get null columns
+ let (_, batches) = join_collect_with_filter(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ filter.clone(),
+ Full,
+ )
+ .await?;
+ let result = batches_to_sort_string(&batches);
+ assert_snapshot!(result, @r"
+ +-----+-------+-----+-------+
+ | key | val_l | key | val_r |
+ +-----+-------+-----+-------+
+ | | | 4 | 400 |
+ | | | 5 | 500 |
+ | | | 6 | 600 |
+ | 1 | 10 | 1 | 100 |
+ | 2 | 20 | 2 | 200 |
+ | 3 | 30 | 3 | 300 |
+ | 4 | 40 | | |
+ | 5 | 50 | | |
+ | 6 | 60 | | |
+ +-----+-------+-----+-------+
+ ");
+
+ Ok(())
+}
+
/// Returns the column names on the schema
fn columns(schema: &Schema) -> Vec<String> {
schema.fields().iter().map(|f| f.name().clone()).collect()
}
+
+// ==================== BitwiseSortMergeJoinStream direct tests ====================
+//
+// These tests construct a BitwiseSortMergeJoinStream directly (bypassing exec)
+// to exercise async re-entry and spill edge cases using PendingStream.
+
+/// Create test memory/spill resources for stream-level tests.
+fn test_stream_resources(
+ inner_schema: SchemaRef,
+ metrics: &ExecutionPlanMetricsSet,
+) -> (
+ datafusion_execution::memory_pool::MemoryReservation,
+ SpillManager,
+ Arc<datafusion_execution::runtime_env::RuntimeEnv>,
+) {
+ let ctx = TaskContext::default();
+ let runtime_env = ctx.runtime_env();
+ let reservation = MemoryConsumer::new("test").register(ctx.memory_pool());
+ let spill_manager = SpillManager::new(
+ Arc::clone(&runtime_env),
+ SpillMetrics::new(metrics, 0),
+ inner_schema,
+ );
+ (reservation, spill_manager, runtime_env)
+}
+
+/// A RecordBatch stream that yields Poll::Pending once before delivering
+/// each batch at a specified index. This simulates the behavior of
+/// repartitioned tokio::sync::mpsc channels where data isn't immediately
+/// available.
+struct PendingStream {
+ batches: Vec<RecordBatch>,
+ index: usize,
+ /// If pending_before[i] is true, yield Pending once before delivering
+ /// the batch at index i.
+ pending_before: Vec<bool>,
+ /// True if we've already yielded Pending for the current index.
+ yielded_pending: bool,
+ schema: SchemaRef,
+}
+
+impl PendingStream {
+ fn new(batches: Vec<RecordBatch>, pending_before: Vec<bool>) -> Self {
+ assert_eq!(batches.len(), pending_before.len());
+ let schema = batches[0].schema();
+ Self {
+ batches,
+ index: 0,
+ pending_before,
+ yielded_pending: false,
+ schema,
+ }
+ }
+}
+
+impl Stream for PendingStream {
+ type Item = Result<RecordBatch>;
+
+ fn poll_next(
+ mut self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ ) -> Poll<Option<Self::Item>> {
+ if self.index >= self.batches.len() {
+ return Poll::Ready(None);
+ }
+ if self.pending_before[self.index] && !self.yielded_pending {
+ self.yielded_pending = true;
+ cx.waker().wake_by_ref();
+ return Poll::Pending;
+ }
+ self.yielded_pending = false;
+ let batch = self.batches[self.index].clone();
+ self.index += 1;
+ Poll::Ready(Some(Ok(batch)))
+ }
+}
+
+impl RecordBatchStream for PendingStream {
+ fn schema(&self) -> SchemaRef {
+ Arc::clone(&self.schema)
+ }
+}
+
+/// Helper: collect all output from a BitwiseSortMergeJoinStream.
+async fn collect_stream(stream: BitwiseSortMergeJoinStream) -> Result<Vec<RecordBatch>> {
+ common::collect(Box::pin(stream)).await
+}
+
+/// Reproduces the buffer_inner_key_group re-entry bug:
+///
+/// When buffer_inner_key_group buffers inner rows across batch boundaries
+/// and poll_next_inner_batch returns Pending mid-way, the ready! macro
+/// exits poll_join. On re-entry, the merge-scan reaches Equal again and
+/// calls buffer_inner_key_group a second time -- which starts with
+/// clear(), destroying the partially collected inner rows. Previously
+/// consumed batches are gone, so re-buffering misses them.
+///
+/// Setup:
+/// - Inner: 3 single-row batches, all with key=1, filter values c2=[10, 20, 30]
+/// - Outer: 1 row, key=1, filter value c1=10
+/// - Filter: c1 == c2 (only first inner row c2=10 matches)
+/// - Pending injected before 3rd inner batch
+///
+/// Without the bug: outer row emitted (match via c2=10)
+/// With the bug: outer row missing (c2=10 batch lost on re-entry)
+#[tokio::test]
+async fn filter_buffer_pending_loses_inner_rows() -> Result<()> {
+ let left_schema = Arc::new(Schema::new(vec![
+ Field::new("a1", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c1", DataType::Int32, false),
+ ]));
+ let right_schema = Arc::new(Schema::new(vec![
+ Field::new("a2", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ]));
+
+ // Outer: 1 row, key=1, c1=10
+ let outer_batch = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![1])), // join key
+ Arc::new(Int32Array::from(vec![10])), // filter value
+ ],
+ )?;
+
+ // Inner: 3 single-row batches, key=1, c2=[10, 20, 30]
+ let inner_batch1 = RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![100])),
+ Arc::new(Int32Array::from(vec![1])), // join key
+ Arc::new(Int32Array::from(vec![10])), // matches filter
+ ],
+ )?;
+ let inner_batch2 = RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![200])),
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![20])), // doesn't match
+ ],
+ )?;
+ let inner_batch3 = RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![300])),
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![30])), // doesn't match
+ ],
+ )?;
+
+ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
+ vec![outer_batch],
+ vec![false], // outer delivers immediately
+ ));
+ let inner: SendableRecordBatchStream = Box::pin(PendingStream::new(
+ vec![inner_batch1, inner_batch2, inner_batch3],
+ vec![false, false, true], // Pending before 3rd batch
+ ));
+
+ // Filter: c1 == c2
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("c1", 0)),
+ Operator::Eq,
+ Arc::new(Column::new("c2", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ])),
+ );
+
+ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let inner_schema = inner.schema();
+ let (reservation, spill_manager, runtime_env) =
+ test_stream_resources(inner_schema, &metrics);
+ let stream = BitwiseSortMergeJoinStream::try_new(
+ left_schema, // output schema = outer schema for semi
+ vec![SortOptions::default()],
+ NullEquality::NullEqualsNothing,
+ outer,
+ inner,
+ on_outer,
+ on_inner,
+ Some(filter),
+ LeftSemi,
+ 8192,
+ 0,
+ &metrics,
+ reservation,
+ spill_manager,
+ runtime_env,
+ )?;
+
+ let batches = collect_stream(stream).await?;
+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+ assert_eq!(
+ total, 1,
+ "LeftSemi with filter: outer row should be emitted because \
+ inner row c2=10 matches filter c1==c2. Got {total} rows."
+ );
+ Ok(())
+}
+
+/// Reproduces the no-filter boundary Pending re-entry bug:
+///
+/// When an outer key group spans a batch boundary, the no-filter path
+/// emits the current batch, then polls for the next outer batch. If
+/// poll returns Pending, poll_join exits. On re-entry, without the
+/// PendingBoundary fix, the new batch is processed fresh by the
+/// merge-scan. Since inner already advanced past this key, the outer
+/// rows with the matching key are skipped via Ordering::Less.
+///
+/// Setup:
+/// - Outer: 2 single-row batches, both with key=1 (key group spans boundary)
+/// - Inner: 1 row with key=1
+/// - Pending injected on outer before 2nd batch
+///
+/// Without fix: only first outer row emitted (second lost on re-entry)
+/// With fix: both outer rows emitted
+#[tokio::test]
+async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {
+ let left_schema = Arc::new(Schema::new(vec![
+ Field::new("a1", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c1", DataType::Int32, false),
+ ]));
+ let right_schema = Arc::new(Schema::new(vec![
+ Field::new("a2", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ]));
+
+ // Outer: 2 single-row batches, both key=1
+ let outer_batch1 = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![10])),
+ ],
+ )?;
+ let outer_batch2 = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![2])),
+ Arc::new(Int32Array::from(vec![1])), // same key
+ Arc::new(Int32Array::from(vec![20])),
+ ],
+ )?;
+
+ // Inner: 1 row, key=1
+ let inner_batch = RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![100])),
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![50])),
+ ],
+ )?;
+
+ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
+ vec![outer_batch1, outer_batch2],
+ vec![false, true], // Pending before 2nd outer batch
+ ));
+ let inner: SendableRecordBatchStream =
+ Box::pin(PendingStream::new(vec![inner_batch], vec![false]));
+
+ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let inner_schema = inner.schema();
+ let (reservation, spill_manager, runtime_env) =
+ test_stream_resources(inner_schema, &metrics);
+ let stream = BitwiseSortMergeJoinStream::try_new(
+ left_schema,
+ vec![SortOptions::default()],
+ NullEquality::NullEqualsNothing,
+ outer,
+ inner,
+ on_outer,
+ on_inner,
+ None, // no filter
+ LeftSemi,
+ 8192,
+ 0,
+ &metrics,
+ reservation,
+ spill_manager,
+ runtime_env,
+ )?;
+
+ let batches = collect_stream(stream).await?;
+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+ assert_eq!(
+ total, 2,
+ "LeftSemi no filter: both outer rows (key=1) should be emitted \
+ because inner has key=1. Got {total} rows."
+ );
+ Ok(())
+}
+
+/// Tests the filtered boundary Pending re-entry: outer key group spans
+/// batches with a filter, and poll_next_outer_batch returns Pending.
+///
+/// Setup:
+/// - Outer: 2 single-row batches, both key=1, c1=[10, 20]
+/// - Inner: 1 row, key=1, c2=10
+/// - Filter: c1 == c2 (first outer row matches, second doesn't)
+/// - Pending before 2nd outer batch
+///
+/// Expected: 1 row (only the first outer row c1=10 passes the filter)
+#[tokio::test]
+async fn filtered_boundary_pending_outer_rows() -> Result<()> {
+ let left_schema = Arc::new(Schema::new(vec![
+ Field::new("a1", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c1", DataType::Int32, false),
+ ]));
+ let right_schema = Arc::new(Schema::new(vec![
+ Field::new("a2", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ]));
+
+ let outer_batch1 = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![10])), // matches filter
+ ],
+ )?;
+ let outer_batch2 = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![2])),
+ Arc::new(Int32Array::from(vec![1])), // same key
+ Arc::new(Int32Array::from(vec![20])), // doesn't match
+ ],
+ )?;
+
+ let inner_batch = RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![100])),
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![10])),
+ ],
+ )?;
+
+ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
+ vec![outer_batch1, outer_batch2],
+ vec![false, true], // Pending before 2nd outer batch
+ ));
+ let inner: SendableRecordBatchStream =
+ Box::pin(PendingStream::new(vec![inner_batch], vec![false]));
+
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("c1", 0)),
+ Operator::Eq,
+ Arc::new(Column::new("c2", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ])),
+ );
+
+ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let inner_schema = inner.schema();
+ let (reservation, spill_manager, runtime_env) =
+ test_stream_resources(inner_schema, &metrics);
+ let stream = BitwiseSortMergeJoinStream::try_new(
+ left_schema,
+ vec![SortOptions::default()],
+ NullEquality::NullEqualsNothing,
+ outer,
+ inner,
+ on_outer,
+ on_inner,
+ Some(filter),
+ LeftSemi,
+ 8192,
+ 0,
+ &metrics,
+ reservation,
+ spill_manager,
+ runtime_env,
+ )?;
+
+ let batches = collect_stream(stream).await?;
+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+ assert_eq!(
+ total, 1,
+ "LeftSemi filtered boundary: only first outer row (c1=10) matches \
+ filter c1==c2. Got {total} rows."
+ );
+ Ok(())
+}
+
+// ── Bitwise stream spill tests ─────────────────────────────────────────────
+
+/// Exercises inner key group spilling under memory pressure.
+///
+/// Uses a tiny memory limit (100 bytes) with disk spilling enabled. Since our
+/// operator only buffers inner rows when a filter is present, this test includes
+/// a filter (c1 < c2, always true). Verifies:
+/// 1. Spill metrics are recorded (spill_count, spilled_bytes, spilled_rows > 0)
+/// 2. Results match a non-spilled run
+#[tokio::test]
+async fn bitwise_spill_with_filter() -> Result<()> {
+ let left = build_table(
+ ("a1", &vec![1, 2, 3, 4, 5, 6]),
+ ("b1", &vec![1, 2, 3, 4, 5, 6]),
+ ("c1", &vec![4, 5, 6, 7, 8, 9]),
+ );
+ let right = build_table(
+ ("a2", &vec![10, 20, 30, 40, 50]),
+ ("b1", &vec![1, 3, 4, 6, 8]),
+ ("c2", &vec![50, 60, 70, 80, 90]),
+ );
+ let on = vec![(
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
+ )];
+ let sort_options = vec![SortOptions::default(); on.len()];
+
+ // c1 < c2 is always true for matching keys
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("c1", 0)),
+ Operator::Lt,
+ Arc::new(Column::new("c2", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ])),
+ );
+
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(100, 1.0)
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()?;
+
+ for batch_size in [1, 50] {
+ let session_config = SessionConfig::default().with_batch_size(batch_size);
+
+ for join_type in [LeftSemi, LeftAnti, RightSemi, RightAnti] {
+ let task_ctx = Arc::new(
+ TaskContext::default()
+ .with_session_config(session_config.clone())
+ .with_runtime(Arc::clone(&runtime)),
+ );
+
+ let join = SortMergeJoinExec::try_new(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ Some(filter.clone()),
+ join_type,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join.execute(0, task_ctx)?;
+ let spilled_result = common::collect(stream).await.unwrap();
+
+ assert!(
+ join.metrics().is_some(),
+ "metrics missing for {join_type:?}"
+ );
+ let metrics = join.metrics().unwrap();
+ assert!(
+ metrics.spill_count().unwrap() > 0,
+ "expected spill_count > 0 for {join_type:?}, batch_size={batch_size}"
+ );
+ assert!(
+ metrics.spilled_bytes().unwrap() > 0,
+ "expected spilled_bytes > 0 for {join_type:?}, batch_size={batch_size}"
+ );
+ assert!(
+ metrics.spilled_rows().unwrap() > 0,
+ "expected spilled_rows > 0 for {join_type:?}, batch_size={batch_size}"
+ );
+
+ // Run without spilling and compare results
+ let task_ctx_no_spill = Arc::new(
+ TaskContext::default().with_session_config(session_config.clone()),
+ );
+ let join_no_spill = SortMergeJoinExec::try_new(
+ Arc::clone(&left),
+ Arc::clone(&right),
+ on.clone(),
+ Some(filter.clone()),
+ join_type,
+ sort_options.clone(),
+ NullEquality::NullEqualsNothing,
+ )?;
+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;
+ let no_spill_result = common::collect(stream).await.unwrap();
+
+ let no_spill_metrics = join_no_spill.metrics().unwrap();
+ assert_eq!(
+ no_spill_metrics.spill_count(),
+ Some(0),
+ "unexpected spill for {join_type:?} without memory limit"
+ );
+
+ assert_eq!(
+ spilled_result, no_spill_result,
+ "spilled vs non-spilled results differ for {join_type:?}, batch_size={batch_size}"
+ );
+ }
+ }
+
+ Ok(())
+}
+
+/// Reproduces a bug where `resume_boundary` for the Filtered pending case
+/// only checks `inner_key_buffer.is_empty()` but ignores `inner_key_spill`.
+/// After spilling, the in-memory buffer is cleared while the spill file
+/// holds the data. If the outer key group spans a batch boundary, the
+/// second outer batch's rows are never evaluated against the inner group.
+///
+/// Setup:
+/// - Outer: 2 single-row batches, both key=1, c1=[10, 10]
+/// - Inner: 1 batch with many rows all key=1 (enough to trigger spill)
+/// - Filter: c1 == c2 (matches when c2=10)
+/// - Memory limit: tiny (100 bytes) to force spilling
+/// - Pending before 2nd outer batch to trigger boundary re-entry
+///
+/// Expected: both outer rows match (semi=2 rows, anti=0 rows)
+/// Bug: second outer row is skipped because resume_boundary sees empty
+/// inner_key_buffer and skips re-evaluation.
+#[tokio::test]
+async fn spill_filtered_boundary_loses_outer_rows() -> Result<()> {
+ let left_schema = Arc::new(Schema::new(vec![
+ Field::new("a1", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c1", DataType::Int32, false),
+ ]));
+ let right_schema = Arc::new(Schema::new(vec![
+ Field::new("a2", DataType::Int32, false),
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ]));
+
+ // Two single-row outer batches with the same key -- key group spans boundary
+ let outer_batch1 = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1])),
+ Arc::new(Int32Array::from(vec![1])), // key=1
+ Arc::new(Int32Array::from(vec![10])), // matches filter
+ ],
+ )?;
+ let outer_batch2 = RecordBatch::try_new(
+ Arc::clone(&left_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![2])),
+ Arc::new(Int32Array::from(vec![1])), // same key=1
+ Arc::new(Int32Array::from(vec![10])), // also matches filter
+ ],
+ )?;
+
+ // Inner: many rows with key=1 to force spilling, followed by key=2.
+ // c2=10 so the filter c1==c2 passes for both outer rows.
+ // The key=2 row ensures the inner cursor advances past the key group
+ // (buffer_inner_key_group returns Ok(false) instead of Ok(true)).
+ let n_inner = 200;
+ let mut inner_a = vec![100; n_inner];
+ inner_a.push(101);
+ let mut inner_b = vec![1; n_inner];
+ inner_b.push(2); // different key -- forces inner cursor past key=1
+ let mut inner_c = vec![10; n_inner];
+ inner_c.push(10);
+ let inner_batch = RecordBatch::try_new(
+ Arc::clone(&right_schema),
+ vec![
+ Arc::new(Int32Array::from(inner_a)),
+ Arc::new(Int32Array::from(inner_b)),
+ Arc::new(Int32Array::from(inner_c)),
+ ],
+ )?;
+
+ // Filter: c1 == c2
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("c1", 0)),
+ Operator::Eq,
+ Arc::new(Column::new("c2", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ])),
+ );
+
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_limit(100, 1.0)
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()?;
+
+ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];
+
+ for join_type in [LeftSemi, LeftAnti] {
+ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(
+ vec![outer_batch1.clone(), outer_batch2.clone()],
+ vec![false, true], // Pending before 2nd outer batch
+ ));
+ let inner: SendableRecordBatchStream =
+ Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);
+ let spill_manager = SpillManager::new(
+ Arc::clone(&runtime),
+ SpillMetrics::new(&metrics, 0),
+ Arc::clone(&right_schema),
+ );
+
+ let stream = BitwiseSortMergeJoinStream::try_new(
+ Arc::clone(&left_schema),
+ vec![SortOptions::default()],
+ NullEquality::NullEqualsNothing,
+ outer,
+ inner,
+ on_outer.clone(),
+ on_inner.clone(),
+ Some(filter.clone()),
+ join_type,
+ 8192,
+ 0,
+ &metrics,
+ reservation,
+ spill_manager,
+ Arc::clone(&runtime),
+ )?;
+
+ let batches = collect_stream(stream).await?;
+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+
+ match join_type {
+ LeftSemi => {
+ assert_eq!(
+ total, 2,
+ "LeftSemi spill+boundary: both outer rows match filter, \
+ expected 2 rows, got {total}"
+ );
+ }
+ LeftAnti => {
+ assert_eq!(
+ total, 0,
+ "LeftAnti spill+boundary: both outer rows match filter, \
+ expected 0 rows, got {total}"
+ );
+ }
+ _ => unreachable!(),
+ }
+ }
+
+ Ok(())
+}
diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.slt
index 15f867d900..6244ab70c1 100644
--- a/datafusion/sqllogictest/test_files/sort_merge_join.slt
+++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt
@@ -941,6 +941,36 @@ WHERE t2.a > 3 OR t2.a IN (SELECT t3.x FROM t3 WHERE t2.b < 150)
----
4 101 1001
+# Regression test for https://github.com/apache/datafusion/issues/21197
+# LeftMark join with cross-table join filter on non-nullable columns.
+# The OR forces a LeftMark join. The cross-table filter (t2_data <> t1_data)
+# becomes a join filter referencing both sides. At scale, unmatched rows in
+# the mark join produce null right-side indices, and fetch_right_columns_by_idxs
+# maps those to null values in columns the schema declares non-nullable,
+# causing RecordBatch::try_new to fail.
+query I
+WITH t1_sorted AS (
+ SELECT value % 1000 as key, value as data
+ FROM range(10000)
+ ORDER BY key, data
+),
+t2_sorted AS (
+ SELECT value % 1000 as key, value as data
+ FROM range(100000)
+ ORDER BY key, data
+)
+SELECT count(*)
+FROM t1_sorted
+WHERE t1_sorted.data < 0
+ OR EXISTS (
+ SELECT 1 FROM t2_sorted
+ WHERE t2_sorted.key = t1_sorted.key
+ AND t2_sorted.data <> t1_sorted.data
+ AND t2_sorted.data % 100 = 0
+)
+----
+100
+
# Config reset
statement ok
RESET datafusion.optimizer.prefer_hash_join;