Apache DataFusion SQL Query Engine
perf: sort-merge join (SMJ) batch deferred filtering and move mark joins to bitwise stream. Near-unique LEFT and FULL SMJ 20-50x faster (#21184)
Matt Butrovich0be5982a913e
diff --git a/benchmarks/src/smj.rs b/benchmarks/src/smj.rsindex 1829b77a3d..3d173b7116 100644--- a/benchmarks/src/smj.rs+++ b/benchmarks/src/smj.rs@@ -39,7 +39,7 @@ use futures::StreamExt;#[derive(Debug, Args, Clone)]#[command(verbatim_doc_comment)]pub struct RunOpt {- /// Query number (between 1 and 23). If not specified, runs all queries+ /// Query number (between 1 and 26). If not specified, runs all queries#[arg(short, long)]query: Option<usize>,@@ -456,6 +456,72 @@ const SMJ_QUERIES: &[&str] = &[ON t1_sorted.key = t2_sorted.keyAND t1_sorted.data + t2_sorted.data < 10000000"#,+ // Q24: LEFT MARK 1M x 10M | 1:10 | 1%+ r#"+ WITH t1_sorted AS (+ SELECT value % 100000 as key, value as data+ FROM range(1000000)+ ORDER BY key, data+ ),+ t2_sorted AS (+ SELECT value % 100000 as key, value as data+ FROM range(10000000)+ ORDER BY key, data+ )+ SELECT t1_sorted.key, t1_sorted.data+ FROM t1_sorted+ WHERE t1_sorted.data < 0+ OR EXISTS (+ SELECT 1 FROM t2_sorted+ WHERE t2_sorted.key = t1_sorted.key+ AND t2_sorted.data <> t1_sorted.data+ AND t2_sorted.data % 100 = 0+ )+ "#,+ // Q25: LEFT MARK 1M x 10M | 1:10 | 50%+ r#"+ WITH t1_sorted AS (+ SELECT value % 100000 as key, value as data+ FROM range(1000000)+ ORDER BY key, data+ ),+ t2_sorted AS (+ SELECT value % 100000 as key, value as data+ FROM range(10000000)+ ORDER BY key, data+ )+ SELECT t1_sorted.key, t1_sorted.data+ FROM t1_sorted+ WHERE t1_sorted.data < 0+ OR EXISTS (+ SELECT 1 FROM t2_sorted+ WHERE t2_sorted.key = t1_sorted.key+ AND t2_sorted.data <> t1_sorted.data+ AND t2_sorted.data % 2 = 0+ )+ "#,+ // Q26: LEFT MARK 1M x 10M | 1:10 | 90%+ r#"+ WITH t1_sorted AS (+ SELECT value % 100000 as key, value as data+ FROM range(1000000)+ ORDER BY key, data+ ),+ t2_sorted AS (+ SELECT value % 100000 as key, value as data+ FROM range(10000000)+ ORDER BY key, data+ )+ SELECT t1_sorted.key, t1_sorted.data+ FROM t1_sorted+ WHERE t1_sorted.data < 0+ OR EXISTS (+ SELECT 1 FROM t2_sorted+ WHERE t2_sorted.key = t1_sorted.key+ AND t2_sorted.data <> t1_sorted.data+ AND t2_sorted.data % 10 <> 0+ )+ "#,];impl RunOpt {@@ -489,7 +555,10 @@ impl RunOpt {let sql = SMJ_QUERIES[query_index];benchmark_run.start_new_case(&format!("Query {query_id}"));- let query_run = self.benchmark_query(sql, &query_id.to_string(), &ctx).await;+ let expect_mark = query_id >= 24;+ let query_run = self+ .benchmark_query(sql, &query_id.to_string(), expect_mark, &ctx)+ .await;match query_run {Ok(query_results) => {for iter in query_results {@@ -513,6 +582,7 @@ impl RunOpt {&self,sql: &str,query_name: &str,+ expect_mark: bool,ctx: &SessionContext,) -> Result<Vec<QueryResult>> {let mut query_results = vec![];@@ -528,6 +598,12 @@ impl RunOpt {));}+ if expect_mark && !plan_string.contains("LeftMark") {+ return Err(exec_datafusion_err!(+ "Query {query_name} expected LeftMark join. Physical plan: {plan_string}"+ ));+ }+for i in 0..self.common.iterations {let start = Instant::now();diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rsindex 669b98e39f..f570335e92 100644--- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs@@ -38,6 +38,9 @@ use datafusion::physical_plan::joins::{};use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::{NullEquality, ScalarValue};+use datafusion_execution::TaskContext;+use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};+use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_physical_expr::PhysicalExprRef;use datafusion_physical_expr::expressions::Literal;@@ -1125,6 +1128,138 @@ impl JoinFuzzTestCase {}}+/// Fuzz test: compare SMJ (with spilling) against HJ (no spill) for filtered+/// outer joins under memory pressure. This exercises the deferred filtering ++/// spill read-back path that unit tests can't easily cover with random data.+#[tokio::test]+async fn test_filtered_join_spill_fuzz() {+ let join_types = [JoinType::Left, JoinType::Right, JoinType::Full];++ let runtime_spill = RuntimeEnvBuilder::new()+ .with_memory_limit(4096, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()+ .unwrap();++ for join_type in &join_types {+ for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {+ let input1 = make_staggered_batches_i32(1000, left_extra);+ let input2 = make_staggered_batches_i32(1000, right_extra);++ let schema1 = input1[0].schema();+ let schema2 = input2[0].schema();+ let filter = col_lt_col_filter(schema1.clone(), schema2.clone());++ let on = vec![+ (+ Arc::new(Column::new_with_schema("a", &schema1).unwrap()) as _,+ Arc::new(Column::new_with_schema("a", &schema2).unwrap()) as _,+ ),+ (+ Arc::new(Column::new_with_schema("b", &schema1).unwrap()) as _,+ Arc::new(Column::new_with_schema("b", &schema2).unwrap()) as _,+ ),+ ];++ for batch_size in [2, 49, 100] {+ let session_config = SessionConfig::new().with_batch_size(batch_size);++ // HJ baseline (no memory limit)+ let left_hj = MemorySourceConfig::try_new_exec(+ std::slice::from_ref(&input1),+ schema1.clone(),+ None,+ )+ .unwrap();+ let right_hj = MemorySourceConfig::try_new_exec(+ std::slice::from_ref(&input2),+ schema2.clone(),+ None,+ )+ .unwrap();+ let hj = Arc::new(+ HashJoinExec::try_new(+ left_hj,+ right_hj,+ on.clone(),+ Some(filter.clone()),+ join_type,+ None,+ PartitionMode::Partitioned,+ NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let ctx_hj = SessionContext::new_with_config(session_config.clone());+ let hj_collected = collect(hj, ctx_hj.task_ctx()).await.unwrap();++ // SMJ with spilling+ let left_smj = MemorySourceConfig::try_new_exec(+ std::slice::from_ref(&input1),+ schema1.clone(),+ None,+ )+ .unwrap();+ let right_smj = MemorySourceConfig::try_new_exec(+ std::slice::from_ref(&input2),+ schema2.clone(),+ None,+ )+ .unwrap();+ let smj = Arc::new(+ SortMergeJoinExec::try_new(+ left_smj,+ right_smj,+ on.clone(),+ Some(filter.clone()),+ *join_type,+ vec![SortOptions::default(); on.len()],+ NullEquality::NullEqualsNothing,+ )+ .unwrap(),+ );+ let task_ctx_spill = Arc::new(+ TaskContext::default()+ .with_session_config(session_config)+ .with_runtime(Arc::clone(&runtime_spill)),+ );+ let smj_collected = collect(smj, task_ctx_spill).await.unwrap();++ let hj_rows: usize = hj_collected.iter().map(|b| b.num_rows()).sum();+ let smj_rows: usize = smj_collected.iter().map(|b| b.num_rows()).sum();++ assert_eq!(+ hj_rows, smj_rows,+ "Row count mismatch for {join_type:?} batch_size={batch_size} \+ left_extra={left_extra} right_extra={right_extra}: \+ HJ={hj_rows} SMJ={smj_rows}"+ );++ if hj_rows > 0 {+ let hj_fmt =+ pretty_format_batches(&hj_collected).unwrap().to_string();+ let smj_fmt =+ pretty_format_batches(&smj_collected).unwrap().to_string();++ let mut hj_sorted: Vec<&str> = hj_fmt.trim().lines().collect();+ hj_sorted.sort_unstable();+ let mut smj_sorted: Vec<&str> = smj_fmt.trim().lines().collect();+ smj_sorted.sort_unstable();++ assert_eq!(+ hj_sorted, smj_sorted,+ "Content mismatch for {join_type:?} batch_size={batch_size} \+ left_extra={left_extra} right_extra={right_extra}"+ );+ }+ }+ }+ }+}+/// Return randomly sized record batches with:/// two sorted int32 columns 'a', 'b' ranged from 0..99 as join columns/// two random int32 columns 'x', 'y' as other columnsdiff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex 0a198b81a9..2cdfa1e6ac 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -34,7 +34,6 @@ mod cross_join;mod hash_join;mod nested_loop_join;mod piecewise_merge_join;-pub(crate) mod semi_anti_sort_merge_join;mod sort_merge_join;mod stream_join_utils;mod symmetric_hash_join;diff --git a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rs b/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rsdeleted file mode 100644index 5c5a2727b6..0000000000--- a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rs+++ /dev/null@@ -1,25 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! Specialized Sort Merge Join stream for Semi/Anti joins.-//!-//! Used internally by `SortMergeJoinExec` for semi/anti join types.--pub(crate) mod stream;--#[cfg(test)]-mod tests;diff --git a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rsdeleted file mode 100644index 92dc78503a..0000000000--- a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rs+++ /dev/null@@ -1,801 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--use std::pin::Pin;-use std::sync::Arc;-use std::task::{Context, Poll};--use super::stream::SemiAntiSortMergeJoinStream;-use crate::ExecutionPlan;-use crate::RecordBatchStream;-use crate::common;-use crate::expressions::Column;-use crate::joins::SortMergeJoinExec;-use crate::joins::utils::{ColumnIndex, JoinFilter};-use crate::metrics::ExecutionPlanMetricsSet;-use crate::metrics::{MetricBuilder, SpillMetrics};-use crate::spill::spill_manager::SpillManager;-use crate::test::TestMemoryExec;--use arrow::array::{Int32Array, RecordBatch};-use arrow::compute::SortOptions;-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};-use datafusion_common::JoinSide;-use datafusion_common::JoinType::*;-use datafusion_common::{NullEquality, Result};-use datafusion_execution::memory_pool::MemoryConsumer;-use datafusion_execution::{SendableRecordBatchStream, TaskContext};-use datafusion_expr::Operator;-use datafusion_physical_expr::expressions::BinaryExpr;-use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;-use futures::Stream;--/// Create test memory/spill resources for stream-level tests.-fn test_stream_resources(- inner_schema: SchemaRef,- metrics: &ExecutionPlanMetricsSet,-) -> (- datafusion_execution::memory_pool::MemoryReservation,- crate::metrics::Gauge,- SpillManager,- Arc<datafusion_execution::runtime_env::RuntimeEnv>,-) {- let ctx = TaskContext::default();- let runtime_env = ctx.runtime_env();- let reservation = MemoryConsumer::new("test").register(ctx.memory_pool());- let peak_mem_used = MetricBuilder::new(metrics).gauge("peak_mem_used", 0);- let spill_manager = SpillManager::new(- Arc::clone(&runtime_env),- SpillMetrics::new(metrics, 0),- inner_schema,- );- (reservation, peak_mem_used, spill_manager, runtime_env)-}--fn build_table(- a: (&str, &Vec<i32>),- b: (&str, &Vec<i32>),- c: (&str, &Vec<i32>),-) -> Arc<dyn ExecutionPlan> {- let schema = Arc::new(Schema::new(vec![- Field::new(a.0, DataType::Int32, false),- Field::new(b.0, DataType::Int32, false),- Field::new(c.0, DataType::Int32, false),- ]));- let batch = RecordBatch::try_new(- Arc::clone(&schema),- vec![- Arc::new(Int32Array::from(a.1.clone())),- Arc::new(Int32Array::from(b.1.clone())),- Arc::new(Int32Array::from(c.1.clone())),- ],- )- .unwrap();- TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap()-}--// ── Async re-entry tests using PendingStream ──────────────────────────────-// reproduce bugs that only manifest when async input streams return-// Poll::Pending at specific points.--/// A RecordBatch stream that yields Poll::Pending once before delivering-/// each batch at a specified index. This simulates the behavior of-/// repartitioned tokio::sync::mpsc channels where data isn't immediately-/// available.-struct PendingStream {- batches: Vec<RecordBatch>,- index: usize,- /// If pending_before[i] is true, yield Pending once before delivering- /// the batch at index i.- pending_before: Vec<bool>,- /// True if we've already yielded Pending for the current index.- yielded_pending: bool,- schema: SchemaRef,-}--impl PendingStream {- fn new(batches: Vec<RecordBatch>, pending_before: Vec<bool>) -> Self {- assert_eq!(batches.len(), pending_before.len());- let schema = batches[0].schema();- Self {- batches,- index: 0,- pending_before,- yielded_pending: false,- schema,- }- }-}--impl Stream for PendingStream {- type Item = Result<RecordBatch>;-- fn poll_next(- mut self: Pin<&mut Self>,- cx: &mut Context<'_>,- ) -> Poll<Option<Self::Item>> {- if self.index >= self.batches.len() {- return Poll::Ready(None);- }- if self.pending_before[self.index] && !self.yielded_pending {- self.yielded_pending = true;- cx.waker().wake_by_ref();- return Poll::Pending;- }- self.yielded_pending = false;- let batch = self.batches[self.index].clone();- self.index += 1;- Poll::Ready(Some(Ok(batch)))- }-}--impl RecordBatchStream for PendingStream {- fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)- }-}--/// Helper: collect all output from a SemiAntiSortMergeJoinStream.-async fn collect_stream(stream: SemiAntiSortMergeJoinStream) -> Result<Vec<RecordBatch>> {- common::collect(Box::pin(stream)).await-}--/// Reproduces the buffer_inner_key_group re-entry bug:-///-/// When buffer_inner_key_group buffers inner rows across batch boundaries-/// and poll_next_inner_batch returns Pending mid-way, the ready! macro-/// exits poll_join. On re-entry, the merge-scan reaches Equal again and-/// calls buffer_inner_key_group a second time — which starts with-/// clear(), destroying the partially collected inner rows. Previously-/// consumed batches are gone, so re-buffering misses them.-///-/// Setup:-/// - Inner: 3 single-row batches, all with key=1, filter values c2=[10, 20, 30]-/// - Outer: 1 row, key=1, filter value c1=10-/// - Filter: c1 == c2 (only first inner row c2=10 matches)-/// - Pending injected before 3rd inner batch-///-/// Without the bug: outer row emitted (match via c2=10)-/// With the bug: outer row missing (c2=10 batch lost on re-entry)-#[tokio::test]-async fn filter_buffer_pending_loses_inner_rows() -> Result<()> {- let left_schema = Arc::new(Schema::new(vec![- Field::new("a1", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c1", DataType::Int32, false),- ]));- let right_schema = Arc::new(Schema::new(vec![- Field::new("a2", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ]));-- // Outer: 1 row, key=1, c1=10- let outer_batch = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![1])), // join key- Arc::new(Int32Array::from(vec![10])), // filter value- ],- )?;-- // Inner: 3 single-row batches, key=1, c2=[10, 20, 30]- let inner_batch1 = RecordBatch::try_new(- Arc::clone(&right_schema),- vec![- Arc::new(Int32Array::from(vec![100])),- Arc::new(Int32Array::from(vec![1])), // join key- Arc::new(Int32Array::from(vec![10])), // matches filter- ],- )?;- let inner_batch2 = RecordBatch::try_new(- Arc::clone(&right_schema),- vec![- Arc::new(Int32Array::from(vec![200])),- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![20])), // doesn't match- ],- )?;- let inner_batch3 = RecordBatch::try_new(- Arc::clone(&right_schema),- vec![- Arc::new(Int32Array::from(vec![300])),- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![30])), // doesn't match- ],- )?;-- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(- vec![outer_batch],- vec![false], // outer delivers immediately- ));- let inner: SendableRecordBatchStream = Box::pin(PendingStream::new(- vec![inner_batch1, inner_batch2, inner_batch3],- vec![false, false, true], // Pending before 3rd batch- ));-- // Filter: c1 == c2- let filter = JoinFilter::new(- Arc::new(BinaryExpr::new(- Arc::new(Column::new("c1", 0)),- Operator::Eq,- Arc::new(Column::new("c2", 1)),- )),- vec![- ColumnIndex {- index: 2,- side: JoinSide::Left,- },- ColumnIndex {- index: 2,- side: JoinSide::Right,- },- ],- Arc::new(Schema::new(vec![- Field::new("c1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ])),- );-- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];-- let metrics = ExecutionPlanMetricsSet::new();- let inner_schema = inner.schema();- let (reservation, peak_mem_used, spill_manager, runtime_env) =- test_stream_resources(inner_schema, &metrics);- let stream = SemiAntiSortMergeJoinStream::try_new(- left_schema, // output schema = outer schema for semi- vec![SortOptions::default()],- NullEquality::NullEqualsNothing,- outer,- inner,- on_outer,- on_inner,- Some(filter),- LeftSemi,- 8192,- 0,- &metrics,- reservation,- peak_mem_used,- spill_manager,- runtime_env,- )?;-- let batches = collect_stream(stream).await?;- let total: usize = batches.iter().map(|b| b.num_rows()).sum();- assert_eq!(- total, 1,- "LeftSemi with filter: outer row should be emitted because \- inner row c2=10 matches filter c1==c2. Got {total} rows."- );- Ok(())-}--/// Reproduces the no-filter boundary Pending re-entry bug:-///-/// When an outer key group spans a batch boundary, the no-filter path-/// emits the current batch, then polls for the next outer batch. If-/// poll returns Pending, poll_join exits. On re-entry, without the-/// PendingBoundary fix, the new batch is processed fresh by the-/// merge-scan. Since inner already advanced past this key, the outer-/// rows with the matching key are skipped via Ordering::Less.-///-/// Setup:-/// - Outer: 2 single-row batches, both with key=1 (key group spans boundary)-/// - Inner: 1 row with key=1-/// - Pending injected on outer before 2nd batch-///-/// Without fix: only first outer row emitted (second lost on re-entry)-/// With fix: both outer rows emitted-#[tokio::test]-async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {- let left_schema = Arc::new(Schema::new(vec![- Field::new("a1", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c1", DataType::Int32, false),- ]));- let right_schema = Arc::new(Schema::new(vec![- Field::new("a2", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ]));-- // Outer: 2 single-row batches, both key=1- let outer_batch1 = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![10])),- ],- )?;- let outer_batch2 = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![2])),- Arc::new(Int32Array::from(vec![1])), // same key- Arc::new(Int32Array::from(vec![20])),- ],- )?;-- // Inner: 1 row, key=1- let inner_batch = RecordBatch::try_new(- Arc::clone(&right_schema),- vec![- Arc::new(Int32Array::from(vec![100])),- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![50])),- ],- )?;-- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(- vec![outer_batch1, outer_batch2],- vec![false, true], // Pending before 2nd outer batch- ));- let inner: SendableRecordBatchStream =- Box::pin(PendingStream::new(vec![inner_batch], vec![false]));-- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];-- let metrics = ExecutionPlanMetricsSet::new();- let inner_schema = inner.schema();- let (reservation, peak_mem_used, spill_manager, runtime_env) =- test_stream_resources(inner_schema, &metrics);- let stream = SemiAntiSortMergeJoinStream::try_new(- left_schema,- vec![SortOptions::default()],- NullEquality::NullEqualsNothing,- outer,- inner,- on_outer,- on_inner,- None, // no filter- LeftSemi,- 8192,- 0,- &metrics,- reservation,- peak_mem_used,- spill_manager,- runtime_env,- )?;-- let batches = collect_stream(stream).await?;- let total: usize = batches.iter().map(|b| b.num_rows()).sum();- assert_eq!(- total, 2,- "LeftSemi no filter: both outer rows (key=1) should be emitted \- because inner has key=1. Got {total} rows."- );- Ok(())-}--/// Tests the filtered boundary Pending re-entry: outer key group spans-/// batches with a filter, and poll_next_outer_batch returns Pending.-///-/// Setup:-/// - Outer: 2 single-row batches, both key=1, c1=[10, 20]-/// - Inner: 1 row, key=1, c2=10-/// - Filter: c1 == c2 (first outer row matches, second doesn't)-/// - Pending before 2nd outer batch-///-/// Expected: 1 row (only the first outer row c1=10 passes the filter)-#[tokio::test]-async fn filtered_boundary_pending_outer_rows() -> Result<()> {- let left_schema = Arc::new(Schema::new(vec![- Field::new("a1", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c1", DataType::Int32, false),- ]));- let right_schema = Arc::new(Schema::new(vec![- Field::new("a2", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ]));-- let outer_batch1 = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![10])), // matches filter- ],- )?;- let outer_batch2 = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![2])),- Arc::new(Int32Array::from(vec![1])), // same key- Arc::new(Int32Array::from(vec![20])), // doesn't match- ],- )?;-- let inner_batch = RecordBatch::try_new(- Arc::clone(&right_schema),- vec![- Arc::new(Int32Array::from(vec![100])),- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![10])),- ],- )?;-- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(- vec![outer_batch1, outer_batch2],- vec![false, true], // Pending before 2nd outer batch- ));- let inner: SendableRecordBatchStream =- Box::pin(PendingStream::new(vec![inner_batch], vec![false]));-- let filter = JoinFilter::new(- Arc::new(BinaryExpr::new(- Arc::new(Column::new("c1", 0)),- Operator::Eq,- Arc::new(Column::new("c2", 1)),- )),- vec![- ColumnIndex {- index: 2,- side: JoinSide::Left,- },- ColumnIndex {- index: 2,- side: JoinSide::Right,- },- ],- Arc::new(Schema::new(vec![- Field::new("c1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ])),- );-- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];-- let metrics = ExecutionPlanMetricsSet::new();- let inner_schema = inner.schema();- let (reservation, peak_mem_used, spill_manager, runtime_env) =- test_stream_resources(inner_schema, &metrics);- let stream = SemiAntiSortMergeJoinStream::try_new(- left_schema,- vec![SortOptions::default()],- NullEquality::NullEqualsNothing,- outer,- inner,- on_outer,- on_inner,- Some(filter),- LeftSemi,- 8192,- 0,- &metrics,- reservation,- peak_mem_used,- spill_manager,- runtime_env,- )?;-- let batches = collect_stream(stream).await?;- let total: usize = batches.iter().map(|b| b.num_rows()).sum();- assert_eq!(- total, 1,- "LeftSemi filtered boundary: only first outer row (c1=10) matches \- filter c1==c2. Got {total} rows."- );- Ok(())-}--// ==================== SPILL TESTS ====================--/// Exercises inner key group spilling under memory pressure.-///-/// Uses a tiny memory limit (100 bytes) with disk spilling enabled. Since our-/// operator only buffers inner rows when a filter is present, this test includes-/// a filter (c1 < c2, always true). Verifies:-/// 1. Spill metrics are recorded (spill_count, spilled_bytes, spilled_rows > 0)-/// 2. Results match a non-spilled run-#[tokio::test]-async fn spill_with_filter() -> Result<()> {- use datafusion_execution::config::SessionConfig;- use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};- use datafusion_execution::runtime_env::RuntimeEnvBuilder;-- let left = build_table(- ("a1", &vec![1, 2, 3, 4, 5, 6]),- ("b1", &vec![1, 2, 3, 4, 5, 6]),- ("c1", &vec![4, 5, 6, 7, 8, 9]),- );- let right = build_table(- ("a2", &vec![10, 20, 30, 40, 50]),- ("b1", &vec![1, 3, 4, 6, 8]),- ("c2", &vec![50, 60, 70, 80, 90]),- );- let on = vec![(- Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,- Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,- )];- let sort_options = vec![SortOptions::default(); on.len()];-- // c1 < c2 is always true for matching keys- let filter = JoinFilter::new(- Arc::new(BinaryExpr::new(- Arc::new(Column::new("c1", 0)),- Operator::Lt,- Arc::new(Column::new("c2", 1)),- )),- vec![- ColumnIndex {- index: 2,- side: JoinSide::Left,- },- ColumnIndex {- index: 2,- side: JoinSide::Right,- },- ],- Arc::new(Schema::new(vec![- Field::new("c1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ])),- );-- let runtime = RuntimeEnvBuilder::new()- .with_memory_limit(100, 1.0)- .with_disk_manager_builder(- DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),- )- .build_arc()?;-- for batch_size in [1, 50] {- let session_config = SessionConfig::default().with_batch_size(batch_size);-- for join_type in [LeftSemi, LeftAnti, RightSemi, RightAnti] {- let task_ctx = Arc::new(- TaskContext::default()- .with_session_config(session_config.clone())- .with_runtime(Arc::clone(&runtime)),- );-- let join = SortMergeJoinExec::try_new(- Arc::clone(&left),- Arc::clone(&right),- on.clone(),- Some(filter.clone()),- join_type,- sort_options.clone(),- NullEquality::NullEqualsNothing,- )?;- let stream = join.execute(0, task_ctx)?;- let spilled_result = common::collect(stream).await.unwrap();-- assert!(- join.metrics().is_some(),- "metrics missing for {join_type:?}"- );- let metrics = join.metrics().unwrap();- assert!(- metrics.spill_count().unwrap() > 0,- "expected spill_count > 0 for {join_type:?}, batch_size={batch_size}"- );- assert!(- metrics.spilled_bytes().unwrap() > 0,- "expected spilled_bytes > 0 for {join_type:?}, batch_size={batch_size}"- );- assert!(- metrics.spilled_rows().unwrap() > 0,- "expected spilled_rows > 0 for {join_type:?}, batch_size={batch_size}"- );-- // Run without spilling and compare results- let task_ctx_no_spill = Arc::new(- TaskContext::default().with_session_config(session_config.clone()),- );- let join_no_spill = SortMergeJoinExec::try_new(- Arc::clone(&left),- Arc::clone(&right),- on.clone(),- Some(filter.clone()),- join_type,- sort_options.clone(),- NullEquality::NullEqualsNothing,- )?;- let stream = join_no_spill.execute(0, task_ctx_no_spill)?;- let no_spill_result = common::collect(stream).await.unwrap();-- let no_spill_metrics = join_no_spill.metrics().unwrap();- assert_eq!(- no_spill_metrics.spill_count(),- Some(0),- "unexpected spill for {join_type:?} without memory limit"- );-- assert_eq!(- spilled_result, no_spill_result,- "spilled vs non-spilled results differ for {join_type:?}, batch_size={batch_size}"- );- }- }-- Ok(())-}--/// Reproduces a bug where `resume_boundary` for the Filtered pending case-/// only checks `inner_key_buffer.is_empty()` but ignores `inner_key_spill`.-/// After spilling, the in-memory buffer is cleared while the spill file-/// holds the data. If the outer key group spans a batch boundary, the-/// second outer batch's rows are never evaluated against the inner group.-///-/// Setup:-/// - Outer: 2 single-row batches, both key=1, c1=[10, 10]-/// - Inner: 1 batch with many rows all key=1 (enough to trigger spill)-/// - Filter: c1 == c2 (matches when c2=10)-/// - Memory limit: tiny (100 bytes) to force spilling-/// - Pending before 2nd outer batch to trigger boundary re-entry-///-/// Expected: both outer rows match (semi=2 rows, anti=0 rows)-/// Bug: second outer row is skipped because resume_boundary sees empty-/// inner_key_buffer and skips re-evaluation.-#[tokio::test]-async fn spill_filtered_boundary_loses_outer_rows() -> Result<()> {- use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};- use datafusion_execution::runtime_env::RuntimeEnvBuilder;-- let left_schema = Arc::new(Schema::new(vec![- Field::new("a1", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c1", DataType::Int32, false),- ]));- let right_schema = Arc::new(Schema::new(vec![- Field::new("a2", DataType::Int32, false),- Field::new("b1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ]));-- // Two single-row outer batches with the same key — key group spans boundary- let outer_batch1 = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![1])),- Arc::new(Int32Array::from(vec![1])), // key=1- Arc::new(Int32Array::from(vec![10])), // matches filter- ],- )?;- let outer_batch2 = RecordBatch::try_new(- Arc::clone(&left_schema),- vec![- Arc::new(Int32Array::from(vec![2])),- Arc::new(Int32Array::from(vec![1])), // same key=1- Arc::new(Int32Array::from(vec![10])), // also matches filter- ],- )?;-- // Inner: many rows with key=1 to force spilling, followed by key=2.- // c2=10 so the filter c1==c2 passes for both outer rows.- // The key=2 row ensures the inner cursor advances past the key group- // (buffer_inner_key_group returns Ok(false) instead of Ok(true)).- let n_inner = 200;- let mut inner_a = vec![100; n_inner];- inner_a.push(101);- let mut inner_b = vec![1; n_inner];- inner_b.push(2); // different key — forces inner cursor past key=1- let mut inner_c = vec![10; n_inner];- inner_c.push(10);- let inner_batch = RecordBatch::try_new(- Arc::clone(&right_schema),- vec![- Arc::new(Int32Array::from(inner_a)),- Arc::new(Int32Array::from(inner_b)),- Arc::new(Int32Array::from(inner_c)),- ],- )?;-- // Filter: c1 == c2- let filter = JoinFilter::new(- Arc::new(BinaryExpr::new(- Arc::new(Column::new("c1", 0)),- Operator::Eq,- Arc::new(Column::new("c2", 1)),- )),- vec![- ColumnIndex {- index: 2,- side: JoinSide::Left,- },- ColumnIndex {- index: 2,- side: JoinSide::Right,- },- ],- Arc::new(Schema::new(vec![- Field::new("c1", DataType::Int32, false),- Field::new("c2", DataType::Int32, false),- ])),- );-- let runtime = RuntimeEnvBuilder::new()- .with_memory_limit(100, 1.0)- .with_disk_manager_builder(- DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),- )- .build_arc()?;-- let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];- let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];-- for join_type in [LeftSemi, LeftAnti] {- let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(- vec![outer_batch1.clone(), outer_batch2.clone()],- vec![false, true], // Pending before 2nd outer batch- ));- let inner: SendableRecordBatchStream =- Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));-- let metrics = ExecutionPlanMetricsSet::new();- let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);- let peak_mem_used = MetricBuilder::new(&metrics).gauge("peak_mem_used", 0);- let spill_manager = SpillManager::new(- Arc::clone(&runtime),- SpillMetrics::new(&metrics, 0),- Arc::clone(&right_schema),- );-- let stream = SemiAntiSortMergeJoinStream::try_new(- Arc::clone(&left_schema),- vec![SortOptions::default()],- NullEquality::NullEqualsNothing,- outer,- inner,- on_outer.clone(),- on_inner.clone(),- Some(filter.clone()),- join_type,- 8192,- 0,- &metrics,- reservation,- peak_mem_used,- spill_manager,- Arc::clone(&runtime),- )?;-- let batches = collect_stream(stream).await?;- let total: usize = batches.iter().map(|b| b.num_rows()).sum();-- match join_type {- LeftSemi => {- assert_eq!(- total, 2,- "LeftSemi spill+boundary: both outer rows match filter, \- expected 2 rows, got {total}"- );- }- LeftAnti => {- assert_eq!(- total, 0,- "LeftAnti spill+boundary: both outer rows match filter, \- expected 0 rows, got {total}"- );- }- _ => unreachable!(),- }- }-- Ok(())-}diff --git a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rssimilarity index 93%rename from datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/stream.rsrename to datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 40e2022d41..2f7c9acb9d 100644--- a/datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -15,14 +15,15 @@// specific language governing permissions and limitations// under the License.-//! Sort-merge join stream specialized for semi/anti joins.+//! Sort-merge join stream specialized for semi/anti/mark joins.//!//! Instantiated by [`SortMergeJoinExec`](crate::joins::sort_merge_join::SortMergeJoinExec)-//! when the join type is `LeftSemi`, `LeftAnti`, `RightSemi`, or `RightAnti`.+//! when the join type is `LeftSemi`, `LeftAnti`, `RightSemi`, `RightAnti`,+//! `LeftMark`, or `RightMark`.//!//! # Motivation//!-//! The general-purpose `SortMergeJoinStream`+//! The general-purpose `MaterializingSortMergeJoinStream`//! handles semi/anti joins by materializing `(outer, inner)` row pairs,//! applying a filter, then using a "corrected filter mask" to deduplicate.//! Semi/anti joins only need a boolean per outer row (does a match exist?),@@ -36,7 +37,8 @@//!//! For `Left*` join types, left is outer and right is inner.//! For `Right*` join types, right is outer and left is inner.-//! The output schema always equals the outer side's schema.+//! The output schema always equals the outer side's schema (for semi/anti)+//! or the outer side's schema plus a boolean mark column (for mark joins).//!//! # Algorithm//!@@ -75,6 +77,7 @@//! On emit://! Semi → filter_record_batch(outer_batch, &matched)//! Anti → filter_record_batch(outer_batch, &NOT(matched))+//! Mark → outer_batch + matched as boolean column//! ```//!//! ## Batch boundaries@@ -245,14 +248,18 @@ enum PendingBoundary {Filtered { saved_keys: Vec<ArrayRef> },}-pub(crate) struct SemiAntiSortMergeJoinStream {- // Decomposed from JoinType to avoid matching on 4 variants in hot paths.- // true for semi (emit matched), false for anti (emit unmatched).- is_semi: bool,+/// Sort-Merge join stream for Semi/Anti/Mark joins.+///+/// Named "bitwise" because it tracks outer-row matches via a per-batch+/// boolean bitset (`BooleanBufferBuilder`) rather than materializing+/// `(outer, inner)` row pairs. Filter results are OR'd into the bitset+/// in `u64` chunks, and emitting applies the bitset directly.+pub(crate) struct BitwiseSortMergeJoinStream {+ join_type: JoinType,// Input streams — in the nested-loop model that sort-merge join// implements, "outer" is the driving loop and "inner" is probed for- // matches. The existing SortMergeJoinStream calls these "streamed"+ // matches. The existing MaterializingSortMergeJoinStream calls these "streamed"// and "buffered" respectively. For Left* joins, outer=left; for// Right* joins, outer=right. Output schema equals the outer side.outer: SendableRecordBatchStream,@@ -330,7 +337,7 @@ pub(crate) struct SemiAntiSortMergeJoinStream {batch_emitted: bool,}-impl SemiAntiSortMergeJoinStream {+impl BitwiseSortMergeJoinStream {#[expect(clippy::too_many_arguments)]pub fn try_new(schema: SchemaRef,@@ -346,21 +353,35 @@ impl SemiAntiSortMergeJoinStream {partition: usize,metrics: &ExecutionPlanMetricsSet,reservation: MemoryReservation,- peak_mem_used: Gauge,spill_manager: SpillManager,runtime_env: Arc<datafusion_execution::runtime_env::RuntimeEnv>,) -> Result<Self> {- let is_semi = matches!(join_type, JoinType::LeftSemi | JoinType::RightSemi);- let outer_is_left = matches!(join_type, JoinType::LeftSemi | JoinType::LeftAnti);+ debug_assert!(+ matches!(+ join_type,+ JoinType::LeftSemi+ | JoinType::RightSemi+ | JoinType::LeftAnti+ | JoinType::RightAnti+ | JoinType::LeftMark+ | JoinType::RightMark+ ),+ "BitwiseSortMergeJoinStream does not handle {join_type:?}"+ );+ let outer_is_left = matches!(+ join_type,+ JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark+ );let join_time = MetricBuilder::new(metrics).subset_time("join_time", partition);let input_batches =MetricBuilder::new(metrics).counter("input_batches", partition);let input_rows = MetricBuilder::new(metrics).counter("input_rows", partition);let baseline_metrics = BaselineMetrics::new(metrics, partition);+ let peak_mem_used = MetricBuilder::new(metrics).gauge("peak_mem_used", partition);Ok(Self {- is_semi,+ join_type,outer,inner,outer_batch: None,@@ -492,17 +513,38 @@ impl SemiAntiSortMergeJoinStream {// finish() converts the bit-packed builder directly to a// BooleanBuffer — no iteration or repacking needed.- let selection = BooleanArray::new(self.matched.finish(), None);-- let selection = if self.is_semi {- selection- } else {- not(&selection)?- };-- let filtered = filter_record_batch(batch, &selection)?;- if filtered.num_rows() > 0 {- self.coalescer.push_batch(filtered)?;+ let matched_buf = self.matched.finish();++ match self.join_type {+ JoinType::LeftMark | JoinType::RightMark => {+ // Mark joins emit ALL outer rows with a boolean match column appended.+ debug_assert_eq!(+ self.schema.fields().len(),+ batch.num_columns() + 1,+ "Mark join output schema should be outer schema + 1 mark column"+ );+ let mark_col = Arc::new(BooleanArray::new(matched_buf, None)) as ArrayRef;+ let mut columns = Vec::with_capacity(batch.num_columns() + 1);+ columns.extend_from_slice(batch.columns());+ columns.push(mark_col);+ let output = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;+ self.coalescer.push_batch(output)?;+ }+ JoinType::LeftSemi | JoinType::RightSemi => {+ let selection = BooleanArray::new(matched_buf, None);+ let filtered = filter_record_batch(batch, &selection)?;+ if filtered.num_rows() > 0 {+ self.coalescer.push_batch(filtered)?;+ }+ }+ JoinType::LeftAnti | JoinType::RightAnti => {+ let selection = not(&BooleanArray::new(matched_buf, None))?;+ let filtered = filter_record_batch(batch, &selection)?;+ if filtered.num_rows() > 0 {+ self.coalescer.push_batch(filtered)?;+ }+ }+ _ => unreachable!(),}Ok(())}@@ -1184,7 +1226,7 @@ fn keys_match(/// Evaluate the join filter for one inner row against a slice of outer rows.///-/// Free function (not a method on SemiAntiSortMergeJoinStream) so that Rust+/// Free function (not a method on BitwiseSortMergeJoinStream) so that Rust/// can split the struct borrow in process_key_match_with_filter: the caller/// holds &mut self.matched and &self.inner_key_buffer simultaneously, which/// is impossible if this borrows all of &self.@@ -1257,7 +1299,7 @@ fn evaluate_filter_for_inner_row(}}-impl Stream for SemiAntiSortMergeJoinStream {+impl Stream for BitwiseSortMergeJoinStream {type Item = Result<RecordBatch>;fn poll_next(@@ -1269,7 +1311,7 @@ impl Stream for SemiAntiSortMergeJoinStream {}}-impl RecordBatchStream for SemiAntiSortMergeJoinStream {+impl RecordBatchStream for BitwiseSortMergeJoinStream {fn schema(&self) -> SchemaRef {Arc::clone(&self.schema)}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex d5d1df4baf..e2c169e558 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -23,17 +23,17 @@ use std::any::Any;use std::fmt::Formatter;use std::sync::Arc;+use super::bitwise_stream::BitwiseSortMergeJoinStream;+use super::materializing_stream::MaterializingSortMergeJoinStream;+use super::metrics::SortMergeJoinMetrics;use crate::execution_plan::{EmissionType, boundedness_from_children};use crate::expressions::PhysicalSortExpr;-use crate::joins::semi_anti_sort_merge_join::stream::SemiAntiSortMergeJoinStream;-use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;-use crate::joins::sort_merge_join::stream::SortMergeJoinStream;use crate::joins::utils::{JoinFilter, JoinOn, JoinOnRef, build_join_schema, check_join_is_valid,estimate_join_statistics, reorder_output_after_swap,symmetric_join_output_partitioning,};-use crate::metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet, SpillMetrics};+use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet, SpillMetrics};use crate::projection::{ProjectionExec, join_allows_pushdown, join_table_borders, new_join_children,physical_to_column_exprs, update_join_on,@@ -337,6 +337,8 @@ impl SortMergeJoinExec {| JoinType::RightSemi| JoinType::LeftAnti| JoinType::RightAnti+ | JoinType::LeftMark+ | JoinType::RightMark) {Ok(Arc::new(new_join))} else {@@ -527,6 +529,12 @@ impl ExecutionPlan for SortMergeJoinExec {let batch_size = context.session_config().batch_size();let reservation = MemoryConsumer::new(format!("SMJStream[{partition}]")).register(context.memory_pool());+ let spill_manager = SpillManager::new(+ context.runtime_env(),+ SpillMetrics::new(&self.metrics, partition),+ buffered.schema(),+ )+ .with_compression_type(context.session_config().spill_compression());if matches!(self.join_type,@@ -534,17 +542,10 @@ impl ExecutionPlan for SortMergeJoinExec {| JoinType::LeftAnti| JoinType::RightSemi| JoinType::RightAnti+ | JoinType::LeftMark+ | JoinType::RightMark) {- let peak_mem_used =- MetricBuilder::new(&self.metrics).gauge("peak_mem_used", partition);- let spill_manager = SpillManager::new(- context.runtime_env(),- SpillMetrics::new(&self.metrics, partition),- buffered.schema(),- )- .with_compression_type(context.session_config().spill_compression());-- Ok(Box::pin(SemiAntiSortMergeJoinStream::try_new(+ Ok(Box::pin(BitwiseSortMergeJoinStream::try_new(Arc::clone(&self.schema),self.sort_options.clone(),self.null_equality,@@ -558,13 +559,11 @@ impl ExecutionPlan for SortMergeJoinExec {partition,&self.metrics,reservation,- peak_mem_used,spill_manager,context.runtime_env(),)?))} else {- Ok(Box::pin(SortMergeJoinStream::try_new(- context.session_config().spill_compression(),+ Ok(Box::pin(MaterializingSortMergeJoinStream::try_new(Arc::clone(&self.schema),self.sort_options.clone(),self.null_equality,@@ -577,6 +576,7 @@ impl ExecutionPlan for SortMergeJoinExec {batch_size,SortMergeJoinMetrics::new(partition, &self.metrics),reservation,+ spill_manager,context.runtime_env(),)?))}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs b/datafusion/physical-plan/src/joins/sort_merge_join/filter.rsindex d598442b65..3a5f0d0f05 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs@@ -19,7 +19,7 @@//!//! This module encapsulates the complexity of join filter evaluation, including://! - Immediate filtering for INNER joins-//! - Deferred filtering for outer/semi/anti/mark joins+//! - Deferred filtering for outer joins//! - Metadata tracking for grouping output rows by input row//! - Correcting filter masks to handle multiple matches per input row@@ -27,9 +27,10 @@ use std::sync::Arc;use arrow::array::{Array, ArrayBuilder, ArrayRef, BooleanArray, BooleanBuilder, RecordBatch,- UInt64Array, UInt64Builder,+ RecordBatchOptions, UInt64Array, UInt64Builder, new_null_array,};-use arrow::compute::{self, concat_batches, filter_record_batch};+use arrow::compute::kernels::zip::zip;+use arrow::compute::{self, filter_record_batch};use arrow::datatypes::SchemaRef;use datafusion_common::{JoinSide, JoinType, Result};@@ -38,9 +39,9 @@ use crate::joins::utils::JoinFilter;/// Metadata for tracking filter results during deferred filtering////// When a join filter is present and we need to ensure each input row produces-/// at least one output (outer joins) or exactly one output (semi joins), we can't-/// filter immediately. Instead, we accumulate all joined rows with metadata,-/// then post-process to determine which rows to output.+/// at least one output (outer joins), we can't filter immediately. Instead,+/// we accumulate all joined rows with metadata, then post-process to determine+/// which rows to output.#[derive(Debug)]pub struct FilterMetadata {/// Did each output row pass the join filter?@@ -96,23 +97,10 @@ impl FilterMetadata {"row_indices and filter_mask must have same length");- for i in 0..row_indices.len() {- if filter_mask.is_null(i) {- self.filter_mask.append_null();- } else if filter_mask.value(i) {- self.filter_mask.append_value(true);- } else {- self.filter_mask.append_value(false);- }-- if row_indices.is_null(i) {- self.row_indices.append_null();- } else {- self.row_indices.append_value(row_indices.value(i));- }-- self.batch_ids.push(batch_id);- }+ self.filter_mask.extend(filter_mask);+ self.row_indices.extend(row_indices);+ self.batch_ids+ .resize(self.batch_ids.len() + row_indices.len(), batch_id);}/// Verify that metadata arrays are aligned (same length)@@ -149,24 +137,12 @@ impl Default for FilterMetadata {/// Deferred filtering is required when:/// - A filter exists AND/// - The join type requires ensuring each input row produces at least one output-/// (or exactly one for semi joins)pub fn needs_deferred_filtering(filter: &Option<JoinFilter>,join_type: JoinType,) -> bool {filter.is_some()- && matches!(- join_type,- JoinType::Left- | JoinType::LeftSemi- | JoinType::LeftMark- | JoinType::Right- | JoinType::RightSemi- | JoinType::RightMark- | JoinType::LeftAnti- | JoinType::RightAnti- | JoinType::Full- )+ && matches!(join_type, JoinType::Left | JoinType::Right | JoinType::Full)}/// Gets the arrays which join filters are applied on@@ -253,9 +229,6 @@ fn last_index_for_row(/// This function groups them by input row and applies join-type-specific logic:////// - **Outer joins**: Keep first matching row, convert rest to nulls, add null-joined for unmatched-/// - **Semi joins**: Keep first matching row, discard rest-/// - **Anti joins**: Keep row only if NO matches passed filter-/// - **Mark joins**: Like semi but first match only////// # Arguments/// * `join_type` - The type of join being performed@@ -267,7 +240,7 @@ fn last_index_for_row(/// # Returns/// Corrected mask indicating which rows to include in final output:/// - `true`: Include this row-/// - `false`: Convert to null-joined row (outer joins) or include as unmatched (anti joins)+/// - `false`: Convert to null-joined row (outer joins)/// - `null`: Discard this rowpub fn get_corrected_filter_mask(join_type: JoinType,@@ -282,314 +255,134 @@ pub fn get_corrected_filter_mask(let mut seen_true = false;match join_type {- JoinType::Left | JoinType::Right => {- // For outer joins: Keep first matching row per input row,- // convert rest to nulls, add null-joined rows for unmatched- for i in 0..row_indices_length {- let last_index =- last_index_for_row(i, row_indices, batch_ids, row_indices_length);- if filter_mask.value(i) {- seen_true = true;- corrected_mask.append_value(true);- } else if seen_true || !filter_mask.value(i) && !last_index {- corrected_mask.append_null(); // to be ignored and not set to output- } else {- corrected_mask.append_value(false); // to be converted to null joined row- }-- if last_index {- seen_true = false;- }- }-- // Generate null joined rows for records which have no matching join key- corrected_mask.append_n(expected_size - corrected_mask.len(), false);- Some(corrected_mask.finish())- }- JoinType::LeftMark | JoinType::RightMark => {- // For mark joins: Like outer but only keep first match, mark with boolean- for i in 0..row_indices_length {- let last_index =- last_index_for_row(i, row_indices, batch_ids, row_indices_length);- if filter_mask.value(i) && !seen_true {- seen_true = true;- corrected_mask.append_value(true);- } else if seen_true || !filter_mask.value(i) && !last_index {- corrected_mask.append_null(); // to be ignored and not set to output- } else {- corrected_mask.append_value(false); // to be converted to null joined row- }-- if last_index {- seen_true = false;- }- }-- // Generate null joined rows for records which have no matching join key- corrected_mask.append_n(expected_size - corrected_mask.len(), false);- Some(corrected_mask.finish())- }- JoinType::LeftSemi | JoinType::RightSemi => {- // For semi joins: Keep only first matching row per input row, discard rest- for i in 0..row_indices_length {- let last_index =- last_index_for_row(i, row_indices, batch_ids, row_indices_length);- if filter_mask.value(i) && !seen_true {- seen_true = true;- corrected_mask.append_value(true);- } else {- corrected_mask.append_null(); // to be ignored and not set to output- }-- if last_index {- seen_true = false;- }- }-- Some(corrected_mask.finish())- }- JoinType::LeftAnti | JoinType::RightAnti => {- // For anti joins: Keep row only if NO matches passed the filter+ JoinType::Left | JoinType::Right | JoinType::Full => {+ // For each input row group: keep first filter-passing row,+ // discard (null) remaining matches, null-join if none passed.+ // Null metadata entries are already-null-joined rows that+ // flow through unchanged to preserve output ordering.for i in 0..row_indices_length {let last_index =last_index_for_row(i, row_indices, batch_ids, row_indices_length);-- if filter_mask.value(i) {- seen_true = true;- }-- if last_index {- if !seen_true {- corrected_mask.append_value(true);- } else {- corrected_mask.append_null();- }-- seen_true = false;- } else {- corrected_mask.append_null();- }- }- // Generate null joined rows for records which have no matching join key,- // for LeftAnti non-matched considered as true- corrected_mask.append_n(expected_size - corrected_mask.len(), true);- Some(corrected_mask.finish())- }- JoinType::Full => {- // For full joins: Similar to outer but handle both sides- for i in 0..row_indices_length {- let last_index =- last_index_for_row(i, row_indices, batch_ids, row_indices_length);-if filter_mask.is_null(i) {- // null joinedcorrected_mask.append_value(true);} else if filter_mask.value(i) {seen_true = true;corrected_mask.append_value(true);} else if seen_true || !filter_mask.value(i) && !last_index {- corrected_mask.append_null(); // to be ignored and not set to output+ corrected_mask.append_null();} else {- corrected_mask.append_value(false); // to be converted to null joined row+ corrected_mask.append_value(false);}if last_index {seen_true = false;}}- // Generate null joined rows for records which have no matching join key+corrected_mask.append_n(expected_size - corrected_mask.len(), false);Some(corrected_mask.finish())}- JoinType::Inner => {- // Inner joins don't need deferred filtering- None+ JoinType::LeftMark+ | JoinType::RightMark+ | JoinType::LeftSemi+ | JoinType::RightSemi+ | JoinType::LeftAnti+ | JoinType::RightAnti => {+ unreachable!("Semi/anti/mark joins are handled by BitwiseSortMergeJoinStream")}+ JoinType::Inner => None,}}/// Applies corrected filter mask to record batch based on join type///-/// Different join types require different handling of filtered results:-/// - Outer joins: Add null-joined rows for false mask values-/// - Semi/Anti joins: May need projection to remove right columns-/// - Full joins: Add null-joined rows for both sides+/// The corrected mask has three possible values per row:+/// - `true`: Keep the row as-is (matched and passed filter)+/// - `false`: Convert to null-joined row (all filter matches failed for this input row)+/// - `null`: Discard the row entirely (duplicate match for an already-output input row)+///+/// This function preserves input row ordering by processing each row in place+/// rather than separating matched/unmatched rows.pub fn filter_record_batch_by_join_type(record_batch: &RecordBatch,corrected_mask: &BooleanArray,join_type: JoinType,schema: &SchemaRef,- streamed_schema: &SchemaRef,buffered_schema: &SchemaRef,) -> Result<RecordBatch> {- let filtered_record_batch = filter_record_batch(record_batch, corrected_mask)?;-match join_type {- JoinType::Left | JoinType::LeftMark => {- // For left joins, add null-joined rows where mask is false- let null_mask = compute::not(corrected_mask)?;- let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;+ JoinType::Left | JoinType::Right | JoinType::Full => {+ if record_batch.num_rows() == 0 {+ return Ok(record_batch.clone());+ }++ // Discard null-masked rows (keep true + false only)+ let keep_mask = compute::is_not_null(corrected_mask)?;+ let kept_batch = filter_record_batch(record_batch, &keep_mask)?;- if null_joined_batch.num_rows() == 0 {- return Ok(filtered_record_batch);+ if kept_batch.num_rows() == 0 {+ return Ok(kept_batch);}- // Create null columns for right side- let null_joined_streamed_batch = create_null_joined_batch(- &null_joined_batch,- buffered_schema,- JoinSide::Left,- join_type,- schema,- )?;-- Ok(concat_batches(- schema,- &[filtered_record_batch, null_joined_streamed_batch],- )?)- }- JoinType::LeftSemi- | JoinType::LeftAnti- | JoinType::RightSemi- | JoinType::RightAnti => {- // For semi/anti joins, project to only include the outer side columns- // Both Left and Right semi/anti use streamed_schema.len() because:- // - For Left: columns are [left, right], so we take first streamed_schema.len()- // - For Right: columns are [right, left], and streamed side is right, so we take first streamed_schema.len()- let output_column_indices: Vec<usize> =- (0..streamed_schema.fields().len()).collect();- Ok(filtered_record_batch.project(&output_column_indices)?)- }- JoinType::Right | JoinType::RightMark => {- // For right joins, add null-joined rows where mask is false- let null_mask = compute::not(corrected_mask)?;- let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;+ let kept_corrected = compute::filter(corrected_mask, &keep_mask)?;+ let kept_corrected = kept_corrected+ .as_any()+ .downcast_ref::<BooleanArray>()+ .unwrap();- if null_joined_batch.num_rows() == 0 {- return Ok(filtered_record_batch);+ // All rows passed the filter — no null-joining needed+ if kept_corrected.true_count() == kept_corrected.len() {+ return Ok(kept_batch);}- // Create null columns for left side (buffered side for RIGHT join)- let null_joined_buffered_batch = create_null_joined_batch(- &null_joined_batch,- buffered_schema, // Pass buffered (left) schema to create nulls for it- JoinSide::Right,- join_type,- schema,- )?;-- Ok(concat_batches(- schema,- &[filtered_record_batch, null_joined_buffered_batch],- )?)- }- JoinType::Full => {- // For full joins, add null-joined rows for both sides- let joined_filter_not_matched_mask = compute::not(corrected_mask)?;- let joined_filter_not_matched_batch =- filter_record_batch(record_batch, &joined_filter_not_matched_mask)?;-- if joined_filter_not_matched_batch.num_rows() == 0 {- return Ok(filtered_record_batch);+ // For false entries: replace the non-preserved side with nulls.+ // This preserves row ordering unlike filter+concat.+ let (null_side_start, null_side_len) = match join_type {+ JoinType::Left => {+ // Left join: null out right (buffered) columns+ let left_cols =+ schema.fields().len() - buffered_schema.fields().len();+ (left_cols, buffered_schema.fields().len())+ }+ JoinType::Right => {+ // Right join: null out left (buffered) columns+ (0, buffered_schema.fields().len())+ }+ JoinType::Full => {+ // Full join: null out buffered columns for streamed rows+ // that matched but failed the filter. Unmatched buffered+ // rows are null-joined on the streamed side separately+ // when the buffered batch is drained.+ let left_cols =+ schema.fields().len() - buffered_schema.fields().len();+ (left_cols, buffered_schema.fields().len())+ }+ _ => unreachable!(),+ };++ let num_rows = kept_batch.num_rows();+ let mut columns: Vec<ArrayRef> = kept_batch.columns().to_vec();++ for col in columns.iter_mut().skip(null_side_start).take(null_side_len) {+ let null_array = new_null_array(col.data_type(), num_rows);+ *col = zip(kept_corrected, &*col, &null_array)?;}- // Create null-joined batches for both sides- let left_null_joined_batch = create_null_joined_batch(- &joined_filter_not_matched_batch,- buffered_schema,- JoinSide::Left,- join_type,- schema,- )?;-- Ok(concat_batches(- schema,- &[filtered_record_batch, left_null_joined_batch],+ let options = RecordBatchOptions::new().with_row_count(Some(num_rows));+ Ok(RecordBatch::try_new_with_options(+ Arc::clone(schema),+ columns,+ &options,)?)}- JoinType::Inner => Ok(filtered_record_batch),+ JoinType::LeftSemi+ | JoinType::LeftAnti+ | JoinType::RightSemi+ | JoinType::RightAnti+ | JoinType::LeftMark+ | JoinType::RightMark => unreachable!(+ "Semi/anti/mark joins are handled by SemiAntiMarkSortMergeJoinStream"+ ),+ JoinType::Inner => Ok(filter_record_batch(record_batch, corrected_mask)?),}}--/// Creates a batch with null columns for the non-joined side-///-/// Note: The input `batch` is assumed to be a fully-joined batch that already contains-/// columns from both sides. We need to extract the data side columns and replace the-/// null side columns with actual nulls.-fn create_null_joined_batch(- batch: &RecordBatch,- null_schema: &SchemaRef,- join_side: JoinSide,- join_type: JoinType,- output_schema: &SchemaRef,-) -> Result<RecordBatch> {- let num_rows = batch.num_rows();-- // The input batch is a fully-joined batch [left_cols..., right_cols...]- // We need to extract the appropriate side and replace the other with nulls (or mark column)- let columns = match (join_side, join_type) {- (JoinSide::Left, JoinType::LeftMark) => {- // For LEFT mark: output is [left_cols..., mark_col]- // Batch is [left_cols..., right_cols...], extract left from beginning- // Number of left columns = output columns - 1 (mark column)- let left_col_count = output_schema.fields().len() - 1;- let mut result: Vec<ArrayRef> = batch.columns()[..left_col_count].to_vec();- result.push(Arc::new(BooleanArray::from(vec![false; num_rows])) as ArrayRef);- result- }- (JoinSide::Right, JoinType::RightMark) => {- // For RIGHT mark: output is [right_cols..., mark_col]- // For RIGHT joins, batch is [right_cols..., left_cols...] (right comes first!)- // Extract right columns from the beginning- let right_col_count = output_schema.fields().len() - 1; // -1 for mark column- let mut result: Vec<ArrayRef> = batch.columns()[..right_col_count].to_vec();- result.push(Arc::new(BooleanArray::from(vec![false; num_rows])) as ArrayRef);- result- }- (JoinSide::Left, _) => {- // For LEFT join: output is [left_cols..., right_cols...]- // Extract left columns, then add null right columns- let null_columns: Vec<ArrayRef> = null_schema- .fields()- .iter()- .map(|field| arrow::array::new_null_array(field.data_type(), num_rows))- .collect();- let left_col_count = output_schema.fields().len() - null_columns.len();- let mut result: Vec<ArrayRef> = batch.columns()[..left_col_count].to_vec();- result.extend(null_columns);- result- }- (JoinSide::Right, _) => {- // For RIGHT join: batch is [left_cols..., right_cols...] (same as schema)- // We want: [null_left..., actual_right...]- // Extract left columns from beginning, replace with nulls, keep right columns- let null_columns: Vec<ArrayRef> = null_schema- .fields()- .iter()- .map(|field| arrow::array::new_null_array(field.data_type(), num_rows))- .collect();- let left_col_count = null_columns.len();- let mut result = null_columns;- // Extract right columns starting after left columns- result.extend_from_slice(&batch.columns()[left_col_count..]);- result- }- (JoinSide::None, _) => {- // This should not happen in normal join operations- unreachable!(- "JoinSide::None should not be used in null-joined batch creation"- )- }- };-- // Create the batch - don't validate nullability since outer joins can have- // null values in columns that were originally non-nullable- use arrow::array::RecordBatchOptions;- let mut options = RecordBatchOptions::new();- options = options.with_row_count(Some(num_rows));- Ok(RecordBatch::try_new_with_options(- Arc::clone(output_schema),- columns,- &options,- )?)-}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rssimilarity index 80%rename from datafusion/physical-plan/src/joins/sort_merge_join/stream.rsrename to datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rsindex 4dcbe1f647..c387e05390 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs@@ -45,14 +45,14 @@ use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream};use arrow::array::{types::UInt64Type, *};use arrow::compute::{- self, BatchCoalescer, SortOptions, concat_batches, filter_record_batch, is_not_null,+ self, BatchCoalescer, SortOptions, concat_batches, filter_record_batch, interleave,take, take_arrays,};use arrow::datatypes::{DataType, SchemaRef, TimeUnit};use arrow::ipc::reader::StreamReader;-use datafusion_common::config::SpillCompression;+use datafusion_common::cast::as_uint64_array;use datafusion_common::{- HashSet, JoinType, NullEquality, Result, exec_err, internal_err, not_impl_err,+ JoinType, NullEquality, Result, exec_err, internal_err, not_impl_err,};use datafusion_execution::disk_manager::RefCountedTempFile;use datafusion_execution::memory_pool::MemoryReservation;@@ -132,10 +132,6 @@ pub(super) struct StreamedBatch {pub num_output_rows: usize,/// Index of currently scanned batch from buffered datapub buffered_batch_idx: Option<usize>,- /// Indices that found a match for the given join filter- /// Used for semi joins to keep track the streaming index which got a join filter match- /// and already emitted to the output.- pub join_filter_matched_idxs: HashSet<u64>,}impl StreamedBatch {@@ -148,7 +144,6 @@ impl StreamedBatch {output_indices: vec![],num_output_rows: 0,buffered_batch_idx: None,- join_filter_matched_idxs: HashSet::new(),}}@@ -160,7 +155,6 @@ impl StreamedBatch {output_indices: vec![],num_output_rows: 0,buffered_batch_idx: None,- join_filter_matched_idxs: HashSet::new(),}}@@ -282,9 +276,12 @@ pub(super) enum BufferedBatchState {Spilled(RefCountedTempFile),}-/// Sort-Merge join stream that consumes streamed and buffered data streams-/// and produces joined output stream.-pub(super) struct SortMergeJoinStream {+/// Sort-Merge join stream for Inner/Left/Right/Full joins.+///+/// Named "materializing" because it builds explicit `(streamed, buffered)` row+/// pairs in [`JoinedRecordBatches`] to produce output columns from both sides+/// of the join.+pub(super) struct MaterializingSortMergeJoinStream {// ========================================================================// PROPERTIES:// These fields are initialized at the start and remain constant throughout@@ -428,16 +425,19 @@ impl JoinedRecordBatches {}}- /// Pushes a batch with null metadata (Full join null-joined rows only)+ /// Pushes a batch with null metadata (rows that need no filter correction)///- /// These buffered rows had NO matching streamed rows. Since we can't group- /// by input row (no input row exists), we use null metadata as a sentinel.+ /// Used for: (1) Full join buffered rows with no streamed match, and+ /// (2) outer join streamed rows with no buffered match. These rows are+ /// already in final form but must flow through the deferred filtering+ /// pipeline to preserve output ordering. Null metadata causes+ /// get_corrected_filter_mask() to pass them through unchanged.////// Maintains invariant: N rows → N metadata entries (nulls)fn push_batch_with_null_metadata(&mut self, batch: RecordBatch, join_type: JoinType) {debug_assert!(- join_type == JoinType::Full,- "push_batch_with_null_metadata should only be called for Full joins"+ matches!(join_type, JoinType::Left | JoinType::Right | JoinType::Full),+ "push_batch_with_null_metadata should only be called for deferred-filtered joins");let num_rows = batch.num_rows();@@ -450,7 +450,7 @@ impl JoinedRecordBatches {.expect("Failed to push batch to BatchCoalescer");}- /// Pushes a batch with filter metadata (filtered outer/semi/anti/mark joins)+ /// Pushes a batch with filter metadata (filtered outer joins)////// Deferred filtering: An input row may join with multiple buffered rows, but we/// don't know yet if all matches failed the filter. We track metadata so@@ -468,19 +468,8 @@ impl JoinedRecordBatches {join_type: JoinType,) {debug_assert!(- matches!(- join_type,- JoinType::Left- | JoinType::LeftSemi- | JoinType::LeftMark- | JoinType::Right- | JoinType::RightSemi- | JoinType::RightMark- | JoinType::LeftAnti- | JoinType::RightAnti- | JoinType::Full- ),- "push_batch_with_filter_metadata should only be called for outer/semi/anti/mark joins that need deferred filtering"+ matches!(join_type, JoinType::Left | JoinType::Right | JoinType::Full),+ "push_batch_with_filter_metadata should only be called for outer joins that need deferred filtering");debug_assert_eq!(@@ -506,7 +495,7 @@ impl JoinedRecordBatches {/// No deferred filtering needed. Either every join match is output (Inner),/// or null-joined rows are handled separately. No need to track which input/// row produced which output row.- fn push_batch_without_metadata(&mut self, batch: RecordBatch, _join_type: JoinType) {+ fn push_batch_without_metadata(&mut self, batch: RecordBatch) {self.joined_batches.push_batch(batch).expect("Failed to push batch to BatchCoalescer");@@ -519,13 +508,13 @@ impl JoinedRecordBatches {self.debug_assert_empty_consistency();}}-impl RecordBatchStream for SortMergeJoinStream {+impl RecordBatchStream for MaterializingSortMergeJoinStream {fn schema(&self) -> SchemaRef {Arc::clone(&self.schema)}}-impl Stream for SortMergeJoinStream {+impl Stream for MaterializingSortMergeJoinStream {type Item = Result<RecordBatch>;fn poll_next(@@ -547,15 +536,35 @@ impl Stream for SortMergeJoinStream {match self.current_ordering {Ordering::Less | Ordering::Equal => {if !streamed_exhausted {+ // Batch deferred filtering: process_filtered_batches()+ // only when >= batch_size rows have accumulated.+ // Without this gate, unique keys cause per-row pipeline+ // execution (concat + correct_mask + filter_by_type),+ // which dominates runtime.+ //+ // Accumulated rows are bounded to ~2*batch_size:+ // one batch_size worth from freeze_dequeuing_buffered()+ // (when an input batch is fully consumed), plus up to+ // batch_size pairs accumulating toward the next freeze.+ // This does not reintroduce the unbounded buffering+ // fixed by PR #20482. Exhausted state flushes remainder.if needs_deferred_filtering(&self.filter,self.join_type,) {- match self.process_filtered_batches()? {- Poll::Ready(Some(batch)) => {- return Poll::Ready(Some(Ok(batch)));+ let accumulated = self.num_unfrozen_pairs()+ + self+ .joined_record_batches+ .filter_metadata+ .filter_mask+ .len();+ if accumulated >= self.batch_size {+ match self.process_filtered_batches()? {+ Poll::Ready(Some(batch)) => {+ return Poll::Ready(Some(Ok(batch)));+ }+ Poll::Ready(None) | Poll::Pending => {}}- Poll::Ready(None) | Poll::Pending => {}}}@@ -734,11 +743,9 @@ impl Stream for SortMergeJoinStream {}}-impl SortMergeJoinStream {+impl MaterializingSortMergeJoinStream {#[expect(clippy::too_many_arguments)]pub fn try_new(- // Configured via `datafusion.execution.spill_compression`.- spill_compression: SpillCompression,schema: SchemaRef,sort_options: Vec<SortOptions>,null_equality: NullEquality,@@ -751,16 +758,19 @@ impl SortMergeJoinStream {batch_size: usize,join_metrics: SortMergeJoinMetrics,reservation: MemoryReservation,+ spill_manager: SpillManager,runtime_env: Arc<RuntimeEnv>,) -> Result<Self> {let streamed_schema = streamed.schema();let buffered_schema = buffered.schema();- let spill_manager = SpillManager::new(- Arc::clone(&runtime_env),- join_metrics.spill_metrics().clone(),- Arc::clone(&buffered_schema),- )- .with_compression_type(spill_compression);+ debug_assert!(+ matches!(+ join_type,+ JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full+ ),+ "MaterializingSortMergeJoinStream does not handle {join_type:?}; \+ semi/anti/mark joins use BitwiseSortMergeJoinStream"+ );Ok(Self {state: SortMergeJoinState::Init,sort_options,@@ -1053,68 +1063,20 @@ impl SortMergeJoinStream {let mut join_streamed = false;// Whether to join buffered rowslet mut join_buffered = false;- // For Mark join we store a dummy id to indicate the row has a match- let mut mark_row_as_match = false;// determine whether we need to join streamed/buffered rowsmatch self.current_ordering {Ordering::Less => {if matches!(self.join_type,- JoinType::Left- | JoinType::Right- | JoinType::Full- | JoinType::LeftAnti- | JoinType::RightAnti- | JoinType::LeftMark- | JoinType::RightMark+ JoinType::Left | JoinType::Right | JoinType::Full) {join_streamed = !self.streamed_joined;}}Ordering::Equal => {- if matches!(- self.join_type,- JoinType::LeftSemi- | JoinType::LeftMark- | JoinType::RightSemi- | JoinType::RightMark- ) {- mark_row_as_match = matches!(- self.join_type,- JoinType::LeftMark | JoinType::RightMark- );- // if the join filter is specified then its needed to output the streamed index- // only if it has not been emitted before- // the `join_filter_matched_idxs` keeps track on if streamed index has a successful- // filter match and prevents the same index to go into output more than once- if self.filter.is_some() {- join_streamed = !self- .streamed_batch- .join_filter_matched_idxs- .contains(&(self.streamed_batch.idx as u64))- && !self.streamed_joined;- // if the join filter specified there can be references to buffered columns- // so buffered columns are needed to access them- join_buffered = join_streamed;- } else {- join_streamed = !self.streamed_joined;- }- }- if matches!(- self.join_type,- JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full- ) {- join_streamed = true;- join_buffered = true;- };-- if matches!(self.join_type, JoinType::LeftAnti | JoinType::RightAnti)- && self.filter.is_some()- {- join_streamed = !self.streamed_joined;- join_buffered = join_streamed;- }+ join_streamed = true;+ join_buffered = true;}Ordering::Greater => {if self.join_type == JoinType::Full {@@ -1162,12 +1124,9 @@ impl SortMergeJoinStream {} else {Some(self.buffered_data.scanning_batch_idx)};- // For Mark join we store a dummy id to indicate the row has a match- let scanning_idx = mark_row_as_match.then_some(0);-self.streamed_batch.append_output_pair(scanning_batch_idx,- scanning_idx,+ None,self.batch_size,);self.buffered_data.scanning_finish();@@ -1267,209 +1226,294 @@ impl SortMergeJoinStream {// Produces and stages record batch for all output indices found// for current streamed batch and clears staged output indices.+ //+ // Null-joined chunks (no buffered match) are pushed immediately.+ // Matched chunks are collected and processed together in+ // freeze_streamed_matched() to amortize filter evaluation overhead.fn freeze_streamed(&mut self) -> Result<()> {+ let mut matched_chunks: Vec<(usize, UInt64Array, UInt64Array)> = Vec::new();+ let mut total_matched_rows: usize = 0;+for chunk in self.streamed_batch.output_indices.iter_mut() {- // The row indices of joined streamed batchlet left_indices = chunk.streamed_indices.finish();-if left_indices.is_empty() {continue;}+ let right_indices: UInt64Array = chunk.buffered_indices.finish();- let mut left_columns = if let Some(range) = is_contiguous_range(&left_indices)- {- // When indices form a contiguous range (common for the streamed- // side which advances sequentially), use zero-copy slice instead- // of the O(n) take kernel.- self.streamed_batch- .batch- .slice(range.start, range.len())- .columns()- .to_vec()- } else {- take_arrays(self.streamed_batch.batch.columns(), &left_indices, None)?- };+ if chunk.buffered_batch_idx.is_none() {+ let left_columns =+ materialize_left_columns(&self.streamed_batch.batch, &left_indices)?;+ let right_columns =+ create_unmatched_columns(&self.buffered_schema, left_indices.len());- // The row indices of joined buffered batch- let right_indices: UInt64Array = chunk.buffered_indices.finish();- let mut right_columns =- if matches!(self.join_type, JoinType::LeftMark | JoinType::RightMark) {- vec![Arc::new(is_not_null(&right_indices)?) as ArrayRef]- } else if matches!(- self.join_type,- JoinType::LeftSemi- | JoinType::LeftAnti- | JoinType::RightAnti- | JoinType::RightSemi- ) {- vec![]- } else if let Some(buffered_idx) = chunk.buffered_batch_idx {- fetch_right_columns_by_idxs(- &self.buffered_data,- buffered_idx,- &right_indices,- )?+ let columns = if self.join_type != JoinType::Right {+ [left_columns, right_columns].concat()} else {- // If buffered batch none, meaning it is null joined batch.- // We need to create null arrays for buffered columns to join with streamed rows.- create_unmatched_columns(- self.join_type,- &self.buffered_schema,- right_indices.len(),- )+ [right_columns, left_columns].concat()};+ let batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;- // Prepare the columns we apply join filter on later.- // Only for joined rows between streamed and buffered.- let filter_columns = if let Some(buffered_batch_idx) =- chunk.buffered_batch_idx- {- if self.join_type != JoinType::Right {- if matches!(- self.join_type,- JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark- ) {- let right_cols = fetch_right_columns_by_idxs(- &self.buffered_data,- buffered_batch_idx,- &right_indices,- )?;-- get_filter_columns(&self.filter, &left_columns, &right_cols)- } else if matches!(- self.join_type,- JoinType::RightAnti | JoinType::RightSemi | JoinType::RightMark- ) {- let right_cols = fetch_right_columns_by_idxs(- &self.buffered_data,- buffered_batch_idx,- &right_indices,- )?;-- get_filter_columns(&self.filter, &right_cols, &left_columns)- } else {- get_filter_columns(&self.filter, &left_columns, &right_columns)- }+ // Null-joined rows (no buffered match) need no filter correction,+ // but must flow through the same pipeline as matched rows to+ // preserve output ordering. Use null metadata as a sentinel so+ // get_corrected_filter_mask() passes them through unchanged.+ if needs_deferred_filtering(&self.filter, self.join_type) {+ self.joined_record_batches+ .push_batch_with_null_metadata(batch, self.join_type);} else {- get_filter_columns(&self.filter, &right_columns, &left_columns)+ self.joined_record_batches+ .push_batch_without_metadata(batch);}- } else {- // This chunk is totally for null joined rows (outer join), we don't need to apply join filter.- // Any join filter applied only on either streamed or buffered side will be pushed already.- vec![]- };+ continue;+ }- let columns = if self.join_type != JoinType::Right {- left_columns.extend(right_columns);- left_columns- } else {- right_columns.extend(left_columns);- right_columns- };+ total_matched_rows += left_indices.len();+ matched_chunks.push((+ chunk.buffered_batch_idx.unwrap(),+ left_indices,+ right_indices,+ ));+ }++ if !matched_chunks.is_empty() {+ self.freeze_streamed_matched(&matched_chunks, total_matched_rows)?;+ }++ self.streamed_batch.output_indices.clear();+ self.streamed_batch.num_output_rows = 0;+ Ok(())+ }++ /// Materializes columns, evaluates the join filter, and pushes output+ /// for all matched chunks in a single batch. This avoids per-chunk+ /// RecordBatch construction and filter evaluation, which dominates+ /// cost when keys are near-unique (1 row per chunk).+ fn freeze_streamed_matched(+ &mut self,+ matched_chunks: &[(usize, UInt64Array, UInt64Array)],+ total_matched_rows: usize,+ ) -> Result<()> {+ debug_assert!(+ !matched_chunks.is_empty(),+ "caller guards this with an is_empty check before calling"+ );+ debug_assert!(+ matched_chunks.iter().all(|(idx, left, right)| {+ left.len() == right.len() && *idx < self.buffered_data.batches.len()+ }),+ "left/right indices are built in pairs from the same streamed×buffered cross, \+ and batch_idx comes from iterating buffered_data.batches"+ );+ debug_assert_eq!(+ matched_chunks+ .iter()+ .map(|(_, l, _)| l.len())+ .sum::<usize>(),+ total_matched_rows,+ "total_matched_rows is accumulated from the same chunks in freeze_streamed"+ );- let output_batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;- // Apply join filter if any- if !filter_columns.is_empty() {- if let Some(f) = &self.filter {- // Construct batch with only filter columns- let filter_batch =- RecordBatch::try_new(Arc::clone(f.schema()), filter_columns)?;-- let filter_result = f- .expression()- .evaluate(&filter_batch)?- .into_array(filter_batch.num_rows())?;-- // The boolean selection mask of the join filter result- let pre_mask =- datafusion_common::cast::as_boolean_array(&filter_result)?;-- // If there are nulls in join filter result, exclude them from selecting- // the rows to output.- let mask = if pre_mask.null_count() > 0 {- compute::prep_null_mask_filter(- datafusion_common::cast::as_boolean_array(&filter_result)?,- )+ let combined_left_indices = if matched_chunks.len() == 1 {+ matched_chunks[0].1.clone()+ } else {+ let refs: Vec<&dyn Array> =+ matched_chunks.iter().map(|c| &c.1 as &dyn Array).collect();+ as_uint64_array(&compute::concat(&refs)?)?.clone()+ };++ let left_columns =+ materialize_left_columns(&self.streamed_batch.batch, &combined_left_indices)?;++ let right_columns =+ self.materialize_right_columns(matched_chunks, total_matched_rows)?;++ let filter_columns = if self.join_type == JoinType::Right {+ get_filter_columns(&self.filter, &right_columns, &left_columns)+ } else {+ get_filter_columns(&self.filter, &left_columns, &right_columns)+ };++ let columns = if self.join_type != JoinType::Right {+ [left_columns, right_columns].concat()+ } else {+ [right_columns, left_columns].concat()+ };+ let output_batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;++ if !filter_columns.is_empty() {+ if let Some(f) = &self.filter {+ let filter_batch =+ RecordBatch::try_new(Arc::clone(f.schema()), filter_columns)?;+ let filter_result = f+ .expression()+ .evaluate(&filter_batch)?+ .into_array(filter_batch.num_rows())?;++ let pre_mask = datafusion_common::cast::as_boolean_array(&filter_result)?;++ let mask = if pre_mask.null_count() > 0 {+ compute::prep_null_mask_filter(pre_mask)+ } else {+ pre_mask.clone()+ };++ if needs_deferred_filtering(&self.filter, self.join_type) {+ // Full join uses pre_mask (preserving nulls) for+ // get_corrected_filter_mask; other outer joins use mask.+ let mask_to_use = if self.join_type != JoinType::Full {+ &mask} else {- pre_mask.clone()+ pre_mask};- // Push the filtered batch which contains rows passing join filter to the output- // For outer/semi/anti/mark joins with deferred filtering, push the unfiltered batch with metadata- // For INNER joins, filter immediately and push without metadata- let needs_deferred_filtering = matches!(+ self.joined_record_batches.push_batch_with_filter_metadata(+ output_batch,+ &combined_left_indices,+ mask_to_use,+ self.streamed_batch_counter.load(Relaxed),self.join_type,- JoinType::Left- | JoinType::LeftSemi- | JoinType::Right- | JoinType::RightSemi- | JoinType::LeftAnti- | JoinType::RightAnti- | JoinType::LeftMark- | JoinType::RightMark- | JoinType::Full);+ } else {+ let filtered_batch = filter_record_batch(&output_batch, &mask)?;+ self.joined_record_batches+ .push_batch_without_metadata(filtered_batch);+ }- if needs_deferred_filtering {- // Outer/semi/anti/mark joins: push unfiltered batch with metadata for deferred filtering- let mask_to_use = if self.join_type != JoinType::Full {- &mask- } else {- pre_mask- };-- self.joined_record_batches.push_batch_with_filter_metadata(- output_batch,- &left_indices,- mask_to_use,- self.streamed_batch_counter.load(Relaxed),- self.join_type,- );- } else {- // INNER joins: filter immediately and push without metadata- let filtered_batch = filter_record_batch(&output_batch, &mask)?;- self.joined_record_batches- .push_batch_without_metadata(filtered_batch, self.join_type);- }+ // Track which buffered rows had all filter matches fail,+ // so full join can emit them as null-joined later.+ if self.join_type == JoinType::Full {+ let mut offset = 0usize;+ for (batch_idx, _left, right) in matched_chunks {+ let chunk_len = right.len();+ let buffered_batch = &mut self.buffered_data.batches[*batch_idx];- // For outer joins, we need to push the null joined rows to the output if- // all joined rows are failed on the join filter.- // I.e., if all rows joined from a streamed row are failed with the join filter,- // we need to join it with nulls as buffered side.- if self.join_type == JoinType::Full {- let buffered_batch = &mut self.buffered_data.batches- [chunk.buffered_batch_idx.unwrap()];-- for i in 0..pre_mask.len() {- // If the buffered row is not joined with streamed side,- // skip it.- if right_indices.is_null(i) {+ for i in 0..chunk_len {+ if right.is_null(i) {continue;}-- let buffered_index = right_indices.value(i);-+ let buffered_index = right.value(i);buffered_batch.join_filter_not_matched_map.insert(buffered_index,*buffered_batch.join_filter_not_matched_map.get(&buffered_index).unwrap_or(&true)- && !pre_mask.value(i),+ && !pre_mask.value(offset + i),);}+ offset += chunk_len;}+ debug_assert_eq!(+ offset, total_matched_rows,+ "offset must advance through every chunk exactly once"+ );}+ }+ } else {+ self.joined_record_batches+ .push_batch_without_metadata(output_batch);+ }++ Ok(())+ }++ /// Materializes right-side columns across all matched chunks.+ ///+ /// When chunks reference a single buffered batch, indices are concatenated+ /// for a single fetch. When multiple batches are involved, `interleave`+ /// gathers columns across sources. A null-row sentinel at source index 0+ /// handles null right indices (unmatched streamed rows).+ fn materialize_right_columns(+ &self,+ matched_chunks: &[(usize, UInt64Array, UInt64Array)],+ total_matched_rows: usize,+ ) -> Result<Vec<ArrayRef>> {+ let first_batch_idx = matched_chunks[0].0;+ let single_source = matched_chunks.iter().all(|c| c.0 == first_batch_idx);++ if single_source {+ let combined_right_indices = if matched_chunks.len() == 1 {+ matched_chunks[0].2.clone()} else {- self.joined_record_batches- .push_batch_without_metadata(output_batch, self.join_type);+ let refs: Vec<&dyn Array> =+ matched_chunks.iter().map(|c| &c.2 as &dyn Array).collect();+ as_uint64_array(&compute::concat(&refs)?)?.clone()+ };+ return fetch_right_columns_by_idxs(+ &self.buffered_data,+ first_batch_idx,+ &combined_right_indices,+ );+ }++ // Multiple source batches: map each buffered_batch_idx to a+ // contiguous source index, reserving source 0 for a null sentinel.+ let mut batch_idx_to_source: HashMap<usize, usize> = HashMap::new();+ let mut source_batches: Vec<usize> = Vec::new();+ for (batch_idx, _, _) in matched_chunks {+ batch_idx_to_source.entry(*batch_idx).or_insert_with(|| {+ let idx = source_batches.len() + 1;+ source_batches.push(*batch_idx);+ idx+ });+ }++ let mut interleave_indices: Vec<(usize, usize)> =+ Vec::with_capacity(total_matched_rows);+ for (batch_idx, _, right) in matched_chunks {+ let source = batch_idx_to_source[batch_idx];+ for i in 0..right.len() {+ if right.is_null(i) {+ interleave_indices.push((0, 0));+ } else {+ interleave_indices.push((source, right.value(i) as usize));+ }}}- self.streamed_batch.output_indices.clear();- self.streamed_batch.num_output_rows = 0;+ let num_right_cols = self.buffered_schema.fields().len();+ let mut right_columns = Vec::with_capacity(num_right_cols);- Ok(())+ // Read each source batch once (spilled batches require disk I/O).+ let source_data: Vec<Option<RecordBatch>> = source_batches+ .iter()+ .map(|&idx| {+ let bb = &self.buffered_data.batches[idx];+ match &bb.batch {+ BufferedBatchState::InMemory(batch) => Some(batch.clone()),+ BufferedBatchState::Spilled(spill_file) => {+ let file = BufReader::new(File::open(spill_file.path()).ok()?);+ let reader = StreamReader::try_new(file, None).ok()?;+ reader.into_iter().next()?.ok()+ }+ }+ })+ .collect();++ for col_idx in 0..num_right_cols {+ let dtype = self.buffered_schema.field(col_idx).data_type();+ let null_array = new_null_array(dtype, 1);++ let mut source_arrays: Vec<&dyn Array> =+ Vec::with_capacity(source_batches.len() + 1);+ source_arrays.push(null_array.as_ref());++ for data in &source_data {+ match data {+ Some(batch) => source_arrays.push(batch.column(col_idx).as_ref()),+ None => {+ return internal_err!(+ "Failed to read spilled buffered batch during interleave"+ );+ }+ }+ }++ right_columns.push(interleave(&source_arrays, &interleave_indices)?);+ }++ Ok(right_columns)}fn filter_joined_batch(&mut self) -> Result<RecordBatch> {@@ -1556,7 +1600,6 @@ impl SortMergeJoinStream {corrected_mask,self.join_type,&self.schema,- &self.streamed_schema,&self.buffered_schema,)?;@@ -1567,22 +1610,26 @@ impl SortMergeJoinStream {}}-fn create_unmatched_columns(- join_type: JoinType,- schema: &SchemaRef,- size: usize,-) -> Vec<ArrayRef> {- if matches!(join_type, JoinType::LeftMark | JoinType::RightMark) {- vec![Arc::new(BooleanArray::from(vec![false; size])) as ArrayRef]+/// Materialize left (streamed) columns using slice or take.+fn materialize_left_columns(+ batch: &RecordBatch,+ indices: &UInt64Array,+) -> Result<Vec<ArrayRef>> {+ if let Some(range) = is_contiguous_range(indices) {+ Ok(batch.slice(range.start, range.len()).columns().to_vec())} else {- schema- .fields()- .iter()- .map(|f| new_null_array(f.data_type(), size))- .collect::<Vec<_>>()+ Ok(take_arrays(batch.columns(), indices, None)?)}}+fn create_unmatched_columns(schema: &SchemaRef, size: usize) -> Vec<ArrayRef> {+ schema+ .fields()+ .iter()+ .map(|f| new_null_array(f.data_type(), size))+ .collect::<Vec<_>>()+}+fn produce_buffered_null_batch(schema: &SchemaRef,streamed_schema: &SchemaRef,diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs b/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rsindex cf76b2ba8c..62efb77f87 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/metrics.rs@@ -19,7 +19,7 @@use crate::metrics::{BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder,- MetricCategory, SpillMetrics, Time,+ MetricCategory, Time,};/// Metrics for SortMergeJoinExec@@ -35,8 +35,6 @@ pub(super) struct SortMergeJoinMetrics {/// Peak memory used for buffered data./// Calculated as sum of peak memory values across partitionspeak_mem_used: Gauge,- /// Metrics related to spilling- spill_metrics: SpillMetrics,}impl SortMergeJoinMetrics {@@ -51,7 +49,6 @@ impl SortMergeJoinMetrics {let peak_mem_used = MetricBuilder::new(metrics).with_category(MetricCategory::Bytes).gauge("peak_mem_used", partition);- let spill_metrics = SpillMetrics::new(metrics, partition);let baseline_metrics = BaselineMetrics::new(metrics, partition);@@ -61,7 +58,6 @@ impl SortMergeJoinMetrics {input_rows,baseline_metrics,peak_mem_used,- spill_metrics,}}@@ -84,8 +80,4 @@ impl SortMergeJoinMetrics {pub fn peak_mem_used(&self) -> Gauge {self.peak_mem_used.clone()}-- pub fn spill_metrics(&self) -> SpillMetrics {- self.spill_metrics.clone()- }}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs b/datafusion/physical-plan/src/joins/sort_merge_join/mod.rsindex 06290ec4d0..2fdb0924e7 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs@@ -19,10 +19,11 @@pub use exec::SortMergeJoinExec;+pub(crate) mod bitwise_stream;mod exec;mod filter;+pub(crate) mod materializing_stream;mod metrics;-mod stream;#[cfg(test)]mod tests;diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex a1043a9d8b..5824c2328d 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -24,15 +24,22 @@//!//! Add relevant tests under the specified sections.+use std::pin::Pin;+use std::sync::Arc;+use std::task::{Context, Poll};++use super::bitwise_stream::BitwiseSortMergeJoinStream;use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn};use crate::joins::{HashJoinExec, PartitionMode, SortMergeJoinExec};+use crate::metrics::{ExecutionPlanMetricsSet, SpillMetrics};+use crate::spill::spill_manager::SpillManager;use crate::test::TestMemoryExec;use crate::test::exec::BarrierExec;use crate::test::{build_table_i32, build_table_i32_two_cols};-use crate::{ExecutionPlan, common};+use crate::{ExecutionPlan, RecordBatchStream, common};use crate::{expressions::Column, joins::sort_merge_join::filter::get_corrected_filter_mask,- joins::sort_merge_join::stream::JoinedRecordBatches,+ joins::sort_merge_join::materializing_stream::JoinedRecordBatches,};use arrow::array::{BinaryArray, BooleanArray, Date32Array, Date64Array, FixedSizeBinaryArray,@@ -53,15 +60,15 @@ use datafusion_common::{use datafusion_common_runtime::JoinSet;use datafusion_execution::config::SessionConfig;use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};+use datafusion_execution::memory_pool::MemoryConsumer;use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::Operator;use datafusion_physical_expr::expressions::BinaryExpr;-use futures::StreamExt;-use insta::{allow_duplicates, assert_snapshot};+use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;+use futures::{Stream, StreamExt};+use insta::assert_snapshot;use itertools::Itertools;-use std::sync::Arc;-use std::task::Poll;fn build_table(a: (&str, &Vec<i32>),@@ -1278,7 +1285,7 @@ async fn join_right_anti_output_two_batches() -> Result<()> {let (_, batches) =join_collect_batch_size_equals_two(left, right, on, LeftAnti).await?;- // SemiAntiSortMergeJoinStream uses a coalescer, so batch boundaries differ+ // BitwiseSortMergeJoinStream uses a coalescer, so batch boundaries differ// from the old stream. Only assert data correctness.let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();assert_eq!(total_rows, 3);@@ -1567,7 +1574,7 @@ async fn join_right_semi_output_two_batches() -> Result<()> {"| 2 | 5 | 8 |","+----+----+----+",];- // SemiAntiSortMergeJoinStream uses a coalescer, so batch boundaries differ+ // BitwiseSortMergeJoinStream uses a coalescer, so batch boundaries differ// from the old stream. Only assert data correctness.let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();assert_eq!(total_rows, 3);@@ -2061,9 +2068,9 @@ async fn overallocation_single_batch_no_spill() -> Result<()> {let sort_options = vec![SortOptions::default(); on.len()];let join_types = vec![- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.- Inner, Left, Right, Full, LeftMark, RightMark,+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.+ Inner, Left, Right, Full,];// Disable DiskManager to prevent spilling@@ -2144,9 +2151,9 @@ async fn overallocation_multi_batch_no_spill() -> Result<()> {let sort_options = vec![SortOptions::default(); on.len()];let join_types = vec![- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.- Inner, Left, Right, Full, LeftMark, RightMark,+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.+ Inner, Left, Right, Full,];// Disable DiskManager to prevent spilling@@ -2206,9 +2213,9 @@ async fn overallocation_single_batch_spill() -> Result<()> {let sort_options = vec![SortOptions::default(); on.len()];let join_types = [- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.- Inner, Left, Right, Full, LeftMark, RightMark,+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.+ Inner, Left, Right, Full,];// Enable DiskManager to allow spilling@@ -2312,9 +2319,9 @@ async fn overallocation_multi_batch_spill() -> Result<()> {let sort_options = vec![SortOptions::default(); on.len()];let join_types = [- // Semi/anti joins use SemiAntiSortMergeJoinStream which only tracks- // inner key buffer memory; tested in semi_anti_sort_merge_join/tests.rs.- Inner, Left, Right, Full, LeftMark, RightMark,+ // Semi/anti/mark joins use BitwiseSortMergeJoinStream which only tracks+ // inner key buffer memory; tested in bitwise_sort_merge_join/tests.rs.+ Inner, Left, Right, Full,];// Enable DiskManager to allow spilling@@ -2377,6 +2384,321 @@ async fn overallocation_multi_batch_spill() -> Result<()> {Ok(())}+/// Build a c1 < c2 filter on the third column of each side.+fn build_c1_lt_c2_filter(left_schema: &Schema, right_schema: &Schema) -> JoinFilter {+ JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("c1", 0)),+ Operator::Lt,+ Arc::new(Column::new("c2", 1)),+ )),+ vec![+ ColumnIndex {+ index: 2,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 2,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ left_schema+ .field_with_name("c1")+ .unwrap()+ .clone()+ .with_nullable(true),+ right_schema+ .field_with_name("c2")+ .unwrap()+ .clone()+ .with_nullable(true),+ ])),+ )+}++#[tokio::test]+async fn spill_with_filter_deferred() -> Result<()> {+ let left = build_table(+ ("a1", &vec![0, 1, 2, 3, 4, 5]),+ ("b1", &vec![1, 2, 3, 4, 5, 6]),+ ("c1", &vec![4, 5, 6, 7, 8, 9]),+ );+ let right = build_table(+ ("a2", &vec![0, 10, 20, 30, 40]),+ ("b2", &vec![1, 3, 4, 6, 8]),+ ("c2", &vec![50, 60, 70, 80, 90]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,+ )];+ let sort_options = vec![SortOptions::default(); on.len()];++ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());++ // Deferred filtering join types handled by the main MaterializingSortMergeJoinStream+ let join_types = [Left, Right, Full];++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(100, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()?;++ for batch_size in [1, 50] {+ let session_config = SessionConfig::default().with_batch_size(batch_size);++ for join_type in &join_types {+ // Run with spilling+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(session_config.clone())+ .with_runtime(Arc::clone(&runtime)),+ );+ let join = join_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ *join_type,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join.execute(0, task_ctx)?;+ let spilled_result = common::collect(stream).await.unwrap();++ assert!(join.metrics().is_some());+ assert!(+ join.metrics().unwrap().spill_count().unwrap() > 0,+ "Expected spilling for {join_type:?} batch_size={batch_size}"+ );++ // Run without spilling+ let task_ctx_no_spill = Arc::new(+ TaskContext::default().with_session_config(session_config.clone()),+ );+ let join_no_spill = join_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ *join_type,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;+ let no_spill_result = common::collect(stream).await.unwrap();++ let spilled_str = batches_to_sort_string(&spilled_result);+ let no_spill_str = batches_to_sort_string(&no_spill_result);+ assert_eq!(+ spilled_str, no_spill_str,+ "Spill vs no-spill mismatch for {join_type:?} batch_size={batch_size}"+ );+ }+ }++ Ok(())+}++#[tokio::test]+async fn spill_with_filter_multi_batch() -> Result<()> {+ let left_batch_1 = build_table_i32(+ ("a1", &vec![0, 1]),+ ("b1", &vec![1, 1]),+ ("c1", &vec![4, 5]),+ );+ let left_batch_2 = build_table_i32(+ ("a1", &vec![2, 3]),+ ("b1", &vec![1, 1]),+ ("c1", &vec![6, 7]),+ );+ let left_batch_3 = build_table_i32(+ ("a1", &vec![4, 5]),+ ("b1", &vec![1, 1]),+ ("c1", &vec![8, 9]),+ );+ let right_batch_1 = build_table_i32(+ ("a2", &vec![0, 10]),+ ("b2", &vec![1, 1]),+ ("c2", &vec![50, 60]),+ );+ let right_batch_2 = build_table_i32(+ ("a2", &vec![20, 30]),+ ("b2", &vec![1, 1]),+ ("c2", &vec![70, 80]),+ );+ let right_batch_3 =+ build_table_i32(("a2", &vec![40]), ("b2", &vec![1]), ("c2", &vec![90]));+ let left = build_table_from_batches(vec![left_batch_1, left_batch_2, left_batch_3]);+ let right =+ build_table_from_batches(vec![right_batch_1, right_batch_2, right_batch_3]);+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,+ )];+ let sort_options = vec![SortOptions::default(); on.len()];++ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());++ let join_types = [Left, Right, Full];++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(500, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()?;++ for batch_size in [1, 50] {+ let session_config = SessionConfig::default().with_batch_size(batch_size);++ for join_type in &join_types {+ // Run with spilling+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(session_config.clone())+ .with_runtime(Arc::clone(&runtime)),+ );+ let join = join_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ *join_type,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join.execute(0, task_ctx)?;+ let spilled_result = common::collect(stream).await.unwrap();++ assert!(join.metrics().is_some());+ assert!(+ join.metrics().unwrap().spill_count().unwrap() > 0,+ "Expected spilling for {join_type:?} batch_size={batch_size}"+ );++ // Run without spilling+ let task_ctx_no_spill = Arc::new(+ TaskContext::default().with_session_config(session_config.clone()),+ );+ let join_no_spill = join_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ *join_type,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;+ let no_spill_result = common::collect(stream).await.unwrap();++ let spilled_str = batches_to_sort_string(&spilled_result);+ let no_spill_str = batches_to_sort_string(&no_spill_result);+ assert_eq!(+ spilled_str, no_spill_str,+ "Spill vs no-spill mismatch for {join_type:?} batch_size={batch_size}"+ );+ }+ }++ Ok(())+}++/// FULL join where all buffered rows match on key but fail the filter.+/// Verifies produce_buffered_not_matched emits null-joined rows under spill.+#[tokio::test]+async fn spill_full_join_filter_not_matched() -> Result<()> {+ // c1 values (100..105) are always > c2 values (1..5), so c1 < c2 always fails+ let left = build_table(+ ("a1", &vec![0, 1, 2, 3, 4]),+ ("b1", &vec![1, 1, 1, 1, 1]),+ ("c1", &vec![100, 101, 102, 103, 104]),+ );+ let right = build_table(+ ("a2", &vec![10, 20, 30, 40, 50]),+ ("b2", &vec![1, 1, 1, 1, 1]),+ ("c2", &vec![1, 2, 3, 4, 5]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,+ )];+ let sort_options = vec![SortOptions::default(); on.len()];++ let filter = build_c1_lt_c2_filter(&left.schema(), &right.schema());++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(100, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()?;++ for batch_size in [1, 50] {+ let session_config = SessionConfig::default().with_batch_size(batch_size);++ // Run with spilling+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(session_config.clone())+ .with_runtime(Arc::clone(&runtime)),+ );+ let join = join_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ Full,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join.execute(0, task_ctx)?;+ let spilled_result = common::collect(stream).await.unwrap();++ assert!(+ join.metrics().unwrap().spill_count().unwrap() > 0,+ "Expected spilling for FULL batch_size={batch_size}"+ );++ // Run without spilling+ let task_ctx_no_spill =+ Arc::new(TaskContext::default().with_session_config(session_config.clone()));+ let join_no_spill = join_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ Full,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;+ let no_spill_result = common::collect(stream).await.unwrap();++ // All filter evaluations fail, so FULL join should produce:+ // - 5 rows with left columns + null right columns (unmatched left)+ // - 5 rows with null left columns + right columns (unmatched right)+ let total_rows: usize = no_spill_result.iter().map(|b| b.num_rows()).sum();+ assert_eq!(+ total_rows, 10,+ "FULL join with all-failing filter should produce 10 rows, got {total_rows}"+ );++ let spilled_str = batches_to_sort_string(&spilled_result);+ let no_spill_str = batches_to_sort_string(&no_spill_result);+ assert_eq!(+ spilled_str, no_spill_str,+ "Spill vs no-spill mismatch for FULL join batch_size={batch_size}"+ );+ }++ Ok(())+}+fn build_joined_record_batches() -> Result<JoinedRecordBatches> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true),@@ -2721,394 +3043,46 @@ async fn test_left_outer_join_filtered_mask() -> Result<()> {Ok(())}-#[tokio::test]-async fn test_semi_join_filtered_mask() -> Result<()> {- for join_type in [LeftSemi, RightSemi] {- let mut joined_batches = build_joined_record_batches()?;- let schema = joined_batches.joined_batches.schema();+#[test]+fn test_partition_statistics() -> Result<()> {+ use crate::ExecutionPlan;+ use datafusion_common::stats::Precision;- let output = joined_batches.concat_batches(&schema)?;- let out_mask = joined_batches.filter_metadata.filter_mask.finish();- let out_indices = joined_batches.filter_metadata.row_indices.finish();+ let left = build_table(+ ("a1", &vec![1, 2, 3]),+ ("b1", &vec![4, 5, 5]),+ ("c1", &vec![7, 8, 9]),+ );+ let right = build_table(+ ("a2", &vec![10, 20, 30]),+ ("b1", &vec![4, 5, 6]),+ ("c2", &vec![70, 80, 90]),+ );- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0]),- &[0usize],- &BooleanArray::from(vec![true]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![true])- );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,+ )];- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0]),- &[0usize],- &BooleanArray::from(vec![false]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![None])- );+ // Test different join types to ensure partition_statistics works correctly for all+ let join_types = vec![+ (Inner, 6), // left cols + right cols+ (Left, 6), // left cols + right cols+ (Right, 6), // left cols + right cols+ (Full, 6), // left cols + right cols+ (LeftSemi, 3), // only left cols+ (LeftAnti, 3), // only left cols+ (RightSemi, 3), // only right cols+ (RightAnti, 3), // only right cols+ ];- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0]),- &[0usize; 2],- &BooleanArray::from(vec![true, true]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![Some(true), None])- );+ for (join_type, expected_cols) in join_types {+ let join_exec =+ join(Arc::clone(&left), Arc::clone(&right), on.clone(), join_type)?;- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![true, true, true]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![Some(true), None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![true, false, true]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![Some(true), None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![false, false, true]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![None, None, Some(true),])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![false, true, true]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![None, Some(true), None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![false, false, false]),- output.num_rows()- )- .unwrap(),- BooleanArray::from(vec![None, None, None])- );-- let corrected_mask = get_corrected_filter_mask(- join_type,- &out_indices,- &joined_batches.filter_metadata.batch_ids,- &out_mask,- output.num_rows(),- )- .unwrap();-- assert_eq!(- corrected_mask,- BooleanArray::from(vec![- Some(true),- None,- Some(true),- None,- Some(true),- None,- None,- None- ])- );-- let filtered_rb = filter_record_batch(&output, &corrected_mask)?;-- assert_batches_eq!(- &[- "+---+----+---+----+",- "| a | b | x | y |",- "+---+----+---+----+",- "| 1 | 10 | 1 | 11 |",- "| 1 | 11 | 1 | 12 |",- "| 1 | 12 | 1 | 13 |",- "+---+----+---+----+",- ],- &[filtered_rb]- );-- // output null rows- let null_mask = arrow::compute::not(&corrected_mask)?;- assert_eq!(- null_mask,- BooleanArray::from(vec![- Some(false),- None,- Some(false),- None,- Some(false),- None,- None,- None- ])- );-- let null_joined_batch = filter_record_batch(&output, &null_mask)?;-- assert_batches_eq!(- &[- "+---+---+---+---+",- "| a | b | x | y |",- "+---+---+---+---+",- "+---+---+---+---+",- ],- &[null_joined_batch]- );- }- Ok(())-}--#[tokio::test]-async fn test_anti_join_filtered_mask() -> Result<()> {- for join_type in [LeftAnti, RightAnti] {- let mut joined_batches = build_joined_record_batches()?;- let schema = joined_batches.joined_batches.schema();-- let output = joined_batches.concat_batches(&schema)?;- let out_mask = joined_batches.filter_metadata.filter_mask.finish();- let out_indices = joined_batches.filter_metadata.row_indices.finish();-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0]),- &[0usize],- &BooleanArray::from(vec![true]),- 1- )- .unwrap(),- BooleanArray::from(vec![None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0]),- &[0usize],- &BooleanArray::from(vec![false]),- 1- )- .unwrap(),- BooleanArray::from(vec![Some(true)])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0]),- &[0usize; 2],- &BooleanArray::from(vec![true, true]),- 2- )- .unwrap(),- BooleanArray::from(vec![None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![true, true, true]),- 3- )- .unwrap(),- BooleanArray::from(vec![None, None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![true, false, true]),- 3- )- .unwrap(),- BooleanArray::from(vec![None, None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![false, false, true]),- 3- )- .unwrap(),- BooleanArray::from(vec![None, None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![false, true, true]),- 3- )- .unwrap(),- BooleanArray::from(vec![None, None, None])- );-- assert_eq!(- get_corrected_filter_mask(- join_type,- &UInt64Array::from(vec![0, 0, 0]),- &[0usize; 3],- &BooleanArray::from(vec![false, false, false]),- 3- )- .unwrap(),- BooleanArray::from(vec![None, None, Some(true)])- );-- let corrected_mask = get_corrected_filter_mask(- join_type,- &out_indices,- &joined_batches.filter_metadata.batch_ids,- &out_mask,- output.num_rows(),- )- .unwrap();-- assert_eq!(- corrected_mask,- BooleanArray::from(vec![- None,- None,- None,- None,- None,- Some(true),- None,- Some(true)- ])- );-- let filtered_rb = filter_record_batch(&output, &corrected_mask)?;-- allow_duplicates! {- assert_snapshot!(batches_to_string(&[filtered_rb]), @r"- +---+----+---+----+- | a | b | x | y |- +---+----+---+----+- | 1 | 13 | 1 | 12 |- | 1 | 14 | 1 | 11 |- +---+----+---+----+- ");- }-- // output null rows- let null_mask = arrow::compute::not(&corrected_mask)?;- assert_eq!(- null_mask,- BooleanArray::from(vec![- None,- None,- None,- None,- None,- Some(false),- None,- Some(false),- ])- );-- let null_joined_batch = filter_record_batch(&output, &null_mask)?;-- allow_duplicates! {- assert_snapshot!(batches_to_string(&[null_joined_batch]), @r"- +---+---+---+---+- | a | b | x | y |- +---+---+---+---+- +---+---+---+---+- ");- }- }-- Ok(())-}--#[test]-fn test_partition_statistics() -> Result<()> {- use crate::ExecutionPlan;- use datafusion_common::stats::Precision;-- let left = build_table(- ("a1", &vec![1, 2, 3]),- ("b1", &vec![4, 5, 5]),- ("c1", &vec![7, 8, 9]),- );- let right = build_table(- ("a2", &vec![10, 20, 30]),- ("b1", &vec![4, 5, 6]),- ("c2", &vec![70, 80, 90]),- );-- let on = vec![(- Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,- Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,- )];-- // Test different join types to ensure partition_statistics works correctly for all- let join_types = vec![- (Inner, 6), // left cols + right cols- (Left, 6), // left cols + right cols- (Right, 6), // left cols + right cols- (Full, 6), // left cols + right cols- (LeftSemi, 3), // only left cols- (LeftAnti, 3), // only left cols- (RightSemi, 3), // only right cols- (RightAnti, 3), // only right cols- ];-- for (join_type, expected_cols) in join_types {- let join_exec =- join(Arc::clone(&left), Arc::clone(&right), on.clone(), join_type)?;-- // Test aggregate statistics (partition = None)- // Should return meaningful statistics computed from both inputs- let stats = join_exec.partition_statistics(None)?;+ // Test aggregate statistics (partition = None)+ // Should return meaningful statistics computed from both inputs+ let stats = join_exec.partition_statistics(None)?;assert_eq!(stats.column_statistics.len(),expected_cols,@@ -3557,7 +3531,878 @@ async fn consume_stream_until_finish_barrier_reached(Ok((output_batched, after_finish_barrier_reached))}+/// Exercises the multi-source interleave path in `materialize_right_columns`.+///+/// When the right (buffered) side is split into many small batches with unique+/// keys, a single `freeze_streamed()` call references multiple `BufferedBatch`es.+/// This forces the `interleave` kernel instead of the single-source `take` path.+/// Without this test, the interleave path has zero coverage from unit tests+/// (fuzz tests use ~100 unique keys across 1000 rows, so all keys fit in one+/// buffered batch).+#[tokio::test]+async fn join_filtered_with_multiple_buffered_batches() -> Result<()> {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("key", DataType::Int32, false),+ Field::new("val_l", DataType::Int32, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("key", DataType::Int32, false),+ Field::new("val_r", DataType::Int32, false),+ ]));++ // Left: single batch, keys 1..=6+ let left_batch = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])),+ Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])),+ ],+ )?;+ let left = build_table_from_batches(vec![left_batch]);++ // Right: one row per batch so each key lives in a separate BufferedBatch+ let right_batches: Vec<RecordBatch> = (1..=6)+ .map(|k| {+ RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![k])),+ Arc::new(Int32Array::from(vec![k * 100])),+ ],+ )+ .unwrap()+ })+ .collect();+ let right = build_table_from_batches(right_batches);++ let on: JoinOn = vec![(+ Arc::new(Column::new_with_schema("key", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("key", &right.schema())?) as _,+ )];++ // Filter: val_l + val_r < 350 — passes for keys 1-3, fails for 4-6+ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("val_l", 0)),+ Operator::Plus,+ Arc::new(Column::new("val_r", 1)),+ )),+ Operator::Lt,+ Arc::new(datafusion_physical_expr::expressions::Literal::new(+ datafusion_common::ScalarValue::Int32(Some(350)),+ )),+ )),+ vec![+ ColumnIndex {+ index: 1,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 1,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ Field::new("val_l", DataType::Int32, true),+ Field::new("val_r", DataType::Int32, true),+ ])),+ );++ // Inner: only rows passing the filter+ let (_, batches) = join_collect_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ Inner,+ )+ .await?;+ let result = batches_to_sort_string(&batches);+ assert_snapshot!(result, @r"+ +-----+-------+-----+-------++ | key | val_l | key | val_r |+ +-----+-------+-----+-------++ | 1 | 10 | 1 | 100 |+ | 2 | 20 | 2 | 200 |+ | 3 | 30 | 3 | 300 |+ +-----+-------+-----+-------++ ");++ // Left: unmatched left rows get null right columns+ let (_, batches) = join_collect_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ Left,+ )+ .await?;+ let result = batches_to_sort_string(&batches);+ assert_snapshot!(result, @r"+ +-----+-------+-----+-------++ | key | val_l | key | val_r |+ +-----+-------+-----+-------++ | 1 | 10 | 1 | 100 |+ | 2 | 20 | 2 | 200 |+ | 3 | 30 | 3 | 300 |+ | 4 | 40 | | |+ | 5 | 50 | | |+ | 6 | 60 | | |+ +-----+-------+-----+-------++ ");++ // Full: unmatched rows on both sides get null columns+ let (_, batches) = join_collect_with_filter(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ filter.clone(),+ Full,+ )+ .await?;+ let result = batches_to_sort_string(&batches);+ assert_snapshot!(result, @r"+ +-----+-------+-----+-------++ | key | val_l | key | val_r |+ +-----+-------+-----+-------++ | | | 4 | 400 |+ | | | 5 | 500 |+ | | | 6 | 600 |+ | 1 | 10 | 1 | 100 |+ | 2 | 20 | 2 | 200 |+ | 3 | 30 | 3 | 300 |+ | 4 | 40 | | |+ | 5 | 50 | | |+ | 6 | 60 | | |+ +-----+-------+-----+-------++ ");++ Ok(())+}+/// Returns the column names on the schemafn columns(schema: &Schema) -> Vec<String> {schema.fields().iter().map(|f| f.name().clone()).collect()}++// ==================== BitwiseSortMergeJoinStream direct tests ====================+//+// These tests construct a BitwiseSortMergeJoinStream directly (bypassing exec)+// to exercise async re-entry and spill edge cases using PendingStream.++/// Create test memory/spill resources for stream-level tests.+fn test_stream_resources(+ inner_schema: SchemaRef,+ metrics: &ExecutionPlanMetricsSet,+) -> (+ datafusion_execution::memory_pool::MemoryReservation,+ SpillManager,+ Arc<datafusion_execution::runtime_env::RuntimeEnv>,+) {+ let ctx = TaskContext::default();+ let runtime_env = ctx.runtime_env();+ let reservation = MemoryConsumer::new("test").register(ctx.memory_pool());+ let spill_manager = SpillManager::new(+ Arc::clone(&runtime_env),+ SpillMetrics::new(metrics, 0),+ inner_schema,+ );+ (reservation, spill_manager, runtime_env)+}++/// A RecordBatch stream that yields Poll::Pending once before delivering+/// each batch at a specified index. This simulates the behavior of+/// repartitioned tokio::sync::mpsc channels where data isn't immediately+/// available.+struct PendingStream {+ batches: Vec<RecordBatch>,+ index: usize,+ /// If pending_before[i] is true, yield Pending once before delivering+ /// the batch at index i.+ pending_before: Vec<bool>,+ /// True if we've already yielded Pending for the current index.+ yielded_pending: bool,+ schema: SchemaRef,+}++impl PendingStream {+ fn new(batches: Vec<RecordBatch>, pending_before: Vec<bool>) -> Self {+ assert_eq!(batches.len(), pending_before.len());+ let schema = batches[0].schema();+ Self {+ batches,+ index: 0,+ pending_before,+ yielded_pending: false,+ schema,+ }+ }+}++impl Stream for PendingStream {+ type Item = Result<RecordBatch>;++ fn poll_next(+ mut self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ ) -> Poll<Option<Self::Item>> {+ if self.index >= self.batches.len() {+ return Poll::Ready(None);+ }+ if self.pending_before[self.index] && !self.yielded_pending {+ self.yielded_pending = true;+ cx.waker().wake_by_ref();+ return Poll::Pending;+ }+ self.yielded_pending = false;+ let batch = self.batches[self.index].clone();+ self.index += 1;+ Poll::Ready(Some(Ok(batch)))+ }+}++impl RecordBatchStream for PendingStream {+ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }+}++/// Helper: collect all output from a BitwiseSortMergeJoinStream.+async fn collect_stream(stream: BitwiseSortMergeJoinStream) -> Result<Vec<RecordBatch>> {+ common::collect(Box::pin(stream)).await+}++/// Reproduces the buffer_inner_key_group re-entry bug:+///+/// When buffer_inner_key_group buffers inner rows across batch boundaries+/// and poll_next_inner_batch returns Pending mid-way, the ready! macro+/// exits poll_join. On re-entry, the merge-scan reaches Equal again and+/// calls buffer_inner_key_group a second time -- which starts with+/// clear(), destroying the partially collected inner rows. Previously+/// consumed batches are gone, so re-buffering misses them.+///+/// Setup:+/// - Inner: 3 single-row batches, all with key=1, filter values c2=[10, 20, 30]+/// - Outer: 1 row, key=1, filter value c1=10+/// - Filter: c1 == c2 (only first inner row c2=10 matches)+/// - Pending injected before 3rd inner batch+///+/// Without the bug: outer row emitted (match via c2=10)+/// With the bug: outer row missing (c2=10 batch lost on re-entry)+#[tokio::test]+async fn filter_buffer_pending_loses_inner_rows() -> Result<()> {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c1", DataType::Int32, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("a2", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ]));++ // Outer: 1 row, key=1, c1=10+ let outer_batch = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![1])), // join key+ Arc::new(Int32Array::from(vec![10])), // filter value+ ],+ )?;++ // Inner: 3 single-row batches, key=1, c2=[10, 20, 30]+ let inner_batch1 = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![100])),+ Arc::new(Int32Array::from(vec![1])), // join key+ Arc::new(Int32Array::from(vec![10])), // matches filter+ ],+ )?;+ let inner_batch2 = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![200])),+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![20])), // doesn't match+ ],+ )?;+ let inner_batch3 = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![300])),+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![30])), // doesn't match+ ],+ )?;++ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(+ vec![outer_batch],+ vec![false], // outer delivers immediately+ ));+ let inner: SendableRecordBatchStream = Box::pin(PendingStream::new(+ vec![inner_batch1, inner_batch2, inner_batch3],+ vec![false, false, true], // Pending before 3rd batch+ ));++ // Filter: c1 == c2+ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("c1", 0)),+ Operator::Eq,+ Arc::new(Column::new("c2", 1)),+ )),+ vec![+ ColumnIndex {+ index: 2,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 2,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ])),+ );++ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];++ let metrics = ExecutionPlanMetricsSet::new();+ let inner_schema = inner.schema();+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(inner_schema, &metrics);+ let stream = BitwiseSortMergeJoinStream::try_new(+ left_schema, // output schema = outer schema for semi+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ on_outer,+ on_inner,+ Some(filter),+ LeftSemi,+ 8192,+ 0,+ &metrics,+ reservation,+ spill_manager,+ runtime_env,+ )?;++ let batches = collect_stream(stream).await?;+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(+ total, 1,+ "LeftSemi with filter: outer row should be emitted because \+ inner row c2=10 matches filter c1==c2. Got {total} rows."+ );+ Ok(())+}++/// Reproduces the no-filter boundary Pending re-entry bug:+///+/// When an outer key group spans a batch boundary, the no-filter path+/// emits the current batch, then polls for the next outer batch. If+/// poll returns Pending, poll_join exits. On re-entry, without the+/// PendingBoundary fix, the new batch is processed fresh by the+/// merge-scan. Since inner already advanced past this key, the outer+/// rows with the matching key are skipped via Ordering::Less.+///+/// Setup:+/// - Outer: 2 single-row batches, both with key=1 (key group spans boundary)+/// - Inner: 1 row with key=1+/// - Pending injected on outer before 2nd batch+///+/// Without fix: only first outer row emitted (second lost on re-entry)+/// With fix: both outer rows emitted+#[tokio::test]+async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c1", DataType::Int32, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("a2", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ]));++ // Outer: 2 single-row batches, both key=1+ let outer_batch1 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![10])),+ ],+ )?;+ let outer_batch2 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![2])),+ Arc::new(Int32Array::from(vec![1])), // same key+ Arc::new(Int32Array::from(vec![20])),+ ],+ )?;++ // Inner: 1 row, key=1+ let inner_batch = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![100])),+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![50])),+ ],+ )?;++ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(+ vec![outer_batch1, outer_batch2],+ vec![false, true], // Pending before 2nd outer batch+ ));+ let inner: SendableRecordBatchStream =+ Box::pin(PendingStream::new(vec![inner_batch], vec![false]));++ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];++ let metrics = ExecutionPlanMetricsSet::new();+ let inner_schema = inner.schema();+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(inner_schema, &metrics);+ let stream = BitwiseSortMergeJoinStream::try_new(+ left_schema,+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ on_outer,+ on_inner,+ None, // no filter+ LeftSemi,+ 8192,+ 0,+ &metrics,+ reservation,+ spill_manager,+ runtime_env,+ )?;++ let batches = collect_stream(stream).await?;+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(+ total, 2,+ "LeftSemi no filter: both outer rows (key=1) should be emitted \+ because inner has key=1. Got {total} rows."+ );+ Ok(())+}++/// Tests the filtered boundary Pending re-entry: outer key group spans+/// batches with a filter, and poll_next_outer_batch returns Pending.+///+/// Setup:+/// - Outer: 2 single-row batches, both key=1, c1=[10, 20]+/// - Inner: 1 row, key=1, c2=10+/// - Filter: c1 == c2 (first outer row matches, second doesn't)+/// - Pending before 2nd outer batch+///+/// Expected: 1 row (only the first outer row c1=10 passes the filter)+#[tokio::test]+async fn filtered_boundary_pending_outer_rows() -> Result<()> {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c1", DataType::Int32, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("a2", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ]));++ let outer_batch1 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![10])), // matches filter+ ],+ )?;+ let outer_batch2 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![2])),+ Arc::new(Int32Array::from(vec![1])), // same key+ Arc::new(Int32Array::from(vec![20])), // doesn't match+ ],+ )?;++ let inner_batch = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(vec![100])),+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![10])),+ ],+ )?;++ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(+ vec![outer_batch1, outer_batch2],+ vec![false, true], // Pending before 2nd outer batch+ ));+ let inner: SendableRecordBatchStream =+ Box::pin(PendingStream::new(vec![inner_batch], vec![false]));++ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("c1", 0)),+ Operator::Eq,+ Arc::new(Column::new("c2", 1)),+ )),+ vec![+ ColumnIndex {+ index: 2,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 2,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ])),+ );++ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];++ let metrics = ExecutionPlanMetricsSet::new();+ let inner_schema = inner.schema();+ let (reservation, spill_manager, runtime_env) =+ test_stream_resources(inner_schema, &metrics);+ let stream = BitwiseSortMergeJoinStream::try_new(+ left_schema,+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ on_outer,+ on_inner,+ Some(filter),+ LeftSemi,+ 8192,+ 0,+ &metrics,+ reservation,+ spill_manager,+ runtime_env,+ )?;++ let batches = collect_stream(stream).await?;+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(+ total, 1,+ "LeftSemi filtered boundary: only first outer row (c1=10) matches \+ filter c1==c2. Got {total} rows."+ );+ Ok(())+}++// ── Bitwise stream spill tests ─────────────────────────────────────────────++/// Exercises inner key group spilling under memory pressure.+///+/// Uses a tiny memory limit (100 bytes) with disk spilling enabled. Since our+/// operator only buffers inner rows when a filter is present, this test includes+/// a filter (c1 < c2, always true). Verifies:+/// 1. Spill metrics are recorded (spill_count, spilled_bytes, spilled_rows > 0)+/// 2. Results match a non-spilled run+#[tokio::test]+async fn bitwise_spill_with_filter() -> Result<()> {+ let left = build_table(+ ("a1", &vec![1, 2, 3, 4, 5, 6]),+ ("b1", &vec![1, 2, 3, 4, 5, 6]),+ ("c1", &vec![4, 5, 6, 7, 8, 9]),+ );+ let right = build_table(+ ("a2", &vec![10, 20, 30, 40, 50]),+ ("b1", &vec![1, 3, 4, 6, 8]),+ ("c2", &vec![50, 60, 70, 80, 90]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,+ )];+ let sort_options = vec![SortOptions::default(); on.len()];++ // c1 < c2 is always true for matching keys+ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("c1", 0)),+ Operator::Lt,+ Arc::new(Column::new("c2", 1)),+ )),+ vec![+ ColumnIndex {+ index: 2,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 2,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ])),+ );++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(100, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()?;++ for batch_size in [1, 50] {+ let session_config = SessionConfig::default().with_batch_size(batch_size);++ for join_type in [LeftSemi, LeftAnti, RightSemi, RightAnti] {+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(session_config.clone())+ .with_runtime(Arc::clone(&runtime)),+ );++ let join = SortMergeJoinExec::try_new(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ Some(filter.clone()),+ join_type,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join.execute(0, task_ctx)?;+ let spilled_result = common::collect(stream).await.unwrap();++ assert!(+ join.metrics().is_some(),+ "metrics missing for {join_type:?}"+ );+ let metrics = join.metrics().unwrap();+ assert!(+ metrics.spill_count().unwrap() > 0,+ "expected spill_count > 0 for {join_type:?}, batch_size={batch_size}"+ );+ assert!(+ metrics.spilled_bytes().unwrap() > 0,+ "expected spilled_bytes > 0 for {join_type:?}, batch_size={batch_size}"+ );+ assert!(+ metrics.spilled_rows().unwrap() > 0,+ "expected spilled_rows > 0 for {join_type:?}, batch_size={batch_size}"+ );++ // Run without spilling and compare results+ let task_ctx_no_spill = Arc::new(+ TaskContext::default().with_session_config(session_config.clone()),+ );+ let join_no_spill = SortMergeJoinExec::try_new(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),+ Some(filter.clone()),+ join_type,+ sort_options.clone(),+ NullEquality::NullEqualsNothing,+ )?;+ let stream = join_no_spill.execute(0, task_ctx_no_spill)?;+ let no_spill_result = common::collect(stream).await.unwrap();++ let no_spill_metrics = join_no_spill.metrics().unwrap();+ assert_eq!(+ no_spill_metrics.spill_count(),+ Some(0),+ "unexpected spill for {join_type:?} without memory limit"+ );++ assert_eq!(+ spilled_result, no_spill_result,+ "spilled vs non-spilled results differ for {join_type:?}, batch_size={batch_size}"+ );+ }+ }++ Ok(())+}++/// Reproduces a bug where `resume_boundary` for the Filtered pending case+/// only checks `inner_key_buffer.is_empty()` but ignores `inner_key_spill`.+/// After spilling, the in-memory buffer is cleared while the spill file+/// holds the data. If the outer key group spans a batch boundary, the+/// second outer batch's rows are never evaluated against the inner group.+///+/// Setup:+/// - Outer: 2 single-row batches, both key=1, c1=[10, 10]+/// - Inner: 1 batch with many rows all key=1 (enough to trigger spill)+/// - Filter: c1 == c2 (matches when c2=10)+/// - Memory limit: tiny (100 bytes) to force spilling+/// - Pending before 2nd outer batch to trigger boundary re-entry+///+/// Expected: both outer rows match (semi=2 rows, anti=0 rows)+/// Bug: second outer row is skipped because resume_boundary sees empty+/// inner_key_buffer and skips re-evaluation.+#[tokio::test]+async fn spill_filtered_boundary_loses_outer_rows() -> Result<()> {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("a1", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c1", DataType::Int32, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("a2", DataType::Int32, false),+ Field::new("b1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ]));++ // Two single-row outer batches with the same key -- key group spans boundary+ let outer_batch1 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![1])),+ Arc::new(Int32Array::from(vec![1])), // key=1+ Arc::new(Int32Array::from(vec![10])), // matches filter+ ],+ )?;+ let outer_batch2 = RecordBatch::try_new(+ Arc::clone(&left_schema),+ vec![+ Arc::new(Int32Array::from(vec![2])),+ Arc::new(Int32Array::from(vec![1])), // same key=1+ Arc::new(Int32Array::from(vec![10])), // also matches filter+ ],+ )?;++ // Inner: many rows with key=1 to force spilling, followed by key=2.+ // c2=10 so the filter c1==c2 passes for both outer rows.+ // The key=2 row ensures the inner cursor advances past the key group+ // (buffer_inner_key_group returns Ok(false) instead of Ok(true)).+ let n_inner = 200;+ let mut inner_a = vec![100; n_inner];+ inner_a.push(101);+ let mut inner_b = vec![1; n_inner];+ inner_b.push(2); // different key -- forces inner cursor past key=1+ let mut inner_c = vec![10; n_inner];+ inner_c.push(10);+ let inner_batch = RecordBatch::try_new(+ Arc::clone(&right_schema),+ vec![+ Arc::new(Int32Array::from(inner_a)),+ Arc::new(Int32Array::from(inner_b)),+ Arc::new(Int32Array::from(inner_c)),+ ],+ )?;++ // Filter: c1 == c2+ let filter = JoinFilter::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("c1", 0)),+ Operator::Eq,+ Arc::new(Column::new("c2", 1)),+ )),+ vec![+ ColumnIndex {+ index: 2,+ side: JoinSide::Left,+ },+ ColumnIndex {+ index: 2,+ side: JoinSide::Right,+ },+ ],+ Arc::new(Schema::new(vec![+ Field::new("c1", DataType::Int32, false),+ Field::new("c2", DataType::Int32, false),+ ])),+ );++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(100, 1.0)+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()?;++ let on_outer: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];+ let on_inner: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("b1", 1))];++ for join_type in [LeftSemi, LeftAnti] {+ let outer: SendableRecordBatchStream = Box::pin(PendingStream::new(+ vec![outer_batch1.clone(), outer_batch2.clone()],+ vec![false, true], // Pending before 2nd outer batch+ ));+ let inner: SendableRecordBatchStream =+ Box::pin(PendingStream::new(vec![inner_batch.clone()], vec![false]));++ let metrics = ExecutionPlanMetricsSet::new();+ let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool);+ let spill_manager = SpillManager::new(+ Arc::clone(&runtime),+ SpillMetrics::new(&metrics, 0),+ Arc::clone(&right_schema),+ );++ let stream = BitwiseSortMergeJoinStream::try_new(+ Arc::clone(&left_schema),+ vec![SortOptions::default()],+ NullEquality::NullEqualsNothing,+ outer,+ inner,+ on_outer.clone(),+ on_inner.clone(),+ Some(filter.clone()),+ join_type,+ 8192,+ 0,+ &metrics,+ reservation,+ spill_manager,+ Arc::clone(&runtime),+ )?;++ let batches = collect_stream(stream).await?;+ let total: usize = batches.iter().map(|b| b.num_rows()).sum();++ match join_type {+ LeftSemi => {+ assert_eq!(+ total, 2,+ "LeftSemi spill+boundary: both outer rows match filter, \+ expected 2 rows, got {total}"+ );+ }+ LeftAnti => {+ assert_eq!(+ total, 0,+ "LeftAnti spill+boundary: both outer rows match filter, \+ expected 0 rows, got {total}"+ );+ }+ _ => unreachable!(),+ }+ }++ Ok(())+}diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.sltindex 15f867d900..6244ab70c1 100644--- a/datafusion/sqllogictest/test_files/sort_merge_join.slt+++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt@@ -941,6 +941,36 @@ WHERE t2.a > 3 OR t2.a IN (SELECT t3.x FROM t3 WHERE t2.b < 150)----4 101 1001+# Regression test for https://github.com/apache/datafusion/issues/21197+# LeftMark join with cross-table join filter on non-nullable columns.+# The OR forces a LeftMark join. The cross-table filter (t2_data <> t1_data)+# becomes a join filter referencing both sides. At scale, unmatched rows in+# the mark join produce null right-side indices, and fetch_right_columns_by_idxs+# maps those to null values in columns the schema declares non-nullable,+# causing RecordBatch::try_new to fail.+query I+WITH t1_sorted AS (+ SELECT value % 1000 as key, value as data+ FROM range(10000)+ ORDER BY key, data+),+t2_sorted AS (+ SELECT value % 1000 as key, value as data+ FROM range(100000)+ ORDER BY key, data+)+SELECT count(*)+FROM t1_sorted+WHERE t1_sorted.data < 0+ OR EXISTS (+ SELECT 1 FROM t2_sorted+ WHERE t2_sorted.key = t1_sorted.key+ AND t2_sorted.data <> t1_sorted.data+ AND t2_sorted.data % 100 = 0+)+----+100+# Config resetstatement okRESET datafusion.optimizer.prefer_hash_join;