Skip to content

Apache DataFusion SQL Query Engine

fix: evaluate sort-merge join filters that reference no columns (#26010)

MrHard201099bae4a6
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
index 96bb9fa291..cdb9d1b8e2 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
@@ -130,7 +130,9 @@ use crate::metrics::{
use crate::spill::in_progress_spill_file::InProgressSpillFile;
use crate::spill::spill_manager::SpillManager;
use crate::stream::{ObservedStream, RecordBatchStreamAdapter};
-use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch};
+use arrow::array::{
+ Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch, RecordBatchOptions,
+};
use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch, not};
use arrow::datatypes::SchemaRef;
use arrow::util::bit_chunk_iterator::UnalignedBitChunk;
@@ -1306,7 +1308,11 @@ fn evaluate_filter_for_inner_row(
}
}
- let filter_batch = RecordBatch::try_new(Arc::clone(filter.schema()), columns)?;
+ let filter_batch = RecordBatch::try_new_with_options(
+ Arc::clone(filter.schema()),
+ columns,
+ &RecordBatchOptions::new().with_row_count(Some(num_outer_rows)),
+ )?;
let result = filter
.expression()
.evaluate(&filter_batch)?
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
index 892be4a2f3..99146627e9 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs
@@ -47,6 +47,7 @@ use arrow::compute::{
take_arrays,
};
use arrow::datatypes::{Schema, SchemaRef};
+use arrow::record_batch::RecordBatchOptions;
use datafusion_common::cast::as_uint64_array;
use datafusion_common::instant::Instant;
use datafusion_common::{
@@ -1704,73 +1705,76 @@ impl MaterializingSortMergeJoinStream {
};
let output_batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
- if !filter_columns.is_empty() {
- if let Some(f) = &self.filter {
- let schema = self.filter_schema.as_ref().unwrap_or_else(|| f.schema());
- let filter_batch =
- RecordBatch::try_new(Arc::clone(schema), filter_columns)?;
- let filter_result = f
- .expression()
- .evaluate(&filter_batch)?
- .into_array(filter_batch.num_rows())?;
-
- let filter_result_mask =
- datafusion_common::cast::as_boolean_array(&filter_result)?;
-
- // Convert NULL filter results to false — NULL means "not satisfied"
- // per SQL semantics, same as Left/Right outer joins.
- let mask = if filter_result_mask.null_count() > 0 {
- compute::prep_null_mask_filter(filter_result_mask)
- } else {
- filter_result_mask.clone()
- };
+ // A filter that reads no columns (e.g. a bound parameter or a volatile
+ // function) still has to be evaluated, so key on the filter itself.
+ if let Some(f) = &self.filter {
+ let schema = self.filter_schema.as_ref().unwrap_or_else(|| f.schema());
+ let filter_batch = RecordBatch::try_new_with_options(
+ Arc::clone(schema),
+ filter_columns,
+ &RecordBatchOptions::new().with_row_count(Some(total_matched_rows)),
+ )?;
+ let filter_result = f
+ .expression()
+ .evaluate(&filter_batch)?
+ .into_array(filter_batch.num_rows())?;
+
+ let filter_result_mask =
+ datafusion_common::cast::as_boolean_array(&filter_result)?;
+
+ // Convert NULL filter results to false — NULL means "not satisfied"
+ // per SQL semantics, same as Left/Right outer joins.
+ let mask = if filter_result_mask.null_count() > 0 {
+ compute::prep_null_mask_filter(filter_result_mask)
+ } else {
+ filter_result_mask.clone()
+ };
- if self.deferred_filtering {
- self.joined_record_batches.push_batch_with_filter_metadata(
- output_batch,
- &combined_left_indices,
- &mask,
- self.streamed_batch_counter,
- self.join_type,
- );
- } else {
- let filtered_batch = filter_record_batch(&output_batch, &mask)?;
- self.joined_record_batches
- .push_batch_without_metadata(filtered_batch);
- }
+ if self.deferred_filtering {
+ self.joined_record_batches.push_batch_with_filter_metadata(
+ output_batch,
+ &combined_left_indices,
+ &mask,
+ self.streamed_batch_counter,
+ self.join_type,
+ );
+ } else {
+ let filtered_batch = filter_record_batch(&output_batch, &mask)?;
+ self.joined_record_batches
+ .push_batch_without_metadata(filtered_batch);
+ }
- // Track which buffered rows had all filter matches fail,
- // so full join can emit them as null-joined later.
- if self.join_type == JoinType::Full {
- let mut offset = 0usize;
- for (batch_idx, _left, right) in matched_chunks {
- let chunk_len = right.len();
- let buffered_batch = &mut self.buffered_data.batches[*batch_idx];
-
- for i in 0..chunk_len {
- if right.is_null(i) {
- continue;
+ // Track which buffered rows had all filter matches fail,
+ // so full join can emit them as null-joined later.
+ if self.join_type == JoinType::Full {
+ let mut offset = 0usize;
+ for (batch_idx, _left, right) in matched_chunks {
+ let chunk_len = right.len();
+ let buffered_batch = &mut self.buffered_data.batches[*batch_idx];
+
+ for i in 0..chunk_len {
+ if right.is_null(i) {
+ continue;
+ }
+ let idx = right.value(i) as usize;
+ match buffered_batch.join_filter_status[idx] {
+ FilterState::SomePassed => {}
+ _ if mask.value(offset + i) => {
+ buffered_batch.join_filter_status[idx] =
+ FilterState::SomePassed;
}
- let idx = right.value(i) as usize;
- match buffered_batch.join_filter_status[idx] {
- FilterState::SomePassed => {}
- _ if mask.value(offset + i) => {
- buffered_batch.join_filter_status[idx] =
- FilterState::SomePassed;
- }
- _ => {
- buffered_batch.join_filter_status[idx] =
- FilterState::AllFailed;
- }
+ _ => {
+ buffered_batch.join_filter_status[idx] =
+ FilterState::AllFailed;
}
}
- offset += chunk_len;
}
- debug_assert_eq!(
- offset, total_matched_rows,
- "offset must advance through every chunk exactly once"
- );
+ offset += chunk_len;
}
+ debug_assert_eq!(
+ offset, total_matched_rows,
+ "offset must advance through every chunk exactly once"
+ );
}
} else {
self.joined_record_batches
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index 9a77bc08f2..35ae5354e5 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -1615,6 +1615,43 @@ async fn join_left_mark_different_columns_count_with_filter() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn join_semi_anti_mark_column_free_filter() -> Result<()> {
+ for join_type in [
+ LeftSemi, LeftAnti, LeftMark, RightSemi, RightAnti, RightMark,
+ ] {
+ let left = build_table(
+ ("a1", &vec![1, 2, 3]),
+ ("b1", &vec![4, 5, 7]),
+ ("c1", &vec![7, 8, 9]),
+ );
+ let right = build_table(
+ ("a2", &vec![10, 20, 30]),
+ ("b1", &vec![4, 5, 6]),
+ ("c2", &vec![70, 80, 90]),
+ );
+ let on = vec![(
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,
+ )];
+ // A filter that reads no columns and rejects every pair.
+ let filter = JoinFilter::new(
+ Arc::new(Literal::new(ScalarValue::Boolean(Some(false)))),
+ vec![],
+ Arc::new(Schema::empty()),
+ );
+ let (_, batches) =
+ join_collect_with_filter(left, right, on, filter, join_type).await?;
+ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+ let expected = match join_type {
+ LeftSemi | RightSemi => 0,
+ _ => 3,
+ };
+ assert_eq!(rows, expected, "{join_type:?}");
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn join_right_mark_different_columns_count_with_filter() -> Result<()> {
// select *
diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.slt
index 5c563c8e81..6fd1d77ec3 100644
--- a/datafusion/sqllogictest/test_files/sort_merge_join.slt
+++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt
@@ -1029,6 +1029,44 @@ DROP TABLE c;
statement ok
set datafusion.execution.target_partitions = 4;
+# Join filter that reads no columns: it must still be evaluated for every
+# candidate pair (hash join already does), so FULL JOIN null-pads both sides.
+statement ok
+CREATE TABLE smj_nocol_a AS SELECT * FROM (VALUES (1, 10), (2, 20), (3, 30)) t(k, v);
+
+statement ok
+CREATE TABLE smj_nocol_b AS SELECT * FROM (VALUES (1, 100), (2, 200), (4, 400)) t(k, w);
+
+statement ok
+set datafusion.execution.target_partitions = 2;
+
+query IIII
+SELECT * FROM smj_nocol_a a FULL JOIN smj_nocol_b b ON a.k = b.k AND random() > 2 ORDER BY a.k, b.k;
+----
+1 10 NULL NULL
+2 20 NULL NULL
+3 30 NULL NULL
+NULL NULL 1 100
+NULL NULL 2 200
+NULL NULL 4 400
+
+query IIII
+SELECT * FROM smj_nocol_a a FULL JOIN smj_nocol_b b ON a.k = b.k AND random() < 2 ORDER BY a.k, b.k;
+----
+1 10 1 100
+2 20 2 200
+3 30 NULL NULL
+NULL NULL 4 400
+
+statement ok
+set datafusion.execution.target_partitions = 4;
+
+statement ok
+DROP TABLE smj_nocol_a;
+
+statement ok
+DROP TABLE smj_nocol_b;
+
# Config reset
statement ok
RESET datafusion.optimizer.prefer_hash_join;