Apache DataFusion SQL Query Engine
fix: evaluate sort-merge join filters that reference no columns (#26010)
MrHard201099bae4a6
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 96bb9fa291..cdb9d1b8e2 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -130,7 +130,9 @@ use crate::metrics::{use crate::spill::in_progress_spill_file::InProgressSpillFile;use crate::spill::spill_manager::SpillManager;use crate::stream::{ObservedStream, RecordBatchStreamAdapter};-use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch};+use arrow::array::{+ Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch, RecordBatchOptions,+};use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch, not};use arrow::datatypes::SchemaRef;use arrow::util::bit_chunk_iterator::UnalignedBitChunk;@@ -1306,7 +1308,11 @@ fn evaluate_filter_for_inner_row(}}- let filter_batch = RecordBatch::try_new(Arc::clone(filter.schema()), columns)?;+ let filter_batch = RecordBatch::try_new_with_options(+ Arc::clone(filter.schema()),+ columns,+ &RecordBatchOptions::new().with_row_count(Some(num_outer_rows)),+ )?;let result = filter.expression().evaluate(&filter_batch)?diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rsindex 892be4a2f3..99146627e9 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs@@ -47,6 +47,7 @@ use arrow::compute::{take_arrays,};use arrow::datatypes::{Schema, SchemaRef};+use arrow::record_batch::RecordBatchOptions;use datafusion_common::cast::as_uint64_array;use datafusion_common::instant::Instant;use datafusion_common::{@@ -1704,73 +1705,76 @@ impl MaterializingSortMergeJoinStream {};let output_batch = RecordBatch::try_new(Arc::clone(&self.schema), columns)?;- if !filter_columns.is_empty() {- if let Some(f) = &self.filter {- let schema = self.filter_schema.as_ref().unwrap_or_else(|| f.schema());- let filter_batch =- RecordBatch::try_new(Arc::clone(schema), filter_columns)?;- let filter_result = f- .expression()- .evaluate(&filter_batch)?- .into_array(filter_batch.num_rows())?;-- let filter_result_mask =- datafusion_common::cast::as_boolean_array(&filter_result)?;-- // Convert NULL filter results to false — NULL means "not satisfied"- // per SQL semantics, same as Left/Right outer joins.- let mask = if filter_result_mask.null_count() > 0 {- compute::prep_null_mask_filter(filter_result_mask)- } else {- filter_result_mask.clone()- };+ // A filter that reads no columns (e.g. a bound parameter or a volatile+ // function) still has to be evaluated, so key on the filter itself.+ if let Some(f) = &self.filter {+ let schema = self.filter_schema.as_ref().unwrap_or_else(|| f.schema());+ let filter_batch = RecordBatch::try_new_with_options(+ Arc::clone(schema),+ filter_columns,+ &RecordBatchOptions::new().with_row_count(Some(total_matched_rows)),+ )?;+ let filter_result = f+ .expression()+ .evaluate(&filter_batch)?+ .into_array(filter_batch.num_rows())?;++ let filter_result_mask =+ datafusion_common::cast::as_boolean_array(&filter_result)?;++ // Convert NULL filter results to false — NULL means "not satisfied"+ // per SQL semantics, same as Left/Right outer joins.+ let mask = if filter_result_mask.null_count() > 0 {+ compute::prep_null_mask_filter(filter_result_mask)+ } else {+ filter_result_mask.clone()+ };- if self.deferred_filtering {- self.joined_record_batches.push_batch_with_filter_metadata(- output_batch,- &combined_left_indices,- &mask,- self.streamed_batch_counter,- self.join_type,- );- } else {- let filtered_batch = filter_record_batch(&output_batch, &mask)?;- self.joined_record_batches- .push_batch_without_metadata(filtered_batch);- }+ if self.deferred_filtering {+ self.joined_record_batches.push_batch_with_filter_metadata(+ output_batch,+ &combined_left_indices,+ &mask,+ self.streamed_batch_counter,+ self.join_type,+ );+ } else {+ let filtered_batch = filter_record_batch(&output_batch, &mask)?;+ self.joined_record_batches+ .push_batch_without_metadata(filtered_batch);+ }- // Track which buffered rows had all filter matches fail,- // so full join can emit them as null-joined later.- if self.join_type == JoinType::Full {- let mut offset = 0usize;- for (batch_idx, _left, right) in matched_chunks {- let chunk_len = right.len();- let buffered_batch = &mut self.buffered_data.batches[*batch_idx];-- for i in 0..chunk_len {- if right.is_null(i) {- continue;+ // Track which buffered rows had all filter matches fail,+ // so full join can emit them as null-joined later.+ if self.join_type == JoinType::Full {+ let mut offset = 0usize;+ for (batch_idx, _left, right) in matched_chunks {+ let chunk_len = right.len();+ let buffered_batch = &mut self.buffered_data.batches[*batch_idx];++ for i in 0..chunk_len {+ if right.is_null(i) {+ continue;+ }+ let idx = right.value(i) as usize;+ match buffered_batch.join_filter_status[idx] {+ FilterState::SomePassed => {}+ _ if mask.value(offset + i) => {+ buffered_batch.join_filter_status[idx] =+ FilterState::SomePassed;}- let idx = right.value(i) as usize;- match buffered_batch.join_filter_status[idx] {- FilterState::SomePassed => {}- _ if mask.value(offset + i) => {- buffered_batch.join_filter_status[idx] =- FilterState::SomePassed;- }- _ => {- buffered_batch.join_filter_status[idx] =- FilterState::AllFailed;- }+ _ => {+ buffered_batch.join_filter_status[idx] =+ FilterState::AllFailed;}}- offset += chunk_len;}- debug_assert_eq!(- offset, total_matched_rows,- "offset must advance through every chunk exactly once"- );+ offset += chunk_len;}+ debug_assert_eq!(+ offset, total_matched_rows,+ "offset must advance through every chunk exactly once"+ );}} else {self.joined_record_batchesdiff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex 9a77bc08f2..35ae5354e5 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -1615,6 +1615,43 @@ async fn join_left_mark_different_columns_count_with_filter() -> Result<()> {Ok(())}+#[tokio::test]+async fn join_semi_anti_mark_column_free_filter() -> Result<()> {+ for join_type in [+ LeftSemi, LeftAnti, LeftMark, RightSemi, RightAnti, RightMark,+ ] {+ let left = build_table(+ ("a1", &vec![1, 2, 3]),+ ("b1", &vec![4, 5, 7]),+ ("c1", &vec![7, 8, 9]),+ );+ let right = build_table(+ ("a2", &vec![10, 20, 30]),+ ("b1", &vec![4, 5, 6]),+ ("c2", &vec![70, 80, 90]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("b1", &right.schema())?) as _,+ )];+ // A filter that reads no columns and rejects every pair.+ let filter = JoinFilter::new(+ Arc::new(Literal::new(ScalarValue::Boolean(Some(false)))),+ vec![],+ Arc::new(Schema::empty()),+ );+ let (_, batches) =+ join_collect_with_filter(left, right, on, filter, join_type).await?;+ let rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ let expected = match join_type {+ LeftSemi | RightSemi => 0,+ _ => 3,+ };+ assert_eq!(rows, expected, "{join_type:?}");+ }+ Ok(())+}+#[tokio::test]async fn join_right_mark_different_columns_count_with_filter() -> Result<()> {// select *diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.sltindex 5c563c8e81..6fd1d77ec3 100644--- a/datafusion/sqllogictest/test_files/sort_merge_join.slt+++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt@@ -1029,6 +1029,44 @@ DROP TABLE c;statement okset datafusion.execution.target_partitions = 4;+# Join filter that reads no columns: it must still be evaluated for every+# candidate pair (hash join already does), so FULL JOIN null-pads both sides.+statement ok+CREATE TABLE smj_nocol_a AS SELECT * FROM (VALUES (1, 10), (2, 20), (3, 30)) t(k, v);++statement ok+CREATE TABLE smj_nocol_b AS SELECT * FROM (VALUES (1, 100), (2, 200), (4, 400)) t(k, w);++statement ok+set datafusion.execution.target_partitions = 2;++query IIII+SELECT * FROM smj_nocol_a a FULL JOIN smj_nocol_b b ON a.k = b.k AND random() > 2 ORDER BY a.k, b.k;+----+1 10 NULL NULL+2 20 NULL NULL+3 30 NULL NULL+NULL NULL 1 100+NULL NULL 2 200+NULL NULL 4 400++query IIII+SELECT * FROM smj_nocol_a a FULL JOIN smj_nocol_b b ON a.k = b.k AND random() < 2 ORDER BY a.k, b.k;+----+1 10 1 100+2 20 2 200+3 30 NULL NULL+NULL NULL 4 400++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+DROP TABLE smj_nocol_a;++statement ok+DROP TABLE smj_nocol_b;+# Config resetstatement okRESET datafusion.optimizer.prefer_hash_join;