Skip to content

Apache DataFusion SQL Query Engine

refactor: Extract sort-merge join filter logic into separate module (#19614)

Liang-Chi Hsieh1736fd2a40b6
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs b/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs
new file mode 100644
index 0000000000..d598442b65
--- /dev/null
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/filter.rs
@@ -0,0 +1,595 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Filter handling for Sort-Merge Join
+//!
+//! This module encapsulates the complexity of join filter evaluation, including:
+//! - Immediate filtering for INNER joins
+//! - Deferred filtering for outer/semi/anti/mark joins
+//! - Metadata tracking for grouping output rows by input row
+//! - Correcting filter masks to handle multiple matches per input row
+
+use std::sync::Arc;
+
+use arrow::array::{
+ Array, ArrayBuilder, ArrayRef, BooleanArray, BooleanBuilder, RecordBatch,
+ UInt64Array, UInt64Builder,
+};
+use arrow::compute::{self, concat_batches, filter_record_batch};
+use arrow::datatypes::SchemaRef;
+use datafusion_common::{JoinSide, JoinType, Result};
+
+use crate::joins::utils::JoinFilter;
+
+/// Metadata for tracking filter results during deferred filtering
+///
+/// When a join filter is present and we need to ensure each input row produces
+/// at least one output (outer joins) or exactly one output (semi joins), we can't
+/// filter immediately. Instead, we accumulate all joined rows with metadata,
+/// then post-process to determine which rows to output.
+#[derive(Debug)]
+pub struct FilterMetadata {
+ /// Did each output row pass the join filter?
+ /// Used to detect if an input row found ANY match
+ pub filter_mask: BooleanBuilder,
+
+ /// Which input row (within batch) produced each output row?
+ /// Used for grouping output rows by input row
+ pub row_indices: UInt64Builder,
+
+ /// Which input batch did each output row come from?
+ /// Used to disambiguate row_indices across multiple batches
+ pub batch_ids: Vec<usize>,
+}
+
+impl FilterMetadata {
+ /// Create new empty filter metadata
+ pub fn new() -> Self {
+ Self {
+ filter_mask: BooleanBuilder::new(),
+ row_indices: UInt64Builder::new(),
+ batch_ids: vec![],
+ }
+ }
+
+ /// Returns (row_indices, filter_mask, batch_ids_ref) and clears builders
+ pub fn finish_metadata(&mut self) -> (UInt64Array, BooleanArray, &[usize]) {
+ let row_indices = self.row_indices.finish();
+ let filter_mask = self.filter_mask.finish();
+ (row_indices, filter_mask, &self.batch_ids)
+ }
+
+ /// Add metadata for null-joined rows (no filter applied)
+ pub fn append_nulls(&mut self, num_rows: usize) {
+ self.filter_mask.append_nulls(num_rows);
+ self.row_indices.append_nulls(num_rows);
+ self.batch_ids.resize(
+ self.batch_ids.len() + num_rows,
+ 0, // batch_id = 0 for null-joined rows
+ );
+ }
+
+ /// Add metadata for filtered rows
+ pub fn append_filter_metadata(
+ &mut self,
+ row_indices: &UInt64Array,
+ filter_mask: &BooleanArray,
+ batch_id: usize,
+ ) {
+ debug_assert_eq!(
+ row_indices.len(),
+ filter_mask.len(),
+ "row_indices and filter_mask must have same length"
+ );
+
+ for i in 0..row_indices.len() {
+ if filter_mask.is_null(i) {
+ self.filter_mask.append_null();
+ } else if filter_mask.value(i) {
+ self.filter_mask.append_value(true);
+ } else {
+ self.filter_mask.append_value(false);
+ }
+
+ if row_indices.is_null(i) {
+ self.row_indices.append_null();
+ } else {
+ self.row_indices.append_value(row_indices.value(i));
+ }
+
+ self.batch_ids.push(batch_id);
+ }
+ }
+
+ /// Verify that metadata arrays are aligned (same length)
+ pub fn debug_assert_metadata_aligned(&self) {
+ if self.filter_mask.len() > 0 {
+ debug_assert_eq!(
+ self.filter_mask.len(),
+ self.row_indices.len(),
+ "filter_mask and row_indices must have same length when metadata is used"
+ );
+ debug_assert_eq!(
+ self.filter_mask.len(),
+ self.batch_ids.len(),
+ "filter_mask and batch_ids must have same length when metadata is used"
+ );
+ } else {
+ debug_assert_eq!(
+ self.filter_mask.len(),
+ 0,
+ "filter_mask should be empty when batches is empty"
+ );
+ }
+ }
+}
+
+impl Default for FilterMetadata {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// Determines if a join type needs deferred filtering
+///
+/// Deferred filtering is required when:
+/// - A filter exists AND
+/// - The join type requires ensuring each input row produces at least one output
+/// (or exactly one for semi joins)
+pub fn needs_deferred_filtering(
+ filter: &Option<JoinFilter>,
+ join_type: JoinType,
+) -> bool {
+ filter.is_some()
+ && matches!(
+ join_type,
+ JoinType::Left
+ | JoinType::LeftSemi
+ | JoinType::LeftMark
+ | JoinType::Right
+ | JoinType::RightSemi
+ | JoinType::RightMark
+ | JoinType::LeftAnti
+ | JoinType::RightAnti
+ | JoinType::Full
+ )
+}
+
+/// Gets the arrays which join filters are applied on
+///
+/// Extracts the columns needed for filter evaluation from left and right batch columns
+pub fn get_filter_columns(
+ join_filter: &Option<JoinFilter>,
+ left_columns: &[ArrayRef],
+ right_columns: &[ArrayRef],
+) -> Vec<ArrayRef> {
+ let mut filter_columns = vec![];
+
+ if let Some(f) = join_filter {
+ let left_columns: Vec<ArrayRef> = f
+ .column_indices()
+ .iter()
+ .filter(|col_index| col_index.side == JoinSide::Left)
+ .map(|i| Arc::clone(&left_columns[i.index]))
+ .collect();
+ let right_columns: Vec<ArrayRef> = f
+ .column_indices()
+ .iter()
+ .filter(|col_index| col_index.side == JoinSide::Right)
+ .map(|i| Arc::clone(&right_columns[i.index]))
+ .collect();
+
+ filter_columns.extend(left_columns);
+ filter_columns.extend(right_columns);
+ }
+
+ filter_columns
+}
+
+/// Determines if current index is the last occurrence of a row
+///
+/// Used during filter mask correction to detect row boundaries when grouping
+/// output rows by input row.
+fn last_index_for_row(
+ row_index: usize,
+ indices: &UInt64Array,
+ batch_ids: &[usize],
+ indices_len: usize,
+) -> bool {
+ debug_assert_eq!(
+ indices.len(),
+ indices_len,
+ "indices.len() should match indices_len parameter"
+ );
+ debug_assert_eq!(
+ batch_ids.len(),
+ indices_len,
+ "batch_ids.len() should match indices_len"
+ );
+ debug_assert!(
+ row_index < indices_len,
+ "row_index {row_index} should be < indices_len {indices_len}",
+ );
+
+ // If this is the last index overall, it's definitely the last for this row
+ if row_index == indices_len - 1 {
+ return true;
+ }
+
+ // Check if next row has different (batch_id, index) pair
+ let current_batch_id = batch_ids[row_index];
+ let next_batch_id = batch_ids[row_index + 1];
+
+ if current_batch_id != next_batch_id {
+ return true;
+ }
+
+ // Same batch_id, check if row index is different
+ // Both current and next should be non-null (already joined rows)
+ if indices.is_null(row_index) || indices.is_null(row_index + 1) {
+ return true;
+ }
+
+ indices.value(row_index) != indices.value(row_index + 1)
+}
+
+/// Corrects the filter mask for joins with deferred filtering
+///
+/// When an input row joins with multiple buffered rows, we get multiple output rows.
+/// This function groups them by input row and applies join-type-specific logic:
+///
+/// - **Outer joins**: Keep first matching row, convert rest to nulls, add null-joined for unmatched
+/// - **Semi joins**: Keep first matching row, discard rest
+/// - **Anti joins**: Keep row only if NO matches passed filter
+/// - **Mark joins**: Like semi but first match only
+///
+/// # Arguments
+/// * `join_type` - The type of join being performed
+/// * `row_indices` - Which input row produced each output row
+/// * `batch_ids` - Which batch each output row came from
+/// * `filter_mask` - Whether each output row passed the filter
+/// * `expected_size` - Total number of input rows (for adding unmatched)
+///
+/// # Returns
+/// Corrected mask indicating which rows to include in final output:
+/// - `true`: Include this row
+/// - `false`: Convert to null-joined row (outer joins) or include as unmatched (anti joins)
+/// - `null`: Discard this row
+pub fn get_corrected_filter_mask(
+ join_type: JoinType,
+ row_indices: &UInt64Array,
+ batch_ids: &[usize],
+ filter_mask: &BooleanArray,
+ expected_size: usize,
+) -> Option<BooleanArray> {
+ let row_indices_length = row_indices.len();
+ let mut corrected_mask: BooleanBuilder =
+ BooleanBuilder::with_capacity(row_indices_length);
+ let mut seen_true = false;
+
+ match join_type {
+ JoinType::Left | JoinType::Right => {
+ // For outer joins: Keep first matching row per input row,
+ // convert rest to nulls, add null-joined rows for unmatched
+ for i in 0..row_indices_length {
+ let last_index =
+ last_index_for_row(i, row_indices, batch_ids, row_indices_length);
+ if filter_mask.value(i) {
+ seen_true = true;
+ corrected_mask.append_value(true);
+ } else if seen_true || !filter_mask.value(i) && !last_index {
+ corrected_mask.append_null(); // to be ignored and not set to output
+ } else {
+ corrected_mask.append_value(false); // to be converted to null joined row
+ }
+
+ if last_index {
+ seen_true = false;
+ }
+ }
+
+ // Generate null joined rows for records which have no matching join key
+ corrected_mask.append_n(expected_size - corrected_mask.len(), false);
+ Some(corrected_mask.finish())
+ }
+ JoinType::LeftMark | JoinType::RightMark => {
+ // For mark joins: Like outer but only keep first match, mark with boolean
+ for i in 0..row_indices_length {
+ let last_index =
+ last_index_for_row(i, row_indices, batch_ids, row_indices_length);
+ if filter_mask.value(i) && !seen_true {
+ seen_true = true;
+ corrected_mask.append_value(true);
+ } else if seen_true || !filter_mask.value(i) && !last_index {
+ corrected_mask.append_null(); // to be ignored and not set to output
+ } else {
+ corrected_mask.append_value(false); // to be converted to null joined row
+ }
+
+ if last_index {
+ seen_true = false;
+ }
+ }
+
+ // Generate null joined rows for records which have no matching join key
+ corrected_mask.append_n(expected_size - corrected_mask.len(), false);
+ Some(corrected_mask.finish())
+ }
+ JoinType::LeftSemi | JoinType::RightSemi => {
+ // For semi joins: Keep only first matching row per input row, discard rest
+ for i in 0..row_indices_length {
+ let last_index =
+ last_index_for_row(i, row_indices, batch_ids, row_indices_length);
+ if filter_mask.value(i) && !seen_true {
+ seen_true = true;
+ corrected_mask.append_value(true);
+ } else {
+ corrected_mask.append_null(); // to be ignored and not set to output
+ }
+
+ if last_index {
+ seen_true = false;
+ }
+ }
+
+ Some(corrected_mask.finish())
+ }
+ JoinType::LeftAnti | JoinType::RightAnti => {
+ // For anti joins: Keep row only if NO matches passed the filter
+ for i in 0..row_indices_length {
+ let last_index =
+ last_index_for_row(i, row_indices, batch_ids, row_indices_length);
+
+ if filter_mask.value(i) {
+ seen_true = true;
+ }
+
+ if last_index {
+ if !seen_true {
+ corrected_mask.append_value(true);
+ } else {
+ corrected_mask.append_null();
+ }
+
+ seen_true = false;
+ } else {
+ corrected_mask.append_null();
+ }
+ }
+ // Generate null joined rows for records which have no matching join key,
+ // for LeftAnti non-matched considered as true
+ corrected_mask.append_n(expected_size - corrected_mask.len(), true);
+ Some(corrected_mask.finish())
+ }
+ JoinType::Full => {
+ // For full joins: Similar to outer but handle both sides
+ for i in 0..row_indices_length {
+ let last_index =
+ last_index_for_row(i, row_indices, batch_ids, row_indices_length);
+
+ if filter_mask.is_null(i) {
+ // null joined
+ corrected_mask.append_value(true);
+ } else if filter_mask.value(i) {
+ seen_true = true;
+ corrected_mask.append_value(true);
+ } else if seen_true || !filter_mask.value(i) && !last_index {
+ corrected_mask.append_null(); // to be ignored and not set to output
+ } else {
+ corrected_mask.append_value(false); // to be converted to null joined row
+ }
+
+ if last_index {
+ seen_true = false;
+ }
+ }
+ // Generate null joined rows for records which have no matching join key
+ corrected_mask.append_n(expected_size - corrected_mask.len(), false);
+ Some(corrected_mask.finish())
+ }
+ JoinType::Inner => {
+ // Inner joins don't need deferred filtering
+ None
+ }
+ }
+}
+
+/// Applies corrected filter mask to record batch based on join type
+///
+/// Different join types require different handling of filtered results:
+/// - Outer joins: Add null-joined rows for false mask values
+/// - Semi/Anti joins: May need projection to remove right columns
+/// - Full joins: Add null-joined rows for both sides
+pub fn filter_record_batch_by_join_type(
+ record_batch: &RecordBatch,
+ corrected_mask: &BooleanArray,
+ join_type: JoinType,
+ schema: &SchemaRef,
+ streamed_schema: &SchemaRef,
+ buffered_schema: &SchemaRef,
+) -> Result<RecordBatch> {
+ let filtered_record_batch = filter_record_batch(record_batch, corrected_mask)?;
+
+ match join_type {
+ JoinType::Left | JoinType::LeftMark => {
+ // For left joins, add null-joined rows where mask is false
+ let null_mask = compute::not(corrected_mask)?;
+ let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;
+
+ if null_joined_batch.num_rows() == 0 {
+ return Ok(filtered_record_batch);
+ }
+
+ // Create null columns for right side
+ let null_joined_streamed_batch = create_null_joined_batch(
+ &null_joined_batch,
+ buffered_schema,
+ JoinSide::Left,
+ join_type,
+ schema,
+ )?;
+
+ Ok(concat_batches(
+ schema,
+ &[filtered_record_batch, null_joined_streamed_batch],
+ )?)
+ }
+ JoinType::LeftSemi
+ | JoinType::LeftAnti
+ | JoinType::RightSemi
+ | JoinType::RightAnti => {
+ // For semi/anti joins, project to only include the outer side columns
+ // Both Left and Right semi/anti use streamed_schema.len() because:
+ // - For Left: columns are [left, right], so we take first streamed_schema.len()
+ // - For Right: columns are [right, left], and streamed side is right, so we take first streamed_schema.len()
+ let output_column_indices: Vec<usize> =
+ (0..streamed_schema.fields().len()).collect();
+ Ok(filtered_record_batch.project(&output_column_indices)?)
+ }
+ JoinType::Right | JoinType::RightMark => {
+ // For right joins, add null-joined rows where mask is false
+ let null_mask = compute::not(corrected_mask)?;
+ let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;
+
+ if null_joined_batch.num_rows() == 0 {
+ return Ok(filtered_record_batch);
+ }
+
+ // Create null columns for left side (buffered side for RIGHT join)
+ let null_joined_buffered_batch = create_null_joined_batch(
+ &null_joined_batch,
+ buffered_schema, // Pass buffered (left) schema to create nulls for it
+ JoinSide::Right,
+ join_type,
+ schema,
+ )?;
+
+ Ok(concat_batches(
+ schema,
+ &[filtered_record_batch, null_joined_buffered_batch],
+ )?)
+ }
+ JoinType::Full => {
+ // For full joins, add null-joined rows for both sides
+ let joined_filter_not_matched_mask = compute::not(corrected_mask)?;
+ let joined_filter_not_matched_batch =
+ filter_record_batch(record_batch, &joined_filter_not_matched_mask)?;
+
+ if joined_filter_not_matched_batch.num_rows() == 0 {
+ return Ok(filtered_record_batch);
+ }
+
+ // Create null-joined batches for both sides
+ let left_null_joined_batch = create_null_joined_batch(
+ &joined_filter_not_matched_batch,
+ buffered_schema,
+ JoinSide::Left,
+ join_type,
+ schema,
+ )?;
+
+ Ok(concat_batches(
+ schema,
+ &[filtered_record_batch, left_null_joined_batch],
+ )?)
+ }
+ JoinType::Inner => Ok(filtered_record_batch),
+ }
+}
+
+/// Creates a batch with null columns for the non-joined side
+///
+/// Note: The input `batch` is assumed to be a fully-joined batch that already contains
+/// columns from both sides. We need to extract the data side columns and replace the
+/// null side columns with actual nulls.
+fn create_null_joined_batch(
+ batch: &RecordBatch,
+ null_schema: &SchemaRef,
+ join_side: JoinSide,
+ join_type: JoinType,
+ output_schema: &SchemaRef,
+) -> Result<RecordBatch> {
+ let num_rows = batch.num_rows();
+
+ // The input batch is a fully-joined batch [left_cols..., right_cols...]
+ // We need to extract the appropriate side and replace the other with nulls (or mark column)
+ let columns = match (join_side, join_type) {
+ (JoinSide::Left, JoinType::LeftMark) => {
+ // For LEFT mark: output is [left_cols..., mark_col]
+ // Batch is [left_cols..., right_cols...], extract left from beginning
+ // Number of left columns = output columns - 1 (mark column)
+ let left_col_count = output_schema.fields().len() - 1;
+ let mut result: Vec<ArrayRef> = batch.columns()[..left_col_count].to_vec();
+ result.push(Arc::new(BooleanArray::from(vec![false; num_rows])) as ArrayRef);
+ result
+ }
+ (JoinSide::Right, JoinType::RightMark) => {
+ // For RIGHT mark: output is [right_cols..., mark_col]
+ // For RIGHT joins, batch is [right_cols..., left_cols...] (right comes first!)
+ // Extract right columns from the beginning
+ let right_col_count = output_schema.fields().len() - 1; // -1 for mark column
+ let mut result: Vec<ArrayRef> = batch.columns()[..right_col_count].to_vec();
+ result.push(Arc::new(BooleanArray::from(vec![false; num_rows])) as ArrayRef);
+ result
+ }
+ (JoinSide::Left, _) => {
+ // For LEFT join: output is [left_cols..., right_cols...]
+ // Extract left columns, then add null right columns
+ let null_columns: Vec<ArrayRef> = null_schema
+ .fields()
+ .iter()
+ .map(|field| arrow::array::new_null_array(field.data_type(), num_rows))
+ .collect();
+ let left_col_count = output_schema.fields().len() - null_columns.len();
+ let mut result: Vec<ArrayRef> = batch.columns()[..left_col_count].to_vec();
+ result.extend(null_columns);
+ result
+ }
+ (JoinSide::Right, _) => {
+ // For RIGHT join: batch is [left_cols..., right_cols...] (same as schema)
+ // We want: [null_left..., actual_right...]
+ // Extract left columns from beginning, replace with nulls, keep right columns
+ let null_columns: Vec<ArrayRef> = null_schema
+ .fields()
+ .iter()
+ .map(|field| arrow::array::new_null_array(field.data_type(), num_rows))
+ .collect();
+ let left_col_count = null_columns.len();
+ let mut result = null_columns;
+ // Extract right columns starting after left columns
+ result.extend_from_slice(&batch.columns()[left_col_count..]);
+ result
+ }
+ (JoinSide::None, _) => {
+ // This should not happen in normal join operations
+ unreachable!(
+ "JoinSide::None should not be used in null-joined batch creation"
+ )
+ }
+ };
+
+ // Create the batch - don't validate nullability since outer joins can have
+ // null values in columns that were originally non-nullable
+ use arrow::array::RecordBatchOptions;
+ let mut options = RecordBatchOptions::new();
+ options = options.with_row_count(Some(num_rows));
+ Ok(RecordBatch::try_new_with_options(
+ Arc::clone(output_schema),
+ columns,
+ &options,
+ )?)
+}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs b/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs
index 82f18e7414..06290ec4d0 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/mod.rs
@@ -20,6 +20,7 @@
pub use exec::SortMergeJoinExec;
mod exec;
+mod filter;
mod metrics;
mod stream;
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
index 12ade0d92b..edbf542005 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
@@ -33,6 +33,10 @@ use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering::Relaxed;
use std::task::{Context, Poll};
+use crate::joins::sort_merge_join::filter::{
+ FilterMetadata, filter_record_batch_by_join_type, get_corrected_filter_mask,
+ get_filter_columns, needs_deferred_filtering,
+};
use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics;
use crate::joins::utils::{JoinFilter, compare_join_arrays};
use crate::metrics::RecordOutput;
@@ -49,8 +53,8 @@ use arrow::error::ArrowError;
use arrow::ipc::reader::StreamReader;
use datafusion_common::config::SpillCompression;
use datafusion_common::{
- DataFusionError, HashSet, JoinSide, JoinType, NullEquality, Result, exec_err,
- internal_err, not_impl_err,
+ DataFusionError, HashSet, JoinType, NullEquality, Result, exec_err, internal_err,
+ not_impl_err,
};
use datafusion_execution::disk_manager::RefCountedTempFile;
use datafusion_execution::memory_pool::MemoryReservation;
@@ -370,12 +374,8 @@ pub(super) struct SortMergeJoinStream {
pub(super) struct JoinedRecordBatches {
/// Joined batches. Each batch is already joined columns from left and right sources
pub(super) joined_batches: BatchCoalescer,
- /// Did each output row pass the join filter? (detect if input row found any match)
- pub(super) filter_mask: BooleanBuilder,
- /// Which input row (within batch) produced each output row? (for grouping by input row)
- pub(super) row_indices: UInt64Builder,
- /// Which input batch did each output row come from? (disambiguate row_indices)
- pub(super) batch_ids: Vec<usize>,
+ /// Filter metadata for deferred filtering
+ pub(super) filter_metadata: FilterMetadata,
}
impl JoinedRecordBatches {
@@ -398,61 +398,28 @@ impl JoinedRecordBatches {
}
}
- /// Finishes and returns the metadata arrays, clearing the builders
- ///
- /// Returns (row_indices, filter_mask, batch_ids_ref)
- /// Note: batch_ids is returned as a reference since it's still needed in the struct
- fn finish_metadata(&mut self) -> (UInt64Array, BooleanArray, &[usize]) {
- let row_indices = self.row_indices.finish();
- let filter_mask = self.filter_mask.finish();
- (row_indices, filter_mask, &self.batch_ids)
- }
-
/// Clears batches without touching metadata (for early return when no filtering needed)
fn clear_batches(&mut self, schema: &SchemaRef, batch_size: usize) {
self.joined_batches = BatchCoalescer::new(Arc::clone(schema), batch_size)
.with_biggest_coalesce_batch_size(Option::from(batch_size / 2));
}
- /// Asserts that internal metadata arrays are consistent with each other
- /// Only checks if metadata is actually being used (i.e., not all empty)
- #[inline]
- fn debug_assert_metadata_aligned(&self) {
- // Metadata arrays should be aligned IF they're being used
- // (For non-filtered joins, they may all be empty)
- if self.filter_mask.len() > 0
- || self.row_indices.len() > 0
- || !self.batch_ids.is_empty()
- {
- debug_assert_eq!(
- self.filter_mask.len(),
- self.row_indices.len(),
- "filter_mask and row_indices must have same length when metadata is used"
- );
- debug_assert_eq!(
- self.filter_mask.len(),
- self.batch_ids.len(),
- "filter_mask and batch_ids must have same length when metadata is used"
- );
- }
- }
-
/// Asserts that if batches is empty, metadata is also empty
#[inline]
fn debug_assert_empty_consistency(&self) {
if self.joined_batches.is_empty() {
debug_assert_eq!(
- self.filter_mask.len(),
+ self.filter_metadata.filter_mask.len(),
0,
"filter_mask should be empty when batches is empty"
);
debug_assert_eq!(
- self.row_indices.len(),
+ self.filter_metadata.row_indices.len(),
0,
"row_indices should be empty when batches is empty"
);
debug_assert_eq!(
- self.batch_ids.len(),
+ self.filter_metadata.batch_ids.len(),
0,
"batch_ids should be empty when batches is empty"
);
@@ -473,14 +440,9 @@ impl JoinedRecordBatches {
let num_rows = batch.num_rows();
- self.filter_mask.append_nulls(num_rows);
- self.row_indices.append_nulls(num_rows);
- self.batch_ids.resize(
- self.batch_ids.len() + num_rows,
- 0, // batch_id = 0 for null-joined rows
- );
+ self.filter_metadata.append_nulls(num_rows);
- self.debug_assert_metadata_aligned();
+ self.filter_metadata.debug_assert_metadata_aligned();
self.joined_batches
.push_batch(batch)
.expect("Failed to push batch to BatchCoalescer");
@@ -525,13 +487,13 @@ impl JoinedRecordBatches {
"row_indices and filter_mask must have same length"
);
- // For Full joins, we keep the pre_mask (with nulls), for others we keep the cleaned mask
- self.filter_mask.extend(filter_mask);
- self.row_indices.extend(row_indices);
- self.batch_ids
- .resize(self.batch_ids.len() + row_indices.len(), streamed_batch_id);
+ self.filter_metadata.append_filter_metadata(
+ row_indices,
+ filter_mask,
+ streamed_batch_id,
+ );
- self.debug_assert_metadata_aligned();
+ self.filter_metadata.debug_assert_metadata_aligned();
self.joined_batches
.push_batch(batch)
.expect("Failed to push batch to BatchCoalescer");
@@ -551,9 +513,7 @@ impl JoinedRecordBatches {
fn clear(&mut self, schema: &SchemaRef, batch_size: usize) {
self.joined_batches = BatchCoalescer::new(Arc::clone(schema), batch_size)
.with_biggest_coalesce_batch_size(Option::from(batch_size / 2));
- self.batch_ids.clear();
- self.filter_mask = BooleanBuilder::new();
- self.row_indices = UInt64Builder::new();
+ self.filter_metadata = FilterMetadata::new();
self.debug_assert_empty_consistency();
}
}
@@ -563,199 +523,6 @@ impl RecordBatchStream for SortMergeJoinStream {
}
}
-/// True if next index refers to either:
-/// - another batch id
-/// - another row index within same batch id
-/// - end of row indices
-#[inline(always)]
-fn last_index_for_row(
- row_index: usize,
- indices: &UInt64Array,
- batch_ids: &[usize],
- indices_len: usize,
-) -> bool {
- debug_assert_eq!(
- indices.len(),
- indices_len,
- "indices.len() should match indices_len parameter"
- );
- debug_assert_eq!(
- batch_ids.len(),
- indices_len,
- "batch_ids.len() should match indices_len"
- );
- debug_assert!(
- row_index < indices_len,
- "row_index {row_index} should be < indices_len {indices_len}",
- );
-
- row_index == indices_len - 1
- || batch_ids[row_index] != batch_ids[row_index + 1]
- || indices.value(row_index) != indices.value(row_index + 1)
-}
-
-// Returns a corrected boolean bitmask for the given join type
-// Values in the corrected bitmask can be: true, false, null
-// `true` - the row found its match and sent to the output
-// `null` - the row ignored, no output
-// `false` - the row sent as NULL joined row
-pub(super) fn get_corrected_filter_mask(
- join_type: JoinType,
- row_indices: &UInt64Array,
- batch_ids: &[usize],
- filter_mask: &BooleanArray,
- expected_size: usize,
-) -> Option<BooleanArray> {
- let row_indices_length = row_indices.len();
- let mut corrected_mask: BooleanBuilder =
- BooleanBuilder::with_capacity(row_indices_length);
- let mut seen_true = false;
-
- match join_type {
- JoinType::Left | JoinType::Right => {
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- if filter_mask.value(i) {
- seen_true = true;
- corrected_mask.append_value(true);
- } else if seen_true || !filter_mask.value(i) && !last_index {
- corrected_mask.append_null(); // to be ignored and not set to output
- } else {
- corrected_mask.append_value(false); // to be converted to null joined row
- }
-
- if last_index {
- seen_true = false;
- }
- }
-
- // Generate null joined rows for records which have no matching join key
- corrected_mask.append_n(expected_size - corrected_mask.len(), false);
- Some(corrected_mask.finish())
- }
- JoinType::LeftMark | JoinType::RightMark => {
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- if filter_mask.value(i) && !seen_true {
- seen_true = true;
- corrected_mask.append_value(true);
- } else if seen_true || !filter_mask.value(i) && !last_index {
- corrected_mask.append_null(); // to be ignored and not set to output
- } else {
- corrected_mask.append_value(false); // to be converted to null joined row
- }
-
- if last_index {
- seen_true = false;
- }
- }
-
- // Generate null joined rows for records which have no matching join key
- corrected_mask.append_n(expected_size - corrected_mask.len(), false);
- Some(corrected_mask.finish())
- }
- JoinType::LeftSemi | JoinType::RightSemi => {
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- if filter_mask.value(i) && !seen_true {
- seen_true = true;
- corrected_mask.append_value(true);
- } else {
- corrected_mask.append_null(); // to be ignored and not set to output
- }
-
- if last_index {
- seen_true = false;
- }
- }
-
- Some(corrected_mask.finish())
- }
- JoinType::LeftAnti | JoinType::RightAnti => {
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
-
- if filter_mask.value(i) {
- seen_true = true;
- }
-
- if last_index {
- if !seen_true {
- corrected_mask.append_value(true);
- } else {
- corrected_mask.append_null();
- }
-
- seen_true = false;
- } else {
- corrected_mask.append_null();
- }
- }
- // Generate null joined rows for records which have no matching join key,
- // for LeftAnti non-matched considered as true
- corrected_mask.append_n(expected_size - corrected_mask.len(), true);
- Some(corrected_mask.finish())
- }
- JoinType::Full => {
- let mut mask: Vec<Option<bool>> = vec![Some(true); row_indices_length];
- let mut last_true_idx = 0;
- let mut first_row_idx = 0;
- let mut seen_false = false;
-
- for i in 0..row_indices_length {
- let last_index =
- last_index_for_row(i, row_indices, batch_ids, row_indices_length);
- let val = filter_mask.value(i);
- let is_null = filter_mask.is_null(i);
-
- if val {
- // memoize the first seen matched row
- if !seen_true {
- last_true_idx = i;
- }
- seen_true = true;
- }
-
- if is_null || val {
- mask[i] = Some(true);
- } else if !is_null && !val && (seen_true || seen_false) {
- mask[i] = None;
- } else {
- mask[i] = Some(false);
- }
-
- if !is_null && !val {
- seen_false = true;
- }
-
- if last_index {
- // If the left row seen as true its needed to output it once
- // To do that we mark all other matches for same row as null to avoid the output
- if seen_true {
- #[expect(clippy::needless_range_loop)]
- for j in first_row_idx..last_true_idx {
- mask[j] = None;
- }
- }
-
- seen_true = false;
- seen_false = false;
- last_true_idx = 0;
- first_row_idx = i + 1;
- }
- }
-
- Some(BooleanArray::from(mask))
- }
- // Only outer joins needs to keep track of processed rows and apply corrected filter mask
- _ => None,
- }
-}
-
impl Stream for SortMergeJoinStream {
type Item = Result<RecordBatch>;
@@ -778,7 +545,10 @@ impl Stream for SortMergeJoinStream {
match self.current_ordering {
Ordering::Less | Ordering::Equal => {
if !streamed_exhausted {
- if self.needs_deferred_filtering() {
+ if needs_deferred_filtering(
+ &self.filter,
+ self.join_type,
+ ) {
match self.process_filtered_batches()? {
Poll::Ready(Some(batch)) => {
return Poll::Ready(Some(Ok(batch)));
@@ -842,10 +612,12 @@ impl Stream for SortMergeJoinStream {
self.freeze_all()?;
// Verify metadata alignment before checking if we have batches to output
- self.joined_record_batches.debug_assert_metadata_aligned();
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
// For filtered joins, skip output and let Init state handle it
- if self.needs_deferred_filtering() {
+ if needs_deferred_filtering(&self.filter, self.join_type) {
continue;
}
@@ -872,10 +644,12 @@ impl Stream for SortMergeJoinStream {
self.freeze_all()?;
// Verify metadata alignment before final output
- self.joined_record_batches.debug_assert_metadata_aligned();
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
// For filtered joins, must concat and filter ALL data at once
- if self.needs_deferred_filtering()
+ if needs_deferred_filtering(&self.filter, self.join_type)
&& !self.joined_record_batches.joined_batches.is_empty()
{
let record_batch = self.filter_joined_batch()?;
@@ -975,9 +749,7 @@ impl SortMergeJoinStream {
joined_record_batches: JoinedRecordBatches {
joined_batches: BatchCoalescer::new(Arc::clone(&schema), batch_size)
.with_biggest_coalesce_batch_size(Option::from(batch_size / 2)),
- filter_mask: BooleanBuilder::new(),
- row_indices: UInt64Builder::new(),
- batch_ids: vec![],
+ filter_metadata: FilterMetadata::new(),
},
output: BatchCoalescer::new(schema, batch_size)
.with_biggest_coalesce_batch_size(Option::from(batch_size / 2)),
@@ -996,26 +768,6 @@ impl SortMergeJoinStream {
self.streamed_batch.num_output_rows()
}
- /// Returns true if this join needs deferred filtering
- ///
- /// Deferred filtering is needed when a filter exists and the join type requires
- /// ensuring each input row produces at least one output row (or exactly one for semi).
- fn needs_deferred_filtering(&self) -> bool {
- self.filter.is_some()
- && matches!(
- self.join_type,
- JoinType::Left
- | JoinType::LeftSemi
- | JoinType::LeftMark
- | JoinType::Right
- | JoinType::RightSemi
- | JoinType::RightMark
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::Full
- )
- }
-
/// Process accumulated batches for filtered joins
///
/// Freezes unfrozen pairs, applies deferred filtering, and outputs if ready.
@@ -1023,7 +775,9 @@ impl SortMergeJoinStream {
fn process_filtered_batches(&mut self) -> Poll<Option<Result<RecordBatch>>> {
self.freeze_all()?;
- self.joined_record_batches.debug_assert_metadata_aligned();
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
if !self.joined_record_batches.joined_batches.is_empty() {
let out_filtered_batch = self.filter_joined_batch()?;
@@ -1399,7 +1153,9 @@ impl SortMergeJoinStream {
self.freeze_streamed()?;
// After freezing, metadata should be aligned
- self.joined_record_batches.debug_assert_metadata_aligned();
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
Ok(())
}
@@ -1414,7 +1170,9 @@ impl SortMergeJoinStream {
self.freeze_buffered(1)?;
// After freezing, metadata should be aligned
- self.joined_record_batches.debug_assert_metadata_aligned();
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
Ok(())
}
@@ -1543,7 +1301,7 @@ impl SortMergeJoinStream {
&right_indices,
)?;
- get_filter_column(&self.filter, &left_columns, &right_cols)
+ get_filter_columns(&self.filter, &left_columns, &right_cols)
} else if matches!(
self.join_type,
JoinType::RightAnti | JoinType::RightSemi | JoinType::RightMark
@@ -1554,12 +1312,12 @@ impl SortMergeJoinStream {
&right_indices,
)?;
- get_filter_column(&self.filter, &right_cols, &left_columns)
+ get_filter_columns(&self.filter, &right_cols, &left_columns)
} else {
- get_filter_column(&self.filter, &left_columns, &right_columns)
+ get_filter_columns(&self.filter, &left_columns, &right_columns)
}
} else {
- get_filter_column(&self.filter, &right_columns, &left_columns)
+ get_filter_columns(&self.filter, &right_columns, &left_columns)
}
} else {
// This chunk is totally for null joined rows (outer join), we don't need to apply join filter.
@@ -1681,11 +1439,13 @@ impl SortMergeJoinStream {
fn filter_joined_batch(&mut self) -> Result<RecordBatch> {
// Metadata should be aligned before processing
- self.joined_record_batches.debug_assert_metadata_aligned();
+ self.joined_record_batches
+ .filter_metadata
+ .debug_assert_metadata_aligned();
let record_batch = self.joined_record_batches.concat_batches(&self.schema)?;
let (mut out_indices, mut out_mask, mut batch_ids) =
- self.joined_record_batches.finish_metadata();
+ self.joined_record_batches.filter_metadata.finish_metadata();
let default_batch_ids = vec![0; record_batch.num_rows()];
// If only nulls come in and indices sizes doesn't match with expected record batch count
@@ -1756,139 +1516,14 @@ impl SortMergeJoinStream {
record_batch: &RecordBatch,
corrected_mask: &BooleanArray,
) -> Result<RecordBatch> {
- // Corrected mask should have length matching or exceeding record_batch rows
- // (for outer joins it may be longer to include null-joined rows)
- debug_assert!(
- corrected_mask.len() >= record_batch.num_rows(),
- "corrected_mask length ({}) should be >= record_batch rows ({})",
- corrected_mask.len(),
- record_batch.num_rows()
- );
-
- let mut filtered_record_batch =
- filter_record_batch(record_batch, corrected_mask)?;
- let left_columns_length = self.streamed_schema.fields.len();
- let right_columns_length = self.buffered_schema.fields.len();
-
- if matches!(
- self.join_type,
- JoinType::Left | JoinType::LeftMark | JoinType::Right | JoinType::RightMark
- ) {
- let null_mask = compute::not(corrected_mask)?;
- let null_joined_batch = filter_record_batch(record_batch, &null_mask)?;
-
- let mut right_columns = create_unmatched_columns(
- self.join_type,
- &self.buffered_schema,
- null_joined_batch.num_rows(),
- );
-
- let columns = match self.join_type {
- JoinType::Right => {
- // The first columns are the right columns.
- let left_columns = null_joined_batch
- .columns()
- .iter()
- .skip(right_columns_length)
- .cloned()
- .collect::<Vec<_>>();
-
- right_columns.extend(left_columns);
- right_columns
- }
- JoinType::Left | JoinType::LeftMark | JoinType::RightMark => {
- // The first columns are the left columns.
- let mut left_columns = null_joined_batch
- .columns()
- .iter()
- .take(left_columns_length)
- .cloned()
- .collect::<Vec<_>>();
-
- left_columns.extend(right_columns);
- left_columns
- }
- _ => exec_err!("Did not expect join type {}", self.join_type)?,
- };
-
- // Push the streamed/buffered batch joined nulls to the output
- let null_joined_streamed_batch =
- RecordBatch::try_new(Arc::clone(&self.schema), columns)?;
-
- filtered_record_batch = concat_batches(
- &self.schema,
- &[filtered_record_batch, null_joined_streamed_batch],
- )?;
- } else if matches!(
+ let filtered_record_batch = filter_record_batch_by_join_type(
+ record_batch,
+ corrected_mask,
self.join_type,
- JoinType::LeftSemi
- | JoinType::LeftAnti
- | JoinType::RightAnti
- | JoinType::RightSemi
- ) {
- let output_column_indices = (0..left_columns_length).collect::<Vec<_>>();
- filtered_record_batch =
- filtered_record_batch.project(&output_column_indices)?;
- } else if matches!(self.join_type, JoinType::Full)
- && corrected_mask.false_count() > 0
- {
- // Find rows which joined by key but Filter predicate evaluated as false
- let joined_filter_not_matched_mask = compute::not(corrected_mask)?;
- let joined_filter_not_matched_batch =
- filter_record_batch(record_batch, &joined_filter_not_matched_mask)?;
-
- // Add left unmatched rows adding the right side as nulls
- let right_null_columns = self
- .buffered_schema
- .fields()
- .iter()
- .map(|f| {
- new_null_array(
- f.data_type(),
- joined_filter_not_matched_batch.num_rows(),
- )
- })
- .collect::<Vec<_>>();
-
- let mut result_joined = joined_filter_not_matched_batch
- .columns()
- .iter()
- .take(left_columns_length)
- .cloned()
- .collect::<Vec<_>>();
-
- result_joined.extend(right_null_columns);
-
- let left_null_joined_batch =
- RecordBatch::try_new(Arc::clone(&self.schema), result_joined)?;
-
- // Add right unmatched rows adding the left side as nulls
- let mut result_joined = self
- .streamed_schema
- .fields()
- .iter()
- .map(|f| {
- new_null_array(
- f.data_type(),
- joined_filter_not_matched_batch.num_rows(),
- )
- })
- .collect::<Vec<_>>();
-
- let right_data = joined_filter_not_matched_batch
- .columns()
- .iter()
- .skip(left_columns_length)
- .cloned()
- .collect::<Vec<_>>();
-
- result_joined.extend(right_data);
-
- filtered_record_batch = concat_batches(
- &self.schema,
- &[filtered_record_batch, left_null_joined_batch],
- )?;
- }
+ &self.schema,
+ &self.streamed_schema,
+ &self.buffered_schema,
+ )?;
self.joined_record_batches
.clear(&self.schema, self.batch_size);
@@ -1913,36 +1548,6 @@ fn create_unmatched_columns(
}
}
-/// Gets the arrays which join filters are applied on.
-fn get_filter_column(
- join_filter: &Option<JoinFilter>,
- streamed_columns: &[ArrayRef],
- buffered_columns: &[ArrayRef],
-) -> Vec<ArrayRef> {
- let mut filter_columns = vec![];
-
- if let Some(f) = join_filter {
- let left_columns = f
- .column_indices()
- .iter()
- .filter(|col_index| col_index.side == JoinSide::Left)
- .map(|i| Arc::clone(&streamed_columns[i.index]))
- .collect::<Vec<_>>();
-
- let right_columns = f
- .column_indices()
- .iter()
- .filter(|col_index| col_index.side == JoinSide::Right)
- .map(|i| Arc::clone(&buffered_columns[i.index]))
- .collect::<Vec<_>>();
-
- filter_columns.extend(left_columns);
- filter_columns.extend(right_columns);
- }
-
- filter_columns
-}
-
fn produce_buffered_null_batch(
schema: &SchemaRef,
streamed_schema: &SchemaRef,
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index d0bcc79636..85cdcc7e7b 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -29,7 +29,6 @@ use std::sync::Arc;
use arrow::array::{
BinaryArray, BooleanArray, Date32Array, Date64Array, FixedSizeBinaryArray,
Int32Array, RecordBatch, UInt64Array,
- builder::{BooleanBuilder, UInt64Builder},
};
use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch};
use arrow::datatypes::{DataType, Field, Schema};
@@ -51,8 +50,8 @@ use datafusion_physical_expr::expressions::BinaryExpr;
use insta::{allow_duplicates, assert_snapshot};
use crate::{
- expressions::Column,
- joins::sort_merge_join::stream::{JoinedRecordBatches, get_corrected_filter_mask},
+ expressions::Column, joins::sort_merge_join::filter::get_corrected_filter_mask,
+ joins::sort_merge_join::stream::JoinedRecordBatches,
};
use crate::joins::SortMergeJoinExec;
@@ -2375,9 +2374,7 @@ fn build_joined_record_batches() -> Result<JoinedRecordBatches> {
let mut batches = JoinedRecordBatches {
joined_batches: BatchCoalescer::new(Arc::clone(&schema), 8192),
- filter_mask: BooleanBuilder::new(),
- row_indices: UInt64Builder::new(),
- batch_ids: vec![],
+ filter_metadata: crate::joins::sort_merge_join::filter::FilterMetadata::new(),
};
// Insert already prejoined non-filtered rows
@@ -2432,44 +2429,73 @@ fn build_joined_record_batches() -> Result<JoinedRecordBatches> {
)?)?;
let streamed_indices = vec![0, 0];
- batches.batch_ids.extend(vec![0; streamed_indices.len()]);
batches
+ .filter_metadata
+ .batch_ids
+ .extend(vec![0; streamed_indices.len()]);
+ batches
+ .filter_metadata
.row_indices
.extend(&UInt64Array::from(streamed_indices));
let streamed_indices = vec![1];
- batches.batch_ids.extend(vec![0; streamed_indices.len()]);
batches
+ .filter_metadata
+ .batch_ids
+ .extend(vec![0; streamed_indices.len()]);
+ batches
+ .filter_metadata
.row_indices
.extend(&UInt64Array::from(streamed_indices));
let streamed_indices = vec![0, 0];
- batches.batch_ids.extend(vec![1; streamed_indices.len()]);
batches
+ .filter_metadata
+ .batch_ids
+ .extend(vec![1; streamed_indices.len()]);
+ batches
+ .filter_metadata
.row_indices
.extend(&UInt64Array::from(streamed_indices));
let streamed_indices = vec![0];
- batches.batch_ids.extend(vec![2; streamed_indices.len()]);
batches
+ .filter_metadata
+ .batch_ids
+ .extend(vec![2; streamed_indices.len()]);
+ batches
+ .filter_metadata
.row_indices
.extend(&UInt64Array::from(streamed_indices));
let streamed_indices = vec![0, 0];
- batches.batch_ids.extend(vec![3; streamed_indices.len()]);
batches
+ .filter_metadata
+ .batch_ids
+ .extend(vec![3; streamed_indices.len()]);
+ batches
+ .filter_metadata
.row_indices
.extend(&UInt64Array::from(streamed_indices));
batches
+ .filter_metadata
.filter_mask
.extend(&BooleanArray::from(vec![true, false]));
- batches.filter_mask.extend(&BooleanArray::from(vec![true]));
batches
+ .filter_metadata
+ .filter_mask
+ .extend(&BooleanArray::from(vec![true]));
+ batches
+ .filter_metadata
.filter_mask
.extend(&BooleanArray::from(vec![false, true]));
- batches.filter_mask.extend(&BooleanArray::from(vec![false]));
batches
+ .filter_metadata
+ .filter_mask
+ .extend(&BooleanArray::from(vec![false]));
+ batches
+ .filter_metadata
.filter_mask
.extend(&BooleanArray::from(vec![false, false]));
@@ -2482,8 +2508,8 @@ async fn test_left_outer_join_filtered_mask() -> Result<()> {
let schema = joined_batches.joined_batches.schema();
let output = joined_batches.concat_batches(&schema)?;
- let out_mask = joined_batches.filter_mask.finish();
- let out_indices = joined_batches.row_indices.finish();
+ let out_mask = joined_batches.filter_metadata.filter_mask.finish();
+ let out_indices = joined_batches.filter_metadata.row_indices.finish();
assert_eq!(
get_corrected_filter_mask(
@@ -2620,7 +2646,7 @@ async fn test_left_outer_join_filtered_mask() -> Result<()> {
let corrected_mask = get_corrected_filter_mask(
Left,
&out_indices,
- &joined_batches.batch_ids,
+ &joined_batches.filter_metadata.batch_ids,
&out_mask,
output.num_rows(),
)
@@ -2689,8 +2715,8 @@ async fn test_semi_join_filtered_mask() -> Result<()> {
let schema = joined_batches.joined_batches.schema();
let output = joined_batches.concat_batches(&schema)?;
- let out_mask = joined_batches.filter_mask.finish();
- let out_indices = joined_batches.row_indices.finish();
+ let out_mask = joined_batches.filter_metadata.filter_mask.finish();
+ let out_indices = joined_batches.filter_metadata.row_indices.finish();
assert_eq!(
get_corrected_filter_mask(
@@ -2791,7 +2817,7 @@ async fn test_semi_join_filtered_mask() -> Result<()> {
let corrected_mask = get_corrected_filter_mask(
join_type,
&out_indices,
- &joined_batches.batch_ids,
+ &joined_batches.filter_metadata.batch_ids,
&out_mask,
output.num_rows(),
)
@@ -2864,8 +2890,8 @@ async fn test_anti_join_filtered_mask() -> Result<()> {
let schema = joined_batches.joined_batches.schema();
let output = joined_batches.concat_batches(&schema)?;
- let out_mask = joined_batches.filter_mask.finish();
- let out_indices = joined_batches.row_indices.finish();
+ let out_mask = joined_batches.filter_metadata.filter_mask.finish();
+ let out_indices = joined_batches.filter_metadata.row_indices.finish();
assert_eq!(
get_corrected_filter_mask(
@@ -2966,7 +2992,7 @@ async fn test_anti_join_filtered_mask() -> Result<()> {
let corrected_mask = get_corrected_filter_mask(
join_type,
&out_indices,
- &joined_batches.batch_ids,
+ &joined_batches.filter_metadata.batch_ids,
&out_mask,
output.num_rows(),
)