Skip to content

Apache DataFusion SQL Query Engine

Fix: Align sort_merge_join filter output with join schema to fix right-anti panic (#18800)

Kumar Ujjawal434a23b1b6ed
diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rs
index b6a5e18da1..24b564da6e 100644
--- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs
+++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs
@@ -245,14 +245,16 @@ async fn test_right_semi_join_1k() {
#[tokio::test]
async fn test_right_semi_join_1k_filtered() {
- JoinFuzzTestCase::new(
- make_staggered_batches_i32(1000, false),
- make_staggered_batches_i32(1000, false),
- JoinType::RightSemi,
- Some(Box::new(col_lt_col_filter)),
- )
- .run_test(&[HjSmj, NljHj], false)
- .await
+ for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
+ JoinFuzzTestCase::new(
+ make_staggered_batches_i32(1000, left_extra),
+ make_staggered_batches_i32(1000, right_extra),
+ JoinType::RightSemi,
+ Some(Box::new(col_lt_col_filter)),
+ )
+ .run_test(&[HjSmj, NljHj], false)
+ .await
+ }
}
#[tokio::test]
@@ -299,14 +301,16 @@ async fn test_right_anti_join_1k() {
#[tokio::test]
async fn test_right_anti_join_1k_filtered() {
- JoinFuzzTestCase::new(
- make_staggered_batches_i32(1000, false),
- make_staggered_batches_i32(1000, false),
- JoinType::RightAnti,
- Some(Box::new(col_lt_col_filter)),
- )
- .run_test(&[HjSmj, NljHj], false)
- .await
+ for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
+ JoinFuzzTestCase::new(
+ make_staggered_batches_i32(1000, left_extra),
+ make_staggered_batches_i32(1000, right_extra),
+ JoinType::RightAnti,
+ Some(Box::new(col_lt_col_filter)),
+ )
+ .run_test(&[HjSmj, NljHj], false)
+ .await
+ }
}
#[tokio::test]
@@ -564,26 +568,30 @@ async fn test_left_anti_join_1k_binary_filtered() {
#[tokio::test]
async fn test_right_anti_join_1k_binary() {
- JoinFuzzTestCase::new(
- make_staggered_batches_binary(1000, false),
- make_staggered_batches_binary(1000, false),
- JoinType::RightAnti,
- None,
- )
- .run_test(&[HjSmj, NljHj], false)
- .await
+ for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
+ JoinFuzzTestCase::new(
+ make_staggered_batches_binary(1000, left_extra),
+ make_staggered_batches_binary(1000, right_extra),
+ JoinType::RightAnti,
+ None,
+ )
+ .run_test(&[HjSmj, NljHj], false)
+ .await
+ }
}
#[tokio::test]
async fn test_right_anti_join_1k_binary_filtered() {
- JoinFuzzTestCase::new(
- make_staggered_batches_binary(1000, false),
- make_staggered_batches_binary(1000, false),
- JoinType::RightAnti,
- Some(Box::new(col_lt_col_filter)),
- )
- .run_test(&[HjSmj, NljHj], false)
- .await
+ for (left_extra, right_extra) in [(true, true), (false, true), (true, false)] {
+ JoinFuzzTestCase::new(
+ make_staggered_batches_binary(1000, left_extra),
+ make_staggered_batches_binary(1000, right_extra),
+ JoinType::RightAnti,
+ Some(Box::new(col_lt_col_filter)),
+ )
+ .run_test(&[HjSmj, NljHj], false)
+ .await
+ }
}
#[tokio::test]
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
index e1d002a436..e266f1b5b7 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/stream.rs
@@ -1595,14 +1595,16 @@ impl SortMergeJoinStream {
&self.schema,
&[filtered_record_batch, null_joined_streamed_batch],
)?;
- } else if matches!(self.join_type, JoinType::LeftSemi | JoinType::LeftAnti) {
+ } else if matches!(
+ self.join_type,
+ JoinType::LeftSemi
+ | JoinType::LeftAnti
+ | JoinType::RightAnti
+ | JoinType::RightSemi
+ ) {
let output_column_indices = (0..left_columns_length).collect::<Vec<_>>();
filtered_record_batch =
filtered_record_batch.project(&output_column_indices)?;
- } else if matches!(self.join_type, JoinType::RightAnti | JoinType::RightSemi) {
- let output_column_indices = (0..right_columns_length).collect::<Vec<_>>();
- filtered_record_batch =
- filtered_record_batch.project(&output_column_indices)?;
} else if matches!(self.join_type, JoinType::Full)
&& corrected_mask.false_count() > 0
{
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index f91bffbed7..2e4725995b 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -1109,6 +1109,61 @@ async fn join_right_anti_two_with_filter() -> Result<()> {
Ok(())
}
+#[tokio::test]
+async fn join_right_anti_filtered_with_mismatched_columns() -> Result<()> {
+ let left = build_table_two_cols(("a1", &vec![31, 31]), ("b1", &vec![32, 33]));
+ let right = build_table(
+ ("a2", &vec![31, 31]),
+ ("b2", &vec![32, 35]),
+ ("c2", &vec![108, 109]),
+ );
+ let on = vec![
+ (
+ Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,
+ ),
+ (
+ Arc::new(Column::new_with_schema("b1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("b2", &right.schema())?) as _,
+ ),
+ ];
+
+ let filter = JoinFilter::new(
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("b1", 0)),
+ Operator::LtEq,
+ Arc::new(Column::new("c2", 1)),
+ )),
+ vec![
+ ColumnIndex {
+ index: 1,
+ side: JoinSide::Left,
+ },
+ ColumnIndex {
+ index: 2,
+ side: JoinSide::Right,
+ },
+ ],
+ Arc::new(Schema::new(vec![
+ Field::new("b1", DataType::Int32, false),
+ Field::new("c2", DataType::Int32, false),
+ ])),
+ );
+
+ let (_, batches) =
+ join_collect_with_filter(left, right, on, filter, RightAnti).await?;
+
+ let expected = [
+ "+----+----+-----+",
+ "| a2 | b2 | c2 |",
+ "+----+----+-----+",
+ "| 31 | 35 | 109 |",
+ "+----+----+-----+",
+ ];
+ assert_batches_eq!(expected, &batches);
+ Ok(())
+}
+
#[tokio::test]
async fn join_right_anti_with_nulls() -> Result<()> {
let left = build_table_i32_nullable(