Apache DataFusion SQL Query Engine
fix: Return `probe_side.len()` for RightMark/Anti count(*) queries (#20710)
Jonathan Chen00e36e8b721b
diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 8af26c1b8a..b31982ea3b 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -713,6 +713,7 @@ impl HashJoinStream {filter,JoinSide::Left,None,+ self.join_type,)?} else {(left_indices, right_indices)@@ -781,6 +782,7 @@ impl HashJoinStream {&right_indices,&self.column_indices,join_side,+ self.join_type,)?;let push_status = self.output_buffer.push_batch(batch)?;@@ -899,6 +901,7 @@ impl HashJoinStream {&right_side,&self.column_indices,JoinSide::Left,+ self.join_type,)?;let push_status = self.output_buffer.push_batch(batch)?;diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex eeb160ae51..4429d1e3fb 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -959,6 +959,7 @@ pub(crate) fn build_side_determined_results(&probe_indices,column_indices,build_hash_joiner.build_side,+ join_type,).map(|batch| (batch.num_rows() > 0).then_some(batch))} else {@@ -1022,6 +1023,7 @@ pub(crate) fn join_with_probe_batch(filter,build_hash_joiner.build_side,None,+ join_type,)?} else {(build_indices, probe_indices)@@ -1060,6 +1062,7 @@ pub(crate) fn join_with_probe_batch(&probe_indices,column_indices,build_hash_joiner.build_side,+ join_type,).map(|batch| (batch.num_rows() > 0).then_some(batch))}diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 34993fcdbd..cf4bf2cd16 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -910,6 +910,7 @@ pub(crate) fn get_final_indices_from_bit_map((left_indices, right_indices)}+#[expect(clippy::too_many_arguments)]pub(crate) fn apply_join_filter_to_indices(build_input_buffer: &RecordBatch,probe_batch: &RecordBatch,@@ -918,6 +919,7 @@ pub(crate) fn apply_join_filter_to_indices(filter: &JoinFilter,build_side: JoinSide,max_intermediate_size: Option<usize>,+ join_type: JoinType,) -> Result<(UInt64Array, UInt32Array)> {if build_indices.is_empty() && probe_indices.is_empty() {return Ok((build_indices, probe_indices));@@ -938,6 +940,7 @@ pub(crate) fn apply_join_filter_to_indices(&probe_indices.slice(i, len),filter.column_indices(),build_side,+ join_type,)?;let filter_result = filter.expression()@@ -959,6 +962,7 @@ pub(crate) fn apply_join_filter_to_indices(&probe_indices,filter.column_indices(),build_side,+ join_type,)?;filter@@ -990,6 +994,7 @@ fn new_empty_schema_batch(schema: &Schema, row_count: usize) -> Result<RecordBat/// Returns a new [RecordBatch] by combining the `left` and `right` according to `indices`./// The resulting batch has [Schema] `schema`.+#[expect(clippy::too_many_arguments)]pub(crate) fn build_batch_from_indices(schema: &Schema,build_input_buffer: &RecordBatch,@@ -998,9 +1003,17 @@ pub(crate) fn build_batch_from_indices(probe_indices: &UInt32Array,column_indices: &[ColumnIndex],build_side: JoinSide,+ join_type: JoinType,) -> Result<RecordBatch> {if schema.fields().is_empty() {- return new_empty_schema_batch(schema, build_indices.len());+ // For RightAnti and RightSemi joins, after `adjust_indices_by_join_type`+ // the build_indices were untouched so only probe_indices hold the actual+ // row count.+ let row_count = match join_type {+ JoinType::RightAnti | JoinType::RightSemi => probe_indices.len(),+ _ => build_indices.len(),+ };+ return new_empty_schema_batch(schema, row_count);}// build the columns of the new [RecordBatch]:diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex 282d7c374b..228918c385 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -5318,3 +5318,46 @@ DROP TABLE issue_20437_small;statement count 0DROP TABLE issue_20437_large;++# Test count(*) with right semi/anti joins returns correct row counts+# issue: https://github.com/apache/datafusion/issues/20669++statement ok+CREATE TABLE t1 (k INT, v INT);++statement ok+CREATE TABLE t2 (k INT, v INT);++statement ok+INSERT INTO t1 SELECT i AS k, i AS v FROM generate_series(1, 100) t(i);++statement ok+INSERT INTO t2 VALUES (1, 1);++query I+WITH t AS (+ SELECT *+ FROM t1+ LEFT ANTI JOIN t2 ON t1.k = t2.k+)+SELECT count(*)+FROM t;+----+99++query I+WITH t AS (+ SELECT *+ FROM t1+ LEFT SEMI JOIN t2 ON t1.k = t2.k+)+SELECT count(*)+FROM t;+----+1++statement count 0+DROP TABLE t1;++statement count 0+DROP TABLE t2;