Skip to content

Apache DataFusion SQL Query Engine

fix(smj): compare all key types across batch boundaries in semi/anti/mark joins (#26830)

doraaf752e5d5620
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
index 6630be5e5d..5c1dd58d51 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
@@ -110,7 +110,7 @@ use std::cmp::Ordering;
use std::sync::Arc;
use crate::EmptyRecordBatchStream;
-use crate::joins::utils::{JoinFilter, JoinKeyComparator, compare_join_arrays};
+use crate::joins::utils::{JoinFilter, JoinKeyComparator};
use crate::metrics::{
BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, Time,
};
@@ -1215,30 +1215,15 @@ fn keys_match(
null_equality: NullEquality,
) -> Result<bool> {
debug_assert!(left_arrays.iter().all(|a| a.len() == 1));
- if left_arrays
- .iter()
- .any(|array| array.data_type().is_floating())
- {
- // The scalar comparator's partial_cmp panics on NaN. Match the merge
- // scan's floating-point equality, including its signed-zero handling.
- let right_keys = slice_keys(right_arrays, 0);
- return Ok(JoinKeyComparator::new(
- left_arrays,
- &right_keys,
- sort_options,
- null_equality,
- )?
- .is_equal(0, 0));
- }
- let cmp = compare_join_arrays(
- left_arrays,
- 0,
- right_arrays,
- 0,
- sort_options,
- null_equality,
- )?;
- Ok(cmp == Ordering::Equal)
+ // Use the merge scan's comparator so every key type it supports, and its
+ // floating-point equality (NaN, signed zero), also applies across batches.
+ // Slicing the right side to one row keeps construction independent of the
+ // batch size.
+ let right_keys = slice_keys(right_arrays, 0);
+ Ok(
+ JoinKeyComparator::new(left_arrays, &right_keys, sort_options, null_equality)?
+ .is_equal(0, 0),
+ )
}
/// Evaluate the join filter for one inner row against a slice of outer rows.
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index ac93f2638a..207113a733 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -5268,32 +5268,114 @@ async fn join_float_key_batch_boundaries(
#[values(false, true)] descending: bool,
#[values(1, 2)] input_batch_size: usize,
) -> Result<()> {
- let input = |unmatched_key| -> Result<Arc<dyn ExecutionPlan>> {
- let mut keys = vec![
+ let keys = |unmatched_key| {
+ arrow::compute::cast(
+ &arrow::array::Float64Array::from(vec![
+ None,
+ Some(f64::NEG_INFINITY),
+ Some(-0.0),
+ Some(0.0),
+ Some(unmatched_key),
+ Some(f64::INFINITY),
+ Some(f64::NAN),
+ Some(f64::NAN),
+ Some(f64::NAN),
+ ]),
+ &key_type,
+ )
+ };
+ assert_key_batch_boundaries(
+ keys(1.0)?,
+ keys(2.0)?,
+ join_type,
+ null_equality,
+ descending,
+ input_batch_size,
+ )
+ .await
+}
+
+// Key types `compare_join_arrays` did not support (#26003) must also compare
+// across batches.
+#[rstest::rstest]
+#[tokio::test]
+async fn join_key_type_batch_boundaries(
+ #[values(
+ DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, Some("UTC".into())),
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+ DataType::Decimal256(40, 2),
+ DataType::Time32(arrow::datatypes::TimeUnit::Second),
+ DataType::Time64(arrow::datatypes::TimeUnit::Nanosecond)
+ )]
+ key_type: DataType,
+ #[values(LeftMark, RightMark, LeftSemi, RightSemi, LeftAnti, RightAnti)]
+ join_type: JoinType,
+ #[values(NullEquality::NullEqualsNothing, NullEquality::NullEqualsNull)]
+ null_equality: NullEquality,
+ #[values(false, true)] descending: bool,
+ #[values(1, 2)] input_batch_size: usize,
+) -> Result<()> {
+ // Single-digit keys so Utf8 dictionary values sort like the integers.
+ let keys = |unmatched_key| -> Result<arrow::array::ArrayRef> {
+ let keys = arrow::array::Int64Array::from(vec![
None,
- Some(f64::NEG_INFINITY),
- Some(-0.0),
- Some(0.0),
+ Some(1),
+ Some(2),
+ Some(3),
Some(unmatched_key),
- Some(f64::INFINITY),
- Some(f64::NAN),
- Some(f64::NAN),
- Some(f64::NAN),
- ];
+ Some(6),
+ Some(7),
+ Some(7),
+ Some(7),
+ ]);
+ // Int64 cannot be cast to Time32 directly.
+ let keys = match key_type {
+ DataType::Time32(_) => arrow::compute::cast(&keys, &DataType::Int32)?,
+ _ => Arc::new(keys),
+ };
+ Ok(arrow::compute::cast(&keys, &key_type)?)
+ };
+ assert_key_batch_boundaries(
+ keys(4)?,
+ keys(5)?,
+ join_type,
+ null_equality,
+ descending,
+ input_batch_size,
+ )
+ .await
+}
+
+/// Joins two inputs of nine ascending keys split into `input_batch_size`-row
+/// batches. Row 0 must be null, row 4 must differ between the sides, and all
+/// remaining rows must match.
+async fn assert_key_batch_boundaries(
+ left_keys: arrow::array::ArrayRef,
+ right_keys: arrow::array::ArrayRef,
+ join_type: JoinType,
+ null_equality: NullEquality,
+ descending: bool,
+ input_batch_size: usize,
+) -> Result<()> {
+ let key_type = left_keys.data_type().clone();
+ let input = |mut keys: arrow::array::ArrayRef| -> Result<Arc<dyn ExecutionPlan>> {
let mut ids: Vec<i32> = (0..keys.len() as i32).collect();
if descending {
- keys.reverse();
ids.reverse();
+ keys = arrow::compute::take(
+ &keys,
+ &arrow::array::UInt32Array::from_iter_values(
+ (0..keys.len() as u32).rev(),
+ ),
+ None,
+ )?;
}
let batch = RecordBatch::try_from_iter(vec![
(
"id",
Arc::new(Int32Array::from(ids)) as arrow::array::ArrayRef,
),
- (
- "key",
- arrow::compute::cast(&arrow::array::Float64Array::from(keys), &key_type)?,
- ),
+ ("key", keys),
])?;
Ok(build_table_from_batches(
(0..batch.num_rows())
@@ -5309,8 +5391,8 @@ async fn join_float_key_batch_boundaries(
Arc::new(Column::new("key", 1)),
)];
let (_, output) = join_collect_with_options(
- input(1.0)?,
- input(2.0)?,
+ input(left_keys)?,
+ input(right_keys)?,
on,
join_type,
vec![SortOptions {
diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.slt
index c24a2383df..be394db7dc 100644
--- a/datafusion/sqllogictest/test_files/sort_merge_join.slt
+++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt
@@ -1054,6 +1054,81 @@ DROP TABLE smj_nocol_a;
statement ok
DROP TABLE smj_nocol_b;
+# Regression test for #26003: semi/anti/mark joins must compare key groups
+# that cross an input batch boundary for key types `compare_join_arrays`
+# did not handle.
+statement ok
+CREATE TABLE smj_tz_events AS
+ SELECT arrow_cast(to_timestamp(value), 'Timestamp(Microsecond, Some("UTC"))') AS ts
+ FROM generate_series(1, 100000);
+
+statement ok
+CREATE TABLE smj_tz_alerts AS
+ SELECT arrow_cast(to_timestamp(value * 7), 'Timestamp(Microsecond, Some("UTC"))') AS ts
+ FROM generate_series(1, 20000);
+
+query I
+SELECT count(*) FROM smj_tz_events e WHERE EXISTS (SELECT 1 FROM smj_tz_alerts a WHERE a.ts = e.ts);
+----
+14285
+
+query I
+SELECT count(*) FROM smj_tz_events e WHERE e.ts IN (SELECT ts FROM smj_tz_alerts);
+----
+14285
+
+query I
+SELECT count(*) FROM smj_tz_events e WHERE NOT EXISTS (SELECT 1 FROM smj_tz_alerts a WHERE a.ts = e.ts);
+----
+85715
+
+statement ok
+DROP TABLE smj_tz_events;
+
+statement ok
+DROP TABLE smj_tz_alerts;
+
+statement ok
+set datafusion.execution.batch_size = 2;
+
+statement ok
+CREATE TABLE smj_key_l AS SELECT
+ arrow_cast(k, 'Dictionary(Int32, Utf8)') AS d,
+ arrow_cast(k, 'Decimal256(40, 2)') AS dec,
+ arrow_cast(arrow_cast(k, 'Int64'), 'Time64(Nanosecond)') AS t
+FROM (VALUES ('1'), ('2'), ('2'), ('2'), ('3'), ('4')) v(k);
+
+statement ok
+CREATE TABLE smj_key_r AS SELECT
+ arrow_cast(k, 'Dictionary(Int32, Utf8)') AS d,
+ arrow_cast(k, 'Decimal256(40, 2)') AS dec,
+ arrow_cast(arrow_cast(k, 'Int64'), 'Time64(Nanosecond)') AS t
+FROM (VALUES ('2'), ('2'), ('2'), ('4'), ('5')) v(k);
+
+query I
+SELECT count(*) FROM smj_key_l l WHERE EXISTS (SELECT 1 FROM smj_key_r r WHERE r.d = l.d);
+----
+4
+
+query I
+SELECT count(*) FROM smj_key_l l WHERE NOT EXISTS (SELECT 1 FROM smj_key_r r WHERE r.dec = l.dec);
+----
+2
+
+query I
+SELECT count(*) FROM smj_key_l l WHERE l.t IN (SELECT t FROM smj_key_r);
+----
+4
+
+statement ok
+DROP TABLE smj_key_l;
+
+statement ok
+DROP TABLE smj_key_r;
+
+statement ok
+RESET datafusion.execution.batch_size;
+
# Config reset
statement ok
RESET datafusion.optimizer.prefer_hash_join;