Apache DataFusion SQL Query Engine
fix(smj): compare all key types across batch boundaries in semi/anti/mark joins (#26830)
doraaf752e5d5620
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 6630be5e5d..5c1dd58d51 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -110,7 +110,7 @@ use std::cmp::Ordering;use std::sync::Arc;use crate::EmptyRecordBatchStream;-use crate::joins::utils::{JoinFilter, JoinKeyComparator, compare_join_arrays};+use crate::joins::utils::{JoinFilter, JoinKeyComparator};use crate::metrics::{BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, Time,};@@ -1215,30 +1215,15 @@ fn keys_match(null_equality: NullEquality,) -> Result<bool> {debug_assert!(left_arrays.iter().all(|a| a.len() == 1));- if left_arrays- .iter()- .any(|array| array.data_type().is_floating())- {- // The scalar comparator's partial_cmp panics on NaN. Match the merge- // scan's floating-point equality, including its signed-zero handling.- let right_keys = slice_keys(right_arrays, 0);- return Ok(JoinKeyComparator::new(- left_arrays,- &right_keys,- sort_options,- null_equality,- )?- .is_equal(0, 0));- }- let cmp = compare_join_arrays(- left_arrays,- 0,- right_arrays,- 0,- sort_options,- null_equality,- )?;- Ok(cmp == Ordering::Equal)+ // Use the merge scan's comparator so every key type it supports, and its+ // floating-point equality (NaN, signed zero), also applies across batches.+ // Slicing the right side to one row keeps construction independent of the+ // batch size.+ let right_keys = slice_keys(right_arrays, 0);+ Ok(+ JoinKeyComparator::new(left_arrays, &right_keys, sort_options, null_equality)?+ .is_equal(0, 0),+ )}/// Evaluate the join filter for one inner row against a slice of outer rows.diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rsindex ac93f2638a..207113a733 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs@@ -5268,32 +5268,114 @@ async fn join_float_key_batch_boundaries(#[values(false, true)] descending: bool,#[values(1, 2)] input_batch_size: usize,) -> Result<()> {- let input = |unmatched_key| -> Result<Arc<dyn ExecutionPlan>> {- let mut keys = vec![+ let keys = |unmatched_key| {+ arrow::compute::cast(+ &arrow::array::Float64Array::from(vec![+ None,+ Some(f64::NEG_INFINITY),+ Some(-0.0),+ Some(0.0),+ Some(unmatched_key),+ Some(f64::INFINITY),+ Some(f64::NAN),+ Some(f64::NAN),+ Some(f64::NAN),+ ]),+ &key_type,+ )+ };+ assert_key_batch_boundaries(+ keys(1.0)?,+ keys(2.0)?,+ join_type,+ null_equality,+ descending,+ input_batch_size,+ )+ .await+}++// Key types `compare_join_arrays` did not support (#26003) must also compare+// across batches.+#[rstest::rstest]+#[tokio::test]+async fn join_key_type_batch_boundaries(+ #[values(+ DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, Some("UTC".into())),+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),+ DataType::Decimal256(40, 2),+ DataType::Time32(arrow::datatypes::TimeUnit::Second),+ DataType::Time64(arrow::datatypes::TimeUnit::Nanosecond)+ )]+ key_type: DataType,+ #[values(LeftMark, RightMark, LeftSemi, RightSemi, LeftAnti, RightAnti)]+ join_type: JoinType,+ #[values(NullEquality::NullEqualsNothing, NullEquality::NullEqualsNull)]+ null_equality: NullEquality,+ #[values(false, true)] descending: bool,+ #[values(1, 2)] input_batch_size: usize,+) -> Result<()> {+ // Single-digit keys so Utf8 dictionary values sort like the integers.+ let keys = |unmatched_key| -> Result<arrow::array::ArrayRef> {+ let keys = arrow::array::Int64Array::from(vec![None,- Some(f64::NEG_INFINITY),- Some(-0.0),- Some(0.0),+ Some(1),+ Some(2),+ Some(3),Some(unmatched_key),- Some(f64::INFINITY),- Some(f64::NAN),- Some(f64::NAN),- Some(f64::NAN),- ];+ Some(6),+ Some(7),+ Some(7),+ Some(7),+ ]);+ // Int64 cannot be cast to Time32 directly.+ let keys = match key_type {+ DataType::Time32(_) => arrow::compute::cast(&keys, &DataType::Int32)?,+ _ => Arc::new(keys),+ };+ Ok(arrow::compute::cast(&keys, &key_type)?)+ };+ assert_key_batch_boundaries(+ keys(4)?,+ keys(5)?,+ join_type,+ null_equality,+ descending,+ input_batch_size,+ )+ .await+}++/// Joins two inputs of nine ascending keys split into `input_batch_size`-row+/// batches. Row 0 must be null, row 4 must differ between the sides, and all+/// remaining rows must match.+async fn assert_key_batch_boundaries(+ left_keys: arrow::array::ArrayRef,+ right_keys: arrow::array::ArrayRef,+ join_type: JoinType,+ null_equality: NullEquality,+ descending: bool,+ input_batch_size: usize,+) -> Result<()> {+ let key_type = left_keys.data_type().clone();+ let input = |mut keys: arrow::array::ArrayRef| -> Result<Arc<dyn ExecutionPlan>> {let mut ids: Vec<i32> = (0..keys.len() as i32).collect();if descending {- keys.reverse();ids.reverse();+ keys = arrow::compute::take(+ &keys,+ &arrow::array::UInt32Array::from_iter_values(+ (0..keys.len() as u32).rev(),+ ),+ None,+ )?;}let batch = RecordBatch::try_from_iter(vec![("id",Arc::new(Int32Array::from(ids)) as arrow::array::ArrayRef,),- (- "key",- arrow::compute::cast(&arrow::array::Float64Array::from(keys), &key_type)?,- ),+ ("key", keys),])?;Ok(build_table_from_batches((0..batch.num_rows())@@ -5309,8 +5391,8 @@ async fn join_float_key_batch_boundaries(Arc::new(Column::new("key", 1)),)];let (_, output) = join_collect_with_options(- input(1.0)?,- input(2.0)?,+ input(left_keys)?,+ input(right_keys)?,on,join_type,vec![SortOptions {diff --git a/datafusion/sqllogictest/test_files/sort_merge_join.slt b/datafusion/sqllogictest/test_files/sort_merge_join.sltindex c24a2383df..be394db7dc 100644--- a/datafusion/sqllogictest/test_files/sort_merge_join.slt+++ b/datafusion/sqllogictest/test_files/sort_merge_join.slt@@ -1054,6 +1054,81 @@ DROP TABLE smj_nocol_a;statement okDROP TABLE smj_nocol_b;+# Regression test for #26003: semi/anti/mark joins must compare key groups+# that cross an input batch boundary for key types `compare_join_arrays`+# did not handle.+statement ok+CREATE TABLE smj_tz_events AS+ SELECT arrow_cast(to_timestamp(value), 'Timestamp(Microsecond, Some("UTC"))') AS ts+ FROM generate_series(1, 100000);++statement ok+CREATE TABLE smj_tz_alerts AS+ SELECT arrow_cast(to_timestamp(value * 7), 'Timestamp(Microsecond, Some("UTC"))') AS ts+ FROM generate_series(1, 20000);++query I+SELECT count(*) FROM smj_tz_events e WHERE EXISTS (SELECT 1 FROM smj_tz_alerts a WHERE a.ts = e.ts);+----+14285++query I+SELECT count(*) FROM smj_tz_events e WHERE e.ts IN (SELECT ts FROM smj_tz_alerts);+----+14285++query I+SELECT count(*) FROM smj_tz_events e WHERE NOT EXISTS (SELECT 1 FROM smj_tz_alerts a WHERE a.ts = e.ts);+----+85715++statement ok+DROP TABLE smj_tz_events;++statement ok+DROP TABLE smj_tz_alerts;++statement ok+set datafusion.execution.batch_size = 2;++statement ok+CREATE TABLE smj_key_l AS SELECT+ arrow_cast(k, 'Dictionary(Int32, Utf8)') AS d,+ arrow_cast(k, 'Decimal256(40, 2)') AS dec,+ arrow_cast(arrow_cast(k, 'Int64'), 'Time64(Nanosecond)') AS t+FROM (VALUES ('1'), ('2'), ('2'), ('2'), ('3'), ('4')) v(k);++statement ok+CREATE TABLE smj_key_r AS SELECT+ arrow_cast(k, 'Dictionary(Int32, Utf8)') AS d,+ arrow_cast(k, 'Decimal256(40, 2)') AS dec,+ arrow_cast(arrow_cast(k, 'Int64'), 'Time64(Nanosecond)') AS t+FROM (VALUES ('2'), ('2'), ('2'), ('4'), ('5')) v(k);++query I+SELECT count(*) FROM smj_key_l l WHERE EXISTS (SELECT 1 FROM smj_key_r r WHERE r.d = l.d);+----+4++query I+SELECT count(*) FROM smj_key_l l WHERE NOT EXISTS (SELECT 1 FROM smj_key_r r WHERE r.dec = l.dec);+----+2++query I+SELECT count(*) FROM smj_key_l l WHERE l.t IN (SELECT t FROM smj_key_r);+----+4++statement ok+DROP TABLE smj_key_l;++statement ok+DROP TABLE smj_key_r;++statement ok+RESET datafusion.execution.batch_size;+# Config resetstatement okRESET datafusion.optimizer.prefer_hash_join;