Skip to content

Apache DataFusion SQL Query Engine

fix(smj): handle floating-point keys across batch boundaries (#25134)

Jeff Wang2abf69d4a8e6
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
index 4ee963beb7..43f1041db5 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
@@ -1193,6 +1193,21 @@ fn keys_match(
null_equality: NullEquality,
) -> Result<bool> {
debug_assert!(left_arrays.iter().all(|a| a.len() == 1));
+ if left_arrays
+ .iter()
+ .any(|array| array.data_type().is_floating())
+ {
+ // The scalar comparator's partial_cmp panics on NaN. Match the merge
+ // scan's floating-point equality, including its signed-zero handling.
+ let right_keys = slice_keys(right_arrays, 0);
+ return Ok(JoinKeyComparator::new(
+ left_arrays,
+ &right_keys,
+ sort_options,
+ null_equality,
+ )?
+ .is_equal(0, 0));
+ }
let cmp = compare_join_arrays(
left_arrays,
0,
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
index bab4850860..4d817df738 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs
@@ -4353,6 +4353,120 @@ fn columns(schema: &Schema) -> Vec<String> {
schema.fields().iter().map(|f| f.name().clone()).collect()
}
+// Floating-point key groups must compare consistently within and across batches.
+#[rstest::rstest]
+#[tokio::test]
+async fn join_float_key_batch_boundaries(
+ #[values(DataType::Float16, DataType::Float32, DataType::Float64)] key_type: DataType,
+ #[values(LeftMark, RightMark, LeftSemi, RightSemi, LeftAnti, RightAnti)]
+ join_type: JoinType,
+ #[values(NullEquality::NullEqualsNothing, NullEquality::NullEqualsNull)]
+ null_equality: NullEquality,
+ #[values(false, true)] descending: bool,
+ #[values(1, 2)] input_batch_size: usize,
+) -> Result<()> {
+ let input = |unmatched_key| -> Result<Arc<dyn ExecutionPlan>> {
+ let mut keys = vec![
+ None,
+ Some(f64::NEG_INFINITY),
+ Some(-0.0),
+ Some(0.0),
+ Some(unmatched_key),
+ Some(f64::INFINITY),
+ Some(f64::NAN),
+ Some(f64::NAN),
+ Some(f64::NAN),
+ ];
+ let mut ids: Vec<i32> = (0..keys.len() as i32).collect();
+ if descending {
+ keys.reverse();
+ ids.reverse();
+ }
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "id",
+ Arc::new(Int32Array::from(ids)) as arrow::array::ArrayRef,
+ ),
+ (
+ "key",
+ arrow::compute::cast(&arrow::array::Float64Array::from(keys), &key_type)?,
+ ),
+ ])?;
+ Ok(build_table_from_batches(
+ (0..batch.num_rows())
+ .step_by(input_batch_size)
+ .map(|offset| {
+ batch.slice(offset, input_batch_size.min(batch.num_rows() - offset))
+ })
+ .collect(),
+ ))
+ };
+ let on: JoinOn = vec![(
+ Arc::new(Column::new("key", 1)),
+ Arc::new(Column::new("key", 1)),
+ )];
+ let (_, output) = join_collect_with_options(
+ input(1.0)?,
+ input(2.0)?,
+ on,
+ join_type,
+ vec![SortOptions {
+ descending,
+ nulls_first: !descending,
+ }],
+ null_equality,
+ )
+ .await?;
+
+ let is_mark = matches!(join_type, LeftMark | RightMark);
+ let is_anti = matches!(join_type, LeftAnti | RightAnti);
+ let mut expected: Vec<_> = (0..9)
+ .map(|id| {
+ let matched =
+ id != 4 && (id != 0 || null_equality == NullEquality::NullEqualsNull);
+ (id, matched)
+ })
+ .filter(|(_, matched)| is_mark || *matched != is_anti)
+ .collect();
+ if descending {
+ expected.reverse();
+ }
+ let mut ids = vec![];
+ let mut marks = vec![];
+ for batch in output {
+ assert_eq!(batch.schema().field(1).data_type(), &key_type);
+ ids.extend_from_slice(
+ batch
+ .column(0)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap()
+ .values(),
+ );
+ if is_mark {
+ marks.extend(
+ batch
+ .column(2)
+ .as_any()
+ .downcast_ref::<BooleanArray>()
+ .unwrap()
+ .iter(),
+ );
+ }
+ }
+ assert_eq!(ids, expected.iter().map(|(id, _)| *id).collect::<Vec<_>>());
+ if is_mark {
+ assert_eq!(
+ marks,
+ expected
+ .iter()
+ .map(|(_, matched)| Some(*matched))
+ .collect::<Vec<_>>()
+ );
+ }
+ Ok(())
+}
+
// ==================== BitwiseSortMergeJoinStream direct tests ====================
//
// These tests construct a BitwiseSortMergeJoinStream directly (bypassing exec)