Apache DataFusion SQL Query Engine
perf: normalize float hash join build keys once (#25972)
dustin801b017bb882
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex bd8103db0f..ce8ee72493 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -82,6 +82,7 @@ use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::utils::memory::{RecordBatchMemoryCounter, estimate_memory_size, get_record_batch_memory_size,};+use datafusion_common::utils::normalize_float_zero;use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,plan_err, project_schema,@@ -299,7 +300,11 @@ struct JoinBuildData {map: Arc<Map>,/// The input rows for the build side.batch: RecordBatch,- /// Evaluated build-side key expressions.+ /// Evaluated build-side key expressions, with float `-0.0` rewritten to+ /// `+0.0` so probe batches can compare against them without scanning the+ /// build side again. The dynamic filters (`bounds`, `membership`) and+ /// `build_side_has_null` were computed from the original keys in+ /// `collect_left_input`.values: Vec<ArrayRef>,/// Bounds computed from the build side; absent for an empty partition.bounds: Option<PartitionBounds>,@@ -370,7 +375,9 @@ impl JoinLeftData {!self.map().is_empty()}- /// returns a reference to the build side expressions values+ /// Returns the build side key values, with float `-0.0` already rewritten+ /// to `+0.0` for key comparison. The dynamic filters and+ /// `build_side_has_null` were derived from the original keys instead.pub(super) fn values(&self) -> &[ArrayRef] {&self.build.values}@@ -3208,12 +3215,20 @@ async fn collect_left_input((Map::HashMap(hashmap), batch, left_values)};+ // Rewrite float `-0.0` to `+0.0` once here, so the per probe batch key+ // comparison does not rescan the build side. Arrays without `-0.0`, and+ // non-float keys, are shared rather than copied.+ let normalized_values: Vec<ArrayRef> =+ left_values.iter().map(normalize_float_zero).collect();+// Join keys that are plain columns share the buffers of `batch`, any other// expression evaluates to new arrays that are kept for the whole join.+ // The same goes for normalized keys: only a rewritten copy adds memory.let mut key_counter = RecordBatchMemoryCounter::new();key_counter.count_batch(&batch);let keys_size = left_values.iter()+ .chain(&normalized_values).map(|values| key_counter.count_array(values.as_ref())).sum::<usize>();reservation.try_grow(keys_size)?;@@ -3276,7 +3291,7 @@ async fn collect_left_input(let build_indices = UInt64Array::from_iter_values(null_mask.values().set_indices().map(|i| i as u64),);- let scope_values = left_values[1..]+ let scope_values = normalized_values[1..].iter().map(|values| Ok(arrow::compute::filter(values.as_ref(), &null_mask)?)).collect::<Result<Vec<_>>>()?;@@ -3399,7 +3414,7 @@ async fn collect_left_input(build: Arc::new(JoinBuildData {map,batch,- values: left_values,+ values: normalized_values,bounds,membership,reservation,@@ -3488,11 +3503,11 @@ mod tests {use arrow::array::{Array, ArrayRef, AsArray, BinaryViewArray, Date32Array, DictionaryArray,- Float64Array, Int32Array, Int64Array, StringArray, StringViewArray, StructArray,- UInt32Array, UInt64Array,+ Float32Array, Float64Array, Int32Array, Int64Array, StringArray, StringViewArray,+ StructArray, UInt32Array, UInt64Array,};use arrow::buffer::NullBuffer;- use arrow::datatypes::{DataType, Field, Int32Type};+ use arrow::datatypes::{DataType, Field, Float32Type, Float64Type, Int32Type};use datafusion_common::hash_utils::create_hashes;use datafusion_common::stats::Precision;use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};@@ -4257,6 +4272,165 @@ mod tests {Ok(())}+ /// Collects a single-column build side keyed on that column.+ async fn collect_float_build(+ keys: ArrayRef,+ ) -> Result<(JoinLeftData, BuildProbeJoinMetrics)> {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "f",+ keys.data_type().clone(),+ true,+ )]));+ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![keys])?;+ let exec = TestMemoryExec::try_new_exec(&[vec![batch]], schema, None)?;+ let stream = exec.execute(0, Arc::new(TaskContext::default()))?;+ let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());+ let metrics = BuildProbeJoinMetrics::new(0, &ExecutionPlanMetricsSet::new());+ let on_left: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("f", 0))];+ let left_data = collect_left_input(+ RandomState::with_seed(0),+ stream,+ on_left,+ metrics.clone(),+ MemoryConsumer::new("HashJoinInput").register(&pool),+ false,+ 1,+ false,+ Arc::new(ConfigOptions::default()),+ NullEquality::NullEqualsNothing,+ None,+ Count::new(),+ BuildMode::Ordinary,+ )+ .await?;+ Ok((left_data, metrics))+ }++ /// The stored build keys have `-0.0` rewritten to `+0.0`, while the build+ /// batch keeps the original value. The rewritten copy is charged to the+ /// build memory, and keys without `-0.0` are shared with the batch.+ #[tokio::test]+ async fn collect_left_input_stores_normalized_float_keys() -> Result<()> {+ let (with_neg_zero, neg_metrics) =+ collect_float_build(Arc::new(Float64Array::from(vec![-0.0, 1.0]))).await?;+ let stored = with_neg_zero.values()[0].as_primitive::<Float64Type>();+ assert_eq!(stored.value(0).to_bits(), 0.0_f64.to_bits());+ assert_eq!(stored.value(1), 1.0);+ let original = with_neg_zero+ .batch()+ .column(0)+ .as_primitive::<Float64Type>();+ assert_eq!(original.value(0).to_bits(), (-0.0_f64).to_bits());++ let (without_neg_zero, pos_metrics) =+ collect_float_build(Arc::new(Float64Array::from(vec![0.0, 1.0]))).await?;+ assert!(Arc::ptr_eq(+ &without_neg_zero.values()[0],+ without_neg_zero.batch().column(0)+ ));++ // Both builds are the same shape, so they differ only by the copy.+ assert_eq!(+ neg_metrics.build_mem_used.value(),+ pos_metrics.build_mem_used.value() + stored.values().inner().capacity()+ );++ let (f32_build, _) =+ collect_float_build(Arc::new(Float32Array::from(vec![-0.0_f32, 1.0])))+ .await?;+ let stored = f32_build.values()[0].as_primitive::<Float32Type>();+ assert_eq!(stored.value(0).to_bits(), 0.0_f32.to_bits());+ let original = f32_build.batch().column(0).as_primitive::<Float32Type>();+ assert_eq!(original.value(0).to_bits(), (-0.0_f32).to_bits());+ Ok(())+ }++ /// A single Float64 key takes the general comparator path. Build and+ /// probe both hold `-0.0` and `0.0`, which must all match each other,+ /// while the output keeps each side's original sign.+ #[rstest]+ #[tokio::test]+ async fn join_inner_single_float_key_negative_zero(+ #[values(8192, 1)] batch_size: usize,+ #[values(NullEquality::NullEqualsNothing, NullEquality::NullEqualsNull)]+ null_equality: NullEquality,+ ) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ let table = |suffix: &str,+ f: Vec<Option<f64>>,+ v: Vec<i32>|+ -> Result<Arc<dyn ExecutionPlan>> {+ let schema = Arc::new(Schema::new(vec![+ Field::new(format!("f{suffix}"), DataType::Float64, true),+ Field::new(format!("v{suffix}"), DataType::Int32, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Float64Array::from(f)),+ Arc::new(Int32Array::from(v)),+ ],+ )?;+ TestMemoryExec::try_new_exec(&[vec![batch]], schema, None)+ .map(|exec| exec as _)+ };++ let left = table(+ "1",+ vec![Some(-0.0), Some(0.0), Some(1.5), None],+ vec![10, 11, 12, 13],+ )?;+ let right = table(+ "2",+ vec![Some(0.0), Some(-0.0), Some(1.5), None],+ vec![20, 21, 22, 23],+ )?;+ let on = vec![(+ Arc::new(Column::new_with_schema("f1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("f2", &right.schema())?) as _,+ )];++ let (_, batches, _) =+ join_collect(left, right, on, &JoinType::Inner, null_equality, task_ctx)+ .await?;++ // The NULL keys only pair up when NULL equals NULL.+ allow_duplicates! {+ match null_equality {+ NullEquality::NullEqualsNothing => {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +------+----+------+----++ | f1 | v1 | f2 | v2 |+ +------+----+------+----++ | -0.0 | 10 | -0.0 | 21 |+ | -0.0 | 10 | 0.0 | 20 |+ | 0.0 | 11 | -0.0 | 21 |+ | 0.0 | 11 | 0.0 | 20 |+ | 1.5 | 12 | 1.5 | 22 |+ +------+----+------+----++ ");+ }+ NullEquality::NullEqualsNull => {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +------+----+------+----++ | f1 | v1 | f2 | v2 |+ +------+----+------+----++ | | 13 | | 23 |+ | -0.0 | 10 | -0.0 | 21 |+ | -0.0 | 10 | 0.0 | 20 |+ | 0.0 | 11 | -0.0 | 21 |+ | 0.0 | 11 | 0.0 | 20 |+ | 1.5 | 12 | 1.5 | 22 |+ +------+----+------+----++ ");+ }+ }+ }++ Ok(())+ }+/// Two probe batches carry different multi-column keys at the same row/// positions. Each batch has to be compared against its own key arrays;/// a comparator left over from the first batch would drop the second@@ -10425,6 +10599,96 @@ mod tests {Ok(())}+ /// Correlated null-aware `LeftAnti` whose two scope keys are Float64, with+ /// `-0.0` on the build side and `0.0` on the probe side. Both scopes must+ /// still match in every direction: the full key lookup, NULL probe values+ /// against all build rows, and NULL build values against the probe rows.+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_left_anti_float_scope_negative_zero(+ batch_size: usize,+ ) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ fn table(+ id: Vec<Option<i32>>,+ g1: Vec<f64>,+ g2: Vec<f64>,+ ) -> Arc<dyn ExecutionPlan> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, true),+ Field::new("g1", DataType::Float64, true),+ Field::new("g2", DataType::Float64, true),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(id)),+ Arc::new(Float64Array::from(g1)),+ Arc::new(Float64Array::from(g2)),+ ],+ )+ .unwrap();+ TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap()+ }++ let left = table(+ vec![Some(1), Some(3), None, Some(4), Some(5), None, Some(6)],+ vec![-0.0, -0.0, -0.0, 2.0, -0.0, 3.0, 2.0],+ vec![1.0, 1.0, 1.0, -0.0, -0.0, -0.0, 1.0],+ );+ let right = table(+ vec![Some(1), Some(2), None, Some(7)],+ vec![0.0, 0.0, 2.0, 2.0],+ vec![1.0, 1.0, 0.0, 1.0],+ );++ let on = ["id", "g1", "g2"]+ .into_iter()+ .map(|name| {+ Ok((+ Arc::new(Column::new_with_schema(name, &left.schema())?) as _,+ Arc::new(Column::new_with_schema(name, &right.schema())?) as _,+ ))+ })+ .collect::<Result<JoinOn>>()?;++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ // Scope (0, 1) holds 1 and 2: `1` matches and is dropped, `3` is kept,+ // and the NULL build value there is UNKNOWN. Scope (2, 0) holds a+ // NULL probe value, so `4` is UNKNOWN. Scopes (0, 0) and (3, 0) are+ // empty, so `5` and the NULL build value in (3, 0) are kept. Scope+ // (2, 1) holds only 7, so `6` is kept.+ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+------+------++ | id | g1 | g2 |+ +----+------+------++ | | 3.0 | -0.0 |+ | 3 | -0.0 | 1.0 |+ | 5 | -0.0 | -0.0 |+ | 6 | 2.0 | 1.0 |+ +----+------+------++ ");+ }++ Ok(())+ }+#[test]fn test_lr_is_preserved() {assert_eq!(lr_is_preserved(JoinType::Inner), (true, true));diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 3068bcb638..dc6a32fa14 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -33,7 +33,7 @@ use crate::joins::hash_join::shared_bounds::{PartitionBounds, PartitionBuildData, SharedBuildAccumulator,};use crate::joins::utils::{- JoinKeyComparator, OnceFut, equal_rows_arr, matchable_join_keys,+ JoinKeyComparator, OnceFut, equal_rows_arr_with_normalized_left, matchable_join_keys,};use crate::stream::EmptyRecordBatchStream;use crate::{@@ -425,6 +425,11 @@ impl RecordBatchStream for HashJoinStream {/// Returns build/probe indices satisfying the equality condition, along with/// (optional) starting point for next iteration.///+/// `build_side_values` must come from [`JoinLeftData::values()`] (or be+/// derived from it), where float `-0.0` is already rewritten to `+0.0`. The+/// key comparison does not normalize the build side again, so raw build keys+/// holding `-0.0` would fail to match `+0.0` on the probe side.+////// # Example////// For `LEFT.b1 = RIGHT.b2`:@@ -496,7 +501,7 @@ pub(super) fn lookup_join_hashmap(let probe_indices_unfiltered: UInt32Array =std::mem::take(probe_indices_buffer).into();- let (build_indices, probe_indices) = equal_rows_arr(+ let (build_indices, probe_indices) = equal_rows_arr_with_normalized_left(&build_indices_unfiltered,&probe_indices_unfiltered,build_side_values,@@ -1568,6 +1573,8 @@ fn mark_null_candidates_for_probe_batch(/// Calls `f` with all correlation-scope matches between `build_scope_values`/// and `probe_scope_values`, as chunks of at most `batch_size` pairs of/// (position in `build_scope_values`, position in `probe_scope_values`).+/// `build_scope_values` must already be normalized, as for+/// [`lookup_join_hashmap`].#[expect(clippy::too_many_arguments)]fn for_each_scope_match(scope_map: &dyn JoinHashMapType,diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 1dfabe3d54..801f7a6e31 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -2301,6 +2301,50 @@ pub(super) fn equal_rows_arr(right_arrays: &[ArrayRef],null_equality: NullEquality,comparator: &mut Option<JoinKeyComparator>,+) -> Result<(UInt64Array, UInt32Array)> {+ equal_rows_arr_impl(+ indices_left,+ indices_right,+ left_arrays,+ right_arrays,+ null_equality,+ comparator,+ true,+ )+}++/// Same as [`equal_rows_arr`], but `left_arrays` must already have float+/// `-0.0` rewritten to `+0.0` (see `normalize_float_zero`), so a comparator+/// built here skips that scan on the left side. Hash join keeps its build+/// keys in that form, which saves rescanning the whole build side for every+/// probe batch.+pub(super) fn equal_rows_arr_with_normalized_left(+ indices_left: &UInt64Array,+ indices_right: &UInt32Array,+ left_arrays: &[ArrayRef],+ right_arrays: &[ArrayRef],+ null_equality: NullEquality,+ comparator: &mut Option<JoinKeyComparator>,+) -> Result<(UInt64Array, UInt32Array)> {+ equal_rows_arr_impl(+ indices_left,+ indices_right,+ left_arrays,+ right_arrays,+ null_equality,+ comparator,+ false,+ )+}++fn equal_rows_arr_impl(+ indices_left: &UInt64Array,+ indices_right: &UInt32Array,+ left_arrays: &[ArrayRef],+ right_arrays: &[ArrayRef],+ null_equality: NullEquality,+ comparator: &mut Option<JoinKeyComparator>,+ normalize_left: bool,) -> Result<(UInt64Array, UInt32Array)> {if indices_left.len() != indices_right.len() {return Err(internal_datafusion_err!(@@ -2346,7 +2390,12 @@ pub(super) fn equal_rows_arr(Some(comparator) => comparator,None => {let sort_options = vec![SortOptions::default(); left_arrays.len()];- comparator.insert(JoinKeyComparator::new(+ let new_comparator = if normalize_left {+ JoinKeyComparator::new+ } else {+ JoinKeyComparator::new_with_normalized_left+ };+ comparator.insert(new_comparator(left_arrays,right_arrays,&sort_options,@@ -2493,6 +2542,35 @@ impl JoinKeyComparator {right_arrays: &[ArrayRef],sort_options: &[SortOptions],null_equality: NullEquality,+ ) -> Result<Self> {+ Self::build(left_arrays, right_arrays, sort_options, null_equality, true)+ }++ /// Like [`Self::new`], but trusts `left_arrays` to already have float+ /// `-0.0` rewritten to `+0.0` and does not scan them again. Only the right+ /// side is normalized here. Passing a left side that still holds `-0.0`+ /// makes it compare unequal to `+0.0`.+ pub(crate) fn new_with_normalized_left(+ left_arrays: &[ArrayRef],+ right_arrays: &[ArrayRef],+ sort_options: &[SortOptions],+ null_equality: NullEquality,+ ) -> Result<Self> {+ Self::build(+ left_arrays,+ right_arrays,+ sort_options,+ null_equality,+ false,+ )+ }++ fn build(+ left_arrays: &[ArrayRef],+ right_arrays: &[ArrayRef],+ sort_options: &[SortOptions],+ null_equality: NullEquality,+ normalize_left: bool,) -> Result<Self> {debug_assert_eq!(left_arrays.len(), right_arrays.len());debug_assert_eq!(left_arrays.len(), sort_options.len());@@ -2508,8 +2586,12 @@ impl JoinKeyComparator {// no-op (Arc::clone) for non-floats and for float arrays// that contain no `-0.0`. `normalize_float_zero` preserves// null positions, so the original null masks below remain- // valid.- let l_norm = normalize_float_zero(l);+ // valid. A left side that is already normalized is used as is.+ let l_norm = if normalize_left {+ normalize_float_zero(l)+ } else {+ Arc::clone(l)+ };let r_norm = normalize_float_zero(r);let inner = make_comparator(l_norm.as_ref(), r_norm.as_ref(), *opts)?;if null_equality == NullEquality::NullEqualsNothing {@@ -4806,6 +4888,54 @@ mod tests {assert_eq!(cmp_nl.compare(1, 1), Ordering::Less);}+ #[test]+ fn join_key_comparator_with_normalized_left_skips_left_normalization() {+ let opts = [SortOptions::default()];+ let neg_zero: ArrayRef = Arc::new(Float64Array::from(vec![-0.0]));+ let pos_zero: ArrayRef = Arc::new(Float64Array::from(vec![0.0]));++ // `new` rewrites -0.0 on both sides, so the keys match.+ let cmp = JoinKeyComparator::new(+ &[Arc::clone(&neg_zero)],+ &[Arc::clone(&pos_zero)],+ &opts,+ NullEquality::NullEqualsNothing,+ )+ .unwrap();+ assert!(cmp.is_equal(0, 0));++ // The left side is taken as already normalized, so a raw -0.0 there+ // is compared bit for bit and does not match +0.0.+ let cmp = JoinKeyComparator::new_with_normalized_left(+ &[Arc::clone(&neg_zero)],+ &[Arc::clone(&pos_zero)],+ &opts,+ NullEquality::NullEqualsNothing,+ )+ .unwrap();+ assert!(!cmp.is_equal(0, 0));++ // The right side is still normalized by both constructors.+ for cmp in [+ JoinKeyComparator::new(+ &[Arc::clone(&pos_zero)],+ &[Arc::clone(&neg_zero)],+ &opts,+ NullEquality::NullEqualsNothing,+ )+ .unwrap(),+ JoinKeyComparator::new_with_normalized_left(+ &[Arc::clone(&pos_zero)],+ &[Arc::clone(&neg_zero)],+ &opts,+ NullEquality::NullEqualsNothing,+ )+ .unwrap(),+ ] {+ assert!(cmp.is_equal(0, 0));+ }+ }+#[test]fn test_equal_rows_arr_filters_candidate_pairs() {let left_a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 2, 3]));