Skip to content

Apache DataFusion SQL Query Engine

refactor: Centralize float normalization (#26595)

Michael Kleen1bc8b2f89802
diff --git a/datafusion/common/src/float_semantics.rs b/datafusion/common/src/float_semantics.rs
new file mode 100644
index 0000000000..a6998143b7
--- /dev/null
+++ b/datafusion/common/src/float_semantics.rs
@@ -0,0 +1,347 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Float semantics for DataFusion.
+//!
+//! Arrow's comparison kernels work on the bit pattern of a float,
+//! i.e. IEEE 754 totalOrder. DataFusion deviates from that in one
+//! respect: `+0.0` and `-0.0` are equal, hash the same and fall into the same
+//! group. See [`normalize_float`], [`normalize_floats`] and
+//! [`normalize_float_scalar`]. Ordering still follows totalOrder, so
+//! [`compare_floats`] sorts `-0.0` before `+0.0`.
+//!
+//! Every comparison, hash or grouping boundary should be built from these
+//! helpers rather than inspecting float bits itself:
+//!
+//! - Equality and grouping: normalize first ([`normalize_float`],
+//! [`normalize_floats`], [`normalize_float_scalar`]), then compare bits or
+//! call Arrow's `eq`, `RowConverter` or `make_comparator`. Hash
+//! [`hash_input`].
+//! - Ordering: use [`compare_floats`], [`float_lt`] or [`float_gt`]. Sorting
+//! does not normalize yet, so `-0.0` still sorts before `+0.0`.
+//!
+
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, make_array};
+use arrow::datatypes::{
+ ArrowPrimitiveType, DataType, Float16Type, Float32Type, Float64Type,
+};
+use half::f16;
+
+use crate::ScalarValue;
+
+/// The floating point types the functions in this module are defined for:
+/// [`f16`](struct@f16), [`f32`] and [`f64`].
+pub trait FloatSemantics: Copy {
+ /// Whether [`normalize_float`] would change this value. This is a cheap
+ /// check that lets callers skip rewriting data which is already
+ /// normalized.
+ fn needs_normalization(self) -> bool;
+
+ /// See [`normalize_float`].
+ fn normalize(self) -> Self;
+
+ /// See [`compare_floats`].
+ fn compare(self, other: Self) -> Ordering;
+}
+
+macro_rules! impl_float_semantics {
+ ($(($t:ty, $bits:ty, $zero:expr)),+ $(,)?) => {
+ $(impl FloatSemantics for $t {
+ #[inline]
+ fn needs_normalization(self) -> bool {
+ // `-0.0` has only the sign bit set; no other value shares
+ // this bit pattern.
+ self.to_bits() == 1 << (<$bits>::BITS - 1)
+ }
+
+ #[inline]
+ fn normalize(self) -> Self {
+ if self.needs_normalization() { $zero } else { self }
+ }
+
+ #[inline]
+ fn compare(self, other: Self) -> Ordering {
+ self.total_cmp(&other)
+ }
+ })+
+ };
+}
+impl_float_semantics!((f16, u16, f16::ZERO), (f32, u32, 0.0), (f64, u64, 0.0));
+
+/// `-0.0` becomes `0.0`, so two values that DataFusion treats as equal also
+/// have the same bits. All other values, including NaN payloads, are kept as
+/// they are.
+#[inline]
+pub fn normalize_float<T: FloatSemantics>(v: T) -> T {
+ v.normalize()
+}
+
+/// Compares two values in the order DataFusion sorts them.
+///
+/// This is IEEE 754 totalOrder (`total_cmp`), which matches Arrow's sort
+/// kernels and row encoding. Note that it orders `-0.0` before `+0.0`.
+/// The grouped `MIN`/`MAX` accumulators start from the extremes of this
+/// order, so they must change together with it.
+#[inline]
+pub fn compare_floats<T: FloatSemantics>(left: T, right: T) -> Ordering {
+ left.compare(right)
+}
+
+/// Whether `left` sorts before `right`, the same as
+/// `compare_floats(left, right).is_lt()`.
+#[inline]
+pub fn float_lt<T: FloatSemantics>(left: T, right: T) -> bool {
+ compare_floats(left, right).is_lt()
+}
+
+/// Whether `left` sorts after `right`, the same as
+/// `compare_floats(left, right).is_gt()`.
+#[inline]
+pub fn float_gt<T: FloatSemantics>(left: T, right: T) -> bool {
+ compare_floats(left, right).is_gt()
+}
+
+/// The value that DataFusion hashes in place of a float. Values that DataFusion
+/// treats as equal hash the same, so this is the value [`normalize_float`]
+/// returns.
+#[inline]
+pub fn hash_input<T: FloatSemantics>(v: T) -> T {
+ normalize_float(v)
+}
+
+/// Applies [`normalize_float`] to every float in `array`, including floats
+/// nested inside lists, structs, maps, unions, dictionaries and run-end
+/// encoded arrays. Arrays without floats are returned unchanged.
+///
+/// Callers use this before handing float data to Arrow kernels that compare
+/// bit patterns, so that `+0.0 = -0.0` holds.
+///
+/// The common case - nothing to normalize - is allocation-free: a single
+/// read-only scan of the values buffer (auto-vectorizable to an OR-reduction)
+/// decides whether to fall through to the rewriting path. Only arrays that
+/// contain values to normalize pay for a new buffer.
+pub fn normalize_floats(array: &ArrayRef) -> ArrayRef {
+ match array.data_type() {
+ DataType::Float16 => normalize_primitive::<Float16Type>(array),
+ DataType::Float32 => normalize_primitive::<Float32Type>(array),
+ DataType::Float64 => normalize_primitive::<Float64Type>(array),
+ dt if has_float_leaf(dt) => {
+ let data = array.to_data();
+ let children = data
+ .child_data()
+ .iter()
+ .map(|child| normalize_floats(&make_array(child.clone())).to_data())
+ .collect::<Vec<_>>();
+ if children
+ .iter()
+ .zip(data.child_data())
+ .all(|(new, old)| new.ptr_eq(old))
+ {
+ return Arc::clone(array);
+ }
+ make_array(
+ data.into_builder()
+ .child_data(children)
+ .build()
+ .expect("rewriting float leaves preserves the array layout"),
+ )
+ }
+ _ => Arc::clone(array),
+ }
+}
+
+fn normalize_primitive<T>(array: &ArrayRef) -> ArrayRef
+where
+ T: ArrowPrimitiveType,
+ T::Native: FloatSemantics,
+{
+ let arr = array.as_primitive::<T>();
+ if !arr.values().iter().any(|v| v.needs_normalization()) {
+ return Arc::clone(array);
+ }
+ Arc::new(arr.unary::<_, T>(normalize_float))
+}
+
+/// Applies [`normalize_float`] to a `Float16`, `Float32` or `Float64` scalar.
+/// Other scalars are returned as is.
+pub fn normalize_float_scalar(scalar: ScalarValue) -> ScalarValue {
+ match scalar {
+ ScalarValue::Float16(Some(v)) => ScalarValue::Float16(Some(normalize_float(v))),
+ ScalarValue::Float32(Some(v)) => ScalarValue::Float32(Some(normalize_float(v))),
+ ScalarValue::Float64(Some(v)) => ScalarValue::Float64(Some(normalize_float(v))),
+ other => other,
+ }
+}
+
+/// Whether `data_type` is a float or contains a float at any depth of its
+/// nested children.
+pub fn has_float_leaf(data_type: &DataType) -> bool {
+ match data_type {
+ DataType::Float16 | DataType::Float32 | DataType::Float64 => true,
+ DataType::List(f)
+ | DataType::LargeList(f)
+ | DataType::ListView(f)
+ | DataType::LargeListView(f)
+ | DataType::FixedSizeList(f, _)
+ | DataType::Map(f, _)
+ | DataType::RunEndEncoded(_, f) => has_float_leaf(f.data_type()),
+ DataType::Struct(fields) => fields.iter().any(|f| has_float_leaf(f.data_type())),
+ DataType::Union(fields, _) => {
+ fields.iter().any(|(_, f)| has_float_leaf(f.data_type()))
+ }
+ DataType::Dictionary(_, values) => has_float_leaf(values),
+ _ => false,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::{Float16Array, Float32Array, Float64Array, Int32Array, ListArray};
+ use arrow::datatypes::Int32Type;
+
+ #[test]
+ fn normalize_float_folds_negative_zero() {
+ assert_eq!(normalize_float(-0.0_f64).to_bits(), 0.0_f64.to_bits());
+ assert_eq!(normalize_float(-0.0_f32).to_bits(), 0.0_f32.to_bits());
+ assert_eq!(
+ normalize_float(f16::NEG_ZERO).to_bits(),
+ f16::ZERO.to_bits()
+ );
+ assert_eq!(
+ hash_input(-0.0_f64).to_bits(),
+ hash_input(0.0_f64).to_bits()
+ );
+ assert!((-0.0_f64).needs_normalization());
+ assert!(!0.0_f64.needs_normalization());
+ }
+
+ #[test]
+ fn normalize_float_keeps_other_values() {
+ let neg_nan = f64::from_bits(f64::NAN.to_bits() | (1 << 63));
+ let payload_nan = f64::from_bits(f64::NAN.to_bits() | 1);
+ for v in [
+ 1.5,
+ -1.5,
+ f64::INFINITY,
+ f64::NEG_INFINITY,
+ f64::MIN_POSITIVE,
+ -f64::MIN_POSITIVE,
+ f64::NAN,
+ neg_nan,
+ payload_nan,
+ ] {
+ assert!(!v.needs_normalization());
+ assert_eq!(normalize_float(v).to_bits(), v.to_bits());
+ }
+ }
+
+ #[test]
+ fn compare_floats_is_total_order() {
+ let neg_nan = f64::from_bits(f64::NAN.to_bits() | (1 << 63));
+ assert_eq!(compare_floats(1.0_f64, 2.0), Ordering::Less);
+ assert_eq!(compare_floats(-0.0_f64, 0.0), Ordering::Less);
+ assert_eq!(compare_floats(f64::NAN, f64::INFINITY), Ordering::Greater);
+ assert_eq!(compare_floats(neg_nan, f64::NEG_INFINITY), Ordering::Less);
+ assert_eq!(compare_floats(f16::ONE, f16::ZERO), Ordering::Greater);
+ assert!(float_lt(1.0_f64, 2.0) && !float_lt(2.0_f64, 1.0));
+ assert!(float_gt(f64::NAN, f64::INFINITY) && !float_gt(1.0_f64, 1.0));
+ }
+
+ #[test]
+ fn normalize_floats_rewrites_negative_zero() {
+ let array: ArrayRef =
+ Arc::new(Float64Array::from(vec![Some(-0.0), None, Some(1.0)]));
+ let result = normalize_floats(&array);
+ let result = result.as_primitive::<Float64Type>();
+ assert_eq!(result.value(0).to_bits(), 0.0_f64.to_bits());
+ assert!(result.is_null(1));
+ assert_eq!(result.value(2), 1.0);
+
+ let array: ArrayRef = Arc::new(Float32Array::from(vec![-0.0_f32]));
+ let result = normalize_floats(&array);
+ assert_eq!(
+ result.as_primitive::<Float32Type>().value(0).to_bits(),
+ 0.0_f32.to_bits()
+ );
+
+ let array: ArrayRef = Arc::new(Float16Array::from(vec![f16::NEG_ZERO]));
+ let result = normalize_floats(&array);
+ assert_eq!(
+ result.as_primitive::<Float16Type>().value(0).to_bits(),
+ f16::ZERO.to_bits()
+ );
+ }
+
+ #[test]
+ fn normalize_floats_reuses_normalized_input() {
+ let array: ArrayRef = Arc::new(Float64Array::from(vec![0.0, 1.0, f64::NAN]));
+ assert!(Arc::ptr_eq(&normalize_floats(&array), &array));
+
+ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
+ assert!(Arc::ptr_eq(&normalize_floats(&array), &array));
+ }
+
+ #[test]
+ fn normalize_floats_rewrites_nested_floats() {
+ let list = ListArray::from_iter_primitive::<Float64Type, _, _>(vec![
+ Some(vec![Some(-0.0), Some(2.0)]),
+ None,
+ ]);
+ let array: ArrayRef = Arc::new(list);
+ let result = normalize_floats(&array);
+ assert!(!Arc::ptr_eq(&result, &array));
+ let values = result.as_list::<i32>().value(0);
+ assert_eq!(
+ values.as_primitive::<Float64Type>().value(0).to_bits(),
+ 0.0_f64.to_bits()
+ );
+ assert!(result.is_null(1));
+
+ let list =
+ ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![Some(1)])]);
+ let array: ArrayRef = Arc::new(list);
+ assert!(Arc::ptr_eq(&normalize_floats(&array), &array));
+ }
+
+ #[test]
+ fn normalize_float_scalar_folds_negative_zero() {
+ assert_eq!(
+ normalize_float_scalar(ScalarValue::Float64(Some(-0.0))),
+ ScalarValue::Float64(Some(0.0))
+ );
+ assert_eq!(
+ normalize_float_scalar(ScalarValue::Float32(Some(-0.0))),
+ ScalarValue::Float32(Some(0.0))
+ );
+ assert_eq!(
+ normalize_float_scalar(ScalarValue::Float16(Some(f16::NEG_ZERO))),
+ ScalarValue::Float16(Some(f16::ZERO))
+ );
+ assert_eq!(
+ normalize_float_scalar(ScalarValue::Float64(None)),
+ ScalarValue::Float64(None)
+ );
+ assert_eq!(
+ normalize_float_scalar(ScalarValue::Int32(Some(0))),
+ ScalarValue::Int32(Some(0))
+ );
+ }
+}
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs
index 181cde28f9..ef29cc3ef8 100644
--- a/datafusion/common/src/hash_utils.rs
+++ b/datafusion/common/src/hash_utils.rs
@@ -89,6 +89,7 @@ use crate::cast::{
};
use crate::error::Result;
use crate::error::{_internal_datafusion_err, _internal_err};
+use crate::float_semantics::hash_input;
#[cfg(not(feature = "force_hash_collisions"))]
use crate::utils::offset_span;
use std::cell::RefCell;
@@ -258,24 +259,20 @@ hash_value!(i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128);
hash_value!(bool, str, [u8], IntervalDayTime, IntervalMonthDayNano);
macro_rules! hash_float_value {
- ($(($t:ty, $i:ty)),+) => {
+ ($($t:ty),+) => {
$(impl HashValue for $t {
+ // Hash the normalized bits so values SQL considers equal
+ // (e.g. +0.0 and -0.0) hash the same.
fn hash_one<S: BuildHasher>(&self, state: &S) -> u64 {
- // +0.0 and -0.0 differ only in the sign bit but compare equal
- // under IEEE 754; normalize -0.0 → +0.0 so Hash agrees with Eq.
- let bits = <$i>::from_ne_bytes(self.to_ne_bytes());
- let bits = if bits << 1 == 0 { 0 } else { bits };
- state.hash_one(bits)
+ state.hash_one(hash_input(*self).to_bits())
}
fn hash_write(&self, hasher: &mut impl Hasher) {
- let bits = <$i>::from_ne_bytes(self.to_ne_bytes());
- let bits: $i = if bits << 1 == 0 { 0 } else { bits };
- hasher.write(&bits.to_ne_bytes())
+ hasher.write(&hash_input(*self).to_bits().to_ne_bytes())
}
})+
};
}
-hash_float_value!((half::f16, u16), (f32, u32), (f64, u64));
+hash_float_value!(half::f16, f32, f64);
#[cfg(not(feature = "force_hash_collisions"))]
trait ChildHashing {
diff --git a/datafusion/common/src/hash_utils/build_hasher.rs b/datafusion/common/src/hash_utils/build_hasher.rs
index 23ee6f12e3..365680e4c7 100644
--- a/datafusion/common/src/hash_utils/build_hasher.rs
+++ b/datafusion/common/src/hash_utils/build_hasher.rs
@@ -31,6 +31,8 @@ use crate::cast::{
};
use crate::error::Result;
use crate::error::{_internal_datafusion_err, _internal_err};
+#[cfg(not(feature = "force_hash_collisions"))]
+use crate::float_semantics::hash_input;
#[cfg(feature = "force_hash_collisions")]
use arrow::array::Array;
#[cfg(not(feature = "force_hash_collisions"))]
@@ -168,18 +170,16 @@ build_hasher_hash_value!(i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128)
build_hasher_hash_value!(bool, str, [u8], IntervalDayTime, IntervalMonthDayNano);
macro_rules! build_hasher_hash_float_value {
- ($(($t:ty, $i:ty)),+) => {
+ ($($t:ty),+) => {
$(#[cfg(not(feature = "force_hash_collisions"))]
impl BuildHasherHashValue for $t {
fn hash_one_with_hasher<S: BuildHasher>(&self, state: &S) -> u64 {
- let bits = <$i>::from_ne_bytes(self.to_ne_bytes());
- let bits = if bits << 1 == 0 { 0 } else { bits };
- state.hash_one(bits)
+ state.hash_one(hash_input(*self).to_bits())
}
})+
};
}
-build_hasher_hash_float_value!((half::f16, u16), (f32, u32), (f64, u64));
+build_hasher_hash_float_value!(half::f16, f32, f64);
#[cfg(not(feature = "force_hash_collisions"))]
fn hash_null_with_hasher<S: BuildHasher>(
diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs
index 2eebfe4963..890f430ae9 100644
--- a/datafusion/common/src/lib.rs
+++ b/datafusion/common/src/lib.rs
@@ -46,6 +46,7 @@ pub mod encryption;
pub mod error;
pub mod extensions;
pub mod file_options;
+pub mod float_semantics;
pub mod format;
pub mod hash_utils;
pub mod heap_size;
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index 0ff108ff36..7811f522ea 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -1436,132 +1436,24 @@ fn fsl_values_row_number(list_size: i32, array_len: usize) -> Result<Int32Array>
}
/// Replace `-0.0` with `+0.0` in any `Float16`, `Float32`, or `Float64` array.
-/// For non-float arrays returns the input unchanged. NaN payloads are
-/// preserved.
-///
-/// Arrow's comparison kernels (`arrow::compute::kernels::cmp::eq` etc.) and
-/// row-encoding (`arrow::row::RowConverter`) use IEEE 754 totalOrder
-/// semantics, which treats `-0.0` and `+0.0` as distinct. SQL semantics
-/// (PostgreSQL / IEEE 754 equality) require them to compare equal, so
-/// callers normalize before invoking those kernels.
-///
-/// The common case - no `-0.0` present - is allocation-free: a single
-/// read-only scan of the underlying buffer (auto-vectorizable to an
-/// OR-reduction) decides whether to fall through to the rewriting path.
-/// Only arrays that actually contain `-0.0` pay for a new buffer.
+#[deprecated(
+ since = "56.0.0",
+ note = "use `datafusion_common::float_semantics::normalize_floats` instead"
+)]
pub fn normalize_float_zero(array: &ArrayRef) -> ArrayRef {
- use arrow::array::{Float16Array, Float32Array, Float64Array, make_array};
- use arrow::datatypes::{Float16Type, Float32Type, Float64Type};
- // -0.0 has only the sign bit set; no other finite or NaN value shares
- // this bit pattern, so a strict-equality scan reliably gates the rewrite.
- const NEG_ZERO_F16_BITS: u16 = half::f16::NEG_ZERO.to_bits();
- const NEG_ZERO_F32_BITS: u32 = (-0.0_f32).to_bits();
- const NEG_ZERO_F64_BITS: u64 = (-0.0_f64).to_bits();
- match array.data_type() {
- DataType::Float32 => {
- let arr: &Float32Array = array.as_primitive::<Float32Type>();
- if !arr
- .values()
- .iter()
- .any(|v| v.to_bits() == NEG_ZERO_F32_BITS)
- {
- return Arc::clone(array);
- }
- let normalized: Float32Array =
- arr.unary(|v| if v.to_bits() << 1 == 0 { 0.0_f32 } else { v });
- Arc::new(normalized)
- }
- DataType::Float64 => {
- let arr: &Float64Array = array.as_primitive::<Float64Type>();
- if !arr
- .values()
- .iter()
- .any(|v| v.to_bits() == NEG_ZERO_F64_BITS)
- {
- return Arc::clone(array);
- }
- let normalized: Float64Array =
- arr.unary(|v| if v.to_bits() << 1 == 0 { 0.0_f64 } else { v });
- Arc::new(normalized)
- }
- DataType::Float16 => {
- let arr: &Float16Array = array.as_primitive::<Float16Type>();
- if !arr
- .values()
- .iter()
- .any(|v| v.to_bits() == NEG_ZERO_F16_BITS)
- {
- return Arc::clone(array);
- }
- let normalized: Float16Array = arr.unary(|v| {
- if v.to_bits() << 1 == 0 {
- half::f16::from_bits(0)
- } else {
- v
- }
- });
- Arc::new(normalized)
- }
- dt if has_float_leaf(dt) => {
- let data = array.to_data();
- let children = data
- .child_data()
- .iter()
- .map(|child| normalize_float_zero(&make_array(child.clone())).to_data())
- .collect::<Vec<_>>();
- if children
- .iter()
- .zip(data.child_data())
- .all(|(new, old)| new.ptr_eq(old))
- {
- return Arc::clone(array);
- }
- make_array(
- data.into_builder()
- .child_data(children)
- .build()
- .expect("rewriting float leaves preserves the array layout"),
- )
- }
- _ => Arc::clone(array),
- }
+ crate::float_semantics::normalize_floats(array)
}
-pub fn has_float_leaf(data_type: &DataType) -> bool {
- match data_type {
- DataType::Float16 | DataType::Float32 | DataType::Float64 => true,
- DataType::List(f)
- | DataType::LargeList(f)
- | DataType::ListView(f)
- | DataType::LargeListView(f)
- | DataType::FixedSizeList(f, _)
- | DataType::Map(f, _)
- | DataType::RunEndEncoded(_, f) => has_float_leaf(f.data_type()),
- DataType::Struct(fields) => fields.iter().any(|f| has_float_leaf(f.data_type())),
- DataType::Union(fields, _) => {
- fields.iter().any(|(_, f)| has_float_leaf(f.data_type()))
- }
- DataType::Dictionary(_, values) => has_float_leaf(values),
- _ => false,
- }
-}
+pub use crate::float_semantics::has_float_leaf;
/// Replace `-0.0` with `+0.0` in `Float16`, `Float32`, or `Float64` scalar
-/// values. Other variants are returned unchanged. See [`normalize_float_zero`]
-/// for context.
+/// values.
+#[deprecated(
+ since = "56.0.0",
+ note = "use `datafusion_common::float_semantics::normalize_float_scalar` instead"
+)]
pub fn normalize_float_zero_scalar(scalar: ScalarValue) -> ScalarValue {
- match scalar {
- ScalarValue::Float32(Some(v)) if v.to_bits() << 1 == 0 => {
- ScalarValue::Float32(Some(0.0))
- }
- ScalarValue::Float64(Some(v)) if v.to_bits() << 1 == 0 => {
- ScalarValue::Float64(Some(0.0))
- }
- ScalarValue::Float16(Some(v)) if v.to_bits() << 1 == 0 => {
- ScalarValue::Float16(Some(half::f16::from_bits(0)))
- }
- other => other,
- }
+ crate::float_semantics::normalize_float_scalar(scalar)
}
/// Apply a struct's nulls to one of its fields.
diff --git a/datafusion/functions-aggregate-common/src/min_max.rs b/datafusion/functions-aggregate-common/src/min_max.rs
index f0a4a52a4a..de2ca7e620 100644
--- a/datafusion/functions-aggregate-common/src/min_max.rs
+++ b/datafusion/functions-aggregate-common/src/min_max.rs
@@ -31,6 +31,7 @@ use arrow::array::{
};
use arrow::compute;
use arrow::datatypes::{DataType, IntervalUnit, TimeUnit};
+use datafusion_common::float_semantics::{FloatSemantics, compare_floats};
use datafusion_common::{
DataFusionError, Result, ScalarValue, downcast_value, internal_err,
};
@@ -64,17 +65,16 @@ fn min_max_option<T: Clone + Ord>(
}
}
-fn min_max_float_option<T: Copy>(
+fn min_max_float_option<T: FloatSemantics>(
lhs: Option<&T>,
rhs: Option<&T>,
ordering: Ordering,
- cmp: impl Fn(&T, &T) -> Ordering,
) -> Option<T> {
match (lhs, rhs) {
(None, None) => None,
(Some(a), None) => Some(*a),
(None, Some(b)) => Some(*b),
- (Some(a), Some(b)) if cmp(a, b) == ordering => Some(*b),
+ (Some(a), Some(b)) if compare_floats(*a, *b) == ordering => Some(*b),
(Some(a), Some(_)) => Some(*a),
}
}
@@ -247,15 +247,13 @@ fn min_max_scalar_same_variant(
ScalarValue::Boolean(min_max_option(lhs.as_ref(), rhs.as_ref(), ordering))
}
(ScalarValue::Float64(lhs), ScalarValue::Float64(rhs)) => ScalarValue::Float64(
- min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering, f64::total_cmp),
+ min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering),
),
(ScalarValue::Float32(lhs), ScalarValue::Float32(rhs)) => ScalarValue::Float32(
- min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering, f32::total_cmp),
+ min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering),
),
(ScalarValue::Float16(lhs), ScalarValue::Float16(rhs)) => ScalarValue::Float16(
- min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering, |a, b| {
- a.total_cmp(b)
- }),
+ min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering),
),
(ScalarValue::UInt64(lhs), ScalarValue::UInt64(rhs)) => {
ScalarValue::UInt64(min_max_option(lhs.as_ref(), rhs.as_ref(), ordering))
diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs
index e12adad906..03d80dd75b 100644
--- a/datafusion/functions-aggregate/src/min_max.rs
+++ b/datafusion/functions-aggregate/src/min_max.rs
@@ -23,6 +23,7 @@ mod min_max_struct;
use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};
use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};
+use datafusion_common::float_semantics::{float_gt, float_lt};
use datafusion_common::stats::Precision;
use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
@@ -124,7 +125,7 @@ macro_rules! primitive_max_accumulator {
($t:ty, $data_type:ident, total, $bits:ident) => {{
Ok(Box::new(
PrimitiveGroupsAccumulator::<$t, _>::new($data_type, |cur, new| {
- if new.total_cmp(cur) == Ordering::Greater {
+ if float_gt(new, *cur) {
*cur = new
}
})
@@ -160,7 +161,7 @@ macro_rules! primitive_min_accumulator {
($t:ty, $data_type:ident, total, $bits:ident) => {{
Ok(Box::new(
PrimitiveGroupsAccumulator::<$t, _>::new(&$data_type, |cur, new| {
- if new.total_cmp(cur) == Ordering::Less {
+ if float_lt(new, *cur) {
*cur = new
}
})
diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rs
index 5d7ae70ae7..e783913204 100644
--- a/datafusion/functions-nested/src/set_ops.rs
+++ b/datafusion/functions-nested/src/set_ops.rs
@@ -28,7 +28,8 @@ use arrow::datatypes::DataType::{LargeList, List, Null};
use arrow::datatypes::{DataType, Field, FieldRef};
use arrow::row::{RowConverter, SortField};
use datafusion_common::cast::{as_large_list_array, as_list_array};
-use datafusion_common::utils::{ListCoercion, normalize_float_zero, offset_span};
+use datafusion_common::float_semantics::normalize_floats;
+use datafusion_common::utils::{ListCoercion, offset_span};
use datafusion_common::{
Result, assert_eq_or_internal_err, exec_err, internal_err, utils::take_function_args,
};
@@ -334,9 +335,9 @@ pub(crate) fn normalize_visible_values<OffsetSize: OffsetSizeTrait>(
) -> ArrayRef {
let (first, len) = offset_span(array.offsets());
if first == 0 && len == array.values().len() {
- normalize_float_zero(array.values())
+ normalize_floats(array.values())
} else {
- normalize_float_zero(&array.values().slice(first, len))
+ normalize_floats(&array.values().slice(first, len))
}
}
diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs
index 5309deb34a..7a2fab3010 100644
--- a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs
+++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs
@@ -28,7 +28,7 @@
//! `false` so that later rules can prune the plan they filter.
use super::utils::{is_false, is_null};
-use datafusion_common::utils::normalize_float_zero_scalar;
+use datafusion_common::float_semantics::normalize_float_scalar;
use datafusion_common::{Column, DataFusionError, Result, ScalarValue, internal_err};
use datafusion_expr::utils::{
conjunction, disjunction, split_binary_owned, split_conjunction_owned,
@@ -422,7 +422,7 @@ fn op_and_literal(predicate: &Expr) -> Result<(Operator, &ScalarValue)> {
fn normalize_literal(expr: &mut Expr) {
if let Expr::Literal(value, _) = expr {
let original = std::mem::replace(value, ScalarValue::Null);
- *value = normalize_float_zero_scalar(original);
+ *value = normalize_float_scalar(original);
}
}
diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs
index 34ead9a69d..b0ae7a7f17 100644
--- a/datafusion/optimizer/src/simplify_expressions/utils.rs
+++ b/datafusion/optimizer/src/simplify_expressions/utils.rs
@@ -18,7 +18,7 @@
//! Utility functions for expression simplification
use datafusion_common::cse::NormalizeEq;
-use datafusion_common::utils::normalize_float_zero_scalar;
+use datafusion_common::float_semantics::normalize_float_scalar;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
Case, Expr, Like, Operator,
@@ -239,8 +239,8 @@ pub fn is_eq_and_ne_with_different_literal(eq_expr: &Expr, ne_expr: &Expr) -> bo
Expr::Literal(right_value, right_metadata),
) => {
left_metadata != right_metadata
- || normalize_float_zero_scalar(left_value.clone())
- != normalize_float_zero_scalar(right_value.clone())
+ || normalize_float_scalar(left_value.clone())
+ != normalize_float_scalar(right_value.clone())
}
_ => left != right,
}
diff --git a/datafusion/physical-expr-common/src/datum.rs b/datafusion/physical-expr-common/src/datum.rs
index 416507c618..182eebf880 100644
--- a/datafusion/physical-expr-common/src/datum.rs
+++ b/datafusion/physical-expr-common/src/datum.rs
@@ -23,8 +23,8 @@ use arrow::compute::kernels::cmp::{
};
use arrow::compute::{SortOptions, ilike, like, nilike, nlike};
use arrow::error::ArrowError;
-use datafusion_common::utils::{
- has_float_leaf, normalize_float_zero, normalize_float_zero_scalar,
+use datafusion_common::float_semantics::{
+ has_float_leaf, normalize_float_scalar, normalize_floats,
};
use datafusion_common::{Result, ScalarValue};
use datafusion_common::{arrow_datafusion_err, assert_or_internal_err, internal_err};
@@ -99,9 +99,9 @@ pub fn apply_cmp(
fn normalize_cmp_input(cv: &ColumnarValue) -> ColumnarValue {
match cv {
- ColumnarValue::Array(a) => ColumnarValue::Array(normalize_float_zero(a)),
+ ColumnarValue::Array(a) => ColumnarValue::Array(normalize_floats(a)),
ColumnarValue::Scalar(s) => {
- ColumnarValue::Scalar(normalize_float_zero_scalar(s.clone()))
+ ColumnarValue::Scalar(normalize_float_scalar(s.clone()))
}
}
}
@@ -151,7 +151,7 @@ pub fn compare_with_eq(
fn normalize_nested_float_zero(array: &dyn Array) -> Option<ArrayRef> {
has_float_leaf(array.data_type())
- .then(|| normalize_float_zero(&make_array(array.to_data())))
+ .then(|| normalize_floats(&make_array(array.to_data())))
}
/// Compare on nested type List, Struct, and so on
diff --git a/datafusion/physical-expr/src/intervals/cp_solver.rs b/datafusion/physical-expr/src/intervals/cp_solver.rs
index b28a42af5b..08d961f286 100644
--- a/datafusion/physical-expr/src/intervals/cp_solver.rs
+++ b/datafusion/physical-expr/src/intervals/cp_solver.rs
@@ -299,12 +299,11 @@ pub fn propagate_arithmetic(
/// Compare two scalars the way runtime `Eq` does, instead of with
/// `ScalarValue`'s bit-wise equality. `ScalarValue::PartialEq` is bit-wise for
/// floats (`to_bits`), under which `-0.0 != +0.0`; runtime `Eq` normalizes
-/// them via `normalize_float_zero_scalar` (see
+/// them via `normalize_float_scalar` (see
/// `physical-expr-common/src/datum.rs`).
fn singleton_values_equal(left: &ScalarValue, right: &ScalarValue) -> bool {
- use datafusion_common::utils::normalize_float_zero_scalar;
- normalize_float_zero_scalar(left.clone())
- == normalize_float_zero_scalar(right.clone())
+ use datafusion_common::float_semantics::normalize_float_scalar;
+ normalize_float_scalar(left.clone()) == normalize_float_scalar(right.clone())
}
/// This function refines intervals `left_child` and `right_child` by applying
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
index 7ab70bf0c8..2836b3177e 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
@@ -45,9 +45,9 @@ use arrow::datatypes::{
BinaryViewType, DataType, Field, Int8Type, Int16Type, Int32Type, Int64Type, Schema,
SchemaRef, StringViewType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
};
+use datafusion_common::float_semantics::{has_float_leaf, normalize_floats};
use datafusion_common::hash_utils::RandomState;
use datafusion_common::hash_utils::create_hashes;
-use datafusion_common::utils::{has_float_leaf, normalize_float_zero};
use datafusion_common::{Result, not_impl_err};
use datafusion_execution::memory_pool::proxy::{HashTableAllocExt, VecAllocExt};
use datafusion_expr::{EmitTo, GroupSelection};
@@ -1070,7 +1070,7 @@ impl<const STREAMING: bool> GroupValues for GroupValuesColumn<STREAMING> {
let normalized: Option<Vec<ArrayRef>> = cols
.iter()
.any(|col| col.data_type().is_nested() && has_float_leaf(col.data_type()))
- .then(|| cols.iter().map(normalize_float_zero).collect());
+ .then(|| cols.iter().map(normalize_floats).collect());
let cols = normalized.as_deref().unwrap_or(cols);
// `try_new` and the reset points in `emit` / `clear_shrink` keep
diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs
index ebd1e071cf..99ecee10c0 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs
@@ -98,9 +98,9 @@ where
} else {
unsafe { *array_values.get_unchecked(rhs_row) }
};
- // `left` was already canonicalized on append; canonicalize the
+ // `left` was already normalized on append; normalize the
// input so ±0 (and any future equivalence class) compares equal.
- if left.is_eq(right.canonicalize()) {
+ if left.is_eq(right.normalize_key()) {
cmp_buf[i / 8] |= 1 << (i % 8);
}
}
@@ -141,7 +141,7 @@ where
continue;
}
- if !self.group_values[lhs_row].is_eq(array.value(rhs_row).canonicalize()) {
+ if !self.group_values[lhs_row].is_eq(array.value(rhs_row).normalize_key()) {
equal_to_results.set_bit(idx, false);
}
}
@@ -165,7 +165,7 @@ where
}
self.group_values[lhs_row]
- .is_eq(array.as_primitive::<T>().value(rhs_row).canonicalize())
+ .is_eq(array.as_primitive::<T>().value(rhs_row).normalize_key())
}
fn append_val(&mut self, array: &ArrayRef, row: usize) -> Result<()> {
@@ -177,11 +177,11 @@ where
} else {
self.nulls.append_non_null();
self.group_values
- .push(array.as_primitive::<T>().value(row).canonicalize());
+ .push(array.as_primitive::<T>().value(row).normalize_key());
}
} else {
self.group_values
- .push(array.as_primitive::<T>().value(row).canonicalize());
+ .push(array.as_primitive::<T>().value(row).normalize_key());
}
Ok(())
@@ -227,7 +227,7 @@ where
self.group_values.push(T::default_value());
} else {
self.nulls.append_non_null();
- self.group_values.push(arr.value(row).canonicalize());
+ self.group_values.push(arr.value(row).normalize_key());
}
}
}
@@ -235,7 +235,7 @@ where
(true, Nulls::None) => {
self.nulls.append_n_non_nulls(rows.len());
for &row in rows {
- self.group_values.push(arr.value(row).canonicalize());
+ self.group_values.push(arr.value(row).normalize_key());
}
}
@@ -247,7 +247,7 @@ where
(false, _) => {
for &row in rows {
- self.group_values.push(arr.value(row).canonicalize());
+ self.group_values.push(arr.value(row).normalize_key());
}
}
}
diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rs
index a2ff0b6adb..ee3f360886 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/row.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs
@@ -25,9 +25,9 @@ use arrow::compute::cast;
use arrow::datatypes::{DataType, SchemaRef};
use arrow::row::{RowConverter, Rows, SortField};
use datafusion_common::Result;
+use datafusion_common::float_semantics::normalize_floats;
use datafusion_common::hash_utils::RandomState;
use datafusion_common::hash_utils::create_hashes;
-use datafusion_common::utils::normalize_float_zero;
use datafusion_execution::memory_pool::proxy::VecAllocExt;
use datafusion_expr::{EmitTo, GroupSelection};
use hashbrown::hash_table::HashTable;
@@ -125,8 +125,7 @@ impl GroupValues for GroupValuesRows {
// Normalize -0.0 → +0.0 so RowConverter (IEEE 754 totalOrder) and
// primitive hashing both group ±0 together. No-op for non-float
// columns.
- let normalized_cols: Vec<ArrayRef> =
- cols.iter().map(normalize_float_zero).collect();
+ let normalized_cols: Vec<ArrayRef> = cols.iter().map(normalize_floats).collect();
let cols = normalized_cols.as_slice();
// Convert the group keys into the row format
diff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs
index 816b020c39..05f2a26ca0 100644
--- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs
+++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs
@@ -23,6 +23,7 @@ use arrow::array::{
};
use arrow::datatypes::{DataType, i256};
use datafusion_common::Result;
+use datafusion_common::float_semantics::normalize_float;
use datafusion_common::hash_utils::RandomState;
use datafusion_common::utils::split_vec_min_alloc;
use datafusion_execution::memory_pool::proxy::VecAllocExt;
@@ -39,11 +40,10 @@ pub trait HashValue {
/// Return a canonical representative whose bit pattern is identical for
/// all values that should be grouped together. Default is the identity;
- /// floats override this to fold `-0.0` into `+0.0` so the bit-equal
- /// `is_eq` check used during insertion treats them as the same group.
- /// NaN payload bits are preserved.
+ /// floats override this with [`normalize_float`] so the bit-equal
+ /// `is_eq` check used during insertion follows SQL equality.
#[inline]
- fn canonicalize(self) -> Self
+ fn normalize_key(self) -> Self
where
Self: Sized,
{
@@ -75,7 +75,7 @@ macro_rules! hash_float {
$(impl HashValue for $t {
#[cfg(not(feature = "force_hash_collisions"))]
fn hash(&self, state: &RandomState) -> u64 {
- state.hash_one(self.canonicalize().to_bits())
+ state.hash_one(datafusion_common::float_semantics::hash_input(*self).to_bits())
}
#[cfg(feature = "force_hash_collisions")]
@@ -84,10 +84,8 @@ macro_rules! hash_float {
}
#[inline]
- fn canonicalize(self) -> Self {
- let bits = self.to_bits();
- let bits = if bits << 1 == 0 { 0 } else { bits };
- Self::from_bits(bits)
+ fn normalize_key(self) -> Self {
+ normalize_float(self)
}
})+
};
@@ -149,7 +147,7 @@ where
// Fold equivalence-class duplicates (e.g. `-0.0` → `+0.0`)
// so the bit-equal `is_eq` matches and the stored value is
// the canonical representative.
- let key = key.canonicalize();
+ let key = key.normalize_key();
let state = &self.random_state;
let hash = key.hash(state);
let insert = self.map.entry(
diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rs
index 89ddaaa0be..519c16d5b8 100644
--- a/datafusion/physical-plan/src/aggregates/topk/heap.rs
+++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs
@@ -32,6 +32,7 @@ use arrow::buffer::ScalarBuffer;
use arrow::datatypes::{DataType, i256};
use datafusion_common::Result;
use datafusion_common::exec_datafusion_err;
+use datafusion_common::float_semantics::compare_floats;
use half::f16;
use std::cmp::Ordering;
@@ -550,7 +551,7 @@ macro_rules! compare_float {
$(impl Comparable for Option<$t> {
fn comp(&self, other: &Self) -> Ordering {
match (self, other) {
- (Some(me), Some(other)) => me.total_cmp(other),
+ (Some(me), Some(other)) => compare_floats(*me, *other),
(Some(_), None) => Ordering::Greater,
(None, Some(_)) => Ordering::Less,
(None, None) => Ordering::Equal,
@@ -560,7 +561,7 @@ macro_rules! compare_float {
$(impl Comparable for $t {
fn comp(&self, other: &Self) -> Ordering {
- self.total_cmp(other)
+ compare_floats(*self, *other)
}
})+
};
diff --git a/datafusion/physical-plan/src/joins/asof_join.rs b/datafusion/physical-plan/src/joins/asof_join.rs
index bc38c28b43..0029159e15 100644
--- a/datafusion/physical-plan/src/joins/asof_join.rs
+++ b/datafusion/physical-plan/src/joins/asof_join.rs
@@ -79,10 +79,10 @@ use arrow::array::{
use arrow::buffer::NullBuffer;
use arrow::compute::{SortOptions, interleave};
use arrow::datatypes::{Schema, SchemaRef};
+use datafusion_common::float_semantics::normalize_floats;
use datafusion_common::stats::Precision;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::memory::RecordBatchMemoryCounter;
-use datafusion_common::utils::normalize_float_zero;
use datafusion_common::{
ColumnStatistics, JoinSide, JoinType, NullEquality, Result, Statistics,
assert_eq_or_internal_err, internal_err, plan_err, project_schema,
@@ -915,7 +915,7 @@ impl InputCursor {
// Arrow's comparator distinguishes -0.0 from +0.0, while SQL does
// not. Normalizing once here is cheaper than fixing every row-level
// comparison and lets the cached comparator use the arrays directly.
- self.match_array = Some(normalize_float_zero(&match_array));
+ self.match_array = Some(normalize_floats(&match_array));
self.key_batch_id += 1;
self.batch = Some(batch);
}
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index b374764033..135c1c4482 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -78,12 +78,12 @@ use arrow::record_batch::RecordBatch;
use arrow::util::bit_util;
use arrow_schema::{DataType, Schema};
use datafusion_common::config::ConfigOptions;
+use datafusion_common::float_semantics::normalize_floats;
use datafusion_common::stats::Precision;
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_common::utils::memory::{
RecordBatchMemoryCounter, estimate_memory_size, get_record_batch_memory_size,
};
-use datafusion_common::utils::normalize_float_zero;
use datafusion_common::{
JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,
plan_err, project_schema,
@@ -3293,7 +3293,7 @@ fn build_in_memory(
// comparison does not rescan the build side. Arrays without `-0.0`, and
// non-float keys, are shared rather than copied.
let normalized_values: Vec<ArrayRef> =
- left_values.iter().map(normalize_float_zero).collect();
+ left_values.iter().map(normalize_floats).collect();
// Join keys that are plain columns share the buffers of `batch`, any other
// expression evaluates to new arrays that are kept for the whole join.
diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs
index 98b32b3151..1a7b4da6da 100644
--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs
+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs
@@ -26,7 +26,7 @@ use arrow::{
};
use arrow_schema::{DataType, Schema, SchemaRef, SortOptions};
use datafusion_common::NullEquality;
-use datafusion_common::utils::normalize_float_zero;
+use datafusion_common::float_semantics::normalize_floats;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};
use datafusion_expr::{ColumnarValue, JoinType, Operator};
@@ -608,7 +608,7 @@ fn buffered_extreme(
let last = values.slice(end - 1, 1);
// `apply_cmp` normalizes `-0.0` only in flat float scalars, not inside a
// `ScalarValue::Dictionary`, so normalize the key before taking it.
- let extreme = normalize_float_zero(&last);
+ let extreme = normalize_floats(&last);
Ok(Some(ColumnarValue::Scalar(ScalarValue::try_from_array(
&extreme, 0,
)?)))
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs
index 6bee6c0ebe..06ad205639 100644
--- a/datafusion/physical-plan/src/joins/utils.rs
+++ b/datafusion/physical-plan/src/joins/utils.rs
@@ -68,11 +68,11 @@ use arrow::datatypes::{
use arrow_ord::ord::{DynComparator, make_comparator};
use arrow_schema::{DataType, SortOptions, TimeUnit};
use datafusion_common::cast::as_boolean_array;
+use datafusion_common::float_semantics::normalize_floats;
use datafusion_common::hash_utils::RandomState;
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::stats::Precision;
use datafusion_common::utils::memory::RecordBatchMemoryCounter;
-use datafusion_common::utils::normalize_float_zero;
use datafusion_common::{
DataFusionError, JoinSide, JoinType, NullEquality, Result, SharedResult,
internal_datafusion_err, not_impl_err, plan_err,
@@ -2444,7 +2444,7 @@ pub(super) fn equal_rows_arr(
}
/// Same as [`equal_rows_arr`], but `left_arrays` must already have float
-/// `-0.0` rewritten to `+0.0` (see `normalize_float_zero`), so a comparator
+/// `-0.0` rewritten to `+0.0` (see `normalize_floats`), so a comparator
/// built here skips that scan on the left side. Hash join keeps its build
/// keys in that form, which saves rescanning the whole build side for every
/// probe batch.
@@ -2714,15 +2714,15 @@ impl JoinKeyComparator {
// treats `-0.0` / `+0.0` as distinct. Normalize float arrays
// so SMJ / piecewise-merge equi-keys honor SQL equality;
// no-op (Arc::clone) for non-floats and for float arrays
- // that contain no `-0.0`. `normalize_float_zero` preserves
+ // that contain no `-0.0`. `normalize_floats` preserves
// null positions, so the original null masks below remain
// valid. A left side that is already normalized is used as is.
let l_norm = if normalize_left {
- normalize_float_zero(l)
+ normalize_floats(l)
} else {
Arc::clone(l)
};
- let r_norm = normalize_float_zero(r);
+ let r_norm = normalize_floats(r);
let inner = make_comparator(l_norm.as_ref(), r_norm.as_ref(), *opts)?;
if null_equality == NullEquality::NullEqualsNothing {
let ln = l.logical_nulls().filter(|n| n.null_count() > 0);
diff --git a/datafusion/physical-plan/src/repartition/range.rs b/datafusion/physical-plan/src/repartition/range.rs
index c05f1e531e..a718cfb341 100644
--- a/datafusion/physical-plan/src/repartition/range.rs
+++ b/datafusion/physical-plan/src/repartition/range.rs
@@ -24,6 +24,7 @@ use arrow::array::*;
use arrow::compute::SortOptions;
use arrow::datatypes::*;
use arrow::row::{Row, RowConverter, Rows, SortField};
+use datafusion_common::float_semantics::compare_floats;
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_err, not_impl_err, plan_err,
validate_range_split_points,
@@ -471,14 +472,14 @@ macro_rules! impl_float_values_router {
if !descending {
for (idx, &val) in values.iter().enumerate() {
let p = split_points.partition_point(|&sp| {
- sp.total_cmp(&val) != Ordering::Greater
+ compare_floats(sp, val) != Ordering::Greater
});
emit(idx, p);
}
} else {
for (idx, &val) in values.iter().enumerate() {
let p = split_points.partition_point(|&sp| {
- sp.total_cmp(&val) != Ordering::Less
+ compare_floats(sp, val) != Ordering::Less
});
emit(idx, p);
}
@@ -492,11 +493,11 @@ macro_rules! impl_float_values_router {
let val = array.value(idx);
let p = if !descending {
split_points.partition_point(|&sp| {
- sp.total_cmp(&val) != Ordering::Greater
+ compare_floats(sp, val) != Ordering::Greater
})
} else {
split_points.partition_point(|&sp| {
- sp.total_cmp(&val) != Ordering::Less
+ compare_floats(sp, val) != Ordering::Less
})
};
emit(idx, p);