Apache DataFusion SQL Query Engine
refactor: Centralize float normalization (#26595)
Michael Kleen1bc8b2f89802
diff --git a/datafusion/common/src/float_semantics.rs b/datafusion/common/src/float_semantics.rsnew file mode 100644index 0000000000..a6998143b7--- /dev/null+++ b/datafusion/common/src/float_semantics.rs@@ -0,0 +1,347 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Float semantics for DataFusion.+//!+//! Arrow's comparison kernels work on the bit pattern of a float,+//! i.e. IEEE 754 totalOrder. DataFusion deviates from that in one+//! respect: `+0.0` and `-0.0` are equal, hash the same and fall into the same+//! group. See [`normalize_float`], [`normalize_floats`] and+//! [`normalize_float_scalar`]. Ordering still follows totalOrder, so+//! [`compare_floats`] sorts `-0.0` before `+0.0`.+//!+//! Every comparison, hash or grouping boundary should be built from these+//! helpers rather than inspecting float bits itself:+//!+//! - Equality and grouping: normalize first ([`normalize_float`],+//! [`normalize_floats`], [`normalize_float_scalar`]), then compare bits or+//! call Arrow's `eq`, `RowConverter` or `make_comparator`. Hash+//! [`hash_input`].+//! - Ordering: use [`compare_floats`], [`float_lt`] or [`float_gt`]. Sorting+//! does not normalize yet, so `-0.0` still sorts before `+0.0`.+//!++use std::cmp::Ordering;+use std::sync::Arc;++use arrow::array::{Array, ArrayRef, AsArray, make_array};+use arrow::datatypes::{+ ArrowPrimitiveType, DataType, Float16Type, Float32Type, Float64Type,+};+use half::f16;++use crate::ScalarValue;++/// The floating point types the functions in this module are defined for:+/// [`f16`](struct@f16), [`f32`] and [`f64`].+pub trait FloatSemantics: Copy {+ /// Whether [`normalize_float`] would change this value. This is a cheap+ /// check that lets callers skip rewriting data which is already+ /// normalized.+ fn needs_normalization(self) -> bool;++ /// See [`normalize_float`].+ fn normalize(self) -> Self;++ /// See [`compare_floats`].+ fn compare(self, other: Self) -> Ordering;+}++macro_rules! impl_float_semantics {+ ($(($t:ty, $bits:ty, $zero:expr)),+ $(,)?) => {+ $(impl FloatSemantics for $t {+ #[inline]+ fn needs_normalization(self) -> bool {+ // `-0.0` has only the sign bit set; no other value shares+ // this bit pattern.+ self.to_bits() == 1 << (<$bits>::BITS - 1)+ }++ #[inline]+ fn normalize(self) -> Self {+ if self.needs_normalization() { $zero } else { self }+ }++ #[inline]+ fn compare(self, other: Self) -> Ordering {+ self.total_cmp(&other)+ }+ })++ };+}+impl_float_semantics!((f16, u16, f16::ZERO), (f32, u32, 0.0), (f64, u64, 0.0));++/// `-0.0` becomes `0.0`, so two values that DataFusion treats as equal also+/// have the same bits. All other values, including NaN payloads, are kept as+/// they are.+#[inline]+pub fn normalize_float<T: FloatSemantics>(v: T) -> T {+ v.normalize()+}++/// Compares two values in the order DataFusion sorts them.+///+/// This is IEEE 754 totalOrder (`total_cmp`), which matches Arrow's sort+/// kernels and row encoding. Note that it orders `-0.0` before `+0.0`.+/// The grouped `MIN`/`MAX` accumulators start from the extremes of this+/// order, so they must change together with it.+#[inline]+pub fn compare_floats<T: FloatSemantics>(left: T, right: T) -> Ordering {+ left.compare(right)+}++/// Whether `left` sorts before `right`, the same as+/// `compare_floats(left, right).is_lt()`.+#[inline]+pub fn float_lt<T: FloatSemantics>(left: T, right: T) -> bool {+ compare_floats(left, right).is_lt()+}++/// Whether `left` sorts after `right`, the same as+/// `compare_floats(left, right).is_gt()`.+#[inline]+pub fn float_gt<T: FloatSemantics>(left: T, right: T) -> bool {+ compare_floats(left, right).is_gt()+}++/// The value that DataFusion hashes in place of a float. Values that DataFusion+/// treats as equal hash the same, so this is the value [`normalize_float`]+/// returns.+#[inline]+pub fn hash_input<T: FloatSemantics>(v: T) -> T {+ normalize_float(v)+}++/// Applies [`normalize_float`] to every float in `array`, including floats+/// nested inside lists, structs, maps, unions, dictionaries and run-end+/// encoded arrays. Arrays without floats are returned unchanged.+///+/// Callers use this before handing float data to Arrow kernels that compare+/// bit patterns, so that `+0.0 = -0.0` holds.+///+/// The common case - nothing to normalize - is allocation-free: a single+/// read-only scan of the values buffer (auto-vectorizable to an OR-reduction)+/// decides whether to fall through to the rewriting path. Only arrays that+/// contain values to normalize pay for a new buffer.+pub fn normalize_floats(array: &ArrayRef) -> ArrayRef {+ match array.data_type() {+ DataType::Float16 => normalize_primitive::<Float16Type>(array),+ DataType::Float32 => normalize_primitive::<Float32Type>(array),+ DataType::Float64 => normalize_primitive::<Float64Type>(array),+ dt if has_float_leaf(dt) => {+ let data = array.to_data();+ let children = data+ .child_data()+ .iter()+ .map(|child| normalize_floats(&make_array(child.clone())).to_data())+ .collect::<Vec<_>>();+ if children+ .iter()+ .zip(data.child_data())+ .all(|(new, old)| new.ptr_eq(old))+ {+ return Arc::clone(array);+ }+ make_array(+ data.into_builder()+ .child_data(children)+ .build()+ .expect("rewriting float leaves preserves the array layout"),+ )+ }+ _ => Arc::clone(array),+ }+}++fn normalize_primitive<T>(array: &ArrayRef) -> ArrayRef+where+ T: ArrowPrimitiveType,+ T::Native: FloatSemantics,+{+ let arr = array.as_primitive::<T>();+ if !arr.values().iter().any(|v| v.needs_normalization()) {+ return Arc::clone(array);+ }+ Arc::new(arr.unary::<_, T>(normalize_float))+}++/// Applies [`normalize_float`] to a `Float16`, `Float32` or `Float64` scalar.+/// Other scalars are returned as is.+pub fn normalize_float_scalar(scalar: ScalarValue) -> ScalarValue {+ match scalar {+ ScalarValue::Float16(Some(v)) => ScalarValue::Float16(Some(normalize_float(v))),+ ScalarValue::Float32(Some(v)) => ScalarValue::Float32(Some(normalize_float(v))),+ ScalarValue::Float64(Some(v)) => ScalarValue::Float64(Some(normalize_float(v))),+ other => other,+ }+}++/// Whether `data_type` is a float or contains a float at any depth of its+/// nested children.+pub fn has_float_leaf(data_type: &DataType) -> bool {+ match data_type {+ DataType::Float16 | DataType::Float32 | DataType::Float64 => true,+ DataType::List(f)+ | DataType::LargeList(f)+ | DataType::ListView(f)+ | DataType::LargeListView(f)+ | DataType::FixedSizeList(f, _)+ | DataType::Map(f, _)+ | DataType::RunEndEncoded(_, f) => has_float_leaf(f.data_type()),+ DataType::Struct(fields) => fields.iter().any(|f| has_float_leaf(f.data_type())),+ DataType::Union(fields, _) => {+ fields.iter().any(|(_, f)| has_float_leaf(f.data_type()))+ }+ DataType::Dictionary(_, values) => has_float_leaf(values),+ _ => false,+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{Float16Array, Float32Array, Float64Array, Int32Array, ListArray};+ use arrow::datatypes::Int32Type;++ #[test]+ fn normalize_float_folds_negative_zero() {+ assert_eq!(normalize_float(-0.0_f64).to_bits(), 0.0_f64.to_bits());+ assert_eq!(normalize_float(-0.0_f32).to_bits(), 0.0_f32.to_bits());+ assert_eq!(+ normalize_float(f16::NEG_ZERO).to_bits(),+ f16::ZERO.to_bits()+ );+ assert_eq!(+ hash_input(-0.0_f64).to_bits(),+ hash_input(0.0_f64).to_bits()+ );+ assert!((-0.0_f64).needs_normalization());+ assert!(!0.0_f64.needs_normalization());+ }++ #[test]+ fn normalize_float_keeps_other_values() {+ let neg_nan = f64::from_bits(f64::NAN.to_bits() | (1 << 63));+ let payload_nan = f64::from_bits(f64::NAN.to_bits() | 1);+ for v in [+ 1.5,+ -1.5,+ f64::INFINITY,+ f64::NEG_INFINITY,+ f64::MIN_POSITIVE,+ -f64::MIN_POSITIVE,+ f64::NAN,+ neg_nan,+ payload_nan,+ ] {+ assert!(!v.needs_normalization());+ assert_eq!(normalize_float(v).to_bits(), v.to_bits());+ }+ }++ #[test]+ fn compare_floats_is_total_order() {+ let neg_nan = f64::from_bits(f64::NAN.to_bits() | (1 << 63));+ assert_eq!(compare_floats(1.0_f64, 2.0), Ordering::Less);+ assert_eq!(compare_floats(-0.0_f64, 0.0), Ordering::Less);+ assert_eq!(compare_floats(f64::NAN, f64::INFINITY), Ordering::Greater);+ assert_eq!(compare_floats(neg_nan, f64::NEG_INFINITY), Ordering::Less);+ assert_eq!(compare_floats(f16::ONE, f16::ZERO), Ordering::Greater);+ assert!(float_lt(1.0_f64, 2.0) && !float_lt(2.0_f64, 1.0));+ assert!(float_gt(f64::NAN, f64::INFINITY) && !float_gt(1.0_f64, 1.0));+ }++ #[test]+ fn normalize_floats_rewrites_negative_zero() {+ let array: ArrayRef =+ Arc::new(Float64Array::from(vec![Some(-0.0), None, Some(1.0)]));+ let result = normalize_floats(&array);+ let result = result.as_primitive::<Float64Type>();+ assert_eq!(result.value(0).to_bits(), 0.0_f64.to_bits());+ assert!(result.is_null(1));+ assert_eq!(result.value(2), 1.0);++ let array: ArrayRef = Arc::new(Float32Array::from(vec![-0.0_f32]));+ let result = normalize_floats(&array);+ assert_eq!(+ result.as_primitive::<Float32Type>().value(0).to_bits(),+ 0.0_f32.to_bits()+ );++ let array: ArrayRef = Arc::new(Float16Array::from(vec![f16::NEG_ZERO]));+ let result = normalize_floats(&array);+ assert_eq!(+ result.as_primitive::<Float16Type>().value(0).to_bits(),+ f16::ZERO.to_bits()+ );+ }++ #[test]+ fn normalize_floats_reuses_normalized_input() {+ let array: ArrayRef = Arc::new(Float64Array::from(vec![0.0, 1.0, f64::NAN]));+ assert!(Arc::ptr_eq(&normalize_floats(&array), &array));++ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));+ assert!(Arc::ptr_eq(&normalize_floats(&array), &array));+ }++ #[test]+ fn normalize_floats_rewrites_nested_floats() {+ let list = ListArray::from_iter_primitive::<Float64Type, _, _>(vec![+ Some(vec![Some(-0.0), Some(2.0)]),+ None,+ ]);+ let array: ArrayRef = Arc::new(list);+ let result = normalize_floats(&array);+ assert!(!Arc::ptr_eq(&result, &array));+ let values = result.as_list::<i32>().value(0);+ assert_eq!(+ values.as_primitive::<Float64Type>().value(0).to_bits(),+ 0.0_f64.to_bits()+ );+ assert!(result.is_null(1));++ let list =+ ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![Some(1)])]);+ let array: ArrayRef = Arc::new(list);+ assert!(Arc::ptr_eq(&normalize_floats(&array), &array));+ }++ #[test]+ fn normalize_float_scalar_folds_negative_zero() {+ assert_eq!(+ normalize_float_scalar(ScalarValue::Float64(Some(-0.0))),+ ScalarValue::Float64(Some(0.0))+ );+ assert_eq!(+ normalize_float_scalar(ScalarValue::Float32(Some(-0.0))),+ ScalarValue::Float32(Some(0.0))+ );+ assert_eq!(+ normalize_float_scalar(ScalarValue::Float16(Some(f16::NEG_ZERO))),+ ScalarValue::Float16(Some(f16::ZERO))+ );+ assert_eq!(+ normalize_float_scalar(ScalarValue::Float64(None)),+ ScalarValue::Float64(None)+ );+ assert_eq!(+ normalize_float_scalar(ScalarValue::Int32(Some(0))),+ ScalarValue::Int32(Some(0))+ );+ }+}diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex 181cde28f9..ef29cc3ef8 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -89,6 +89,7 @@ use crate::cast::{};use crate::error::Result;use crate::error::{_internal_datafusion_err, _internal_err};+use crate::float_semantics::hash_input;#[cfg(not(feature = "force_hash_collisions"))]use crate::utils::offset_span;use std::cell::RefCell;@@ -258,24 +259,20 @@ hash_value!(i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128);hash_value!(bool, str, [u8], IntervalDayTime, IntervalMonthDayNano);macro_rules! hash_float_value {- ($(($t:ty, $i:ty)),+) => {+ ($($t:ty),+) => {$(impl HashValue for $t {+ // Hash the normalized bits so values SQL considers equal+ // (e.g. +0.0 and -0.0) hash the same.fn hash_one<S: BuildHasher>(&self, state: &S) -> u64 {- // +0.0 and -0.0 differ only in the sign bit but compare equal- // under IEEE 754; normalize -0.0 → +0.0 so Hash agrees with Eq.- let bits = <$i>::from_ne_bytes(self.to_ne_bytes());- let bits = if bits << 1 == 0 { 0 } else { bits };- state.hash_one(bits)+ state.hash_one(hash_input(*self).to_bits())}fn hash_write(&self, hasher: &mut impl Hasher) {- let bits = <$i>::from_ne_bytes(self.to_ne_bytes());- let bits: $i = if bits << 1 == 0 { 0 } else { bits };- hasher.write(&bits.to_ne_bytes())+ hasher.write(&hash_input(*self).to_bits().to_ne_bytes())}})+};}-hash_float_value!((half::f16, u16), (f32, u32), (f64, u64));+hash_float_value!(half::f16, f32, f64);#[cfg(not(feature = "force_hash_collisions"))]trait ChildHashing {diff --git a/datafusion/common/src/hash_utils/build_hasher.rs b/datafusion/common/src/hash_utils/build_hasher.rsindex 23ee6f12e3..365680e4c7 100644--- a/datafusion/common/src/hash_utils/build_hasher.rs+++ b/datafusion/common/src/hash_utils/build_hasher.rs@@ -31,6 +31,8 @@ use crate::cast::{};use crate::error::Result;use crate::error::{_internal_datafusion_err, _internal_err};+#[cfg(not(feature = "force_hash_collisions"))]+use crate::float_semantics::hash_input;#[cfg(feature = "force_hash_collisions")]use arrow::array::Array;#[cfg(not(feature = "force_hash_collisions"))]@@ -168,18 +170,16 @@ build_hasher_hash_value!(i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128)build_hasher_hash_value!(bool, str, [u8], IntervalDayTime, IntervalMonthDayNano);macro_rules! build_hasher_hash_float_value {- ($(($t:ty, $i:ty)),+) => {+ ($($t:ty),+) => {$(#[cfg(not(feature = "force_hash_collisions"))]impl BuildHasherHashValue for $t {fn hash_one_with_hasher<S: BuildHasher>(&self, state: &S) -> u64 {- let bits = <$i>::from_ne_bytes(self.to_ne_bytes());- let bits = if bits << 1 == 0 { 0 } else { bits };- state.hash_one(bits)+ state.hash_one(hash_input(*self).to_bits())}})+};}-build_hasher_hash_float_value!((half::f16, u16), (f32, u32), (f64, u64));+build_hasher_hash_float_value!(half::f16, f32, f64);#[cfg(not(feature = "force_hash_collisions"))]fn hash_null_with_hasher<S: BuildHasher>(diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rsindex 2eebfe4963..890f430ae9 100644--- a/datafusion/common/src/lib.rs+++ b/datafusion/common/src/lib.rs@@ -46,6 +46,7 @@ pub mod encryption;pub mod error;pub mod extensions;pub mod file_options;+pub mod float_semantics;pub mod format;pub mod hash_utils;pub mod heap_size;diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rsindex 0ff108ff36..7811f522ea 100644--- a/datafusion/common/src/utils/mod.rs+++ b/datafusion/common/src/utils/mod.rs@@ -1436,132 +1436,24 @@ fn fsl_values_row_number(list_size: i32, array_len: usize) -> Result<Int32Array>}/// Replace `-0.0` with `+0.0` in any `Float16`, `Float32`, or `Float64` array.-/// For non-float arrays returns the input unchanged. NaN payloads are-/// preserved.-///-/// Arrow's comparison kernels (`arrow::compute::kernels::cmp::eq` etc.) and-/// row-encoding (`arrow::row::RowConverter`) use IEEE 754 totalOrder-/// semantics, which treats `-0.0` and `+0.0` as distinct. SQL semantics-/// (PostgreSQL / IEEE 754 equality) require them to compare equal, so-/// callers normalize before invoking those kernels.-///-/// The common case - no `-0.0` present - is allocation-free: a single-/// read-only scan of the underlying buffer (auto-vectorizable to an-/// OR-reduction) decides whether to fall through to the rewriting path.-/// Only arrays that actually contain `-0.0` pay for a new buffer.+#[deprecated(+ since = "56.0.0",+ note = "use `datafusion_common::float_semantics::normalize_floats` instead"+)]pub fn normalize_float_zero(array: &ArrayRef) -> ArrayRef {- use arrow::array::{Float16Array, Float32Array, Float64Array, make_array};- use arrow::datatypes::{Float16Type, Float32Type, Float64Type};- // -0.0 has only the sign bit set; no other finite or NaN value shares- // this bit pattern, so a strict-equality scan reliably gates the rewrite.- const NEG_ZERO_F16_BITS: u16 = half::f16::NEG_ZERO.to_bits();- const NEG_ZERO_F32_BITS: u32 = (-0.0_f32).to_bits();- const NEG_ZERO_F64_BITS: u64 = (-0.0_f64).to_bits();- match array.data_type() {- DataType::Float32 => {- let arr: &Float32Array = array.as_primitive::<Float32Type>();- if !arr- .values()- .iter()- .any(|v| v.to_bits() == NEG_ZERO_F32_BITS)- {- return Arc::clone(array);- }- let normalized: Float32Array =- arr.unary(|v| if v.to_bits() << 1 == 0 { 0.0_f32 } else { v });- Arc::new(normalized)- }- DataType::Float64 => {- let arr: &Float64Array = array.as_primitive::<Float64Type>();- if !arr- .values()- .iter()- .any(|v| v.to_bits() == NEG_ZERO_F64_BITS)- {- return Arc::clone(array);- }- let normalized: Float64Array =- arr.unary(|v| if v.to_bits() << 1 == 0 { 0.0_f64 } else { v });- Arc::new(normalized)- }- DataType::Float16 => {- let arr: &Float16Array = array.as_primitive::<Float16Type>();- if !arr- .values()- .iter()- .any(|v| v.to_bits() == NEG_ZERO_F16_BITS)- {- return Arc::clone(array);- }- let normalized: Float16Array = arr.unary(|v| {- if v.to_bits() << 1 == 0 {- half::f16::from_bits(0)- } else {- v- }- });- Arc::new(normalized)- }- dt if has_float_leaf(dt) => {- let data = array.to_data();- let children = data- .child_data()- .iter()- .map(|child| normalize_float_zero(&make_array(child.clone())).to_data())- .collect::<Vec<_>>();- if children- .iter()- .zip(data.child_data())- .all(|(new, old)| new.ptr_eq(old))- {- return Arc::clone(array);- }- make_array(- data.into_builder()- .child_data(children)- .build()- .expect("rewriting float leaves preserves the array layout"),- )- }- _ => Arc::clone(array),- }+ crate::float_semantics::normalize_floats(array)}-pub fn has_float_leaf(data_type: &DataType) -> bool {- match data_type {- DataType::Float16 | DataType::Float32 | DataType::Float64 => true,- DataType::List(f)- | DataType::LargeList(f)- | DataType::ListView(f)- | DataType::LargeListView(f)- | DataType::FixedSizeList(f, _)- | DataType::Map(f, _)- | DataType::RunEndEncoded(_, f) => has_float_leaf(f.data_type()),- DataType::Struct(fields) => fields.iter().any(|f| has_float_leaf(f.data_type())),- DataType::Union(fields, _) => {- fields.iter().any(|(_, f)| has_float_leaf(f.data_type()))- }- DataType::Dictionary(_, values) => has_float_leaf(values),- _ => false,- }-}+pub use crate::float_semantics::has_float_leaf;/// Replace `-0.0` with `+0.0` in `Float16`, `Float32`, or `Float64` scalar-/// values. Other variants are returned unchanged. See [`normalize_float_zero`]-/// for context.+/// values.+#[deprecated(+ since = "56.0.0",+ note = "use `datafusion_common::float_semantics::normalize_float_scalar` instead"+)]pub fn normalize_float_zero_scalar(scalar: ScalarValue) -> ScalarValue {- match scalar {- ScalarValue::Float32(Some(v)) if v.to_bits() << 1 == 0 => {- ScalarValue::Float32(Some(0.0))- }- ScalarValue::Float64(Some(v)) if v.to_bits() << 1 == 0 => {- ScalarValue::Float64(Some(0.0))- }- ScalarValue::Float16(Some(v)) if v.to_bits() << 1 == 0 => {- ScalarValue::Float16(Some(half::f16::from_bits(0)))- }- other => other,- }+ crate::float_semantics::normalize_float_scalar(scalar)}/// Apply a struct's nulls to one of its fields.diff --git a/datafusion/functions-aggregate-common/src/min_max.rs b/datafusion/functions-aggregate-common/src/min_max.rsindex f0a4a52a4a..de2ca7e620 100644--- a/datafusion/functions-aggregate-common/src/min_max.rs+++ b/datafusion/functions-aggregate-common/src/min_max.rs@@ -31,6 +31,7 @@ use arrow::array::{};use arrow::compute;use arrow::datatypes::{DataType, IntervalUnit, TimeUnit};+use datafusion_common::float_semantics::{FloatSemantics, compare_floats};use datafusion_common::{DataFusionError, Result, ScalarValue, downcast_value, internal_err,};@@ -64,17 +65,16 @@ fn min_max_option<T: Clone + Ord>(}}-fn min_max_float_option<T: Copy>(+fn min_max_float_option<T: FloatSemantics>(lhs: Option<&T>,rhs: Option<&T>,ordering: Ordering,- cmp: impl Fn(&T, &T) -> Ordering,) -> Option<T> {match (lhs, rhs) {(None, None) => None,(Some(a), None) => Some(*a),(None, Some(b)) => Some(*b),- (Some(a), Some(b)) if cmp(a, b) == ordering => Some(*b),+ (Some(a), Some(b)) if compare_floats(*a, *b) == ordering => Some(*b),(Some(a), Some(_)) => Some(*a),}}@@ -247,15 +247,13 @@ fn min_max_scalar_same_variant(ScalarValue::Boolean(min_max_option(lhs.as_ref(), rhs.as_ref(), ordering))}(ScalarValue::Float64(lhs), ScalarValue::Float64(rhs)) => ScalarValue::Float64(- min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering, f64::total_cmp),+ min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering),),(ScalarValue::Float32(lhs), ScalarValue::Float32(rhs)) => ScalarValue::Float32(- min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering, f32::total_cmp),+ min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering),),(ScalarValue::Float16(lhs), ScalarValue::Float16(rhs)) => ScalarValue::Float16(- min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering, |a, b| {- a.total_cmp(b)- }),+ min_max_float_option(lhs.as_ref(), rhs.as_ref(), ordering),),(ScalarValue::UInt64(lhs), ScalarValue::UInt64(rhs)) => {ScalarValue::UInt64(min_max_option(lhs.as_ref(), rhs.as_ref(), ordering))diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rsindex e12adad906..03d80dd75b 100644--- a/datafusion/functions-aggregate/src/min_max.rs+++ b/datafusion/functions-aggregate/src/min_max.rs@@ -23,6 +23,7 @@ mod min_max_struct;use arrow::array::{ArrayRef, ArrowPrimitiveType, downcast_primitive};use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type};+use datafusion_common::float_semantics::{float_gt, float_lt};use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, Result, exec_err, internal_err};use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;@@ -124,7 +125,7 @@ macro_rules! primitive_max_accumulator {($t:ty, $data_type:ident, total, $bits:ident) => {{Ok(Box::new(PrimitiveGroupsAccumulator::<$t, _>::new($data_type, |cur, new| {- if new.total_cmp(cur) == Ordering::Greater {+ if float_gt(new, *cur) {*cur = new}})@@ -160,7 +161,7 @@ macro_rules! primitive_min_accumulator {($t:ty, $data_type:ident, total, $bits:ident) => {{Ok(Box::new(PrimitiveGroupsAccumulator::<$t, _>::new(&$data_type, |cur, new| {- if new.total_cmp(cur) == Ordering::Less {+ if float_lt(new, *cur) {*cur = new}})diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rsindex 5d7ae70ae7..e783913204 100644--- a/datafusion/functions-nested/src/set_ops.rs+++ b/datafusion/functions-nested/src/set_ops.rs@@ -28,7 +28,8 @@ use arrow::datatypes::DataType::{LargeList, List, Null};use arrow::datatypes::{DataType, Field, FieldRef};use arrow::row::{RowConverter, SortField};use datafusion_common::cast::{as_large_list_array, as_list_array};-use datafusion_common::utils::{ListCoercion, normalize_float_zero, offset_span};+use datafusion_common::float_semantics::normalize_floats;+use datafusion_common::utils::{ListCoercion, offset_span};use datafusion_common::{Result, assert_eq_or_internal_err, exec_err, internal_err, utils::take_function_args,};@@ -334,9 +335,9 @@ pub(crate) fn normalize_visible_values<OffsetSize: OffsetSizeTrait>() -> ArrayRef {let (first, len) = offset_span(array.offsets());if first == 0 && len == array.values().len() {- normalize_float_zero(array.values())+ normalize_floats(array.values())} else {- normalize_float_zero(&array.values().slice(first, len))+ normalize_floats(&array.values().slice(first, len))}}diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rsindex 5309deb34a..7a2fab3010 100644--- a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs+++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs@@ -28,7 +28,7 @@//! `false` so that later rules can prune the plan they filter.use super::utils::{is_false, is_null};-use datafusion_common::utils::normalize_float_zero_scalar;+use datafusion_common::float_semantics::normalize_float_scalar;use datafusion_common::{Column, DataFusionError, Result, ScalarValue, internal_err};use datafusion_expr::utils::{conjunction, disjunction, split_binary_owned, split_conjunction_owned,@@ -422,7 +422,7 @@ fn op_and_literal(predicate: &Expr) -> Result<(Operator, &ScalarValue)> {fn normalize_literal(expr: &mut Expr) {if let Expr::Literal(value, _) = expr {let original = std::mem::replace(value, ScalarValue::Null);- *value = normalize_float_zero_scalar(original);+ *value = normalize_float_scalar(original);}}diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rsindex 34ead9a69d..b0ae7a7f17 100644--- a/datafusion/optimizer/src/simplify_expressions/utils.rs+++ b/datafusion/optimizer/src/simplify_expressions/utils.rs@@ -18,7 +18,7 @@//! Utility functions for expression simplificationuse datafusion_common::cse::NormalizeEq;-use datafusion_common::utils::normalize_float_zero_scalar;+use datafusion_common::float_semantics::normalize_float_scalar;use datafusion_common::{Result, ScalarValue, internal_err};use datafusion_expr::{Case, Expr, Like, Operator,@@ -239,8 +239,8 @@ pub fn is_eq_and_ne_with_different_literal(eq_expr: &Expr, ne_expr: &Expr) -> boExpr::Literal(right_value, right_metadata),) => {left_metadata != right_metadata- || normalize_float_zero_scalar(left_value.clone())- != normalize_float_zero_scalar(right_value.clone())+ || normalize_float_scalar(left_value.clone())+ != normalize_float_scalar(right_value.clone())}_ => left != right,}diff --git a/datafusion/physical-expr-common/src/datum.rs b/datafusion/physical-expr-common/src/datum.rsindex 416507c618..182eebf880 100644--- a/datafusion/physical-expr-common/src/datum.rs+++ b/datafusion/physical-expr-common/src/datum.rs@@ -23,8 +23,8 @@ use arrow::compute::kernels::cmp::{};use arrow::compute::{SortOptions, ilike, like, nilike, nlike};use arrow::error::ArrowError;-use datafusion_common::utils::{- has_float_leaf, normalize_float_zero, normalize_float_zero_scalar,+use datafusion_common::float_semantics::{+ has_float_leaf, normalize_float_scalar, normalize_floats,};use datafusion_common::{Result, ScalarValue};use datafusion_common::{arrow_datafusion_err, assert_or_internal_err, internal_err};@@ -99,9 +99,9 @@ pub fn apply_cmp(fn normalize_cmp_input(cv: &ColumnarValue) -> ColumnarValue {match cv {- ColumnarValue::Array(a) => ColumnarValue::Array(normalize_float_zero(a)),+ ColumnarValue::Array(a) => ColumnarValue::Array(normalize_floats(a)),ColumnarValue::Scalar(s) => {- ColumnarValue::Scalar(normalize_float_zero_scalar(s.clone()))+ ColumnarValue::Scalar(normalize_float_scalar(s.clone()))}}}@@ -151,7 +151,7 @@ pub fn compare_with_eq(fn normalize_nested_float_zero(array: &dyn Array) -> Option<ArrayRef> {has_float_leaf(array.data_type())- .then(|| normalize_float_zero(&make_array(array.to_data())))+ .then(|| normalize_floats(&make_array(array.to_data())))}/// Compare on nested type List, Struct, and so ondiff --git a/datafusion/physical-expr/src/intervals/cp_solver.rs b/datafusion/physical-expr/src/intervals/cp_solver.rsindex b28a42af5b..08d961f286 100644--- a/datafusion/physical-expr/src/intervals/cp_solver.rs+++ b/datafusion/physical-expr/src/intervals/cp_solver.rs@@ -299,12 +299,11 @@ pub fn propagate_arithmetic(/// Compare two scalars the way runtime `Eq` does, instead of with/// `ScalarValue`'s bit-wise equality. `ScalarValue::PartialEq` is bit-wise for/// floats (`to_bits`), under which `-0.0 != +0.0`; runtime `Eq` normalizes-/// them via `normalize_float_zero_scalar` (see+/// them via `normalize_float_scalar` (see/// `physical-expr-common/src/datum.rs`).fn singleton_values_equal(left: &ScalarValue, right: &ScalarValue) -> bool {- use datafusion_common::utils::normalize_float_zero_scalar;- normalize_float_zero_scalar(left.clone())- == normalize_float_zero_scalar(right.clone())+ use datafusion_common::float_semantics::normalize_float_scalar;+ normalize_float_scalar(left.clone()) == normalize_float_scalar(right.clone())}/// This function refines intervals `left_child` and `right_child` by applyingdiff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rsindex 7ab70bf0c8..2836b3177e 100644--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs@@ -45,9 +45,9 @@ use arrow::datatypes::{BinaryViewType, DataType, Field, Int8Type, Int16Type, Int32Type, Int64Type, Schema,SchemaRef, StringViewType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,};+use datafusion_common::float_semantics::{has_float_leaf, normalize_floats};use datafusion_common::hash_utils::RandomState;use datafusion_common::hash_utils::create_hashes;-use datafusion_common::utils::{has_float_leaf, normalize_float_zero};use datafusion_common::{Result, not_impl_err};use datafusion_execution::memory_pool::proxy::{HashTableAllocExt, VecAllocExt};use datafusion_expr::{EmitTo, GroupSelection};@@ -1070,7 +1070,7 @@ impl<const STREAMING: bool> GroupValues for GroupValuesColumn<STREAMING> {let normalized: Option<Vec<ArrayRef>> = cols.iter().any(|col| col.data_type().is_nested() && has_float_leaf(col.data_type()))- .then(|| cols.iter().map(normalize_float_zero).collect());+ .then(|| cols.iter().map(normalize_floats).collect());let cols = normalized.as_deref().unwrap_or(cols);// `try_new` and the reset points in `emit` / `clear_shrink` keepdiff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rsindex ebd1e071cf..99ecee10c0 100644--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs@@ -98,9 +98,9 @@ where} else {unsafe { *array_values.get_unchecked(rhs_row) }};- // `left` was already canonicalized on append; canonicalize the+ // `left` was already normalized on append; normalize the// input so ±0 (and any future equivalence class) compares equal.- if left.is_eq(right.canonicalize()) {+ if left.is_eq(right.normalize_key()) {cmp_buf[i / 8] |= 1 << (i % 8);}}@@ -141,7 +141,7 @@ wherecontinue;}- if !self.group_values[lhs_row].is_eq(array.value(rhs_row).canonicalize()) {+ if !self.group_values[lhs_row].is_eq(array.value(rhs_row).normalize_key()) {equal_to_results.set_bit(idx, false);}}@@ -165,7 +165,7 @@ where}self.group_values[lhs_row]- .is_eq(array.as_primitive::<T>().value(rhs_row).canonicalize())+ .is_eq(array.as_primitive::<T>().value(rhs_row).normalize_key())}fn append_val(&mut self, array: &ArrayRef, row: usize) -> Result<()> {@@ -177,11 +177,11 @@ where} else {self.nulls.append_non_null();self.group_values- .push(array.as_primitive::<T>().value(row).canonicalize());+ .push(array.as_primitive::<T>().value(row).normalize_key());}} else {self.group_values- .push(array.as_primitive::<T>().value(row).canonicalize());+ .push(array.as_primitive::<T>().value(row).normalize_key());}Ok(())@@ -227,7 +227,7 @@ whereself.group_values.push(T::default_value());} else {self.nulls.append_non_null();- self.group_values.push(arr.value(row).canonicalize());+ self.group_values.push(arr.value(row).normalize_key());}}}@@ -235,7 +235,7 @@ where(true, Nulls::None) => {self.nulls.append_n_non_nulls(rows.len());for &row in rows {- self.group_values.push(arr.value(row).canonicalize());+ self.group_values.push(arr.value(row).normalize_key());}}@@ -247,7 +247,7 @@ where(false, _) => {for &row in rows {- self.group_values.push(arr.value(row).canonicalize());+ self.group_values.push(arr.value(row).normalize_key());}}}diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rsindex a2ff0b6adb..ee3f360886 100644--- a/datafusion/physical-plan/src/aggregates/group_values/row.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs@@ -25,9 +25,9 @@ use arrow::compute::cast;use arrow::datatypes::{DataType, SchemaRef};use arrow::row::{RowConverter, Rows, SortField};use datafusion_common::Result;+use datafusion_common::float_semantics::normalize_floats;use datafusion_common::hash_utils::RandomState;use datafusion_common::hash_utils::create_hashes;-use datafusion_common::utils::normalize_float_zero;use datafusion_execution::memory_pool::proxy::VecAllocExt;use datafusion_expr::{EmitTo, GroupSelection};use hashbrown::hash_table::HashTable;@@ -125,8 +125,7 @@ impl GroupValues for GroupValuesRows {// Normalize -0.0 → +0.0 so RowConverter (IEEE 754 totalOrder) and// primitive hashing both group ±0 together. No-op for non-float// columns.- let normalized_cols: Vec<ArrayRef> =- cols.iter().map(normalize_float_zero).collect();+ let normalized_cols: Vec<ArrayRef> = cols.iter().map(normalize_floats).collect();let cols = normalized_cols.as_slice();// Convert the group keys into the row formatdiff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rsindex 816b020c39..05f2a26ca0 100644--- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs@@ -23,6 +23,7 @@ use arrow::array::{};use arrow::datatypes::{DataType, i256};use datafusion_common::Result;+use datafusion_common::float_semantics::normalize_float;use datafusion_common::hash_utils::RandomState;use datafusion_common::utils::split_vec_min_alloc;use datafusion_execution::memory_pool::proxy::VecAllocExt;@@ -39,11 +40,10 @@ pub trait HashValue {/// Return a canonical representative whose bit pattern is identical for/// all values that should be grouped together. Default is the identity;- /// floats override this to fold `-0.0` into `+0.0` so the bit-equal- /// `is_eq` check used during insertion treats them as the same group.- /// NaN payload bits are preserved.+ /// floats override this with [`normalize_float`] so the bit-equal+ /// `is_eq` check used during insertion follows SQL equality.#[inline]- fn canonicalize(self) -> Self+ fn normalize_key(self) -> SelfwhereSelf: Sized,{@@ -75,7 +75,7 @@ macro_rules! hash_float {$(impl HashValue for $t {#[cfg(not(feature = "force_hash_collisions"))]fn hash(&self, state: &RandomState) -> u64 {- state.hash_one(self.canonicalize().to_bits())+ state.hash_one(datafusion_common::float_semantics::hash_input(*self).to_bits())}#[cfg(feature = "force_hash_collisions")]@@ -84,10 +84,8 @@ macro_rules! hash_float {}#[inline]- fn canonicalize(self) -> Self {- let bits = self.to_bits();- let bits = if bits << 1 == 0 { 0 } else { bits };- Self::from_bits(bits)+ fn normalize_key(self) -> Self {+ normalize_float(self)}})+};@@ -149,7 +147,7 @@ where// Fold equivalence-class duplicates (e.g. `-0.0` → `+0.0`)// so the bit-equal `is_eq` matches and the stored value is// the canonical representative.- let key = key.canonicalize();+ let key = key.normalize_key();let state = &self.random_state;let hash = key.hash(state);let insert = self.map.entry(diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rsindex 89ddaaa0be..519c16d5b8 100644--- a/datafusion/physical-plan/src/aggregates/topk/heap.rs+++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs@@ -32,6 +32,7 @@ use arrow::buffer::ScalarBuffer;use arrow::datatypes::{DataType, i256};use datafusion_common::Result;use datafusion_common::exec_datafusion_err;+use datafusion_common::float_semantics::compare_floats;use half::f16;use std::cmp::Ordering;@@ -550,7 +551,7 @@ macro_rules! compare_float {$(impl Comparable for Option<$t> {fn comp(&self, other: &Self) -> Ordering {match (self, other) {- (Some(me), Some(other)) => me.total_cmp(other),+ (Some(me), Some(other)) => compare_floats(*me, *other),(Some(_), None) => Ordering::Greater,(None, Some(_)) => Ordering::Less,(None, None) => Ordering::Equal,@@ -560,7 +561,7 @@ macro_rules! compare_float {$(impl Comparable for $t {fn comp(&self, other: &Self) -> Ordering {- self.total_cmp(other)+ compare_floats(*self, *other)}})+};diff --git a/datafusion/physical-plan/src/joins/asof_join.rs b/datafusion/physical-plan/src/joins/asof_join.rsindex bc38c28b43..0029159e15 100644--- a/datafusion/physical-plan/src/joins/asof_join.rs+++ b/datafusion/physical-plan/src/joins/asof_join.rs@@ -79,10 +79,10 @@ use arrow::array::{use arrow::buffer::NullBuffer;use arrow::compute::{SortOptions, interleave};use arrow::datatypes::{Schema, SchemaRef};+use datafusion_common::float_semantics::normalize_floats;use datafusion_common::stats::Precision;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::memory::RecordBatchMemoryCounter;-use datafusion_common::utils::normalize_float_zero;use datafusion_common::{ColumnStatistics, JoinSide, JoinType, NullEquality, Result, Statistics,assert_eq_or_internal_err, internal_err, plan_err, project_schema,@@ -915,7 +915,7 @@ impl InputCursor {// Arrow's comparator distinguishes -0.0 from +0.0, while SQL does// not. Normalizing once here is cheaper than fixing every row-level// comparison and lets the cached comparator use the arrays directly.- self.match_array = Some(normalize_float_zero(&match_array));+ self.match_array = Some(normalize_floats(&match_array));self.key_batch_id += 1;self.batch = Some(batch);}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex b374764033..135c1c4482 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -78,12 +78,12 @@ use arrow::record_batch::RecordBatch;use arrow::util::bit_util;use arrow_schema::{DataType, Schema};use datafusion_common::config::ConfigOptions;+use datafusion_common::float_semantics::normalize_floats;use datafusion_common::stats::Precision;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::utils::memory::{RecordBatchMemoryCounter, estimate_memory_size, get_record_batch_memory_size,};-use datafusion_common::utils::normalize_float_zero;use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,plan_err, project_schema,@@ -3293,7 +3293,7 @@ fn build_in_memory(// comparison does not rescan the build side. Arrays without `-0.0`, and// non-float keys, are shared rather than copied.let normalized_values: Vec<ArrayRef> =- left_values.iter().map(normalize_float_zero).collect();+ left_values.iter().map(normalize_floats).collect();// Join keys that are plain columns share the buffers of `batch`, any other// expression evaluates to new arrays that are kept for the whole join.diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rsindex 98b32b3151..1a7b4da6da 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs@@ -26,7 +26,7 @@ use arrow::{};use arrow_schema::{DataType, Schema, SchemaRef, SortOptions};use datafusion_common::NullEquality;-use datafusion_common::utils::normalize_float_zero;+use datafusion_common::float_semantics::normalize_floats;use datafusion_common::{Result, ScalarValue, internal_err};use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};use datafusion_expr::{ColumnarValue, JoinType, Operator};@@ -608,7 +608,7 @@ fn buffered_extreme(let last = values.slice(end - 1, 1);// `apply_cmp` normalizes `-0.0` only in flat float scalars, not inside a// `ScalarValue::Dictionary`, so normalize the key before taking it.- let extreme = normalize_float_zero(&last);+ let extreme = normalize_floats(&last);Ok(Some(ColumnarValue::Scalar(ScalarValue::try_from_array(&extreme, 0,)?)))diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 6bee6c0ebe..06ad205639 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -68,11 +68,11 @@ use arrow::datatypes::{use arrow_ord::ord::{DynComparator, make_comparator};use arrow_schema::{DataType, SortOptions, TimeUnit};use datafusion_common::cast::as_boolean_array;+use datafusion_common::float_semantics::normalize_floats;use datafusion_common::hash_utils::RandomState;use datafusion_common::hash_utils::create_hashes;use datafusion_common::stats::Precision;use datafusion_common::utils::memory::RecordBatchMemoryCounter;-use datafusion_common::utils::normalize_float_zero;use datafusion_common::{DataFusionError, JoinSide, JoinType, NullEquality, Result, SharedResult,internal_datafusion_err, not_impl_err, plan_err,@@ -2444,7 +2444,7 @@ pub(super) fn equal_rows_arr(}/// Same as [`equal_rows_arr`], but `left_arrays` must already have float-/// `-0.0` rewritten to `+0.0` (see `normalize_float_zero`), so a comparator+/// `-0.0` rewritten to `+0.0` (see `normalize_floats`), so a comparator/// built here skips that scan on the left side. Hash join keeps its build/// keys in that form, which saves rescanning the whole build side for every/// probe batch.@@ -2714,15 +2714,15 @@ impl JoinKeyComparator {// treats `-0.0` / `+0.0` as distinct. Normalize float arrays// so SMJ / piecewise-merge equi-keys honor SQL equality;// no-op (Arc::clone) for non-floats and for float arrays- // that contain no `-0.0`. `normalize_float_zero` preserves+ // that contain no `-0.0`. `normalize_floats` preserves// null positions, so the original null masks below remain// valid. A left side that is already normalized is used as is.let l_norm = if normalize_left {- normalize_float_zero(l)+ normalize_floats(l)} else {Arc::clone(l)};- let r_norm = normalize_float_zero(r);+ let r_norm = normalize_floats(r);let inner = make_comparator(l_norm.as_ref(), r_norm.as_ref(), *opts)?;if null_equality == NullEquality::NullEqualsNothing {let ln = l.logical_nulls().filter(|n| n.null_count() > 0);diff --git a/datafusion/physical-plan/src/repartition/range.rs b/datafusion/physical-plan/src/repartition/range.rsindex c05f1e531e..a718cfb341 100644--- a/datafusion/physical-plan/src/repartition/range.rs+++ b/datafusion/physical-plan/src/repartition/range.rs@@ -24,6 +24,7 @@ use arrow::array::*;use arrow::compute::SortOptions;use arrow::datatypes::*;use arrow::row::{Row, RowConverter, Rows, SortField};+use datafusion_common::float_semantics::compare_floats;use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err, not_impl_err, plan_err,validate_range_split_points,@@ -471,14 +472,14 @@ macro_rules! impl_float_values_router {if !descending {for (idx, &val) in values.iter().enumerate() {let p = split_points.partition_point(|&sp| {- sp.total_cmp(&val) != Ordering::Greater+ compare_floats(sp, val) != Ordering::Greater});emit(idx, p);}} else {for (idx, &val) in values.iter().enumerate() {let p = split_points.partition_point(|&sp| {- sp.total_cmp(&val) != Ordering::Less+ compare_floats(sp, val) != Ordering::Less});emit(idx, p);}@@ -492,11 +493,11 @@ macro_rules! impl_float_values_router {let val = array.value(idx);let p = if !descending {split_points.partition_point(|&sp| {- sp.total_cmp(&val) != Ordering::Greater+ compare_floats(sp, val) != Ordering::Greater})} else {split_points.partition_point(|&sp| {- sp.total_cmp(&val) != Ordering::Less+ compare_floats(sp, val) != Ordering::Less})};emit(idx, p);