Apache DataFusion SQL Query Engine
IN LIST: treat signed zeros as equal (#25186)
Geoffrey Claude94486b1e9355
diff --git a/datafusion/common/src/float_semantics.rs b/datafusion/common/src/float_semantics.rsindex 04ae911800..fb120ed523 100644--- a/datafusion/common/src/float_semantics.rs+++ b/datafusion/common/src/float_semantics.rs@@ -189,8 +189,12 @@ pub fn has_float_leaf(data_type: &DataType) -> bool {#[cfg(test)]mod tests {use super::*;- use arrow::array::{Float16Array, Float32Array, Float64Array, Int32Array, ListArray};- use arrow::datatypes::Int32Type;+ use crate::{Result, ScalarValue};+ use arrow::array::{+ DictionaryArray, Float16Array, Float32Array, Float64Array, Int8Array, Int32Array,+ ListArray,+ };+ use arrow::datatypes::{Field, Int8Type, Int32Type};#[test]fn normalize_float_folds_negative_zero() {@@ -295,4 +299,100 @@ mod tests {let array: ArrayRef = Arc::new(list);assert!(Arc::ptr_eq(&normalize_floats(&array), &array));}++ #[test]+ fn normalize_floats_in_dictionary_arrays_and_scalars() -> Result<()> {+ let nan = f64::from_bits(0x7ff8_0000_0000_0001);+ let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);+ let array: ArrayRef = Arc::new(DictionaryArray::try_new(+ keys.clone(),+ Arc::new(Float64Array::from(vec![-0.0, nan, 1.0])),+ )?);++ let normalized = normalize_floats(&array);+ let dictionary = normalized.as_dictionary::<Int8Type>();+ assert_eq!(dictionary.keys(), &keys);+ let values = dictionary.values().as_primitive::<Float64Type>();+ assert_eq!(values.value(0).to_bits(), 0.0_f64.to_bits());+ assert_eq!(values.value(1).to_bits(), nan.to_bits());+ assert_eq!(values.value(2), 1.0);++ assert!(Arc::ptr_eq(&normalize_floats(&normalized), &normalized));++ assert_eq!(+ ScalarValue::try_from_array(&array, 0)?.normalize(),+ ScalarValue::try_from_array(&normalized, 0)?+ );++ Ok(())+ }++ #[test]+ fn normalize_preserves_union_and_run_metadata() -> Result<()> {+ use arrow::datatypes::{UnionFields, UnionMode};++ let run_ends = Arc::new(Field::new("ends", DataType::Int32, false));+ let values = Arc::new(Field::new("samples", DataType::Float64, true));+ let fields = UnionFields::try_new(+ [7, 42],+ [+ Field::new(+ "runs",+ DataType::RunEndEncoded(Arc::clone(&run_ends), Arc::clone(&values)),+ true,+ ),+ Field::new("other", DataType::Int32, true),+ ],+ )?;+ let nan = f64::from_bits(0x7ff8_0000_0000_0001);+ for mode in [UnionMode::Sparse, UnionMode::Dense] {+ let wrap = |value| {+ ScalarValue::Union(+ Some((+ 7,+ Box::new(ScalarValue::RunEndEncoded(+ Arc::clone(&run_ends),+ Arc::clone(&values),+ Box::new(ScalarValue::Float64(value)),+ )),+ )),+ fields.clone(),+ mode,+ )+ };+ for (value, expected) in [+ (Some(-0.0), Some(0.0)),+ (Some(nan), Some(nan)),+ (None, None),+ ] {+ assert_eq!(wrap(value).normalize(), wrap(expected));+ }+ let null = ScalarValue::Union(None, fields.clone(), mode);+ assert_eq!(null.clone().normalize(), null);+ }+ Ok(())+ }++ #[test]+ fn normalize_preserves_sliced_list() {+ let nan = f64::from_bits(0x7ff8_0000_0000_0001);+ let list = Arc::new(+ ListArray::from_iter_primitive::<Float64Type, _, _>([+ Some(vec![Some(99.0)]),+ Some(vec![Some(-0.0), Some(nan), None]),+ Some(vec![Some(42.0)]),+ ])+ .slice(1, 1),+ );+ let ScalarValue::List(normalized) = ScalarValue::List(list).normalize() else {+ panic!("normalization must preserve the scalar variant");+ };+ assert_eq!(normalized.len(), 1);+ let child = normalized.value(0);+ let child = child.as_primitive::<Float64Type>();+ assert_eq!(child.len(), 3);+ assert_eq!(child.value(0).to_bits(), 0.0_f64.to_bits());+ assert_eq!(child.value(1).to_bits(), nan.to_bits());+ assert!(child.is_null(2));+ }}diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 3aa9a1d3da..fa7d4e3278 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -62,7 +62,7 @@ use arrow::array::{Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, UnionArray,- downcast_run_array, make_comparator, new_empty_array, new_null_array,+ downcast_array, downcast_run_array, make_comparator, new_empty_array, new_null_array,};use arrow::buffer::{BooleanBuffer, ScalarBuffer};use arrow::compute::kernels::cast::{CastOptions, cast_with_options};@@ -2762,24 +2762,43 @@ impl ScalarValue {)}- /// Applies [`normalize_float`] to a `Float16`, `Float32` or `Float64`- /// value, so `-0.0` becomes `0.0`. Other values are returned as is.+ /// Applies DataFusion's floating-point equality semantics to this value,+ /// including floats inside nested and encoded values. Negative zero becomes+ /// positive zero; nulls, NaN payloads and other values are preserved.///- /// [`normalize_float`]: crate::float_semantics::normalize_float- pub fn normalize(self) -> Self {- use crate::float_semantics::normalize_float;- match self {- ScalarValue::Float16(Some(v)) => {- ScalarValue::Float16(Some(normalize_float(v)))- }- ScalarValue::Float32(Some(v)) => {- ScalarValue::Float32(Some(normalize_float(v)))+ /// See [`crate::float_semantics`] for the shared normalization rules.+ pub fn normalize(mut self) -> Self {+ use crate::float_semantics::{normalize_float, normalize_floats};++ fn normalize_array<A: Array + From<ArrayData> + 'static>(array: &mut Arc<A>) {+ let original: ArrayRef = Arc::clone(array) as ArrayRef;+ let normalized = normalize_floats(&original);+ if !Arc::ptr_eq(&original, &normalized) {+ *array = Arc::new(downcast_array(normalized.as_ref()));}- ScalarValue::Float64(Some(v)) => {- ScalarValue::Float64(Some(normalize_float(v)))+ }++ fn normalize(scalar: &mut ScalarValue) {+ match scalar {+ ScalarValue::Float16(Some(value)) => *value = normalize_float(*value),+ ScalarValue::Float32(Some(value)) => *value = normalize_float(*value),+ ScalarValue::Float64(Some(value)) => *value = normalize_float(*value),+ ScalarValue::FixedSizeList(array) => normalize_array(array),+ ScalarValue::List(array) => normalize_array(array),+ ScalarValue::LargeList(array) => normalize_array(array),+ ScalarValue::ListView(array) => normalize_array(array),+ ScalarValue::LargeListView(array) => normalize_array(array),+ ScalarValue::Struct(array) => normalize_array(array),+ ScalarValue::Map(array) => normalize_array(array),+ ScalarValue::Union(Some((_, value)), _, _)+ | ScalarValue::Dictionary(_, value)+ | ScalarValue::RunEndEncoded(_, _, value) => normalize(value),+ _ => {}}- other => other,}++ normalize(&mut self);+ self}/// whether this value is null or not.diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex af5ad6d43b..e1c625fdf5 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -16,6 +16,7 @@ use std::sync::Arc;use std::sync::LazyLock;use datafusion_common::config::ConfigOptions;+use datafusion_common::float_semantics::has_float_leaf;use datafusion_common::nested_struct::has_one_of_more_common_fields;use datafusion_common::{DFSchema, DataFusionError, Result, ScalarValue, exec_datafusion_err, internal_err,@@ -2234,8 +2235,9 @@ fn are_inlist_and_eq_and_match_neg(fn inlists_have_set_comparable_literals(left: &Expr, right: &Expr) -> bool {match (left, right) {(Expr::InList(l), Expr::InList(r)) => l.list.iter().chain(&r.list).all(|item| {- item.as_literal()- .is_some_and(|value| !value.is_null() && !value.data_type().is_floating())+ item.as_literal().is_some_and(|value| {+ !value.is_null() && !has_float_leaf(&value.data_type())+ })}),_ => false,}diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex bb42936052..f1d772ae26 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -19,6 +19,7 @@ use arrow::compute::kernels::boolean::{not, or_kleene};use arrow::compute::kernels::cmp::eq as arrow_eq;use arrow::datatypes::*;+use datafusion_common::float_semantics::normalize_floats;use datafusion_common::tree_node::TreeNode;use datafusion_common::{DFSchema, Result, ScalarValue, assert_or_internal_err, exec_err,@@ -71,6 +72,13 @@ fn supports_arrow_eq(dt: &DataType) -> bool {}}+fn normalize_in_list_value(value: ColumnarValue) -> ColumnarValue {+ match value {+ ColumnarValue::Array(array) => ColumnarValue::Array(normalize_floats(&array)),+ ColumnarValue::Scalar(scalar) => ColumnarValue::Scalar(scalar.normalize()),+ }+}+/// Evaluates the list of expressions into an array, flattening any dictionariesfn evaluate_list(list: &[Arc<dyn PhysicalExpr>],@@ -368,12 +376,14 @@ impl PhysicalExpr for InListExpr {// Use Arrow's vectorized eq kernel for types it supports (primitive,// boolean, string, binary, dictionary), falling back to row-by-row// comparator for unsupported types (nested, RunEndEncoded, etc.).- let value = value.into_array(num_rows)?;+ // Normalize the left side once for the whole list. Doing this+ // outside `compare_one` avoids rescanning it for every item.+ let value = normalize_in_list_value(value).into_array(num_rows)?;let lhs_supports_arrow_eq = supports_arrow_eq(value.data_type());// Helper: compare value against a single list expressionlet compare_one = |expr: &Arc<dyn PhysicalExpr>| -> Result<BooleanArray> {- match expr.evaluate(batch)? {+ match normalize_in_list_value(expr.evaluate(batch)?) {ColumnarValue::Array(array) => {if lhs_supports_arrow_eq&& supports_arrow_eq(array.data_type())@@ -1287,20 +1297,28 @@ mod tests {Arc::new(schema.clone()),vec![Arc::new(Int32Array::from(vec![0, 0, 0]))],)?;- let needle = lit(ScalarValue::Dictionary(- Box::new(DataType::Int16),- Box::new(ScalarValue::Dictionary(- Box::new(DataType::Int8),- Box::new(ScalarValue::Int32(Some(2))),- )),- ));+ for (value, list) in [+ (ScalarValue::Int32(Some(2)), vec![lit(1_i32), lit(2_i32)]),+ (+ ScalarValue::Float64(Some(-0.0)),+ vec![lit(0.0_f64), lit(1.0_f64)],+ ),+ ] {+ let needle = lit(ScalarValue::Dictionary(+ Box::new(DataType::Int16),+ Box::new(ScalarValue::Dictionary(+ Box::new(DataType::Int8),+ Box::new(value),+ )),+ ));- let expr = in_list(needle, vec![lit(1_i32), lit(2_i32)], &false, &schema)?;- let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;- assert_eq!(- as_boolean_array(&result),- &BooleanArray::from(vec![true, true, true])- );+ let expr = in_list(needle, list, &false, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(+ as_boolean_array(&result),+ &BooleanArray::from(vec![true, true, true])+ );+ }Ok(())}@@ -3391,6 +3409,44 @@ mod tests {Ok(())}+ #[test]+ fn test_in_list_with_columns_float_signed_zero() -> Result<()> {+ use arrow::compute::cast;++ for (data_type, dictionary) in [+ (DataType::Float32, false),+ (DataType::Float64, false),+ (DataType::Float64, true),+ ] {+ let mut left = cast(&Float64Array::from(vec![0.0, -0.0, 1.0]), &data_type)?;+ let mut right = cast(&Float64Array::from(vec![-0.0, 0.0, 2.0]), &data_type)?;+ if dictionary {+ left = wrap_in_dict(left);+ right = wrap_in_dict(right);+ }+ let scalar = lit(ScalarValue::try_from_array(left.as_ref(), 1)?);+ let batch = RecordBatch::try_from_iter([("a", left), ("b", right)])?;+ let schema = batch.schema();++ // Exercise array and scalar values on both sides, including scalar+ // normalization before the left-hand side is broadcast.+ for (left, right) in [+ (col("a", &schema)?, col("b", &schema)?),+ (col("a", &schema)?, Arc::clone(&scalar)),+ (scalar, col("a", &schema)?),+ ] {+ let expr = make_in_list_with_columns(left, vec![right], false);+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(+ as_boolean_array(&result),+ &BooleanArray::from(vec![true, true, false]),+ "{data_type:?}, dictionary={dictionary}: {expr}"+ );+ }+ }+ Ok(())+ }+/// Tests that short-circuit evaluation produces correct results./// When all rows match after the first list item, remaining items/// should be skipped without affecting correctness.@@ -3916,6 +3972,23 @@ mod tests {Ok(())}+ #[test]+ fn test_try_new_from_array_dict_haystack_float64_signed_zero() -> Result<()> {+ // One value beyond the branchless limit selects the hash-set strategy.+ let list_len =+ <Float64Type as branchless_filter::BranchlessFilterType>::MAX_LIST_LEN + 1;+ let haystack = make_f64_dict_array(vec![Some(-0.0); list_len]);+ let needles: ArrayRef = Arc::new(Float64Array::from(vec![0.0]));+ for needles in [Arc::clone(&needles), wrap_in_dict(needles)] {+ assert_eq!(+ eval_in_list_from_array(needles, Arc::clone(&haystack))?,+ BooleanArray::from(vec![true])+ );+ }++ Ok(())+ }+#[test]fn test_try_new_from_array_type_mismatch_rejects() -> Result<()> {let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);diff --git a/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs b/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rsindex 9df0c53e4e..d2822982c8 100644--- a/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs@@ -2,12 +2,13 @@// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements-use arrow::array::{Array, ArrayRef, BooleanArray, make_comparator};+use arrow::array::{Array, ArrayRef, BooleanArray, make_array, make_comparator};use arrow::buffer::{BooleanBuffer, NullBuffer};use arrow::compute::SortOptions;use arrow::datatypes::DataType;use arrow::util::bit_iterator::BitIndexIterator;use datafusion_common::Result;+use datafusion_common::float_semantics::{has_float_leaf, normalize_floats};use datafusion_common::hash_utils::{RandomState, with_hashes};use hashbrown::HashTable;@@ -40,6 +41,8 @@ impl ArrayStaticFilter {});}+ // Hashing treats both signed zeros alike; the comparator must do so too.+ let in_array = normalize_floats(&in_array);let state = RandomState::default();let table = Self::build_haystack_table(&in_array, &state)?;@@ -125,6 +128,11 @@ impl StaticFilter for ArrayStaticFilter {));}- self.find_needles_in_haystack(v, negated)+ if has_float_leaf(v.data_type()) {+ let normalized = normalize_floats(&make_array(v.to_data()));+ self.find_needles_in_haystack(normalized.as_ref(), negated)+ } else {+ self.find_needles_in_haystack(v, negated)+ }}}diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rsindex bc2ab9d272..7d7f9c7c2f 100644--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs@@ -4,10 +4,13 @@use std::sync::Arc;-use arrow::array::{ArrayRef, AsArray};-use arrow::compute::take;-use arrow::datatypes::DataType;+use arrow::array::{ArrayRef, ArrowNativeTypeOp, AsArray, PrimitiveArray};+use arrow::compute::{concat, take};+use arrow::datatypes::{+ ArrowPrimitiveType, DataType, Float16Type, Float32Type, Float64Type,+};use datafusion_common::Result;+use datafusion_common::float_semantics::FloatSemantics;use super::array_static_filter::ArrayStaticFilter;use super::byte_view_filter::instantiate_byte_view_filter;@@ -21,6 +24,13 @@ pub(super) fn instantiate_static_filter(needle_data_type: &DataType,) -> Result<StaticFilterRef> {let in_array = flatten_dictionary_haystack(in_array)?;+ // Expand before choosing a filter so its cutoff counts every physical key.+ let in_array = match in_array.data_type() {+ DataType::Float16 => add_missing_signed_zero::<Float16Type>(in_array)?,+ DataType::Float32 => add_missing_signed_zero::<Float32Type>(in_array)?,+ DataType::Float64 => add_missing_signed_zero::<Float64Type>(in_array)?,+ _ => in_array,+ };let filter = if view_types_match(needle_data_type, in_array.data_type())&& let Some(filter) = instantiate_byte_view_filter(&in_array)?@@ -43,6 +53,32 @@ pub(super) fn instantiate_static_filter(}}+/// SQL membership equates the zero signs, while the primitive filters compare bits.+fn add_missing_signed_zero<T: ArrowPrimitiveType>(in_array: ArrayRef) -> Result<ArrayRef>+where+ T::Native: FloatSemantics,+{+ let array = in_array.as_primitive::<T>();+ let positive_zero = T::Native::ZERO;+ let negative_zero = positive_zero.neg_wrapping();+ let mut positive_present = false;+ let mut negative_present = false;+ for value in array.iter().flatten() {+ positive_present = positive_present || value.is_eq(positive_zero);+ negative_present = negative_present || value.needs_normalization();+ if positive_present && negative_present {+ return Ok(in_array);+ }+ }+ let missing = match (positive_present, negative_present) {+ (true, false) => negative_zero,+ (false, true) => positive_zero,+ _ => return Ok(in_array),+ };+ let alias = PrimitiveArray::<T>::from_value(missing, 1);+ Ok(concat(&[array, &alias])?)+}+/// Raw view access requires the expression and list to use the same view type./// Dictionary wrappers do not change the expression's value type.fn view_types_match(needle_type: &DataType, list_type: &DataType) -> bool {@@ -72,9 +108,10 @@ mod tests {use std::sync::Arc;use arrow::array::{- BooleanArray, DictionaryArray, Int8Array, Int16Array, Int32Array,+ BooleanArray, DictionaryArray, Float32Array, Int8Array, Int16Array, Int32Array,};+ use super::super::branchless_filter::BranchlessFilterType;use super::*;fn nested_dictionary(keys: Int16Array) -> Result<ArrayRef> {@@ -116,6 +153,95 @@ mod tests {Ok(())}+ #[test]+ fn float_zero_expansion_preserves_values_and_reuses_complete_lists() -> Result<()> {+ let nan = f32::from_bits(0x7fc0_0001);+ for zero in [0.0_f32, -0.0_f32] {+ let array: ArrayRef = Arc::new(+ Float32Array::from(vec![Some(9.0), None, Some(nan), Some(zero)])+ .slice(1, 3),+ );+ let expanded = add_missing_signed_zero::<Float32Type>(array)?;+ let bits = expanded+ .as_primitive::<Float32Type>()+ .iter()+ .map(|value| value.map(f32::to_bits))+ .collect::<Vec<_>>();+ assert_eq!(+ bits,+ vec![+ None,+ Some(nan.to_bits()),+ Some(zero.to_bits()),+ Some((-zero).to_bits())+ ]+ );+ assert!(Arc::ptr_eq(+ &expanded,+ &add_missing_signed_zero::<Float32Type>(Arc::clone(&expanded))?+ ));+ }++ let unchanged: [ArrayRef; 4] = [+ Arc::new(Float32Array::from(Vec::<f32>::new())),+ Arc::new(Float32Array::from(vec![None::<f32>])),+ Arc::new(Float32Array::from(vec![0.0_f32, -0.0_f32])),+ Arc::new(Float32Array::from(vec![0.0_f32, 1.0]).slice(1, 1)),+ ];+ for array in unchanged {+ assert!(Arc::ptr_eq(+ &array,+ &add_missing_signed_zero::<Float32Type>(Arc::clone(&array))?+ ));+ }+ Ok(())+ }++ #[test]+ fn static_float_filters_match_both_zeros_at_cutoffs() -> Result<()> {+ fn check<T: BranchlessFilterType>() -> Result<()>+ where+ T::Native: FloatSemantics,+ {+ let positive_zero = T::Native::ZERO;+ let negative_zero = positive_zero.neg_wrapping();+ let needles = PrimitiveArray::<T>::from_iter([+ Some(positive_zero),+ Some(negative_zero),+ Some(T::Native::ONE),+ None,+ ]);+ for len in [T::MAX_LIST_LEN - 1, T::MAX_LIST_LEN] {+ for zero in [positive_zero, negative_zero] {+ let array = Arc::new(PrimitiveArray::<T>::from_value(zero, len));+ let filter = instantiate_static_filter(array, &T::DATA_TYPE)?;+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![+ Some(true),+ Some(true),+ Some(false),+ None+ ])+ );+ assert_eq!(+ filter.contains(&needles, true)?,+ BooleanArray::from(vec![+ Some(false),+ Some(false),+ Some(true),+ None+ ])+ );+ }+ }+ Ok(())+ }+ check::<Float16Type>()?;+ check::<Float32Type>()?;+ check::<Float64Type>()+ }+#[test]fn byte_view_routing_requires_same_physical_type() {let dict_view =diff --git a/datafusion/sqllogictest/test_files/negative_zero.slt b/datafusion/sqllogictest/test_files/negative_zero.sltindex 08648c0dd5..609296e6c5 100644--- a/datafusion/sqllogictest/test_files/negative_zero.slt+++ b/datafusion/sqllogictest/test_files/negative_zero.slt@@ -34,6 +34,58 @@ SELECT 0.0 IS DISTINCT FROM -0.0 AS is_distinct;----false+#####+## IN / NOT IN predicates+#####++statement ok+CREATE TABLE negative_zero_in_list AS+SELECT arrow_cast(-0.0, 'Float16') AS negative_f16,+ arrow_cast(0.0, 'Float32') AS positive_f32,+ 0.0 AS positive_f64,+ -0.0 AS negative_f64,+ arrow_cast(-0.0, 'Dictionary(Int32, Float64)') AS negative_dict_f64;++# Short Float32/Float64 literal lists retain static membership filters.+# Check IN and NOT IN across float widths, list lengths, and both zero signs.+query BBBBBBBBBBB+SELECT+ -0.0 IN (0.0, 1, 2, 3),+ positive_f64 IN (-0.0, 1, 2),+ negative_f16 IN (arrow_cast(0.0, 'Float16'), 1, 2, 3),+ positive_f32 IN (arrow_cast(-0.0, 'Float32'), 1, 2, 3),+ positive_f64 IN (-0.0, 1, 2, 3),+ positive_f64 NOT IN (-0.0, 1, 2, 3),+ negative_f64 IN (1, 2, 3, 4),+ negative_f16 IN (arrow_cast(0.0, 'Float16'), 1, 2, 3, 4, 5, 6, 7, 8),+ negative_f64 IN (0.0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16),+ negative_f64 NOT IN (0.0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16),+ negative_f64 IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)+FROM negative_zero_in_list;+----+true true true true true false false true true false false++# Dictionary encoding must preserve equality on both sides of the rewrite threshold.+query BB+SELECT+ negative_dict_f64 IN (0.0),+ negative_dict_f64 IN (0.0, 1, 2, 3)+FROM negative_zero_in_list;+----+true true++# A column-valued list item forces non-static evaluation. Check both directions.+query BB+SELECT+ positive_f64 IN (negative_f64, 1, 2, 3),+ negative_f64 IN (positive_f64, 1, 2, 3)+FROM negative_zero_in_list;+----+true true++statement ok+DROP TABLE negative_zero_in_list;+####### SELECT DISTINCT with +0.0 / -0.0 (Float64)#####@@ -228,16 +280,85 @@ SELECT [0.0] = [-0.0] AS eq,----true false true+# Four constants keep the static membership filter. Nested scalar inputs+# must match their own values, including negative zero.+query BB+SELECT [-0.0] IN ([-0.0], [1.0], [2.0], [3.0]),+ [-0.0] NOT IN ([-0.0], [1.0], [2.0], [3.0]);+----+true false+statement okCREATE TABLE nested_zeros(id INT, a DOUBLE[]) AS VALUES (1, [0.0]), (2, [-0.0]);-query II-SELECT l.id, r.id FROM nested_zeros l JOIN nested_zeros r ON l.a = r.a ORDER BY l.id, r.id;+# Column-valued list items exercise dynamic array comparisons in both directions.+query IIBB+SELECT l.id, r.id,+ l.a IN (r.a, [1.0], [2.0], [3.0]),+ l.a NOT IN (r.a, [1.0], [2.0], [3.0])+FROM nested_zeros l JOIN nested_zeros r ON l.a = r.a+ORDER BY l.id, r.id;+----+1 1 true false+1 2 true false+2 1 true false+2 2 true false++# Short lists become equality comparisons; four-item constant lists use a static filter.+query IBBBB+SELECT id,+ a IN ([0.0]),+ a IN ([0.0], [1.0], [2.0], [3.0]),+ a IN ([-0.0], [1.0], [2.0], [3.0]),+ a NOT IN ([-0.0], [1.0], [2.0], [3.0])+FROM nested_zeros ORDER BY id;+----+1 true true true false+2 true true true false++# The nonmatching column-derived item keeps scalar RHS comparisons dynamic.+# Also check scalar LHS values against array-valued list items.+query IBBBB+SELECT id,+ a IN ([-0.0], [1.0], [2.0], [CAST(id AS DOUBLE)]),+ a NOT IN ([0.0], [1.0], [2.0], [CAST(id AS DOUBLE)]),+ [-0.0] IN (a, [1.0], [2.0], [3.0]),+ [0.0] NOT IN (a, [1.0], [2.0], [3.0])+FROM nested_zeros ORDER BY id;+----+1 true false true false+2 true false true false++# Structural literal equality must not drive IN intersection or difference for nested floats.+query IBB+SELECT id,+ a IN ([0.0], [1.0], [2.0], [3.0])+ AND a IN ([-0.0], [4.0], [5.0], [6.0]),+ a IN ([0.0], [1.0], [2.0], [3.0])+ AND a NOT IN ([-0.0], [4.0], [5.0], [6.0])+FROM nested_zeros ORDER BY id;+----+1 true false+2 true false++statement ok+INSERT INTO nested_zeros VALUES (3, NULL), (4, [NULL]), (5, [9.0]);++# Matches win over an outer NULL list item; nested NULL elements remain comparable.+# A missing match or a NULL input produces NULL, for both static and dynamic lists.+query IBBBB+SELECT id,+ a IN ([-0.0], [NULL], [1.0], NULL),+ a NOT IN ([-0.0], [NULL], [1.0], NULL),+ a IN ([-0.0], [NULL], [CAST(id AS DOUBLE)], NULL),+ a NOT IN ([-0.0], [NULL], [CAST(id AS DOUBLE)], NULL)+FROM nested_zeros ORDER BY id;-----1 1-1 2-2 1-2 2+1 true false true false+2 true false true false+3 NULL NULL NULL NULL+4 true false true false+5 NULL NULL NULL NULLstatement okDROP TABLE nested_zeros;