Apache DataFusion SQL Query Engine
IN LIST: add Float16 bitmap filter (#23311)
Geoffrey Claude34274f32dfea
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rsindex 3eff1f5cf3..c70f6da2a4 100644--- a/datafusion/physical-expr/benches/in_list_strategy.rs+++ b/datafusion/physical-expr/benches/in_list_strategy.rs@@ -34,7 +34,7 @@//! | Case | Types | Characteristics | List Sizes Tested |//! |------|-------|-----------------|-------------------|//! | Narrow integer cases | UInt8 | small value domain | 4, 16 |-//! | Narrow integer cases | Int16 | larger value domain | 4, 64, 256 |+//! | Narrow integer cases | Int16, Float16 | larger value domain | 4, 64, 256 |//! | 32-bit primitive cases | Int32, Float32 | small and large lists | 4, 32, 64, 256 |//! | 64-bit primitive cases | Int64, TimestampNs | small and large lists | 4, 16, 32, 128 |//! | Utf8 short-string cases | Utf8 | 8-byte strings | 4, 64, 256 |@@ -51,6 +51,7 @@ use arrow::record_batch::RecordBatch;use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};use datafusion_common::ScalarValue;use datafusion_physical_expr::expressions::{col, in_list, lit};+use half::f16;use rand::distr::Alphanumeric;use rand::prelude::*;use std::sync::Arc;@@ -392,6 +393,23 @@ fn bench_narrow_integer(c: &mut Criterion) {);}}++ // Float16: same 65,536-value bit-pattern domain as Int16/UInt16.+ for list_size in [4, 64, 256] {+ for match_pct in MATCH_RATES {+ bench_numeric::<f16, Float16Array>(+ c,+ "narrow_integer",+ &format!("f16/list={list_size}/match={match_pct}%"),+ &NumericBenchConfig::new(+ list_size,+ match_pct as f64 / 100.0,+ |rng| f16::from_f32(rng.random::<f32>() * 1000.0),+ |v| ScalarValue::Float16(Some(v)),+ ),+ );+ }+ }}// =============================================================================diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex 50ff393693..2764083f31 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -3525,6 +3525,7 @@ mod tests {DataType::UInt16,DataType::UInt32,DataType::UInt64,+ DataType::Float16,DataType::Float32,DataType::Float64,];diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rsindex 8607584901..8f8d9bad04 100644--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs@@ -79,7 +79,7 @@ impl BitmapStorage for Box<[u64; 1024]> {////// Arrow already defines the Rust value type as `T::Native`. This trait only/// supplies the bitmap storage size and maps values to their bit-pattern index-/// for the two integer domains that are small enough to represent with one bit+/// for the primitive domains that are small enough to represent with one bit/// per possible value.pub(super) trait BitmapFilterType:ArrowPrimitiveType + Send + Sync + 'static@@ -134,6 +134,17 @@ impl BitmapFilterType for UInt16Type {}}+/// `Float16` has 65,536 possible bit patterns, so 1,024 `u64` words cover the+/// full domain.+impl BitmapFilterType for Float16Type {+ type Storage = Box<[u64; 1024]>;++ #[inline(always)]+ fn index(value: Self::Native) -> usize {+ value.to_bits() as usize+ }+}+/// `IN` filter backed by one bit per possible value.////// Building the filter scans the non-null values in the IN-list and turns on@@ -418,7 +429,10 @@ mod tests {use super::*;use std::sync::Arc;- use arrow::array::{DictionaryArray, Int8Array, Int16Array, UInt8Array, UInt16Array};+ use arrow::array::{+ DictionaryArray, Float16Array, Int8Array, Int16Array, UInt8Array, UInt16Array,+ };+ use half::f16;fn assert_contains(filter: &dyn StaticFilter,@@ -534,4 +548,40 @@ mod tests {Ok(())}++ #[test]+ fn bitmap_filter_f16_handles_bit_patterns_and_slices() -> Result<()> {+ let nan_a = f16::from_bits(0x7e01);+ let nan_b = f16::from_bits(0x7e02);+ let haystack: ArrayRef = Arc::new(+ Float16Array::from(vec![+ Some(f16::from_f32(9.0)),+ Some(f16::from_f32(1.5)),+ None,+ Some(f16::from_f32(-0.0)),+ Some(nan_a),+ ])+ .slice(1, 4),+ );+ let filter = BitmapFilter::<Float16Type>::try_new(&haystack)?;+ let needles = Float16Array::from(vec![+ Some(f16::from_f32(0.0)),+ Some(f16::from_f32(-0.0)),+ Some(nan_a),+ Some(nan_b),+ None,+ ])+ .slice(1, 4);++ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), Some(true), None, None])+ );+ assert_eq!(+ filter.contains(&needles, true)?,+ BooleanArray::from(vec![Some(false), Some(false), None, None])+ );++ Ok(())+ }}diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rsindex bfdc83fa14..9db90ea4fa 100644--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs@@ -19,7 +19,9 @@ use std::sync::Arc;use arrow::array::ArrayRef;use arrow::compute::cast;-use arrow::datatypes::{DataType, Int8Type, Int16Type, UInt8Type, UInt16Type};+use arrow::datatypes::{+ DataType, Float16Type, Int8Type, Int16Type, UInt8Type, UInt16Type,+};use datafusion_common::Result;use super::array_static_filter::ArrayStaticFilter;@@ -41,6 +43,9 @@ pub(super) fn instantiate_static_filter(DataType::UInt8 => Ok(Arc::new(BitmapFilter::<UInt8Type>::try_new(&in_array)?)),DataType::Int16 => Ok(Arc::new(BitmapFilter::<Int16Type>::try_new(&in_array)?)),DataType::UInt16 => Ok(Arc::new(BitmapFilter::<UInt16Type>::try_new(&in_array)?)),+ DataType::Float16 => {+ Ok(Arc::new(BitmapFilter::<Float16Type>::try_new(&in_array)?))+ }DataType::Int32 => Ok(Arc::new(Int32StaticFilter::try_new(&in_array)?)),DataType::Int64 => Ok(Arc::new(Int64StaticFilter::try_new(&in_array)?)),DataType::UInt32 => Ok(Arc::new(UInt32StaticFilter::try_new(&in_array)?)),