Skip to content

Apache DataFusion SQL Query Engine

IN LIST: add Float16 bitmap filter (#23311)

Geoffrey Claude34274f32dfea
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs
index 3eff1f5cf3..c70f6da2a4 100644
--- a/datafusion/physical-expr/benches/in_list_strategy.rs
+++ b/datafusion/physical-expr/benches/in_list_strategy.rs
@@ -34,7 +34,7 @@
//! | Case | Types | Characteristics | List Sizes Tested |
//! |------|-------|-----------------|-------------------|
//! | Narrow integer cases | UInt8 | small value domain | 4, 16 |
-//! | Narrow integer cases | Int16 | larger value domain | 4, 64, 256 |
+//! | Narrow integer cases | Int16, Float16 | larger value domain | 4, 64, 256 |
//! | 32-bit primitive cases | Int32, Float32 | small and large lists | 4, 32, 64, 256 |
//! | 64-bit primitive cases | Int64, TimestampNs | small and large lists | 4, 16, 32, 128 |
//! | Utf8 short-string cases | Utf8 | 8-byte strings | 4, 64, 256 |
@@ -51,6 +51,7 @@ use arrow::record_batch::RecordBatch;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
use datafusion_physical_expr::expressions::{col, in_list, lit};
+use half::f16;
use rand::distr::Alphanumeric;
use rand::prelude::*;
use std::sync::Arc;
@@ -392,6 +393,23 @@ fn bench_narrow_integer(c: &mut Criterion) {
);
}
}
+
+ // Float16: same 65,536-value bit-pattern domain as Int16/UInt16.
+ for list_size in [4, 64, 256] {
+ for match_pct in MATCH_RATES {
+ bench_numeric::<f16, Float16Array>(
+ c,
+ "narrow_integer",
+ &format!("f16/list={list_size}/match={match_pct}%"),
+ &NumericBenchConfig::new(
+ list_size,
+ match_pct as f64 / 100.0,
+ |rng| f16::from_f32(rng.random::<f32>() * 1000.0),
+ |v| ScalarValue::Float16(Some(v)),
+ ),
+ );
+ }
+ }
}
// =============================================================================
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs
index 50ff393693..2764083f31 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -3525,6 +3525,7 @@ mod tests {
DataType::UInt16,
DataType::UInt32,
DataType::UInt64,
+ DataType::Float16,
DataType::Float32,
DataType::Float64,
];
diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
index 8607584901..8f8d9bad04 100644
--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
@@ -79,7 +79,7 @@ impl BitmapStorage for Box<[u64; 1024]> {
///
/// Arrow already defines the Rust value type as `T::Native`. This trait only
/// supplies the bitmap storage size and maps values to their bit-pattern index
-/// for the two integer domains that are small enough to represent with one bit
+/// for the primitive domains that are small enough to represent with one bit
/// per possible value.
pub(super) trait BitmapFilterType:
ArrowPrimitiveType + Send + Sync + 'static
@@ -134,6 +134,17 @@ impl BitmapFilterType for UInt16Type {
}
}
+/// `Float16` has 65,536 possible bit patterns, so 1,024 `u64` words cover the
+/// full domain.
+impl BitmapFilterType for Float16Type {
+ type Storage = Box<[u64; 1024]>;
+
+ #[inline(always)]
+ fn index(value: Self::Native) -> usize {
+ value.to_bits() as usize
+ }
+}
+
/// `IN` filter backed by one bit per possible value.
///
/// Building the filter scans the non-null values in the IN-list and turns on
@@ -418,7 +429,10 @@ mod tests {
use super::*;
use std::sync::Arc;
- use arrow::array::{DictionaryArray, Int8Array, Int16Array, UInt8Array, UInt16Array};
+ use arrow::array::{
+ DictionaryArray, Float16Array, Int8Array, Int16Array, UInt8Array, UInt16Array,
+ };
+ use half::f16;
fn assert_contains(
filter: &dyn StaticFilter,
@@ -534,4 +548,40 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn bitmap_filter_f16_handles_bit_patterns_and_slices() -> Result<()> {
+ let nan_a = f16::from_bits(0x7e01);
+ let nan_b = f16::from_bits(0x7e02);
+ let haystack: ArrayRef = Arc::new(
+ Float16Array::from(vec![
+ Some(f16::from_f32(9.0)),
+ Some(f16::from_f32(1.5)),
+ None,
+ Some(f16::from_f32(-0.0)),
+ Some(nan_a),
+ ])
+ .slice(1, 4),
+ );
+ let filter = BitmapFilter::<Float16Type>::try_new(&haystack)?;
+ let needles = Float16Array::from(vec![
+ Some(f16::from_f32(0.0)),
+ Some(f16::from_f32(-0.0)),
+ Some(nan_a),
+ Some(nan_b),
+ None,
+ ])
+ .slice(1, 4);
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), Some(true), None, None])
+ );
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), Some(false), None, None])
+ );
+
+ Ok(())
+ }
}
diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
index bfdc83fa14..9db90ea4fa 100644
--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
@@ -19,7 +19,9 @@ use std::sync::Arc;
use arrow::array::ArrayRef;
use arrow::compute::cast;
-use arrow::datatypes::{DataType, Int8Type, Int16Type, UInt8Type, UInt16Type};
+use arrow::datatypes::{
+ DataType, Float16Type, Int8Type, Int16Type, UInt8Type, UInt16Type,
+};
use datafusion_common::Result;
use super::array_static_filter::ArrayStaticFilter;
@@ -41,6 +43,9 @@ pub(super) fn instantiate_static_filter(
DataType::UInt8 => Ok(Arc::new(BitmapFilter::<UInt8Type>::try_new(&in_array)?)),
DataType::Int16 => Ok(Arc::new(BitmapFilter::<Int16Type>::try_new(&in_array)?)),
DataType::UInt16 => Ok(Arc::new(BitmapFilter::<UInt16Type>::try_new(&in_array)?)),
+ DataType::Float16 => {
+ Ok(Arc::new(BitmapFilter::<Float16Type>::try_new(&in_array)?))
+ }
DataType::Int32 => Ok(Arc::new(Int32StaticFilter::try_new(&in_array)?)),
DataType::Int64 => Ok(Arc::new(Int64StaticFilter::try_new(&in_array)?)),
DataType::UInt32 => Ok(Arc::new(UInt32StaticFilter::try_new(&in_array)?)),