Apache DataFusion SQL Query Engine
IN LIST: add UInt16 bitmap filter (#23012)
Geoffrey Claudebde8e5b24e77
diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rsindex e7647e5adb..8242ba09bd 100644--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs@@ -104,6 +104,81 @@ impl StaticFilter for UInt8BitmapFilter {}}+/// Bitmap filter for O(1) `UInt16` set membership via single bit test.+///+/// `UInt16` has 65,536 possible values, so the filter stores membership in an+/// 8 KiB heap-allocated bitmap instead of using a hash table.+pub(super) struct UInt16BitmapFilter {+ null_count: usize,+ bits: Box<[u64; 1024]>,+}++impl UInt16BitmapFilter {+ pub(super) fn try_new(in_array: &ArrayRef) -> Result<Self> {+ let prim_array = in_array.as_primitive_opt::<UInt16Type>().ok_or_else(|| {+ exec_datafusion_err!("UInt16BitmapFilter: expected UInt16 array")+ })?;+ let mut bits = Box::new([0u64; 1024]);+ let mut set_bit = |v: u16| {+ let index = usize::from(v);+ bits[index / 64] |= 1u64 << (index % 64);+ };++ let values = prim_array.values();+ match prim_array.nulls() {+ None => {+ for &v in values {+ set_bit(v);+ }+ }+ Some(nulls) => {+ for i in+ BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len())+ {+ set_bit(values[i]);+ }+ }+ }+ Ok(Self {+ null_count: prim_array.null_count(),+ bits,+ })+ }++ #[inline(always)]+ fn check(&self, needle: u16) -> bool {+ let index = needle as usize;+ (self.bits[index / 64] >> (index % 64)) & 1 != 0+ }+}++impl StaticFilter for UInt16BitmapFilter {+ fn null_count(&self) -> usize {+ self.null_count+ }++ fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {+ handle_dictionary!(self, v, negated);+ let v = v.as_primitive_opt::<UInt16Type>().ok_or_else(|| {+ exec_datafusion_err!("UInt16BitmapFilter: expected UInt16 array")+ })?;+ let input_values = v.values();+ Ok(build_in_list_result(+ v.len(),+ v.nulls(),+ self.null_count > 0,+ negated,+ #[inline(always)]+ |i| {+ // SAFETY: `build_in_list_result` invokes this closure for+ // indices in `0..v.len()`, which matches `input_values.len()`.+ let needle = unsafe { *input_values.get_unchecked(i) };+ self.check(needle)+ },+ ))+ }+}+/// Wrapper for f32 that implements Hash and Eq using bit comparison./// This treats NaN values as equal to each other when they have the same bit pattern.#[derive(Clone, Copy)]@@ -294,7 +369,6 @@ primitive_static_filter!(Int8StaticFilter, Int8Type);primitive_static_filter!(Int16StaticFilter, Int16Type);primitive_static_filter!(Int32StaticFilter, Int32Type);primitive_static_filter!(Int64StaticFilter, Int64Type);-primitive_static_filter!(UInt16StaticFilter, UInt16Type);primitive_static_filter!(UInt32StaticFilter, UInt32Type);primitive_static_filter!(UInt64StaticFilter, UInt64Type);@@ -315,10 +389,10 @@ mod tests {use super::*;use std::sync::Arc;- use arrow::array::{DictionaryArray, Int8Array, UInt8Array};+ use arrow::array::{DictionaryArray, Int8Array, UInt8Array, UInt16Array};fn assert_contains(- filter: &UInt8BitmapFilter,+ filter: &dyn StaticFilter,needles: &dyn Array,expected: Vec<Option<bool>>,) -> Result<()> {@@ -355,4 +429,29 @@ mod tests {assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])}++ #[test]+ fn bitmap_filter_u16_handles_boundaries_and_nulls() -> Result<()> {+ let haystack: ArrayRef = Arc::new(UInt16Array::from(vec![+ Some(0),+ None,+ Some(1024),+ Some(u16::MAX),+ ]));+ let filter = UInt16BitmapFilter::try_new(&haystack)?;+ let needles =+ UInt16Array::from(vec![Some(0), Some(1), Some(1024), Some(u16::MAX), None]);++ assert_contains(+ &filter,+ &needles,+ vec![Some(true), None, Some(true), Some(true), None],+ )?;+ assert_eq!(+ filter.contains(&needles, true)?,+ BooleanArray::from(vec![Some(false), None, Some(false), Some(false), None])+ );++ Ok(())+ }}diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rsindex 1fb8e03fe2..aec94bddb9 100644--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs@@ -43,7 +43,7 @@ pub(super) fn instantiate_static_filter(DataType::Int32 => Ok(Arc::new(Int32StaticFilter::try_new(&in_array)?)),DataType::Int64 => Ok(Arc::new(Int64StaticFilter::try_new(&in_array)?)),DataType::UInt8 => Ok(Arc::new(UInt8BitmapFilter::try_new(&in_array)?)),- DataType::UInt16 => Ok(Arc::new(UInt16StaticFilter::try_new(&in_array)?)),+ DataType::UInt16 => Ok(Arc::new(UInt16BitmapFilter::try_new(&in_array)?)),DataType::UInt32 => Ok(Arc::new(UInt32StaticFilter::try_new(&in_array)?)),DataType::UInt64 => Ok(Arc::new(UInt64StaticFilter::try_new(&in_array)?)),// Float primitive types (use ordered wrappers for Hash/Eq)