Apache DataFusion SQL Query Engine
IN LIST: reuse primitive filters for FixedSizeBinary (#24102)
Geoffrey Claudee2e94ae64fd1
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rsindex 762c2c97a1..00a7c476f5 100644--- a/datafusion/physical-expr/benches/in_list_strategy.rs+++ b/datafusion/physical-expr/benches/in_list_strategy.rs@@ -15,10 +15,10 @@// specific language governing permissions and limitations// under the License.-//! Focused benchmarks for `InList` cases.+//! Benchmarks for static `IN LIST` filters.//!-//! This benchmark file adds targeted coverage for representative `IN LIST`-//! workloads with controlled parameters:+//! The cases control match rate and list size across several value types and+//! string layouts://!//! - **Controlled match rates**: Exercises both hit-heavy and miss-heavy paths//! - **List size scaling**: Measures behavior across small and large `IN` lists@@ -27,7 +27,7 @@//! - **Shared-prefix strings**: Adds collision-heavy string cases where values//! only differ late in the string//! - **Mixed-length strings**: Covers inputs that combine short and long values-//! - **Null handling**: Includes representative `NULL` and `NOT IN` cases+//! - **Null handling**: Covers `NULL` and `NOT IN` cases//!//! # Case Coverage//!@@ -45,14 +45,19 @@//! | Utf8View length-12 cases | Utf8View | 12-byte strings | 16, 64 |//! | Utf8View long-string cases | Utf8View | 24-byte strings | 4, 16, 64, 256 |//! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 |-//! | Fixed-size binary cases | FixedSizeBinary(16) | fixed-width binary values | 4, 64, 256, 10000 |+//! | Fixed-size binary direct-comparison case | FixedSizeBinary(1) | direct-comparison cutoff | 16 |+//! | Fixed-size binary direct-comparison case | FixedSizeBinary(16) | direct-comparison cutoff | 4 |+//! | Fixed-size binary bitmap case | FixedSizeBinary(2) | bitmap lookup | 64 |+//! | Fixed-size binary hash-set cases | FixedSizeBinary(16) | hash-set scaling | 64, 256, 10000 |+//! | Fixed-size binary unaligned case | FixedSizeBinary(16) | per-evaluation alignment copy | 64 |use arrow::array::types::IntervalMonthDayNano;use arrow::array::*;+use arrow::buffer::{Buffer, MutableBuffer};use arrow::datatypes::{Field, Int32Type, IntervalMonthDayNanoType, Schema};use arrow::record_batch::RecordBatch;use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};-use datafusion_common::ScalarValue;+use datafusion_common::{HashSet, ScalarValue};use datafusion_physical_expr::expressions::{col, in_list, lit};use half::f16;use rand::distr::Alphanumeric;@@ -870,7 +875,7 @@ fn bench_dictionary(c: &mut Criterion) {// NULL HANDLING BENCHMARKS// =============================================================================//-// Tests representative null-containing inputs across primitive and string cases.+// Null-containing primitive and string cases.fn bench_nulls(c: &mut Criterion) {// =========================================================================@@ -1014,72 +1019,180 @@ fn bench_nulls(c: &mut Criterion) {}// =============================================================================-// FIXED SIZE BINARY BENCHMARKS (FixedSizeBinary<16>, e.g. UUIDs)+// FIXED SIZE BINARY BENCHMARKS// =============================================================================-/// Generates a random 16-byte value (UUID-sized).-fn random_fixed_binary_16(rng: &mut StdRng) -> Vec<u8> {- let mut buf = vec![0u8; 16];+fn random_fixed_binary(rng: &mut StdRng, width: i32) -> Vec<u8> {+ let mut buf = vec![0u8; width as usize];rng.fill(&mut buf[..]);buf}-/// Benchmarks FixedSizeBinary(16) IN list evaluation.-/// FixedSizeBinary doesn't use the generic numeric helpers since its array-/// construction differs from primitive types.-fn bench_fixed_size_binary_inner(- c: &mut Criterion,- name: &str,+#[derive(Clone, Copy)]+enum InputLayout {+ Aligned,+ UnalignedI128,+}++#[derive(Clone, Copy)]+struct FixedSizeBinaryBenchConfig {+ width: i32,+ list_size: usize,+ input_layout: InputLayout,+}++impl FixedSizeBinaryBenchConfig {+ const fn aligned(width: i32, list_size: usize) -> Self {+ Self {+ width,+ list_size,+ input_layout: InputLayout::Aligned,+ }+ }++ const fn unaligned_i128(list_size: usize) -> Self {+ Self {+ width: 16,+ list_size,+ input_layout: InputLayout::UnalignedI128,+ }+ }+}++const FIXED_SIZE_BINARY_CASES: [FixedSizeBinaryBenchConfig; 7] = [+ FixedSizeBinaryBenchConfig::aligned(1, 16),+ FixedSizeBinaryBenchConfig::aligned(2, 64),+ FixedSizeBinaryBenchConfig::aligned(16, 4),+ FixedSizeBinaryBenchConfig::aligned(16, 64),+ FixedSizeBinaryBenchConfig::aligned(16, 256),+ FixedSizeBinaryBenchConfig::aligned(16, 10000),+ // 8,192 rows at 16 bytes each copy 128 KiB per evaluation.+ FixedSizeBinaryBenchConfig::unaligned_i128(64),+];++fn generate_fixed_size_binary_data(+ rng: &mut StdRng,+ width: i32,list_size: usize,match_rate: f64,-) {- let seed = 0xF1ED_B1A7_u64.wrapping_add(list_size as u64 * 0x6666);- let mut rng = StdRng::seed_from_u64(seed);+) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {+ if let Some(domain_size) = match width {+ 1 => Some(1_usize << 8),+ 2 => Some(1_usize << 16),+ _ => None,+ } {+ // The value generator needs at least one value outside the haystack.+ assert!(list_size < domain_size);+ }- // Generate IN list values (16-byte each)- let haystack: Vec<Vec<u8>> = (0..list_size)- .map(|_| random_fixed_binary_16(&mut rng))- .collect();+ // Keep the number of distinct haystack values equal to the configured list size.+ let mut haystack_set = HashSet::with_capacity(list_size);+ let mut haystack = Vec::with_capacity(list_size);+ while haystack.len() < list_size {+ let value = random_fixed_binary(rng, width);+ if haystack_set.insert(value.clone()) {+ haystack.push(value);+ }+ }- // Generate array with controlled match rate- let values: Vec<Vec<u8>> = (0..ARRAY_SIZE)+ // Generate values with the configured match rate.+ let values = (0..ARRAY_SIZE).map(|_| {if !haystack.is_empty() && rng.random_bool(match_rate) {- haystack.choose(&mut rng).unwrap().clone()+ haystack.choose(rng).unwrap().clone()} else {- random_fixed_binary_16(&mut rng)+ loop {+ let value = random_fixed_binary(rng, width);+ if !haystack_set.contains(&value) {+ break value;+ }+ }}}).collect();- let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();- let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap();+ (haystack, values)+}++fn unaligned_fixed_size_binary_16(values: &[Vec<u8>]) -> FixedSizeBinaryArray {+ const WIDTH: usize = 16;+ let payload_len = values.len() * WIDTH;+ let mut bytes = MutableBuffer::with_capacity(payload_len + 1);+ bytes.push(0_u8);+ for value in values {+ assert_eq!(value.len(), WIDTH);+ bytes.extend_from_slice(value);+ }++ // MutableBuffer starts at an Arrow-aligned address. Fixed-size binary+ // values only require byte alignment, so slicing off this padding byte+ // creates a valid Arrow buffer that models unaligned external input.+ let buffer = Buffer::from(bytes).slice(1);+ assert!(+ !buffer.as_ptr().cast::<i128>().is_aligned(),+ "benchmark input must be unaligned"+ );+ FixedSizeBinaryArray::new(WIDTH as i32, buffer, None)+}++/// FixedSizeBinary doesn't use the generic numeric helpers since its array+/// construction differs from primitive types.+fn bench_fixed_size_binary_inner(+ c: &mut Criterion,+ config: FixedSizeBinaryBenchConfig,+ match_pct: u32,+) {+ assert!(match_pct <= 100);+ let match_rate = f64::from(match_pct) / 100.0;++ let seed = 0xF1ED_B1A7_u64+ .wrapping_add(config.list_size as u64 * 0x6666)+ .wrapping_add(config.width as u64 * 0x7777);+ let mut rng = StdRng::seed_from_u64(seed);++ let (haystack, values) = generate_fixed_size_binary_data(+ &mut rng,+ config.width,+ config.list_size,+ match_rate,+ );++ let array = match config.input_layout {+ InputLayout::Aligned => {+ FixedSizeBinaryArray::try_from_iter(values.iter().map(Vec::as_slice)).unwrap()+ }+ InputLayout::UnalignedI128 => unaligned_fixed_size_binary_16(&values),+ };let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]);let exprs: Vec<_> = haystack.iter()- .map(|v| lit(ScalarValue::FixedSizeBinary(16, Some(v.clone()))))+ .map(|v| lit(ScalarValue::FixedSizeBinary(config.width, Some(v.clone())))).collect();let expr = in_list(col("a", &schema).unwrap(), exprs, &false, &schema).unwrap();let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array) as ArrayRef]).unwrap();c.bench_with_input(- BenchmarkId::new("fixed_size_binary", name),+ BenchmarkId::new("fixed_size_binary", {+ let name = format!(+ "fsb{}/list={}/match={match_pct}%",+ config.width, config.list_size+ );+ match config.input_layout {+ InputLayout::Aligned => name,+ InputLayout::UnalignedI128 => format!("{name}/input=unaligned"),+ }+ }),&batch,|b, batch| b.iter(|| expr.evaluate(batch).unwrap()),);}fn bench_fixed_size_binary(c: &mut Criterion) {- for list_size in [4, 64, 256, 10000] {+ for config in FIXED_SIZE_BINARY_CASES {for match_pct in MATCH_RATES {- bench_fixed_size_binary_inner(- c,- &format!("fsb16/list={list_size}/match={match_pct}%"),- list_size,- match_pct as f64 / 100.0,- );+ bench_fixed_size_binary_inner(c, config, match_pct);}}}diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex 0fb978cd0b..154decfd8b 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -38,6 +38,7 @@ use datafusion_expr::{ColumnarValue, expr_vec_fmt};mod array_static_filter;mod branchless_filter;+mod fixed_size_binary_filter;mod primitive_filter;mod result;mod static_filter;@@ -3548,6 +3549,38 @@ mod tests {);}+ // FixedSizeBinary in_array, FixedSizeBinary and Dictionary needles+ let fsb_in = Arc::new(FixedSizeBinaryArray::try_from_iter(+ [+ [1, 2, 3, 4].as_slice(),+ [5, 6, 7, 8].as_slice(),+ [9, 10, 11, 12].as_slice(),+ ]+ .into_iter(),+ )?) as ArrayRef;+ let fsb_needle = Arc::new(FixedSizeBinaryArray::try_from_iter(+ [+ [1, 2, 3, 4].as_slice(),+ [13, 14, 15, 16].as_slice(),+ [5, 6, 7, 8].as_slice(),+ ]+ .into_iter(),+ )?) as ArrayRef;+ assert_eq!(+ expected,+ eval_in_list_from_array(Arc::clone(&fsb_needle), Arc::clone(&fsb_in))?+ );+ // The dictionary does not reference its second value, so that value+ // must not become a member of the flattened list.+ let dict_fsb_in = Arc::new(DictionaryArray::new(+ Int32Array::from(vec![0, 2]),+ Arc::clone(&fsb_in),+ ));+ assert_eq!(+ BooleanArray::from(vec![Some(true), Some(false), Some(false)]),+ eval_in_list_from_array(wrap_in_dict(fsb_needle), dict_fsb_in)?+ );+// Utf8 (falls through to ArrayStaticFilter)let utf8_in = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef;let utf8_needle = Arc::new(StringArray::from(vec!["a", "d", "b"])) as ArrayRef;diff --git a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rsnew file mode 100644index 0000000000..421c02ea1f--- /dev/null+++ b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs@@ -0,0 +1,371 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Optimized filters for fixed-size binary `IN` lists.+//!+//! Supported widths use an Arrow primitive representation with the same+//! in-memory size:+//!+//! | Width | Primitive representation |+//! |------:|--------------------------|+//! | 1 | `UInt8` |+//! | 2 | `UInt16` |+//! | 4 | `UInt32` |+//! | 8 | `UInt64` |+//! | 16 | `Decimal128` |+//!+//! The shared primitive selector applies the native primitive branchless cutoffs+//! and chooses the bitmap or hash-set fallback.+//!+//! The list and input bytes are read the same way, so each primitive value is+//! an exact key for comparison, bitmap lookup, or hashing. No arithmetic,+//! ordering, or decimal operations are used.+//!+//! Reinterpreting an aligned Arrow buffer is zero-copy. An unaligned buffer is+//! copied into aligned primitive storage before filter construction or probing.++use std::marker::PhantomData;+use std::mem::size_of;+use std::sync::Arc;++use arrow::array::{+ Array, ArrayRef, AsArray, BooleanArray, FixedSizeBinaryArray, PrimitiveArray,+};+use arrow::buffer::{Buffer, ScalarBuffer};+use arrow::datatypes::{+ ArrowPrimitiveType, DataType, Decimal128Type, UInt8Type, UInt16Type, UInt32Type,+ UInt64Type,+};+use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};++use super::primitive_filter::instantiate_primitive_filter;+use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary};++/// Reinterpret fixed-size binary values as same-width primitive values.+///+/// Arrow buffers are normally sufficiently aligned, making this zero-copy. A+/// valid Arrow array can still be constructed from a sliced, unaligned buffer;+/// in that case, copy each value into aligned primitive storage.+fn reinterpret_as_primitive<T>(array: &FixedSizeBinaryArray) -> Result<PrimitiveArray<T>>+where+ T: ArrowPrimitiveType,+{+ let width = size_of::<T::Native>();+ if array.value_size() != width {+ return Err(internal_datafusion_err!(+ "FixedSizeBinary filter: expected {width}-byte values, got {}",+ array.value_size()+ ));+ }++ let source = array.values();+ let values = if source.as_ptr().cast::<T::Native>().is_aligned() {+ ScalarBuffer::new(source.clone(), 0, array.len())+ } else {+ // `Buffer::from(&[u8])` copies into Arrow-aligned storage.+ ScalarBuffer::new(Buffer::from(source.as_slice()), 0, array.len())+ };++ Ok(PrimitiveArray::<T>::new(values, array.nulls().cloned()))+}++/// Adapts a primitive filter to concrete, same-width `FixedSizeBinary` arrays.+struct FixedSizeBinaryFilter<T: ArrowPrimitiveType> {+ data_type: DataType,+ inner: StaticFilterRef,+ _marker: PhantomData<T>,+}++impl<T> StaticFilter for FixedSizeBinaryFilter<T>+where+ T: ArrowPrimitiveType + Send + Sync + 'static,+{+ fn null_count(&self) -> usize {+ self.inner.null_count()+ }++ fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {+ handle_dictionary!(self, v, negated);++ if v.data_type() != &self.data_type {+ return Err(exec_datafusion_err!(+ "FixedSizeBinary filter: expected {} array, got {}",+ self.data_type,+ v.data_type()+ ));+ }+ let array = v.as_fixed_size_binary_opt().ok_or_else(|| {+ exec_datafusion_err!(+ "FixedSizeBinary filter: expected concrete {} array",+ self.data_type+ )+ })?;+ let primitive = reinterpret_as_primitive::<T>(array)?;+ self.inner.contains(&primitive, negated)+ }+}++fn instantiate_for_primitive<T>(array: &FixedSizeBinaryArray) -> Result<StaticFilterRef>+where+ T: ArrowPrimitiveType + Send + Sync + 'static,+{+ let primitive: ArrayRef = Arc::new(reinterpret_as_primitive::<T>(array)?);+ let inner = instantiate_primitive_filter(&primitive)?.ok_or_else(|| {+ internal_datafusion_err!(+ "FixedSizeBinary filter: no primitive filter for {}",+ primitive.data_type()+ )+ })?;+ Ok(Arc::new(FixedSizeBinaryFilter::<T> {+ data_type: array.data_type().clone(),+ inner,+ _marker: PhantomData,+ }))+}++/// Creates an optimized filter for supported concrete `FixedSizeBinary` arrays.+pub(super) fn instantiate_fixed_size_binary_filter(+ in_array: &ArrayRef,+) -> Result<Option<StaticFilterRef>> {+ let DataType::FixedSizeBinary(width) = in_array.data_type() else {+ return Ok(None);+ };+ let Some(array) = in_array.as_fixed_size_binary_opt() else {+ return Ok(None);+ };++ let filter = match width {+ 1 => instantiate_for_primitive::<UInt8Type>(array)?,+ 2 => instantiate_for_primitive::<UInt16Type>(array)?,+ 4 => instantiate_for_primitive::<UInt32Type>(array)?,+ 8 => instantiate_for_primitive::<UInt64Type>(array)?,+ 16 => instantiate_for_primitive::<Decimal128Type>(array)?,+ _ => return Ok(None),+ };+ Ok(Some(filter))+}++#[cfg(test)]+mod tests {+ use arrow::array::{DictionaryArray, Int8Array, StringArray};+ use arrow::buffer::{Buffer, MutableBuffer, NullBuffer};+ use arrow::datatypes::Int8Type;++ use super::*;++ fn value(width: i32, index: usize, miss: bool) -> Vec<u8> {+ let mut value = (index as u128).to_le_bytes()[..width as usize].to_vec();+ let last = value.last_mut().unwrap();+ if miss {+ *last |= 0x80;+ } else {+ *last &= 0x7f;+ }+ value+ }++ fn array(width: i32, values: &[Option<Vec<u8>>]) -> FixedSizeBinaryArray {+ FixedSizeBinaryArray::try_from_sparse_iter_with_size(+ values.iter().map(|value| value.as_deref()),+ width,+ )+ .unwrap()+ }++ fn make_filter(width: i32, values: &[Option<Vec<u8>>]) -> Result<StaticFilterRef> {+ let in_array: ArrayRef = Arc::new(array(width, values));+ Ok(instantiate_fixed_size_binary_filter(&in_array)?.unwrap())+ }++ #[test]+ fn filters_supported_widths_across_strategy_thresholds() -> Result<()> {+ for (width, list_len) in [+ (1, 16),+ (1, 17),+ (2, 8),+ (2, 9),+ (4, 32),+ (4, 33),+ (8, 16),+ (8, 17),+ (16, 4),+ (16, 5),+ ] {+ let mut hit = vec![0x80; width as usize];+ hit[width as usize - 1] = 0xff;+ let mut miss = hit.clone();+ miss[width as usize - 1] ^= 1;++ let mut haystack = (0..list_len - 1)+ .map(|index| Some(value(width, index, false)))+ .collect::<Vec<_>>();+ haystack.push(Some(hit.clone()));+ let filter = make_filter(width, &haystack)?;+ let needles = array(width, &[Some(hit), Some(miss), None]);+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), Some(false), None]),+ "width={width}, list_len={list_len}"+ );+ }+ Ok(())+ }++ #[test]+ fn handles_slices_nulls_and_not_in() -> Result<()> {+ let width = 16;+ let parent = array(+ width,+ &[+ Some(value(width, 0, false)),+ Some(value(width, 1, false)),+ None,+ Some(value(width, 2, false)),+ Some(value(width, 3, false)),+ Some(value(width, 4, false)),+ Some(value(width, 5, false)),+ Some(value(width, 6, false)),+ ],+ );+ // Five non-null values select the hash-set path.+ let in_array: ArrayRef = Arc::new(parent.slice(1, 6));+ let filter = instantiate_fixed_size_binary_filter(&in_array)?.unwrap();+ let needles = array(+ width,+ &[+ Some(value(width, 2, false)),+ Some(value(width, 0, false)),+ Some(value(width, 6, false)),+ Some(value(width, 7, false)),+ None,+ ],+ );++ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), None, None, None, None])+ );+ assert_eq!(+ filter.contains(&needles, true)?,+ BooleanArray::from(vec![Some(false), None, None, None, None])+ );+ Ok(())+ }++ #[test]+ fn handles_dictionary_needles() -> Result<()> {+ let filter = make_filter(4, &[Some(value(4, 7, false))])?;+ let dictionary_values: ArrayRef = Arc::new(array(+ 4,+ &[Some(value(4, 7, false)), Some(value(4, 8, false))],+ ));+ let keys = Int8Array::from(vec![Some(0), Some(1), None]);+ let needles =+ DictionaryArray::<Int8Type>::try_new(keys, dictionary_values).unwrap();++ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), Some(false), None])+ );+ assert_eq!(+ filter.contains(&needles, true)?,+ BooleanArray::from(vec![Some(false), Some(true), None])+ );+ Ok(())+ }++ #[test]+ fn rejects_unsupported_arrays() -> Result<()> {+ let filter = make_filter(4, &[Some(value(4, 1, false))])?;+ let wrong_width = array(8, &[Some(value(8, 1, false))]);+ let error = filter+ .contains(&wrong_width, false)+ .unwrap_err()+ .to_string();+ assert!(+ error.contains("expected FixedSizeBinary(4) array, got FixedSizeBinary(8)"),+ "{error}"+ );++ let wrong_type = StringArray::from(vec!["one"]);+ let error = filter.contains(&wrong_type, false).unwrap_err().to_string();+ assert!(+ error.contains("expected FixedSizeBinary(4) array, got Utf8"),+ "{error}"+ );++ for width in [0, 3, 5, 15, 17] {+ let unsupported: ArrayRef =+ Arc::new(FixedSizeBinaryArray::new_null(width, 1));+ assert!(+ instantiate_fixed_size_binary_filter(&unsupported)?.is_none(),+ "width={width}"+ );+ }++ Ok(())+ }++ fn unaligned_i128_array(+ values: &[Vec<u8>],+ nulls: Option<NullBuffer>,+ ) -> FixedSizeBinaryArray {+ let width = size_of::<i128>();+ let mut bytes = MutableBuffer::with_capacity(1 + width * values.len());+ bytes.push(0_u8);+ for value in values {+ assert_eq!(value.len(), width);+ bytes.extend_from_slice(value);+ }+ let buffer = Buffer::from(bytes).slice(1);+ assert!(+ !buffer.as_ptr().cast::<i128>().is_aligned(),+ "test buffer must be unaligned"+ );+ FixedSizeBinaryArray::new(width as i32, buffer, nulls)+ }++ #[test]+ fn handles_aligned_and_unaligned_buffers() -> Result<()> {+ let buffer = Buffer::from_vec(vec![1_u64, 2, 3]);+ let source_ptr = buffer.as_ptr();+ let array = FixedSizeBinaryArray::new(8, buffer, None);+ let primitive = reinterpret_as_primitive::<UInt64Type>(&array)?;+ assert_eq!(primitive.values().inner().as_ptr(), source_ptr);++ let width = 16;+ let haystack_values = (0..5)+ .map(|index| value(width, index, false))+ .collect::<Vec<_>>();+ let haystack: ArrayRef = Arc::new(unaligned_i128_array(&haystack_values, None));+ let needles = unaligned_i128_array(+ &[+ value(width, 3, false),+ value(width, 8, true),+ value(width, 9, true),+ ],+ Some(NullBuffer::from(vec![true, false, true])),+ );+ let filter = instantiate_fixed_size_binary_filter(&haystack)?.unwrap();++ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), None, Some(false)])+ );+ Ok(())+ }+}diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rsindex d008217ee1..98bd698507 100644--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs@@ -23,12 +23,17 @@ use arrow::datatypes::DataType;use datafusion_common::Result;use super::array_static_filter::ArrayStaticFilter;+use super::fixed_size_binary_filter::instantiate_fixed_size_binary_filter;use super::primitive_filter::instantiate_primitive_filter;use super::static_filter::StaticFilterRef;pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result<StaticFilterRef> {let in_array = flatten_dictionary_haystack(in_array)?;+ if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? {+ return Ok(filter);+ }+if let Some(filter) = instantiate_primitive_filter(&in_array)? {return Ok(filter);}