Skip to content

Apache DataFusion SQL Query Engine

IN LIST: reuse primitive filters for FixedSizeBinary (#24102)

Geoffrey Claudee2e94ae64fd1
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs
index 762c2c97a1..00a7c476f5 100644
--- a/datafusion/physical-expr/benches/in_list_strategy.rs
+++ b/datafusion/physical-expr/benches/in_list_strategy.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-//! Focused benchmarks for `InList` cases.
+//! Benchmarks for static `IN LIST` filters.
//!
-//! This benchmark file adds targeted coverage for representative `IN LIST`
-//! workloads with controlled parameters:
+//! The cases control match rate and list size across several value types and
+//! string layouts:
//!
//! - **Controlled match rates**: Exercises both hit-heavy and miss-heavy paths
//! - **List size scaling**: Measures behavior across small and large `IN` lists
@@ -27,7 +27,7 @@
//! - **Shared-prefix strings**: Adds collision-heavy string cases where values
//! only differ late in the string
//! - **Mixed-length strings**: Covers inputs that combine short and long values
-//! - **Null handling**: Includes representative `NULL` and `NOT IN` cases
+//! - **Null handling**: Covers `NULL` and `NOT IN` cases
//!
//! # Case Coverage
//!
@@ -45,14 +45,19 @@
//! | Utf8View length-12 cases | Utf8View | 12-byte strings | 16, 64 |
//! | Utf8View long-string cases | Utf8View | 24-byte strings | 4, 16, 64, 256 |
//! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 |
-//! | Fixed-size binary cases | FixedSizeBinary(16) | fixed-width binary values | 4, 64, 256, 10000 |
+//! | Fixed-size binary direct-comparison case | FixedSizeBinary(1) | direct-comparison cutoff | 16 |
+//! | Fixed-size binary direct-comparison case | FixedSizeBinary(16) | direct-comparison cutoff | 4 |
+//! | Fixed-size binary bitmap case | FixedSizeBinary(2) | bitmap lookup | 64 |
+//! | Fixed-size binary hash-set cases | FixedSizeBinary(16) | hash-set scaling | 64, 256, 10000 |
+//! | Fixed-size binary unaligned case | FixedSizeBinary(16) | per-evaluation alignment copy | 64 |
use arrow::array::types::IntervalMonthDayNano;
use arrow::array::*;
+use arrow::buffer::{Buffer, MutableBuffer};
use arrow::datatypes::{Field, Int32Type, IntervalMonthDayNanoType, Schema};
use arrow::record_batch::RecordBatch;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use datafusion_common::ScalarValue;
+use datafusion_common::{HashSet, ScalarValue};
use datafusion_physical_expr::expressions::{col, in_list, lit};
use half::f16;
use rand::distr::Alphanumeric;
@@ -870,7 +875,7 @@ fn bench_dictionary(c: &mut Criterion) {
// NULL HANDLING BENCHMARKS
// =============================================================================
//
-// Tests representative null-containing inputs across primitive and string cases.
+// Null-containing primitive and string cases.
fn bench_nulls(c: &mut Criterion) {
// =========================================================================
@@ -1014,72 +1019,180 @@ fn bench_nulls(c: &mut Criterion) {
}
// =============================================================================
-// FIXED SIZE BINARY BENCHMARKS (FixedSizeBinary<16>, e.g. UUIDs)
+// FIXED SIZE BINARY BENCHMARKS
// =============================================================================
-/// Generates a random 16-byte value (UUID-sized).
-fn random_fixed_binary_16(rng: &mut StdRng) -> Vec<u8> {
- let mut buf = vec![0u8; 16];
+fn random_fixed_binary(rng: &mut StdRng, width: i32) -> Vec<u8> {
+ let mut buf = vec![0u8; width as usize];
rng.fill(&mut buf[..]);
buf
}
-/// Benchmarks FixedSizeBinary(16) IN list evaluation.
-/// FixedSizeBinary doesn't use the generic numeric helpers since its array
-/// construction differs from primitive types.
-fn bench_fixed_size_binary_inner(
- c: &mut Criterion,
- name: &str,
+#[derive(Clone, Copy)]
+enum InputLayout {
+ Aligned,
+ UnalignedI128,
+}
+
+#[derive(Clone, Copy)]
+struct FixedSizeBinaryBenchConfig {
+ width: i32,
+ list_size: usize,
+ input_layout: InputLayout,
+}
+
+impl FixedSizeBinaryBenchConfig {
+ const fn aligned(width: i32, list_size: usize) -> Self {
+ Self {
+ width,
+ list_size,
+ input_layout: InputLayout::Aligned,
+ }
+ }
+
+ const fn unaligned_i128(list_size: usize) -> Self {
+ Self {
+ width: 16,
+ list_size,
+ input_layout: InputLayout::UnalignedI128,
+ }
+ }
+}
+
+const FIXED_SIZE_BINARY_CASES: [FixedSizeBinaryBenchConfig; 7] = [
+ FixedSizeBinaryBenchConfig::aligned(1, 16),
+ FixedSizeBinaryBenchConfig::aligned(2, 64),
+ FixedSizeBinaryBenchConfig::aligned(16, 4),
+ FixedSizeBinaryBenchConfig::aligned(16, 64),
+ FixedSizeBinaryBenchConfig::aligned(16, 256),
+ FixedSizeBinaryBenchConfig::aligned(16, 10000),
+ // 8,192 rows at 16 bytes each copy 128 KiB per evaluation.
+ FixedSizeBinaryBenchConfig::unaligned_i128(64),
+];
+
+fn generate_fixed_size_binary_data(
+ rng: &mut StdRng,
+ width: i32,
list_size: usize,
match_rate: f64,
-) {
- let seed = 0xF1ED_B1A7_u64.wrapping_add(list_size as u64 * 0x6666);
- let mut rng = StdRng::seed_from_u64(seed);
+) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
+ if let Some(domain_size) = match width {
+ 1 => Some(1_usize << 8),
+ 2 => Some(1_usize << 16),
+ _ => None,
+ } {
+ // The value generator needs at least one value outside the haystack.
+ assert!(list_size < domain_size);
+ }
- // Generate IN list values (16-byte each)
- let haystack: Vec<Vec<u8>> = (0..list_size)
- .map(|_| random_fixed_binary_16(&mut rng))
- .collect();
+ // Keep the number of distinct haystack values equal to the configured list size.
+ let mut haystack_set = HashSet::with_capacity(list_size);
+ let mut haystack = Vec::with_capacity(list_size);
+ while haystack.len() < list_size {
+ let value = random_fixed_binary(rng, width);
+ if haystack_set.insert(value.clone()) {
+ haystack.push(value);
+ }
+ }
- // Generate array with controlled match rate
- let values: Vec<Vec<u8>> = (0..ARRAY_SIZE)
+ // Generate values with the configured match rate.
+ let values = (0..ARRAY_SIZE)
.map(|_| {
if !haystack.is_empty() && rng.random_bool(match_rate) {
- haystack.choose(&mut rng).unwrap().clone()
+ haystack.choose(rng).unwrap().clone()
} else {
- random_fixed_binary_16(&mut rng)
+ loop {
+ let value = random_fixed_binary(rng, width);
+ if !haystack_set.contains(&value) {
+ break value;
+ }
+ }
}
})
.collect();
- let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();
- let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap();
+ (haystack, values)
+}
+
+fn unaligned_fixed_size_binary_16(values: &[Vec<u8>]) -> FixedSizeBinaryArray {
+ const WIDTH: usize = 16;
+ let payload_len = values.len() * WIDTH;
+ let mut bytes = MutableBuffer::with_capacity(payload_len + 1);
+ bytes.push(0_u8);
+ for value in values {
+ assert_eq!(value.len(), WIDTH);
+ bytes.extend_from_slice(value);
+ }
+
+ // MutableBuffer starts at an Arrow-aligned address. Fixed-size binary
+ // values only require byte alignment, so slicing off this padding byte
+ // creates a valid Arrow buffer that models unaligned external input.
+ let buffer = Buffer::from(bytes).slice(1);
+ assert!(
+ !buffer.as_ptr().cast::<i128>().is_aligned(),
+ "benchmark input must be unaligned"
+ );
+ FixedSizeBinaryArray::new(WIDTH as i32, buffer, None)
+}
+
+/// FixedSizeBinary doesn't use the generic numeric helpers since its array
+/// construction differs from primitive types.
+fn bench_fixed_size_binary_inner(
+ c: &mut Criterion,
+ config: FixedSizeBinaryBenchConfig,
+ match_pct: u32,
+) {
+ assert!(match_pct <= 100);
+ let match_rate = f64::from(match_pct) / 100.0;
+
+ let seed = 0xF1ED_B1A7_u64
+ .wrapping_add(config.list_size as u64 * 0x6666)
+ .wrapping_add(config.width as u64 * 0x7777);
+ let mut rng = StdRng::seed_from_u64(seed);
+
+ let (haystack, values) = generate_fixed_size_binary_data(
+ &mut rng,
+ config.width,
+ config.list_size,
+ match_rate,
+ );
+
+ let array = match config.input_layout {
+ InputLayout::Aligned => {
+ FixedSizeBinaryArray::try_from_iter(values.iter().map(Vec::as_slice)).unwrap()
+ }
+ InputLayout::UnalignedI128 => unaligned_fixed_size_binary_16(&values),
+ };
let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]);
let exprs: Vec<_> = haystack
.iter()
- .map(|v| lit(ScalarValue::FixedSizeBinary(16, Some(v.clone()))))
+ .map(|v| lit(ScalarValue::FixedSizeBinary(config.width, Some(v.clone()))))
.collect();
let expr = in_list(col("a", &schema).unwrap(), exprs, &false, &schema).unwrap();
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array) as ArrayRef])
.unwrap();
c.bench_with_input(
- BenchmarkId::new("fixed_size_binary", name),
+ BenchmarkId::new("fixed_size_binary", {
+ let name = format!(
+ "fsb{}/list={}/match={match_pct}%",
+ config.width, config.list_size
+ );
+ match config.input_layout {
+ InputLayout::Aligned => name,
+ InputLayout::UnalignedI128 => format!("{name}/input=unaligned"),
+ }
+ }),
&batch,
|b, batch| b.iter(|| expr.evaluate(batch).unwrap()),
);
}
fn bench_fixed_size_binary(c: &mut Criterion) {
- for list_size in [4, 64, 256, 10000] {
+ for config in FIXED_SIZE_BINARY_CASES {
for match_pct in MATCH_RATES {
- bench_fixed_size_binary_inner(
- c,
- &format!("fsb16/list={list_size}/match={match_pct}%"),
- list_size,
- match_pct as f64 / 100.0,
- );
+ bench_fixed_size_binary_inner(c, config, match_pct);
}
}
}
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs
index 0fb978cd0b..154decfd8b 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -38,6 +38,7 @@ use datafusion_expr::{ColumnarValue, expr_vec_fmt};
mod array_static_filter;
mod branchless_filter;
+mod fixed_size_binary_filter;
mod primitive_filter;
mod result;
mod static_filter;
@@ -3548,6 +3549,38 @@ mod tests {
);
}
+ // FixedSizeBinary in_array, FixedSizeBinary and Dictionary needles
+ let fsb_in = Arc::new(FixedSizeBinaryArray::try_from_iter(
+ [
+ [1, 2, 3, 4].as_slice(),
+ [5, 6, 7, 8].as_slice(),
+ [9, 10, 11, 12].as_slice(),
+ ]
+ .into_iter(),
+ )?) as ArrayRef;
+ let fsb_needle = Arc::new(FixedSizeBinaryArray::try_from_iter(
+ [
+ [1, 2, 3, 4].as_slice(),
+ [13, 14, 15, 16].as_slice(),
+ [5, 6, 7, 8].as_slice(),
+ ]
+ .into_iter(),
+ )?) as ArrayRef;
+ assert_eq!(
+ expected,
+ eval_in_list_from_array(Arc::clone(&fsb_needle), Arc::clone(&fsb_in))?
+ );
+ // The dictionary does not reference its second value, so that value
+ // must not become a member of the flattened list.
+ let dict_fsb_in = Arc::new(DictionaryArray::new(
+ Int32Array::from(vec![0, 2]),
+ Arc::clone(&fsb_in),
+ ));
+ assert_eq!(
+ BooleanArray::from(vec![Some(true), Some(false), Some(false)]),
+ eval_in_list_from_array(wrap_in_dict(fsb_needle), dict_fsb_in)?
+ );
+
// Utf8 (falls through to ArrayStaticFilter)
let utf8_in = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef;
let utf8_needle = Arc::new(StringArray::from(vec!["a", "d", "b"])) as ArrayRef;
diff --git a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs
new file mode 100644
index 0000000000..421c02ea1f
--- /dev/null
+++ b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs
@@ -0,0 +1,371 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Optimized filters for fixed-size binary `IN` lists.
+//!
+//! Supported widths use an Arrow primitive representation with the same
+//! in-memory size:
+//!
+//! | Width | Primitive representation |
+//! |------:|--------------------------|
+//! | 1 | `UInt8` |
+//! | 2 | `UInt16` |
+//! | 4 | `UInt32` |
+//! | 8 | `UInt64` |
+//! | 16 | `Decimal128` |
+//!
+//! The shared primitive selector applies the native primitive branchless cutoffs
+//! and chooses the bitmap or hash-set fallback.
+//!
+//! The list and input bytes are read the same way, so each primitive value is
+//! an exact key for comparison, bitmap lookup, or hashing. No arithmetic,
+//! ordering, or decimal operations are used.
+//!
+//! Reinterpreting an aligned Arrow buffer is zero-copy. An unaligned buffer is
+//! copied into aligned primitive storage before filter construction or probing.
+
+use std::marker::PhantomData;
+use std::mem::size_of;
+use std::sync::Arc;
+
+use arrow::array::{
+ Array, ArrayRef, AsArray, BooleanArray, FixedSizeBinaryArray, PrimitiveArray,
+};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{
+ ArrowPrimitiveType, DataType, Decimal128Type, UInt8Type, UInt16Type, UInt32Type,
+ UInt64Type,
+};
+use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};
+
+use super::primitive_filter::instantiate_primitive_filter;
+use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary};
+
+/// Reinterpret fixed-size binary values as same-width primitive values.
+///
+/// Arrow buffers are normally sufficiently aligned, making this zero-copy. A
+/// valid Arrow array can still be constructed from a sliced, unaligned buffer;
+/// in that case, copy each value into aligned primitive storage.
+fn reinterpret_as_primitive<T>(array: &FixedSizeBinaryArray) -> Result<PrimitiveArray<T>>
+where
+ T: ArrowPrimitiveType,
+{
+ let width = size_of::<T::Native>();
+ if array.value_size() != width {
+ return Err(internal_datafusion_err!(
+ "FixedSizeBinary filter: expected {width}-byte values, got {}",
+ array.value_size()
+ ));
+ }
+
+ let source = array.values();
+ let values = if source.as_ptr().cast::<T::Native>().is_aligned() {
+ ScalarBuffer::new(source.clone(), 0, array.len())
+ } else {
+ // `Buffer::from(&[u8])` copies into Arrow-aligned storage.
+ ScalarBuffer::new(Buffer::from(source.as_slice()), 0, array.len())
+ };
+
+ Ok(PrimitiveArray::<T>::new(values, array.nulls().cloned()))
+}
+
+/// Adapts a primitive filter to concrete, same-width `FixedSizeBinary` arrays.
+struct FixedSizeBinaryFilter<T: ArrowPrimitiveType> {
+ data_type: DataType,
+ inner: StaticFilterRef,
+ _marker: PhantomData<T>,
+}
+
+impl<T> StaticFilter for FixedSizeBinaryFilter<T>
+where
+ T: ArrowPrimitiveType + Send + Sync + 'static,
+{
+ fn null_count(&self) -> usize {
+ self.inner.null_count()
+ }
+
+ fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {
+ handle_dictionary!(self, v, negated);
+
+ if v.data_type() != &self.data_type {
+ return Err(exec_datafusion_err!(
+ "FixedSizeBinary filter: expected {} array, got {}",
+ self.data_type,
+ v.data_type()
+ ));
+ }
+ let array = v.as_fixed_size_binary_opt().ok_or_else(|| {
+ exec_datafusion_err!(
+ "FixedSizeBinary filter: expected concrete {} array",
+ self.data_type
+ )
+ })?;
+ let primitive = reinterpret_as_primitive::<T>(array)?;
+ self.inner.contains(&primitive, negated)
+ }
+}
+
+fn instantiate_for_primitive<T>(array: &FixedSizeBinaryArray) -> Result<StaticFilterRef>
+where
+ T: ArrowPrimitiveType + Send + Sync + 'static,
+{
+ let primitive: ArrayRef = Arc::new(reinterpret_as_primitive::<T>(array)?);
+ let inner = instantiate_primitive_filter(&primitive)?.ok_or_else(|| {
+ internal_datafusion_err!(
+ "FixedSizeBinary filter: no primitive filter for {}",
+ primitive.data_type()
+ )
+ })?;
+ Ok(Arc::new(FixedSizeBinaryFilter::<T> {
+ data_type: array.data_type().clone(),
+ inner,
+ _marker: PhantomData,
+ }))
+}
+
+/// Creates an optimized filter for supported concrete `FixedSizeBinary` arrays.
+pub(super) fn instantiate_fixed_size_binary_filter(
+ in_array: &ArrayRef,
+) -> Result<Option<StaticFilterRef>> {
+ let DataType::FixedSizeBinary(width) = in_array.data_type() else {
+ return Ok(None);
+ };
+ let Some(array) = in_array.as_fixed_size_binary_opt() else {
+ return Ok(None);
+ };
+
+ let filter = match width {
+ 1 => instantiate_for_primitive::<UInt8Type>(array)?,
+ 2 => instantiate_for_primitive::<UInt16Type>(array)?,
+ 4 => instantiate_for_primitive::<UInt32Type>(array)?,
+ 8 => instantiate_for_primitive::<UInt64Type>(array)?,
+ 16 => instantiate_for_primitive::<Decimal128Type>(array)?,
+ _ => return Ok(None),
+ };
+ Ok(Some(filter))
+}
+
+#[cfg(test)]
+mod tests {
+ use arrow::array::{DictionaryArray, Int8Array, StringArray};
+ use arrow::buffer::{Buffer, MutableBuffer, NullBuffer};
+ use arrow::datatypes::Int8Type;
+
+ use super::*;
+
+ fn value(width: i32, index: usize, miss: bool) -> Vec<u8> {
+ let mut value = (index as u128).to_le_bytes()[..width as usize].to_vec();
+ let last = value.last_mut().unwrap();
+ if miss {
+ *last |= 0x80;
+ } else {
+ *last &= 0x7f;
+ }
+ value
+ }
+
+ fn array(width: i32, values: &[Option<Vec<u8>>]) -> FixedSizeBinaryArray {
+ FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+ values.iter().map(|value| value.as_deref()),
+ width,
+ )
+ .unwrap()
+ }
+
+ fn make_filter(width: i32, values: &[Option<Vec<u8>>]) -> Result<StaticFilterRef> {
+ let in_array: ArrayRef = Arc::new(array(width, values));
+ Ok(instantiate_fixed_size_binary_filter(&in_array)?.unwrap())
+ }
+
+ #[test]
+ fn filters_supported_widths_across_strategy_thresholds() -> Result<()> {
+ for (width, list_len) in [
+ (1, 16),
+ (1, 17),
+ (2, 8),
+ (2, 9),
+ (4, 32),
+ (4, 33),
+ (8, 16),
+ (8, 17),
+ (16, 4),
+ (16, 5),
+ ] {
+ let mut hit = vec![0x80; width as usize];
+ hit[width as usize - 1] = 0xff;
+ let mut miss = hit.clone();
+ miss[width as usize - 1] ^= 1;
+
+ let mut haystack = (0..list_len - 1)
+ .map(|index| Some(value(width, index, false)))
+ .collect::<Vec<_>>();
+ haystack.push(Some(hit.clone()));
+ let filter = make_filter(width, &haystack)?;
+ let needles = array(width, &[Some(hit), Some(miss), None]);
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), Some(false), None]),
+ "width={width}, list_len={list_len}"
+ );
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn handles_slices_nulls_and_not_in() -> Result<()> {
+ let width = 16;
+ let parent = array(
+ width,
+ &[
+ Some(value(width, 0, false)),
+ Some(value(width, 1, false)),
+ None,
+ Some(value(width, 2, false)),
+ Some(value(width, 3, false)),
+ Some(value(width, 4, false)),
+ Some(value(width, 5, false)),
+ Some(value(width, 6, false)),
+ ],
+ );
+ // Five non-null values select the hash-set path.
+ let in_array: ArrayRef = Arc::new(parent.slice(1, 6));
+ let filter = instantiate_fixed_size_binary_filter(&in_array)?.unwrap();
+ let needles = array(
+ width,
+ &[
+ Some(value(width, 2, false)),
+ Some(value(width, 0, false)),
+ Some(value(width, 6, false)),
+ Some(value(width, 7, false)),
+ None,
+ ],
+ );
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), None, None, None, None])
+ );
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), None, None, None, None])
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn handles_dictionary_needles() -> Result<()> {
+ let filter = make_filter(4, &[Some(value(4, 7, false))])?;
+ let dictionary_values: ArrayRef = Arc::new(array(
+ 4,
+ &[Some(value(4, 7, false)), Some(value(4, 8, false))],
+ ));
+ let keys = Int8Array::from(vec![Some(0), Some(1), None]);
+ let needles =
+ DictionaryArray::<Int8Type>::try_new(keys, dictionary_values).unwrap();
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), Some(false), None])
+ );
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), Some(true), None])
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_unsupported_arrays() -> Result<()> {
+ let filter = make_filter(4, &[Some(value(4, 1, false))])?;
+ let wrong_width = array(8, &[Some(value(8, 1, false))]);
+ let error = filter
+ .contains(&wrong_width, false)
+ .unwrap_err()
+ .to_string();
+ assert!(
+ error.contains("expected FixedSizeBinary(4) array, got FixedSizeBinary(8)"),
+ "{error}"
+ );
+
+ let wrong_type = StringArray::from(vec!["one"]);
+ let error = filter.contains(&wrong_type, false).unwrap_err().to_string();
+ assert!(
+ error.contains("expected FixedSizeBinary(4) array, got Utf8"),
+ "{error}"
+ );
+
+ for width in [0, 3, 5, 15, 17] {
+ let unsupported: ArrayRef =
+ Arc::new(FixedSizeBinaryArray::new_null(width, 1));
+ assert!(
+ instantiate_fixed_size_binary_filter(&unsupported)?.is_none(),
+ "width={width}"
+ );
+ }
+
+ Ok(())
+ }
+
+ fn unaligned_i128_array(
+ values: &[Vec<u8>],
+ nulls: Option<NullBuffer>,
+ ) -> FixedSizeBinaryArray {
+ let width = size_of::<i128>();
+ let mut bytes = MutableBuffer::with_capacity(1 + width * values.len());
+ bytes.push(0_u8);
+ for value in values {
+ assert_eq!(value.len(), width);
+ bytes.extend_from_slice(value);
+ }
+ let buffer = Buffer::from(bytes).slice(1);
+ assert!(
+ !buffer.as_ptr().cast::<i128>().is_aligned(),
+ "test buffer must be unaligned"
+ );
+ FixedSizeBinaryArray::new(width as i32, buffer, nulls)
+ }
+
+ #[test]
+ fn handles_aligned_and_unaligned_buffers() -> Result<()> {
+ let buffer = Buffer::from_vec(vec![1_u64, 2, 3]);
+ let source_ptr = buffer.as_ptr();
+ let array = FixedSizeBinaryArray::new(8, buffer, None);
+ let primitive = reinterpret_as_primitive::<UInt64Type>(&array)?;
+ assert_eq!(primitive.values().inner().as_ptr(), source_ptr);
+
+ let width = 16;
+ let haystack_values = (0..5)
+ .map(|index| value(width, index, false))
+ .collect::<Vec<_>>();
+ let haystack: ArrayRef = Arc::new(unaligned_i128_array(&haystack_values, None));
+ let needles = unaligned_i128_array(
+ &[
+ value(width, 3, false),
+ value(width, 8, true),
+ value(width, 9, true),
+ ],
+ Some(NullBuffer::from(vec![true, false, true])),
+ );
+ let filter = instantiate_fixed_size_binary_filter(&haystack)?.unwrap();
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), None, Some(false)])
+ );
+ Ok(())
+ }
+}
diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
index d008217ee1..98bd698507 100644
--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
@@ -23,12 +23,17 @@ use arrow::datatypes::DataType;
use datafusion_common::Result;
use super::array_static_filter::ArrayStaticFilter;
+use super::fixed_size_binary_filter::instantiate_fixed_size_binary_filter;
use super::primitive_filter::instantiate_primitive_filter;
use super::static_filter::StaticFilterRef;
pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result<StaticFilterRef> {
let in_array = flatten_dictionary_haystack(in_array)?;
+ if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? {
+ return Ok(filter);
+ }
+
if let Some(filter) = instantiate_primitive_filter(&in_array)? {
return Ok(filter);
}