Skip to content

Apache DataFusion SQL Query Engine

IN LIST: add branchless filter for small primitive lists (#23014)

Geoffrey Claudee3e2cb227928
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs
index c70f6da2a4..c69af192b9 100644
--- a/datafusion/physical-expr/benches/in_list_strategy.rs
+++ b/datafusion/physical-expr/benches/in_list_strategy.rs
@@ -37,6 +37,7 @@
//! | Narrow integer cases | Int16, Float16 | larger value domain | 4, 64, 256 |
//! | 32-bit primitive cases | Int32, Float32 | small and large lists | 4, 32, 64, 256 |
//! | 64-bit primitive cases | Int64, TimestampNs | small and large lists | 4, 16, 32, 128 |
+//! | 128-bit interval cases | IntervalMonthDayNano | small lists | 4 |
//! | Utf8 short-string cases | Utf8 | 8-byte strings | 4, 64, 256 |
//! | Utf8 long-string cases | Utf8 | 24-byte strings | 4, 64, 256 |
//! | Utf8View short-string cases | Utf8View | 8-byte strings | 4, 16, 64, 256 |
@@ -45,8 +46,9 @@
//! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 |
//! | Fixed-size binary cases | FixedSizeBinary(16) | fixed-width binary values | 4, 64, 256, 10000 |
+use arrow::array::types::IntervalMonthDayNano;
use arrow::array::*;
-use arrow::datatypes::{Field, Int32Type, Schema};
+use arrow::datatypes::{Field, Int32Type, IntervalMonthDayNanoType, Schema};
use arrow::record_batch::RecordBatch;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
@@ -528,6 +530,28 @@ fn bench_timestamp_ns(c: &mut Criterion) {
}
}
+fn bench_interval_month_day_nano(c: &mut Criterion) {
+ for match_pct in MATCH_RATES {
+ bench_numeric::<IntervalMonthDayNano, IntervalMonthDayNanoArray>(
+ c,
+ "interval_month_day_nano",
+ &format!("small_list/list=4/match={match_pct}%"),
+ &NumericBenchConfig::new(
+ 4,
+ match_pct as f64 / 100.0,
+ |rng| {
+ IntervalMonthDayNanoType::make_value(
+ rng.random_range(-120..=120),
+ rng.random_range(-31..=31),
+ rng.random_range(-1_000_000_000..=1_000_000_000),
+ )
+ },
+ |v| ScalarValue::IntervalMonthDayNano(Some(v)),
+ ),
+ );
+ }
+}
+
// =============================================================================
// UTF8 STRING CASE BENCHMARKS
// =============================================================================
@@ -1049,7 +1073,7 @@ fn bench_fixed_size_binary(c: &mut Criterion) {
criterion_group! {
name = benches;
config = Criterion::default();
- targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary
+ targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary
}
criterion_main!(benches);
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs
index 2764083f31..e4ec72285c 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -2900,10 +2900,9 @@ mod tests {
#[test]
fn test_in_list_esoteric_types() -> Result<()> {
- // Test esoteric/less common types to validate the transform and mapping flow.
- // These types are reinterpreted to base primitive types (e.g., Timestamp -> UInt64,
- // Interval -> Decimal128, Float16 -> UInt16). We just need to verify basic
- // functionality works - no need for comprehensive null handling tests.
+ // Test less common types covered by IN-list evaluation. Some of these
+ // use specialized filters, and others fall back to the generic path;
+ // this keeps the end-to-end behavior covered either way.
// Helper: simple IN test that expects [Some(true), Some(false)]
let test_type = |data_type: DataType,
@@ -2926,7 +2925,7 @@ mod tests {
Ok(())
};
- // Timestamp types (all units map to Int64 -> UInt64)
+ // Timestamp types
test_type(
DataType::Timestamp(TimeUnit::Second, None),
Arc::new(TimestampSecondArray::from(vec![Some(1000), Some(2000)])),
@@ -2960,7 +2959,7 @@ mod tests {
],
)?;
- // Time32 and Time64 (map to Int32 -> UInt32 and Int64 -> UInt64 respectively)
+ // Time32 and Time64
test_type(
DataType::Time32(TimeUnit::Second),
Arc::new(Time32SecondArray::from(vec![Some(3600), Some(7200)])),
@@ -3006,7 +3005,7 @@ mod tests {
],
)?;
- // Duration types (map to Int64 -> UInt64)
+ // Duration types
test_type(
DataType::Duration(TimeUnit::Second),
Arc::new(DurationSecondArray::from(vec![Some(86400), Some(172800)])),
@@ -3052,7 +3051,7 @@ mod tests {
],
)?;
- // Interval types (map to 16-byte Decimal128Type)
+ // Interval types
test_type(
DataType::Interval(IntervalUnit::YearMonth),
Arc::new(IntervalYearMonthArray::from(vec![Some(12), Some(24)])),
@@ -3114,8 +3113,7 @@ mod tests {
],
)?;
- // Decimal256 (maps to Decimal128Type for 16-byte width)
- // Need to use with_precision_and_scale() to set the metadata
+ // Decimal256. Need to use with_precision_and_scale() to set the metadata.
let precision = 38;
let scale = 10;
test_type(
diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
index 8f8d9bad04..e802e1d024 100644
--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
@@ -19,14 +19,15 @@
//!
//! This module provides membership tests for Arrow primitive types.
-use arrow::array::{Array, ArrayRef, AsArray, BooleanArray};
-use arrow::buffer::{BooleanBuffer, NullBuffer};
+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, PrimitiveArray};
+use arrow::buffer::{BooleanBuffer, NullBuffer, ScalarBuffer};
use arrow::datatypes::*;
use arrow::util::bit_iterator::BitIndexIterator;
use datafusion_common::{HashSet, Result, exec_datafusion_err};
use std::hash::{Hash, Hasher};
+use std::mem::size_of;
-use super::result::build_in_list_result;
+use super::result::{build_in_list_result, build_result_from_contains};
use super::static_filter::{StaticFilter, handle_dictionary};
/// Storage for the bits used by [`BitmapFilter`].
@@ -222,6 +223,276 @@ where
}
}
+pub(super) type BranchlessNative<T> =
+ <<T as BranchlessFilterType>::CompareType as ArrowPrimitiveType>::Native;
+
+/// Maximum list size for branchless lookup on 1-byte primitives.
+///
+/// Sixteen 1-byte values fit in one 128-bit SIMD vector, so this keeps the
+/// branchless list small enough for a single vectorized membership check.
+const BRANCHLESS_MAX_1B: usize = 16;
+
+/// Maximum list size for branchless lookup on 2-byte primitives.
+///
+/// Eight 2-byte values fit in one 128-bit SIMD vector, so this keeps the
+/// branchless list small enough for a single vectorized membership check.
+const BRANCHLESS_MAX_2B: usize = 8;
+
+/// Maximum list size for branchless lookup on 4-byte primitives.
+///
+/// Thirty-two 4-byte values keep the inline list at 128 bytes. Beyond that,
+/// the comparison chain and filter footprint grow enough that the hash/generic
+/// fallback is a better fit.
+const BRANCHLESS_MAX_4B: usize = 32;
+
+/// Maximum list size for branchless lookup on 8-byte primitives.
+///
+/// Sixteen 8-byte values use the same 128-byte inline-list budget as 4-byte
+/// primitives. Larger lists are left to the hash/generic fallback.
+const BRANCHLESS_MAX_8B: usize = 16;
+
+/// Maximum list size for branchless lookup on 16-byte primitives.
+///
+/// These comparisons are wider, so this path is limited to four values.
+/// Larger lists are left to the generic fallback.
+const BRANCHLESS_MAX_16B: usize = 4;
+
+/// Arrow primitive types supported by [`BranchlessFilter`].
+///
+/// `T` is the logical Arrow type accepted by the filter. `CompareType` is the
+/// same-width type used for the fixed comparison chain. Signed integers,
+/// floats, and temporal values use an unsigned comparison type so they compare
+/// by their raw bit pattern.
+pub(super) trait BranchlessFilterType:
+ ArrowPrimitiveType + Send + Sync + 'static
+{
+ type CompareType: ArrowPrimitiveType + Send + Sync + 'static;
+
+ /// Maximum number of non-null IN-list values to handle with
+ /// [`BranchlessFilter`] for this primitive type.
+ const MAX_LIST_LEN: usize;
+}
+
+macro_rules! branchless_filter_type {
+ ($logical:ty, $compare:ty, $max_len:expr) => {
+ // The branchless filter reads the same Arrow value buffer as the
+ // comparison type. That is only valid when both native types have the
+ // same width, so catch any bad mapping here at compile time.
+ const _: () = assert!(
+ size_of::<<$logical as ArrowPrimitiveType>::Native>()
+ == size_of::<<$compare as ArrowPrimitiveType>::Native>(),
+ "BranchlessFilterType::CompareType must use the same native width"
+ );
+
+ impl BranchlessFilterType for $logical {
+ type CompareType = $compare;
+ const MAX_LIST_LEN: usize = $max_len;
+ }
+ };
+}
+
+branchless_filter_type!(Int8Type, UInt8Type, BRANCHLESS_MAX_1B);
+branchless_filter_type!(UInt8Type, UInt8Type, BRANCHLESS_MAX_1B);
+branchless_filter_type!(Int16Type, UInt16Type, BRANCHLESS_MAX_2B);
+branchless_filter_type!(UInt16Type, UInt16Type, BRANCHLESS_MAX_2B);
+branchless_filter_type!(Float16Type, UInt16Type, BRANCHLESS_MAX_2B);
+
+branchless_filter_type!(Int32Type, UInt32Type, BRANCHLESS_MAX_4B);
+branchless_filter_type!(UInt32Type, UInt32Type, BRANCHLESS_MAX_4B);
+branchless_filter_type!(Float32Type, UInt32Type, BRANCHLESS_MAX_4B);
+branchless_filter_type!(Date32Type, UInt32Type, BRANCHLESS_MAX_4B);
+branchless_filter_type!(Time32SecondType, UInt32Type, BRANCHLESS_MAX_4B);
+branchless_filter_type!(Time32MillisecondType, UInt32Type, BRANCHLESS_MAX_4B);
+
+branchless_filter_type!(Int64Type, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(UInt64Type, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(Float64Type, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(Date64Type, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(Time64MicrosecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(Time64NanosecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(TimestampSecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(TimestampMillisecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(TimestampMicrosecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(TimestampNanosecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(DurationSecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(DurationMillisecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(DurationMicrosecondType, UInt64Type, BRANCHLESS_MAX_8B);
+branchless_filter_type!(DurationNanosecondType, UInt64Type, BRANCHLESS_MAX_8B);
+
+branchless_filter_type!(Decimal128Type, Decimal128Type, BRANCHLESS_MAX_16B);
+branchless_filter_type!(
+ IntervalMonthDayNanoType,
+ IntervalMonthDayNanoType,
+ BRANCHLESS_MAX_16B
+);
+
+/// Checks each input value against the `IN`-list values.
+type MembershipCheck<C> = fn(in_list_values: &[C], input_values: &[C]) -> BooleanBuffer;
+
+/// A branchless filter for fixed-width primitive `IN` lists up to
+/// `T::MAX_LIST_LEN` values.
+///
+/// The filter stores the non-null `IN`-list values in a slice and chooses a
+/// comparison function for that length. Keeping the length out of
+/// `BranchlessFilter` avoids generating a full copy of the filter for every
+/// supported length.
+pub(super) struct BranchlessFilter<T: BranchlessFilterType> {
+ expected_data_type: DataType,
+ null_count: usize,
+ in_list_values: Box<[BranchlessNative<T>]>,
+ check_values: MembershipCheck<BranchlessNative<T>>,
+}
+
+impl<T> BranchlessFilter<T>
+where
+ T: BranchlessFilterType,
+ BranchlessNative<T>: Copy + PartialEq,
+{
+ pub(super) fn try_new(in_array: &ArrayRef) -> Result<Self> {
+ let in_array = in_array.as_primitive_opt::<T>().ok_or_else(|| {
+ exec_datafusion_err!("BranchlessFilter: expected {} array", T::DATA_TYPE)
+ })?;
+ let non_null_count = in_array.len() - in_array.null_count();
+ // `try_new` can be called on its own, so check the limit here too.
+ if non_null_count > T::MAX_LIST_LEN {
+ return Err(exec_datafusion_err!(
+ "BranchlessFilter: supports at most {} non-null values, got {non_null_count}",
+ T::MAX_LIST_LEN
+ ));
+ }
+
+ let all_values = branchless_values::<T>(in_array);
+ let mut in_list_values = Vec::with_capacity(non_null_count);
+
+ match in_array.nulls() {
+ None => {
+ in_list_values.extend(all_values.iter().copied());
+ }
+ Some(nulls) => {
+ for row in
+ BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len())
+ {
+ in_list_values.push(all_values[row]);
+ }
+ }
+ }
+
+ debug_assert_eq!(in_list_values.len(), non_null_count);
+ let in_list_values = in_list_values.into_boxed_slice();
+ let check_values = membership_check_for_len::<T>(in_list_values.len());
+
+ Ok(Self {
+ expected_data_type: in_array.data_type().clone(),
+ null_count: in_array.null_count(),
+ in_list_values,
+ check_values,
+ })
+ }
+}
+
+impl<T> StaticFilter for BranchlessFilter<T>
+where
+ T: BranchlessFilterType,
+ BranchlessNative<T>: Copy + PartialEq + Send + Sync,
+{
+ fn null_count(&self) -> usize {
+ self.null_count
+ }
+
+ fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {
+ handle_dictionary!(self, v, negated);
+
+ // Arrow compatibility ignores timestamp timezone and decimal precision/scale
+ // while still requiring the same primitive representation.
+ if !PrimitiveArray::<T>::is_compatible(v.data_type()) {
+ return Err(exec_datafusion_err!(
+ "BranchlessFilter: expected {} array, got {}",
+ self.expected_data_type,
+ v.data_type()
+ ));
+ }
+
+ let v = v.as_primitive_opt::<T>().ok_or_else(|| {
+ exec_datafusion_err!("BranchlessFilter: expected {} array", T::DATA_TYPE)
+ })?;
+ let input_values = branchless_values::<T>(v);
+ let matches =
+ (self.check_values)(self.in_list_values.as_ref(), input_values.as_ref());
+ Ok(build_result_from_contains(
+ v.nulls(),
+ self.null_count > 0,
+ negated,
+ matches,
+ ))
+ }
+}
+
+/// Picks the comparison function for `len` non-null `IN`-list values.
+///
+/// A length of zero is used when the list contains only nulls. The comparisons
+/// return false, and the caller then applies the usual SQL null behavior.
+fn membership_check_for_len<T>(len: usize) -> MembershipCheck<BranchlessNative<T>>
+where
+ T: BranchlessFilterType,
+ BranchlessNative<T>: Copy + PartialEq,
+{
+ macro_rules! choose {
+ ($($n:literal),* $(,)?) => {
+ match len {
+ $($n => check_values::<BranchlessNative<T>, $n>,)*
+ _ => unreachable!("list length exceeds the configured limit"),
+ }
+ };
+ }
+
+ // Avoid creating checks for lengths a type does not support.
+ match T::MAX_LIST_LEN {
+ 4 => choose!(0, 1, 2, 3, 4),
+ 8 => choose!(0, 1, 2, 3, 4, 5, 6, 7, 8),
+ 16 => choose!(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16),
+ 32 => choose!(
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
+ 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
+ ),
+ _ => unreachable!("list-size limits must be 4, 8, 16, or 32"),
+ }
+}
+
+#[inline]
+fn check_values<C, const N: usize>(
+ in_list_values: &[C],
+ input_values: &[C],
+) -> BooleanBuffer
+where
+ C: Copy + PartialEq,
+{
+ let in_list_values: &[C; N] = in_list_values
+ .try_into()
+ .expect("comparison length matches IN-list values");
+
+ BooleanBuffer::collect_bool(input_values.len(), |i| {
+ // SAFETY: `collect_bool` invokes this closure for indices in
+ // `0..input_values.len()`.
+ let input_value = unsafe { *input_values.get_unchecked(i) };
+ // `|` checks every list value; `||` would stop after the first match.
+ in_list_values
+ .iter()
+ .fold(false, |acc, &value| acc | (value == input_value))
+ })
+}
+
+fn branchless_values<T>(array: &PrimitiveArray<T>) -> ScalarBuffer<BranchlessNative<T>>
+where
+ T: BranchlessFilterType,
+{
+ let data = array.to_data();
+ ScalarBuffer::<BranchlessNative<T>>::new(
+ data.buffers()[0].clone(),
+ data.offset(),
+ data.len(),
+ )
+}
+
/// Wrapper for f32 that implements Hash and Eq using bit comparison.
/// This treats NaN values as equal to each other when they have the same bit pattern.
#[derive(Clone, Copy)]
@@ -430,7 +701,9 @@ mod tests {
use std::sync::Arc;
use arrow::array::{
- DictionaryArray, Float16Array, Int8Array, Int16Array, UInt8Array, UInt16Array,
+ Decimal128Array, DictionaryArray, Float16Array, Float32Array, Float64Array,
+ Int8Array, Int16Array, IntervalMonthDayNanoArray, TimestampMillisecondArray,
+ TimestampNanosecondArray, UInt8Array, UInt16Array,
};
use half::f16;
@@ -584,4 +857,206 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn branchless_filter_u8_handles_nulls() -> Result<()> {
+ let haystack: ArrayRef = Arc::new(UInt8Array::from(vec![Some(1), None, Some(3)]));
+ let filter = BranchlessFilter::<UInt8Type>::try_new(&haystack)?;
+ let needles = UInt8Array::from(vec![Some(1), Some(2), None, Some(3)]);
+
+ assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])?;
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), None, None, Some(false)])
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_all_null_list_preserves_sql_null_semantics() -> Result<()> {
+ let haystack: ArrayRef = Arc::new(UInt8Array::from(vec![None, None]));
+ let filter = BranchlessFilter::<UInt8Type>::try_new(&haystack)?;
+ let needles = UInt8Array::from(vec![Some(1), None]);
+ let expected = BooleanArray::from(vec![None, None]);
+
+ assert_eq!(filter.contains(&needles, false)?, expected);
+ assert_eq!(filter.contains(&needles, true)?, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_i8_handles_signed_boundaries_and_slices() -> Result<()> {
+ let haystack: ArrayRef = Arc::new(
+ Int8Array::from(vec![Some(99), Some(i8::MIN), None, Some(-1), Some(42)])
+ .slice(1, 3),
+ );
+ let filter = BranchlessFilter::<Int8Type>::try_new(&haystack)?;
+ let needles =
+ Int8Array::from(vec![Some(7), Some(i8::MIN), Some(-1), None]).slice(1, 3);
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), Some(true), None])
+ );
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), Some(false), None])
+ );
+
+ let wrong_type = UInt8Array::from(vec![Some(128), Some(u8::MAX)]);
+ let err = filter.contains(&wrong_type, false).unwrap_err().to_string();
+ assert!(err.contains("expected Int8 array, got UInt8"), "{err}");
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_f16_handles_bit_patterns_and_slices() -> Result<()> {
+ let nan_a = f16::from_bits(0x7e01);
+ let nan_b = f16::from_bits(0x7e02);
+ let haystack: ArrayRef = Arc::new(
+ Float16Array::from(vec![
+ Some(f16::from_f32(9.0)),
+ Some(f16::from_f32(-0.0)),
+ Some(nan_a),
+ None,
+ ])
+ .slice(1, 3),
+ );
+ let filter = BranchlessFilter::<Float16Type>::try_new(&haystack)?;
+ let needles = Float16Array::from(vec![
+ Some(f16::from_f32(0.0)),
+ Some(f16::from_f32(-0.0)),
+ Some(nan_a),
+ Some(nan_b),
+ None,
+ ]);
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![None, Some(true), Some(true), None, None])
+ );
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![None, Some(false), Some(false), None, None])
+ );
+
+ let wrong_type = UInt16Array::from(vec![Some(0x8000), Some(0x7e01)]);
+ let err = filter.contains(&wrong_type, false).unwrap_err().to_string();
+ assert!(err.contains("expected Float16 array, got UInt16"), "{err}");
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_floats_use_bit_equality() -> Result<()> {
+ let nan_a = f32::from_bits(0x7fc0_0001);
+ let nan_b = f32::from_bits(0x7fc0_0002);
+ let haystack: ArrayRef =
+ Arc::new(Float32Array::from(vec![Some(-0.0), Some(nan_a)]));
+ let filter = BranchlessFilter::<Float32Type>::try_new(&haystack)?;
+ let needles =
+ Float32Array::from(vec![Some(0.0), Some(-0.0), Some(nan_a), Some(nan_b)]);
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(false), Some(true), Some(true), Some(false)])
+ );
+
+ let nan_a = f64::from_bits(0x7ff8_0000_0000_0001);
+ let nan_b = f64::from_bits(0x7ff8_0000_0000_0002);
+ let haystack: ArrayRef =
+ Arc::new(Float64Array::from(vec![Some(-0.0), Some(nan_a)]));
+ let filter = BranchlessFilter::<Float64Type>::try_new(&haystack)?;
+ let needles =
+ Float64Array::from(vec![Some(0.0), Some(-0.0), Some(nan_a), Some(nan_b)]);
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(false), Some(true), Some(true), Some(false)])
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_timestamp_uses_physical_compatibility() -> Result<()> {
+ let haystack: ArrayRef = Arc::new(
+ TimestampNanosecondArray::from(vec![Some(1), Some(3)]).with_timezone("UTC"),
+ );
+ let filter = BranchlessFilter::<TimestampNanosecondType>::try_new(&haystack)?;
+ let needles = TimestampNanosecondArray::from(vec![Some(1), Some(2), None])
+ .with_timezone("UTC");
+
+ assert_contains(&filter, &needles, vec![Some(true), Some(false), None])?;
+
+ let different_timezone = TimestampNanosecondArray::from(vec![Some(1), Some(2)])
+ .with_timezone("Europe/Paris");
+ assert_contains(&filter, &different_timezone, vec![Some(true), Some(false)])?;
+
+ let different_unit = TimestampMillisecondArray::from(vec![Some(1)]);
+ let err = filter
+ .contains(&different_unit, false)
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("Timestamp(ns"), "{err}");
+ assert!(err.contains("Timestamp(ms"), "{err}");
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_decimal128_handles_precision_scale_and_nulls() -> Result<()> {
+ let haystack: ArrayRef = Arc::new(
+ Decimal128Array::from(vec![Some(12345), None, Some(-700), Some(42)])
+ .with_precision_and_scale(10, 2)?,
+ );
+ let filter = BranchlessFilter::<Decimal128Type>::try_new(&haystack)?;
+ let needles =
+ Decimal128Array::from(vec![Some(12345), Some(999), None, Some(-700)])
+ .with_precision_and_scale(10, 2)?;
+
+ assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])?;
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), None, None, Some(false)])
+ );
+
+ let compatible_metadata =
+ Decimal128Array::from(vec![Some(12345)]).with_precision_and_scale(11, 3)?;
+ assert_contains(&filter, &compatible_metadata, vec![Some(true)])?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_filter_interval_month_day_nano_handles_nulls() -> Result<()> {
+ let one_month = IntervalMonthDayNanoType::make_value(1, 0, 0);
+ let two_days = IntervalMonthDayNanoType::make_value(0, 2, 0);
+ let three_nanos = IntervalMonthDayNanoType::make_value(0, 0, 3);
+ let absent = IntervalMonthDayNanoType::make_value(4, 5, 6);
+ let haystack: ArrayRef = Arc::new(IntervalMonthDayNanoArray::from(vec![
+ Some(one_month),
+ None,
+ Some(two_days),
+ Some(three_nanos),
+ ]));
+ let filter = BranchlessFilter::<IntervalMonthDayNanoType>::try_new(&haystack)?;
+ let needles = IntervalMonthDayNanoArray::from(vec![
+ Some(one_month),
+ Some(absent),
+ None,
+ Some(three_nanos),
+ ]);
+
+ assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])?;
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), None, None, Some(false)])
+ );
+
+ Ok(())
+ }
}
diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
index 9db90ea4fa..be4dce8dfd 100644
--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
@@ -20,7 +20,13 @@ use std::sync::Arc;
use arrow::array::ArrayRef;
use arrow::compute::cast;
use arrow::datatypes::{
- DataType, Float16Type, Int8Type, Int16Type, UInt8Type, UInt16Type,
+ DataType, Date32Type, Date64Type, Decimal128Type, DurationMicrosecondType,
+ DurationMillisecondType, DurationNanosecondType, DurationSecondType, Float16Type,
+ Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
+ IntervalMonthDayNanoType, IntervalUnit, Time32MillisecondType, Time32SecondType,
+ Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType,
+ TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type,
+ UInt16Type, UInt32Type, UInt64Type,
};
use datafusion_common::Result;
@@ -28,24 +34,88 @@ use super::array_static_filter::ArrayStaticFilter;
use super::primitive_filter::*;
use super::static_filter::StaticFilter;
-pub(super) fn instantiate_static_filter(
- in_array: ArrayRef,
-) -> Result<Arc<dyn StaticFilter + Send + Sync>> {
+type StaticFilterRef = Arc<dyn StaticFilter + Send + Sync>;
+
+pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result<StaticFilterRef> {
+ let in_array = flatten_dictionary_haystack(in_array)?;
+
+ if let Some(filter) = instantiate_branchless_filter(&in_array)? {
+ return Ok(filter);
+ }
+
+ instantiate_standard_filter(in_array)
+}
+
+fn flatten_dictionary_haystack(in_array: ArrayRef) -> Result<ArrayRef> {
// Flatten dictionary-encoded haystacks to their value type so that
// specialized filters (e.g. Int32StaticFilter) are used instead of
// falling through to the generic ArrayStaticFilter.
- let in_array = match in_array.data_type() {
- DataType::Dictionary(_, value_type) => cast(&in_array, value_type.as_ref())?,
- _ => in_array,
- };
match in_array.data_type() {
- DataType::Int8 => Ok(Arc::new(BitmapFilter::<Int8Type>::try_new(&in_array)?)),
- DataType::UInt8 => Ok(Arc::new(BitmapFilter::<UInt8Type>::try_new(&in_array)?)),
- DataType::Int16 => Ok(Arc::new(BitmapFilter::<Int16Type>::try_new(&in_array)?)),
- DataType::UInt16 => Ok(Arc::new(BitmapFilter::<UInt16Type>::try_new(&in_array)?)),
- DataType::Float16 => {
- Ok(Arc::new(BitmapFilter::<Float16Type>::try_new(&in_array)?))
+ DataType::Dictionary(_, value_type) => Ok(cast(&in_array, value_type.as_ref())?),
+ _ => Ok(in_array),
+ }
+}
+
+fn instantiate_branchless_filter(in_array: &ArrayRef) -> Result<Option<StaticFilterRef>> {
+ let non_null_count = in_array.len() - in_array.null_count();
+
+ macro_rules! filter {
+ ($arrow_type:ty) => {
+ branchless_filter::<$arrow_type>(in_array, non_null_count)
+ };
+ }
+
+ match in_array.data_type() {
+ DataType::Int8 => filter!(Int8Type),
+ DataType::UInt8 => filter!(UInt8Type),
+ DataType::Int16 => filter!(Int16Type),
+ DataType::UInt16 => filter!(UInt16Type),
+ DataType::Float16 => filter!(Float16Type),
+ DataType::Int32 => filter!(Int32Type),
+ DataType::UInt32 => filter!(UInt32Type),
+ DataType::Float32 => filter!(Float32Type),
+ DataType::Date32 => filter!(Date32Type),
+ DataType::Time32(unit) => match unit {
+ TimeUnit::Second => filter!(Time32SecondType),
+ TimeUnit::Millisecond => filter!(Time32MillisecondType),
+ _ => Ok(None),
+ },
+ DataType::Int64 => filter!(Int64Type),
+ DataType::UInt64 => filter!(UInt64Type),
+ DataType::Float64 => filter!(Float64Type),
+ DataType::Date64 => filter!(Date64Type),
+ DataType::Time64(unit) => match unit {
+ TimeUnit::Microsecond => filter!(Time64MicrosecondType),
+ TimeUnit::Nanosecond => filter!(Time64NanosecondType),
+ _ => Ok(None),
+ },
+ DataType::Timestamp(unit, _) => match unit {
+ TimeUnit::Second => filter!(TimestampSecondType),
+ TimeUnit::Millisecond => filter!(TimestampMillisecondType),
+ TimeUnit::Microsecond => filter!(TimestampMicrosecondType),
+ TimeUnit::Nanosecond => filter!(TimestampNanosecondType),
+ },
+ DataType::Duration(unit) => match unit {
+ TimeUnit::Second => filter!(DurationSecondType),
+ TimeUnit::Millisecond => filter!(DurationMillisecondType),
+ TimeUnit::Microsecond => filter!(DurationMicrosecondType),
+ TimeUnit::Nanosecond => filter!(DurationNanosecondType),
+ },
+ DataType::Decimal128(_, _) => filter!(Decimal128Type),
+ DataType::Interval(IntervalUnit::MonthDayNano) => {
+ filter!(IntervalMonthDayNanoType)
}
+ _ => Ok(None),
+ }
+}
+
+fn instantiate_standard_filter(in_array: ArrayRef) -> Result<StaticFilterRef> {
+ match in_array.data_type() {
+ DataType::Int8 => bitmap_filter::<Int8Type>(&in_array),
+ DataType::UInt8 => bitmap_filter::<UInt8Type>(&in_array),
+ DataType::Int16 => bitmap_filter::<Int16Type>(&in_array),
+ DataType::UInt16 => bitmap_filter::<UInt16Type>(&in_array),
+ DataType::Float16 => bitmap_filter::<Float16Type>(&in_array),
DataType::Int32 => Ok(Arc::new(Int32StaticFilter::try_new(&in_array)?)),
DataType::Int64 => Ok(Arc::new(Int64StaticFilter::try_new(&in_array)?)),
DataType::UInt32 => Ok(Arc::new(UInt32StaticFilter::try_new(&in_array)?)),
@@ -54,8 +124,71 @@ pub(super) fn instantiate_static_filter(
DataType::Float32 => Ok(Arc::new(Float32StaticFilter::try_new(&in_array)?)),
DataType::Float64 => Ok(Arc::new(Float64StaticFilter::try_new(&in_array)?)),
_ => {
- /* fall through to generic implementation for unsupported types (Struct, etc.) */
+ // Fall through to generic implementation for unsupported types
+ // (Struct, etc.).
Ok(Arc::new(ArrayStaticFilter::try_new(in_array)?))
}
}
}
+
+fn bitmap_filter<T>(in_array: &ArrayRef) -> Result<StaticFilterRef>
+where
+ T: BitmapFilterType,
+{
+ Ok(Arc::new(BitmapFilter::<T>::try_new(in_array)?))
+}
+
+fn branchless_filter<T>(
+ in_array: &ArrayRef,
+ non_null_count: usize,
+) -> Result<Option<StaticFilterRef>>
+where
+ T: BranchlessFilterType,
+ BranchlessNative<T>: Copy + PartialEq + Send + Sync,
+{
+ // Larger lists use the standard filter. `try_new` checks the limit again.
+ if non_null_count > T::MAX_LIST_LEN {
+ return Ok(None);
+ }
+
+ Ok(Some(Arc::new(BranchlessFilter::<T>::try_new(in_array)?)))
+}
+
+#[cfg(test)]
+mod tests {
+ use arrow::array::UInt32Array;
+ use arrow::datatypes::UInt32Type;
+
+ use super::super::primitive_filter::BranchlessFilterType;
+ use super::*;
+
+ fn uint32_array(values: Vec<Option<u32>>) -> ArrayRef {
+ Arc::new(UInt32Array::from(values))
+ }
+
+ #[test]
+ fn branchless_routing_respects_max_list_len() -> Result<()> {
+ let max_len = <UInt32Type as BranchlessFilterType>::MAX_LIST_LEN;
+
+ let values = (0..max_len)
+ .map(|value| Some(value as u32))
+ .collect::<Vec<_>>();
+ assert!(instantiate_branchless_filter(&uint32_array(values))?.is_some());
+
+ let values = (0..=max_len)
+ .map(|value| Some(value as u32))
+ .collect::<Vec<_>>();
+ assert!(instantiate_branchless_filter(&uint32_array(values))?.is_none());
+
+ Ok(())
+ }
+
+ #[test]
+ fn branchless_routing_handles_zero_non_null_values() -> Result<()> {
+ let array = uint32_array(vec![None; 3]);
+
+ assert!(instantiate_branchless_filter(&array)?.is_some());
+
+ Ok(())
+ }
+}