Skip to content

Apache DataFusion SQL Query Engine

IN LIST: Handle dictionary inputs once (#24662)

Geoffrey Claude1bf24c9fed3e
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs
index 00a7c476f5..24ca0ae176 100644
--- a/datafusion/physical-expr/benches/in_list_strategy.rs
+++ b/datafusion/physical-expr/benches/in_list_strategy.rs
@@ -45,6 +45,7 @@
//! | Utf8View length-12 cases | Utf8View | 12-byte strings | 16, 64 |
//! | Utf8View long-string cases | Utf8View | 24-byte strings | 4, 16, 64, 256 |
//! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 |
+//! | Dictionary dispatch cases | Int32, Dictionary<Int32> | per-batch dispatch overhead | 4, 64 |
//! | Fixed-size binary direct-comparison case | FixedSizeBinary(1) | direct-comparison cutoff | 16 |
//! | Fixed-size binary direct-comparison case | FixedSizeBinary(16) | direct-comparison cutoff | 4 |
//! | Fixed-size binary bitmap case | FixedSizeBinary(2) | bitmap lookup | 64 |
@@ -871,6 +872,60 @@ fn bench_dictionary(c: &mut Criterion) {
bench_dict_string(c, "utf8_short/dict=500/list=20", 500, 20, 10);
}
+/// Measures the fixed per-batch cost of handling dictionary and plain inputs.
+/// Small batches make dispatch overhead visible, while the standard 8,192-row
+/// batch shows whether it matters in the usual vectorized case.
+fn bench_dictionary_dispatch(c: &mut Criterion) {
+ for list_size in [4_usize, 64] {
+ let strategy = if list_size == 4 {
+ "branchless"
+ } else {
+ "hash_set"
+ };
+ let haystack = (0..list_size as i32).collect::<Vec<_>>();
+
+ for batch_size in [1, 8, 64, ARRAY_SIZE] {
+ let keys = (0..batch_size)
+ .map(|index| (index % 8) as i32)
+ .collect::<Vec<_>>();
+ let values = keys.iter().map(|key| key * 2).collect::<Vec<_>>();
+
+ for dictionary in [false, true] {
+ let array: ArrayRef = if dictionary {
+ Arc::new(
+ DictionaryArray::<Int32Type>::try_new(
+ Int32Array::from(keys.clone()),
+ Arc::new(Int32Array::from_iter_values((0..8).map(|v| v * 2))),
+ )
+ .unwrap(),
+ )
+ } else {
+ Arc::new(Int32Array::from(values.clone()))
+ };
+
+ let schema =
+ Schema::new(vec![Field::new("a", array.data_type().clone(), false)]);
+ let exprs = haystack.iter().map(|value| lit(*value)).collect();
+ let expr =
+ in_list(col("a", &schema).unwrap(), exprs, &false, &schema).unwrap();
+ let batch = RecordBatch::try_new(Arc::new(schema), vec![array]).unwrap();
+ let encoding = if dictionary { "dictionary" } else { "plain" };
+
+ c.bench_with_input(
+ BenchmarkId::new(
+ "dictionary_dispatch",
+ format!(
+ "i32/{strategy}/{encoding}/list={list_size}/batch={batch_size}"
+ ),
+ ),
+ &batch,
+ |b, batch| b.iter(|| expr.evaluate(batch).unwrap()),
+ );
+ }
+ }
+ }
+}
+
// =============================================================================
// NULL HANDLING BENCHMARKS
// =============================================================================
@@ -1204,7 +1259,7 @@ fn bench_fixed_size_binary(c: &mut Criterion) {
criterion_group! {
name = benches;
config = Criterion::default();
- targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary
+ targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_dictionary_dispatch, bench_nulls, bench_fixed_size_binary
}
criterion_main!(benches);
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs
index 154decfd8b..3c260190b8 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -38,6 +38,7 @@ use datafusion_expr::{ColumnarValue, expr_vec_fmt};
mod array_static_filter;
mod branchless_filter;
+mod dictionary_filter;
mod fixed_size_binary_filter;
mod primitive_filter;
mod result;
@@ -216,7 +217,7 @@ impl InListExpr {
expr,
list,
negated,
- Some(instantiate_static_filter(array)?),
+ Some(instantiate_static_filter(array, &expr_data_type)?),
))
}
@@ -243,7 +244,7 @@ impl InListExpr {
// Try to create a static filter if all list expressions are constants
let static_filter = match try_evaluate_constant_list(&list, schema)? {
- Some(in_array) => Some(instantiate_static_filter(in_array)?),
+ Some(in_array) => Some(instantiate_static_filter(in_array, &expr_data_type)?),
None => None, // Non-constant expressions, fall back to dynamic evaluation
};
@@ -1265,6 +1266,31 @@ mod tests {
Ok(())
}
+ #[test]
+ fn in_list_nested_dictionary_scalar() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let batch = RecordBatch::try_new(
+ Arc::new(schema.clone()),
+ vec![Arc::new(Int32Array::from(vec![0, 0, 0]))],
+ )?;
+ let needle = lit(ScalarValue::Dictionary(
+ Box::new(DataType::Int16),
+ Box::new(ScalarValue::Dictionary(
+ Box::new(DataType::Int8),
+ Box::new(ScalarValue::Int32(Some(2))),
+ )),
+ ));
+
+ let expr = in_list(needle, vec![lit(1_i32), lit(2_i32)], &false, &schema)?;
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ assert_eq!(
+ as_boolean_array(&result),
+ &BooleanArray::from(vec![true, true, true])
+ );
+
+ Ok(())
+ }
+
#[test]
fn in_list_utf8_with_dict_types() -> Result<()> {
fn dict_lit(key_type: DataType, value: &str) -> Arc<dyn PhysicalExpr> {
diff --git a/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs b/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs
index 75e92dbcc5..c2d3b47282 100644
--- a/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs
@@ -15,12 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{
- Array, ArrayRef, BooleanArray, downcast_array, downcast_dictionary_array,
- make_comparator,
-};
+use arrow::array::{Array, ArrayRef, BooleanArray, make_comparator};
use arrow::buffer::{BooleanBuffer, NullBuffer};
-use arrow::compute::{SortOptions, take};
+use arrow::compute::SortOptions;
use arrow::datatypes::DataType;
use arrow::util::bit_iterator::BitIndexIterator;
use datafusion_common::Result;
@@ -141,22 +138,6 @@ impl StaticFilter for ArrayStaticFilter {
));
}
- // Unwrap dictionary-encoded needles when the value type matches
- // in_array, evaluating against the dictionary values and mapping
- // back via keys.
- downcast_dictionary_array! {
- v => {
- // Only unwrap when the haystack (in_array) type matches
- // the dictionary value type
- if v.values().data_type() == self.in_array.data_type() {
- let values_contains = self.contains(v.values().as_ref(), negated)?;
- let result = take(&values_contains, v.keys(), None)?;
- return Ok(downcast_array(result.as_ref()));
- }
- }
- _ => {}
- }
-
self.find_needles_in_haystack(v, negated)
}
}
diff --git a/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs b/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs
index 8539dc7956..b4d4c10fd7 100644
--- a/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs
@@ -77,7 +77,7 @@ use arrow::util::bit_iterator::BitIndexIterator;
use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};
use super::result::build_result_from_contains;
-use super::static_filter::{StaticFilter, handle_dictionary};
+use super::static_filter::StaticFilter;
pub(super) type BranchlessNative<T> =
<<T as BranchlessFilterType>::CompareType as ArrowPrimitiveType>::Native;
@@ -256,8 +256,6 @@ where
}
fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {
- handle_dictionary!(self, v, negated);
-
// Arrow compatibility ignores timestamp timezone and decimal precision/scale
// while still requiring the same primitive representation.
if !PrimitiveArray::<T>::is_compatible(v.data_type()) {
diff --git a/datafusion/physical-expr/src/expressions/in_list/dictionary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/dictionary_filter.rs
new file mode 100644
index 0000000000..ff9a9cc699
--- /dev/null
+++ b/datafusion/physical-expr/src/expressions/in_list/dictionary_filter.rs
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{Array, AsArray, BooleanArray};
+use arrow::compute::take;
+use datafusion_common::Result;
+
+use super::static_filter::{StaticFilter, StaticFilterRef};
+
+/// Adds dictionary-encoded needle support to a filter that expects plain arrays.
+/// This wrapper is only used when the input expression returns dictionaries.
+pub(super) struct DictionaryFilter {
+ inner: StaticFilterRef,
+}
+
+impl DictionaryFilter {
+ pub(super) fn new(inner: StaticFilterRef) -> Self {
+ Self { inner }
+ }
+}
+
+impl StaticFilter for DictionaryFilter {
+ fn null_count(&self) -> usize {
+ self.inner.null_count()
+ }
+
+ fn contains(&self, needles: &dyn Array, negated: bool) -> Result<BooleanArray> {
+ let Some(dictionary) = needles.as_any_dictionary_opt() else {
+ return self.inner.contains(needles, negated);
+ };
+ let values_contains = self.contains(dictionary.values().as_ref(), negated)?;
+ let result = take(&values_contains, dictionary.keys(), None)?;
+ Ok(result.as_boolean().clone())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use arrow::array::{
+ ArrayRef, Decimal128Array, DictionaryArray, Int8Array, Int16Array, Int32Array,
+ Int64Array, TimestampNanosecondArray, UInt8Array, UInt16Array, UInt32Array,
+ UInt64Array,
+ };
+
+ use super::super::strategy::instantiate_static_filter;
+ use super::*;
+
+ #[test]
+ fn dictionary_needles_support_all_key_types() -> Result<()> {
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
+ let expected = BooleanArray::from(vec![Some(true), Some(false), None]);
+
+ macro_rules! check_keys {
+ ($keys:expr) => {{
+ let needles = DictionaryArray::try_new($keys, Arc::clone(&values))?;
+ let filter = instantiate_static_filter(
+ Arc::new(Int32Array::from(vec![1, 3])),
+ needles.data_type(),
+ )?;
+ assert_eq!(filter.contains(&needles, false)?, expected);
+ }};
+ }
+
+ check_keys!(Int8Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(Int16Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(Int32Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(Int64Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(UInt8Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(UInt16Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(UInt32Array::from(vec![Some(0), Some(1), None]));
+ check_keys!(UInt64Array::from(vec![Some(0), Some(1), None]));
+
+ Ok(())
+ }
+
+ #[test]
+ fn nested_dictionary_needles_preserve_nulls() -> Result<()> {
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));
+ let inner: ArrayRef = Arc::new(DictionaryArray::try_new(
+ Int8Array::from(vec![0, 1, 2]),
+ values,
+ )?);
+ let needles = DictionaryArray::try_new(
+ Int16Array::from(vec![Some(0), Some(1), Some(2), None]),
+ inner,
+ )?;
+ let filter = instantiate_static_filter(
+ Arc::new(Int32Array::from(vec![1, 3])),
+ needles.data_type(),
+ )?;
+
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), Some(false), None, None])
+ );
+ assert_eq!(
+ filter.contains(&needles, true)?,
+ BooleanArray::from(vec![Some(false), Some(true), None, None])
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn dictionary_timestamp_needles_keep_timezone_compatibility() -> Result<()> {
+ let timestamps: ArrayRef =
+ Arc::new(TimestampNanosecondArray::from(vec![1, 3]).with_timezone("UTC"));
+ let values: ArrayRef = Arc::new(
+ TimestampNanosecondArray::from(vec![1, 2]).with_timezone("Europe/Paris"),
+ );
+ let needles = DictionaryArray::try_new(Int8Array::from(vec![0, 1]), values)?;
+ let filter = instantiate_static_filter(timestamps, needles.data_type())?;
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![true, false])
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn dictionary_decimal_needles_keep_precision_scale_compatibility() -> Result<()> {
+ // Five list values use the hash-set filter.
+ let decimals: ArrayRef = Arc::new(
+ Decimal128Array::from(vec![1, 3, 5, 7, 9]).with_precision_and_scale(10, 2)?,
+ );
+ let values: ArrayRef =
+ Arc::new(Decimal128Array::from(vec![1, 2]).with_precision_and_scale(11, 3)?);
+ let needles = DictionaryArray::try_new(Int8Array::from(vec![0, 1]), values)?;
+ let filter = instantiate_static_filter(decimals, needles.data_type())?;
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![true, false])
+ );
+
+ Ok(())
+ }
+}
diff --git a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs
index 421c02ea1f..ddad5a7099 100644
--- a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs
@@ -53,7 +53,7 @@ use arrow::datatypes::{
use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};
use super::primitive_filter::instantiate_primitive_filter;
-use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary};
+use super::static_filter::{StaticFilter, StaticFilterRef};
/// Reinterpret fixed-size binary values as same-width primitive values.
///
@@ -99,8 +99,6 @@ where
}
fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {
- handle_dictionary!(self, v, negated);
-
if v.data_type() != &self.data_type {
return Err(exec_datafusion_err!(
"FixedSizeBinary filter: expected {} array, got {}",
@@ -165,6 +163,7 @@ mod tests {
use arrow::buffer::{Buffer, MutableBuffer, NullBuffer};
use arrow::datatypes::Int8Type;
+ use super::super::dictionary_filter::DictionaryFilter;
use super::*;
fn value(width: i32, index: usize, miss: bool) -> Vec<u8> {
@@ -268,7 +267,7 @@ mod tests {
#[test]
fn handles_dictionary_needles() -> Result<()> {
- let filter = make_filter(4, &[Some(value(4, 7, false))])?;
+ let filter = DictionaryFilter::new(make_filter(4, &[Some(value(4, 7, false))])?);
let dictionary_values: ArrayRef = Arc::new(array(
4,
&[Some(value(4, 7, false)), Some(value(4, 8, false))],
diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
index bfafae31ed..ceb9bd525b 100644
--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs
@@ -19,17 +19,18 @@
//!
//! This module provides membership tests for Arrow primitive types.
+use std::hash::{Hash, Hasher};
+use std::marker::PhantomData;
+use std::sync::Arc;
+
use arrow::array::{Array, ArrayRef, AsArray, BooleanArray};
use arrow::datatypes::*;
use arrow::util::bit_iterator::BitIndexIterator;
use datafusion_common::{HashSet, Result, exec_datafusion_err};
-use std::hash::{Hash, Hasher};
-use std::marker::PhantomData;
-use std::sync::Arc;
use super::branchless_filter::{BranchlessFilter, BranchlessFilterType};
use super::result::build_in_list_result;
-use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary};
+use super::static_filter::{StaticFilter, StaticFilterRef};
/// Selects an optimized filter for a primitive representation.
///
@@ -311,7 +312,6 @@ where
}
fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {
- handle_dictionary!(self, v, negated);
let v = v.as_primitive_opt::<T>().ok_or_else(|| {
exec_datafusion_err!("BitmapFilter: expected {} array", T::DATA_TYPE)
})?;
@@ -432,8 +432,6 @@ where
}
fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {
- handle_dictionary!(self, v, negated);
-
let v = v.as_primitive_opt::<T>().ok_or_else(|| {
exec_datafusion_err!(
"PrimitiveHashSetFilter: expected {} array",
@@ -465,6 +463,8 @@ mod tests {
};
use half::f16;
+ use super::super::dictionary_filter::DictionaryFilter;
+
fn uint32_array(values: Vec<Option<u32>>) -> ArrayRef {
Arc::new(UInt32Array::from(values))
}
@@ -553,13 +553,19 @@ mod tests {
#[test]
fn bitmap_filter_u8_handles_dictionary_needles() -> Result<()> {
let haystack: ArrayRef = Arc::new(UInt8Array::from(vec![Some(1), None, Some(3)]));
- let filter = BitmapFilter::<UInt8Type>::try_new(&haystack)?;
+ let inner: StaticFilterRef =
+ Arc::new(BitmapFilter::<UInt8Type>::try_new(&haystack)?);
+ let filter = DictionaryFilter::new(inner);
let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);
let values = Arc::new(UInt8Array::from(vec![Some(1), Some(2), Some(3)]));
let needles = DictionaryArray::try_new(keys, values)?;
- assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), None, None, Some(true)])
+ );
+ Ok(())
}
#[test]
diff --git a/datafusion/physical-expr/src/expressions/in_list/static_filter.rs b/datafusion/physical-expr/src/expressions/in_list/static_filter.rs
index e74ea9131e..255d677c94 100644
--- a/datafusion/physical-expr/src/expressions/in_list/static_filter.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/static_filter.rs
@@ -20,39 +20,19 @@ use std::sync::Arc;
use arrow::array::{Array, BooleanArray};
use datafusion_common::Result;
-pub(super) type StaticFilterRef = Arc<dyn StaticFilter + Send + Sync>;
+pub(super) type StaticFilterRef = Arc<dyn StaticFilter>;
/// Trait for InList static filters.
///
/// Static filters store a pre-computed set of values (the haystack) and check
/// whether needle values are contained in that set. The haystack is always
/// represented in its non-dictionary (value) type. Dictionary haystacks are
-/// flattened via `cast()` before construction.
+/// flattened before construction.
///
-/// Dictionary-encoded needles are unwrapped inside `contains()` and
-/// evaluated against the dictionary's values.
-pub(super) trait StaticFilter {
+/// Dictionary-encoded needles are unwrapped before the concrete filter is called.
+pub(super) trait StaticFilter: Send + Sync {
fn null_count(&self) -> usize;
- /// Checks if values in `v` (needle) are contained in this filter's
- /// haystack. `v` may be dictionary-encoded, in which case the
- /// implementation unwraps the dictionary and operates on its values.
- fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray>;
+ /// Checks if non-dictionary needles are contained in this filter's haystack.
+ fn contains(&self, needles: &dyn Array, negated: bool) -> Result<BooleanArray>;
}
-
-/// Evaluate dictionary-encoded needles by applying a filter to dictionary
-/// values and remapping the result through the keys.
-macro_rules! handle_dictionary {
- ($self:ident, $v:ident, $negated:ident) => {
- arrow::array::downcast_dictionary_array! {
- $v => {
- let values_contains = $self.contains($v.values().as_ref(), $negated)?;
- let result = arrow::compute::take(&values_contains, $v.keys(), None)?;
- return Ok(arrow::array::downcast_array(result.as_ref()))
- }
- _ => {}
- }
- };
-}
-
-pub(super) use handle_dictionary;
diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
index 98bd698507..093752cce0 100644
--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs
+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs
@@ -17,36 +17,96 @@
use std::sync::Arc;
-use arrow::array::ArrayRef;
-use arrow::compute::cast;
+use arrow::array::{ArrayRef, AsArray};
+use arrow::compute::take;
use arrow::datatypes::DataType;
use datafusion_common::Result;
use super::array_static_filter::ArrayStaticFilter;
+use super::dictionary_filter::DictionaryFilter;
use super::fixed_size_binary_filter::instantiate_fixed_size_binary_filter;
use super::primitive_filter::instantiate_primitive_filter;
use super::static_filter::StaticFilterRef;
-pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result<StaticFilterRef> {
+pub(super) fn instantiate_static_filter(
+ in_array: ArrayRef,
+ needle_data_type: &DataType,
+) -> Result<StaticFilterRef> {
let in_array = flatten_dictionary_haystack(in_array)?;
- if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? {
- return Ok(filter);
+ let filter = if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? {
+ filter
+ } else if let Some(filter) = instantiate_primitive_filter(&in_array)? {
+ filter
+ } else {
+ Arc::new(ArrayStaticFilter::try_new(in_array)?)
+ };
+
+ // Plain inputs can call the concrete filter directly. Dictionary inputs
+ // share one adapter across all concrete filter types.
+ if matches!(needle_data_type, DataType::Dictionary(_, _)) {
+ Ok(Arc::new(DictionaryFilter::new(filter)))
+ } else {
+ Ok(filter)
}
+}
- if let Some(filter) = instantiate_primitive_filter(&in_array)? {
- return Ok(filter);
+fn flatten_dictionary_haystack(mut in_array: ArrayRef) -> Result<ArrayRef> {
+ // Flatten every dictionary layer so the final value type can use a
+ // specialized filter.
+ while let Some(dictionary) = in_array.as_any_dictionary_opt() {
+ in_array = take(dictionary.values().as_ref(), dictionary.keys(), None)?;
}
- Ok(Arc::new(ArrayStaticFilter::try_new(in_array)?))
+ Ok(in_array)
}
-fn flatten_dictionary_haystack(in_array: ArrayRef) -> Result<ArrayRef> {
- // Flatten dictionary-encoded haystacks to their value type so that
- // specialized primitive filters are used instead of falling through to the
- // generic ArrayStaticFilter.
- match in_array.data_type() {
- DataType::Dictionary(_, value_type) => Ok(cast(&in_array, value_type.as_ref())?),
- _ => Ok(in_array),
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use arrow::array::{
+ BooleanArray, DictionaryArray, Int8Array, Int16Array, Int32Array,
+ };
+
+ use super::*;
+
+ fn nested_dictionary(keys: Int16Array) -> Result<ArrayRef> {
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
+ // This dictionary represents [1, 3, NULL].
+ let inner: ArrayRef = Arc::new(DictionaryArray::try_new(
+ Int8Array::from(vec![0, 2, 1]),
+ values,
+ )?);
+ Ok(Arc::new(DictionaryArray::try_new(keys, inner)?))
+ }
+
+ #[test]
+ fn nested_dictionary_haystacks_only_include_referenced_values() -> Result<()> {
+ let needles = Int32Array::from(vec![1, 2, 3]);
+
+ // The null in the inner dictionary is not referenced by the outer one.
+ let filter = instantiate_static_filter(
+ nested_dictionary(Int16Array::from(vec![0, 1]))?,
+ &DataType::Int32,
+ )?;
+ assert_eq!(filter.null_count(), 0);
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![true, false, true])
+ );
+
+ // Referencing that same value gives the list normal SQL null semantics.
+ let filter = instantiate_static_filter(
+ nested_dictionary(Int16Array::from(vec![0, 2]))?,
+ &DataType::Int32,
+ )?;
+ assert_eq!(filter.null_count(), 1);
+ assert_eq!(
+ filter.contains(&needles, false)?,
+ BooleanArray::from(vec![Some(true), None, None])
+ );
+
+ Ok(())
}
}