Apache DataFusion SQL Query Engine
IN LIST: Handle dictionary inputs once (#24662)
Geoffrey Claude1bf24c9fed3e
diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rsindex 00a7c476f5..24ca0ae176 100644--- a/datafusion/physical-expr/benches/in_list_strategy.rs+++ b/datafusion/physical-expr/benches/in_list_strategy.rs@@ -45,6 +45,7 @@//! | Utf8View length-12 cases | Utf8View | 12-byte strings | 16, 64 |//! | Utf8View long-string cases | Utf8View | 24-byte strings | 4, 16, 64, 256 |//! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 |+//! | Dictionary dispatch cases | Int32, Dictionary<Int32> | per-batch dispatch overhead | 4, 64 |//! | Fixed-size binary direct-comparison case | FixedSizeBinary(1) | direct-comparison cutoff | 16 |//! | Fixed-size binary direct-comparison case | FixedSizeBinary(16) | direct-comparison cutoff | 4 |//! | Fixed-size binary bitmap case | FixedSizeBinary(2) | bitmap lookup | 64 |@@ -871,6 +872,60 @@ fn bench_dictionary(c: &mut Criterion) {bench_dict_string(c, "utf8_short/dict=500/list=20", 500, 20, 10);}+/// Measures the fixed per-batch cost of handling dictionary and plain inputs.+/// Small batches make dispatch overhead visible, while the standard 8,192-row+/// batch shows whether it matters in the usual vectorized case.+fn bench_dictionary_dispatch(c: &mut Criterion) {+ for list_size in [4_usize, 64] {+ let strategy = if list_size == 4 {+ "branchless"+ } else {+ "hash_set"+ };+ let haystack = (0..list_size as i32).collect::<Vec<_>>();++ for batch_size in [1, 8, 64, ARRAY_SIZE] {+ let keys = (0..batch_size)+ .map(|index| (index % 8) as i32)+ .collect::<Vec<_>>();+ let values = keys.iter().map(|key| key * 2).collect::<Vec<_>>();++ for dictionary in [false, true] {+ let array: ArrayRef = if dictionary {+ Arc::new(+ DictionaryArray::<Int32Type>::try_new(+ Int32Array::from(keys.clone()),+ Arc::new(Int32Array::from_iter_values((0..8).map(|v| v * 2))),+ )+ .unwrap(),+ )+ } else {+ Arc::new(Int32Array::from(values.clone()))+ };++ let schema =+ Schema::new(vec![Field::new("a", array.data_type().clone(), false)]);+ let exprs = haystack.iter().map(|value| lit(*value)).collect();+ let expr =+ in_list(col("a", &schema).unwrap(), exprs, &false, &schema).unwrap();+ let batch = RecordBatch::try_new(Arc::new(schema), vec![array]).unwrap();+ let encoding = if dictionary { "dictionary" } else { "plain" };++ c.bench_with_input(+ BenchmarkId::new(+ "dictionary_dispatch",+ format!(+ "i32/{strategy}/{encoding}/list={list_size}/batch={batch_size}"+ ),+ ),+ &batch,+ |b, batch| b.iter(|| expr.evaluate(batch).unwrap()),+ );+ }+ }+ }+}+// =============================================================================// NULL HANDLING BENCHMARKS// =============================================================================@@ -1204,7 +1259,7 @@ fn bench_fixed_size_binary(c: &mut Criterion) {criterion_group! {name = benches;config = Criterion::default();- targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary+ targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_dictionary_dispatch, bench_nulls, bench_fixed_size_binary}criterion_main!(benches);diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex 154decfd8b..3c260190b8 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -38,6 +38,7 @@ use datafusion_expr::{ColumnarValue, expr_vec_fmt};mod array_static_filter;mod branchless_filter;+mod dictionary_filter;mod fixed_size_binary_filter;mod primitive_filter;mod result;@@ -216,7 +217,7 @@ impl InListExpr {expr,list,negated,- Some(instantiate_static_filter(array)?),+ Some(instantiate_static_filter(array, &expr_data_type)?),))}@@ -243,7 +244,7 @@ impl InListExpr {// Try to create a static filter if all list expressions are constantslet static_filter = match try_evaluate_constant_list(&list, schema)? {- Some(in_array) => Some(instantiate_static_filter(in_array)?),+ Some(in_array) => Some(instantiate_static_filter(in_array, &expr_data_type)?),None => None, // Non-constant expressions, fall back to dynamic evaluation};@@ -1265,6 +1266,31 @@ mod tests {Ok(())}+ #[test]+ fn in_list_nested_dictionary_scalar() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let batch = RecordBatch::try_new(+ Arc::new(schema.clone()),+ vec![Arc::new(Int32Array::from(vec![0, 0, 0]))],+ )?;+ let needle = lit(ScalarValue::Dictionary(+ Box::new(DataType::Int16),+ Box::new(ScalarValue::Dictionary(+ Box::new(DataType::Int8),+ Box::new(ScalarValue::Int32(Some(2))),+ )),+ ));++ let expr = in_list(needle, vec![lit(1_i32), lit(2_i32)], &false, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ assert_eq!(+ as_boolean_array(&result),+ &BooleanArray::from(vec![true, true, true])+ );++ Ok(())+ }+#[test]fn in_list_utf8_with_dict_types() -> Result<()> {fn dict_lit(key_type: DataType, value: &str) -> Arc<dyn PhysicalExpr> {diff --git a/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs b/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rsindex 75e92dbcc5..c2d3b47282 100644--- a/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/array_static_filter.rs@@ -15,12 +15,9 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{- Array, ArrayRef, BooleanArray, downcast_array, downcast_dictionary_array,- make_comparator,-};+use arrow::array::{Array, ArrayRef, BooleanArray, make_comparator};use arrow::buffer::{BooleanBuffer, NullBuffer};-use arrow::compute::{SortOptions, take};+use arrow::compute::SortOptions;use arrow::datatypes::DataType;use arrow::util::bit_iterator::BitIndexIterator;use datafusion_common::Result;@@ -141,22 +138,6 @@ impl StaticFilter for ArrayStaticFilter {));}- // Unwrap dictionary-encoded needles when the value type matches- // in_array, evaluating against the dictionary values and mapping- // back via keys.- downcast_dictionary_array! {- v => {- // Only unwrap when the haystack (in_array) type matches- // the dictionary value type- if v.values().data_type() == self.in_array.data_type() {- let values_contains = self.contains(v.values().as_ref(), negated)?;- let result = take(&values_contains, v.keys(), None)?;- return Ok(downcast_array(result.as_ref()));- }- }- _ => {}- }-self.find_needles_in_haystack(v, negated)}}diff --git a/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs b/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rsindex 8539dc7956..b4d4c10fd7 100644--- a/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/branchless_filter.rs@@ -77,7 +77,7 @@ use arrow::util::bit_iterator::BitIndexIterator;use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};use super::result::build_result_from_contains;-use super::static_filter::{StaticFilter, handle_dictionary};+use super::static_filter::StaticFilter;pub(super) type BranchlessNative<T> =<<T as BranchlessFilterType>::CompareType as ArrowPrimitiveType>::Native;@@ -256,8 +256,6 @@ where}fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {- handle_dictionary!(self, v, negated);-// Arrow compatibility ignores timestamp timezone and decimal precision/scale// while still requiring the same primitive representation.if !PrimitiveArray::<T>::is_compatible(v.data_type()) {diff --git a/datafusion/physical-expr/src/expressions/in_list/dictionary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/dictionary_filter.rsnew file mode 100644index 0000000000..ff9a9cc699--- /dev/null+++ b/datafusion/physical-expr/src/expressions/in_list/dictionary_filter.rs@@ -0,0 +1,154 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::{Array, AsArray, BooleanArray};+use arrow::compute::take;+use datafusion_common::Result;++use super::static_filter::{StaticFilter, StaticFilterRef};++/// Adds dictionary-encoded needle support to a filter that expects plain arrays.+/// This wrapper is only used when the input expression returns dictionaries.+pub(super) struct DictionaryFilter {+ inner: StaticFilterRef,+}++impl DictionaryFilter {+ pub(super) fn new(inner: StaticFilterRef) -> Self {+ Self { inner }+ }+}++impl StaticFilter for DictionaryFilter {+ fn null_count(&self) -> usize {+ self.inner.null_count()+ }++ fn contains(&self, needles: &dyn Array, negated: bool) -> Result<BooleanArray> {+ let Some(dictionary) = needles.as_any_dictionary_opt() else {+ return self.inner.contains(needles, negated);+ };+ let values_contains = self.contains(dictionary.values().as_ref(), negated)?;+ let result = take(&values_contains, dictionary.keys(), None)?;+ Ok(result.as_boolean().clone())+ }+}++#[cfg(test)]+mod tests {+ use std::sync::Arc;++ use arrow::array::{+ ArrayRef, Decimal128Array, DictionaryArray, Int8Array, Int16Array, Int32Array,+ Int64Array, TimestampNanosecondArray, UInt8Array, UInt16Array, UInt32Array,+ UInt64Array,+ };++ use super::super::strategy::instantiate_static_filter;+ use super::*;++ #[test]+ fn dictionary_needles_support_all_key_types() -> Result<()> {+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));+ let expected = BooleanArray::from(vec![Some(true), Some(false), None]);++ macro_rules! check_keys {+ ($keys:expr) => {{+ let needles = DictionaryArray::try_new($keys, Arc::clone(&values))?;+ let filter = instantiate_static_filter(+ Arc::new(Int32Array::from(vec![1, 3])),+ needles.data_type(),+ )?;+ assert_eq!(filter.contains(&needles, false)?, expected);+ }};+ }++ check_keys!(Int8Array::from(vec![Some(0), Some(1), None]));+ check_keys!(Int16Array::from(vec![Some(0), Some(1), None]));+ check_keys!(Int32Array::from(vec![Some(0), Some(1), None]));+ check_keys!(Int64Array::from(vec![Some(0), Some(1), None]));+ check_keys!(UInt8Array::from(vec![Some(0), Some(1), None]));+ check_keys!(UInt16Array::from(vec![Some(0), Some(1), None]));+ check_keys!(UInt32Array::from(vec![Some(0), Some(1), None]));+ check_keys!(UInt64Array::from(vec![Some(0), Some(1), None]));++ Ok(())+ }++ #[test]+ fn nested_dictionary_needles_preserve_nulls() -> Result<()> {+ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2), None]));+ let inner: ArrayRef = Arc::new(DictionaryArray::try_new(+ Int8Array::from(vec![0, 1, 2]),+ values,+ )?);+ let needles = DictionaryArray::try_new(+ Int16Array::from(vec![Some(0), Some(1), Some(2), None]),+ inner,+ )?;+ let filter = instantiate_static_filter(+ Arc::new(Int32Array::from(vec![1, 3])),+ needles.data_type(),+ )?;++ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), Some(false), None, None])+ );+ assert_eq!(+ filter.contains(&needles, true)?,+ BooleanArray::from(vec![Some(false), Some(true), None, None])+ );++ Ok(())+ }++ #[test]+ fn dictionary_timestamp_needles_keep_timezone_compatibility() -> Result<()> {+ let timestamps: ArrayRef =+ Arc::new(TimestampNanosecondArray::from(vec![1, 3]).with_timezone("UTC"));+ let values: ArrayRef = Arc::new(+ TimestampNanosecondArray::from(vec![1, 2]).with_timezone("Europe/Paris"),+ );+ let needles = DictionaryArray::try_new(Int8Array::from(vec![0, 1]), values)?;+ let filter = instantiate_static_filter(timestamps, needles.data_type())?;+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![true, false])+ );++ Ok(())+ }++ #[test]+ fn dictionary_decimal_needles_keep_precision_scale_compatibility() -> Result<()> {+ // Five list values use the hash-set filter.+ let decimals: ArrayRef = Arc::new(+ Decimal128Array::from(vec![1, 3, 5, 7, 9]).with_precision_and_scale(10, 2)?,+ );+ let values: ArrayRef =+ Arc::new(Decimal128Array::from(vec![1, 2]).with_precision_and_scale(11, 3)?);+ let needles = DictionaryArray::try_new(Int8Array::from(vec![0, 1]), values)?;+ let filter = instantiate_static_filter(decimals, needles.data_type())?;+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![true, false])+ );++ Ok(())+ }+}diff --git a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rsindex 421c02ea1f..ddad5a7099 100644--- a/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/fixed_size_binary_filter.rs@@ -53,7 +53,7 @@ use arrow::datatypes::{use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err};use super::primitive_filter::instantiate_primitive_filter;-use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary};+use super::static_filter::{StaticFilter, StaticFilterRef};/// Reinterpret fixed-size binary values as same-width primitive values.///@@ -99,8 +99,6 @@ where}fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {- handle_dictionary!(self, v, negated);-if v.data_type() != &self.data_type {return Err(exec_datafusion_err!("FixedSizeBinary filter: expected {} array, got {}",@@ -165,6 +163,7 @@ mod tests {use arrow::buffer::{Buffer, MutableBuffer, NullBuffer};use arrow::datatypes::Int8Type;+ use super::super::dictionary_filter::DictionaryFilter;use super::*;fn value(width: i32, index: usize, miss: bool) -> Vec<u8> {@@ -268,7 +267,7 @@ mod tests {#[test]fn handles_dictionary_needles() -> Result<()> {- let filter = make_filter(4, &[Some(value(4, 7, false))])?;+ let filter = DictionaryFilter::new(make_filter(4, &[Some(value(4, 7, false))])?);let dictionary_values: ArrayRef = Arc::new(array(4,&[Some(value(4, 7, false)), Some(value(4, 8, false))],diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rsindex bfafae31ed..ceb9bd525b 100644--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs@@ -19,17 +19,18 @@//!//! This module provides membership tests for Arrow primitive types.+use std::hash::{Hash, Hasher};+use std::marker::PhantomData;+use std::sync::Arc;+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray};use arrow::datatypes::*;use arrow::util::bit_iterator::BitIndexIterator;use datafusion_common::{HashSet, Result, exec_datafusion_err};-use std::hash::{Hash, Hasher};-use std::marker::PhantomData;-use std::sync::Arc;use super::branchless_filter::{BranchlessFilter, BranchlessFilterType};use super::result::build_in_list_result;-use super::static_filter::{StaticFilter, StaticFilterRef, handle_dictionary};+use super::static_filter::{StaticFilter, StaticFilterRef};/// Selects an optimized filter for a primitive representation.///@@ -311,7 +312,6 @@ where}fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {- handle_dictionary!(self, v, negated);let v = v.as_primitive_opt::<T>().ok_or_else(|| {exec_datafusion_err!("BitmapFilter: expected {} array", T::DATA_TYPE)})?;@@ -432,8 +432,6 @@ where}fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {- handle_dictionary!(self, v, negated);-let v = v.as_primitive_opt::<T>().ok_or_else(|| {exec_datafusion_err!("PrimitiveHashSetFilter: expected {} array",@@ -465,6 +463,8 @@ mod tests {};use half::f16;+ use super::super::dictionary_filter::DictionaryFilter;+fn uint32_array(values: Vec<Option<u32>>) -> ArrayRef {Arc::new(UInt32Array::from(values))}@@ -553,13 +553,19 @@ mod tests {#[test]fn bitmap_filter_u8_handles_dictionary_needles() -> Result<()> {let haystack: ArrayRef = Arc::new(UInt8Array::from(vec![Some(1), None, Some(3)]));- let filter = BitmapFilter::<UInt8Type>::try_new(&haystack)?;+ let inner: StaticFilterRef =+ Arc::new(BitmapFilter::<UInt8Type>::try_new(&haystack)?);+ let filter = DictionaryFilter::new(inner);let keys = Int8Array::from(vec![Some(0), Some(1), None, Some(2)]);let values = Arc::new(UInt8Array::from(vec![Some(1), Some(2), Some(3)]));let needles = DictionaryArray::try_new(keys, values)?;- assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), None, None, Some(true)])+ );+ Ok(())}#[test]diff --git a/datafusion/physical-expr/src/expressions/in_list/static_filter.rs b/datafusion/physical-expr/src/expressions/in_list/static_filter.rsindex e74ea9131e..255d677c94 100644--- a/datafusion/physical-expr/src/expressions/in_list/static_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/static_filter.rs@@ -20,39 +20,19 @@ use std::sync::Arc;use arrow::array::{Array, BooleanArray};use datafusion_common::Result;-pub(super) type StaticFilterRef = Arc<dyn StaticFilter + Send + Sync>;+pub(super) type StaticFilterRef = Arc<dyn StaticFilter>;/// Trait for InList static filters.////// Static filters store a pre-computed set of values (the haystack) and check/// whether needle values are contained in that set. The haystack is always/// represented in its non-dictionary (value) type. Dictionary haystacks are-/// flattened via `cast()` before construction.+/// flattened before construction.///-/// Dictionary-encoded needles are unwrapped inside `contains()` and-/// evaluated against the dictionary's values.-pub(super) trait StaticFilter {+/// Dictionary-encoded needles are unwrapped before the concrete filter is called.+pub(super) trait StaticFilter: Send + Sync {fn null_count(&self) -> usize;- /// Checks if values in `v` (needle) are contained in this filter's- /// haystack. `v` may be dictionary-encoded, in which case the- /// implementation unwraps the dictionary and operates on its values.- fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray>;+ /// Checks if non-dictionary needles are contained in this filter's haystack.+ fn contains(&self, needles: &dyn Array, negated: bool) -> Result<BooleanArray>;}--/// Evaluate dictionary-encoded needles by applying a filter to dictionary-/// values and remapping the result through the keys.-macro_rules! handle_dictionary {- ($self:ident, $v:ident, $negated:ident) => {- arrow::array::downcast_dictionary_array! {- $v => {- let values_contains = $self.contains($v.values().as_ref(), $negated)?;- let result = arrow::compute::take(&values_contains, $v.keys(), None)?;- return Ok(arrow::array::downcast_array(result.as_ref()))- }- _ => {}- }- };-}--pub(super) use handle_dictionary;diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rsindex 98bd698507..093752cce0 100644--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs@@ -17,36 +17,96 @@use std::sync::Arc;-use arrow::array::ArrayRef;-use arrow::compute::cast;+use arrow::array::{ArrayRef, AsArray};+use arrow::compute::take;use arrow::datatypes::DataType;use datafusion_common::Result;use super::array_static_filter::ArrayStaticFilter;+use super::dictionary_filter::DictionaryFilter;use super::fixed_size_binary_filter::instantiate_fixed_size_binary_filter;use super::primitive_filter::instantiate_primitive_filter;use super::static_filter::StaticFilterRef;-pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result<StaticFilterRef> {+pub(super) fn instantiate_static_filter(+ in_array: ArrayRef,+ needle_data_type: &DataType,+) -> Result<StaticFilterRef> {let in_array = flatten_dictionary_haystack(in_array)?;- if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? {- return Ok(filter);+ let filter = if let Some(filter) = instantiate_fixed_size_binary_filter(&in_array)? {+ filter+ } else if let Some(filter) = instantiate_primitive_filter(&in_array)? {+ filter+ } else {+ Arc::new(ArrayStaticFilter::try_new(in_array)?)+ };++ // Plain inputs can call the concrete filter directly. Dictionary inputs+ // share one adapter across all concrete filter types.+ if matches!(needle_data_type, DataType::Dictionary(_, _)) {+ Ok(Arc::new(DictionaryFilter::new(filter)))+ } else {+ Ok(filter)}+}- if let Some(filter) = instantiate_primitive_filter(&in_array)? {- return Ok(filter);+fn flatten_dictionary_haystack(mut in_array: ArrayRef) -> Result<ArrayRef> {+ // Flatten every dictionary layer so the final value type can use a+ // specialized filter.+ while let Some(dictionary) = in_array.as_any_dictionary_opt() {+ in_array = take(dictionary.values().as_ref(), dictionary.keys(), None)?;}- Ok(Arc::new(ArrayStaticFilter::try_new(in_array)?))+ Ok(in_array)}-fn flatten_dictionary_haystack(in_array: ArrayRef) -> Result<ArrayRef> {- // Flatten dictionary-encoded haystacks to their value type so that- // specialized primitive filters are used instead of falling through to the- // generic ArrayStaticFilter.- match in_array.data_type() {- DataType::Dictionary(_, value_type) => Ok(cast(&in_array, value_type.as_ref())?),- _ => Ok(in_array),+#[cfg(test)]+mod tests {+ use std::sync::Arc;++ use arrow::array::{+ BooleanArray, DictionaryArray, Int8Array, Int16Array, Int32Array,+ };++ use super::*;++ fn nested_dictionary(keys: Int16Array) -> Result<ArrayRef> {+ let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));+ // This dictionary represents [1, 3, NULL].+ let inner: ArrayRef = Arc::new(DictionaryArray::try_new(+ Int8Array::from(vec![0, 2, 1]),+ values,+ )?);+ Ok(Arc::new(DictionaryArray::try_new(keys, inner)?))+ }++ #[test]+ fn nested_dictionary_haystacks_only_include_referenced_values() -> Result<()> {+ let needles = Int32Array::from(vec![1, 2, 3]);++ // The null in the inner dictionary is not referenced by the outer one.+ let filter = instantiate_static_filter(+ nested_dictionary(Int16Array::from(vec![0, 1]))?,+ &DataType::Int32,+ )?;+ assert_eq!(filter.null_count(), 0);+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![true, false, true])+ );++ // Referencing that same value gives the list normal SQL null semantics.+ let filter = instantiate_static_filter(+ nested_dictionary(Int16Array::from(vec![0, 2]))?,+ &DataType::Int32,+ )?;+ assert_eq!(filter.null_count(), 1);+ assert_eq!(+ filter.contains(&needles, false)?,+ BooleanArray::from(vec![Some(true), None, None])+ );++ Ok(())}}