Apache DataFusion SQL Query Engine
Deduplicate InList primitive static filters (#21932)
Geoffrey Claude985f0a4ea3fe
diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rsindex 2c184b8ea0..2c084a1cb2 100644--- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs@@ -79,9 +79,17 @@ impl From<f64> for OrderedFloat64 {// Macro to generate specialized StaticFilter implementations for primitive typesmacro_rules! primitive_static_filter {($Name:ident, $ArrowType:ty) => {+ primitive_static_filter!(+ $Name,+ $ArrowType,+ <$ArrowType as ArrowPrimitiveType>::Native,+ |v| v+ );+ };+ ($Name:ident, $ArrowType:ty, $SetValueType:ty, $to_set_value:expr) => {pub(super) struct $Name {null_count: usize,- values: HashSet<<$ArrowType as ArrowPrimitiveType>::Native>,+ values: HashSet<$SetValueType>,}impl $Name {@@ -94,7 +102,7 @@ macro_rules! primitive_static_filter {let null_count = in_array.null_count();for v in in_array.iter().flatten() {- values.insert(v);+ values.insert(($to_set_value)(v));}Ok(Self { null_count, values })@@ -146,11 +154,11 @@ macro_rules! primitive_static_filter {// This ignores nulls - we handle them separatelylet contains_buffer = if negated {BooleanBuffer::collect_bool(needle_values.len(), |i| {- !self.values.contains(&needle_values[i])+ !self.values.contains(&($to_set_value)(needle_values[i]))})} else {BooleanBuffer::collect_bool(needle_values.len(), |i| {- self.values.contains(&needle_values[i])+ self.values.contains(&($to_set_value)(needle_values[i]))})};@@ -216,126 +224,7 @@ primitive_static_filter!(UInt64StaticFilter, UInt64Type);// Floats require a wrapper type (OrderedFloat*) to implement Hash/Eq due to NaN semanticsmacro_rules! float_static_filter {($Name:ident, $ArrowType:ty, $OrderedType:ty) => {- pub(super) struct $Name {- null_count: usize,- values: HashSet<$OrderedType>,- }-- impl $Name {- pub(super) fn try_new(in_array: &ArrayRef) -> Result<Self> {- let in_array = in_array- .as_primitive_opt::<$ArrowType>()- .ok_or_else(|| exec_datafusion_err!("Failed to downcast an array to a '{}' array", stringify!($ArrowType)))?;-- let mut values = HashSet::with_capacity(in_array.len());- let null_count = in_array.null_count();-- for v in in_array.iter().flatten() {- values.insert(<$OrderedType>::from(v));- }-- Ok(Self { null_count, values })- }- }-- impl StaticFilter for $Name {- fn null_count(&self) -> usize {- self.null_count- }-- fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> {- // Handle dictionary arrays by recursing on the values- downcast_dictionary_array! {- v => {- let values_contains = self.contains(v.values().as_ref(), negated)?;- let result = take(&values_contains, v.keys(), None)?;- return Ok(downcast_array(result.as_ref()))- }- _ => {}- }-- let v = v- .as_primitive_opt::<$ArrowType>()- .ok_or_else(|| exec_datafusion_err!("Failed to downcast an array to a '{}' array", stringify!($ArrowType)))?;-- let haystack_has_nulls = self.null_count > 0;- let needle_values = v.values();- let needle_nulls = v.nulls();- let needle_has_nulls = v.null_count() > 0;-- // Truth table for `value [NOT] IN (set)` with SQL three-valued logic:- // ("-" means the value doesn't affect the result)- //- // | needle_null | haystack_null | negated | in set? | result |- // |-------------|---------------|---------|---------|--------|- // | true | - | false | - | null |- // | true | - | true | - | null |- // | false | true | false | yes | true |- // | false | true | false | no | null |- // | false | true | true | yes | false |- // | false | true | true | no | null |- // | false | false | false | yes | true |- // | false | false | false | no | false |- // | false | false | true | yes | false |- // | false | false | true | no | true |-- // Compute the "contains" result using collect_bool (fast batched approach)- // This ignores nulls - we handle them separately- let contains_buffer = if negated {- BooleanBuffer::collect_bool(needle_values.len(), |i| {- !self.values.contains(&<$OrderedType>::from(needle_values[i]))- })- } else {- BooleanBuffer::collect_bool(needle_values.len(), |i| {- self.values.contains(&<$OrderedType>::from(needle_values[i]))- })- };-- // Compute the null mask- // Output is null when:- // 1. needle value is null, OR- // 2. needle value is not in set AND haystack has nulls- let result_nulls = match (needle_has_nulls, haystack_has_nulls) {- (false, false) => {- // No nulls anywhere- None- }- (true, false) => {- // Only needle has nulls - just use needle's null mask- needle_nulls.cloned()- }- (false, true) => {- // Only haystack has nulls - result is null when value not in set- // Valid (not null) when original "in set" is true- // For NOT IN: contains_buffer = !original, so validity = !contains_buffer- let validity = if negated {- !&contains_buffer- } else {- contains_buffer.clone()- };- Some(NullBuffer::new(validity))- }- (true, true) => {- // Both have nulls - combine needle nulls with haystack-induced nulls- let needle_validity = needle_nulls.map(|n| n.inner().clone())- .unwrap_or_else(|| BooleanBuffer::new_set(needle_values.len()));-- // Valid when original "in set" is true (see above)- let haystack_validity = if negated {- !&contains_buffer- } else {- contains_buffer.clone()- };-- // Combined validity: valid only where both are valid- let combined_validity = &needle_validity & &haystack_validity;- Some(NullBuffer::new(combined_validity))- }- };-- Ok(BooleanArray::new(contains_buffer, result_nulls))- }- }+ primitive_static_filter!($Name, $ArrowType, $OrderedType, <$OrderedType>::from);};}