Apache DataFusion SQL Query Engine
feat: Improve InListExpr types, flatten dict haystacks and validate in try_new_from_array (#21402)
Burak Şen059929d31af6
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rsindex 44e6a99c8a..e2251d8e63 100644--- a/datafusion/physical-expr/src/expressions/in_list.rs+++ b/datafusion/physical-expr/src/expressions/in_list.rs@@ -123,6 +123,23 @@ fn try_evaluate_constant_list(}}+/// Asserts that the InList expression's data type matches a list element's+/// data type. `DataType::Null` list elements are accepted unconditionally so+/// that null literals and `NullArray` haystacks remain compatible with any+/// expression type.+fn assert_inlist_data_types_match(+ expr_data_type: &DataType,+ list_data_type: &DataType,+) -> Result<()> {+ if *list_data_type != DataType::Null {+ assert_or_internal_err!(+ DFSchema::datatype_is_logically_equal(expr_data_type, list_data_type),+ "The data type inlist should be same, the value type is {expr_data_type}, one of list expr type is {list_data_type}"+ );+ }+ Ok(())+}+impl InListExpr {/// Create a new InList expressionfn new(@@ -164,20 +181,28 @@ impl InListExpr {/// Create a new InList expression directly from an array, bypassing expression evaluation.///- /// This is more efficient than `in_list()` when you already have the list as an array,- /// as it avoids the conversion: `ArrayRef -> Vec<PhysicalExpr> -> ArrayRef -> StaticFilter`.- /// Instead it goes directly: `ArrayRef -> StaticFilter`.+ /// This is more efficient than [`InListExpr::try_new`] when you already have the list+ /// as an array, as it builds the static filter directly from the array instead of+ /// reconstructing an intermediate array from literal expressions.+ ///+ /// The `list` field is populated with literal expressions extracted from+ /// the array, and the array is used to build a static filter for+ /// efficient set membership evaluation.///- /// The `list` field will be empty when using this constructor, as the array is stored- /// directly in the static filter.+ /// The `array` may be dictionary-encoded — it will be flattened to its+ /// value type such that specialized filters are used.///- /// This does not make the expression any more performant at runtime, but it does make it slightly- /// cheaper to build.+ /// Returns an error if the expression's data type and the array's data type+ /// are not logically equal. Null arrays are always accepted.pub fn try_new_from_array(expr: Arc<dyn PhysicalExpr>,array: ArrayRef,negated: bool,+ schema: &Schema,) -> Result<Self> {+ let expr_data_type = expr.data_type(schema)?;+ assert_inlist_data_types_match(&expr_data_type, array.data_type())?;+let list = (0..array.len()).map(|i| {let scalar = ScalarValue::try_from_array(array.as_ref(), i)?;@@ -210,13 +235,7 @@ impl InListExpr {let expr_data_type = expr.data_type(schema)?;for list_expr in list.iter() {let list_expr_data_type = list_expr.data_type(schema)?;- assert_or_internal_err!(- DFSchema::datatype_is_logically_equal(- &expr_data_type,- &list_expr_data_type- ),- "The data type inlist should be same, the value type is {expr_data_type}, one of list expr type is {list_expr_data_type}"- );+ assert_inlist_data_types_match(&expr_data_type, &list_expr_data_type)?;}// Try to create a static filter if all list expressions are constants@@ -1835,6 +1854,7 @@ mod tests {Arc::clone(&col_a),array,false,+ &schema,)?) as Arc<dyn PhysicalExpr>;// Create test data: [1, 2, 3, 4, null]@@ -1964,6 +1984,7 @@ mod tests {Arc::clone(&col_a),null_array,false,+ &schema,)?) as Arc<dyn PhysicalExpr>;let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;@@ -1992,6 +2013,7 @@ mod tests {Arc::clone(&col_a),null_array,false,+ &schema,)?) as Arc<dyn PhysicalExpr>;let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;@@ -3428,8 +3450,9 @@ mod tests {let schema =Schema::new(vec![Field::new("a", needle.data_type().clone(), false)]);let col_a = col("a", &schema)?;- let expr = Arc::new(InListExpr::try_new_from_array(col_a, in_array, false)?)- as Arc<dyn PhysicalExpr>;+ let expr = Arc::new(InListExpr::try_new_from_array(+ col_a, in_array, false, &schema,+ )?) as Arc<dyn PhysicalExpr>;let batch = RecordBatch::try_new(Arc::new(schema), vec![needle])?;let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;Ok(as_boolean_array(&result).clone())@@ -3562,41 +3585,237 @@ mod tests {Ok(())}+ fn make_int32_dict_array(values: Vec<Option<i32>>) -> ArrayRef {+ let mut builder = PrimitiveDictionaryBuilder::<Int8Type, Int32Type>::new();+ for v in values {+ match v {+ Some(val) => builder.append_value(val),+ None => builder.append_null(),+ }+ }+ Arc::new(builder.finish())+ }++ fn make_f64_dict_array(values: Vec<Option<f64>>) -> ArrayRef {+ let mut builder = PrimitiveDictionaryBuilder::<Int8Type, Float64Type>::new();+ for v in values {+ match v {+ Some(val) => builder.append_value(val),+ None => builder.append_null(),+ }+ }+ Arc::new(builder.finish())+ }++ #[test]+ fn test_try_new_from_array_dict_haystack_int32() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let needle = Int32Array::from(vec![1, 2, 3, 4]);+ let batch =+ RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(needle)])?;++ let haystack = make_int32_dict_array(vec![Some(1), None, Some(3)]);++ let col_a = col("a", &schema)?;+ let expr = InListExpr::try_new_from_array(col_a, haystack, false, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(true), None, Some(true), None])+ );++ Ok(())+ }+#[test]fn test_in_list_from_array_type_mismatch_errors() -> Result<()> {- // Utf8 needle, Dict(Utf8) in_array- let err = eval_in_list_from_array(- Arc::new(StringArray::from(vec!["a", "d", "b"])),- wrap_in_dict(Arc::new(StringArray::from(vec!["a", "b", "c"]))),- )- .unwrap_err()- .to_string();- assert!(- err.contains("Can't compare arrays of different types"),- "{err}"+ // Utf8 needle, Dict(Utf8) in_array: now works with dict haystack support+ assert_eq!(+ BooleanArray::from(vec![Some(true), Some(false), Some(true)]),+ eval_in_list_from_array(+ Arc::new(StringArray::from(vec!["a", "d", "b"])),+ wrap_in_dict(Arc::new(StringArray::from(vec!["a", "b", "c"]))),+ )?);- // Dict(Utf8) needle, Int64 in_array: specialized Int64StaticFilter- // rejects the Utf8 dictionary values at construction time+ // Dict(Utf8) needle, Int64 in_array: type validation rejects at constructionlet err = eval_in_list_from_array(wrap_in_dict(Arc::new(StringArray::from(vec!["a", "d", "b"]))),Arc::new(Int64Array::from(vec![1, 2, 3])),).unwrap_err().to_string();- assert!(err.contains("Failed to downcast"), "{err}");+ assert!(err.contains("The data type inlist should be same"), "{err}");// Dict(Int64) needle, Dict(Utf8) in_array: both Dict but different- // value types, make_comparator rejects the comparison+ // value types, type validation rejects at constructionlet err = eval_in_list_from_array(wrap_in_dict(Arc::new(Int64Array::from(vec![1, 4, 2]))),wrap_in_dict(Arc::new(StringArray::from(vec!["a", "b", "c"]))),).unwrap_err().to_string();- assert!(- err.contains("Can't compare arrays of different types"),- "{err}"+ assert!(err.contains("The data type inlist should be same"), "{err}");++ Ok(())+ }++ #[test]+ fn test_try_new_from_array_dict_haystack_negated() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let needle = Int32Array::from(vec![1, 2, 3, 4]);+ let batch =+ RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(needle)])?;++ let haystack = make_int32_dict_array(vec![Some(1), None, Some(3)]);++ let col_a = col("a", &schema)?;+ let expr = InListExpr::try_new_from_array(col_a, haystack, true, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(false), None, Some(false), None])+ );++ Ok(())+ }++ #[test]+ fn test_try_new_from_array_dict_haystack_utf8() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]);+ let needle = StringArray::from(vec!["a", "b", "c"]);+ let batch =+ RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(needle)])?;++ let dict_builder = StringDictionaryBuilder::<Int8Type>::new();+ let mut builder = dict_builder;+ builder.append_value("a");+ builder.append_value("c");+ let haystack: ArrayRef = Arc::new(builder.finish());++ let col_a = col("a", &schema)?;+ let expr = InListExpr::try_new_from_array(col_a, haystack, false, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(true), Some(false), Some(true)])+ );++ Ok(())+ }++ #[test]+ fn test_try_new_from_array_dict_needle_and_plain_haystack() -> Result<()> {+ let schema = Schema::new(vec![Field::new(+ "a",+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int32)),+ false,+ )]);++ let needle = make_int32_dict_array(vec![Some(1), Some(2), Some(3), Some(4)]);+ let batch =+ RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::clone(&needle)])?;++ let haystack: ArrayRef = Arc::new(Int32Array::from(vec![1, 3]));+ let col_a = col("a", &schema)?;+ let expr = InListExpr::try_new_from_array(col_a, haystack, false, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(true), Some(false), Some(true), Some(false)])+ );++ Ok(())+ }++ #[test]+ fn test_try_new_from_array_dict_haystack_float64() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);+ let needle = Float64Array::from(vec![1.0, 2.0, 3.0]);+ let batch =+ RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(needle)])?;++ let haystack = make_f64_dict_array(vec![Some(1.0), Some(3.0)]);++ let col_a = col("a", &schema)?;+ let expr = InListExpr::try_new_from_array(col_a, haystack, false, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(true), Some(false), Some(true)])+ );++ Ok(())+ }++ #[test]+ fn test_try_new_from_array_type_mismatch_rejects() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let col_a = col("a", &schema)?;+ let haystack: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));++ let result = InListExpr::try_new_from_array(col_a, haystack, false, &schema);+ assert!(result.is_err());+ Ok(())+ }++ #[test]+ fn test_try_new_from_array_struct_haystack() -> Result<()> {+ let struct_fields = Fields::from(vec![+ Field::new("x", DataType::Int32, false),+ Field::new("y", DataType::Utf8, false),+ ]);+ let struct_dt = DataType::Struct(struct_fields.clone());+ let schema = Schema::new(vec![Field::new("a", struct_dt, true)]);++ // Needle: [{1,"a"}, {2,"b"}, NULL, {4,"d"}]+ let needle = Arc::new(StructArray::new(+ struct_fields.clone(),+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3, 4])),+ Arc::new(StringArray::from(vec!["a", "b", "c", "d"])),+ ],+ Some(vec![true, true, false, true].into()),+ ));+ let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![needle])?;++ // Haystack: [{1,"a"}, {4,"d"}]+ let haystack: ArrayRef = Arc::new(StructArray::new(+ struct_fields,+ vec![+ Arc::new(Int32Array::from(vec![1, 4])),+ Arc::new(StringArray::from(vec!["a", "d"])),+ ],+ None,+ ));++ let col_a = col("a", &schema)?;+ let expr = InListExpr::try_new_from_array(+ Arc::clone(&col_a),+ Arc::clone(&haystack),+ false,+ &schema,+ )?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ // {1,"a"} -> true, {2,"b"} -> false, NULL -> NULL, {4,"d"} -> true+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(true), Some(false), None, Some(true)])+ );++ // Negated path+ let expr = InListExpr::try_new_from_array(col_a, haystack, true, &schema)?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = as_boolean_array(&result);+ assert_eq!(+ result,+ &BooleanArray::from(vec![Some(false), Some(true), None, Some(false)]));Ok(())diff --git a/datafusion/physical-expr/src/expressions/in_list/static_filter.rs b/datafusion/physical-expr/src/expressions/in_list/static_filter.rsindex 47bffb85ad..218bd27950 100644--- a/datafusion/physical-expr/src/expressions/in_list/static_filter.rs+++ b/datafusion/physical-expr/src/expressions/in_list/static_filter.rs@@ -18,10 +18,20 @@use arrow::array::{Array, BooleanArray};use datafusion_common::Result;-/// Trait for InList static filters+/// Trait for InList static filters.+///+/// Static filters store a pre-computed set of values (the haystack) and check+/// whether needle values are contained in that set. The haystack is always+/// represented in its non-dictionary (value) type. Dictionary haystacks are+/// flattened via `cast()` before construction.+///+/// Dictionary-encoded needles are unwrapped inside `contains()` and+/// evaluated against the dictionary's values.pub(super) trait StaticFilter {fn null_count(&self) -> usize;- /// Checks if values in `v` are contained in the filter+ /// Checks if values in `v` (needle) are contained in this filter's+ /// haystack. `v` may be dictionary-encoded, in which case the+ /// implementation unwraps the dictionary and operates on its values.fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray>;}diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rsindex 955ab5ad29..b7ee3dd1a3 100644--- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs+++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs@@ -18,6 +18,7 @@use std::sync::Arc;use arrow::array::ArrayRef;+use arrow::compute::cast;use arrow::datatypes::DataType;use datafusion_common::Result;@@ -28,6 +29,13 @@ use super::static_filter::StaticFilter;pub(super) fn instantiate_static_filter(in_array: ArrayRef,) -> Result<Arc<dyn StaticFilter + Send + Sync>> {+ // Flatten dictionary-encoded haystacks to their value type so that+ // specialized filters (e.g. Int32StaticFilter) are used instead of+ // falling through to the generic ArrayStaticFilter.+ let in_array = match in_array.data_type() {+ DataType::Dictionary(_, value_type) => cast(&in_array, value_type.as_ref())?,+ _ => in_array,+ };match in_array.data_type() {// Integer primitive typesDataType::Int8 => Ok(Arc::new(Int8StaticFilter::try_new(&in_array)?)),diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex 23ca14f5ba..fba6b2c2db 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -120,11 +120,12 @@ fn create_membership_predicate()) as Arc<dyn PhysicalExpr>};- // Use in_list_from_array() helper to create InList with static_filter optimization (hash-based lookup)+ // Use InListExpr::try_new_from_array() to build an InList with static_filter optimization (hash-based lookup)Ok(Some(Arc::new(InListExpr::try_new_from_array(expr,in_list_array,false,+ schema,)?)))}// Use hash table lookup for large build sides