Apache DataFusion SQL Query Engine
fix: preserve dictionary-value nulls in scalar regex operators (#23966)
discord9cc1326a6098d
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex 8b71b3ec40..170c1a4d02 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -3353,6 +3353,78 @@ mod tests {Ok(())}+ #[test]+ fn regex_scalar_with_dictionary_nulls() -> Result<()> {+ let dictionary_values = Arc::new(StringArray::from(vec![+ Some("abc"),+ None,+ Some("ABC"),+ Some("def"),+ ]));+ let keys = UInt32Array::from(vec![Some(0), None, Some(1), Some(2), Some(3)]);+ let dictionary =+ Arc::new(DictionaryArray::try_new(keys, dictionary_values)?) as ArrayRef;+ let utf8 = cast(&dictionary, &DataType::Utf8)?;+ let pattern = ScalarValue::Utf8(Some("^abc$".to_string()));+ let dictionary_schema = Arc::new(Schema::new(vec![Field::new(+ "a",+ dictionary.data_type().clone(),+ true,+ )]));+ let utf8_schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)]));++ let evaluate =+ |schema: &SchemaRef, array: &ArrayRef, op: Operator| -> Result<ArrayRef> {+ let expr = binary(col("a", schema)?, op, lit(pattern.clone()), schema)?;+ let batch =+ RecordBatch::try_new(Arc::clone(schema), vec![Arc::clone(array)])?;+ Ok(expr+ .evaluate(&batch)?+ .into_array(batch.num_rows())+ .expect("Failed to convert to array"))+ };++ for (op, expected) in [+ (+ Operator::RegexMatch,+ BooleanArray::from(vec![+ Some(true),+ None,+ None,+ Some(false),+ Some(false),+ ]),+ ),+ (+ Operator::RegexIMatch,+ BooleanArray::from(vec![Some(true), None, None, Some(true), Some(false)]),+ ),+ (+ Operator::RegexNotMatch,+ BooleanArray::from(vec![Some(false), None, None, Some(true), Some(true)]),+ ),+ (+ Operator::RegexNotIMatch,+ BooleanArray::from(vec![+ Some(false),+ None,+ None,+ Some(false),+ Some(true),+ ]),+ ),+ ] {+ let dictionary_result = evaluate(&dictionary_schema, &dictionary, op)?;+ let utf8_result = evaluate(&utf8_schema, &utf8, op)?;++ assert_eq!(dictionary_result.as_ref(), &expected);+ assert_eq!(&dictionary_result, &utf8_result);+ }++ Ok(())+ }+#[test]fn regex_mismatched_array_types_error() -> Result<()> {// The analyzer coerces both operands of a regex operator to a commondiff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rsindex fca824c14b..a123fba1f9 100644--- a/datafusion/physical-expr/src/expressions/binary/kernels.rs+++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs@@ -270,7 +270,8 @@ pub(crate) fn regex_match_dyn_scalar(regexp_is_match_flag_scalar!(left, right, LargeStringArray, not_match, flag)}DataType::Dictionary(_, _) => {- let values = left.as_any_dictionary().values();+ let dictionary = left.as_any_dictionary();+ let values = dictionary.values();match values.data_type() {DataType::Utf8 => regexp_is_match_flag_scalar!(values, right, StringArray, not_match, flag),@@ -280,16 +281,15 @@ pub(crate) fn regex_match_dyn_scalar("Data type {} not supported as a dictionary value type for operation 'regex_match_dyn_scalar' on string array",other),- }.map(- // downcast_dictionary_array duplicates code per possible key type, so we aim to do all prep work before- |evaluated_values| downcast_dictionary_array! {- left => {- let unpacked_dict = evaluated_values.take_iter(left.keys().iter().map(|opt| opt.map(|v| v as _))).collect::<BooleanArray>();- Arc::new(unpacked_dict) as ArrayRef- },- _ => unreachable!(),- }- )+ }+ .and_then(|evaluated_values| {+ // Expand back to rows while preserving nulls from both keys and values.+ Ok(arrow::compute::take(+ evaluated_values.as_ref(),+ dictionary.keys(),+ None,+ )?)+ })}other => internal_err!("Data type {} not supported for operation 'regex_match_dyn_scalar' on string array",