Skip to content

Apache DataFusion SQL Query Engine

fix: preserve dictionary-value nulls in scalar regex operators (#23966)

discord9cc1326a6098d
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index 8b71b3ec40..170c1a4d02 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -3353,6 +3353,78 @@ mod tests {
Ok(())
}
+ #[test]
+ fn regex_scalar_with_dictionary_nulls() -> Result<()> {
+ let dictionary_values = Arc::new(StringArray::from(vec![
+ Some("abc"),
+ None,
+ Some("ABC"),
+ Some("def"),
+ ]));
+ let keys = UInt32Array::from(vec![Some(0), None, Some(1), Some(2), Some(3)]);
+ let dictionary =
+ Arc::new(DictionaryArray::try_new(keys, dictionary_values)?) as ArrayRef;
+ let utf8 = cast(&dictionary, &DataType::Utf8)?;
+ let pattern = ScalarValue::Utf8(Some("^abc$".to_string()));
+ let dictionary_schema = Arc::new(Schema::new(vec![Field::new(
+ "a",
+ dictionary.data_type().clone(),
+ true,
+ )]));
+ let utf8_schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)]));
+
+ let evaluate =
+ |schema: &SchemaRef, array: &ArrayRef, op: Operator| -> Result<ArrayRef> {
+ let expr = binary(col("a", schema)?, op, lit(pattern.clone()), schema)?;
+ let batch =
+ RecordBatch::try_new(Arc::clone(schema), vec![Arc::clone(array)])?;
+ Ok(expr
+ .evaluate(&batch)?
+ .into_array(batch.num_rows())
+ .expect("Failed to convert to array"))
+ };
+
+ for (op, expected) in [
+ (
+ Operator::RegexMatch,
+ BooleanArray::from(vec![
+ Some(true),
+ None,
+ None,
+ Some(false),
+ Some(false),
+ ]),
+ ),
+ (
+ Operator::RegexIMatch,
+ BooleanArray::from(vec![Some(true), None, None, Some(true), Some(false)]),
+ ),
+ (
+ Operator::RegexNotMatch,
+ BooleanArray::from(vec![Some(false), None, None, Some(true), Some(true)]),
+ ),
+ (
+ Operator::RegexNotIMatch,
+ BooleanArray::from(vec![
+ Some(false),
+ None,
+ None,
+ Some(false),
+ Some(true),
+ ]),
+ ),
+ ] {
+ let dictionary_result = evaluate(&dictionary_schema, &dictionary, op)?;
+ let utf8_result = evaluate(&utf8_schema, &utf8, op)?;
+
+ assert_eq!(dictionary_result.as_ref(), &expected);
+ assert_eq!(&dictionary_result, &utf8_result);
+ }
+
+ Ok(())
+ }
+
#[test]
fn regex_mismatched_array_types_error() -> Result<()> {
// The analyzer coerces both operands of a regex operator to a common
diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs
index fca824c14b..a123fba1f9 100644
--- a/datafusion/physical-expr/src/expressions/binary/kernels.rs
+++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs
@@ -270,7 +270,8 @@ pub(crate) fn regex_match_dyn_scalar(
regexp_is_match_flag_scalar!(left, right, LargeStringArray, not_match, flag)
}
DataType::Dictionary(_, _) => {
- let values = left.as_any_dictionary().values();
+ let dictionary = left.as_any_dictionary();
+ let values = dictionary.values();
match values.data_type() {
DataType::Utf8 => regexp_is_match_flag_scalar!(values, right, StringArray, not_match, flag),
@@ -280,16 +281,15 @@ pub(crate) fn regex_match_dyn_scalar(
"Data type {} not supported as a dictionary value type for operation 'regex_match_dyn_scalar' on string array",
other
),
- }.map(
- // downcast_dictionary_array duplicates code per possible key type, so we aim to do all prep work before
- |evaluated_values| downcast_dictionary_array! {
- left => {
- let unpacked_dict = evaluated_values.take_iter(left.keys().iter().map(|opt| opt.map(|v| v as _))).collect::<BooleanArray>();
- Arc::new(unpacked_dict) as ArrayRef
- },
- _ => unreachable!(),
- }
- )
+ }
+ .and_then(|evaluated_values| {
+ // Expand back to rows while preserving nulls from both keys and values.
+ Ok(arrow::compute::take(
+ evaluated_values.as_ref(),
+ dictionary.keys(),
+ None,
+ )?)
+ })
}
other => internal_err!(
"Data type {} not supported for operation 'regex_match_dyn_scalar' on string array",