Apache DataFusion SQL Query Engine
fix: accept empty flags in regexp_like and regexp_match (#25046)
Peter Lee935748de2291
diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rsindex 614fcc5af0..3877251c66 100644--- a/datafusion/functions/src/regex/mod.rs+++ b/datafusion/functions/src/regex/mod.rs@@ -17,7 +17,10 @@//! "regex" DataFusion functions+use arrow::array::ArrayRef;+use arrow::compute::kernels::{cmp::eq, nullif::nullif};use arrow::error::ArrowError;+use datafusion_common::{Result, ScalarValue};use regex::Regex;use std::collections::HashMap;use std::collections::hash_map::Entry;@@ -28,6 +31,14 @@ pub mod regexplike;pub mod regexpmatch;pub mod regexpreplace;+/// Arrow's regex kernels treat null flags as no flags, but reject empty flags.+/// Normalize empty strings without copying the string buffers.+fn normalize_empty_flags(flags: &ArrayRef) -> Result<ArrayRef> {+ let empty =+ ScalarValue::try_from_string(String::new(), flags.data_type())?.to_scalar()?;+ Ok(nullif(flags, &eq(flags, &empty)?)?)+}+// create UDFsmake_udf_function!(regexpcount::RegexpCountFunc, regexp_count);make_udf_function!(regexpinstr::RegexpInstrFunc, regexp_instr);@@ -185,6 +196,108 @@ pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowErrmod tests {use super::start_to_byte_offset;+ #[test]+ fn empty_flags_match_omitted_flags() {+ use arrow::array::StringArray;+ use arrow::compute::cast;+ use arrow::datatypes::{DataType, Field};+ use datafusion_common::config::ConfigOptions;+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};++ use super::*;++ for udf in [regexp_like(), regexp_match()] {+ for data_type in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {+ // Exercise every scalar/array combination, bypassing simplification.+ for shape in 0..8 {+ for pattern in ["b..", "B..", "", "(b)(..)"] {+ let args = ["foobarbaz", pattern, ""]+ .into_iter()+ .enumerate()+ .map(|(i, value)| {+ let scalar = ScalarValue::try_from_string(+ value.to_string(),+ &data_type,+ )+ .unwrap();+ if shape & (1 << i) == 0 {+ ColumnarValue::Scalar(scalar)+ } else {+ ColumnarValue::Array(+ scalar.to_array_of_size(3).unwrap(),+ )+ }+ })+ .collect::<Vec<_>>();+ let invoke = |args: Vec<ColumnarValue>| {+ let types = args+ .iter()+ .map(ColumnarValue::data_type)+ .collect::<Vec<_>>();+ let return_type = udf.return_type(&types).unwrap();+ udf.invoke_with_args(ScalarFunctionArgs {+ arg_fields: types+ .into_iter()+ .map(|t| Arc::new(Field::new("arg", t, true)))+ .collect(),+ args,+ number_rows: 3,+ return_field: Arc::new(Field::new(+ "result",+ return_type,+ true,+ )),+ config_options: Arc::new(ConfigOptions::default()),+ })+ .unwrap()+ .to_array(3)+ .unwrap()+ };+ assert_eq!(+ &invoke(args.clone()),+ &invoke(args[..2].to_vec()),+ "{} {data_type:?} shape={shape} pattern={pattern:?}",+ udf.name()+ );+ }+ }++ // Empty and null flags both mean no flags; preserve nonempty flags+ // and null values/patterns in the same batch.+ let arrays = [+ vec![+ Some("abc"),+ Some("ABC"),+ Some("ABC"),+ None,+ Some("abc"),+ Some("abc"),+ ],+ vec![Some("a"), Some("a"), Some("a"), Some("a"), None, Some("a")],+ vec![Some(""), Some("i"), Some(""), Some(""), Some(""), None],+ ]+ .map(|values| cast(&StringArray::from(values), &data_type).unwrap());+ let expected_flags = cast(+ &StringArray::from(vec![None, Some("i"), None, None, None, None]),+ &data_type,+ )+ .unwrap();+ let kernel = if udf.name() == "regexp_like" {+ regexplike::regexp_like+ } else {+ regexpmatch::regexp_match+ };+ let expected = kernel(&[+ Arc::clone(&arrays[0]),+ Arc::clone(&arrays[1]),+ expected_flags,+ ])+ .unwrap();+ assert_eq!(&kernel(&arrays).unwrap(), &expected);+ }+ }+ }+#[test]fn start_to_byte_offset_ascii() {assert_eq!(start_to_byte_offset("abc", 1), Some(0));diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rsindex 18114931cf..08cd7b0651 100644--- a/datafusion/functions/src/regex/regexplike.rs+++ b/datafusion/functions/src/regex/regexplike.rs@@ -180,7 +180,7 @@ impl ScalarUDFImpl for RegexpLikeFunc {return true;}- let pattern = match flags {+ let pattern = match flags.filter(|flags| !flags.is_empty()) {Some(flags) => Cow::Owned(format!("(?{flags}){pattern}")),None => Cow::Borrowed(pattern),};@@ -308,10 +308,11 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {match args.len() {2 => handle_regexp_like(&args[0], &args[1], None),3 => {- let flags = match args[2].data_type() {- Utf8 => args[2].as_string::<i32>(),+ let flags = super::normalize_empty_flags(&args[2])?;+ let flags = match flags.data_type() {+ Utf8 => flags.as_string::<i32>(),LargeUtf8 => {- let large_string_array = args[2].as_string::<i64>();+ let large_string_array = flags.as_string::<i64>();let string_vec: Vec<Option<&str>> = (0..large_string_array.len()).map(|i| {if large_string_array.is_null(i) {@@ -325,7 +326,7 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {&GenericStringArray::<i32>::from(string_vec)}_ => {- let string_view_array = args[2].as_string_view();+ let string_view_array = flags.as_string_view();let string_vec: Vec<Option<String>> = (0..string_view_array.len()).map(|i| {if string_view_array.is_null(i) {@@ -374,6 +375,7 @@ fn regexp_like_array_scalar(let Some(pattern) = pattern else {return Ok(Arc::new(BooleanArray::new_null(values.len())));};+ let flags = flags.filter(|flags| !flags.is_empty());let array = match values.data_type() {Utf8 => {let array = values.as_string::<i32>();@@ -412,7 +414,7 @@ fn regexp_like_scalar(let value = value.unwrap();let pattern = pattern.unwrap();- let pattern = match flags {+ let pattern = match flags.filter(|flags| !flags.is_empty()) {Some(flagz) => format!("(?{flagz}){pattern}"),None => pattern.to_string(),};@@ -679,6 +681,7 @@ mod tests {assert!(should_evaluate(args("^a+$", None)));assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", None)));+ assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", Some(""))));assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", Some("m"))));let null_value = vec![diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rsindex ce7c437a54..7ca8ba82d5 100644--- a/datafusion/functions/src/regex/regexpmatch.rs+++ b/datafusion/functions/src/regex/regexpmatch.rs@@ -192,7 +192,10 @@ fn regexp_match_scalar_pattern(args: &[ColumnarValue]) -> Result<Option<ArrayRef}let pattern = pattern.to_scalar()?;- let flags = flags.map(ScalarValue::to_scalar).transpose()?;+ let flags = flags+ .filter(|flags| flags.try_as_str() != Some(Some("")))+ .map(ScalarValue::to_scalar)+ .transpose()?;regexp::regexp_match(values,@@ -237,7 +240,8 @@ pub fn regexp_match(args: &[ArrayRef]) -> Result<ArrayRef> {}}- regexp::regexp_match(&args[0], &args[1], Some(&args[2]))+ let flags = super::normalize_empty_flags(&args[2])?;+ regexp::regexp_match(&args[0], &args[1], Some(&flags)).map_err(|e| arrow_datafusion_err!(e))}other => exec_err!(diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt b/datafusion/sqllogictest/test_files/regexp/regexp_like.sltindex 30fa913896..14531fd466 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt@@ -368,3 +368,36 @@ select * from regexp_test where regexp_like('f', regexp_replace((('v\r') like ('statement okdrop table if exists dict_table;++# Empty flags behave like omitted flags for literals and table columns.+query BBB+SELECT regexp_like('foobarbaz', 'b..', ''),+ regexp_like('fooBAR', 'b..', ''),+ regexp_like('foobarbaz', '', '');+----+true false true++statement ok+CREATE TABLE regexp_empty_flags (str VARCHAR, pattern VARCHAR, flags VARCHAR) AS VALUES+ ('foobarbaz', 'b..', ''),+ ('fooBAR', 'b..', ''),+ ('fooBAR', 'b..', 'i'),+ ('foobarbaz', '', ''),+ (NULL, 'b..', ''),+ ('foobarbaz', NULL, ''),+ ('foobarbaz', 'b..', NULL);++query BB+SELECT regexp_like(str, pattern, flags), regexp_like(str, 'b..', '')+FROM regexp_empty_flags;+----+true true+false false+true false+true true+NULL NULL+NULL true+true true++statement ok+DROP TABLE regexp_empty_flags;diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt b/datafusion/sqllogictest/test_files/regexp/regexp_match.sltindex e79af4774a..1bc97b5be6 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt@@ -199,3 +199,36 @@ query Bselect null !~* 'abc';----NULL++# Empty flags behave like omitted flags for literals and table columns.+query ???+SELECT regexp_match('foobarbaz', 'b..', ''),+ regexp_match('fooBAR', 'b..', ''),+ regexp_match('foobarbaz', '', '');+----+[bar] NULL []++statement ok+CREATE TABLE regexp_empty_flags (str VARCHAR, pattern VARCHAR, flags VARCHAR) AS VALUES+ ('foobarbaz', 'b..', ''),+ ('fooBAR', 'b..', ''),+ ('fooBAR', 'b..', 'i'),+ ('foobarbaz', '', ''),+ (NULL, 'b..', ''),+ ('foobarbaz', NULL, ''),+ ('foobarbaz', 'b..', NULL);++query ??+SELECT regexp_match(str, pattern, flags), regexp_match(str, 'b..', '')+FROM regexp_empty_flags;+----+[bar] [bar]+NULL NULL+[BAR] NULL+[] [bar]+NULL NULL+NULL [bar]+[bar] [bar]++statement ok+DROP TABLE regexp_empty_flags;