Skip to content

Apache DataFusion SQL Query Engine

fix: accept empty flags in regexp_like and regexp_match (#25046)

Peter Lee935748de2291
diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rs
index 614fcc5af0..3877251c66 100644
--- a/datafusion/functions/src/regex/mod.rs
+++ b/datafusion/functions/src/regex/mod.rs
@@ -17,7 +17,10 @@
//! "regex" DataFusion functions
+use arrow::array::ArrayRef;
+use arrow::compute::kernels::{cmp::eq, nullif::nullif};
use arrow::error::ArrowError;
+use datafusion_common::{Result, ScalarValue};
use regex::Regex;
use std::collections::HashMap;
use std::collections::hash_map::Entry;
@@ -28,6 +31,14 @@ pub mod regexplike;
pub mod regexpmatch;
pub mod regexpreplace;
+/// Arrow's regex kernels treat null flags as no flags, but reject empty flags.
+/// Normalize empty strings without copying the string buffers.
+fn normalize_empty_flags(flags: &ArrayRef) -> Result<ArrayRef> {
+ let empty =
+ ScalarValue::try_from_string(String::new(), flags.data_type())?.to_scalar()?;
+ Ok(nullif(flags, &eq(flags, &empty)?)?)
+}
+
// create UDFs
make_udf_function!(regexpcount::RegexpCountFunc, regexp_count);
make_udf_function!(regexpinstr::RegexpInstrFunc, regexp_instr);
@@ -185,6 +196,108 @@ pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowErr
mod tests {
use super::start_to_byte_offset;
+ #[test]
+ fn empty_flags_match_omitted_flags() {
+ use arrow::array::StringArray;
+ use arrow::compute::cast;
+ use arrow::datatypes::{DataType, Field};
+ use datafusion_common::config::ConfigOptions;
+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+
+ use super::*;
+
+ for udf in [regexp_like(), regexp_match()] {
+ for data_type in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {
+ // Exercise every scalar/array combination, bypassing simplification.
+ for shape in 0..8 {
+ for pattern in ["b..", "B..", "", "(b)(..)"] {
+ let args = ["foobarbaz", pattern, ""]
+ .into_iter()
+ .enumerate()
+ .map(|(i, value)| {
+ let scalar = ScalarValue::try_from_string(
+ value.to_string(),
+ &data_type,
+ )
+ .unwrap();
+ if shape & (1 << i) == 0 {
+ ColumnarValue::Scalar(scalar)
+ } else {
+ ColumnarValue::Array(
+ scalar.to_array_of_size(3).unwrap(),
+ )
+ }
+ })
+ .collect::<Vec<_>>();
+ let invoke = |args: Vec<ColumnarValue>| {
+ let types = args
+ .iter()
+ .map(ColumnarValue::data_type)
+ .collect::<Vec<_>>();
+ let return_type = udf.return_type(&types).unwrap();
+ udf.invoke_with_args(ScalarFunctionArgs {
+ arg_fields: types
+ .into_iter()
+ .map(|t| Arc::new(Field::new("arg", t, true)))
+ .collect(),
+ args,
+ number_rows: 3,
+ return_field: Arc::new(Field::new(
+ "result",
+ return_type,
+ true,
+ )),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap()
+ .to_array(3)
+ .unwrap()
+ };
+ assert_eq!(
+ &invoke(args.clone()),
+ &invoke(args[..2].to_vec()),
+ "{} {data_type:?} shape={shape} pattern={pattern:?}",
+ udf.name()
+ );
+ }
+ }
+
+ // Empty and null flags both mean no flags; preserve nonempty flags
+ // and null values/patterns in the same batch.
+ let arrays = [
+ vec![
+ Some("abc"),
+ Some("ABC"),
+ Some("ABC"),
+ None,
+ Some("abc"),
+ Some("abc"),
+ ],
+ vec![Some("a"), Some("a"), Some("a"), Some("a"), None, Some("a")],
+ vec![Some(""), Some("i"), Some(""), Some(""), Some(""), None],
+ ]
+ .map(|values| cast(&StringArray::from(values), &data_type).unwrap());
+ let expected_flags = cast(
+ &StringArray::from(vec![None, Some("i"), None, None, None, None]),
+ &data_type,
+ )
+ .unwrap();
+ let kernel = if udf.name() == "regexp_like" {
+ regexplike::regexp_like
+ } else {
+ regexpmatch::regexp_match
+ };
+ let expected = kernel(&[
+ Arc::clone(&arrays[0]),
+ Arc::clone(&arrays[1]),
+ expected_flags,
+ ])
+ .unwrap();
+ assert_eq!(&kernel(&arrays).unwrap(), &expected);
+ }
+ }
+ }
+
#[test]
fn start_to_byte_offset_ascii() {
assert_eq!(start_to_byte_offset("abc", 1), Some(0));
diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rs
index 18114931cf..08cd7b0651 100644
--- a/datafusion/functions/src/regex/regexplike.rs
+++ b/datafusion/functions/src/regex/regexplike.rs
@@ -180,7 +180,7 @@ impl ScalarUDFImpl for RegexpLikeFunc {
return true;
}
- let pattern = match flags {
+ let pattern = match flags.filter(|flags| !flags.is_empty()) {
Some(flags) => Cow::Owned(format!("(?{flags}){pattern}")),
None => Cow::Borrowed(pattern),
};
@@ -308,10 +308,11 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {
match args.len() {
2 => handle_regexp_like(&args[0], &args[1], None),
3 => {
- let flags = match args[2].data_type() {
- Utf8 => args[2].as_string::<i32>(),
+ let flags = super::normalize_empty_flags(&args[2])?;
+ let flags = match flags.data_type() {
+ Utf8 => flags.as_string::<i32>(),
LargeUtf8 => {
- let large_string_array = args[2].as_string::<i64>();
+ let large_string_array = flags.as_string::<i64>();
let string_vec: Vec<Option<&str>> = (0..large_string_array.len())
.map(|i| {
if large_string_array.is_null(i) {
@@ -325,7 +326,7 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {
&GenericStringArray::<i32>::from(string_vec)
}
_ => {
- let string_view_array = args[2].as_string_view();
+ let string_view_array = flags.as_string_view();
let string_vec: Vec<Option<String>> = (0..string_view_array.len())
.map(|i| {
if string_view_array.is_null(i) {
@@ -374,6 +375,7 @@ fn regexp_like_array_scalar(
let Some(pattern) = pattern else {
return Ok(Arc::new(BooleanArray::new_null(values.len())));
};
+ let flags = flags.filter(|flags| !flags.is_empty());
let array = match values.data_type() {
Utf8 => {
let array = values.as_string::<i32>();
@@ -412,7 +414,7 @@ fn regexp_like_scalar(
let value = value.unwrap();
let pattern = pattern.unwrap();
- let pattern = match flags {
+ let pattern = match flags.filter(|flags| !flags.is_empty()) {
Some(flagz) => format!("(?{flagz}){pattern}"),
None => pattern.to_string(),
};
@@ -679,6 +681,7 @@ mod tests {
assert!(should_evaluate(args("^a+$", None)));
assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", None)));
+ assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", Some(""))));
assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", Some("m"))));
let null_value = vec![
diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rs
index ce7c437a54..7ca8ba82d5 100644
--- a/datafusion/functions/src/regex/regexpmatch.rs
+++ b/datafusion/functions/src/regex/regexpmatch.rs
@@ -192,7 +192,10 @@ fn regexp_match_scalar_pattern(args: &[ColumnarValue]) -> Result<Option<ArrayRef
}
let pattern = pattern.to_scalar()?;
- let flags = flags.map(ScalarValue::to_scalar).transpose()?;
+ let flags = flags
+ .filter(|flags| flags.try_as_str() != Some(Some("")))
+ .map(ScalarValue::to_scalar)
+ .transpose()?;
regexp::regexp_match(
values,
@@ -237,7 +240,8 @@ pub fn regexp_match(args: &[ArrayRef]) -> Result<ArrayRef> {
}
}
- regexp::regexp_match(&args[0], &args[1], Some(&args[2]))
+ let flags = super::normalize_empty_flags(&args[2])?;
+ regexp::regexp_match(&args[0], &args[1], Some(&flags))
.map_err(|e| arrow_datafusion_err!(e))
}
other => exec_err!(
diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
index 30fa913896..14531fd466 100644
--- a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
+++ b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
@@ -368,3 +368,36 @@ select * from regexp_test where regexp_like('f', regexp_replace((('v\r') like ('
statement ok
drop table if exists dict_table;
+
+# Empty flags behave like omitted flags for literals and table columns.
+query BBB
+SELECT regexp_like('foobarbaz', 'b..', ''),
+ regexp_like('fooBAR', 'b..', ''),
+ regexp_like('foobarbaz', '', '');
+----
+true false true
+
+statement ok
+CREATE TABLE regexp_empty_flags (str VARCHAR, pattern VARCHAR, flags VARCHAR) AS VALUES
+ ('foobarbaz', 'b..', ''),
+ ('fooBAR', 'b..', ''),
+ ('fooBAR', 'b..', 'i'),
+ ('foobarbaz', '', ''),
+ (NULL, 'b..', ''),
+ ('foobarbaz', NULL, ''),
+ ('foobarbaz', 'b..', NULL);
+
+query BB
+SELECT regexp_like(str, pattern, flags), regexp_like(str, 'b..', '')
+FROM regexp_empty_flags;
+----
+true true
+false false
+true false
+true true
+NULL NULL
+NULL true
+true true
+
+statement ok
+DROP TABLE regexp_empty_flags;
diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt
index e79af4774a..1bc97b5be6 100644
--- a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt
+++ b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt
@@ -199,3 +199,36 @@ query B
select null !~* 'abc';
----
NULL
+
+# Empty flags behave like omitted flags for literals and table columns.
+query ???
+SELECT regexp_match('foobarbaz', 'b..', ''),
+ regexp_match('fooBAR', 'b..', ''),
+ regexp_match('foobarbaz', '', '');
+----
+[bar] NULL []
+
+statement ok
+CREATE TABLE regexp_empty_flags (str VARCHAR, pattern VARCHAR, flags VARCHAR) AS VALUES
+ ('foobarbaz', 'b..', ''),
+ ('fooBAR', 'b..', ''),
+ ('fooBAR', 'b..', 'i'),
+ ('foobarbaz', '', ''),
+ (NULL, 'b..', ''),
+ ('foobarbaz', NULL, ''),
+ ('foobarbaz', 'b..', NULL);
+
+query ??
+SELECT regexp_match(str, pattern, flags), regexp_match(str, 'b..', '')
+FROM regexp_empty_flags;
+----
+[bar] [bar]
+NULL NULL
+[BAR] NULL
+[] [bar]
+NULL NULL
+NULL [bar]
+[bar] [bar]
+
+statement ok
+DROP TABLE regexp_empty_flags;