Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize scalar fast path for `regexp_like` and rejects g inside combined flags like ig (#20354)

Kumar Ujjawalf488a9071b53
diff --git a/datafusion/functions/benches/regx.rs b/datafusion/functions/benches/regx.rs
index c5014655a8..a46b548236 100644
--- a/datafusion/functions/benches/regx.rs
+++ b/datafusion/functions/benches/regx.rs
@@ -15,23 +15,27 @@
// specific language governing permissions and limitations
// under the License.
+use std::hint::black_box;
+use std::iter;
+use std::sync::Arc;
+
use arrow::array::builder::StringBuilder;
use arrow::array::{ArrayRef, AsArray, Int64Array, StringArray, StringViewArray};
use arrow::compute::cast;
-use arrow::datatypes::DataType;
+use arrow::datatypes::{DataType, Field};
use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
use datafusion_functions::regex::regexpcount::regexp_count_func;
use datafusion_functions::regex::regexpinstr::regexp_instr_func;
-use datafusion_functions::regex::regexplike::regexp_like;
+use datafusion_functions::regex::regexplike::{RegexpLikeFunc, regexp_like};
use datafusion_functions::regex::regexpmatch::regexp_match;
use datafusion_functions::regex::regexpreplace::regexp_replace;
use rand::Rng;
use rand::distr::Alphanumeric;
use rand::prelude::IndexedRandom;
use rand::rngs::ThreadRng;
-use std::hint::black_box;
-use std::iter;
-use std::sync::Arc;
fn data(rng: &mut ThreadRng) -> StringArray {
let mut data: Vec<String> = vec![];
for _ in 0..1000 {
@@ -105,6 +109,8 @@ fn subexp(rng: &mut ThreadRng) -> Int64Array {
}
fn criterion_benchmark(c: &mut Criterion) {
+ let regexp_like_func = RegexpLikeFunc::new();
+ let config_options = Arc::new(ConfigOptions::default());
c.bench_function("regexp_count_1000 string", |b| {
let mut rng = rand::rng();
let data = Arc::new(data(&mut rng)) as ArrayRef;
@@ -219,6 +225,32 @@ fn criterion_benchmark(c: &mut Criterion) {
})
});
+ let scalar_args = vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("foobarbequebaz".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("(bar)(beque)".to_string()))),
+ ];
+ let scalar_arg_fields = vec![
+ Field::new("arg_0", DataType::Utf8, false).into(),
+ Field::new("arg_1", DataType::Utf8, false).into(),
+ ];
+ let return_field = Field::new("f", DataType::Boolean, true).into();
+
+ c.bench_function("regexp_like scalar utf8", |b| {
+ b.iter(|| {
+ black_box(
+ regexp_like_func
+ .invoke_with_args(ScalarFunctionArgs {
+ args: scalar_args.clone(),
+ arg_fields: scalar_arg_fields.clone(),
+ number_rows: 1,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .expect("regexp_like scalar should work on valid values"),
+ )
+ })
+ });
+
c.bench_function("regexp_match_1000", |b| {
let mut rng = rand::rng();
let data = Arc::new(data(&mut rng)) as ArrayRef;
diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rs
index 439a2dba06..b2df38a679 100644
--- a/datafusion/functions/src/regex/regexplike.rs
+++ b/datafusion/functions/src/regex/regexplike.rs
@@ -17,7 +17,7 @@
//! Regex expressions
-use arrow::array::{Array, ArrayRef, AsArray, GenericStringArray};
+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, GenericStringArray};
use arrow::compute::kernels::regexp;
use arrow::datatypes::DataType;
use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};
@@ -34,6 +34,7 @@ use datafusion_macros::user_doc;
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
use datafusion_expr_common::operator::Operator;
use datafusion_expr_common::type_coercion::binary::BinaryTypeCoercer;
+use regex::Regex;
use std::any::Any;
use std::sync::Arc;
@@ -130,28 +131,45 @@ impl ScalarUDFImpl for RegexpLikeFunc {
args: datafusion_expr::ScalarFunctionArgs,
) -> Result<ColumnarValue> {
let args = &args.args;
-
- let len = args
- .iter()
- .fold(Option::<usize>::None, |acc, arg| match arg {
- ColumnarValue::Scalar(_) => acc,
- ColumnarValue::Array(a) => Some(a.len()),
- });
-
- let is_scalar = len.is_none();
- let inferred_length = len.unwrap_or(1);
- let args = args
- .iter()
- .map(|arg| arg.to_array(inferred_length))
- .collect::<Result<Vec<_>>>()?;
-
- let result = regexp_like(&args);
- if is_scalar {
- // If all inputs are scalar, keeps output as scalar
- let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0));
- result.map(ColumnarValue::Scalar)
- } else {
- result.map(ColumnarValue::Array)
+ match args.as_slice() {
+ [ColumnarValue::Scalar(value), ColumnarValue::Scalar(pattern)] => {
+ let value = scalar_string(value)?;
+ let pattern = scalar_string(pattern)?;
+ regexp_like_scalar(value, pattern, None)
+ }
+ [
+ ColumnarValue::Scalar(value),
+ ColumnarValue::Scalar(pattern),
+ ColumnarValue::Scalar(flags),
+ ] => {
+ let value = scalar_string(value)?;
+ let pattern = scalar_string(pattern)?;
+ let flags = scalar_string(flags)?;
+ regexp_like_scalar(value, pattern, flags)
+ }
+ [ColumnarValue::Array(values), ColumnarValue::Scalar(pattern)] => {
+ let pattern = scalar_string(pattern)?;
+ let array = regexp_like_array_scalar(values, pattern, None)?;
+ Ok(ColumnarValue::Array(array))
+ }
+ [
+ ColumnarValue::Array(values),
+ ColumnarValue::Scalar(pattern),
+ ColumnarValue::Scalar(flags),
+ ] => {
+ let flags = scalar_string(flags)?;
+ if flags.is_some_and(|flagz| flagz.contains('g')) {
+ plan_err!("regexp_like() does not support the \"global\" option")
+ } else {
+ let pattern = scalar_string(pattern)?;
+ let array = regexp_like_array_scalar(values, pattern, flags)?;
+ Ok(ColumnarValue::Array(array))
+ }
+ }
+ _ => {
+ let args = ColumnarValue::values_to_arrays(args)?;
+ regexp_like(&args).map(ColumnarValue::Array)
+ }
}
}
@@ -302,7 +320,10 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {
}
};
- if flags.iter().any(|s| s == Some("g")) {
+ if flags
+ .iter()
+ .any(|s| s.is_some_and(|flagz| flagz.contains('g')))
+ {
return plan_err!("regexp_like() does not support the \"global\" option");
}
@@ -314,6 +335,83 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {
}
}
+fn scalar_string(value: &ScalarValue) -> Result<Option<&str>> {
+ match value.try_as_str() {
+ Some(v) => Ok(v),
+ None => internal_err!(
+ "Unsupported data type {:?} for function `regexp_like`",
+ value.data_type()
+ ),
+ }
+}
+
+fn regexp_like_array_scalar(
+ values: &ArrayRef,
+ pattern: Option<&str>,
+ flags: Option<&str>,
+) -> Result<ArrayRef> {
+ use DataType::*;
+
+ let Some(pattern) = pattern else {
+ return Ok(Arc::new(BooleanArray::new_null(values.len())));
+ };
+ let array = match values.data_type() {
+ Utf8 => {
+ let array = values.as_string::<i32>();
+ regexp::regexp_is_match_scalar(array, pattern, flags)?
+ }
+ Utf8View => {
+ let array = values.as_string_view();
+ regexp::regexp_is_match_scalar(array, pattern, flags)?
+ }
+ LargeUtf8 => {
+ let array = values.as_string::<i64>();
+ regexp::regexp_is_match_scalar(array, pattern, flags)?
+ }
+ other => {
+ return internal_err!(
+ "Unsupported data type {other:?} for function `regexp_like`"
+ );
+ }
+ };
+
+ Ok(Arc::new(array))
+}
+
+fn regexp_like_scalar(
+ value: Option<&str>,
+ pattern: Option<&str>,
+ flags: Option<&str>,
+) -> Result<ColumnarValue> {
+ if flags.is_some_and(|flagz| flagz.contains('g')) {
+ return plan_err!("regexp_like() does not support the \"global\" option");
+ }
+
+ if value.is_none() || pattern.is_none() {
+ return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None)));
+ }
+
+ let value = value.unwrap();
+ let pattern = pattern.unwrap();
+ let pattern = match flags {
+ Some(flagz) => format!("(?{flagz}){pattern}"),
+ None => pattern.to_string(),
+ };
+
+ let result = if pattern.is_empty() {
+ true
+ } else {
+ let re = Regex::new(pattern.as_str()).map_err(|e| {
+ datafusion_common::DataFusionError::Execution(format!(
+ "Regular expression did not compile: {e:?}"
+ ))
+ })?;
+ re.is_match(value)
+ };
+
+ Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(result))))
+}
+
fn handle_regexp_like(
values: &ArrayRef,
patterns: &ArrayRef,
@@ -399,8 +497,37 @@ mod tests {
use arrow::array::StringArray;
use arrow::array::{BooleanBuilder, StringViewArray};
+ use arrow::datatypes::{DataType, Field};
+ use datafusion_common::config::ConfigOptions;
+ use datafusion_common::{Result, ScalarValue};
+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
- use crate::regex::regexplike::regexp_like;
+ use crate::regex::regexplike::{RegexpLikeFunc, regexp_like};
+
+ fn invoke_regexp_like(args: Vec<ColumnarValue>) -> Result<ColumnarValue> {
+ let number_rows = args
+ .iter()
+ .find_map(|arg| match arg {
+ ColumnarValue::Array(array) => Some(array.len()),
+ _ => None,
+ })
+ .unwrap_or(1);
+ let arg_fields = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| {
+ Arc::new(Field::new(format!("arg_{idx}"), arg.data_type(), true))
+ })
+ .collect::<Vec<_>>();
+
+ RegexpLikeFunc::new().invoke_with_args(ScalarFunctionArgs {
+ args,
+ arg_fields,
+ number_rows,
+ return_field: Arc::new(Field::new("f", DataType::Boolean, true)),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ }
#[test]
fn test_case_sensitive_regexp_like_utf8() {
@@ -499,4 +626,66 @@ mod tests {
"Error during planning: regexp_like() does not support the \"global\" option"
);
}
+
+ #[test]
+ fn test_regexp_like_scalar_invoke() {
+ let args = vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("foobarbequebaz".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("(bar)(beque)".to_string()))),
+ ];
+ let result = invoke_regexp_like(args).unwrap();
+ match result {
+ ColumnarValue::Scalar(ScalarValue::Boolean(Some(true))) => {}
+ other => panic!("Unexpected result {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_regexp_like_array_scalar_invoke() {
+ let values = Arc::new(StringArray::from(vec!["abc", "xyz"]));
+ let args = vec![
+ ColumnarValue::Array(values),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("^(a)".to_string()))),
+ ];
+ let result = invoke_regexp_like(args).unwrap();
+ let mut expected_builder = BooleanBuilder::new();
+ expected_builder.append_value(true);
+ expected_builder.append_value(false);
+ let expected = expected_builder.finish();
+ match result {
+ ColumnarValue::Array(array) => {
+ assert_eq!(array.as_ref(), &expected);
+ }
+ other => panic!("Unexpected result {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_regexp_like_scalar_flags_with_global() {
+ let args = vec![
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("abc".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("^(a)".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("ig".to_string()))),
+ ];
+ let err = invoke_regexp_like(args).expect_err("global flag should be rejected");
+ assert_eq!(
+ err.strip_backtrace(),
+ "Error during planning: regexp_like() does not support the \"global\" option"
+ );
+ }
+
+ #[test]
+ fn test_regexp_like_array_scalar_flags_with_global() {
+ let values = Arc::new(StringArray::from(vec!["abc", "xyz"]));
+ let args = vec![
+ ColumnarValue::Array(values),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("^(a)".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("ig".to_string()))),
+ ];
+ let err = invoke_regexp_like(args).expect_err("global flag should be rejected");
+ assert_eq!(
+ err.strip_backtrace(),
+ "Error during planning: regexp_like() does not support the \"global\" option"
+ );
+ }
}