Apache DataFusion SQL Query Engine
perf: Optimize scalar fast path for `regexp_like` and rejects g inside combined flags like ig (#20354)
Kumar Ujjawalf488a9071b53
diff --git a/datafusion/functions/benches/regx.rs b/datafusion/functions/benches/regx.rsindex c5014655a8..a46b548236 100644--- a/datafusion/functions/benches/regx.rs+++ b/datafusion/functions/benches/regx.rs@@ -15,23 +15,27 @@// specific language governing permissions and limitations// under the License.+use std::hint::black_box;+use std::iter;+use std::sync::Arc;+use arrow::array::builder::StringBuilder;use arrow::array::{ArrayRef, AsArray, Int64Array, StringArray, StringViewArray};use arrow::compute::cast;-use arrow::datatypes::DataType;+use arrow::datatypes::{DataType, Field};use criterion::{Criterion, criterion_group, criterion_main};+use datafusion_common::ScalarValue;+use datafusion_common::config::ConfigOptions;+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};use datafusion_functions::regex::regexpcount::regexp_count_func;use datafusion_functions::regex::regexpinstr::regexp_instr_func;-use datafusion_functions::regex::regexplike::regexp_like;+use datafusion_functions::regex::regexplike::{RegexpLikeFunc, regexp_like};use datafusion_functions::regex::regexpmatch::regexp_match;use datafusion_functions::regex::regexpreplace::regexp_replace;use rand::Rng;use rand::distr::Alphanumeric;use rand::prelude::IndexedRandom;use rand::rngs::ThreadRng;-use std::hint::black_box;-use std::iter;-use std::sync::Arc;fn data(rng: &mut ThreadRng) -> StringArray {let mut data: Vec<String> = vec![];for _ in 0..1000 {@@ -105,6 +109,8 @@ fn subexp(rng: &mut ThreadRng) -> Int64Array {}fn criterion_benchmark(c: &mut Criterion) {+ let regexp_like_func = RegexpLikeFunc::new();+ let config_options = Arc::new(ConfigOptions::default());c.bench_function("regexp_count_1000 string", |b| {let mut rng = rand::rng();let data = Arc::new(data(&mut rng)) as ArrayRef;@@ -219,6 +225,32 @@ fn criterion_benchmark(c: &mut Criterion) {})});+ let scalar_args = vec![+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("foobarbequebaz".to_string()))),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("(bar)(beque)".to_string()))),+ ];+ let scalar_arg_fields = vec![+ Field::new("arg_0", DataType::Utf8, false).into(),+ Field::new("arg_1", DataType::Utf8, false).into(),+ ];+ let return_field = Field::new("f", DataType::Boolean, true).into();++ c.bench_function("regexp_like scalar utf8", |b| {+ b.iter(|| {+ black_box(+ regexp_like_func+ .invoke_with_args(ScalarFunctionArgs {+ args: scalar_args.clone(),+ arg_fields: scalar_arg_fields.clone(),+ number_rows: 1,+ return_field: Arc::clone(&return_field),+ config_options: Arc::clone(&config_options),+ })+ .expect("regexp_like scalar should work on valid values"),+ )+ })+ });+c.bench_function("regexp_match_1000", |b| {let mut rng = rand::rng();let data = Arc::new(data(&mut rng)) as ArrayRef;diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rsindex 439a2dba06..b2df38a679 100644--- a/datafusion/functions/src/regex/regexplike.rs+++ b/datafusion/functions/src/regex/regexplike.rs@@ -17,7 +17,7 @@//! Regex expressions-use arrow::array::{Array, ArrayRef, AsArray, GenericStringArray};+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, GenericStringArray};use arrow::compute::kernels::regexp;use arrow::datatypes::DataType;use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};@@ -34,6 +34,7 @@ use datafusion_macros::user_doc;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};use datafusion_expr_common::operator::Operator;use datafusion_expr_common::type_coercion::binary::BinaryTypeCoercer;+use regex::Regex;use std::any::Any;use std::sync::Arc;@@ -130,28 +131,45 @@ impl ScalarUDFImpl for RegexpLikeFunc {args: datafusion_expr::ScalarFunctionArgs,) -> Result<ColumnarValue> {let args = &args.args;-- let len = args- .iter()- .fold(Option::<usize>::None, |acc, arg| match arg {- ColumnarValue::Scalar(_) => acc,- ColumnarValue::Array(a) => Some(a.len()),- });-- let is_scalar = len.is_none();- let inferred_length = len.unwrap_or(1);- let args = args- .iter()- .map(|arg| arg.to_array(inferred_length))- .collect::<Result<Vec<_>>>()?;-- let result = regexp_like(&args);- if is_scalar {- // If all inputs are scalar, keeps output as scalar- let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0));- result.map(ColumnarValue::Scalar)- } else {- result.map(ColumnarValue::Array)+ match args.as_slice() {+ [ColumnarValue::Scalar(value), ColumnarValue::Scalar(pattern)] => {+ let value = scalar_string(value)?;+ let pattern = scalar_string(pattern)?;+ regexp_like_scalar(value, pattern, None)+ }+ [+ ColumnarValue::Scalar(value),+ ColumnarValue::Scalar(pattern),+ ColumnarValue::Scalar(flags),+ ] => {+ let value = scalar_string(value)?;+ let pattern = scalar_string(pattern)?;+ let flags = scalar_string(flags)?;+ regexp_like_scalar(value, pattern, flags)+ }+ [ColumnarValue::Array(values), ColumnarValue::Scalar(pattern)] => {+ let pattern = scalar_string(pattern)?;+ let array = regexp_like_array_scalar(values, pattern, None)?;+ Ok(ColumnarValue::Array(array))+ }+ [+ ColumnarValue::Array(values),+ ColumnarValue::Scalar(pattern),+ ColumnarValue::Scalar(flags),+ ] => {+ let flags = scalar_string(flags)?;+ if flags.is_some_and(|flagz| flagz.contains('g')) {+ plan_err!("regexp_like() does not support the \"global\" option")+ } else {+ let pattern = scalar_string(pattern)?;+ let array = regexp_like_array_scalar(values, pattern, flags)?;+ Ok(ColumnarValue::Array(array))+ }+ }+ _ => {+ let args = ColumnarValue::values_to_arrays(args)?;+ regexp_like(&args).map(ColumnarValue::Array)+ }}}@@ -302,7 +320,10 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {}};- if flags.iter().any(|s| s == Some("g")) {+ if flags+ .iter()+ .any(|s| s.is_some_and(|flagz| flagz.contains('g')))+ {return plan_err!("regexp_like() does not support the \"global\" option");}@@ -314,6 +335,83 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {}}+fn scalar_string(value: &ScalarValue) -> Result<Option<&str>> {+ match value.try_as_str() {+ Some(v) => Ok(v),+ None => internal_err!(+ "Unsupported data type {:?} for function `regexp_like`",+ value.data_type()+ ),+ }+}++fn regexp_like_array_scalar(+ values: &ArrayRef,+ pattern: Option<&str>,+ flags: Option<&str>,+) -> Result<ArrayRef> {+ use DataType::*;++ let Some(pattern) = pattern else {+ return Ok(Arc::new(BooleanArray::new_null(values.len())));+ };+ let array = match values.data_type() {+ Utf8 => {+ let array = values.as_string::<i32>();+ regexp::regexp_is_match_scalar(array, pattern, flags)?+ }+ Utf8View => {+ let array = values.as_string_view();+ regexp::regexp_is_match_scalar(array, pattern, flags)?+ }+ LargeUtf8 => {+ let array = values.as_string::<i64>();+ regexp::regexp_is_match_scalar(array, pattern, flags)?+ }+ other => {+ return internal_err!(+ "Unsupported data type {other:?} for function `regexp_like`"+ );+ }+ };++ Ok(Arc::new(array))+}++fn regexp_like_scalar(+ value: Option<&str>,+ pattern: Option<&str>,+ flags: Option<&str>,+) -> Result<ColumnarValue> {+ if flags.is_some_and(|flagz| flagz.contains('g')) {+ return plan_err!("regexp_like() does not support the \"global\" option");+ }++ if value.is_none() || pattern.is_none() {+ return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None)));+ }++ let value = value.unwrap();+ let pattern = pattern.unwrap();+ let pattern = match flags {+ Some(flagz) => format!("(?{flagz}){pattern}"),+ None => pattern.to_string(),+ };++ let result = if pattern.is_empty() {+ true+ } else {+ let re = Regex::new(pattern.as_str()).map_err(|e| {+ datafusion_common::DataFusionError::Execution(format!(+ "Regular expression did not compile: {e:?}"+ ))+ })?;+ re.is_match(value)+ };++ Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(result))))+}+fn handle_regexp_like(values: &ArrayRef,patterns: &ArrayRef,@@ -399,8 +497,37 @@ mod tests {use arrow::array::StringArray;use arrow::array::{BooleanBuilder, StringViewArray};+ use arrow::datatypes::{DataType, Field};+ use datafusion_common::config::ConfigOptions;+ use datafusion_common::{Result, ScalarValue};+ use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};- use crate::regex::regexplike::regexp_like;+ use crate::regex::regexplike::{RegexpLikeFunc, regexp_like};++ fn invoke_regexp_like(args: Vec<ColumnarValue>) -> Result<ColumnarValue> {+ let number_rows = args+ .iter()+ .find_map(|arg| match arg {+ ColumnarValue::Array(array) => Some(array.len()),+ _ => None,+ })+ .unwrap_or(1);+ let arg_fields = args+ .iter()+ .enumerate()+ .map(|(idx, arg)| {+ Arc::new(Field::new(format!("arg_{idx}"), arg.data_type(), true))+ })+ .collect::<Vec<_>>();++ RegexpLikeFunc::new().invoke_with_args(ScalarFunctionArgs {+ args,+ arg_fields,+ number_rows,+ return_field: Arc::new(Field::new("f", DataType::Boolean, true)),+ config_options: Arc::new(ConfigOptions::default()),+ })+ }#[test]fn test_case_sensitive_regexp_like_utf8() {@@ -499,4 +626,66 @@ mod tests {"Error during planning: regexp_like() does not support the \"global\" option");}++ #[test]+ fn test_regexp_like_scalar_invoke() {+ let args = vec![+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("foobarbequebaz".to_string()))),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("(bar)(beque)".to_string()))),+ ];+ let result = invoke_regexp_like(args).unwrap();+ match result {+ ColumnarValue::Scalar(ScalarValue::Boolean(Some(true))) => {}+ other => panic!("Unexpected result {other:?}"),+ }+ }++ #[test]+ fn test_regexp_like_array_scalar_invoke() {+ let values = Arc::new(StringArray::from(vec!["abc", "xyz"]));+ let args = vec![+ ColumnarValue::Array(values),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("^(a)".to_string()))),+ ];+ let result = invoke_regexp_like(args).unwrap();+ let mut expected_builder = BooleanBuilder::new();+ expected_builder.append_value(true);+ expected_builder.append_value(false);+ let expected = expected_builder.finish();+ match result {+ ColumnarValue::Array(array) => {+ assert_eq!(array.as_ref(), &expected);+ }+ other => panic!("Unexpected result {other:?}"),+ }+ }++ #[test]+ fn test_regexp_like_scalar_flags_with_global() {+ let args = vec![+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("abc".to_string()))),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("^(a)".to_string()))),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("ig".to_string()))),+ ];+ let err = invoke_regexp_like(args).expect_err("global flag should be rejected");+ assert_eq!(+ err.strip_backtrace(),+ "Error during planning: regexp_like() does not support the \"global\" option"+ );+ }++ #[test]+ fn test_regexp_like_array_scalar_flags_with_global() {+ let values = Arc::new(StringArray::from(vec!["abc", "xyz"]));+ let args = vec![+ ColumnarValue::Array(values),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("^(a)".to_string()))),+ ColumnarValue::Scalar(ScalarValue::Utf8(Some("ig".to_string()))),+ ];+ let err = invoke_regexp_like(args).expect_err("global flag should be rejected");+ assert_eq!(+ err.strip_backtrace(),+ "Error during planning: regexp_like() does not support the \"global\" option"+ );+ }}