Apache DataFusion SQL Query Engine
fix: report regex compile failures as DataFusion errors, consistently across the regexp family (#25352)
Adrian Garcia Badaracco28c0dec0a4c0
diff --git a/Cargo.lock b/Cargo.lockindex 2c4e8d3e7e..f0677c38d8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2462,6 +2462,7 @@ dependencies = ["parking_lot","pin-project","rand 0.9.4",+ "regex",][[package]]diff --git a/datafusion-examples/examples/builtin_functions/regexp.rs b/datafusion-examples/examples/builtin_functions/regexp.rsindex 97dc71b94e..94f8315018 100644--- a/datafusion-examples/examples/builtin_functions/regexp.rs+++ b/datafusion-examples/examples/builtin_functions/regexp.rs@@ -162,7 +162,8 @@ pub async fn regexp() -> Result<()> {.collect().await;- let expected = "Regular expression did not compile: CompiledTooBig";+ let expected =+ "Regular expression did not compile: Compiled regex exceeds size limit";assert_contains!(result.unwrap_err().to_string(), expected);//diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rsindex 3877251c66..bf113b9c03 100644--- a/datafusion/functions/src/regex/mod.rs+++ b/datafusion/functions/src/regex/mod.rs@@ -19,12 +19,16 @@use arrow::array::ArrayRef;use arrow::compute::kernels::{cmp::eq, nullif::nullif};-use arrow::error::ArrowError;use datafusion_common::{Result, ScalarValue};-use regex::Regex;-use std::collections::HashMap;-use std::collections::hash_map::Entry;use std::sync::Arc;++pub(crate) use datafusion_physical_expr_common::regex::explain_regexp_kernel_error;+// The compilation of a regular expression is shared with the physical+// expressions, so that every caller reports a failure in the same way. These+// re-exports keep the paths that callers of this crate already use.+pub use datafusion_physical_expr_common::regex::{+ compile_and_cache_regex, compile_regex,+};pub mod regexpcount;pub mod regexpinstr;pub mod regexplike;@@ -139,24 +143,6 @@ pub fn functions() -> Vec<Arc<datafusion_expr::ScalarUDF>> {]}-pub fn compile_and_cache_regex<'strings, 'cache>(- regex: &'strings str,- flags: Option<&'strings str>,- regex_cache: &'cache mut HashMap<(&'strings str, Option<&'strings str>), Regex>,-) -> Result<&'cache Regex, ArrowError>-where- 'strings: 'cache,-{- let result = match regex_cache.entry((regex, flags)) {- Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),- Entry::Vacant(vacant_entry) => {- let compiled = compile_regex(regex, flags)?;- vacant_entry.insert(compiled)- }- };- Ok(result)-}-/// Maps `start`, a 1-based character position, to a byte offset in `value`./// Positions `1..=n` (for an `n`-character string) map to the corresponding/// character's first byte; position `n + 1`, the end of the string, maps to@@ -173,25 +159,6 @@ pub(crate) fn start_to_byte_offset(value: &str, start: i64) -> Option<usize> {.nth(start_index)}-pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowError> {- let pattern = match flags {- None | Some("") => regex.to_string(),- Some(flags) => {- if flags.contains('g') {- return Err(ArrowError::ComputeError(- "regexp_count()/regexp_instr() does not support the global flag"- .to_string(),- ));- }- format!("(?{flags}){regex}")- }- };-- Regex::new(&pattern).map_err(|_| {- ArrowError::ComputeError(format!("Regular expression did not compile: {pattern}"))- })-}-#[cfg(test)]mod tests {use super::start_to_byte_offset;diff --git a/datafusion/functions/src/regex/regexpcount.rs b/datafusion/functions/src/regex/regexpcount.rsindex 2920b687ed..c8ada077af 100644--- a/datafusion/functions/src/regex/regexpcount.rs+++ b/datafusion/functions/src/regex/regexpcount.rs@@ -21,7 +21,6 @@ use arrow::datatypes::{DataType, Int64Type};use arrow::datatypes::{DataType::Int64, DataType::LargeUtf8, DataType::Utf8, DataType::Utf8View,};-use arrow::error::ArrowError;use datafusion_common::{Result, ScalarValue, exec_err, internal_err};use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -155,7 +154,6 @@ pub fn regexp_count_func(args: &[ArrayRef]) -> Result<ArrayRef> {if args_len > 2 { Some(&args[2]) } else { None },if args_len > 3 { Some(&args[3]) } else { None },)- .map_err(|e| e.into())}/// `arrow-rs` style implementation of `regexp_count` function.@@ -178,7 +176,7 @@ fn regexp_count(regex_array: &dyn Datum,start_array: Option<&dyn Datum>,flags_array: Option<&dyn Datum>,-) -> Result<ArrayRef, ArrowError> {+) -> Result<ArrayRef> {let (regex_array, is_regex_scalar) = regex_array.get();let (start_array, is_start_scalar) = start_array.map_or((None, true), |start| {let (start, is_start_scalar) = start.get();@@ -199,15 +197,17 @@ fn regexp_count(None,is_flags_scalar,),- (Utf8, Utf8, Some(flags_array)) if *flags_array.data_type() == Utf8 => regexp_count_inner(- &values.as_string::<i32>(),- ®ex_array.as_string::<i32>(),- is_regex_scalar,- start_array.map(|start| start.as_primitive::<Int64Type>()),- is_start_scalar,- Some(&flags_array.as_string::<i32>()),- is_flags_scalar,- ),+ (Utf8, Utf8, Some(flags_array)) if *flags_array.data_type() == Utf8 => {+ regexp_count_inner(+ &values.as_string::<i32>(),+ ®ex_array.as_string::<i32>(),+ is_regex_scalar,+ start_array.map(|start| start.as_primitive::<Int64Type>()),+ is_start_scalar,+ Some(&flags_array.as_string::<i32>()),+ is_flags_scalar,+ )+ }(LargeUtf8, LargeUtf8, None) => regexp_count_inner(&values.as_string::<i64>(),®ex_array.as_string::<i64>(),@@ -217,15 +217,19 @@ fn regexp_count(None,is_flags_scalar,),- (LargeUtf8, LargeUtf8, Some(flags_array)) if *flags_array.data_type() == LargeUtf8 => regexp_count_inner(- &values.as_string::<i64>(),- ®ex_array.as_string::<i64>(),- is_regex_scalar,- start_array.map(|start| start.as_primitive::<Int64Type>()),- is_start_scalar,- Some(&flags_array.as_string::<i64>()),- is_flags_scalar,- ),+ (LargeUtf8, LargeUtf8, Some(flags_array))+ if *flags_array.data_type() == LargeUtf8 =>+ {+ regexp_count_inner(+ &values.as_string::<i64>(),+ ®ex_array.as_string::<i64>(),+ is_regex_scalar,+ start_array.map(|start| start.as_primitive::<Int64Type>()),+ is_start_scalar,+ Some(&flags_array.as_string::<i64>()),+ is_flags_scalar,+ )+ }(Utf8View, Utf8View, None) => regexp_count_inner(&values.as_string_view(),®ex_array.as_string_view(),@@ -235,18 +239,22 @@ fn regexp_count(None,is_flags_scalar,),- (Utf8View, Utf8View, Some(flags_array)) if *flags_array.data_type() == Utf8View => regexp_count_inner(- &values.as_string_view(),- ®ex_array.as_string_view(),- is_regex_scalar,- start_array.map(|start| start.as_primitive::<Int64Type>()),- is_start_scalar,- Some(&flags_array.as_string_view()),- is_flags_scalar,+ (Utf8View, Utf8View, Some(flags_array))+ if *flags_array.data_type() == Utf8View =>+ {+ regexp_count_inner(+ &values.as_string_view(),+ ®ex_array.as_string_view(),+ is_regex_scalar,+ start_array.map(|start| start.as_primitive::<Int64Type>()),+ is_start_scalar,+ Some(&flags_array.as_string_view()),+ is_flags_scalar,+ )+ }+ _ => internal_err!(+ "regexp_count() expected the input arrays to be of type Utf8, LargeUtf8, or Utf8View and the data types of the values, regex_array, and flags_array to match"),- _ => Err(ArrowError::ComputeError(- "regexp_count() expected the input arrays to be of type Utf8, LargeUtf8, or Utf8View and the data types of the values, regex_array, and flags_array to match".to_string(),- )),}}@@ -258,7 +266,7 @@ fn regexp_count_inner<'a, S>(is_start_scalar: bool,flags_array: Option<&S>,is_flags_scalar: bool,-) -> Result<ArrayRef, ArrowError>+) -> Result<ArrayRef>whereS: StringArrayType<'a>,{@@ -293,23 +301,23 @@ wherematch (regex_scalar, is_start_scalar, is_flags_scalar) {(Some(regex), true, true) => {- let pattern = compile_regex(regex, flags_scalar)?;+ let pattern = compile_regex("regexp_count", regex, flags_scalar)?;Ok(Arc::new(values.iter().map(|value| count_matches(value, &pattern, start_scalar))- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(Some(regex), true, false) => {let flags_array = flags_array.unwrap();if values.len() != flags_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("flags_array must be the same length as values array; got {} and {}",flags_array.len(),- values.len(),- )));+ values.len()+ );}Ok(Arc::new(@@ -322,17 +330,18 @@ where};let pattern = compile_and_cache_regex(+ "regexp_count",regex,Some(flags),&mut regex_cache,)?;count_matches(value, pattern, start_scalar)})- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(Some(regex), false, true) => {- let pattern = compile_regex(regex, flags_scalar)?;+ let pattern = compile_regex("regexp_count", regex, flags_scalar)?;let start_array = start_array.unwrap();@@ -341,17 +350,17 @@ where.iter().zip(start_array.iter()).map(|(value, start)| count_matches(value, &pattern, start))- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(Some(regex), false, false) => {let flags_array = flags_array.unwrap();if values.len() != flags_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("flags_array must be the same length as values array; got {} and {}",flags_array.len(),- values.len(),- )));+ values.len()+ );}Ok(Arc::new(@@ -365,21 +374,25 @@ wherereturn Ok(None);};- let pattern =- compile_and_cache_regex(regex, Some(flags), &mut regex_cache)?;+ let pattern = compile_and_cache_regex(+ "regexp_count",+ regex,+ Some(flags),+ &mut regex_cache,+ )?;count_matches(value, pattern, start)})- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(None, true, true) => {if values.len() != regex_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("regex_array must be the same length as values array; got {} and {}",regex_array.len(),- values.len(),- )));+ values.len()+ );}Ok(Arc::new(@@ -392,31 +405,32 @@ where};let pattern = compile_and_cache_regex(+ "regexp_count",regex,flags_scalar,&mut regex_cache,)?;count_matches(value, pattern, start_scalar)})- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(None, true, false) => {if values.len() != regex_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("regex_array must be the same length as values array; got {} and {}",regex_array.len(),- values.len(),- )));+ values.len()+ );}let flags_array = flags_array.unwrap();if values.len() != flags_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("flags_array must be the same length as values array; got {} and {}",flags_array.len(),- values.len(),- )));+ values.len()+ );}Ok(Arc::new(@@ -427,6 +441,7 @@ where};let pattern = compile_and_cache_regex(+ "regexp_count",regex,Some(flags),&mut regex_cache,@@ -434,25 +449,25 @@ wherecount_matches(value, pattern, start_scalar)})- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(None, false, true) => {if values.len() != regex_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("regex_array must be the same length as values array; got {} and {}",regex_array.len(),- values.len(),- )));+ values.len()+ );}let start_array = start_array.unwrap();if values.len() != start_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("start_array must be the same length as values array; got {} and {}",start_array.len(),- values.len(),- )));+ values.len()+ );}Ok(Arc::new(@@ -463,40 +478,41 @@ where};let pattern = compile_and_cache_regex(+ "regexp_count",regex,flags_scalar,&mut regex_cache,)?;count_matches(value, pattern, start)})- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}(None, false, false) => {if values.len() != regex_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("regex_array must be the same length as values array; got {} and {}",regex_array.len(),- values.len(),- )));+ values.len()+ );}let start_array = start_array.unwrap();if values.len() != start_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("start_array must be the same length as values array; got {} and {}",start_array.len(),- values.len(),- )));+ values.len()+ );}let flags_array = flags_array.unwrap();if values.len() != flags_array.len() {- return Err(ArrowError::ComputeError(format!(+ return exec_err!("flags_array must be the same length as values array; got {} and {}",flags_array.len(),- values.len(),- )));+ values.len()+ );}Ok(Arc::new(@@ -511,11 +527,15 @@ wherereturn Ok(None);};- let pattern =- compile_and_cache_regex(regex, Some(flags), &mut regex_cache)?;+ let pattern = compile_and_cache_regex(+ "regexp_count",+ regex,+ Some(flags),+ &mut regex_cache,+ )?;count_matches(value, pattern, start)})- .collect::<Result<Int64Array, ArrowError>>()?,+ .collect::<Result<Int64Array>>()?,))}}@@ -525,16 +545,14 @@ fn count_matches(value: Option<&str>,pattern: &Regex,start: Option<i64>,-) -> Result<Option<i64>, ArrowError> {+) -> Result<Option<i64>> {// A NULL value or start position produces a NULL result.let (Some(value), Some(start)) = (value, start) else {return Ok(None);};if start < 1 {- return Err(ArrowError::ComputeError(- "regexp_count() requires start to be 1 based".to_string(),- ));+ return exec_err!("regexp_count() requires start to be 1 based");}let Some(byte_offset) = start_to_byte_offset(value, start) else {diff --git a/datafusion/functions/src/regex/regexpinstr.rs b/datafusion/functions/src/regex/regexpinstr.rsindex de460c56f6..5e1f5293b4 100644--- a/datafusion/functions/src/regex/regexpinstr.rs+++ b/datafusion/functions/src/regex/regexpinstr.rs@@ -23,7 +23,6 @@ use arrow::datatypes::{DataType, Int64Type};use arrow::datatypes::{DataType::Int64, DataType::LargeUtf8, DataType::Utf8, DataType::Utf8View,};-use arrow::error::ArrowError;use datafusion_common::{Result, ScalarValue, exec_err, internal_err};use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -175,7 +174,6 @@ pub fn regexp_instr_func(args: &[ArrayRef]) -> Result<ArrayRef> {if args_len > 4 { Some(&args[4]) } else { None },if args_len > 5 { Some(&args[5]) } else { None },)- .map_err(|e| e.into())}/// `arrow-rs` style implementation of `regexp_instr` function.@@ -203,7 +201,7 @@ fn regexp_instr(nth_array: Option<&dyn Datum>,flags_array: Option<&dyn Datum>,subexpr_array: Option<&dyn Datum>,-) -> Result<ArrayRef, ArrowError> {+) -> Result<ArrayRef> {let (regex_array, _) = regex_array.get();let start_array = start_array.map(|start| {let (start, _) = start.get();@@ -231,14 +229,16 @@ fn regexp_instr(None,subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),),- (Utf8, Utf8, Some(flags_array)) if *flags_array.data_type() == Utf8 => regexp_instr_inner(- &values.as_string::<i32>(),- ®ex_array.as_string::<i32>(),- start_array.map(|start| start.as_primitive::<Int64Type>()),- nth_array.map(|nth| nth.as_primitive::<Int64Type>()),- Some(&flags_array.as_string::<i32>()),- subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),- ),+ (Utf8, Utf8, Some(flags_array)) if *flags_array.data_type() == Utf8 => {+ regexp_instr_inner(+ &values.as_string::<i32>(),+ ®ex_array.as_string::<i32>(),+ start_array.map(|start| start.as_primitive::<Int64Type>()),+ nth_array.map(|nth| nth.as_primitive::<Int64Type>()),+ Some(&flags_array.as_string::<i32>()),+ subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),+ )+ }(LargeUtf8, LargeUtf8, None) => regexp_instr_inner(&values.as_string::<i64>(),®ex_array.as_string::<i64>(),@@ -247,14 +247,18 @@ fn regexp_instr(None,subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),),- (LargeUtf8, LargeUtf8, Some(flags_array)) if *flags_array.data_type() == LargeUtf8 => regexp_instr_inner(- &values.as_string::<i64>(),- ®ex_array.as_string::<i64>(),- start_array.map(|start| start.as_primitive::<Int64Type>()),- nth_array.map(|nth| nth.as_primitive::<Int64Type>()),- Some(&flags_array.as_string::<i64>()),- subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),- ),+ (LargeUtf8, LargeUtf8, Some(flags_array))+ if *flags_array.data_type() == LargeUtf8 =>+ {+ regexp_instr_inner(+ &values.as_string::<i64>(),+ ®ex_array.as_string::<i64>(),+ start_array.map(|start| start.as_primitive::<Int64Type>()),+ nth_array.map(|nth| nth.as_primitive::<Int64Type>()),+ Some(&flags_array.as_string::<i64>()),+ subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),+ )+ }(Utf8View, Utf8View, None) => regexp_instr_inner(&values.as_string_view(),®ex_array.as_string_view(),@@ -263,17 +267,21 @@ fn regexp_instr(None,subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),),- (Utf8View, Utf8View, Some(flags_array)) if *flags_array.data_type() == Utf8View => regexp_instr_inner(- &values.as_string_view(),- ®ex_array.as_string_view(),- start_array.map(|start| start.as_primitive::<Int64Type>()),- nth_array.map(|nth| nth.as_primitive::<Int64Type>()),- Some(&flags_array.as_string_view()),- subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),+ (Utf8View, Utf8View, Some(flags_array))+ if *flags_array.data_type() == Utf8View =>+ {+ regexp_instr_inner(+ &values.as_string_view(),+ ®ex_array.as_string_view(),+ start_array.map(|start| start.as_primitive::<Int64Type>()),+ nth_array.map(|nth| nth.as_primitive::<Int64Type>()),+ Some(&flags_array.as_string_view()),+ subexpr_array.map(|subexpr| subexpr.as_primitive::<Int64Type>()),+ )+ }+ _ => internal_err!(+ "regexp_instr() expected the input arrays to be of type Utf8, LargeUtf8, or Utf8View and the data types of the values, regex_array, and flags_array to match"),- _ => Err(ArrowError::ComputeError(- "regexp_instr() expected the input arrays to be of type Utf8, LargeUtf8, or Utf8View and the data types of the values, regex_array, and flags_array to match".to_string(),- )),}}@@ -284,7 +292,7 @@ fn regexp_instr_inner<'a, S>(nth_array: Option<&Int64Array>,flags_array: Option<&S>,subexp_array: Option<&Int64Array>,-) -> Result<ArrayRef, ArrowError>+) -> Result<ArrayRef>whereS: StringArrayType<'a>,{@@ -342,7 +350,7 @@ impl<'a> RegexCache<'a> {&mut self,regex: &'a str,flags: Option<&'a str>,- ) -> Result<&Regex, ArrowError> {+ ) -> Result<&Regex> {let key = (regex, flags);let index = match self.last {Some((last_key, index)) if last_key == key => index,@@ -350,7 +358,8 @@ impl<'a> RegexCache<'a> {let index = match self.indices.entry(key) {Entry::Occupied(entry) => *entry.get(),Entry::Vacant(entry) => {- self.compiled.push(compile_regex(regex, flags)?);+ self.compiled+ .push(compile_regex("regexp_instr", regex, flags)?);*entry.insert(self.compiled.len() - 1)}};@@ -372,17 +381,13 @@ fn get_index(start: i64,n: i64,subexpr: i64,-) -> Result<i64, ArrowError> {+) -> Result<i64> {if start < 1 {- return Err(ArrowError::ComputeError(- "regexp_instr() requires start to be 1-based".to_string(),- ));+ return exec_err!("regexp_instr() requires start to be 1-based");}if n < 1 {- return Err(ArrowError::ComputeError(- "N must be 1 or greater".to_string(),- ));+ return exec_err!("N must be 1 or greater");}let Some(byte_start_offset) = start_to_byte_offset(value, start) else {diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rsindex 08cd7b0651..546e1dcfb5 100644--- a/datafusion/functions/src/regex/regexplike.rs+++ b/datafusion/functions/src/regex/regexplike.rs@@ -17,14 +17,14 @@//! Regex expressions-use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, GenericStringArray};+use arrow::array::{+ Array, ArrayRef, AsArray, BooleanArray, GenericStringArray, StringArray,+};use arrow::compute::kernels::regexp;use arrow::datatypes::DataType;use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};use datafusion_common::types::logical_string;-use datafusion_common::{- Result, ScalarValue, arrow_datafusion_err, exec_err, internal_err, plan_err,-};+use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err};use datafusion_expr::{Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDFImpl,Signature, TypeSignature, TypeSignatureClass, Volatility, binary_expr, cast,@@ -36,7 +36,7 @@ use datafusion_expr::simplify::{};use datafusion_expr_common::operator::Operator;use datafusion_expr_common::type_coercion::binary::BinaryTypeCoercer;-use regex::{Error as RegexError, Regex, RegexBuilder};+use regex::{Error as RegexError, RegexBuilder};use std::borrow::Cow;use std::sync::Arc;@@ -377,24 +377,35 @@ fn regexp_like_array_scalar(};let flags = flags.filter(|flags| !flags.is_empty());let array = match values.data_type() {- Utf8 => {- let array = values.as_string::<i32>();- regexp::regexp_is_match_scalar(array, pattern, flags)?- }+ Utf8 => regexp::regexp_is_match_scalar(values.as_string::<i32>(), pattern, flags),Utf8View => {- let array = values.as_string_view();- regexp::regexp_is_match_scalar(array, pattern, flags)?+ regexp::regexp_is_match_scalar(values.as_string_view(), pattern, flags)}LargeUtf8 => {- let array = values.as_string::<i64>();- regexp::regexp_is_match_scalar(array, pattern, flags)?+ regexp::regexp_is_match_scalar(values.as_string::<i64>(), pattern, flags)}other => {return internal_err!("Unsupported data type {other:?} for function `regexp_like`");}- };+ }+ // The kernel compiles the pattern itself. Describe the scalar pattern and+ // flags as arrays of one value, so that a failure is explained the same+ // way as on the paths that pass arrays.+ .map_err(|error| {+ let patterns = StringArray::from(vec![pattern]);+ let flags = flags.map(|flags| StringArray::from(vec![flags]));+ super::explain_regexp_kernel_error(+ "regexp_like",+ error,+ // The kernel compiles the one pattern up front, whatever the+ // values are.+ None,+ &patterns,+ flags.as_ref().map(|flags| flags as &dyn Array),+ )+ })?;Ok(Arc::new(array))}@@ -414,20 +425,15 @@ fn regexp_like_scalar(let value = value.unwrap();let pattern = pattern.unwrap();- let pattern = match flags.filter(|flags| !flags.is_empty()) {- Some(flagz) => format!("(?{flagz}){pattern}"),- None => pattern.to_string(),- };+ let flags = flags.filter(|flags| !flags.is_empty());- let result = if pattern.is_empty() {+ // An empty pattern matches every value and needs no compilation. Every+ // other pattern is compiled exactly once, as on the paths that call a+ // kernel.+ let result = if pattern.is_empty() && flags.is_none() {true} else {- let re = Regex::new(pattern.as_str()).map_err(|e| {- datafusion_common::DataFusionError::Execution(format!(- "Regular expression did not compile: {e:?}"- ))- })?;- re.is_match(value)+ super::compile_regex("regexp_like", pattern, flags)?.is_match(value)};Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(result))))@@ -444,70 +450,74 @@ fn handle_regexp_like(let pattern = patterns.as_string::<i32>();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(Utf8View, Utf8View) => {let value = values.as_string_view();let pattern = patterns.as_string_view();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(Utf8View, LargeUtf8) => {let value = values.as_string_view();let pattern = patterns.as_string::<i64>();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(Utf8, Utf8) => {let value = values.as_string::<i32>();let pattern = patterns.as_string::<i32>();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(Utf8, Utf8View) => {let value = values.as_string::<i32>();let pattern = patterns.as_string_view();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(Utf8, LargeUtf8) => {let value = values.as_string::<i32>();let pattern = patterns.as_string::<i64>();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(LargeUtf8, Utf8) => {let value = values.as_string::<i64>();let pattern = patterns.as_string::<i32>();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(LargeUtf8, Utf8View) => {let value = values.as_string::<i64>();let pattern = patterns.as_string_view();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}(LargeUtf8, LargeUtf8) => {let value = values.as_string::<i64>();let pattern = patterns.as_string::<i64>();regexp::regexp_is_match(value, pattern, flags)- .map_err(|e| arrow_datafusion_err!(e))?}other => {return internal_err!("Unsupported data type {other:?} for function `regexp_like`");}- };+ }+ // Every arm hands its pattern to the kernel, which compiles it. Explain a+ // failure in one place, for every arm.+ .map_err(|error| {+ super::explain_regexp_kernel_error(+ "regexp_like",+ error,+ // The kernel compiles the pattern of a row only if that row has a+ // value.+ Some(values.as_ref()),+ patterns.as_ref(),+ flags.map(|flags| flags as &dyn Array),+ )+ })?;Ok(Arc::new(array) as ArrayRef)}diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rsindex 7ca8ba82d5..e6ffe04c27 100644--- a/datafusion/functions/src/regex/regexpmatch.rs+++ b/datafusion/functions/src/regex/regexpmatch.rs@@ -23,7 +23,7 @@ use arrow::datatypes::Field;use datafusion_common::Result;use datafusion_common::ScalarValue;use datafusion_common::exec_err;-use datafusion_common::{arrow_datafusion_err, plan_err};+use datafusion_common::plan_err;use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, TypeSignature};use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};use datafusion_macros::user_doc;@@ -185,7 +185,11 @@ fn regexp_match_scalar_pattern(args: &[ColumnarValue]) -> Result<Option<ArrayRefif !matches!(pattern.try_as_str(), Some(Some(_)))|| &pattern.data_type() != value_type|| flags.is_some_and(|flags| {- flags.try_as_str() == Some(Some("g")) || &flags.data_type() != value_type+ flags+ .try_as_str()+ .flatten()+ .is_some_and(|flags| flags.contains('g'))+ || &flags.data_type() != value_type}){return Ok(None);@@ -203,31 +207,62 @@ fn regexp_match_scalar_pattern(args: &[ColumnarValue]) -> Result<Option<ArrayRefflags.as_ref().map(|flags| flags as &dyn Datum),).map(Some)- .map_err(|e| arrow_datafusion_err!(e))+ // The kernel compiles the pattern. A scalar argument reaches it as an+ // array of one value.+ .map_err(|error| {+ super::explain_regexp_kernel_error(+ "regexp_match",+ error,+ // The kernel compiles the one pattern up front, whatever the+ // values are.+ None,+ pattern.get().0,+ flags.as_ref().map(|flags| flags.get().0),+ )+ })}pub fn regexp_match(args: &[ArrayRef]) -> Result<ArrayRef> {match args.len() {- 2 => regexp::regexp_match(&args[0], &args[1], None)- .map_err(|e| arrow_datafusion_err!(e)),+ 2 => regexp::regexp_match(&args[0], &args[1], None).map_err(|error| {+ super::explain_regexp_kernel_error(+ "regexp_match",+ error,+ Some(args[0].as_ref()),+ args[1].as_ref(),+ None,+ )+ }),3 => {match args[2].data_type() {DataType::Utf8View => {- if args[2].as_string_view().iter().any(|s| s == Some("g")) {+ if args[2]+ .as_string_view()+ .iter()+ .any(|s| s.is_some_and(|s| s.contains('g')))+ {return plan_err!("regexp_match() does not support the \"global\" option");}}DataType::Utf8 => {- if args[2].as_string::<i32>().iter().any(|s| s == Some("g")) {+ if args[2]+ .as_string::<i32>()+ .iter()+ .any(|s| s.is_some_and(|s| s.contains('g')))+ {return plan_err!("regexp_match() does not support the \"global\" option");}}DataType::LargeUtf8 => {- if args[2].as_string::<i64>().iter().any(|s| s == Some("g")) {+ if args[2]+ .as_string::<i64>()+ .iter()+ .any(|s| s.is_some_and(|s| s.contains('g')))+ {return plan_err!("regexp_match() does not support the \"global\" option");@@ -241,8 +276,15 @@ pub fn regexp_match(args: &[ArrayRef]) -> Result<ArrayRef> {}let flags = super::normalize_empty_flags(&args[2])?;- regexp::regexp_match(&args[0], &args[1], Some(&flags))- .map_err(|e| arrow_datafusion_err!(e))+ regexp::regexp_match(&args[0], &args[1], Some(&flags)).map_err(|error| {+ super::explain_regexp_kernel_error(+ "regexp_match",+ error,+ Some(args[0].as_ref()),+ args[1].as_ref(),+ Some(flags.as_ref()),+ )+ })}other => exec_err!("regexp_match was called with {other} arguments. It requires at least 2 and at most 3."diff --git a/datafusion/functions/src/regex/regexpreplace.rs b/datafusion/functions/src/regex/regexpreplace.rsindex f5bd6f182a..6ad138de15 100644--- a/datafusion/functions/src/regex/regexpreplace.rs+++ b/datafusion/functions/src/regex/regexpreplace.rs@@ -32,9 +32,7 @@ use datafusion_common::cast::{};use datafusion_common::exec_err;use datafusion_common::plan_err;-use datafusion_common::{- DataFusionError, Result, cast::as_generic_string_array, internal_err,-};+use datafusion_common::{Result, cast::as_generic_string_array, internal_err};use datafusion_expr::ColumnarValue;use datafusion_expr::TypeSignature;use datafusion_expr::function::Hint;@@ -43,6 +41,8 @@ use datafusion_expr::{};use datafusion_macros::user_doc;use regex::{CaptureLocations, Regex};++use super::compile_regex;use std::borrow::Cow;use std::collections::HashMap;use std::sync::{Arc, LazyLock};@@ -361,15 +361,15 @@ where// if patterns hashmap already has regexp then use else create and returnlet re = match patterns.get(pattern) {Some(re) => Ok(re),- None => match Regex::new(pattern) {- Ok(re) => {- patterns.insert(pattern.to_string(), re);- Ok(patterns.get(pattern).unwrap())- }- Err(err) => {- Err(DataFusionError::External(Box::new(err)))+ None => {+ match compile_regex("regexp_replace", pattern, None) {+ Ok(re) => {+ patterns.insert(pattern.to_string(), re);+ Ok(patterns.get(pattern).unwrap())+ }+ Err(err) => Err(err),}- },+ }};Some(re.map(|re| re.replace(string, replacement.as_str())))@@ -420,14 +420,16 @@ where// if patterns hashmap already has regexp then use else create and returnlet re = match patterns.get(&pattern) {Some(re) => Ok(re),- None => match Regex::new(pattern.as_str()) {+ None => match compile_regex(+ "regexp_replace",+ pattern.as_str(),+ None,+ ) {Ok(re) => {patterns.insert(pattern.clone(), re);Ok(patterns.get(&pattern).unwrap())}- Err(err) => {- Err(DataFusionError::External(Box::new(err)))- }+ Err(err) => Err(err),},};@@ -537,8 +539,7 @@ fn regexp_replace_static_pattern_replace<T: OffsetSizeTrait>(None => (pattern.to_string(), 1),};- let re =- Regex::new(&pattern).map_err(|err| DataFusionError::External(Box::new(err)))?;+ let re = compile_regex("regexp_replace", &pattern, None)?;// Replaces the posix groups in the replacement string// with rust ones.@@ -976,7 +977,7 @@ mod tests {let pattern_err = re.expect_err("broken pattern should have failed");assert_eq!(pattern_err.strip_backtrace(),- "External error: regex parse error:\n [\n ^\nerror: unclosed character class"+ "Execution error: Regular expression did not compile: regex parse error:\n [\n ^\nerror: unclosed character class");}diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rsindex 97417bc112..05176f7220 100644--- a/datafusion/optimizer/src/simplify_expressions/regex.rs+++ b/datafusion/optimizer/src/simplify_expressions/regex.rs@@ -16,7 +16,7 @@// under the License.use datafusion_common::tree_node::Transformed;-use datafusion_common::{DataFusionError, Result, ScalarValue};+use datafusion_common::{Result, ScalarValue, plan_err};use datafusion_expr::{BinaryExpr, Expr, Like, Operator, lit};use regex_syntax::hir::{Capture, Hir, HirKind, Literal, Look};@@ -98,11 +98,11 @@ pub fn simplify_regex_expr(}}Err(e) => {- // error out early since the execution may fail anyways- return Err(DataFusionError::Context(- "Invalid regex".to_owned(),- Box::new(DataFusionError::External(Box::new(e))),- ));+ // A literal pattern that does not compile is an error in the query+ // text, and it is known here, before execution. Report it with the+ // diagnosis of the `regex_syntax` crate, in the same shape as the+ // error that the regexp functions report at execution time.+ return plan_err!("Regular expression did not compile: {e}");}}diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.tomlindex 4276814fd0..a4f8ebf91b 100644--- a/datafusion/physical-expr-common/Cargo.toml+++ b/datafusion/physical-expr-common/Cargo.toml@@ -60,6 +60,7 @@ indexmap = { workspace = true }itertools = { workspace = true }parking_lot = { workspace = true }pin-project = { workspace = true }+regex = { workspace = true }[dev-dependencies]criterion = { workspace = true }diff --git a/datafusion/physical-expr-common/src/lib.rs b/datafusion/physical-expr-common/src/lib.rsindex b6eaacdca2..131bea24ff 100644--- a/datafusion/physical-expr-common/src/lib.rs+++ b/datafusion/physical-expr-common/src/lib.rs@@ -35,6 +35,7 @@ pub mod binary_view_map;pub mod datum;pub mod metrics;pub mod physical_expr;+pub mod regex;pub mod sort_expr;pub mod tree_node;pub mod utils;diff --git a/datafusion/physical-expr-common/src/regex.rs b/datafusion/physical-expr-common/src/regex.rsnew file mode 100644index 0000000000..e0c32f0473--- /dev/null+++ b/datafusion/physical-expr-common/src/regex.rs@@ -0,0 +1,195 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Compilation of the regular expressions of SQL.+//!+//! The regexp functions and the `~` family of operators share this module, so+//! that a pattern that does not compile is reported in one way, wherever the+//! pattern came from.++use arrow::array::{Array, AsArray, LargeStringArray, StringArray, StringViewArray};+use arrow::datatypes::DataType;+use arrow::error::ArrowError;+use datafusion_common::{+ DataFusionError, Result, arrow_datafusion_err, exec_datafusion_err, plan_err,+};+use regex::Regex;+use std::collections::HashMap;+use std::collections::hash_map::Entry;++/// Compiles `regex` with [`compile_regex`], keeping the compiled pattern in+/// `regex_cache` under the key `(regex, flags)`.+pub fn compile_and_cache_regex<'strings, 'cache>(+ function_name: &str,+ regex: &'strings str,+ flags: Option<&'strings str>,+ regex_cache: &'cache mut HashMap<(&'strings str, Option<&'strings str>), Regex>,+) -> Result<&'cache Regex>+where+ 'strings: 'cache,+{+ let result = match regex_cache.entry((regex, flags)) {+ Entry::Occupied(occupied_entry) => occupied_entry.into_mut(),+ Entry::Vacant(vacant_entry) => {+ let compiled = compile_regex(function_name, regex, flags)?;+ vacant_entry.insert(compiled)+ }+ };+ Ok(result)+}++/// Compiles `regex`, applying `flags` as inline regex flags.+///+/// `function_name` names the SQL function that the user called. It appears in+/// the error that reports an unsupported flag, so that every function reports+/// its own name.+///+/// A pattern that does not compile is reported as+/// [`DataFusionError::Execution`] carrying the diagnosis of the `regex` crate,+/// which names the position and the reason the pattern was rejected.+pub fn compile_regex(+ function_name: &str,+ regex: &str,+ flags: Option<&str>,+) -> Result<Regex> {+ let pattern = match flags {+ None | Some("") => regex.to_string(),+ Some(flags) => {+ if flags.contains('g') {+ return plan_err!(+ "{function_name}() does not support the \"global\" option"+ );+ }+ format!("(?{flags}){regex}")+ }+ };++ Regex::new(&pattern)+ .map_err(|e| exec_datafusion_err!("Regular expression did not compile: {e}"))+}++/// Explains a failure reported by one of the arrow regexp kernels.+///+/// The kernels compile the patterns themselves and report a pattern that does+/// not compile as an opaque [`ArrowError::ComputeError`]. This compiles the+/// patterns that were given to the kernel and reports the first one that does+/// not compile, with the diagnosis of the `regex` crate. The diagnosis of a+/// syntax error quotes the pattern that caused it.+///+/// This runs only after the kernel has failed, so a query that succeeds never+/// compiles a pattern twice.+///+/// `patterns` and `flags` are the arrays that the kernel received. An argument+/// that was a scalar is held as an array of one value, which applies to every+/// row. If every pattern compiles, the failure has a different cause and the+/// original error is kept.+///+/// `values` is the array of strings that the kernel matched against, given+/// only by the callers of a kernel that compiles the pattern of a row as it+/// reaches that row. Such a kernel produces NULL for a row whose value is+/// NULL without compiling that row's pattern, so a pattern that the kernel+/// never reached must not be reported as the one that failed. A kernel that+/// compiles a single pattern up front, before it reads any value, takes+/// `None`: it compiles that pattern whatever the values are.+///+/// This is `pub` only so that the crates that call the kernels can reach it.+// Not public API.+#[doc(hidden)]+pub fn explain_regexp_kernel_error(+ function_name: &str,+ error: ArrowError,+ values: Option<&dyn Array>,+ patterns: &dyn Array,+ flags: Option<&dyn Array>,+) -> DataFusionError {+ let Some(patterns) = StringValues::new(patterns) else {+ return arrow_datafusion_err!(error);+ };+ let flags = match flags.map(StringValues::new) {+ None => None,+ Some(Some(flags)) => Some(flags),+ // Flags of some other type are not what made the kernel fail.+ Some(None) => return arrow_datafusion_err!(error),+ };++ let rows = patterns+ .len()+ .max(flags.as_ref().map_or(0, StringValues::len))+ .max(values.map_or(0, Array::len));+ for row in 0..rows {+ // The kernel skips a row whose value is NULL, so it never compiled+ // the pattern of that row.+ if values.is_some_and(|values| row < values.len() && values.is_null(row)) {+ continue;+ }+ // A NULL pattern or NULL flags produce a NULL result, not an error.+ let Some(pattern) = patterns.broadcast_value(row) else {+ continue;+ };+ let flags = flags.as_ref().and_then(|flags| flags.broadcast_value(row));+ if let Err(error) = compile_regex(function_name, pattern, flags) {+ return error;+ }+ }++ arrow_datafusion_err!(error)+}++/// A string array of any of the three string types, read by row.+///+/// This borrows the array that the kernel received, so that explaining an+/// error reads the rows it needs and allocates nothing, however long the+/// array is.+enum StringValues<'a> {+ Utf8(&'a StringArray),+ LargeUtf8(&'a LargeStringArray),+ Utf8View(&'a StringViewArray),+}++impl<'a> StringValues<'a> {+ /// Borrows `array`, or returns `None` for an array of any other type.+ fn new(array: &'a dyn Array) -> Option<Self> {+ match array.data_type() {+ DataType::Utf8 => Some(Self::Utf8(array.as_string::<i32>())),+ DataType::LargeUtf8 => Some(Self::LargeUtf8(array.as_string::<i64>())),+ DataType::Utf8View => Some(Self::Utf8View(array.as_string_view())),+ _ => None,+ }+ }++ fn len(&self) -> usize {+ match self {+ Self::Utf8(array) => array.len(),+ Self::LargeUtf8(array) => array.len(),+ Self::Utf8View(array) => array.len(),+ }+ }++ /// Reads the value of `row`, treating an array of a single value as a+ /// scalar that applies to every row.+ fn broadcast_value(&self, row: usize) -> Option<&'a str> {+ let row = if self.len() == 1 { 0 } else { row };+ if row >= self.len() {+ return None;+ }+ match *self {+ Self::Utf8(array) => (!array.is_null(row)).then(|| array.value(row)),+ Self::LargeUtf8(array) => (!array.is_null(row)).then(|| array.value(row)),+ Self::Utf8View(array) => (!array.is_null(row)).then(|| array.value(row)),+ }+ }+}diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rsindex 94612af9f6..a0e7ac3fc8 100644--- a/datafusion/physical-expr/src/expressions/binary/kernels.rs+++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs@@ -28,6 +28,7 @@ use arrow::compute::kernels::comparison::{regexp_is_match, regexp_is_match_scalause arrow::datatypes::DataType;use datafusion_common::{Result, ScalarValue};use datafusion_common::{exec_err, internal_err, plan_err};+use datafusion_physical_expr_common::regex::explain_regexp_kernel_error;use std::sync::Arc;@@ -159,6 +160,16 @@ create_left_integral_dyn_scalar_kernel!(bitwise_shift_left_scalar);+/// The SQL spelling of the operator, for error messages.+fn operator_name(not_match: bool, case_insensitive: bool) -> &'static str {+ match (not_match, case_insensitive) {+ (false, false) => "~",+ (false, true) => "~*",+ (true, false) => "!~",+ (true, true) => "!~*",+ }+}+/// Invoke a compute kernel on a pair of binary data arrays with flagsmacro_rules! regexp_is_match_flag {($LEFT:expr, $RIGHT:expr, $ARRAYTYPE:ident, $NOT:expr, $FLAG:expr) => {{@@ -183,7 +194,19 @@ macro_rules! regexp_is_match_flag {} else {None};- let mut array = regexp_is_match(ll, rr, flag.as_ref())?;+ // The kernel compiles the pattern of each row. A pattern that does+ // not compile is explained only after the kernel has failed.+ let mut array = regexp_is_match(ll, rr, flag.as_ref()).map_err(|error| {+ explain_regexp_kernel_error(+ operator_name($NOT, $FLAG),+ error,+ // The kernel compiles the pattern of a row only if that row+ // has a value.+ Some(ll as &dyn Array),+ rr,+ flag.as_ref().map(|flag| flag as &dyn Array),+ )+ })?;if $NOT {array = not(&array).unwrap();}@@ -233,7 +256,22 @@ macro_rules! regexp_is_match_flag_scalar {}Ok(Arc::new(array))}- Err(e) => internal_err!("failed to call 'regex_match_dyn_scalar' {}", e),+ Err(error) => {+ // Describe the scalar pattern and flags as arrays of one+ // value, so that the failure is explained the same way as+ // on the paths that pass arrays.+ let patterns = StringArray::from(vec![string_value]);+ let flags = flag.map(|flag| StringArray::from(vec![flag]));+ Err(explain_regexp_kernel_error(+ operator_name($NOT, $FLAG),+ error,+ // The kernel compiles the one pattern up front,+ // whatever the values are.+ None,+ &patterns,+ flags.as_ref().map(|flags| flags as &dyn Array),+ ))+ }}} else {internal_err!(diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_count.slt b/datafusion/sqllogictest/test_files/regexp/regexp_count.sltindex 1f1500a6ca..9c06cbbd91 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_count.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_count.slt@@ -91,16 +91,13 @@ SELECT regexp_count('abc', 'x*', 4);----1-statement error-External error: query failed: DataFusion error: Arrow error: Compute error: regexp_count() requires start to be 1 based+statement error DataFusion error: Execution error: regexp_count\(\) requires start to be 1 basedSELECT regexp_count('123123123123', '123', 0);-statement error-External error: query failed: DataFusion error: Arrow error: Compute error: regexp_count() requires start to be 1 based+statement error DataFusion error: Execution error: regexp_count\(\) requires start to be 1 basedSELECT regexp_count('123123123123', '123', -3);-statement error-External error: statement failed: DataFusion error: Arrow error: Compute error: regexp_count() does not support global flag+statement error DataFusion error: Error during planning: regexp_count\(\) does not support the "global" optionSELECT regexp_count('123123123123', '123', 1, 'g');query I@@ -394,6 +391,32 @@ NULLNULLNULL+#+# Invalid patterns and flags report the reason from the regex parser+#++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_count(str, 'a(b') from empty_table;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_count('abc', 'a(b');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_count('abc', 'a', 1, 'z');++query error DataFusion error: Error during planning: regexp_count\(\) does not support the "global" option+SELECT regexp_count('abc', 'a', 1, 'g');++# A pattern that varies per row is reported in the same way.+statement ok+CREATE TABLE t_invalid_pattern(str varchar, pattern varchar) AS VALUES ('abc', 'a(b');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_count(str, pattern) FROM t_invalid_pattern;++statement ok+DROP TABLE t_invalid_pattern;+statement okdrop table t_stringview;diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_instr.slt b/datafusion/sqllogictest/test_files/regexp/regexp_instr.sltindex 14e1ffbdf7..18a07afaf8 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_instr.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_instr.slt@@ -73,13 +73,13 @@ SELECT----11-statement error DataFusion error: Arrow error: Compute error: regexp_instr\(\) requires start to be 1-based+statement error DataFusion error: Execution error: regexp_instr\(\) requires start to be 1-basedSELECT regexp_instr('123123123123', '123', 0);-statement error DataFusion error: Arrow error: Compute error: regexp_instr\(\) requires start to be 1-based+statement error DataFusion error: Execution error: regexp_instr\(\) requires start to be 1-basedSELECT regexp_instr('123123123123', '123', -3);-statement error DataFusion error: Arrow error: Compute error: N must be 1 or greater+statement error DataFusion error: Execution error: N must be 1 or greaterSELECT regexp_instr('abcabcabc', 'abc', 1, 0);query I@@ -246,6 +246,32 @@ SELECT regexp_instr(str, pattern) FROM t_alternating_pattern;41+#+# Invalid patterns and flags report the reason from the regex parser+#++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_instr(str, 'a(b') FROM empty_table;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_instr('abc', 'a(b');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_instr('abc', 'a', 1, 1, 'z');++query error DataFusion error: Error during planning: regexp_instr\(\) does not support the "global" option+SELECT regexp_instr('abc', 'a', 1, 1, 'g');++# A pattern that varies per row is reported in the same way.+statement ok+CREATE TABLE t_invalid_pattern(str varchar, pattern varchar) AS VALUES ('abc', 'a(b');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error:+SELECT regexp_instr(str, pattern) FROM t_invalid_pattern;++statement ok+DROP TABLE t_invalid_pattern;+statement okDROP TABLE t_stringview;diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt b/datafusion/sqllogictest/test_files/regexp/regexp_like.sltindex 4a649a1966..225fdb2215 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt@@ -148,7 +148,7 @@ SELECT regexp_like('bb-1', '.*-(\d)', 'g');query error Error during planning: regexp_like\(\) does not support the "global" optionSELECT regexp_like('bb-1', '.*-(\d)', 'g');-query error Arrow error: Compute error: Regular expression did not compile: CompiledTooBig\(10485760\)+query error DataFusion error: Execution error: Regular expression did not compile: Compiled regex exceeds size limit of 10485760 bytesSELECT regexp_like('aaaaa', 'a{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}');# look-around is not supported and will just return false@@ -401,3 +401,96 @@ true truestatement okDROP TABLE regexp_empty_flags;++# A literal pattern without flags, like every `~` operator, is simplified at+# planning time, so a pattern that does not compile is reported there, with the+# same wording that the function uses at execution time.+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Error during planning: Regular expression did not compile: regex parse error+SELECT regexp_like('abc', 'a(b');++statement ok+CREATE TABLE t_invalid_pattern(str varchar, pattern varchar) AS VALUES ('abc', 'a(b'), ('abc', NULL);++query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Error during planning: Regular expression did not compile: regex parse error+SELECT regexp_like(str, 'a(b') FROM t_invalid_pattern;++query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Error during planning: Regular expression did not compile: regex parse error+SELECT str ~ 'a(b' FROM t_invalid_pattern;++# Flags that have no operator equivalent keep the call as a function, which+# gives the same diagnosis at execution time.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_like('abc', 'a(b', 'm');++# An unknown flag makes the pattern fail to compile in the same way.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_like('abc', 'a', 'z');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_like(str, 'a', 'z') FROM t_invalid_pattern;++# A pattern that varies per row is simplified to the `~` operator, which+# compiles each row's pattern inside the arrow kernel. The kernel failure is+# explained in the same way.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_like(str, pattern) FROM t_invalid_pattern;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT str ~ pattern FROM t_invalid_pattern;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT str ~* pattern FROM t_invalid_pattern;++# A NULL value or a NULL pattern gives NULL, and compiles nothing.+query B+SELECT regexp_like(NULL, 'a(b');+----+NULL++query B+SELECT regexp_like('abc', NULL);+----+NULL++statement ok+DROP TABLE t_invalid_pattern;++# A row whose value or whose pattern is NULL gives NULL, and the kernel never+# compiles that row's pattern. The diagnosis therefore names the pattern that+# made the kernel fail, not the one of a row the kernel skipped.+statement ok+CREATE TABLE t_null_value(str varchar, pattern varchar) AS VALUES (NULL, 'a(b'), ('abc', NULL), ('abc', 'c[d');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error[\s\S]*unclosed character class+SELECT str ~ pattern FROM t_null_value;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error[\s\S]*unclosed character class+SELECT regexp_like(str, pattern) FROM t_null_value;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error[\s\S]*unclosed character class+SELECT regexp_like(str, pattern, 'm') FROM t_null_value;++# The control: on their own, the rows that the kernel skips give NULL, and the+# pattern that does not compile is never reported.+query B+SELECT str ~ pattern FROM t_null_value WHERE str IS NULL;+----+NULL++query B+SELECT str ~ pattern FROM t_null_value WHERE pattern IS NULL;+----+NULL++query B+SELECT regexp_like(str, pattern, 'm') FROM t_null_value WHERE str IS NULL;+----+NULL++query B+SELECT regexp_like(str, pattern, 'm') FROM t_null_value WHERE pattern IS NULL;+----+NULL++statement ok+DROP TABLE t_null_value;diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt b/datafusion/sqllogictest/test_files/regexp/regexp_match.sltindex b339812c12..a62477d6ba 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt@@ -125,7 +125,7 @@ SELECT regexp_match('bb-1', '.*-(\d)', 'g');query error Error during planning: regexp_match\(\) does not support the "global" optionSELECT regexp_match('bb-1', '.*-(\d)', 'g');-query error Arrow error: Compute error: Regular expression did not compile: CompiledTooBig\(10485760\)+query error DataFusion error: Execution error: Regular expression did not compile: Compiled regex exceeds size limit of 10485760 bytesSELECT regexp_match('aaaaa', 'a{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}{5}');# look-around is not supported and will just return null@@ -232,3 +232,79 @@ NULL [bar]statement okDROP TABLE regexp_empty_flags;++# A pattern that does not compile is reported with the diagnosis of the `regex`+# crate. The kernel compiles the pattern, and the diagnosis is produced only+# after the kernel has failed, so a query that succeeds compiles it once.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_match('abc', 'a(b');++statement ok+CREATE TABLE t_invalid_pattern(str varchar, pattern varchar) AS VALUES ('abc', 'a(b'), ('abc', NULL);++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_match(str, 'a(b') FROM t_invalid_pattern;++# A pattern that varies per row is reported in the same way.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_match(str, pattern) FROM t_invalid_pattern;++# An unknown flag makes the pattern fail to compile in the same way.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_match('abc', 'a', 'z');++# A NULL value or a NULL pattern gives NULL, and compiles nothing.+query ?+SELECT regexp_match(NULL, 'a(b');+----+NULL++query ?+SELECT regexp_match('abc', NULL);+----+NULL++query ?+SELECT regexp_match(str, pattern) FROM t_invalid_pattern WHERE pattern IS NULL;+----+NULL++statement ok+DROP TABLE t_invalid_pattern;++# A row whose value or whose pattern is NULL gives NULL, and the kernel never+# compiles that row's pattern. The diagnosis therefore names the pattern that+# made the kernel fail, not the one of a row the kernel skipped.+statement ok+CREATE TABLE t_null_value(str varchar, pattern varchar) AS VALUES (NULL, 'a(b'), ('abc', NULL), ('abc', 'c[d');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error[\s\S]*unclosed character class+SELECT regexp_match(str, pattern) FROM t_null_value;++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error[\s\S]*unclosed character class+SELECT regexp_match(str, pattern, 'm') FROM t_null_value;++# The control: on their own, the rows that the kernel skips give NULL, and the+# pattern that does not compile is never reported.+query ?+SELECT regexp_match(str, pattern) FROM t_null_value WHERE str IS NULL;+----+NULL++query ?+SELECT regexp_match(str, pattern) FROM t_null_value WHERE pattern IS NULL;+----+NULL++query ?+SELECT regexp_match(str, pattern, 'm') FROM t_null_value WHERE str IS NULL;+----+NULL++query ?+SELECT regexp_match(str, pattern, 'm') FROM t_null_value WHERE pattern IS NULL;+----+NULL++statement ok+DROP TABLE t_null_value;diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_replace.slt b/datafusion/sqllogictest/test_files/regexp/regexp_replace.sltindex 61f41eb2a5..937c41dcf5 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_replace.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_replace.slt@@ -274,3 +274,28 @@ not-a-url true# cleanupstatement okDROP TABLE regexp_replace_optimized_cases;++# A pattern that does not compile is reported with the diagnosis of the `regex`+# crate, in the same shape as the other regexp functions.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_replace('abc', 'a(b', 'x');++statement ok+CREATE TABLE t_invalid_pattern(str varchar, pattern varchar) AS VALUES ('abc', 'a(b');++query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_replace(str, pattern, 'x') FROM t_invalid_pattern;++statement ok+DROP TABLE t_invalid_pattern;++# An unknown flag makes the pattern fail to compile in the same way.+query error DataFusion error: Execution error: Regular expression did not compile: regex parse error+SELECT regexp_replace('abc', 'a', 'x', 'z');++# The "global" flag is what regexp_replace uses to replace every match, so it+# stays supported here.+query T+SELECT regexp_replace('abcabc', 'b', 'x', 'g');+----+axcaxc