Apache DataFusion SQL Query Engine
perf: optimize char -> byte offset mapping in `regexp_count` (#24153)
Neil Conway33ad1cc14611
diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rsindex 75cc5d9514..6724171203 100644--- a/datafusion/functions/src/regex/mod.rs+++ b/datafusion/functions/src/regex/mod.rs@@ -146,6 +146,22 @@ whereOk(result)}+/// Maps `start`, a 1-based character position, to a byte offset in `value`.+/// Positions `1..=n` (for an `n`-character string) map to the corresponding+/// character's first byte; position `n + 1`, the end of the string, maps to+/// `value.len()`. Returns `None` for larger positions. Callers must validate+/// `start >= 1`.+pub(crate) fn start_to_byte_offset(value: &str, start: i64) -> Option<usize> {+ // If `start - 1` does not fit in `usize`, it is necessarily past the end+ // of the string.+ let start_index = usize::try_from(start - 1).ok()?;+ value+ .char_indices()+ .map(|(offset, _)| offset)+ .chain(std::iter::once(value.len()))+ .nth(start_index)+}+pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowError> {let pattern = match flags {None | Some("") => regex.to_string(),@@ -164,3 +180,32 @@ pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowErrArrowError::ComputeError(format!("Regular expression did not compile: {pattern}"))})}++#[cfg(test)]+mod tests {+ use super::start_to_byte_offset;++ #[test]+ fn start_to_byte_offset_ascii() {+ assert_eq!(start_to_byte_offset("abc", 1), Some(0));+ assert_eq!(start_to_byte_offset("abc", 3), Some(2));+ // The end of the string is a valid position.+ assert_eq!(start_to_byte_offset("abc", 4), Some(3));+ assert_eq!(start_to_byte_offset("abc", 5), None);+ assert_eq!(start_to_byte_offset("abc", i64::MAX), None);+ }++ #[test]+ fn start_to_byte_offset_empty_string() {+ assert_eq!(start_to_byte_offset("", 1), Some(0));+ assert_eq!(start_to_byte_offset("", 2), None);+ }++ #[test]+ fn start_to_byte_offset_multibyte() {+ assert_eq!(start_to_byte_offset("πa", 1), Some(0));+ assert_eq!(start_to_byte_offset("πa", 2), Some(4));+ assert_eq!(start_to_byte_offset("πa", 3), Some(5));+ assert_eq!(start_to_byte_offset("πa", 4), None);+ }+}diff --git a/datafusion/functions/src/regex/regexpcount.rs b/datafusion/functions/src/regex/regexpcount.rsindex 40d9ba05e5..0ecedc7cbb 100644--- a/datafusion/functions/src/regex/regexpcount.rs+++ b/datafusion/functions/src/regex/regexpcount.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use crate::regex::{compile_and_cache_regex, compile_regex};+use crate::regex::{compile_and_cache_regex, compile_regex, start_to_byte_offset};use arrow::array::{Array, ArrayRef, AsArray, Datum, Int64Array, StringArrayType};use arrow::datatypes::{DataType, Int64Type};use arrow::datatypes::{@@ -560,27 +560,10 @@ fn count_matches());}- let char_len = value.chars().count();- let start_index = (start as usize).saturating_sub(1);-- if start_index > char_len {+ let Some(byte_offset) = start_to_byte_offset(value, start) else {return Ok(0);- }-- // Find the byte offset for the start position (1-based character index)- let byte_offset = if start_index == char_len {- value.len()- } else {- value- .char_indices()- .nth(start_index)- .map(|(idx, _)| idx)- .unwrap_or(value.len())};-- // Use string slicing instead of collecting chars into a new String- let find_slice = &value[byte_offset..];- let count = pattern.find_iter(find_slice).count();+ let count = pattern.find_iter(&value[byte_offset..]).count();Ok(count as i64)} else {let count = pattern.find_iter(value).count();diff --git a/datafusion/functions/src/regex/regexpinstr.rs b/datafusion/functions/src/regex/regexpinstr.rsindex e8a8f5286b..30385672df 100644--- a/datafusion/functions/src/regex/regexpinstr.rs+++ b/datafusion/functions/src/regex/regexpinstr.rs@@ -34,7 +34,7 @@ use std::collections::HashMap;use std::collections::hash_map::Entry;use std::sync::Arc;-use crate::regex::compile_regex;+use crate::regex::{compile_regex, start_to_byte_offset};#[user_doc(doc_section(label = "Regular Expression Functions"),@@ -383,17 +383,7 @@ fn get_index());}- let Ok(start_index) = usize::try_from(start - 1) else {- return Ok(0);- };- // Include the terminal byte boundary so an empty pattern can match after- // the last character, including in an empty string.- let Some(byte_start_offset) = value- .char_indices()- .map(|(offset, _)| offset)- .chain(std::iter::once(value.len()))- .nth(start_index)- else {+ let Some(byte_start_offset) = start_to_byte_offset(value, start) else {return Ok(0);};let search_slice = &value[byte_start_offset..];diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_count.slt b/datafusion/sqllogictest/test_files/regexp/regexp_count.sltindex c87c194fa6..1fd43eeb46 100644--- a/datafusion/sqllogictest/test_files/regexp/regexp_count.slt+++ b/datafusion/sqllogictest/test_files/regexp/regexp_count.slt@@ -76,6 +76,21 @@ SELECT regexp_count('abc', '', 5);----0+query I+SELECT regexp_count('', '');+----+1++query I+SELECT regexp_count('π', '', 2);+----+1++query I+SELECT regexp_count('abc', 'x*', 4);+----+1+statement errorExternal error: query failed: DataFusion error: Arrow error: Compute error: regexp_count() requires start to be 1 basedSELECT regexp_count('123123123123', '123', 0);