Skip to content

Apache DataFusion SQL Query Engine

perf: optimize char -> byte offset mapping in `regexp_count` (#24153)

Neil Conway33ad1cc14611
diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rs
index 75cc5d9514..6724171203 100644
--- a/datafusion/functions/src/regex/mod.rs
+++ b/datafusion/functions/src/regex/mod.rs
@@ -146,6 +146,22 @@ where
Ok(result)
}
+/// Maps `start`, a 1-based character position, to a byte offset in `value`.
+/// Positions `1..=n` (for an `n`-character string) map to the corresponding
+/// character's first byte; position `n + 1`, the end of the string, maps to
+/// `value.len()`. Returns `None` for larger positions. Callers must validate
+/// `start >= 1`.
+pub(crate) fn start_to_byte_offset(value: &str, start: i64) -> Option<usize> {
+ // If `start - 1` does not fit in `usize`, it is necessarily past the end
+ // of the string.
+ let start_index = usize::try_from(start - 1).ok()?;
+ value
+ .char_indices()
+ .map(|(offset, _)| offset)
+ .chain(std::iter::once(value.len()))
+ .nth(start_index)
+}
+
pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowError> {
let pattern = match flags {
None | Some("") => regex.to_string(),
@@ -164,3 +180,32 @@ pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowErr
ArrowError::ComputeError(format!("Regular expression did not compile: {pattern}"))
})
}
+
+#[cfg(test)]
+mod tests {
+ use super::start_to_byte_offset;
+
+ #[test]
+ fn start_to_byte_offset_ascii() {
+ assert_eq!(start_to_byte_offset("abc", 1), Some(0));
+ assert_eq!(start_to_byte_offset("abc", 3), Some(2));
+ // The end of the string is a valid position.
+ assert_eq!(start_to_byte_offset("abc", 4), Some(3));
+ assert_eq!(start_to_byte_offset("abc", 5), None);
+ assert_eq!(start_to_byte_offset("abc", i64::MAX), None);
+ }
+
+ #[test]
+ fn start_to_byte_offset_empty_string() {
+ assert_eq!(start_to_byte_offset("", 1), Some(0));
+ assert_eq!(start_to_byte_offset("", 2), None);
+ }
+
+ #[test]
+ fn start_to_byte_offset_multibyte() {
+ assert_eq!(start_to_byte_offset("πŸ˜€a", 1), Some(0));
+ assert_eq!(start_to_byte_offset("πŸ˜€a", 2), Some(4));
+ assert_eq!(start_to_byte_offset("πŸ˜€a", 3), Some(5));
+ assert_eq!(start_to_byte_offset("πŸ˜€a", 4), None);
+ }
+}
diff --git a/datafusion/functions/src/regex/regexpcount.rs b/datafusion/functions/src/regex/regexpcount.rs
index 40d9ba05e5..0ecedc7cbb 100644
--- a/datafusion/functions/src/regex/regexpcount.rs
+++ b/datafusion/functions/src/regex/regexpcount.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::regex::{compile_and_cache_regex, compile_regex};
+use crate::regex::{compile_and_cache_regex, compile_regex, start_to_byte_offset};
use arrow::array::{Array, ArrayRef, AsArray, Datum, Int64Array, StringArrayType};
use arrow::datatypes::{DataType, Int64Type};
use arrow::datatypes::{
@@ -560,27 +560,10 @@ fn count_matches(
));
}
- let char_len = value.chars().count();
- let start_index = (start as usize).saturating_sub(1);
-
- if start_index > char_len {
+ let Some(byte_offset) = start_to_byte_offset(value, start) else {
return Ok(0);
- }
-
- // Find the byte offset for the start position (1-based character index)
- let byte_offset = if start_index == char_len {
- value.len()
- } else {
- value
- .char_indices()
- .nth(start_index)
- .map(|(idx, _)| idx)
- .unwrap_or(value.len())
};
-
- // Use string slicing instead of collecting chars into a new String
- let find_slice = &value[byte_offset..];
- let count = pattern.find_iter(find_slice).count();
+ let count = pattern.find_iter(&value[byte_offset..]).count();
Ok(count as i64)
} else {
let count = pattern.find_iter(value).count();
diff --git a/datafusion/functions/src/regex/regexpinstr.rs b/datafusion/functions/src/regex/regexpinstr.rs
index e8a8f5286b..30385672df 100644
--- a/datafusion/functions/src/regex/regexpinstr.rs
+++ b/datafusion/functions/src/regex/regexpinstr.rs
@@ -34,7 +34,7 @@ use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::sync::Arc;
-use crate::regex::compile_regex;
+use crate::regex::{compile_regex, start_to_byte_offset};
#[user_doc(
doc_section(label = "Regular Expression Functions"),
@@ -383,17 +383,7 @@ fn get_index(
));
}
- let Ok(start_index) = usize::try_from(start - 1) else {
- return Ok(0);
- };
- // Include the terminal byte boundary so an empty pattern can match after
- // the last character, including in an empty string.
- let Some(byte_start_offset) = value
- .char_indices()
- .map(|(offset, _)| offset)
- .chain(std::iter::once(value.len()))
- .nth(start_index)
- else {
+ let Some(byte_start_offset) = start_to_byte_offset(value, start) else {
return Ok(0);
};
let search_slice = &value[byte_start_offset..];
diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_count.slt b/datafusion/sqllogictest/test_files/regexp/regexp_count.slt
index c87c194fa6..1fd43eeb46 100644
--- a/datafusion/sqllogictest/test_files/regexp/regexp_count.slt
+++ b/datafusion/sqllogictest/test_files/regexp/regexp_count.slt
@@ -76,6 +76,21 @@ SELECT regexp_count('abc', '', 5);
----
0
+query I
+SELECT regexp_count('', '');
+----
+1
+
+query I
+SELECT regexp_count('πŸ˜€', '', 2);
+----
+1
+
+query I
+SELECT regexp_count('abc', 'x*', 4);
+----
+1
+
statement error
External error: query failed: DataFusion error: Arrow error: Compute error: regexp_count() requires start to be 1 based
SELECT regexp_count('123123123123', '123', 0);