Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): preserve regex semantics when splitting text (#4669)

LingYif40d9697923e
diff --git a/docreader/tests/test_utils_split.py b/docreader/tests/test_utils_split.py
index 1c901842b..e763ee058 100644
--- a/docreader/tests/test_utils_split.py
+++ b/docreader/tests/test_utils_split.py
@@ -34,6 +34,20 @@ class SplitUtilsTest(unittest.TestCase):
def test_split_by_regex_keeps_separators(self):
self.assertEqual(split_by_regex("\n")("a\nb\nc"), ["a", "\n", "b", "\n", "c"])
+ def test_split_by_regex_keeps_nested_captures_once(self):
+ self.assertEqual(
+ split_by_regex(r"(([,;]))")("a,b;c"), ["a", ",", "b", ";", "c"]
+ )
+
+ def test_split_by_regex_preserves_backreferences(self):
+ self.assertEqual(split_by_regex(r"(\w)\1")("book"), ["b", "oo", "k"])
+
+ def test_split_by_regex_accepts_inline_flags(self):
+ self.assertEqual(split_by_regex(r"(?i)sep")("aSePb"), ["a", "SeP", "b"])
+
+ def test_split_by_regex_keeps_zero_width_boundaries(self):
+ self.assertEqual(split_by_regex(r"(?=b)")("abc"), ["a", "bc"])
+
def test_match_by_regex(self):
self.assertTrue(match_by_regex(r"^\d+")("123"))
self.assertFalse(match_by_regex(r"^\d+")("abc"))
diff --git a/docreader/utils/split.py b/docreader/utils/split.py
index 4950f7ac2..1e299b877 100644
--- a/docreader/utils/split.py
+++ b/docreader/utils/split.py
@@ -57,12 +57,21 @@ def split_by_regex(regex: str) -> Callable[[str], List[str]]:
Returns:
A callable function that takes text and returns a list of split strings
- The regex pattern is captured, so the separators are included in the result
+ The full separator matches are included in the result.
"""
- # Compile regex with capturing group to keep separators in result
- pattern = re.compile(f"({regex})")
- # Split by pattern and filter out None/empty values
- return lambda text: list(filter(None, pattern.split(text)))
+ # Preserve capture numbering and inline flags in the supplied pattern.
+ pattern = re.compile(regex)
+
+ def split(text: str) -> List[str]:
+ parts = []
+ start = 0
+ for match in pattern.finditer(text):
+ parts.extend((text[start:match.start()], match.group()))
+ start = match.end()
+ parts.append(text[start:])
+ return list(filter(None, parts))
+
+ return split
def match_by_regex(regex: str) -> Callable[[str], bool]: