Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): preserve regex semantics when splitting text (#4669)
LingYif40d9697923e
diff --git a/docreader/tests/test_utils_split.py b/docreader/tests/test_utils_split.pyindex 1c901842b..e763ee058 100644--- a/docreader/tests/test_utils_split.py+++ b/docreader/tests/test_utils_split.py@@ -34,6 +34,20 @@ class SplitUtilsTest(unittest.TestCase):def test_split_by_regex_keeps_separators(self):self.assertEqual(split_by_regex("\n")("a\nb\nc"), ["a", "\n", "b", "\n", "c"])+ def test_split_by_regex_keeps_nested_captures_once(self):+ self.assertEqual(+ split_by_regex(r"(([,;]))")("a,b;c"), ["a", ",", "b", ";", "c"]+ )++ def test_split_by_regex_preserves_backreferences(self):+ self.assertEqual(split_by_regex(r"(\w)\1")("book"), ["b", "oo", "k"])++ def test_split_by_regex_accepts_inline_flags(self):+ self.assertEqual(split_by_regex(r"(?i)sep")("aSePb"), ["a", "SeP", "b"])++ def test_split_by_regex_keeps_zero_width_boundaries(self):+ self.assertEqual(split_by_regex(r"(?=b)")("abc"), ["a", "bc"])+def test_match_by_regex(self):self.assertTrue(match_by_regex(r"^\d+")("123"))self.assertFalse(match_by_regex(r"^\d+")("abc"))diff --git a/docreader/utils/split.py b/docreader/utils/split.pyindex 4950f7ac2..1e299b877 100644--- a/docreader/utils/split.py+++ b/docreader/utils/split.py@@ -57,12 +57,21 @@ def split_by_regex(regex: str) -> Callable[[str], List[str]]:Returns:A callable function that takes text and returns a list of split strings- The regex pattern is captured, so the separators are included in the result+ The full separator matches are included in the result."""- # Compile regex with capturing group to keep separators in result- pattern = re.compile(f"({regex})")- # Split by pattern and filter out None/empty values- return lambda text: list(filter(None, pattern.split(text)))+ # Preserve capture numbering and inline flags in the supplied pattern.+ pattern = re.compile(regex)++ def split(text: str) -> List[str]:+ parts = []+ start = 0+ for match in pattern.finditer(text):+ parts.extend((text[start:match.start()], match.group()))+ start = match.end()+ parts.append(text[start:])+ return list(filter(None, parts))++ return splitdef match_by_regex(regex: str) -> Callable[[str], bool]: