Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
test(docreader): add unit tests for utils.split text-splitting helpers
Jay0130-a54cb6e9f9be8
diff --git a/docreader/tests/test_utils_split.py b/docreader/tests/test_utils_split.pynew file mode 100644index 000000000..1c901842b--- /dev/null+++ b/docreader/tests/test_utils_split.py@@ -0,0 +1,43 @@+import unittest++from docreader.utils.split import (+ match_by_regex,+ split_by_char,+ split_by_regex,+ split_by_sep,+ split_text_keep_separator,+)+++class SplitUtilsTest(unittest.TestCase):+ def test_split_text_keep_separator_basic(self):+ self.assertEqual(+ split_text_keep_separator("Hello\nWorld\nTest", "\n"),+ ["Hello", "\nWorld", "\nTest"],+ )++ def test_split_text_keep_separator_no_separator(self):+ self.assertEqual(split_text_keep_separator("no sep", ","), ["no sep"])++ def test_split_text_keep_separator_filters_empty_parts(self):+ self.assertEqual(split_text_keep_separator("a,,b", ","), ["a", ",", ",b"])++ def test_split_by_sep_keep(self):+ self.assertEqual(split_by_sep(",")("a,b,c"), ["a", ",b", ",c"])++ def test_split_by_sep_discard(self):+ self.assertEqual(split_by_sep(",", keep_sep=False)("a,,b"), ["a", "", "b"])++ def test_split_by_char(self):+ self.assertEqual(split_by_char()("abc"), ["a", "b", "c"])++ def test_split_by_regex_keeps_separators(self):+ self.assertEqual(split_by_regex("\n")("a\nb\nc"), ["a", "\n", "b", "\n", "c"])++ def test_match_by_regex(self):+ self.assertTrue(match_by_regex(r"^\d+")("123"))+ self.assertFalse(match_by_regex(r"^\d+")("abc"))+++if __name__ == "__main__":+ unittest.main()