The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
text-splitters[patch]: Modified SpacyTextSplitter to fully keep whitespace when strip_whitespace is false (#23272)
Matthew DeGenaro66828f4ecc25
diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.pyindex d83aea9a00..447a3e4296 100644--- a/libs/text-splitters/langchain_text_splitters/spacy.py+++ b/libs/text-splitters/langchain_text_splitters/spacy.py@@ -20,6 +20,8 @@ class SpacyTextSplitter(TextSplitter):separator: str = "\n\n",pipeline: str = "en_core_web_sm",max_length: int = 1_000_000,+ *,+ strip_whitespace: bool = True,**kwargs: Any,) -> None:"""Initialize the spacy text splitter."""@@ -28,10 +30,14 @@ class SpacyTextSplitter(TextSplitter):pipeline, max_length=max_length)self._separator = separator+ self._strip_whitespace = strip_whitespacedef split_text(self, text: str) -> List[str]:"""Split incoming text and return chunks."""- splits = (s.text for s in self._tokenizer(text).sents)+ splits = (+ s.text if self._strip_whitespace else s.text_with_ws+ for s in self._tokenizer(text).sents+ )return self._merge_splits(splits, self._separator)diff --git a/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py b/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.pyindex 402d01655d..b19e8bf86e 100644--- a/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py+++ b/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py@@ -37,3 +37,16 @@ def test_spacy_text_splitter(pipeline: str) -> None:output = splitter.split_text(text)expected_output = [f"This is sentence one.{separator}And this is sentence two."]assert output == expected_output++[email protected]("pipeline", ["sentencizer", "en_core_web_sm"])+def test_spacy_text_splitter_strip_whitespace(pipeline: str) -> None:+ """Test splitting by sentence using Spacy."""+ text = "This is sentence one. And this is sentence two."+ separator = "|||"+ splitter = SpacyTextSplitter(+ separator=separator, pipeline=pipeline, strip_whitespace=False+ )+ output = splitter.split_text(text)+ expected_output = [f"This is sentence one. {separator}And this is sentence two."]+ assert output == expected_output