The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
text-splitters: Add keep_separator arg to HTMLSemanticPreservingSplitter (#31588)
Tom-Trumper532e6455e93a
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.pyindex 405055b03b..a917f6a9c3 100644--- a/libs/text-splitters/langchain_text_splitters/html.py+++ b/libs/text-splitters/langchain_text_splitters/html.py@@ -10,10 +10,12 @@ from typing import (Dict,Iterable,List,+ Literal,Optional,Sequence,Tuple,TypedDict,+ Union,cast,)@@ -535,6 +537,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):preserve_parent_metadata (bool): Whether to pass through parent documentmetadata to split documents when calling``transform_documents/atransform_documents()``.+ keep_separator (Union[bool, Literal["start", "end"]]): Whether separators+ should be at the beginning of a chunk, at the end, or not at all.Example:.. code-block:: python@@ -584,6 +588,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):allowlist_tags: Optional[List[str]] = None,denylist_tags: Optional[List[str]] = None,preserve_parent_metadata: bool = False,+ keep_separator: Union[bool, Literal["start", "end"]] = True,):"""Initialize splitter."""try:@@ -611,6 +616,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):self._external_metadata = external_metadata or {}self._allowlist_tags = allowlist_tagsself._preserve_parent_metadata = preserve_parent_metadata+ self._keep_separator = keep_separatorif allowlist_tags:self._allowlist_tags = list(set(allowlist_tags + [header[0] for header in headers_to_split_on])@@ -625,12 +631,15 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):if separators:self._recursive_splitter = RecursiveCharacterTextSplitter(separators=separators,+ keep_separator=keep_separator,chunk_size=max_chunk_size,chunk_overlap=chunk_overlap,)else:self._recursive_splitter = RecursiveCharacterTextSplitter(- chunk_size=max_chunk_size, chunk_overlap=chunk_overlap+ keep_separator=keep_separator,+ chunk_size=max_chunk_size,+ chunk_overlap=chunk_overlap,)if self._stopword_removal:diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex bc94dca89e..85c6caace1 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -3375,6 +3375,148 @@ def test_html_splitter_with_media_preservation() -> None:assert documents == expected[email protected]("bs4")+def test_html_splitter_keep_separator_true() -> None:+ """Test HTML splitting with keep_separator=True"""+ html_content = """+ <h1>Section 1</h1>+ <p>This is some text. This is some other text.</p>+ """+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ max_chunk_size=10,+ separators=[". "],+ keep_separator=True,+ )+ documents = splitter.split_text(html_content)++ expected = [+ Document(+ page_content="This is some text",+ metadata={"Header 1": "Section 1"},+ ),+ Document(+ page_content=". This is some other text.",+ metadata={"Header 1": "Section 1"},+ ),+ ]++ assert documents == expected++[email protected]("bs4")+def test_html_splitter_keep_separator_false() -> None:+ """Test HTML splitting with keep_separator=False"""+ html_content = """+ <h1>Section 1</h1>+ <p>This is some text. This is some other text.</p>+ """+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ max_chunk_size=10,+ separators=[". "],+ keep_separator=False,+ )+ documents = splitter.split_text(html_content)++ expected = [+ Document(+ page_content="This is some text",+ metadata={"Header 1": "Section 1"},+ ),+ Document(+ page_content="This is some other text.",+ metadata={"Header 1": "Section 1"},+ ),+ ]++ assert documents == expected++[email protected]("bs4")+def test_html_splitter_keep_separator_start() -> None:+ """Test HTML splitting with keep_separator="start" """+ html_content = """+ <h1>Section 1</h1>+ <p>This is some text. This is some other text.</p>+ """+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ max_chunk_size=10,+ separators=[". "],+ keep_separator="start",+ )+ documents = splitter.split_text(html_content)++ expected = [+ Document(+ page_content="This is some text",+ metadata={"Header 1": "Section 1"},+ ),+ Document(+ page_content=". This is some other text.",+ metadata={"Header 1": "Section 1"},+ ),+ ]++ assert documents == expected++[email protected]("bs4")+def test_html_splitter_keep_separator_end() -> None:+ """Test HTML splitting with keep_separator="end" """+ html_content = """+ <h1>Section 1</h1>+ <p>This is some text. This is some other text.</p>+ """+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ max_chunk_size=10,+ separators=[". "],+ keep_separator="end",+ )+ documents = splitter.split_text(html_content)++ expected = [+ Document(+ page_content="This is some text.",+ metadata={"Header 1": "Section 1"},+ ),+ Document(+ page_content="This is some other text.",+ metadata={"Header 1": "Section 1"},+ ),+ ]++ assert documents == expected++[email protected]("bs4")+def test_html_splitter_keep_separator_default() -> None:+ """Test HTML splitting with keep_separator not set"""+ html_content = """+ <h1>Section 1</h1>+ <p>This is some text. This is some other text.</p>+ """+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")], max_chunk_size=10, separators=[". "]+ )+ documents = splitter.split_text(html_content)++ expected = [+ Document(+ page_content="This is some text",+ metadata={"Header 1": "Section 1"},+ ),+ Document(+ page_content=". This is some other text.",+ metadata={"Header 1": "Section 1"},+ ),+ ]++ assert documents == expected++def test_character_text_splitter_discard_regex_separator_on_merge() -> None:"""Test that regex lookahead separator is not re-inserted when merging."""text = "SCE191 First chunk. SCE103 Second chunk."