The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(text-splitters): HTMLSemanticPreservingSplitter nested preserved … (#34587)
Julia (Juli) Huangcd5b36456a3f
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.pyindex 65613bde39..b98a1c445e 100644--- a/libs/text-splitters/langchain_text_splitters/html.py+++ b/libs/text-splitters/langchain_text_splitters/html.py@@ -849,28 +849,6 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):placeholder_count: int,) -> tuple[list[Document], dict[str, str], list[str], dict[str, str], int]:for elem in element:- if elem.name.lower() in {"html", "body", "div", "main"}:- children = _find_all_tags(elem, recursive=False)- (- documents,- current_headers,- current_content,- preserved_elements,- placeholder_count,- ) = _process_element(- children,- documents,- current_headers,- current_content,- preserved_elements,- placeholder_count,- )- content = " ".join(_find_all_strings(elem, recursive=False))- if content:- content = self._normalize_and_clean_text(content)- current_content.append(content)- continue-if elem.name in [h[0] for h in self._headers_to_split_on]:if current_content:documents.extend(@@ -892,9 +870,39 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):current_content.append(placeholder)placeholder_count += 1else:- content = _get_element_text(elem)- if content:- current_content.append(content)+ # Recursively process children to find nested headers or+ # preserved elements.+ children = _find_all_tags(elem, recursive=False)+ if children:+ # Element has children - recursively process them.+ (+ documents,+ current_headers,+ current_content,+ preserved_elements,+ placeholder_count,+ ) = _process_element(+ children,+ documents,+ current_headers,+ current_content,+ preserved_elements,+ placeholder_count,+ )+ # After processing children, extract only text+ # strings from this element (not its children). Used+ # recursive=False to avoid double-counting.+ content = " ".join(_find_all_strings(elem, recursive=False))+ if content:+ content = self._normalize_and_clean_text(content)+ current_content.append(content)+ else:+ # Leaf element with no children, so we extract its+ # text and add to current content. Handles+ # text-only elements like <p>, <span>, <div>+ content = _get_element_text(elem)+ if content:+ current_content.append(content)return (documents,diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex 8919c2db27..5bf84d09cb 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -3427,6 +3427,100 @@ def test_html_splitter_with_preserved_elements() -> None:assert documents == expected # Shouldn't split the table or ul[email protected]("bs4")+def test_html_splitter_with_nested_preserved_elements() -> None:+ """Test HTML splitter with preserved elements nested in containers.++ Test that preserved elements are correctly preserved even when they are+ nested inside other container elements like <section> or <article>.+ This is a regression test for issue #31569+ """+ html_content = """+ <article>+ <h1>Section 1</h1>+ <section>+ <p>Some context about the data:</p>+ <table>+ <tr><td>Col1</td><td>Col2</td></tr>+ <tr><td>Data1</td><td>Data2</td></tr>+ </table>+ <p>Conclusion about data.</p>+ </section>+ </article>+ """+ with suppress_langchain_beta_warning():+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ elements_to_preserve=["table"],+ max_chunk_size=1000,+ )+ documents = splitter.split_text(html_content)++ # The table should be preserved in the output+ assert len(documents) == 1+ content = documents[0].page_content+ # Check that the table structure is maintained (not flattened)+ assert "Col1" in content+ assert "Col2" in content+ assert "Data1" in content+ assert "Data2" in content+ # Check metadata+ assert documents[0].metadata == {"Header 1": "Section 1"}++[email protected]("bs4")+def test_html_splitter_with_nested_div_preserved() -> None:+ """Test HTML splitter preserving nested div elements.++ Nested div elements should be preserved when specified in elements_to_preserve+ """+ html_content = """+ <div>+ <h1>Header</h1>+ <p>outer text</p>+ <div>inner div content</div>+ <p>more outer text</p>+ </div>+ """+ with suppress_langchain_beta_warning():+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ elements_to_preserve=["div"],+ max_chunk_size=1000,+ )+ documents = splitter.split_text(html_content)++ assert len(documents) == 1+ content = documents[0].page_content+ # The inner div content should be preserved+ assert "inner div content" in content+ assert "outer text" in content+ assert "more outer text" in content++[email protected]("bs4")+def test_html_splitter_preserve_nested_in_paragraph() -> None:+ """Test preserving deeply nested elements (code inside paragraph).++ tests the case where a preserved element (<code>) is nested+ inside a non-container element (<p>)+ """+ html_content = "<p>before <code>KEEP_THIS</code> after</p>"+ with suppress_langchain_beta_warning():+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[],+ elements_to_preserve=["code"],+ )+ documents = splitter.split_text(html_content)++ assert len(documents) == 1+ content = documents[0].page_content+ # All text should be preserved+ assert "before" in content+ assert "KEEP_THIS" in content+ assert "after" in content++@pytest.mark.requires("bs4")def test_html_splitter_with_no_further_splits() -> None:"""Test HTML splitting that requires no further splits beyond sections."""diff --git a/libs/text-splitters/uv.lock b/libs/text-splitters/uv.lockindex 2a5c2382b9..8ec7d5995c 100644--- a/libs/text-splitters/uv.lock+++ b/libs/text-splitters/uv.lock@@ -1150,7 +1150,7 @@ wheels = [[[package]]name = "langchain-core"-version = "1.2.5"+version = "1.2.6"source = { editable = "../core" }dependencies = [{ name = "jsonpatch" },