The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(text-splitters): reverse preserved elements iterator in `HTMLSemanticPreservingSplitter` (#34080)
Rohan Disa16f2c7d13b0e
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.pyindex 16e7237667..d0c2c67b68 100644--- a/libs/text-splitters/langchain_text_splitters/html.py+++ b/libs/text-splitters/langchain_text_splitters/html.py@@ -1055,7 +1055,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):Returns:The content with placeholders replaced by preserved elements."""- for placeholder, preserved_content in preserved_elements.items():+ for placeholder, preserved_content in reversed(preserved_elements.items()):content = content.replace(placeholder, preserved_content.strip())return contentdiff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex 582674ce74..a520962bea 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport randomimport reimport string+import textwrapfrom typing import TYPE_CHECKING, Anyimport pytest@@ -3973,6 +3974,79 @@ def test_html_splitter_keep_separator_default() -> None:assert documents == expected[email protected]("bs4")+def test_html_splitter_preserved_elements_reverse_order() -> None:+ """Test HTML splitter with preserved elements and conflicting placeholders.++ This test validates that preserved elements are reinserted in reverse order+ to prevent conflicts when one placeholder might be a substring of another.+ """+ html_content = """+ <h1>Section 1</h1>+ <table>+ <tr><td>Table 1 content</td></tr>+ </table>+ <p>Some text between tables</p>+ <table>+ <tr><td>Table 10 content</td></tr>+ </table>+ <ul>+ <li>List item 1</li>+ <li>List item 10</li>+ </ul>+ """+ with suppress_langchain_beta_warning():+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[("h1", "Header 1")],+ elements_to_preserve=["table", "ul"],+ max_chunk_size=100,+ )+ documents = splitter.split_text(html_content)++ # Verify that all preserved elements are correctly reinserted+ # This would fail if placeholders were processed in forward order+ # when one placeholder is a substring of another+ assert len(documents) >= 1+ # Check that table content is preserved+ content = " ".join(doc.page_content for doc in documents)+ assert "Table 1 content" in content+ assert "Table 10 content" in content+ assert "List item 1" in content+ assert "List item 10" in content++[email protected]("bs4")+def test_html_splitter_replacement_order() -> None:+ body = textwrap.dedent(+ """+ <p>Hello1</p>+ <p>Hello2</p>+ <p>Hello3</p>+ <p>Hello4</p>+ <p>Hello5</p>+ <p>Hello6</p>+ <p>Hello7</p>+ <p>Hello8</p>+ <p>Hello9</p>+ <p>Hello10</p>+ <p>Hello11</p>+ <p>Hello12</p>+ <p>Hello13</p>+ <p>Hello14</p>+ """+ )++ with suppress_langchain_beta_warning():+ splitter = HTMLSemanticPreservingSplitter(+ headers_to_split_on=[],+ elements_to_preserve=["p"],+ )+ documents = splitter.split_text(body)+ assert len(documents) == 1+ content = documents[0].page_content+ assert content == " ".join([f"Hello{i}" for i in range(1, 15)])++def test_character_text_splitter_discard_regex_separator_on_merge() -> None:"""Test that regex lookahead separator is not re-inserted when merging."""text = "SCE191 First chunk. SCE103 Second chunk."