Skip to content

The agent engineering platform.

fix(text-splitters): reverse preserved elements iterator in `HTMLSemanticPreservingSplitter` (#34080)

Rohan Disa16f2c7d13b0e
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 16e7237667..d0c2c67b68 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -1055,7 +1055,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
Returns:
The content with placeholders replaced by preserved elements.
"""
- for placeholder, preserved_content in preserved_elements.items():
+ for placeholder, preserved_content in reversed(preserved_elements.items()):
content = content.replace(placeholder, preserved_content.strip())
return content
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index 582674ce74..a520962bea 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import random
import re
import string
+import textwrap
from typing import TYPE_CHECKING, Any
import pytest
@@ -3973,6 +3974,79 @@ def test_html_splitter_keep_separator_default() -> None:
assert documents == expected
+def test_html_splitter_preserved_elements_reverse_order() -> None:
+ """Test HTML splitter with preserved elements and conflicting placeholders.
+
+ This test validates that preserved elements are reinserted in reverse order
+ to prevent conflicts when one placeholder might be a substring of another.
+ """
+ html_content = """
+ <h1>Section 1</h1>
+ <table>
+ <tr><td>Table 1 content</td></tr>
+ </table>
+ <p>Some text between tables</p>
+ <table>
+ <tr><td>Table 10 content</td></tr>
+ </table>
+ <ul>
+ <li>List item 1</li>
+ <li>List item 10</li>
+ </ul>
+ """
+ with suppress_langchain_beta_warning():
+ splitter = HTMLSemanticPreservingSplitter(
+ headers_to_split_on=[("h1", "Header 1")],
+ elements_to_preserve=["table", "ul"],
+ max_chunk_size=100,
+ )
+ documents = splitter.split_text(html_content)
+
+ # Verify that all preserved elements are correctly reinserted
+ # This would fail if placeholders were processed in forward order
+ # when one placeholder is a substring of another
+ assert len(documents) >= 1
+ # Check that table content is preserved
+ content = " ".join(doc.page_content for doc in documents)
+ assert "Table 1 content" in content
+ assert "Table 10 content" in content
+ assert "List item 1" in content
+ assert "List item 10" in content
+
+
+def test_html_splitter_replacement_order() -> None:
+ body = textwrap.dedent(
+ """
+ <p>Hello1</p>
+ <p>Hello2</p>
+ <p>Hello3</p>
+ <p>Hello4</p>
+ <p>Hello5</p>
+ <p>Hello6</p>
+ <p>Hello7</p>
+ <p>Hello8</p>
+ <p>Hello9</p>
+ <p>Hello10</p>
+ <p>Hello11</p>
+ <p>Hello12</p>
+ <p>Hello13</p>
+ <p>Hello14</p>
+ """
+ )
+
+ with suppress_langchain_beta_warning():
+ splitter = HTMLSemanticPreservingSplitter(
+ headers_to_split_on=[],
+ elements_to_preserve=["p"],
+ )
+ documents = splitter.split_text(body)
+ assert len(documents) == 1
+ content = documents[0].page_content
+ assert content == " ".join([f"Hello{i}" for i in range(1, 15)])
+
+
def test_character_text_splitter_discard_regex_separator_on_merge() -> None:
"""Test that regex lookahead separator is not re-inserted when merging."""
text = "SCE191 First chunk. SCE103 Second chunk."