Skip to content

The agent engineering platform.

fix(text-splitters): HTMLSemanticPreservingSplitter nested preserved … (#34587)

Julia (Juli) Huangcd5b36456a3f
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 65613bde39..b98a1c445e 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -849,28 +849,6 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
placeholder_count: int,
) -> tuple[list[Document], dict[str, str], list[str], dict[str, str], int]:
for elem in element:
- if elem.name.lower() in {"html", "body", "div", "main"}:
- children = _find_all_tags(elem, recursive=False)
- (
- documents,
- current_headers,
- current_content,
- preserved_elements,
- placeholder_count,
- ) = _process_element(
- children,
- documents,
- current_headers,
- current_content,
- preserved_elements,
- placeholder_count,
- )
- content = " ".join(_find_all_strings(elem, recursive=False))
- if content:
- content = self._normalize_and_clean_text(content)
- current_content.append(content)
- continue
-
if elem.name in [h[0] for h in self._headers_to_split_on]:
if current_content:
documents.extend(
@@ -892,9 +870,39 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
current_content.append(placeholder)
placeholder_count += 1
else:
- content = _get_element_text(elem)
- if content:
- current_content.append(content)
+ # Recursively process children to find nested headers or
+ # preserved elements.
+ children = _find_all_tags(elem, recursive=False)
+ if children:
+ # Element has children - recursively process them.
+ (
+ documents,
+ current_headers,
+ current_content,
+ preserved_elements,
+ placeholder_count,
+ ) = _process_element(
+ children,
+ documents,
+ current_headers,
+ current_content,
+ preserved_elements,
+ placeholder_count,
+ )
+ # After processing children, extract only text
+ # strings from this element (not its children). Used
+ # recursive=False to avoid double-counting.
+ content = " ".join(_find_all_strings(elem, recursive=False))
+ if content:
+ content = self._normalize_and_clean_text(content)
+ current_content.append(content)
+ else:
+ # Leaf element with no children, so we extract its
+ # text and add to current content. Handles
+ # text-only elements like <p>, <span>, <div>
+ content = _get_element_text(elem)
+ if content:
+ current_content.append(content)
return (
documents,
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index 8919c2db27..5bf84d09cb 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -3427,6 +3427,100 @@ def test_html_splitter_with_preserved_elements() -> None:
assert documents == expected # Shouldn't split the table or ul
+def test_html_splitter_with_nested_preserved_elements() -> None:
+ """Test HTML splitter with preserved elements nested in containers.
+
+ Test that preserved elements are correctly preserved even when they are
+ nested inside other container elements like <section> or <article>.
+ This is a regression test for issue #31569
+ """
+ html_content = """
+ <article>
+ <h1>Section 1</h1>
+ <section>
+ <p>Some context about the data:</p>
+ <table>
+ <tr><td>Col1</td><td>Col2</td></tr>
+ <tr><td>Data1</td><td>Data2</td></tr>
+ </table>
+ <p>Conclusion about data.</p>
+ </section>
+ </article>
+ """
+ with suppress_langchain_beta_warning():
+ splitter = HTMLSemanticPreservingSplitter(
+ headers_to_split_on=[("h1", "Header 1")],
+ elements_to_preserve=["table"],
+ max_chunk_size=1000,
+ )
+ documents = splitter.split_text(html_content)
+
+ # The table should be preserved in the output
+ assert len(documents) == 1
+ content = documents[0].page_content
+ # Check that the table structure is maintained (not flattened)
+ assert "Col1" in content
+ assert "Col2" in content
+ assert "Data1" in content
+ assert "Data2" in content
+ # Check metadata
+ assert documents[0].metadata == {"Header 1": "Section 1"}
+
+
+def test_html_splitter_with_nested_div_preserved() -> None:
+ """Test HTML splitter preserving nested div elements.
+
+ Nested div elements should be preserved when specified in elements_to_preserve
+ """
+ html_content = """
+ <div>
+ <h1>Header</h1>
+ <p>outer text</p>
+ <div>inner div content</div>
+ <p>more outer text</p>
+ </div>
+ """
+ with suppress_langchain_beta_warning():
+ splitter = HTMLSemanticPreservingSplitter(
+ headers_to_split_on=[("h1", "Header 1")],
+ elements_to_preserve=["div"],
+ max_chunk_size=1000,
+ )
+ documents = splitter.split_text(html_content)
+
+ assert len(documents) == 1
+ content = documents[0].page_content
+ # The inner div content should be preserved
+ assert "inner div content" in content
+ assert "outer text" in content
+ assert "more outer text" in content
+
+
+def test_html_splitter_preserve_nested_in_paragraph() -> None:
+ """Test preserving deeply nested elements (code inside paragraph).
+
+ tests the case where a preserved element (<code>) is nested
+ inside a non-container element (<p>)
+ """
+ html_content = "<p>before <code>KEEP_THIS</code> after</p>"
+ with suppress_langchain_beta_warning():
+ splitter = HTMLSemanticPreservingSplitter(
+ headers_to_split_on=[],
+ elements_to_preserve=["code"],
+ )
+ documents = splitter.split_text(html_content)
+
+ assert len(documents) == 1
+ content = documents[0].page_content
+ # All text should be preserved
+ assert "before" in content
+ assert "KEEP_THIS" in content
+ assert "after" in content
+
+
@pytest.mark.requires("bs4")
def test_html_splitter_with_no_further_splits() -> None:
"""Test HTML splitting that requires no further splits beyond sections."""
diff --git a/libs/text-splitters/uv.lock b/libs/text-splitters/uv.lock
index 2a5c2382b9..8ec7d5995c 100644
--- a/libs/text-splitters/uv.lock
+++ b/libs/text-splitters/uv.lock
@@ -1150,7 +1150,7 @@ wheels = [
[[package]]
name = "langchain-core"
-version = "1.2.5"
+version = "1.2.6"
source = { editable = "../core" }
dependencies = [
{ name = "jsonpatch" },