Skip to content

The agent engineering platform.

text-splitters[patch]: Fix HTMLSectionSplitter (#22812)

Jiejun Tanc8c67dde6f3e
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 8911331396..5cd1fce74d 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -233,9 +233,7 @@ class HTMLSectionSplitter:
documents.append(new_doc)
return documents
- def split_html_by_headers(
- self, html_doc: str
- ) -> Dict[str, Dict[str, Optional[str]]]:
+ def split_html_by_headers(self, html_doc: str) -> List[Dict[str, Optional[str]]]:
try:
from bs4 import BeautifulSoup, PageElement # type: ignore[import-untyped]
except ImportError as e:
@@ -247,7 +245,7 @@ class HTMLSectionSplitter:
soup = BeautifulSoup(html_doc, "html.parser")
headers = list(self.headers_to_split_on.keys())
- sections: Dict[str, Dict[str, Optional[str]]] = {}
+ sections: list[dict[str, str | None]] = []
headers = soup.find_all(["body"] + headers)
@@ -269,10 +267,13 @@ class HTMLSectionSplitter:
content = " ".join(section_content).strip()
if content != "":
- sections[current_header] = {
- "content": content,
- "tag_name": current_header_tag,
- }
+ sections.append(
+ {
+ "header": current_header,
+ "content": content,
+ "tag_name": current_header_tag,
+ }
+ )
return sections
@@ -307,12 +308,12 @@ class HTMLSectionSplitter:
return [
Document(
- cast(str, sections[section_key]["content"]),
+ cast(str, section["content"]),
metadata={
- self.headers_to_split_on[
- str(sections[section_key]["tag_name"])
- ]: section_key
+ self.headers_to_split_on[str(section["tag_name"])]: section[
+ "header"
+ ]
},
)
- for section_key in sections.keys()
+ for section in sections
]
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index 9f9c76b98d..d3ca82862f 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -1650,6 +1650,59 @@ def test_section_splitter_accepts_an_absolute_path() -> None:
sec_splitter.split_text(html_string)
+def test_happy_path_splitting_with_duplicate_header_tag() -> None:
+ # arrange
+ html_string = """<!DOCTYPE html>
+ <html>
+ <body>
+ <div>
+ <h1>Foo</h1>
+ <p>Some intro text about Foo.</p>
+ <div>
+ <h2>Bar main section</h2>
+ <p>Some intro text about Bar.</p>
+ <h3>Bar subsection 1</h3>
+ <p>Some text about the first subtopic of Bar.</p>
+ <h3>Bar subsection 2</h3>
+ <p>Some text about the second subtopic of Bar.</p>
+ </div>
+ <div>
+ <h2>Foo</h2>
+ <p>Some text about Baz</p>
+ </div>
+ <h1>Foo</h1>
+ <br>
+ <p>Some concluding text about Foo</p>
+ </div>
+ </body>
+ </html>"""
+
+ sec_splitter = HTMLSectionSplitter(
+ headers_to_split_on=[("h1", "Header 1"), ("h2", "Header 2")]
+ )
+
+ docs = sec_splitter.split_text(html_string)
+
+ assert len(docs) == 4
+ assert docs[0].page_content == "Foo \n Some intro text about Foo."
+ assert docs[0].metadata["Header 1"] == "Foo"
+
+ assert docs[1].page_content == (
+ "Bar main section \n Some intro text about Bar. \n "
+ "Bar subsection 1 \n Some text about the first subtopic of Bar. \n "
+ "Bar subsection 2 \n Some text about the second subtopic of Bar."
+ )
+ assert docs[1].metadata["Header 2"] == "Bar main section"
+
+ assert docs[2].page_content == "Foo \n Some text about Baz"
+ assert docs[2].metadata["Header 2"] == "Foo"
+
+ assert docs[3].page_content == "Foo \n \n Some concluding text about Foo"
+ assert docs[3].metadata["Header 1"] == "Foo"
+
+
def test_split_json() -> None:
"""Test json text splitter"""
max_chunk = 800