Skip to content

The agent engineering platform.

text-splitters[minor]: Replace lxml and XSLT with BeautifulSoup in HTMLHeaderTextSplitter for Improved Large HTML File Processing (#27678)

Ahmed Tammaad3ed9b86be31
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 3613937d99..56d545f6fc 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import copy
import pathlib
import re
-from io import BytesIO, StringIO
+from io import StringIO
from typing import (
Any,
Callable,
@@ -34,148 +34,291 @@ class ElementType(TypedDict):
class HTMLHeaderTextSplitter:
- """Splitting HTML files based on specified headers.
+ """Split HTML content into structured Documents based on specified headers.
- Requires lxml package.
+ Splits HTML content by detecting specified header tags (e.g., <h1>, <h2>) and
+ creating hierarchical Document objects that reflect the semantic structure
+ of the original content. For each identified section, the splitter associates
+ the extracted text with metadata corresponding to the encountered headers.
+
+ If no specified headers are found, the entire content is returned as a single
+ Document. This allows for flexible handling of HTML input, ensuring that
+ information is organized according to its semantic headers.
+
+ The splitter provides the option to return each HTML element as a separate
+ Document or aggregate them into semantically meaningful chunks. It also
+ gracefully handles multiple levels of nested headers, creating a rich,
+ hierarchical representation of the content.
+
+ Args:
+ headers_to_split_on (List[Tuple[str, str]]): A list of (header_tag,
+ header_name) pairs representing the headers that define splitting
+ boundaries. For example, [("h1", "Header 1"), ("h2", "Header 2")]
+ will split content by <h1> and <h2> tags, assigning their textual
+ content to the Document metadata.
+ return_each_element (bool): If True, every HTML element encountered
+ (including headers, paragraphs, etc.) is returned as a separate
+ Document. If False, content under the same header hierarchy is
+ aggregated into fewer Documents.
+
+ Returns:
+ List[Document]: A list of Document objects. Each Document contains
+ `page_content` holding the extracted text and `metadata` that maps
+ the header hierarchy to their corresponding titles.
+
+ Example:
+ .. code-block:: python
+
+ from langchain_text_splitters.html_header_text_splitter import (
+ HTMLHeaderTextSplitter,
+ )
+
+ # Define headers for splitting on h1 and h2 tags.
+ headers_to_split_on = [("h1", "Main Topic"), ("h2", "Sub Topic")]
+
+ splitter = HTMLHeaderTextSplitter(
+ headers_to_split_on=headers_to_split_on,
+ return_each_element=False
+ )
+
+ html_content = \"\"\"
+ <html>
+ <body>
+ <h1>Introduction</h1>
+ <p>Welcome to the introduction section.</p>
+ <h2>Background</h2>
+ <p>Some background details here.</p>
+ <h1>Conclusion</h1>
+ <p>Final thoughts.</p>
+ </body>
+ </html>
+ \"\"\"
+
+ documents = splitter.split_text(html_content)
+
+ # 'documents' now contains Document objects reflecting the hierarchy:
+ # - Document with metadata={"Main Topic": "Introduction"} and
+ # content="Introduction"
+ # - Document with metadata={"Main Topic": "Introduction"} and
+ # content="Welcome to the introduction section."
+ # - Document with metadata={"Main Topic": "Introduction",
+ # "Sub Topic": "Background"} and content="Background"
+ # - Document with metadata={"Main Topic": "Introduction",
+ # "Sub Topic": "Background"} and content="Some background details here."
+ # - Document with metadata={"Main Topic": "Conclusion"} and
+ # content="Conclusion"
+ # - Document with metadata={"Main Topic": "Conclusion"} and
+ # content="Final thoughts."
"""
def __init__(
self,
headers_to_split_on: List[Tuple[str, str]],
return_each_element: bool = False,
- ):
- """Create a new HTMLHeaderTextSplitter.
+ ) -> None:
+ """Initialize with headers to split on.
Args:
- headers_to_split_on: list of tuples of headers we want to track mapped to
- (arbitrary) keys for metadata. Allowed header values: h1, h2, h3, h4,
- h5, h6 e.g. [("h1", "Header 1"), ("h2", "Header 2)].
- return_each_element: Return each element w/ associated headers.
+ headers_to_split_on: A list of tuples where
+ each tuple contains a header tag and its corresponding value.
+ return_each_element: Whether to return each HTML
+ element as a separate Document. Defaults to False.
"""
- # Output element-by-element or aggregated into chunks w/ common headers
+ self.headers_to_split_on = sorted(
+ headers_to_split_on, key=lambda x: int(x[0][1])
+ )
+ self.header_mapping = dict(self.headers_to_split_on)
+ self.header_tags = [tag for tag, _ in self.headers_to_split_on]
self.return_each_element = return_each_element
- self.headers_to_split_on = sorted(headers_to_split_on)
- def aggregate_elements_to_chunks(
- self, elements: List[ElementType]
- ) -> List[Document]:
- """Combine elements with common metadata into chunks.
+ def split_text(self, text: str) -> List[Document]:
+ """Split the given text into a list of Document objects.
Args:
- elements: HTML element content with associated identifying info and metadata
+ text: The HTML text to split.
+
+ Returns:
+ A list of split Document objects.
"""
- aggregated_chunks: List[ElementType] = []
+ return self.split_text_from_file(StringIO(text))
- for element in elements:
- if (
- aggregated_chunks
- and aggregated_chunks[-1]["metadata"] == element["metadata"]
- ):
- # If the last element in the aggregated list
- # has the same metadata as the current element,
- # append the current content to the last element's content
- aggregated_chunks[-1]["content"] += " \n" + element["content"]
- else:
- # Otherwise, append the current element to the aggregated list
- aggregated_chunks.append(element)
+ def split_text_from_url(
+ self, url: str, timeout: int = 10, **kwargs: Any
+ ) -> List[Document]:
+ """Fetch text content from a URL and split it into documents.
- return [
- Document(page_content=chunk["content"], metadata=chunk["metadata"])
- for chunk in aggregated_chunks
- ]
+ Args:
+ url: The URL to fetch content from.
+ timeout: Timeout for the request. Defaults to 10.
+ **kwargs: Additional keyword arguments for the request.
- def split_text_from_url(self, url: str, **kwargs: Any) -> List[Document]:
- """Split HTML from web URL.
+ Returns:
+ A list of split Document objects.
- Args:
- url: web URL
- **kwargs: Arbitrary additional keyword arguments. These are usually passed
- to the fetch url content request.
+ Raises:
+ requests.RequestException: If the HTTP request fails.
"""
- r = requests.get(url, **kwargs)
- return self.split_text_from_file(BytesIO(r.content))
-
- def split_text(self, text: str) -> List[Document]:
- """Split HTML text string.
+ kwargs.setdefault("timeout", timeout)
+ response = requests.get(url, **kwargs)
+ response.raise_for_status()
+ return self.split_text(response.text)
+
+ def _header_level(self, tag_name: str) -> int:
+ """Determine the heading level of a tag."""
+ if tag_name.lower() in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+ return int(tag_name[1])
+ # Returns high level if it isn't a header
+ return 9999
+
+ def _dom_depth(self, element: Any) -> int:
+ """Determine the DOM depth of an element by counting its parents."""
+ depth = 0
+ for _ in element.parents:
+ depth += 1
+ return depth
+
+ def _get_elements(self, html_content: str) -> List[Any]:
+ """Parse HTML content and return a list of BeautifulSoup elements.
+
+ This helper function takes HTML content as input,
+ parses it using BeautifulSoup4, and returns all HTML elements
+ found in the document body. If no body tag exists,
+ it returns all elements in the full document.
Args:
- text: HTML text
- """
- return self.split_text_from_file(StringIO(text))
+ html_content: Raw HTML content to be parsed.
- def split_text_from_file(self, file: Any) -> List[Document]:
- """Split HTML file.
+ Returns:
+ List[Any]: A list of BeautifulSoup elements found in the HTML document.
- Args:
- file: HTML file
+ Raises:
+ ImportError: If the BeautifulSoup4 package is not installed.
"""
try:
- from lxml import etree
+ from bs4 import BeautifulSoup # type: ignore[import-untyped]
except ImportError as e:
raise ImportError(
- "Unable to import lxml, please install with `pip install lxml`."
+ "Unable to import BeautifulSoup/PageElement, \
+ please install with `pip install \
+ bs4`."
) from e
- # use lxml library to parse html document and return xml ElementTree
- # Explicitly encoding in utf-8 allows non-English
- # html files to be processed without garbled characters
- parser = etree.HTMLParser(encoding="utf-8")
- tree = etree.parse(file, parser)
-
- # document transformation for "structure-aware" chunking is handled with xsl.
- # see comments in html_chunks_with_headers.xslt for more detailed information.
- xslt_path = pathlib.Path(__file__).parent / "xsl/html_chunks_with_headers.xslt"
- xslt_tree = etree.parse(xslt_path)
- transform = etree.XSLT(xslt_tree)
- result = transform(tree)
- result_dom = etree.fromstring(str(result))
-
- # create filter and mapping for header metadata
- header_filter = [header[0] for header in self.headers_to_split_on]
- header_mapping = dict(self.headers_to_split_on)
-
- # map xhtml namespace prefix
- ns_map = {"h": "http://www.w3.org/1999/xhtml"}
-
- # build list of elements from DOM
- elements = []
- for element in result_dom.findall("*//*", ns_map):
- if element.findall("*[@class='headers']") or element.findall(
- "*[@class='chunk']"
- ):
- elements.append(
- ElementType(
- url=file,
- xpath="".join(
- [
- node.text or ""
- for node in element.findall("*[@class='xpath']", ns_map)
- ]
- ),
- content="".join(
- [
- node.text or ""
- for node in element.findall("*[@class='chunk']", ns_map)
- ]
- ),
- metadata={
- # Add text of specified headers to metadata using header
- # mapping.
- header_mapping[node.tag]: node.text or ""
- for node in filter(
- lambda x: x.tag in header_filter,
- element.findall("*[@class='headers']/*", ns_map),
- )
- },
- )
+ soup = BeautifulSoup(html_content, "html.parser")
+ body = soup.body if soup.body else soup
+ return body.find_all()
+
+ def split_text_from_file(self, file: Any) -> List[Document]:
+ """Split HTML content from a file into a list of Document objects.
+
+ Args:
+ file: A file path or a file-like object containing HTML content.
+
+ Returns:
+ A list of split Document objects.
+ """
+ if isinstance(file, str):
+ with open(file, "r", encoding="utf-8") as f:
+ html_content = f.read()
+ else:
+ html_content = file.read()
+ elements = self._get_elements(html_content)
+ documents: List[Document] = []
+ active_headers: Dict[str, Tuple[str, int, int]] = {}
+ current_chunk: List[str] = []
+ chunk_dom_depth = 0
+
+ def finalize_chunk() -> None:
+ if current_chunk:
+ final_meta = {
+ key: content
+ for key, (content, level, dom_depth) in active_headers.items()
+ if chunk_dom_depth >= dom_depth
+ }
+ combined_text = " \n".join(
+ line for line in current_chunk if line.strip()
)
+ if combined_text.strip():
+ documents.append(
+ Document(page_content=combined_text, metadata=final_meta)
+ )
+ current_chunk.clear()
+
+ for element in elements:
+ tag = element.name
+ if not tag:
+ continue
+ text = " ".join(
+ t
+ for t in element.find_all(string=True, recursive=False)
+ if isinstance(t, str)
+ ).strip()
+ if not text:
+ continue
+
+ level = self._header_level(tag)
+ dom_depth = self._dom_depth(element)
+
+ if tag in self.header_tags:
+ if not self.return_each_element:
+ finalize_chunk()
+
+ # Remove headers at same or deeper level
+ headers_to_remove = [
+ key for key, (_, lvl, _) in active_headers.items() if lvl >= level
+ ]
+ for key in headers_to_remove:
+ del active_headers[key]
+
+ header_key = self.header_mapping[tag]
+ active_headers[header_key] = (text, level, dom_depth)
+
+ # Produce a document for the header itself
+ header_meta = {
+ key: content
+ for key, (content, lvl, dd) in active_headers.items()
+ if dom_depth >= dd
+ }
+ documents.append(Document(page_content=text, metadata=header_meta))
+ # After encountering a header,
+ # no immediate content goes to current_chunk
+ # (if return_each_element is False, we wait for next content)
+ # (if return_each_element is True, we create docs per element anyway)
+ else:
+ # Non-header element logic
+ # Remove headers that don't apply if dom_depth < their dom_depth
+ headers_to_remove = [
+ key for key, (_, _, dd) in active_headers.items() if dom_depth < dd
+ ]
+ for key in headers_to_remove:
+ del active_headers[key]
+
+ if self.return_each_element:
+ # Produce a doc for this element immediately
+ element_meta = {
+ key: content
+ for key, (content, lvl, dd) in active_headers.items()
+ if dom_depth >= dd
+ }
+ if text.strip():
+ documents.append(
+ Document(page_content=text, metadata=element_meta)
+ )
+ else:
+ # Accumulate content in current_chunk
+ if text.strip():
+ current_chunk.append(text)
+ chunk_dom_depth = max(chunk_dom_depth, dom_depth)
if not self.return_each_element:
- return self.aggregate_elements_to_chunks(elements)
- else:
- return [
- Document(page_content=chunk["content"], metadata=chunk["metadata"])
- for chunk in elements
- ]
+ # finalize any remaining chunk
+ finalize_chunk()
+
+ # If no headers were found at all and return_each_element=False, behavior is:
+ # The entire content should be in one document.
+ # The logic above naturally handles it:
+ # If no recognized headers, we never split; we ended up just accumulating text
+ # in current_chunk and finalizing once at the end.
+
+ return documents
class HTMLSectionSplitter:
@@ -269,7 +412,10 @@ class HTMLSectionSplitter:
- 'tag_name': The name of the header tag (e.g., "h1", "h2").
"""
try:
- from bs4 import BeautifulSoup, PageElement # type: ignore[import-untyped]
+ from bs4 import (
+ BeautifulSoup, # type: ignore[import-untyped]
+ PageElement,
+ )
except ImportError as e:
raise ImportError(
"Unable to import BeautifulSoup/PageElement, \
@@ -343,10 +489,13 @@ class HTMLSectionSplitter:
return str(result)
def split_text_from_file(self, file: Any) -> List[Document]:
- """Split HTML file.
+ """Split HTML content from a file into a list of Document objects.
Args:
- file: HTML file
+ file: A file path or a file-like object containing HTML content.
+
+ Returns:
+ A list of split Document objects.
"""
file_content = file.getvalue()
file_content = self.convert_possible_tags_to_header(file_content)
@@ -844,3 +993,6 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
for placeholder, preserved_content in preserved_elements.items():
content = content.replace(placeholder, preserved_content.strip())
return content
+
+
+# %%
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index dd6d0023d1..4e5c0bca4c 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -4,7 +4,7 @@ import random
import re
import string
from pathlib import Path
-from typing import Any, List
+from typing import Any, Callable, List, Tuple
import pytest
from langchain_core.documents import Document
@@ -2039,49 +2039,476 @@ def test_haskell_code_splitter() -> None:
assert chunks == expected_chunks
-def test_html_header_text_splitter(tmp_path: Path) -> None:
- splitter = HTMLHeaderTextSplitter(
- headers_to_split_on=[("h1", "Header 1"), ("h2", "Header 2")]
- )
+def html_header_splitter_splitter_factory() -> (
+ Callable[[List[Tuple[str, str]]], HTMLHeaderTextSplitter]
+):
+ """
+ Fixture to create an HTMLHeaderTextSplitter instance with given headers.
+ This factory allows dynamic creation of splitters with different headers.
+ """
- content = """
-<h1>Sample Document</h1>
- <h2>Section</h2>
- <p id="1234">Reference content.</p>
+ def _create_splitter(
+ headers_to_split_on: List[Tuple[str, str]],
+ ) -> HTMLHeaderTextSplitter:
+ return HTMLHeaderTextSplitter(headers_to_split_on=headers_to_split_on)
- <h2>Lists</h2>
- <ul>
- <li>Item 1</li>
- <li>Item 2</li>
- <li>Item 3</li>
- </ul>
+ return _create_splitter
- <h3>A block</h3>
- <div class="amazing">
- <p>Some text</p>
- <p>Some more text</p>
- </div>
+
+ "headers_to_split_on, html_input, expected_documents, test_case",
+ [
+ (
+ # Test Case 1: Split on h1 and h2
+ [("h1", "Header 1"), ("h2", "Header 2")],
+ """
+ <html>
+ <body>
+ <h1>Introduction</h1>
+ <p>This is the introduction.</p>
+ <h2>Background</h2>
+ <p>Background information.</p>
+ <h1>Conclusion</h1>
+ <p>Final thoughts.</p>
+ </body>
+ </html>
+ """,
+ [
+ Document(
+ page_content="Introduction", metadata={"Header 1": "Introduction"}
+ ),
+ Document(
+ page_content="This is the introduction.",
+ metadata={"Header 1": "Introduction"},
+ ),
+ Document(
+ page_content="Background",
+ metadata={"Header 1": "Introduction", "Header 2": "Background"},
+ ),
+ Document(
+ page_content="Background information.",
+ metadata={"Header 1": "Introduction", "Header 2": "Background"},
+ ),
+ Document(
+ page_content="Conclusion", metadata={"Header 1": "Conclusion"}
+ ),
+ Document(
+ page_content="Final thoughts.", metadata={"Header 1": "Conclusion"}
+ ),
+ ],
+ "Simple headers and paragraphs",
+ ),
+ (
+ # Test Case 2: Nested headers with h1, h2, and h3
+ [("h1", "Header 1"), ("h2", "Header 2"), ("h3", "Header 3")],
+ """
+ <html>
+ <body>
+ <div>
+ <h1>Main Title</h1>
+ <div>
+ <h2>Subsection</h2>
+ <p>Details of subsection.</p>
+ <div>
+ <h3>Sub-subsection</h3>
+ <p>More details.</p>
+ </div>
+ </div>
+ </div>
+ <h1>Another Main Title</h1>
+ <p>Content under another main title.</p>
+ </body>
+ </html>
+ """,
+ [
+ Document(
+ page_content="Main Title", metadata={"Header 1": "Main Title"}
+ ),
+ Document(
+ page_content="Subsection",
+ metadata={"Header 1": "Main Title", "Header 2": "Subsection"},
+ ),
+ Document(
+ page_content="Details of subsection.",
+ metadata={"Header 1": "Main Title", "Header 2": "Subsection"},
+ ),
+ Document(
+ page_content="Sub-subsection",
+ metadata={
+ "Header 1": "Main Title",
+ "Header 2": "Subsection",
+ "Header 3": "Sub-subsection",
+ },
+ ),
+ Document(
+ page_content="More details.",
+ metadata={
+ "Header 1": "Main Title",
+ "Header 2": "Subsection",
+ "Header 3": "Sub-subsection",
+ },
+ ),
+ Document(
+ page_content="Another Main Title",
+ metadata={"Header 1": "Another Main Title"},
+ ),
+ Document(
+ page_content="Content under another main title.",
+ metadata={"Header 1": "Another Main Title"},
+ ),
+ ],
+ "Nested headers with h1, h2, and h3",
+ ),
+ (
+ # Test Case 3: No headers
+ [("h1", "Header 1")],
+ """
+ <html>
+ <body>
+ <p>Paragraph one.</p>
+ <p>Paragraph two.</p>
+ <div>
+ <p>Paragraph three.</p>
+ </div>
+ </body>
+ </html>
+ """,
+ [
+ Document(
+ page_content="Paragraph one. \nParagraph two. \nParagraph three.",
+ metadata={},
+ )
+ ],
+ "No headers present",
+ ),
+ (
+ # Test Case 4: Multiple headers of the same level
+ [("h1", "Header 1")],
+ """
+ <html>
+ <body>
+ <h1>Chapter 1</h1>
+ <p>Content of chapter 1.</p>
+ <h1>Chapter 2</h1>
+ <p>Content of chapter 2.</p>
+ <h1>Chapter 3</h1>
+ <p>Content of chapter 3.</p>
+ </body>
+ </html>
+ """,
+ [
+ Document(page_content="Chapter 1", metadata={"Header 1": "Chapter 1"}),
+ Document(
+ page_content="Content of chapter 1.",
+ metadata={"Header 1": "Chapter 1"},
+ ),
+ Document(page_content="Chapter 2", metadata={"Header 1": "Chapter 2"}),
+ Document(
+ page_content="Content of chapter 2.",
+ metadata={"Header 1": "Chapter 2"},
+ ),
+ Document(page_content="Chapter 3", metadata={"Header 1": "Chapter 3"}),
+ Document(
+ page_content="Content of chapter 3.",
+ metadata={"Header 1": "Chapter 3"},
+ ),
+ ],
+ "Multiple headers of the same level",
+ ),
+ (
+ # Test Case 5: Headers with no content
+ [("h1", "Header 1"), ("h2", "Header 2")],
+ """
+ <html>
+ <body>
+ <h1>Header 1</h1>
+ <h2>Header 2</h2>
+ <h1>Header 3</h1>
+ </body>
+ </html>
+ """,
+ [
+ Document(page_content="Header 1", metadata={"Header 1": "Header 1"}),
+ Document(
+ page_content="Header 2",
+ metadata={"Header 1": "Header 1", "Header 2": "Header 2"},
+ ),
+ Document(page_content="Header 3", metadata={"Header 1": "Header 3"}),
+ ],
+ "Headers with no associated content",
+ ),
+ ],
+)
+def test_html_header_text_splitter(
+ html_header_splitter_splitter_factory: Any,
+ headers_to_split_on: List[Tuple[str, str]],
+ html_input: str,
+ expected_documents: List[Document],
+ test_case: str,
+) -> None:
+ """
+ Test the HTML header text splitter.
+
+ Args:
+ html_header_splitter_splitter_factory (Any): Factory function to create
+ the HTML header splitter.
+ headers_to_split_on (List[Tuple[str, str]]): List of headers to split on.
+ html_input (str): The HTML input string to be split.
+ expected_documents (List[Document]): List of expected Document objects.
+ test_case (str): Description of the test case.
+
+ Raises:
+ AssertionError: If the number of documents or their content/metadata
+ does not match the expected values.
"""
- docs = splitter.split_text(content)
- expected = [
- Document(
- page_content="Reference content.",
- metadata={"Header 1": "Sample Document", "Header 2": "Section"},
+ splitter = html_header_splitter_splitter_factory(
+ headers_to_split_on=headers_to_split_on
+ )
+ docs = splitter.split_text(html_input)
+
+ assert len(docs) == len(expected_documents), (
+ f"Test Case '{test_case}' Failed: Number of documents mismatch. "
+ f"Expected {len(expected_documents)}, got {len(docs)}."
+ )
+ for idx, (doc, expected) in enumerate(zip(docs, expected_documents), start=1):
+ assert doc.page_content == expected.page_content, (
+ f"Test Case '{test_case}' Failed at Document {idx}: "
+ f"Content mismatch.\nExpected: {expected.page_content}"
+ "\nGot: {doc.page_content}"
+ )
+ assert doc.metadata == expected.metadata, (
+ f"Test Case '{test_case}' Failed at Document {idx}: "
+ f"Metadata mismatch.\nExpected: {expected.metadata}\nGot: {doc.metadata}"
+ )
+
+
+ "headers_to_split_on, html_content, expected_output, test_case",
+ [
+ (
+ # Test Case A: Split on h1 and h2 with h3 in content
+ [("h1", "Header 1"), ("h2", "Header 2"), ("h3", "Header 3")],
+ """
+ <!DOCTYPE html>
+ <html>
+ <body>
+ <div>
+ <h1>Foo</h1>
+ <p>Some intro text about Foo.</p>
+ <div>
+ <h2>Bar main section</h2>
+ <p>Some intro text about Bar.</p>
+ <h3>Bar subsection 1</h3>
+ <p>Some text about the first subtopic of Bar.</p>
+ <h3>Bar subsection 2</h3>
+ <p>Some text about the second subtopic of Bar.</p>
+ </div>
+ <div>
+ <h2>Baz</h2>
+ <p>Some text about Baz</p>
+ </div>
+ <br>
+ <p>Some concluding text about Foo</p>
+ </div>
+ </body>
+ </html>
+ """,
+ [
+ Document(metadata={"Header 1": "Foo"}, page_content="Foo"),
+ Document(
+ metadata={"Header 1": "Foo"},
+ page_content="Some intro text about Foo.",
+ ),
+ Document(
+ metadata={"Header 1": "Foo", "Header 2": "Bar main section"},
+ page_content="Bar main section",
+ ),
+ Document(
+ metadata={"Header 1": "Foo", "Header 2": "Bar main section"},
+ page_content="Some intro text about Bar.",
+ ),
+ Document(
+ metadata={
+ "Header 1": "Foo",
+ "Header 2": "Bar main section",
+ "Header 3": "Bar subsection 1",
+ },
+ page_content="Bar subsection 1",
+ ),
+ Document(
+ metadata={
+ "Header 1": "Foo",
+ "Header 2": "Bar main section",
+ "Header 3": "Bar subsection 1",
+ },
+ page_content="Some text about the first subtopic of Bar.",
+ ),
+ Document(
+ metadata={
+ "Header 1": "Foo",
+ "Header 2": "Bar main section",
+ "Header 3": "Bar subsection 2",
+ },
+ page_content="Bar subsection 2",
+ ),
+ Document(
+ metadata={
+ "Header 1": "Foo",
+ "Header 2": "Bar main section",
+ "Header 3": "Bar subsection 2",
+ },
+ page_content="Some text about the second subtopic of Bar.",
+ ),
+ Document(
+ metadata={"Header 1": "Foo", "Header 2": "Baz"}, page_content="Baz"
+ ),
+ Document(
+ metadata={"Header 1": "Foo"},
+ page_content=(
+ "Some text about Baz \nSome concluding text about Foo"
+ ),
+ ),
+ ],
+ "Test Case A: Split on h1, h2, and h3 with nested headers",
),
- Document(
- page_content="Item 1 Item 2 Item 3 \nSome text \nSome more text",
- metadata={"Header 1": "Sample Document", "Header 2": "Lists"},
+ (
+ # Test Case B: Split on h1 only without any headers
+ [("h1", "Header 1")],
+ """
+ <html>
+ <body>
+ <p>Paragraph one.</p>
+ <p>Paragraph two.</p>
+ <p>Paragraph three.</p>
+ </body>
+ </html>
+ """,
+ [
+ Document(
+ metadata={},
+ page_content="Paragraph one. \nParagraph two. \nParagraph three.",
+ )
+ ],
+ "Test Case B: Split on h1 only without any headers",
),
- ]
- assert docs == expected
+ ],
+)
+def test_additional_html_header_text_splitter(
+ html_header_splitter_splitter_factory: Any,
+ headers_to_split_on: List[Tuple[str, str]],
+ html_content: str,
+ expected_output: List[Document],
+ test_case: str,
+) -> None:
+ """
+ Test the HTML header text splitter.
+
+ Args:
+ html_header_splitter_splitter_factory (Any): Factory function to create
+ the HTML header splitter.
+ headers_to_split_on (List[Tuple[str, str]]): List of headers to split on.
+ html_content (str): HTML content to be split.
+ expected_output (List[Document]): Expected list of Document objects.
+ test_case (str): Description of the test case.
+
+ Raises:
+ AssertionError: If the number of documents or their content/metadata
+ does not match the expected output.
+ """
+ splitter = html_header_splitter_splitter_factory(
+ headers_to_split_on=headers_to_split_on
+ )
+ docs = splitter.split_text(html_content)
+
+ assert len(docs) == len(expected_output), (
+ f"{test_case} Failed: Number of documents mismatch. "
+ f"Expected {len(expected_output)}, got {len(docs)}."
+ )
+ for idx, (doc, expected) in enumerate(zip(docs, expected_output), start=1):
+ assert doc.page_content == expected.page_content, (
+ f"{test_case} Failed at Document {idx}: "
+ f"Content mismatch.\nExpected: {expected.page_content}\n"
+ "Got: {doc.page_content}"
+ )
+ assert doc.metadata == expected.metadata, (
+ f"{test_case} Failed at Document {idx}: "
+ f"Metadata mismatch.\nExpected: {expected.metadata}\nGot: {doc.metadata}"
+ )
- with open(tmp_path / "doc.html", "w") as tmp:
- tmp.write(content)
- docs_from_file = splitter.split_text_from_file(tmp_path / "doc.html")
- assert docs_from_file == expected
+ "headers_to_split_on, html_content, expected_output, test_case",
+ [
+ (
+ # Test Case C: Split on h1, h2, and h3 with no headers present
+ [("h1", "Header 1"), ("h2", "Header 2"), ("h3", "Header 3")],
+ """
+ <html>
+ <body>
+ <p>Just some random text without headers.</p>
+ <div>
+ <span>More text here.</span>
+ </div>
+ </body>
+ </html>
+ """,
+ [
+ Document(
+ page_content="Just some random text without headers."
+ " \nMore text here.",
+ metadata={},
+ )
+ ],
+ "Test Case C: Split on h1, h2, and h3 without any headers",
+ )
+ ],
+)
+def test_html_no_headers_with_multiple_splitters(
+ html_header_splitter_splitter_factory: Any,
+ headers_to_split_on: List[Tuple[str, str]],
+ html_content: str,
+ expected_output: List[Document],
+ test_case: str,
+) -> None:
+ """
+ Test HTML content splitting without headers using multiple splitters.
+ Args:
+ html_header_splitter_splitter_factory (Any): Factory to create the
+ HTML header splitter.
+ headers_to_split_on (List[Tuple[str, str]]): List of headers to split on.
+ html_content (str): HTML content to be split.
+ expected_output (List[Document]): Expected list of Document objects
+ after splitting.
+ test_case (str): Description of the test case.
+ Raises:
+ AssertionError: If the number of documents or their content/metadata
+ does not match the expected output.
+ """
+ splitter = html_header_splitter_splitter_factory(
+ headers_to_split_on=headers_to_split_on
+ )
+ docs = splitter.split_text(html_content)
+
+ assert len(docs) == len(expected_output), (
+ f"{test_case} Failed: Number of documents mismatch. "
+ f"Expected {len(expected_output)}, got {len(docs)}."
+ )
+ for idx, (doc, expected) in enumerate(zip(docs, expected_output), start=1):
+ assert doc.page_content == expected.page_content, (
+ f"{test_case} Failed at Document {idx}: "
+ f"Content mismatch.\nExpected: {expected.page_content}\n"
+ "Got: {doc.page_content}"
+ )
+ assert doc.metadata == expected.metadata, (
+ f"{test_case} Failed at Document {idx}: "
+ f"Metadata mismatch.\nExpected: {expected.metadata}\nGot: {doc.metadata}"
+ )
def test_split_text_on_tokens() -> None:
@@ -2480,7 +2907,8 @@ def test_html_splitter_with_custom_extractor() -> None:
expected = [
Document(
- page_content="This is an iframe: [iframe:http://example.com](http://example.com)",
+ page_content="This is an iframe: "
+ "[iframe:http://example.com](http://example.com)",
metadata={"Header 1": "Section 1"},
),
]
@@ -2598,7 +3026,7 @@ def test_html_splitter_with_small_chunk_size() -> None:
"""Test HTML splitting with a very small chunk size to validate chunking."""
html_content = """
<h1>Section 1</h1>
- <p>This is some long text that should be split into multiple chunks due to the
+ <p>This is some long text that should be split into multiple chunks due to the
small chunk size.</p>
"""
splitter = HTMLSemanticPreservingSplitter(