Skip to content

The agent engineering platform.

style(text-splitters): lint (#34865)

Mason Daugherty51f13f7bff7b
diff --git a/libs/text-splitters/langchain_text_splitters/__init__.py b/libs/text-splitters/langchain_text_splitters/__init__.py
index 623582f8d9..7dc07f50b2 100644
--- a/libs/text-splitters/langchain_text_splitters/__init__.py
+++ b/libs/text-splitters/langchain_text_splitters/__init__.py
@@ -1,6 +1,7 @@
"""Text Splitters are classes for splitting text.
!!! note
+
`MarkdownHeaderTextSplitter` and `HTMLHeaderTextSplitter` do not derive from
`TextSplitter`.
"""
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py
index 6e4f9b0ec9..19114b253d 100644
--- a/libs/text-splitters/langchain_text_splitters/base.py
+++ b/libs/text-splitters/langchain_text_splitters/base.py
@@ -53,7 +53,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
add_start_index: bool = False, # noqa: FBT001,FBT002
strip_whitespace: bool = True, # noqa: FBT001,FBT002
) -> None:
- """Create a new TextSplitter.
+ """Create a new `TextSplitter`.
Args:
chunk_size: Maximum size of chunks to return
@@ -68,7 +68,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
Raises:
ValueError: If `chunk_size` is less than or equal to 0
ValueError: If `chunk_overlap` is less than 0
- ValueError: If `chunk_overlap` is greater than chunk_size
+ ValueError: If `chunk_overlap` is greater than `chunk_size`
"""
if chunk_size <= 0:
msg = f"chunk_size must be > 0, got {chunk_size}"
@@ -204,7 +204,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
Returns:
An instance of `TextSplitter` using the Hugging Face tokenizer for length
- calculation.
+ calculation.
"""
if not _HAS_TRANSFORMERS:
msg = (
@@ -235,8 +235,9 @@ class TextSplitter(BaseDocumentTransformer, ABC):
Args:
encoding_name: The name of the tiktoken encoding to use.
- model_name: The name of the model to use. If provided, this will
- override the `encoding_name`.
+ model_name: The name of the model to use.
+
+ If provided, this will override the `encoding_name`.
allowed_special: Special tokens that are allowed during encoding.
disallowed_special: Special tokens that are disallowed during encoding.
@@ -309,8 +310,9 @@ class TokenTextSplitter(TextSplitter):
Args:
encoding_name: The name of the tiktoken encoding to use.
- model_name: The name of the model to use. If provided, this will
- override the `encoding_name`.
+ model_name: The name of the model to use.
+
+ If provided, this will override the `encoding_name`.
allowed_special: Special tokens that are allowed during encoding.
disallowed_special: Special tokens that are disallowed during encoding.
@@ -406,10 +408,13 @@ class Tokenizer:
chunk_overlap: int
"""Overlap in tokens between chunks"""
+
tokens_per_chunk: int
"""Maximum number of tokens per chunk"""
+
decode: Callable[[list[int]], str]
""" Function to decode a list of token IDs to a string"""
+
encode: Callable[[str], list[int]]
""" Function to encode a string to a list of token IDs"""
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 8247ecc538..16e7237667 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -152,13 +152,17 @@ class HTMLHeaderTextSplitter:
Args:
headers_to_split_on: A list of `(header_tag,
header_name)` pairs representing the headers that define splitting
- boundaries. For example, `[("h1", "Header 1"), ("h2", "Header 2")]`
- will split content by `h1` and `h2` tags, assigning their textual
- content to the `Document` metadata.
+ boundaries.
+
+ For example, `[("h1", "Header 1"), ("h2", "Header 2")]` will split
+ content by `h1` and `h2` tags, assigning their textual content to the
+ `Document` metadata.
return_each_element: If `True`, every HTML element encountered
(including headers, paragraphs, etc.) is returned as a separate
- `Document`. If `False`, content under the same header hierarchy is
- aggregated into fewer `Document` objects.
+ `Document`.
+
+ If `False`, content under the same header hierarchy is aggregated into
+ fewer `Document` objects.
"""
# Sort headers by their numeric level so that h1 < h2 < h3...
self.headers_to_split_on = sorted(
@@ -175,9 +179,10 @@ class HTMLHeaderTextSplitter:
text: The HTML text to split.
Returns:
- A list of split Document objects. Each `Document` contains
- `page_content` holding the extracted text and `metadata` that maps
- the header hierarchy to their corresponding titles.
+ A list of split `Document` objects.
+
+ Each `Document` contains `page_content` holding the extracted text and
+ `metadata` that maps the header hierarchy to their corresponding titles.
"""
return self.split_text_from_file(StringIO(text))
@@ -192,9 +197,10 @@ class HTMLHeaderTextSplitter:
**kwargs: Additional keyword arguments for the request.
Returns:
- A list of split Document objects. Each `Document` contains
- `page_content` holding the extracted text and `metadata` that maps
- the header hierarchy to their corresponding titles.
+ A list of split `Document` objects.
+
+ Each `Document` contains `page_content` holding the extracted text and
+ `metadata` that maps the header hierarchy to their corresponding titles.
Raises:
requests.RequestException: If the HTTP request fails.
@@ -210,9 +216,10 @@ class HTMLHeaderTextSplitter:
file: A file path or a file-like object containing HTML content.
Returns:
- A list of split Document objects. Each `Document` contains
- `page_content` holding the extracted text and `metadata` that maps
- the header hierarchy to their corresponding titles.
+ A list of split `Document` objects.
+
+ Each `Document` contains `page_content` holding the extracted text and
+ `metadata` that maps the header hierarchy to their corresponding titles.
"""
if isinstance(file, str):
html_content = pathlib.Path(file).read_text(encoding="utf-8")
@@ -223,9 +230,9 @@ class HTMLHeaderTextSplitter:
def _generate_documents(self, html_content: str) -> Iterator[Document]:
"""Private method that performs a DFS traversal over the DOM and yields.
- Document objects on-the-fly. This approach maintains the same splitting
- logic (headers vs. non-headers, chunking, etc.) while walking the DOM
- explicitly in code.
+ Document objects on-the-fly. This approach maintains the same splitting logic
+ (headers vs. non-headers, chunking, etc.) while walking the DOM explicitly in
+ code.
Args:
html_content: The raw HTML content.
@@ -341,7 +348,7 @@ class HTMLHeaderTextSplitter:
class HTMLSectionSplitter:
"""Splitting HTML files based on specified tag and font sizes.
- Requires `lxml` package.
+ Requires lxml package.
"""
def __init__(
@@ -352,9 +359,11 @@ class HTMLSectionSplitter:
"""Create a new `HTMLSectionSplitter`.
Args:
- headers_to_split_on: list of tuples of headers we want to track mapped to
- (arbitrary) keys for metadata. Allowed header values: `h1`, `h2`, `h3`,
- `h4`, `h5`, `h6` e.g. `[("h1", "Header 1"), ("h2", "Header 2"]`.
+ headers_to_split_on: List of tuples of headers we want to track mapped to
+ (arbitrary) keys for metadata.
+
+ Allowed header values: `h1`, `h2`, `h3`, `h4`, `h5`, `h6`, e.g.:
+ `[("h1", "Header 1"), ("h2", "Header 2"]`.
**kwargs: Additional optional arguments for customizations.
"""
@@ -431,7 +440,9 @@ class HTMLSectionSplitter:
html_doc: The HTML document to be split into sections.
Returns:
- A list of dictionaries representing sections. Each dictionary contains:
+ A list of dictionaries representing sections.
+
+ Each dictionary contains:
* `'header'`: The header text or a default title for the first section.
* `'content'`: The content under the header.
@@ -523,7 +534,7 @@ class HTMLSectionSplitter:
file: A file path or a file-like object containing HTML content.
Returns:
- A list of split Document objects.
+ A list of split `Document` objects.
"""
file_content = file.getvalue()
file_content = self.convert_possible_tags_to_header(file_content)
@@ -548,7 +559,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
Splits HTML content by headers into generalized chunks, preserving semantic
structure. If chunks exceed the maximum chunk size, it uses
- RecursiveCharacterTextSplitter for further splitting.
+ `RecursiveCharacterTextSplitter` for further splitting.
The splitter preserves full HTML elements and converts links to Markdown-like links.
It can also preserve images, videos, and audio elements by converting them into
@@ -610,10 +621,10 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
"""Initialize splitter.
Args:
- headers_to_split_on: HTML headers (e.g., `h1`, `h2`)
- that define content sections.
- max_chunk_size: Maximum size for each chunk, with allowance for
- exceeding this limit to preserve semantics.
+ headers_to_split_on: HTML headers (e.g., `h1`, `h2`) that define content
+ sections.
+ max_chunk_size: Maximum size for each chunk, with allowance for exceeding
+ this limit to preserve semantics.
chunk_overlap: Number of characters to overlap between chunks to ensure
contextual continuity.
separators: Delimiters used by `RecursiveCharacterTextSplitter` for
@@ -622,26 +633,24 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
intact during splitting.
preserve_links: Converts `a` tags to Markdown links (`[text](url)`).
preserve_images: Converts `img` tags to Markdown images (`![alt](src)`).
- preserve_videos: Converts `video` tags to Markdown
- video links (`![video](src)`).
- preserve_audio: Converts `audio` tags to Markdown
- audio links (`![audio](src)`).
- custom_handlers: Optional custom handlers for
- specific HTML tags, allowing tailored extraction or processing.
+ preserve_videos: Converts `video` tags to Markdown video links
+ (`![video](src)`).
+ preserve_audio: Converts `audio` tags to Markdown audio links
+ (`![audio](src)`).
+ custom_handlers: Optional custom handlers for specific HTML tags, allowing
+ tailored extraction or processing.
stopword_removal: Optionally remove stopwords from the text.
stopword_lang: The language of stopwords to remove.
- normalize_text: Optionally normalize text
- (e.g., lowercasing, removing punctuation).
- external_metadata: Additional metadata to attach to
- the Document objects.
- allowlist_tags: Only these tags will be retained in
- the HTML.
+ normalize_text: Optionally normalize text (e.g., lowercasing, removing
+ punctuation).
+ external_metadata: Additional metadata to attach to the Document objects.
+ allowlist_tags: Only these tags will be retained in the HTML.
denylist_tags: These tags will be removed from the HTML.
- preserve_parent_metadata: Whether to pass through parent document
- metadata to split documents when calling
+ preserve_parent_metadata: Whether to pass through parent document metadata
+ to split documents when calling
`transform_documents/atransform_documents()`.
- keep_separator: Whether separators
- should be at the beginning of a chunk, at the end, or not at all.
+ keep_separator: Whether separators should be at the beginning of a chunk, at
+ the end, or not at all.
Raises:
ImportError: If BeautifulSoup or NLTK (when stopword removal is enabled)
@@ -985,8 +994,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
"""Creates Document objects from the provided headers, content, and elements.
Args:
- headers: The headers to attach as metadata to the Document.
- content: The content of the Document.
+ headers: The headers to attach as metadata to the `Document`.
+ content: The content of the `Document`.
preserved_elements: Preserved elements to be reinserted into the content.
Returns:
diff --git a/libs/text-splitters/langchain_text_splitters/json.py b/libs/text-splitters/langchain_text_splitters/json.py
index 8c80b9d7a5..1162aa0aa1 100644
--- a/libs/text-splitters/langchain_text_splitters/json.py
+++ b/libs/text-splitters/langchain_text_splitters/json.py
@@ -20,9 +20,11 @@ class RecursiveJsonSplitter:
max_chunk_size: int = 2000
"""The maximum size for each chunk."""
+
min_chunk_size: int = 1800
"""The minimum size for each chunk, derived from `max_chunk_size` if not
- explicitly provided."""
+ explicitly provided.
+ """
def __init__(
self, max_chunk_size: int = 2000, min_chunk_size: int | None = None
@@ -35,8 +37,10 @@ class RecursiveJsonSplitter:
Args:
max_chunk_size: The maximum size for a chunk.
- min_chunk_size: The minimum size for a chunk. If `None`,
- defaults to the maximum chunk size minus 200, with a lower bound of 50.
+ min_chunk_size: The minimum size for a chunk.
+
+ If `None`, defaults to the maximum chunk size minus 200, with a lower
+ bound of 50.
"""
super().__init__()
self.max_chunk_size = max_chunk_size
diff --git a/libs/text-splitters/langchain_text_splitters/jsx.py b/libs/text-splitters/langchain_text_splitters/jsx.py
index 4ca23324d8..e335ae034b 100644
--- a/libs/text-splitters/langchain_text_splitters/jsx.py
+++ b/libs/text-splitters/langchain_text_splitters/jsx.py
@@ -9,20 +9,20 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter
class JSFrameworkTextSplitter(RecursiveCharacterTextSplitter):
"""Text splitter that handles React (JSX), Vue, and Svelte code.
- This splitter extends RecursiveCharacterTextSplitter to handle
- React (JSX), Vue, and Svelte code by:
+ This splitter extends `RecursiveCharacterTextSplitter` to handle React (JSX), Vue,
+ and Svelte code by:
1. Detecting and extracting custom component tags from the text
2. Using those tags as additional separators along with standard JS syntax
The splitter combines:
- * Custom component tags as separators (e.g. <Component, <div)
+ * Custom component tags as separators (e.g. `<Component`, `<div`)
* JavaScript syntax elements (function, const, if, etc)
* Standard text splitting on newlines
- This allows chunks to break at natural boundaries in
- React, Vue, and Svelte component code.
+ This allows chunks to break at natural boundaries in React, Vue, and Svelte
+ component code.
"""
def __init__(
diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.py
index 3d6dcf9f72..f4c2b05530 100644
--- a/libs/text-splitters/langchain_text_splitters/markdown.py
+++ b/libs/text-splitters/langchain_text_splitters/markdown.py
@@ -15,7 +15,7 @@ class MarkdownTextSplitter(RecursiveCharacterTextSplitter):
"""Attempts to split the text along Markdown-formatted headings."""
def __init__(self, **kwargs: Any) -> None:
- """Initialize a MarkdownTextSplitter."""
+ """Initialize a `MarkdownTextSplitter`."""
separators = self.get_separators_for_language(Language.MARKDOWN)
super().__init__(separators=separators, **kwargs)
@@ -30,15 +30,17 @@ class MarkdownHeaderTextSplitter:
strip_headers: bool = True, # noqa: FBT001,FBT002
custom_header_patterns: dict[str, int] | None = None,
) -> None:
- """Create a new MarkdownHeaderTextSplitter.
+ """Create a new `MarkdownHeaderTextSplitter`.
Args:
headers_to_split_on: Headers we want to track
return_each_line: Return each line w/ associated headers
strip_headers: Strip split headers from the content of the chunk
custom_header_patterns: Optional dict mapping header patterns to their
- levels. For example: {"**": 1, "***": 2} to treat **Header** as
- level 1 and ***Header*** as level 2 headers.
+ levels.
+
+ For example: `{"**": 1, "***": 2}` to treat `**Header**` as level 1 and
+ `***Header***` as level 2 headers.
"""
# Output line-by-line or aggregated into chunks w/ common headers
self.return_each_line = return_each_line
@@ -60,7 +62,7 @@ class MarkdownHeaderTextSplitter:
sep: The separator pattern to match
Returns:
- True if the line matches the custom pattern format
+ `True` if the line matches the custom pattern format
"""
if sep not in self.custom_header_patterns:
return False
@@ -90,7 +92,7 @@ class MarkdownHeaderTextSplitter:
lines: Line of text / associated header metadata
Returns:
- List of Documents with common metadata aggregated.
+ List of `Document` objects with common metadata aggregated.
"""
aggregated_chunks: list[LineType] = []
@@ -140,16 +142,22 @@ class MarkdownHeaderTextSplitter:
"""
# Split the input text by newline character ("\n").
lines = text.split("\n")
+
# Final output
lines_with_metadata: list[LineType] = []
+
# Content and metadata of the chunk currently being processed
current_content: list[str] = []
+
current_metadata: dict[str, str] = {}
+
# Keep track of the nested header structure
header_stack: list[HeaderType] = []
+
initial_metadata: dict[str, str] = {}
in_code_block = False
+
opening_fence = ""
for line in lines:
@@ -273,14 +281,14 @@ class MarkdownHeaderTextSplitter:
class LineType(TypedDict):
- """Line type as typed dict."""
+ """Line type as `TypedDict`."""
metadata: dict[str, str]
content: str
class HeaderType(TypedDict):
- """Header type as typed dict."""
+ """Header type as `TypedDict`."""
level: int
name: str
@@ -292,8 +300,8 @@ class ExperimentalMarkdownSyntaxTextSplitter:
This splitter aims to retain the exact whitespace of the original text while
extracting structured metadata, such as headers. It is a re-implementation of the
- MarkdownHeaderTextSplitter with notable changes to the approach and
- additional features.
+ `MarkdownHeaderTextSplitter` with notable changes to the approach and additional
+ features.
Key Features:
@@ -305,18 +313,18 @@ class ExperimentalMarkdownSyntaxTextSplitter:
`headers_to_split_on` parameter.
Example:
- ```python
- headers_to_split_on = [
- ("#", "Header 1"),
- ("##", "Header 2"),
- ]
- splitter = ExperimentalMarkdownSyntaxTextSplitter(
- headers_to_split_on=headers_to_split_on
- )
- chunks = splitter.split(text)
- for chunk in chunks:
- print(chunk)
- ```
+ ```python
+ headers_to_split_on = [
+ ("#", "Header 1"),
+ ("##", "Header 2"),
+ ]
+ splitter = ExperimentalMarkdownSyntaxTextSplitter(
+ headers_to_split_on=headers_to_split_on
+ )
+ chunks = splitter.split(text)
+ for chunk in chunks:
+ print(chunk)
+ ```
This class is currently experimental and subject to change based on feedback and
further development.
@@ -334,14 +342,14 @@ class ExperimentalMarkdownSyntaxTextSplitter:
chunks based on specified headers and formatting preferences.
Args:
- headers_to_split_on (Union[list[tuple[str, str]], None]):
- A list of tuples, where each tuple contains a header tag (e.g., "h1")
- and its corresponding metadata key. If `None`, default headers are used.
- return_each_line (bool):
- Whether to return each line as an individual chunk.
+ headers_to_split_on: A list of tuples, where each tuple contains a header
+ tag (e.g., "h1") and its corresponding metadata key.
+
+ If `None`, default headers are used.
+ return_each_line: Whether to return each line as an individual chunk.
+
Defaults to `False`, which aggregates lines into larger chunks.
- strip_headers (bool):
- Whether to exclude headers from the resulting chunks.
+ strip_headers: Whether to exclude headers from the resulting chunks.
"""
self.chunks: list[Document] = []
self.current_chunk = Document(page_content="")
@@ -365,9 +373,8 @@ class ExperimentalMarkdownSyntaxTextSplitter:
"""Split the input text into structured chunks.
This method processes the input text line by line, identifying and handling
- specific patterns such as headers, code blocks, and horizontal rules to
- split it into structured chunks based on headers, code blocks, and
- horizontal rules.
+ specific patterns such as headers, code blocks, and horizontal rules to split it
+ into structured chunks based on headers, code blocks, and horizontal rules.
Args:
text: The input text to be split into chunks.
diff --git a/libs/text-splitters/langchain_text_splitters/python.py b/libs/text-splitters/langchain_text_splitters/python.py
index ccab44bcf6..71295f3995 100644
--- a/libs/text-splitters/langchain_text_splitters/python.py
+++ b/libs/text-splitters/langchain_text_splitters/python.py
@@ -12,6 +12,6 @@ class PythonCodeTextSplitter(RecursiveCharacterTextSplitter):
"""Attempts to split the text along Python syntax."""
def __init__(self, **kwargs: Any) -> None:
- """Initialize a PythonCodeTextSplitter."""
+ """Initialize a `PythonCodeTextSplitter`."""
separators = self.get_separators_for_language(Language.PYTHON)
super().__init__(separators=separators, **kwargs)
diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
index 376e48ae9f..03bde0a60c 100644
--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
@@ -27,16 +27,17 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
tokens_per_chunk: int | None = None,
**kwargs: Any,
) -> None:
- """Create a new TextSplitter.
+ """Create a new `TextSplitter`.
Args:
chunk_overlap: The number of tokens to overlap between chunks.
model_name: The name of the sentence transformer model to use.
- tokens_per_chunk: The number of tokens per chunk. If `None`, uses the
- maximum tokens allowed by the model.
+ tokens_per_chunk: The number of tokens per chunk.
+
+ If `None`, uses the maximum tokens allowed by the model.
Raises:
- ImportError: If the sentence_transformers package is not installed.
+ ImportError: If the `sentence_transformers` package is not installed.
"""
super().__init__(**kwargs, chunk_overlap=chunk_overlap)
@@ -82,7 +83,7 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
Returns:
A list of string components derived from the input text after encoding and
- processing.
+ processing.
"""
def encode_strip_start_and_stop_token_ids(text: str) -> list[int]:
@@ -107,7 +108,7 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
text: The input text for which the token count is calculated.
Returns:
- int: The number of tokens in the encoded text.
+ The number of tokens in the encoded text.
"""
return len(self._encode(text))