The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
style(text-splitters): lint (#34865)
Mason Daugherty51f13f7bff7b
diff --git a/libs/text-splitters/langchain_text_splitters/__init__.py b/libs/text-splitters/langchain_text_splitters/__init__.pyindex 623582f8d9..7dc07f50b2 100644--- a/libs/text-splitters/langchain_text_splitters/__init__.py+++ b/libs/text-splitters/langchain_text_splitters/__init__.py@@ -1,6 +1,7 @@"""Text Splitters are classes for splitting text.!!! note+`MarkdownHeaderTextSplitter` and `HTMLHeaderTextSplitter` do not derive from`TextSplitter`."""diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.pyindex 6e4f9b0ec9..19114b253d 100644--- a/libs/text-splitters/langchain_text_splitters/base.py+++ b/libs/text-splitters/langchain_text_splitters/base.py@@ -53,7 +53,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):add_start_index: bool = False, # noqa: FBT001,FBT002strip_whitespace: bool = True, # noqa: FBT001,FBT002) -> None:- """Create a new TextSplitter.+ """Create a new `TextSplitter`.Args:chunk_size: Maximum size of chunks to return@@ -68,7 +68,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):Raises:ValueError: If `chunk_size` is less than or equal to 0ValueError: If `chunk_overlap` is less than 0- ValueError: If `chunk_overlap` is greater than chunk_size+ ValueError: If `chunk_overlap` is greater than `chunk_size`"""if chunk_size <= 0:msg = f"chunk_size must be > 0, got {chunk_size}"@@ -204,7 +204,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):Returns:An instance of `TextSplitter` using the Hugging Face tokenizer for length- calculation.+ calculation."""if not _HAS_TRANSFORMERS:msg = (@@ -235,8 +235,9 @@ class TextSplitter(BaseDocumentTransformer, ABC):Args:encoding_name: The name of the tiktoken encoding to use.- model_name: The name of the model to use. If provided, this will- override the `encoding_name`.+ model_name: The name of the model to use.++ If provided, this will override the `encoding_name`.allowed_special: Special tokens that are allowed during encoding.disallowed_special: Special tokens that are disallowed during encoding.@@ -309,8 +310,9 @@ class TokenTextSplitter(TextSplitter):Args:encoding_name: The name of the tiktoken encoding to use.- model_name: The name of the model to use. If provided, this will- override the `encoding_name`.+ model_name: The name of the model to use.++ If provided, this will override the `encoding_name`.allowed_special: Special tokens that are allowed during encoding.disallowed_special: Special tokens that are disallowed during encoding.@@ -406,10 +408,13 @@ class Tokenizer:chunk_overlap: int"""Overlap in tokens between chunks"""+tokens_per_chunk: int"""Maximum number of tokens per chunk"""+decode: Callable[[list[int]], str]""" Function to decode a list of token IDs to a string"""+encode: Callable[[str], list[int]]""" Function to encode a string to a list of token IDs"""diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.pyindex 8247ecc538..16e7237667 100644--- a/libs/text-splitters/langchain_text_splitters/html.py+++ b/libs/text-splitters/langchain_text_splitters/html.py@@ -152,13 +152,17 @@ class HTMLHeaderTextSplitter:Args:headers_to_split_on: A list of `(header_tag,header_name)` pairs representing the headers that define splitting- boundaries. For example, `[("h1", "Header 1"), ("h2", "Header 2")]`- will split content by `h1` and `h2` tags, assigning their textual- content to the `Document` metadata.+ boundaries.++ For example, `[("h1", "Header 1"), ("h2", "Header 2")]` will split+ content by `h1` and `h2` tags, assigning their textual content to the+ `Document` metadata.return_each_element: If `True`, every HTML element encountered(including headers, paragraphs, etc.) is returned as a separate- `Document`. If `False`, content under the same header hierarchy is- aggregated into fewer `Document` objects.+ `Document`.++ If `False`, content under the same header hierarchy is aggregated into+ fewer `Document` objects."""# Sort headers by their numeric level so that h1 < h2 < h3...self.headers_to_split_on = sorted(@@ -175,9 +179,10 @@ class HTMLHeaderTextSplitter:text: The HTML text to split.Returns:- A list of split Document objects. Each `Document` contains- `page_content` holding the extracted text and `metadata` that maps- the header hierarchy to their corresponding titles.+ A list of split `Document` objects.++ Each `Document` contains `page_content` holding the extracted text and+ `metadata` that maps the header hierarchy to their corresponding titles."""return self.split_text_from_file(StringIO(text))@@ -192,9 +197,10 @@ class HTMLHeaderTextSplitter:**kwargs: Additional keyword arguments for the request.Returns:- A list of split Document objects. Each `Document` contains- `page_content` holding the extracted text and `metadata` that maps- the header hierarchy to their corresponding titles.+ A list of split `Document` objects.++ Each `Document` contains `page_content` holding the extracted text and+ `metadata` that maps the header hierarchy to their corresponding titles.Raises:requests.RequestException: If the HTTP request fails.@@ -210,9 +216,10 @@ class HTMLHeaderTextSplitter:file: A file path or a file-like object containing HTML content.Returns:- A list of split Document objects. Each `Document` contains- `page_content` holding the extracted text and `metadata` that maps- the header hierarchy to their corresponding titles.+ A list of split `Document` objects.++ Each `Document` contains `page_content` holding the extracted text and+ `metadata` that maps the header hierarchy to their corresponding titles."""if isinstance(file, str):html_content = pathlib.Path(file).read_text(encoding="utf-8")@@ -223,9 +230,9 @@ class HTMLHeaderTextSplitter:def _generate_documents(self, html_content: str) -> Iterator[Document]:"""Private method that performs a DFS traversal over the DOM and yields.- Document objects on-the-fly. This approach maintains the same splitting- logic (headers vs. non-headers, chunking, etc.) while walking the DOM- explicitly in code.+ Document objects on-the-fly. This approach maintains the same splitting logic+ (headers vs. non-headers, chunking, etc.) while walking the DOM explicitly in+ code.Args:html_content: The raw HTML content.@@ -341,7 +348,7 @@ class HTMLHeaderTextSplitter:class HTMLSectionSplitter:"""Splitting HTML files based on specified tag and font sizes.- Requires `lxml` package.+ Requires lxml package."""def __init__(@@ -352,9 +359,11 @@ class HTMLSectionSplitter:"""Create a new `HTMLSectionSplitter`.Args:- headers_to_split_on: list of tuples of headers we want to track mapped to- (arbitrary) keys for metadata. Allowed header values: `h1`, `h2`, `h3`,- `h4`, `h5`, `h6` e.g. `[("h1", "Header 1"), ("h2", "Header 2"]`.+ headers_to_split_on: List of tuples of headers we want to track mapped to+ (arbitrary) keys for metadata.++ Allowed header values: `h1`, `h2`, `h3`, `h4`, `h5`, `h6`, e.g.:+ `[("h1", "Header 1"), ("h2", "Header 2"]`.**kwargs: Additional optional arguments for customizations."""@@ -431,7 +440,9 @@ class HTMLSectionSplitter:html_doc: The HTML document to be split into sections.Returns:- A list of dictionaries representing sections. Each dictionary contains:+ A list of dictionaries representing sections.++ Each dictionary contains:* `'header'`: The header text or a default title for the first section.* `'content'`: The content under the header.@@ -523,7 +534,7 @@ class HTMLSectionSplitter:file: A file path or a file-like object containing HTML content.Returns:- A list of split Document objects.+ A list of split `Document` objects."""file_content = file.getvalue()file_content = self.convert_possible_tags_to_header(file_content)@@ -548,7 +559,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):Splits HTML content by headers into generalized chunks, preserving semanticstructure. If chunks exceed the maximum chunk size, it uses- RecursiveCharacterTextSplitter for further splitting.+ `RecursiveCharacterTextSplitter` for further splitting.The splitter preserves full HTML elements and converts links to Markdown-like links.It can also preserve images, videos, and audio elements by converting them into@@ -610,10 +621,10 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):"""Initialize splitter.Args:- headers_to_split_on: HTML headers (e.g., `h1`, `h2`)- that define content sections.- max_chunk_size: Maximum size for each chunk, with allowance for- exceeding this limit to preserve semantics.+ headers_to_split_on: HTML headers (e.g., `h1`, `h2`) that define content+ sections.+ max_chunk_size: Maximum size for each chunk, with allowance for exceeding+ this limit to preserve semantics.chunk_overlap: Number of characters to overlap between chunks to ensurecontextual continuity.separators: Delimiters used by `RecursiveCharacterTextSplitter` for@@ -622,26 +633,24 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):intact during splitting.preserve_links: Converts `a` tags to Markdown links (`[text](url)`).preserve_images: Converts `img` tags to Markdown images (``).- preserve_videos: Converts `video` tags to Markdown- video links (``).- preserve_audio: Converts `audio` tags to Markdown- audio links (``).- custom_handlers: Optional custom handlers for- specific HTML tags, allowing tailored extraction or processing.+ preserve_videos: Converts `video` tags to Markdown video links+ (``).+ preserve_audio: Converts `audio` tags to Markdown audio links+ (``).+ custom_handlers: Optional custom handlers for specific HTML tags, allowing+ tailored extraction or processing.stopword_removal: Optionally remove stopwords from the text.stopword_lang: The language of stopwords to remove.- normalize_text: Optionally normalize text- (e.g., lowercasing, removing punctuation).- external_metadata: Additional metadata to attach to- the Document objects.- allowlist_tags: Only these tags will be retained in- the HTML.+ normalize_text: Optionally normalize text (e.g., lowercasing, removing+ punctuation).+ external_metadata: Additional metadata to attach to the Document objects.+ allowlist_tags: Only these tags will be retained in the HTML.denylist_tags: These tags will be removed from the HTML.- preserve_parent_metadata: Whether to pass through parent document- metadata to split documents when calling+ preserve_parent_metadata: Whether to pass through parent document metadata+ to split documents when calling`transform_documents/atransform_documents()`.- keep_separator: Whether separators- should be at the beginning of a chunk, at the end, or not at all.+ keep_separator: Whether separators should be at the beginning of a chunk, at+ the end, or not at all.Raises:ImportError: If BeautifulSoup or NLTK (when stopword removal is enabled)@@ -985,8 +994,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):"""Creates Document objects from the provided headers, content, and elements.Args:- headers: The headers to attach as metadata to the Document.- content: The content of the Document.+ headers: The headers to attach as metadata to the `Document`.+ content: The content of the `Document`.preserved_elements: Preserved elements to be reinserted into the content.Returns:diff --git a/libs/text-splitters/langchain_text_splitters/json.py b/libs/text-splitters/langchain_text_splitters/json.pyindex 8c80b9d7a5..1162aa0aa1 100644--- a/libs/text-splitters/langchain_text_splitters/json.py+++ b/libs/text-splitters/langchain_text_splitters/json.py@@ -20,9 +20,11 @@ class RecursiveJsonSplitter:max_chunk_size: int = 2000"""The maximum size for each chunk."""+min_chunk_size: int = 1800"""The minimum size for each chunk, derived from `max_chunk_size` if not- explicitly provided."""+ explicitly provided.+ """def __init__(self, max_chunk_size: int = 2000, min_chunk_size: int | None = None@@ -35,8 +37,10 @@ class RecursiveJsonSplitter:Args:max_chunk_size: The maximum size for a chunk.- min_chunk_size: The minimum size for a chunk. If `None`,- defaults to the maximum chunk size minus 200, with a lower bound of 50.+ min_chunk_size: The minimum size for a chunk.++ If `None`, defaults to the maximum chunk size minus 200, with a lower+ bound of 50."""super().__init__()self.max_chunk_size = max_chunk_sizediff --git a/libs/text-splitters/langchain_text_splitters/jsx.py b/libs/text-splitters/langchain_text_splitters/jsx.pyindex 4ca23324d8..e335ae034b 100644--- a/libs/text-splitters/langchain_text_splitters/jsx.py+++ b/libs/text-splitters/langchain_text_splitters/jsx.py@@ -9,20 +9,20 @@ from langchain_text_splitters import RecursiveCharacterTextSplitterclass JSFrameworkTextSplitter(RecursiveCharacterTextSplitter):"""Text splitter that handles React (JSX), Vue, and Svelte code.- This splitter extends RecursiveCharacterTextSplitter to handle- React (JSX), Vue, and Svelte code by:+ This splitter extends `RecursiveCharacterTextSplitter` to handle React (JSX), Vue,+ and Svelte code by:1. Detecting and extracting custom component tags from the text2. Using those tags as additional separators along with standard JS syntaxThe splitter combines:- * Custom component tags as separators (e.g. <Component, <div)+ * Custom component tags as separators (e.g. `<Component`, `<div`)* JavaScript syntax elements (function, const, if, etc)* Standard text splitting on newlines- This allows chunks to break at natural boundaries in- React, Vue, and Svelte component code.+ This allows chunks to break at natural boundaries in React, Vue, and Svelte+ component code."""def __init__(diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.pyindex 3d6dcf9f72..f4c2b05530 100644--- a/libs/text-splitters/langchain_text_splitters/markdown.py+++ b/libs/text-splitters/langchain_text_splitters/markdown.py@@ -15,7 +15,7 @@ class MarkdownTextSplitter(RecursiveCharacterTextSplitter):"""Attempts to split the text along Markdown-formatted headings."""def __init__(self, **kwargs: Any) -> None:- """Initialize a MarkdownTextSplitter."""+ """Initialize a `MarkdownTextSplitter`."""separators = self.get_separators_for_language(Language.MARKDOWN)super().__init__(separators=separators, **kwargs)@@ -30,15 +30,17 @@ class MarkdownHeaderTextSplitter:strip_headers: bool = True, # noqa: FBT001,FBT002custom_header_patterns: dict[str, int] | None = None,) -> None:- """Create a new MarkdownHeaderTextSplitter.+ """Create a new `MarkdownHeaderTextSplitter`.Args:headers_to_split_on: Headers we want to trackreturn_each_line: Return each line w/ associated headersstrip_headers: Strip split headers from the content of the chunkcustom_header_patterns: Optional dict mapping header patterns to their- levels. For example: {"**": 1, "***": 2} to treat **Header** as- level 1 and ***Header*** as level 2 headers.+ levels.++ For example: `{"**": 1, "***": 2}` to treat `**Header**` as level 1 and+ `***Header***` as level 2 headers."""# Output line-by-line or aggregated into chunks w/ common headersself.return_each_line = return_each_line@@ -60,7 +62,7 @@ class MarkdownHeaderTextSplitter:sep: The separator pattern to matchReturns:- True if the line matches the custom pattern format+ `True` if the line matches the custom pattern format"""if sep not in self.custom_header_patterns:return False@@ -90,7 +92,7 @@ class MarkdownHeaderTextSplitter:lines: Line of text / associated header metadataReturns:- List of Documents with common metadata aggregated.+ List of `Document` objects with common metadata aggregated."""aggregated_chunks: list[LineType] = []@@ -140,16 +142,22 @@ class MarkdownHeaderTextSplitter:"""# Split the input text by newline character ("\n").lines = text.split("\n")+# Final outputlines_with_metadata: list[LineType] = []+# Content and metadata of the chunk currently being processedcurrent_content: list[str] = []+current_metadata: dict[str, str] = {}+# Keep track of the nested header structureheader_stack: list[HeaderType] = []+initial_metadata: dict[str, str] = {}in_code_block = False+opening_fence = ""for line in lines:@@ -273,14 +281,14 @@ class MarkdownHeaderTextSplitter:class LineType(TypedDict):- """Line type as typed dict."""+ """Line type as `TypedDict`."""metadata: dict[str, str]content: strclass HeaderType(TypedDict):- """Header type as typed dict."""+ """Header type as `TypedDict`."""level: intname: str@@ -292,8 +300,8 @@ class ExperimentalMarkdownSyntaxTextSplitter:This splitter aims to retain the exact whitespace of the original text whileextracting structured metadata, such as headers. It is a re-implementation of the- MarkdownHeaderTextSplitter with notable changes to the approach and- additional features.+ `MarkdownHeaderTextSplitter` with notable changes to the approach and additional+ features.Key Features:@@ -305,18 +313,18 @@ class ExperimentalMarkdownSyntaxTextSplitter:`headers_to_split_on` parameter.Example:- ```python- headers_to_split_on = [- ("#", "Header 1"),- ("##", "Header 2"),- ]- splitter = ExperimentalMarkdownSyntaxTextSplitter(- headers_to_split_on=headers_to_split_on- )- chunks = splitter.split(text)- for chunk in chunks:- print(chunk)- ```+ ```python+ headers_to_split_on = [+ ("#", "Header 1"),+ ("##", "Header 2"),+ ]+ splitter = ExperimentalMarkdownSyntaxTextSplitter(+ headers_to_split_on=headers_to_split_on+ )+ chunks = splitter.split(text)+ for chunk in chunks:+ print(chunk)+ ```This class is currently experimental and subject to change based on feedback andfurther development.@@ -334,14 +342,14 @@ class ExperimentalMarkdownSyntaxTextSplitter:chunks based on specified headers and formatting preferences.Args:- headers_to_split_on (Union[list[tuple[str, str]], None]):- A list of tuples, where each tuple contains a header tag (e.g., "h1")- and its corresponding metadata key. If `None`, default headers are used.- return_each_line (bool):- Whether to return each line as an individual chunk.+ headers_to_split_on: A list of tuples, where each tuple contains a header+ tag (e.g., "h1") and its corresponding metadata key.++ If `None`, default headers are used.+ return_each_line: Whether to return each line as an individual chunk.+Defaults to `False`, which aggregates lines into larger chunks.- strip_headers (bool):- Whether to exclude headers from the resulting chunks.+ strip_headers: Whether to exclude headers from the resulting chunks."""self.chunks: list[Document] = []self.current_chunk = Document(page_content="")@@ -365,9 +373,8 @@ class ExperimentalMarkdownSyntaxTextSplitter:"""Split the input text into structured chunks.This method processes the input text line by line, identifying and handling- specific patterns such as headers, code blocks, and horizontal rules to- split it into structured chunks based on headers, code blocks, and- horizontal rules.+ specific patterns such as headers, code blocks, and horizontal rules to split it+ into structured chunks based on headers, code blocks, and horizontal rules.Args:text: The input text to be split into chunks.diff --git a/libs/text-splitters/langchain_text_splitters/python.py b/libs/text-splitters/langchain_text_splitters/python.pyindex ccab44bcf6..71295f3995 100644--- a/libs/text-splitters/langchain_text_splitters/python.py+++ b/libs/text-splitters/langchain_text_splitters/python.py@@ -12,6 +12,6 @@ class PythonCodeTextSplitter(RecursiveCharacterTextSplitter):"""Attempts to split the text along Python syntax."""def __init__(self, **kwargs: Any) -> None:- """Initialize a PythonCodeTextSplitter."""+ """Initialize a `PythonCodeTextSplitter`."""separators = self.get_separators_for_language(Language.PYTHON)super().__init__(separators=separators, **kwargs)diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.pyindex 376e48ae9f..03bde0a60c 100644--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py@@ -27,16 +27,17 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):tokens_per_chunk: int | None = None,**kwargs: Any,) -> None:- """Create a new TextSplitter.+ """Create a new `TextSplitter`.Args:chunk_overlap: The number of tokens to overlap between chunks.model_name: The name of the sentence transformer model to use.- tokens_per_chunk: The number of tokens per chunk. If `None`, uses the- maximum tokens allowed by the model.+ tokens_per_chunk: The number of tokens per chunk.++ If `None`, uses the maximum tokens allowed by the model.Raises:- ImportError: If the sentence_transformers package is not installed.+ ImportError: If the `sentence_transformers` package is not installed."""super().__init__(**kwargs, chunk_overlap=chunk_overlap)@@ -82,7 +83,7 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):Returns:A list of string components derived from the input text after encoding and- processing.+ processing."""def encode_strip_start_and_stop_token_ids(text: str) -> list[int]:@@ -107,7 +108,7 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):text: The input text for which the token count is calculated.Returns:- int: The number of tokens in the encoded text.+ The number of tokens in the encoded text."""return len(self._encode(text))