The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
style(text-splitters): fix some ruff preview rules (#34665)
Christophe Bornetfd694254390c
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.pyindex d3bc65adb6..6e4f9b0ec9 100644--- a/libs/text-splitters/langchain_text_splitters/base.py+++ b/libs/text-splitters/langchain_text_splitters/base.py@@ -64,6 +64,11 @@ class TextSplitter(BaseDocumentTransformer, ABC):add_start_index: If `True`, includes chunk's start index in metadatastrip_whitespace: If `True`, strips whitespace from the start and end ofevery document++ Raises:+ ValueError: If `chunk_size` is less than or equal to 0+ ValueError: If `chunk_overlap` is less than 0+ ValueError: If `chunk_overlap` is greater than chunk_size"""if chunk_size <= 0:msg = f"chunk_size must be > 0, got {chunk_size}"@@ -86,12 +91,27 @@ class TextSplitter(BaseDocumentTransformer, ABC):@abstractmethoddef split_text(self, text: str) -> list[str]:- """Split text into multiple components."""+ """Split text into multiple components.++ Args:+ text: The text to split.++ Returns:+ A list of text chunks.+ """def create_documents(self, texts: list[str], metadatas: list[dict[Any, Any]] | None = None) -> list[Document]:- """Create a list of `Document` objects from a list of texts."""+ """Create a list of `Document` objects from a list of texts.++ Args:+ texts: A list of texts to be split and converted into documents.+ metadatas: Optional list of metadata to associate with each document.++ Returns:+ A list of `Document` objects.+ """metadatas_ = metadatas or [{}] * len(texts)documents = []for i, text in enumerate(texts):@@ -109,7 +129,14 @@ class TextSplitter(BaseDocumentTransformer, ABC):return documentsdef split_documents(self, documents: Iterable[Document]) -> list[Document]:- """Split documents."""+ """Split documents.++ Args:+ documents: The documents to split.++ Returns:+ A list of split documents.+ """texts, metadatas = [], []for doc in documents:texts.append(doc.page_content)@@ -170,7 +197,15 @@ class TextSplitter(BaseDocumentTransformer, ABC):def from_huggingface_tokenizer(cls, tokenizer: PreTrainedTokenizerBase, **kwargs: Any) -> TextSplitter:- """Text splitter that uses Hugging Face tokenizer to count length."""+ """Text splitter that uses Hugging Face tokenizer to count length.++ Args:+ tokenizer: The Hugging Face tokenizer to use.++ Returns:+ An instance of `TextSplitter` using the Hugging Face tokenizer for length+ calculation.+ """if not _HAS_TRANSFORMERS:msg = ("Could not import transformers python package. "@@ -196,7 +231,21 @@ class TextSplitter(BaseDocumentTransformer, ABC):disallowed_special: Literal["all"] | Collection[str] = "all",**kwargs: Any,) -> Self:- """Text splitter that uses `tiktoken` encoder to count length."""+ """Text splitter that uses `tiktoken` encoder to count length.++ Args:+ encoding_name: The name of the tiktoken encoding to use.+ model_name: The name of the model to use. If provided, this will+ override the `encoding_name`.+ allowed_special: Special tokens that are allowed during encoding.+ disallowed_special: Special tokens that are disallowed during encoding.++ Returns:+ An instance of `TextSplitter` using tiktoken for length calculation.++ Raises:+ ImportError: If the tiktoken package is not installed.+ """if not _HAS_TIKTOKEN:msg = ("Could not import tiktoken python package. "@@ -234,7 +283,14 @@ class TextSplitter(BaseDocumentTransformer, ABC):def transform_documents(self, documents: Sequence[Document], **kwargs: Any) -> Sequence[Document]:- """Transform sequence of documents by splitting them."""+ """Transform sequence of documents by splitting them.++ Args:+ documents: The sequence of documents to split.++ Returns:+ A list of split documents.+ """return self.split_documents(list(documents))@@ -249,7 +305,18 @@ class TokenTextSplitter(TextSplitter):disallowed_special: Literal["all"] | Collection[str] = "all",**kwargs: Any,) -> None:- """Create a new TextSplitter."""+ """Create a new `TextSplitter`.++ Args:+ encoding_name: The name of the tiktoken encoding to use.+ model_name: The name of the model to use. If provided, this will+ override the `encoding_name`.+ allowed_special: Special tokens that are allowed during encoding.+ disallowed_special: Special tokens that are disallowed during encoding.++ Raises:+ ImportError: If the tiktoken package is not installed.+ """super().__init__(**kwargs)if not _HAS_TIKTOKEN:msg = (@@ -348,7 +415,15 @@ class Tokenizer:def split_text_on_tokens(*, text: str, tokenizer: Tokenizer) -> list[str]:- """Split incoming text and return chunks using tokenizer."""+ """Split incoming text and return chunks using tokenizer.++ Args:+ text: The input text to be split.+ tokenizer: The tokenizer to use for splitting.++ Returns:+ A list of text chunks.+ """splits: list[str] = []input_ids = tokenizer.encode(text)start_idx = 0diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.pyindex 5bcc62b99f..469afdb913 100644--- a/libs/text-splitters/langchain_text_splitters/character.py+++ b/libs/text-splitters/langchain_text_splitters/character.py@@ -23,7 +23,14 @@ class CharacterTextSplitter(TextSplitter):self._is_separator_regex = is_separator_regexdef split_text(self, text: str) -> list[str]:- """Split into chunks without re-inserting lookaround separators."""+ """Split into chunks without re-inserting lookaround separators.++ Args:+ text: The text to split.++ Returns:+ A list of text chunks.+ """# 1. Determine split pattern: raw regex or escaped literalsep_pattern = (self._separator if self._is_separator_regex else re.escape(self._separator)@@ -103,13 +110,13 @@ class RecursiveCharacterTextSplitter(TextSplitter):# Get appropriate separator to useseparator = separators[-1]new_separators = []- for i, _s in enumerate(separators):- separator_ = _s if self._is_separator_regex else re.escape(_s)- if not _s:- separator = _s+ for i, s_ in enumerate(separators):+ separator_ = s_ if self._is_separator_regex else re.escape(s_)+ if not s_:+ separator = s_breakif re.search(separator_, text):- separator = _s+ separator = s_new_separators = separators[i + 1 :]break@@ -177,6 +184,9 @@ class RecursiveCharacterTextSplitter(TextSplitter):Returns:A list of separators appropriate for the specified language.++ Raises:+ ValueError: If the language is not implemented or supported."""if language in {Language.C, Language.CPP}:return [diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.pyindex b98a1c445e..8247ecc538 100644--- a/libs/text-splitters/langchain_text_splitters/html.py+++ b/libs/text-splitters/langchain_text_splitters/html.py@@ -232,6 +232,9 @@ class HTMLHeaderTextSplitter:Yields:Document objects as they are created.++ Raises:+ ImportError: If BeautifulSoup is not installed."""if not _HAS_BS4:msg = (@@ -362,7 +365,14 @@ class HTMLSectionSplitter:self.kwargs = kwargsdef split_documents(self, documents: Iterable[Document]) -> list[Document]:- """Split documents."""+ """Split documents.++ Args:+ documents: Iterable of `Document` objects to be split.++ Returns:+ A list of split `Document` objects.+ """texts, metadatas = [], []for doc in documents:texts.append(doc.page_content)@@ -378,13 +388,24 @@ class HTMLSectionSplitter:Args:text: HTML text++ Returns:+ A list of split `Document` objects."""return self.split_text_from_file(StringIO(text))def create_documents(self, texts: list[str], metadatas: list[dict[Any, Any]] | None = None) -> list[Document]:- """Create a list of `Document` objects from a list of texts."""+ """Create a list of `Document` objects from a list of texts.++ Args:+ texts: A list of texts to be split and converted into documents.+ metadatas: Optional list of metadata to associate with each document.++ Returns:+ A list of `Document` objects.+ """metadatas_ = metadatas or [{}] * len(texts)documents = []for i, text in enumerate(texts):@@ -415,6 +436,9 @@ class HTMLSectionSplitter:* `'header'`: The header text or a default title for the first section.* `'content'`: The content under the header.* `'tag_name'`: The name of the header tag (e.g., `h1`, `h2`).++ Raises:+ ImportError: If BeautifulSoup is not installed."""if not _HAS_BS4:msg = "Unable to import BeautifulSoup/PageElement, \@@ -467,6 +491,9 @@ class HTMLSectionSplitter:Returns:The transformed HTML content as a string.++ Raises:+ ImportError: If the `lxml` library is not installed."""if not _HAS_LXML:msg = "Unable to import lxml, please install with `pip install lxml`."@@ -615,6 +642,10 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):`transform_documents/atransform_documents()`.keep_separator: Whether separatorsshould be at the beginning of a chunk, at the end, or not at all.++ Raises:+ ImportError: If BeautifulSoup or NLTK (when stopword removal is enabled)+ is not installed."""if not _HAS_BS4:msg = (@@ -697,7 +728,14 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):def transform_documents(self, documents: Sequence[Document], **kwargs: Any) -> list[Document]:- """Transform sequence of documents by splitting them."""+ """Transform sequence of documents by splitting them.++ Args:+ documents: A sequence of `Document` objects to be split.++ Returns:+ A sequence of split `Document` objects.+ """transformed = []for doc in documents:splits = self.split_text(doc.page_content)@@ -745,7 +783,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):wrapper.string = markdown_audioaudio_tag.replace_with(wrapper)- def _process_links(self, soup: BeautifulSoup) -> None:+ @staticmethod+ def _process_links(soup: BeautifulSoup) -> None:"""Processes the links in the HTML content.Args:@@ -994,8 +1033,9 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):return result+ @staticmethoddef _reinsert_preserved_elements(- self, content: str, preserved_elements: dict[str, str]+ content: str, preserved_elements: dict[str, str]) -> str:"""Reinserts preserved elements into the content into their original positions.diff --git a/libs/text-splitters/langchain_text_splitters/json.py b/libs/text-splitters/langchain_text_splitters/json.pyindex a306bcad51..8c80b9d7a5 100644--- a/libs/text-splitters/langchain_text_splitters/json.py+++ b/libs/text-splitters/langchain_text_splitters/json.py@@ -114,7 +114,16 @@ class RecursiveJsonSplitter:json_data: dict[str, Any],convert_lists: bool = False, # noqa: FBT001,FBT002) -> list[dict[str, Any]]:- """Splits JSON into a list of JSON chunks."""+ """Splits JSON into a list of JSON chunks.++ Args:+ json_data: The JSON data to be split.+ convert_lists: Whether to convert lists in the JSON to dictionaries+ before splitting.++ Returns:+ A list of JSON chunks.+ """if convert_lists:chunks = self._json_split(self._list_to_dict_preprocessing(json_data))else:@@ -131,7 +140,17 @@ class RecursiveJsonSplitter:convert_lists: bool = False, # noqa: FBT001,FBT002ensure_ascii: bool = True, # noqa: FBT001,FBT002) -> list[str]:- """Splits JSON into a list of JSON formatted strings."""+ """Splits JSON into a list of JSON formatted strings.++ Args:+ json_data: The JSON data to be split.+ convert_lists: Whether to convert lists in the JSON to dictionaries+ before splitting.+ ensure_ascii: Whether to ensure ASCII encoding in the JSON strings.++ Returns:+ A list of JSON formatted strings.+ """chunks = self.split_json(json_data=json_data, convert_lists=convert_lists)# Convert to string@@ -144,7 +163,17 @@ class RecursiveJsonSplitter:ensure_ascii: bool = True, # noqa: FBT001,FBT002metadatas: list[dict[Any, Any]] | None = None,) -> list[Document]:- """Create a list of `Document` objects from a list of json objects (`dict`)."""+ """Create a list of `Document` objects from a list of json objects (`dict`).++ Args:+ texts: A list of JSON data to be split and converted into documents.+ convert_lists: Whether to convert lists to dictionaries before splitting.+ ensure_ascii: Whether to ensure ASCII encoding in the JSON strings.+ metadatas: Optional list of metadata to associate with each document.++ Returns:+ A list of `Document` objects.+ """metadatas_ = metadatas or [{}] * len(texts)documents = []for i, text in enumerate(texts):diff --git a/libs/text-splitters/langchain_text_splitters/konlpy.py b/libs/text-splitters/langchain_text_splitters/konlpy.pyindex 863447e883..e9404431c9 100644--- a/libs/text-splitters/langchain_text_splitters/konlpy.py+++ b/libs/text-splitters/langchain_text_splitters/konlpy.py@@ -4,6 +4,8 @@ from __future__ import annotationsfrom typing import Any+from typing_extensions import override+from langchain_text_splitters.base import TextSplittertry:@@ -25,7 +27,14 @@ class KonlpyTextSplitter(TextSplitter):separator: str = "\n\n",**kwargs: Any,) -> None:- """Initialize the Konlpy text splitter."""+ """Initialize the Konlpy text splitter.++ Args:+ separator: The separator to use when combining splits.++ Raises:+ ImportError: If Konlpy is not installed.+ """super().__init__(**kwargs)self._separator = separatorif not _HAS_KONLPY:@@ -36,7 +45,7 @@ class KonlpyTextSplitter(TextSplitter):raise ImportError(msg)self.kkma = konlpy.tag.Kkma()+ @overridedef split_text(self, text: str) -> list[str]:- """Split incoming text and return chunks."""splits = self.kkma.sentences(text)return self._merge_splits(splits, self._separator)diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.pyindex 41281ce9b8..3d6dcf9f72 100644--- a/libs/text-splitters/langchain_text_splitters/markdown.py+++ b/libs/text-splitters/langchain_text_splitters/markdown.py@@ -88,6 +88,9 @@ class MarkdownHeaderTextSplitter:Args:lines: Line of text / associated header metadata++ Returns:+ List of Documents with common metadata aggregated."""aggregated_chunks: list[LineType] = []@@ -131,6 +134,9 @@ class MarkdownHeaderTextSplitter:Args:text: Markdown file++ Returns:+ List of `Document` objects."""# Split the input text by newline character ("\n").lines = text.split("\n")@@ -455,11 +461,13 @@ class ExperimentalMarkdownSyntaxTextSplitter:return matchreturn None- def _match_code(self, line: str) -> re.Match[str] | None:+ @staticmethod+ def _match_code(line: str) -> re.Match[str] | None:matches = [re.match(rule, line) for rule in [r"^```(.*)", r"^~~~(.*)"]]return next((match for match in matches if match), None)- def _match_horz(self, line: str) -> re.Match[str] | None:+ @staticmethod+ def _match_horz(line: str) -> re.Match[str] | None:matches = [re.match(rule, line) for rule in [r"^\*\*\*+\n", r"^---+\n", r"^___+\n"]]diff --git a/libs/text-splitters/langchain_text_splitters/nltk.py b/libs/text-splitters/langchain_text_splitters/nltk.pyindex 8041d47d49..87f9de051e 100644--- a/libs/text-splitters/langchain_text_splitters/nltk.py+++ b/libs/text-splitters/langchain_text_splitters/nltk.py@@ -4,6 +4,8 @@ from __future__ import annotationsfrom typing import Any+from typing_extensions import override+from langchain_text_splitters.base import TextSplittertry:@@ -25,7 +27,18 @@ class NLTKTextSplitter(TextSplitter):use_span_tokenize: bool = False,**kwargs: Any,) -> None:- """Initialize the NLTK splitter."""+ """Initialize the NLTK splitter.++ Args:+ separator: The separator to use when combining splits.+ language: The language to use.+ use_span_tokenize: Whether to use `span_tokenize` instead of+ `sent_tokenize`.++ Raises:+ ImportError: If NLTK is not installed.+ ValueError: If `use_span_tokenize` is `True` and separator is not `''`.+ """super().__init__(**kwargs)self._separator = separatorself._language = language@@ -41,8 +54,8 @@ class NLTKTextSplitter(TextSplitter):else:self._tokenizer = nltk.tokenize.sent_tokenize+ @overridedef split_text(self, text: str) -> list[str]:- """Split incoming text and return chunks."""# First we naively split the large input into a bunch of smaller ones.if self._use_span_tokenize:spans = list(self._tokenizer.span_tokenize(text))diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.pyindex d354fff583..376e48ae9f 100644--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py@@ -27,7 +27,17 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):tokens_per_chunk: int | None = None,**kwargs: Any,) -> None:- """Create a new TextSplitter."""+ """Create a new TextSplitter.++ Args:+ chunk_overlap: The number of tokens to overlap between chunks.+ model_name: The name of the sentence transformer model to use.+ tokens_per_chunk: The number of tokens per chunk. If `None`, uses the+ maximum tokens allowed by the model.++ Raises:+ ImportError: If the sentence_transformers package is not installed.+ """super().__init__(**kwargs, chunk_overlap=chunk_overlap)if not _HAS_SENTENCE_TRANSFORMERS:diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.pyindex af06bfa8ba..44a0e898ce 100644--- a/libs/text-splitters/langchain_text_splitters/spacy.py+++ b/libs/text-splitters/langchain_text_splitters/spacy.py@@ -4,6 +4,8 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Any+from typing_extensions import override+from langchain_text_splitters.base import TextSplittertry:@@ -47,8 +49,8 @@ class SpacyTextSplitter(TextSplitter):self._separator = separatorself._strip_whitespace = strip_whitespace+ @overridedef split_text(self, text: str) -> list[str]:- """Split incoming text and return chunks."""splits = (s.text if self._strip_whitespace else s.text_with_wsfor s in self._tokenizer(text).sentsdiff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex c7b68bc9bb..98975e8c51 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -510,8 +510,10 @@ def test_jsx_text_splitter() -> None:splits = splitter.split_text(FAKE_JSX_TEXT)expected_splits = [- "\nimport React from 'react';\n"- "import OtherComponent from './OtherComponent';\n",+ (+ "\nimport React from 'react';\n"+ "import OtherComponent from './OtherComponent';\n"+ ),"\nfunction MyComponent() {\n const [count, setCount] = React.useState(0);","\n\n const handleClick = () => {\n setCount(count + 1);\n };","return (",@@ -566,13 +568,17 @@ def test_vue_text_splitter() -> None:"<template>","<div>","<h1>{{ title }}</h1>",- '<button @click="increment">\n Count is: {{ count }}\n'- " </button>\n </div>\n</template>",+ (+ '<button @click="increment">\n Count is: {{ count }}\n'+ " </button>\n </div>\n</template>"+ ),"<script>","export",- " default {\n data() {\n return {\n title: 'Counter App',\n "- "count: 0\n }\n },\n methods: {\n increment() {\n "- "this.count++\n }\n }\n}\n</script>",+ (+ " default {\n data() {\n return {\n title: 'Counter App',\n "+ "count: 0\n }\n },\n methods: {\n increment() {\n "+ "this.count++\n }\n }\n}\n</script>"+ ),"<style>\nbutton {\n color: blue;\n}\n</style>",]assert [s.strip() for s in splits] == [s.strip() for s in expected_splits]@@ -2442,6 +2448,10 @@ def html_header_splitter_splitter_factory() -> Callable["""Fixture to create an `HTMLHeaderTextSplitter` instance with given headers.This factory allows dynamic creation of splitters with different headers.++ Returns:+ Factory function that takes a list of headers to split on and returns an+ `HTMLHeaderTextSplitter` instance."""def _create_splitter(