Skip to content

The agent engineering platform.

style(text-splitters): fix some ruff preview rules (#34665)

Christophe Bornetfd694254390c
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py
index d3bc65adb6..6e4f9b0ec9 100644
--- a/libs/text-splitters/langchain_text_splitters/base.py
+++ b/libs/text-splitters/langchain_text_splitters/base.py
@@ -64,6 +64,11 @@ class TextSplitter(BaseDocumentTransformer, ABC):
add_start_index: If `True`, includes chunk's start index in metadata
strip_whitespace: If `True`, strips whitespace from the start and end of
every document
+
+ Raises:
+ ValueError: If `chunk_size` is less than or equal to 0
+ ValueError: If `chunk_overlap` is less than 0
+ ValueError: If `chunk_overlap` is greater than chunk_size
"""
if chunk_size <= 0:
msg = f"chunk_size must be > 0, got {chunk_size}"
@@ -86,12 +91,27 @@ class TextSplitter(BaseDocumentTransformer, ABC):
@abstractmethod
def split_text(self, text: str) -> list[str]:
- """Split text into multiple components."""
+ """Split text into multiple components.
+
+ Args:
+ text: The text to split.
+
+ Returns:
+ A list of text chunks.
+ """
def create_documents(
self, texts: list[str], metadatas: list[dict[Any, Any]] | None = None
) -> list[Document]:
- """Create a list of `Document` objects from a list of texts."""
+ """Create a list of `Document` objects from a list of texts.
+
+ Args:
+ texts: A list of texts to be split and converted into documents.
+ metadatas: Optional list of metadata to associate with each document.
+
+ Returns:
+ A list of `Document` objects.
+ """
metadatas_ = metadatas or [{}] * len(texts)
documents = []
for i, text in enumerate(texts):
@@ -109,7 +129,14 @@ class TextSplitter(BaseDocumentTransformer, ABC):
return documents
def split_documents(self, documents: Iterable[Document]) -> list[Document]:
- """Split documents."""
+ """Split documents.
+
+ Args:
+ documents: The documents to split.
+
+ Returns:
+ A list of split documents.
+ """
texts, metadatas = [], []
for doc in documents:
texts.append(doc.page_content)
@@ -170,7 +197,15 @@ class TextSplitter(BaseDocumentTransformer, ABC):
def from_huggingface_tokenizer(
cls, tokenizer: PreTrainedTokenizerBase, **kwargs: Any
) -> TextSplitter:
- """Text splitter that uses Hugging Face tokenizer to count length."""
+ """Text splitter that uses Hugging Face tokenizer to count length.
+
+ Args:
+ tokenizer: The Hugging Face tokenizer to use.
+
+ Returns:
+ An instance of `TextSplitter` using the Hugging Face tokenizer for length
+ calculation.
+ """
if not _HAS_TRANSFORMERS:
msg = (
"Could not import transformers python package. "
@@ -196,7 +231,21 @@ class TextSplitter(BaseDocumentTransformer, ABC):
disallowed_special: Literal["all"] | Collection[str] = "all",
**kwargs: Any,
) -> Self:
- """Text splitter that uses `tiktoken` encoder to count length."""
+ """Text splitter that uses `tiktoken` encoder to count length.
+
+ Args:
+ encoding_name: The name of the tiktoken encoding to use.
+ model_name: The name of the model to use. If provided, this will
+ override the `encoding_name`.
+ allowed_special: Special tokens that are allowed during encoding.
+ disallowed_special: Special tokens that are disallowed during encoding.
+
+ Returns:
+ An instance of `TextSplitter` using tiktoken for length calculation.
+
+ Raises:
+ ImportError: If the tiktoken package is not installed.
+ """
if not _HAS_TIKTOKEN:
msg = (
"Could not import tiktoken python package. "
@@ -234,7 +283,14 @@ class TextSplitter(BaseDocumentTransformer, ABC):
def transform_documents(
self, documents: Sequence[Document], **kwargs: Any
) -> Sequence[Document]:
- """Transform sequence of documents by splitting them."""
+ """Transform sequence of documents by splitting them.
+
+ Args:
+ documents: The sequence of documents to split.
+
+ Returns:
+ A list of split documents.
+ """
return self.split_documents(list(documents))
@@ -249,7 +305,18 @@ class TokenTextSplitter(TextSplitter):
disallowed_special: Literal["all"] | Collection[str] = "all",
**kwargs: Any,
) -> None:
- """Create a new TextSplitter."""
+ """Create a new `TextSplitter`.
+
+ Args:
+ encoding_name: The name of the tiktoken encoding to use.
+ model_name: The name of the model to use. If provided, this will
+ override the `encoding_name`.
+ allowed_special: Special tokens that are allowed during encoding.
+ disallowed_special: Special tokens that are disallowed during encoding.
+
+ Raises:
+ ImportError: If the tiktoken package is not installed.
+ """
super().__init__(**kwargs)
if not _HAS_TIKTOKEN:
msg = (
@@ -348,7 +415,15 @@ class Tokenizer:
def split_text_on_tokens(*, text: str, tokenizer: Tokenizer) -> list[str]:
- """Split incoming text and return chunks using tokenizer."""
+ """Split incoming text and return chunks using tokenizer.
+
+ Args:
+ text: The input text to be split.
+ tokenizer: The tokenizer to use for splitting.
+
+ Returns:
+ A list of text chunks.
+ """
splits: list[str] = []
input_ids = tokenizer.encode(text)
start_idx = 0
diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.py
index 5bcc62b99f..469afdb913 100644
--- a/libs/text-splitters/langchain_text_splitters/character.py
+++ b/libs/text-splitters/langchain_text_splitters/character.py
@@ -23,7 +23,14 @@ class CharacterTextSplitter(TextSplitter):
self._is_separator_regex = is_separator_regex
def split_text(self, text: str) -> list[str]:
- """Split into chunks without re-inserting lookaround separators."""
+ """Split into chunks without re-inserting lookaround separators.
+
+ Args:
+ text: The text to split.
+
+ Returns:
+ A list of text chunks.
+ """
# 1. Determine split pattern: raw regex or escaped literal
sep_pattern = (
self._separator if self._is_separator_regex else re.escape(self._separator)
@@ -103,13 +110,13 @@ class RecursiveCharacterTextSplitter(TextSplitter):
# Get appropriate separator to use
separator = separators[-1]
new_separators = []
- for i, _s in enumerate(separators):
- separator_ = _s if self._is_separator_regex else re.escape(_s)
- if not _s:
- separator = _s
+ for i, s_ in enumerate(separators):
+ separator_ = s_ if self._is_separator_regex else re.escape(s_)
+ if not s_:
+ separator = s_
break
if re.search(separator_, text):
- separator = _s
+ separator = s_
new_separators = separators[i + 1 :]
break
@@ -177,6 +184,9 @@ class RecursiveCharacterTextSplitter(TextSplitter):
Returns:
A list of separators appropriate for the specified language.
+
+ Raises:
+ ValueError: If the language is not implemented or supported.
"""
if language in {Language.C, Language.CPP}:
return [
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index b98a1c445e..8247ecc538 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -232,6 +232,9 @@ class HTMLHeaderTextSplitter:
Yields:
Document objects as they are created.
+
+ Raises:
+ ImportError: If BeautifulSoup is not installed.
"""
if not _HAS_BS4:
msg = (
@@ -362,7 +365,14 @@ class HTMLSectionSplitter:
self.kwargs = kwargs
def split_documents(self, documents: Iterable[Document]) -> list[Document]:
- """Split documents."""
+ """Split documents.
+
+ Args:
+ documents: Iterable of `Document` objects to be split.
+
+ Returns:
+ A list of split `Document` objects.
+ """
texts, metadatas = [], []
for doc in documents:
texts.append(doc.page_content)
@@ -378,13 +388,24 @@ class HTMLSectionSplitter:
Args:
text: HTML text
+
+ Returns:
+ A list of split `Document` objects.
"""
return self.split_text_from_file(StringIO(text))
def create_documents(
self, texts: list[str], metadatas: list[dict[Any, Any]] | None = None
) -> list[Document]:
- """Create a list of `Document` objects from a list of texts."""
+ """Create a list of `Document` objects from a list of texts.
+
+ Args:
+ texts: A list of texts to be split and converted into documents.
+ metadatas: Optional list of metadata to associate with each document.
+
+ Returns:
+ A list of `Document` objects.
+ """
metadatas_ = metadatas or [{}] * len(texts)
documents = []
for i, text in enumerate(texts):
@@ -415,6 +436,9 @@ class HTMLSectionSplitter:
* `'header'`: The header text or a default title for the first section.
* `'content'`: The content under the header.
* `'tag_name'`: The name of the header tag (e.g., `h1`, `h2`).
+
+ Raises:
+ ImportError: If BeautifulSoup is not installed.
"""
if not _HAS_BS4:
msg = "Unable to import BeautifulSoup/PageElement, \
@@ -467,6 +491,9 @@ class HTMLSectionSplitter:
Returns:
The transformed HTML content as a string.
+
+ Raises:
+ ImportError: If the `lxml` library is not installed.
"""
if not _HAS_LXML:
msg = "Unable to import lxml, please install with `pip install lxml`."
@@ -615,6 +642,10 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
`transform_documents/atransform_documents()`.
keep_separator: Whether separators
should be at the beginning of a chunk, at the end, or not at all.
+
+ Raises:
+ ImportError: If BeautifulSoup or NLTK (when stopword removal is enabled)
+ is not installed.
"""
if not _HAS_BS4:
msg = (
@@ -697,7 +728,14 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
def transform_documents(
self, documents: Sequence[Document], **kwargs: Any
) -> list[Document]:
- """Transform sequence of documents by splitting them."""
+ """Transform sequence of documents by splitting them.
+
+ Args:
+ documents: A sequence of `Document` objects to be split.
+
+ Returns:
+ A sequence of split `Document` objects.
+ """
transformed = []
for doc in documents:
splits = self.split_text(doc.page_content)
@@ -745,7 +783,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
wrapper.string = markdown_audio
audio_tag.replace_with(wrapper)
- def _process_links(self, soup: BeautifulSoup) -> None:
+ @staticmethod
+ def _process_links(soup: BeautifulSoup) -> None:
"""Processes the links in the HTML content.
Args:
@@ -994,8 +1033,9 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
return result
+ @staticmethod
def _reinsert_preserved_elements(
- self, content: str, preserved_elements: dict[str, str]
+ content: str, preserved_elements: dict[str, str]
) -> str:
"""Reinserts preserved elements into the content into their original positions.
diff --git a/libs/text-splitters/langchain_text_splitters/json.py b/libs/text-splitters/langchain_text_splitters/json.py
index a306bcad51..8c80b9d7a5 100644
--- a/libs/text-splitters/langchain_text_splitters/json.py
+++ b/libs/text-splitters/langchain_text_splitters/json.py
@@ -114,7 +114,16 @@ class RecursiveJsonSplitter:
json_data: dict[str, Any],
convert_lists: bool = False, # noqa: FBT001,FBT002
) -> list[dict[str, Any]]:
- """Splits JSON into a list of JSON chunks."""
+ """Splits JSON into a list of JSON chunks.
+
+ Args:
+ json_data: The JSON data to be split.
+ convert_lists: Whether to convert lists in the JSON to dictionaries
+ before splitting.
+
+ Returns:
+ A list of JSON chunks.
+ """
if convert_lists:
chunks = self._json_split(self._list_to_dict_preprocessing(json_data))
else:
@@ -131,7 +140,17 @@ class RecursiveJsonSplitter:
convert_lists: bool = False, # noqa: FBT001,FBT002
ensure_ascii: bool = True, # noqa: FBT001,FBT002
) -> list[str]:
- """Splits JSON into a list of JSON formatted strings."""
+ """Splits JSON into a list of JSON formatted strings.
+
+ Args:
+ json_data: The JSON data to be split.
+ convert_lists: Whether to convert lists in the JSON to dictionaries
+ before splitting.
+ ensure_ascii: Whether to ensure ASCII encoding in the JSON strings.
+
+ Returns:
+ A list of JSON formatted strings.
+ """
chunks = self.split_json(json_data=json_data, convert_lists=convert_lists)
# Convert to string
@@ -144,7 +163,17 @@ class RecursiveJsonSplitter:
ensure_ascii: bool = True, # noqa: FBT001,FBT002
metadatas: list[dict[Any, Any]] | None = None,
) -> list[Document]:
- """Create a list of `Document` objects from a list of json objects (`dict`)."""
+ """Create a list of `Document` objects from a list of json objects (`dict`).
+
+ Args:
+ texts: A list of JSON data to be split and converted into documents.
+ convert_lists: Whether to convert lists to dictionaries before splitting.
+ ensure_ascii: Whether to ensure ASCII encoding in the JSON strings.
+ metadatas: Optional list of metadata to associate with each document.
+
+ Returns:
+ A list of `Document` objects.
+ """
metadatas_ = metadatas or [{}] * len(texts)
documents = []
for i, text in enumerate(texts):
diff --git a/libs/text-splitters/langchain_text_splitters/konlpy.py b/libs/text-splitters/langchain_text_splitters/konlpy.py
index 863447e883..e9404431c9 100644
--- a/libs/text-splitters/langchain_text_splitters/konlpy.py
+++ b/libs/text-splitters/langchain_text_splitters/konlpy.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from typing import Any
+from typing_extensions import override
+
from langchain_text_splitters.base import TextSplitter
try:
@@ -25,7 +27,14 @@ class KonlpyTextSplitter(TextSplitter):
separator: str = "\n\n",
**kwargs: Any,
) -> None:
- """Initialize the Konlpy text splitter."""
+ """Initialize the Konlpy text splitter.
+
+ Args:
+ separator: The separator to use when combining splits.
+
+ Raises:
+ ImportError: If Konlpy is not installed.
+ """
super().__init__(**kwargs)
self._separator = separator
if not _HAS_KONLPY:
@@ -36,7 +45,7 @@ class KonlpyTextSplitter(TextSplitter):
raise ImportError(msg)
self.kkma = konlpy.tag.Kkma()
+ @override
def split_text(self, text: str) -> list[str]:
- """Split incoming text and return chunks."""
splits = self.kkma.sentences(text)
return self._merge_splits(splits, self._separator)
diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.py
index 41281ce9b8..3d6dcf9f72 100644
--- a/libs/text-splitters/langchain_text_splitters/markdown.py
+++ b/libs/text-splitters/langchain_text_splitters/markdown.py
@@ -88,6 +88,9 @@ class MarkdownHeaderTextSplitter:
Args:
lines: Line of text / associated header metadata
+
+ Returns:
+ List of Documents with common metadata aggregated.
"""
aggregated_chunks: list[LineType] = []
@@ -131,6 +134,9 @@ class MarkdownHeaderTextSplitter:
Args:
text: Markdown file
+
+ Returns:
+ List of `Document` objects.
"""
# Split the input text by newline character ("\n").
lines = text.split("\n")
@@ -455,11 +461,13 @@ class ExperimentalMarkdownSyntaxTextSplitter:
return match
return None
- def _match_code(self, line: str) -> re.Match[str] | None:
+ @staticmethod
+ def _match_code(line: str) -> re.Match[str] | None:
matches = [re.match(rule, line) for rule in [r"^```(.*)", r"^~~~(.*)"]]
return next((match for match in matches if match), None)
- def _match_horz(self, line: str) -> re.Match[str] | None:
+ @staticmethod
+ def _match_horz(line: str) -> re.Match[str] | None:
matches = [
re.match(rule, line) for rule in [r"^\*\*\*+\n", r"^---+\n", r"^___+\n"]
]
diff --git a/libs/text-splitters/langchain_text_splitters/nltk.py b/libs/text-splitters/langchain_text_splitters/nltk.py
index 8041d47d49..87f9de051e 100644
--- a/libs/text-splitters/langchain_text_splitters/nltk.py
+++ b/libs/text-splitters/langchain_text_splitters/nltk.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from typing import Any
+from typing_extensions import override
+
from langchain_text_splitters.base import TextSplitter
try:
@@ -25,7 +27,18 @@ class NLTKTextSplitter(TextSplitter):
use_span_tokenize: bool = False,
**kwargs: Any,
) -> None:
- """Initialize the NLTK splitter."""
+ """Initialize the NLTK splitter.
+
+ Args:
+ separator: The separator to use when combining splits.
+ language: The language to use.
+ use_span_tokenize: Whether to use `span_tokenize` instead of
+ `sent_tokenize`.
+
+ Raises:
+ ImportError: If NLTK is not installed.
+ ValueError: If `use_span_tokenize` is `True` and separator is not `''`.
+ """
super().__init__(**kwargs)
self._separator = separator
self._language = language
@@ -41,8 +54,8 @@ class NLTKTextSplitter(TextSplitter):
else:
self._tokenizer = nltk.tokenize.sent_tokenize
+ @override
def split_text(self, text: str) -> list[str]:
- """Split incoming text and return chunks."""
# First we naively split the large input into a bunch of smaller ones.
if self._use_span_tokenize:
spans = list(self._tokenizer.span_tokenize(text))
diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
index d354fff583..376e48ae9f 100644
--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
@@ -27,7 +27,17 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
tokens_per_chunk: int | None = None,
**kwargs: Any,
) -> None:
- """Create a new TextSplitter."""
+ """Create a new TextSplitter.
+
+ Args:
+ chunk_overlap: The number of tokens to overlap between chunks.
+ model_name: The name of the sentence transformer model to use.
+ tokens_per_chunk: The number of tokens per chunk. If `None`, uses the
+ maximum tokens allowed by the model.
+
+ Raises:
+ ImportError: If the sentence_transformers package is not installed.
+ """
super().__init__(**kwargs, chunk_overlap=chunk_overlap)
if not _HAS_SENTENCE_TRANSFORMERS:
diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.py
index af06bfa8ba..44a0e898ce 100644
--- a/libs/text-splitters/langchain_text_splitters/spacy.py
+++ b/libs/text-splitters/langchain_text_splitters/spacy.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any
+from typing_extensions import override
+
from langchain_text_splitters.base import TextSplitter
try:
@@ -47,8 +49,8 @@ class SpacyTextSplitter(TextSplitter):
self._separator = separator
self._strip_whitespace = strip_whitespace
+ @override
def split_text(self, text: str) -> list[str]:
- """Split incoming text and return chunks."""
splits = (
s.text if self._strip_whitespace else s.text_with_ws
for s in self._tokenizer(text).sents
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index c7b68bc9bb..98975e8c51 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -510,8 +510,10 @@ def test_jsx_text_splitter() -> None:
splits = splitter.split_text(FAKE_JSX_TEXT)
expected_splits = [
- "\nimport React from 'react';\n"
- "import OtherComponent from './OtherComponent';\n",
+ (
+ "\nimport React from 'react';\n"
+ "import OtherComponent from './OtherComponent';\n"
+ ),
"\nfunction MyComponent() {\n const [count, setCount] = React.useState(0);",
"\n\n const handleClick = () => {\n setCount(count + 1);\n };",
"return (",
@@ -566,13 +568,17 @@ def test_vue_text_splitter() -> None:
"<template>",
"<div>",
"<h1>{{ title }}</h1>",
- '<button @click="increment">\n Count is: {{ count }}\n'
- " </button>\n </div>\n</template>",
+ (
+ '<button @click="increment">\n Count is: {{ count }}\n'
+ " </button>\n </div>\n</template>"
+ ),
"<script>",
"export",
- " default {\n data() {\n return {\n title: 'Counter App',\n "
- "count: 0\n }\n },\n methods: {\n increment() {\n "
- "this.count++\n }\n }\n}\n</script>",
+ (
+ " default {\n data() {\n return {\n title: 'Counter App',\n "
+ "count: 0\n }\n },\n methods: {\n increment() {\n "
+ "this.count++\n }\n }\n}\n</script>"
+ ),
"<style>\nbutton {\n color: blue;\n}\n</style>",
]
assert [s.strip() for s in splits] == [s.strip() for s in expected_splits]
@@ -2442,6 +2448,10 @@ def html_header_splitter_splitter_factory() -> Callable[
"""Fixture to create an `HTMLHeaderTextSplitter` instance with given headers.
This factory allows dynamic creation of splitters with different headers.
+
+ Returns:
+ Factory function that takes a list of headers to split on and returns an
+ `HTMLHeaderTextSplitter` instance.
"""
def _create_splitter(