Skip to content

The agent engineering platform.

chore(text-splitters): select ALL rules with exclusions (#32325)

Christophe Bornet0c3e8ccd0eba
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py
index 0ab553cefe..bce83f14fa 100644
--- a/libs/text-splitters/langchain_text_splitters/base.py
+++ b/libs/text-splitters/langchain_text_splitters/base.py
@@ -1,3 +1,5 @@
+"""Text splitter base interface."""
+
from __future__ import annotations
import copy
@@ -16,12 +18,27 @@ from typing import (
)
from langchain_core.documents import BaseDocumentTransformer, Document
-from typing_extensions import Self
+from typing_extensions import Self, override
if TYPE_CHECKING:
from collections.abc import Collection, Iterable, Sequence
from collections.abc import Set as AbstractSet
+
+try:
+ import tiktoken
+
+ _HAS_TIKTOKEN = True
+except ImportError:
+ _HAS_TIKTOKEN = False
+
+try:
+ from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+ _HAS_TRANSFORMERS = True
+except ImportError:
+ _HAS_TRANSFORMERS = False
+
logger = logging.getLogger(__name__)
TS = TypeVar("TS", bound="TextSplitter")
@@ -78,13 +95,13 @@ class TextSplitter(BaseDocumentTransformer, ABC):
self, texts: list[str], metadatas: Optional[list[dict[Any, Any]]] = None
) -> list[Document]:
"""Create documents from a list of texts."""
- _metadatas = metadatas or [{}] * len(texts)
+ metadatas_ = metadatas or [{}] * len(texts)
documents = []
for i, text in enumerate(texts):
index = 0
previous_chunk_len = 0
for chunk in self.split_text(text):
- metadata = copy.deepcopy(_metadatas[i])
+ metadata = copy.deepcopy(metadatas_[i])
if self._add_start_index:
offset = index + previous_chunk_len - self._chunk_overlap
index = text.find(chunk, max(0, offset))
@@ -106,9 +123,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
text = separator.join(docs)
if self._strip_whitespace:
text = text.strip()
- if text == "":
- return None
- return text
+ return text or None
def _merge_splits(self, splits: Iterable[str], separator: str) -> list[str]:
# We now want to combine these smaller pieces into medium size
@@ -119,15 +134,17 @@ class TextSplitter(BaseDocumentTransformer, ABC):
current_doc: list[str] = []
total = 0
for d in splits:
- _len = self._length_function(d)
+ len_ = self._length_function(d)
if (
- total + _len + (separator_len if len(current_doc) > 0 else 0)
+ total + len_ + (separator_len if len(current_doc) > 0 else 0)
> self._chunk_size
):
if total > self._chunk_size:
logger.warning(
- f"Created a chunk of size {total}, "
- f"which is longer than the specified {self._chunk_size}"
+ "Created a chunk of size %s, "
+ "which is longer than the specified %s",
+ total,
+ self._chunk_size,
)
if len(current_doc) > 0:
doc = self._join_docs(current_doc, separator)
@@ -137,7 +154,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
# - we have a larger chunk than in the chunk overlap
# - or if we still have any chunks and the length is long
while total > self._chunk_overlap or (
- total + _len + (separator_len if len(current_doc) > 0 else 0)
+ total + len_ + (separator_len if len(current_doc) > 0 else 0)
> self._chunk_size
and total > 0
):
@@ -146,33 +163,31 @@ class TextSplitter(BaseDocumentTransformer, ABC):
)
current_doc = current_doc[1:]
current_doc.append(d)
- total += _len + (separator_len if len(current_doc) > 1 else 0)
+ total += len_ + (separator_len if len(current_doc) > 1 else 0)
doc = self._join_docs(current_doc, separator)
if doc is not None:
docs.append(doc)
return docs
@classmethod
- def from_huggingface_tokenizer(cls, tokenizer: Any, **kwargs: Any) -> TextSplitter:
+ def from_huggingface_tokenizer(
+ cls, tokenizer: PreTrainedTokenizerBase, **kwargs: Any
+ ) -> TextSplitter:
"""Text splitter that uses HuggingFace tokenizer to count length."""
- try:
- from transformers.tokenization_utils_base import PreTrainedTokenizerBase
-
- if not isinstance(tokenizer, PreTrainedTokenizerBase):
- msg = (
- "Tokenizer received was not an instance of PreTrainedTokenizerBase"
- )
- raise ValueError(msg)
-
- def _huggingface_tokenizer_length(text: str) -> int:
- return len(tokenizer.tokenize(text))
-
- except ImportError as err:
+ if not _HAS_TRANSFORMERS:
msg = (
"Could not import transformers python package. "
"Please install it with `pip install transformers`."
)
- raise ValueError(msg) from err
+ raise ValueError(msg)
+
+ if not isinstance(tokenizer, PreTrainedTokenizerBase):
+ msg = "Tokenizer received was not an instance of PreTrainedTokenizerBase" # type: ignore[unreachable]
+ raise ValueError(msg) # noqa: TRY004
+
+ def _huggingface_tokenizer_length(text: str) -> int:
+ return len(tokenizer.tokenize(text))
+
return cls(length_function=_huggingface_tokenizer_length, **kwargs)
@classmethod
@@ -185,15 +200,13 @@ class TextSplitter(BaseDocumentTransformer, ABC):
**kwargs: Any,
) -> Self:
"""Text splitter that uses tiktoken encoder to count length."""
- try:
- import tiktoken
- except ImportError as err:
+ if not _HAS_TIKTOKEN:
msg = (
"Could not import tiktoken python package. "
"This is needed in order to calculate max_tokens_for_prompt. "
"Please install it with `pip install tiktoken`."
)
- raise ImportError(msg) from err
+ raise ImportError(msg)
if model_name is not None:
enc = tiktoken.encoding_for_model(model_name)
@@ -220,6 +233,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
return cls(length_function=_tiktoken_encoder, **kwargs)
+ @override
def transform_documents(
self, documents: Sequence[Document], **kwargs: Any
) -> Sequence[Document]:
@@ -240,15 +254,13 @@ class TokenTextSplitter(TextSplitter):
) -> None:
"""Create a new TextSplitter."""
super().__init__(**kwargs)
- try:
- import tiktoken
- except ImportError as err:
+ if not _HAS_TIKTOKEN:
msg = (
"Could not import tiktoken python package. "
"This is needed in order to for TokenTextSplitter. "
"Please install it with `pip install tiktoken`."
)
- raise ImportError(msg) from err
+ raise ImportError(msg)
if model_name is not None:
enc = tiktoken.encoding_for_model(model_name)
diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.py
index 90366c5b51..3a6d77d42d 100644
--- a/libs/text-splitters/langchain_text_splitters/character.py
+++ b/libs/text-splitters/langchain_text_splitters/character.py
@@ -1,3 +1,5 @@
+"""Character text splitters."""
+
from __future__ import annotations
import re
@@ -56,24 +58,24 @@ def _split_text_with_regex(
if separator:
if keep_separator:
# The parentheses in the pattern keep the delimiters in the result.
- _splits = re.split(f"({separator})", text)
+ splits_ = re.split(f"({separator})", text)
splits = (
- ([_splits[i] + _splits[i + 1] for i in range(0, len(_splits) - 1, 2)])
+ ([splits_[i] + splits_[i + 1] for i in range(0, len(splits_) - 1, 2)])
if keep_separator == "end"
- else ([_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)])
+ else ([splits_[i] + splits_[i + 1] for i in range(1, len(splits_), 2)])
)
- if len(_splits) % 2 == 0:
- splits += _splits[-1:]
+ if len(splits_) % 2 == 0:
+ splits += splits_[-1:]
splits = (
- ([*splits, _splits[-1]])
+ ([*splits, splits_[-1]])
if keep_separator == "end"
- else ([_splits[0], *splits])
+ else ([splits_[0], *splits])
)
else:
splits = re.split(separator, text)
else:
splits = list(text)
- return [s for s in splits if s != ""]
+ return [s for s in splits if s]
class RecursiveCharacterTextSplitter(TextSplitter):
@@ -102,38 +104,38 @@ class RecursiveCharacterTextSplitter(TextSplitter):
separator = separators[-1]
new_separators = []
for i, _s in enumerate(separators):
- _separator = _s if self._is_separator_regex else re.escape(_s)
- if _s == "":
+ separator_ = _s if self._is_separator_regex else re.escape(_s)
+ if not _s:
separator = _s
break
- if re.search(_separator, text):
+ if re.search(separator_, text):
separator = _s
new_separators = separators[i + 1 :]
break
- _separator = separator if self._is_separator_regex else re.escape(separator)
+ separator_ = separator if self._is_separator_regex else re.escape(separator)
splits = _split_text_with_regex(
- text, _separator, keep_separator=self._keep_separator
+ text, separator_, keep_separator=self._keep_separator
)
# Now go merging things, recursively splitting longer texts.
- _good_splits = []
- _separator = "" if self._keep_separator else separator
+ good_splits = []
+ separator_ = "" if self._keep_separator else separator
for s in splits:
if self._length_function(s) < self._chunk_size:
- _good_splits.append(s)
+ good_splits.append(s)
else:
- if _good_splits:
- merged_text = self._merge_splits(_good_splits, _separator)
+ if good_splits:
+ merged_text = self._merge_splits(good_splits, separator_)
final_chunks.extend(merged_text)
- _good_splits = []
+ good_splits = []
if not new_separators:
final_chunks.append(s)
else:
other_info = self._split_text(s, new_separators)
final_chunks.extend(other_info)
- if _good_splits:
- merged_text = self._merge_splits(_good_splits, _separator)
+ if good_splits:
+ merged_text = self._merge_splits(good_splits, separator_)
final_chunks.extend(merged_text)
return final_chunks
@@ -177,7 +179,7 @@ class RecursiveCharacterTextSplitter(TextSplitter):
Returns:
List[str]: A list of separators appropriate for the specified language.
"""
- if language in (Language.C, Language.CPP):
+ if language in {Language.C, Language.CPP}:
return [
# Split along class definitions
"\nclass ",
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 4d78fbcc35..35471fb2ff 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -1,3 +1,5 @@
+"""HTML text splitters."""
+
from __future__ import annotations
import copy
@@ -5,6 +7,7 @@ import pathlib
import re
from io import StringIO
from typing import (
+ IO,
TYPE_CHECKING,
Any,
Callable,
@@ -18,13 +21,36 @@ from typing import (
import requests
from langchain_core._api import beta
from langchain_core.documents import BaseDocumentTransformer, Document
+from typing_extensions import override
from langchain_text_splitters.character import RecursiveCharacterTextSplitter
if TYPE_CHECKING:
- from collections.abc import Iterable, Sequence
+ from collections.abc import Iterable, Iterator, Sequence
+
+ from bs4.element import ResultSet
+
+try:
+ import nltk
+
+ _HAS_NLTK = True
+except ImportError:
+ _HAS_NLTK = False
+
+try:
+ from bs4 import BeautifulSoup, Tag
+ from bs4.element import NavigableString, PageElement
- from bs4.element import PageElement
+ _HAS_BS4 = True
+except ImportError:
+ _HAS_BS4 = False
+
+try:
+ from lxml import etree
+
+ _HAS_LXML = True
+except ImportError:
+ _HAS_LXML = False
class ElementType(TypedDict):
@@ -36,6 +62,29 @@ class ElementType(TypedDict):
metadata: dict[str, str]
+# Unfortunately, BeautifulSoup doesn't define overloads for Tag.find_all.
+# So doing the type resolution ourselves.
+
+
+def _find_all_strings(
+ tag: Tag,
+ *,
+ recursive: bool = True,
+) -> ResultSet[NavigableString]:
+ return cast(
+ "ResultSet[NavigableString]", tag.find_all(string=True, recursive=recursive)
+ )
+
+
+def _find_all_tags(
+ tag: Tag,
+ *,
+ name: Union[bool, str, list[str], None] = None,
+ recursive: bool = True,
+) -> ResultSet[Tag]:
+ return cast("ResultSet[Tag]", tag.find_all(name, recursive=recursive))
+
+
class HTMLHeaderTextSplitter:
"""Split HTML content into structured Documents based on specified headers.
@@ -157,16 +206,11 @@ class HTMLHeaderTextSplitter:
Raises:
requests.RequestException: If the HTTP request fails.
"""
- kwargs.setdefault("timeout", timeout)
- response = requests.get(
- url,
- timeout=kwargs.get("timeout", timeout),
- **{k: v for k, v in kwargs.items() if k != "timeout"},
- )
+ response = requests.get(url, timeout=timeout, **kwargs)
response.raise_for_status()
return self.split_text(response.text)
- def split_text_from_file(self, file: Any) -> list[Document]:
+ def split_text_from_file(self, file: Union[str, IO[str]]) -> list[Document]:
"""Split HTML content from a file into a list of Document objects.
Args:
@@ -178,13 +222,12 @@ class HTMLHeaderTextSplitter:
the header hierarchy to their corresponding titles.
"""
if isinstance(file, str):
- with open(file, encoding="utf-8") as f:
- html_content = f.read()
+ html_content = pathlib.Path(file).read_text(encoding="utf-8")
else:
html_content = file.read()
return list(self._generate_documents(html_content))
- def _generate_documents(self, html_content: str) -> Any:
+ def _generate_documents(self, html_content: str) -> Iterator[Document]:
"""Private method that performs a DFS traversal over the DOM and yields.
Document objects on-the-fly. This approach maintains the same splitting
@@ -197,20 +240,18 @@ class HTMLHeaderTextSplitter:
Yields:
Document objects as they are created.
"""
- try:
- from bs4 import BeautifulSoup
- except ImportError as e:
+ if not _HAS_BS4:
msg = (
"Unable to import BeautifulSoup. Please install via `pip install bs4`."
)
- raise ImportError(msg) from e
+ raise ImportError(msg)
soup = BeautifulSoup(html_content, "html.parser")
- body = soup.body if soup.body else soup
+ body = soup.body or soup
# Dictionary of active headers:
# key = user-defined header name (e.g. "Header 1")
- # value = (header_text, level, dom_depth)
+ # value = tuple of header_text, level, dom_depth
active_headers: dict[str, tuple[str, int, int]] = {}
current_chunk: list[str] = []
@@ -232,7 +273,6 @@ class HTMLHeaderTextSplitter:
while stack:
node = stack.pop()
children = list(node.children)
- from bs4.element import Tag
stack.extend(
child for child in reversed(children) if isinstance(child, Tag)
@@ -243,8 +283,7 @@ class HTMLHeaderTextSplitter:
continue
text_elements = [
- str(child).strip()
- for child in node.find_all(string=True, recursive=False)
+ str(child).strip() for child in _find_all_strings(node, recursive=False)
]
node_text = " ".join(elem for elem in text_elements if elem)
if not node_text:
@@ -353,11 +392,11 @@ class HTMLSectionSplitter:
self, texts: list[str], metadatas: Optional[list[dict[Any, Any]]] = None
) -> list[Document]:
"""Create documents from a list of texts."""
- _metadatas = metadatas or [{}] * len(texts)
+ metadatas_ = metadatas or [{}] * len(texts)
documents = []
for i, text in enumerate(texts):
for chunk in self.split_text(text):
- metadata = copy.deepcopy(_metadatas[i])
+ metadata = copy.deepcopy(metadatas_[i])
for key in chunk.metadata:
if chunk.metadata[key] == "#TITLE#":
@@ -386,38 +425,35 @@ class HTMLSectionSplitter:
* 'content': The content under the header.
* 'tag_name': The name of the header tag (e.g., "h1", "h2").
"""
- try:
- from bs4 import BeautifulSoup
- except ImportError as e:
+ if not _HAS_BS4:
msg = "Unable to import BeautifulSoup/PageElement, \
please install with `pip install \
bs4`."
- raise ImportError(msg) from e
+ raise ImportError(msg)
soup = BeautifulSoup(html_doc, "html.parser")
- headers = list(self.headers_to_split_on.keys())
+ header_names = list(self.headers_to_split_on.keys())
sections: list[dict[str, str | None]] = []
- headers = soup.find_all(["body", *headers]) # type: ignore[assignment]
+ headers = _find_all_tags(soup, name=["body", *header_names])
for i, header in enumerate(headers):
- header_element = cast("PageElement", header)
if i == 0:
current_header = "#TITLE#"
current_header_tag = "h1"
section_content: list[str] = []
else:
- current_header = header_element.text.strip()
- current_header_tag = header_element.name # type: ignore[attr-defined]
+ current_header = header.text.strip()
+ current_header_tag = header.name
section_content = []
- for element in header_element.next_elements:
- if i + 1 < len(headers) and element == headers[i + 1]: # type: ignore[comparison-overlap]
+ for element in header.next_elements:
+ if i + 1 < len(headers) and element == headers[i + 1]:
break
if isinstance(element, str):
section_content.append(element)
content = " ".join(section_content).strip()
- if content != "":
+ if content:
sections.append(
{
"header": current_header,
@@ -441,11 +477,9 @@ class HTMLSectionSplitter:
Returns:
str: The transformed HTML content as a string.
"""
- try:
- from lxml import etree
- except ImportError as e:
+ if not _HAS_LXML:
msg = "Unable to import lxml, please install with `pip install lxml`."
- raise ImportError(msg) from e
+ raise ImportError(msg)
# use lxml library to parse html document and return xml ElementTree
# Create secure parsers to prevent XXE attacks
html_parser = etree.HTMLParser(no_network=True)
@@ -464,7 +498,7 @@ class HTMLSectionSplitter:
result = transform(tree)
return str(result)
- def split_text_from_file(self, file: Any) -> list[Document]:
+ def split_text_from_file(self, file: StringIO) -> list[Document]:
"""Split HTML content from a file into a list of Document objects.
Args:
@@ -546,7 +580,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
preserve_images: bool = False,
preserve_videos: bool = False,
preserve_audio: bool = False,
- custom_handlers: Optional[dict[str, Callable[[Any], str]]] = None,
+ custom_handlers: Optional[dict[str, Callable[[Tag], str]]] = None,
stopword_removal: bool = False,
stopword_lang: str = "english",
normalize_text: bool = False,
@@ -592,17 +626,12 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
keep_separator: Whether separators
should be at the beginning of a chunk, at the end, or not at all.
"""
- try:
- from bs4 import BeautifulSoup, Tag
-
- self._BeautifulSoup = BeautifulSoup
- self._Tag = Tag
- except ImportError as err:
+ if not _HAS_BS4:
msg = (
"Could not import BeautifulSoup. "
"Please install it with 'pip install bs4'."
)
- raise ImportError(msg) from err
+ raise ImportError(msg)
self._headers_to_split_on = sorted(headers_to_split_on)
self._max_chunk_size = max_chunk_size
@@ -645,16 +674,13 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
)
if self._stopword_removal:
- try:
- import nltk
-
- nltk.download("stopwords")
- self._stopwords = set(nltk.corpus.stopwords.words(self._stopword_lang))
- except ImportError as err:
+ if not _HAS_NLTK:
msg = (
"Could not import nltk. Please install it with 'pip install nltk'."
)
- raise ImportError(msg) from err
+ raise ImportError(msg)
+ nltk.download("stopwords")
+ self._stopwords = set(nltk.corpus.stopwords.words(self._stopword_lang))
def split_text(self, text: str) -> list[Document]:
"""Splits the provided HTML text into smaller chunks based on the configuration.
@@ -665,7 +691,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
Returns:
List[Document]: A list of Document objects containing the split content.
"""
- soup = self._BeautifulSoup(text, "html.parser")
+ soup = BeautifulSoup(text, "html.parser")
self._process_media(soup)
@@ -677,6 +703,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
return self._process_html(soup)
+ @override
def transform_documents(
self, documents: Sequence[Document], **kwargs: Any
) -> list[Document]:
@@ -695,17 +722,17 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
transformed.extend(splits)
return transformed
- def _process_media(self, soup: Any) -> None:
+ def _process_media(self, soup: BeautifulSoup) -> None:
"""Processes the media elements.
Process elements in the HTML content by wrapping them in a <media-wrapper> tag
and converting them to Markdown format.
Args:
- soup (Any): Parsed HTML content using BeautifulSoup.
+ soup: Parsed HTML content using BeautifulSoup.
"""
if self._preserve_images:
- for img_tag in soup.find_all("img"):
+ for img_tag in _find_all_tags(soup, name="img"):
img_src = img_tag.get("src", "")
markdown_img = f"![image:{img_src}]({img_src})"
wrapper = soup.new_tag("media-wrapper")
@@ -713,7 +740,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
img_tag.replace_with(wrapper)
if self._preserve_videos:
- for video_tag in soup.find_all("video"):
+ for video_tag in _find_all_tags(soup, name="video"):
video_src = video_tag.get("src", "")
markdown_video = f"![video:{video_src}]({video_src})"
wrapper = soup.new_tag("media-wrapper")
@@ -721,40 +748,40 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
video_tag.replace_with(wrapper)
if self._preserve_audio:
- for audio_tag in soup.find_all("audio"):
+ for audio_tag in _find_all_tags(soup, name="audio"):
audio_src = audio_tag.get("src", "")
markdown_audio = f"![audio:{audio_src}]({audio_src})"
wrapper = soup.new_tag("media-wrapper")
wrapper.string = markdown_audio
audio_tag.replace_with(wrapper)
- def _process_links(self, soup: Any) -> None:
+ def _process_links(self, soup: BeautifulSoup) -> None:
"""Processes the links in the HTML content.
Args:
- soup (Any): Parsed HTML content using BeautifulSoup.
+ soup: Parsed HTML content using BeautifulSoup.
"""
- for a_tag in soup.find_all("a"):
+ for a_tag in _find_all_tags(soup, name="a"):
a_href = a_tag.get("href", "")
a_text = a_tag.get_text(strip=True)
markdown_link = f"[{a_text}]({a_href})"
wrapper = soup.new_tag("link-wrapper")
wrapper.string = markdown_link
- a_tag.replace_with(markdown_link)
+ a_tag.replace_with(NavigableString(markdown_link))
- def _filter_tags(self, soup: Any) -> None:
+ def _filter_tags(self, soup: BeautifulSoup) -> None:
"""Filters the HTML content based on the allowlist and denylist tags.
Args:
- soup (Any): Parsed HTML content using BeautifulSoup.
+ soup: Parsed HTML content using BeautifulSoup.
"""
if self._allowlist_tags:
- for tag in soup.find_all(name=True):
+ for tag in _find_all_tags(soup, name=True):
if tag.name not in self._allowlist_tags:
tag.decompose()
if self._denylist_tags:
- for tag in soup.find_all(self._denylist_tags):
+ for tag in _find_all_tags(soup, name=self._denylist_tags):
tag.decompose()
def _normalize_and_clean_text(self, text: str) -> str:
@@ -778,14 +805,14 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
return text
- def _process_html(self, soup: Any) -> list[Document]:
+ def _process_html(self, soup: BeautifulSoup) -> list[Document]:
"""Processes the HTML content using BeautifulSoup and splits it using headers.
Args:
- soup (Any): Parsed HTML content using BeautifulSoup.
+ soup: Parsed HTML content using BeautifulSoup.
Returns:
- List[Document]: A list of Document objects containing the split content.
+ A list of Document objects containing the split content.
"""
documents: list[Document] = []
current_headers: dict[str, str] = {}
@@ -793,17 +820,18 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
preserved_elements: dict[str, str] = {}
placeholder_count: int = 0
- def _get_element_text(element: Any) -> str:
+ def _get_element_text(element: PageElement) -> str:
"""Recursively extracts and processes the text of an element.
Applies custom handlers where applicable, and ensures correct spacing.
Args:
- element (Any): The HTML element to process.
+ element: The HTML element to process.
Returns:
- str: The processed text of the element.
+ The processed text of the element.
"""
+ element = cast("Union[Tag, NavigableString]", element)
if element.name in self._custom_handlers:
return self._custom_handlers[element.name](element)
@@ -820,10 +848,10 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
return self._normalize_and_clean_text(text)
- elements = soup.find_all(recursive=False)
+ elements = _find_all_tags(soup, recursive=False)
def _process_element(
- element: list[Any],
+ element: ResultSet[Tag],
documents: list[Document],
current_headers: dict[str, str],
current_content: list[str],
@@ -831,8 +859,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
placeholder_count: int,
) -> tuple[list[Document], dict[str, str], list[str], dict[str, str], int]:
for elem in element:
- if elem.name.lower() in ["html", "body", "div", "main"]:
- children = elem.find_all(recursive=False)
+ if elem.name.lower() in {"html", "body", "div", "main"}:
+ children = _find_all_tags(elem, recursive=False)
(
documents,
current_headers,
@@ -847,7 +875,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
preserved_elements,
placeholder_count,
)
- content = " ".join(elem.find_all(string=True, recursive=False))
+ content = " ".join(_find_all_strings(elem, recursive=False))
if content:
content = self._normalize_and_clean_text(content)
current_content.append(content)
diff --git a/libs/text-splitters/langchain_text_splitters/json.py b/libs/text-splitters/langchain_text_splitters/json.py
index c2fa8936d2..6e61bc33f9 100644
--- a/libs/text-splitters/langchain_text_splitters/json.py
+++ b/libs/text-splitters/langchain_text_splitters/json.py
@@ -1,3 +1,5 @@
+"""JSON text splitter."""
+
from __future__ import annotations
import copy
@@ -50,13 +52,17 @@ class RecursiveJsonSplitter:
return len(json.dumps(data))
@staticmethod
- def _set_nested_dict(d: dict[str, Any], path: list[str], value: Any) -> None:
+ def _set_nested_dict(
+ d: dict[str, Any],
+ path: list[str],
+ value: Any, # noqa: ANN401
+ ) -> None:
"""Set a value in a nested dictionary based on the given path."""
for key in path[:-1]:
d = d.setdefault(key, {})
d[path[-1]] = value
- def _list_to_dict_preprocessing(self, data: Any) -> Any:
+ def _list_to_dict_preprocessing(self, data: Any) -> Any: # noqa: ANN401
if isinstance(data, dict):
# Process each key-value pair in the dictionary
return {k: self._list_to_dict_preprocessing(v) for k, v in data.items()}
@@ -71,7 +77,7 @@ class RecursiveJsonSplitter:
def _json_split(
self,
- data: Any,
+ data: Any, # noqa: ANN401
current_path: Optional[list[str]] = None,
chunks: Optional[list[dict[str, Any]]] = None,
) -> list[dict[str, Any]]:
@@ -136,13 +142,13 @@ class RecursiveJsonSplitter:
metadatas: Optional[list[dict[Any, Any]]] = None,
) -> list[Document]:
"""Create documents from a list of json objects (Dict)."""
- _metadatas = metadatas or [{}] * len(texts)
+ metadatas_ = metadatas or [{}] * len(texts)
documents = []
for i, text in enumerate(texts):
for chunk in self.split_text(
json_data=text, convert_lists=convert_lists, ensure_ascii=ensure_ascii
):
- metadata = copy.deepcopy(_metadatas[i])
+ metadata = copy.deepcopy(metadatas_[i])
new_doc = Document(page_content=chunk, metadata=metadata)
documents.append(new_doc)
return documents
diff --git a/libs/text-splitters/langchain_text_splitters/jsx.py b/libs/text-splitters/langchain_text_splitters/jsx.py
index 4621670112..1333065a76 100644
--- a/libs/text-splitters/langchain_text_splitters/jsx.py
+++ b/libs/text-splitters/langchain_text_splitters/jsx.py
@@ -1,3 +1,5 @@
+"""JavaScript framework text splitter."""
+
import re
from typing import Any, Optional
diff --git a/libs/text-splitters/langchain_text_splitters/konlpy.py b/libs/text-splitters/langchain_text_splitters/konlpy.py
index fc1763790f..863447e883 100644
--- a/libs/text-splitters/langchain_text_splitters/konlpy.py
+++ b/libs/text-splitters/langchain_text_splitters/konlpy.py
@@ -1,9 +1,18 @@
+"""Konlpy text splitter."""
+
from __future__ import annotations
from typing import Any
from langchain_text_splitters.base import TextSplitter
+try:
+ import konlpy
+
+ _HAS_KONLPY = True
+except ImportError:
+ _HAS_KONLPY = False
+
class KonlpyTextSplitter(TextSplitter):
"""Splitting text using Konlpy package.
@@ -19,14 +28,12 @@ class KonlpyTextSplitter(TextSplitter):
"""Initialize the Konlpy text splitter."""
super().__init__(**kwargs)
self._separator = separator
- try:
- import konlpy
- except ImportError as err:
+ if not _HAS_KONLPY:
msg = """
Konlpy is not installed, please install it with
`pip install konlpy`
"""
- raise ImportError(msg) from err
+ raise ImportError(msg)
self.kkma = konlpy.tag.Kkma()
def split_text(self, text: str) -> list[str]:
diff --git a/libs/text-splitters/langchain_text_splitters/latex.py b/libs/text-splitters/langchain_text_splitters/latex.py
index 55d47f7851..fc9c2ff498 100644
--- a/libs/text-splitters/langchain_text_splitters/latex.py
+++ b/libs/text-splitters/langchain_text_splitters/latex.py
@@ -1,3 +1,5 @@
+"""Latex text splitter."""
+
from __future__ import annotations
from typing import Any
diff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.py
index b5fb63c5d7..a1d40a4322 100644
--- a/libs/text-splitters/langchain_text_splitters/markdown.py
+++ b/libs/text-splitters/langchain_text_splitters/markdown.py
@@ -1,3 +1,5 @@
+"""Markdown text splitters."""
+
from __future__ import annotations
import re
@@ -138,7 +140,6 @@ class MarkdownHeaderTextSplitter:
current_content: list[str] = []
current_metadata: dict[str, str] = {}
# Keep track of the nested header structure
- # header_stack: List[Dict[str, Union[int, str]]] = []
header_stack: list[HeaderType] = []
initial_metadata: dict[str, str] = {}
diff --git a/libs/text-splitters/langchain_text_splitters/nltk.py b/libs/text-splitters/langchain_text_splitters/nltk.py
index 4ac7b0dbf0..8041d47d49 100644
--- a/libs/text-splitters/langchain_text_splitters/nltk.py
+++ b/libs/text-splitters/langchain_text_splitters/nltk.py
@@ -1,9 +1,18 @@
+"""NLTK text splitter."""
+
from __future__ import annotations
from typing import Any
from langchain_text_splitters.base import TextSplitter
+try:
+ import nltk
+
+ _HAS_NLTK = True
+except ImportError:
+ _HAS_NLTK = False
+
class NLTKTextSplitter(TextSplitter):
"""Splitting text using NLTK package."""
@@ -21,19 +30,16 @@ class NLTKTextSplitter(TextSplitter):
self._separator = separator
self._language = language
self._use_span_tokenize = use_span_tokenize
- if self._use_span_tokenize and self._separator != "":
+ if self._use_span_tokenize and self._separator:
msg = "When use_span_tokenize is True, separator should be ''"
raise ValueError(msg)
- try:
- import nltk
-
- if self._use_span_tokenize:
- self._tokenizer = nltk.tokenize._get_punkt_tokenizer(self._language)
- else:
- self._tokenizer = nltk.tokenize.sent_tokenize
- except ImportError as err:
+ if not _HAS_NLTK:
msg = "NLTK is not installed, please install it with `pip install nltk`."
- raise ImportError(msg) from err
+ raise ImportError(msg)
+ if self._use_span_tokenize:
+ self._tokenizer = nltk.tokenize._get_punkt_tokenizer(self._language) # noqa: SLF001
+ else:
+ self._tokenizer = nltk.tokenize.sent_tokenize
def split_text(self, text: str) -> list[str]:
"""Split incoming text and return chunks."""
diff --git a/libs/text-splitters/langchain_text_splitters/python.py b/libs/text-splitters/langchain_text_splitters/python.py
index ffd4a10f62..ccab44bcf6 100644
--- a/libs/text-splitters/langchain_text_splitters/python.py
+++ b/libs/text-splitters/langchain_text_splitters/python.py
@@ -1,3 +1,5 @@
+"""Python code text splitter."""
+
from __future__ import annotations
from typing import Any
diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
index 8cbe227495..de3cd27970 100644
--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
@@ -1,9 +1,18 @@
+"""Sentence transformers text splitter."""
+
from __future__ import annotations
from typing import Any, Optional, cast
from langchain_text_splitters.base import TextSplitter, Tokenizer, split_text_on_tokens
+try:
+ from sentence_transformers import SentenceTransformer
+
+ _HAS_SENTENCE_TRANSFORMERS = True
+except ImportError:
+ _HAS_SENTENCE_TRANSFORMERS = False
+
class SentenceTransformersTokenTextSplitter(TextSplitter):
"""Splitting text to tokens using sentence model tokenizer."""
@@ -18,15 +27,13 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
"""Create a new TextSplitter."""
super().__init__(**kwargs, chunk_overlap=chunk_overlap)
- try:
- from sentence_transformers import SentenceTransformer
- except ImportError as err:
+ if not _HAS_SENTENCE_TRANSFORMERS:
msg = (
"Could not import sentence_transformers python package. "
- "This is needed in order to for SentenceTransformersTokenTextSplitter. "
+ "This is needed in order to use SentenceTransformersTokenTextSplitter. "
"Please install it with `pip install sentence-transformers`."
)
- raise ImportError(msg) from err
+ raise ImportError(msg)
self.model_name = model_name
self._model = SentenceTransformer(self.model_name)
diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.py
index af7c8e3e25..cfc6fa6eec 100644
--- a/libs/text-splitters/langchain_text_splitters/spacy.py
+++ b/libs/text-splitters/langchain_text_splitters/spacy.py
@@ -1,9 +1,20 @@
+"""Spacy text splitter."""
+
from __future__ import annotations
from typing import Any
from langchain_text_splitters.base import TextSplitter
+try:
+ import spacy
+ from spacy.lang.en import English
+ from spacy.language import Language
+
+ _HAS_SPACY = True
+except ImportError:
+ _HAS_SPACY = False
+
class SpacyTextSplitter(TextSplitter):
"""Splitting text using Spacy package.
@@ -42,16 +53,12 @@ class SpacyTextSplitter(TextSplitter):
def _make_spacy_pipeline_for_splitting(
pipeline: str, *, max_length: int = 1_000_000
-) -> Any: # avoid importing spacy
- try:
- import spacy
- except ImportError as err:
+) -> Language:
+ if not _HAS_SPACY:
msg = "Spacy is not installed, please install it with `pip install spacy`."
- raise ImportError(msg) from err
+ raise ImportError(msg)
if pipeline == "sentencizer":
- from spacy.lang.en import English
-
- sentencizer: Any = English()
+ sentencizer: Language = English()
sentencizer.add_pipe("sentencizer")
else:
sentencizer = spacy.load(pipeline, exclude=["ner", "tagger"])
diff --git a/libs/text-splitters/pyproject.toml b/libs/text-splitters/pyproject.toml
index 9d4dd95f39..35ced364e4 100644
--- a/libs/text-splitters/pyproject.toml
+++ b/libs/text-splitters/pyproject.toml
@@ -25,7 +25,8 @@ typing = [
"mypy<1.18,>=1.17.1",
"lxml-stubs<1.0.0,>=0.5.1",
"types-requests<3.0.0.0,>=2.31.0.20240218",
- "tiktoken<1.0.0,>=0.8.0",
+ "tiktoken<1.0.0,>=0.11.0",
+ "beautifulsoup4<5.0.0,>=4.13.5",
]
dev = ["jupyter<2.0.0,>=1.0.0", "langchain-core"]
test = [
@@ -66,71 +67,46 @@ ignore_missing_imports = true
target-version = "py39"
[tool.ruff.lint]
-select = [
- "A", # flake8-builtins
- "B", # flake8-bugbear
- "ASYNC", # flake8-async
- "C4", # flake8-comprehensions
- "COM", # flake8-commas
- "D", # pydocstyle
- "E", # pycodestyle error
- "EM", # flake8-errmsg
- "F", # pyflakes
- "FA", # flake8-future-annotations
- "FBT", # flake8-boolean-trap
- "FLY", # flake8-flynt
- "I", # isort
- "ICN", # flake8-import-conventions
- "INT", # flake8-gettext
- "ISC", # isort-comprehensions
- "PGH", # pygrep-hooks
- "PIE", # flake8-pie
- "PERF", # flake8-perf
- "PYI", # flake8-pyi
- "Q", # flake8-quotes
- "RET", # flake8-return
- "RSE", # flake8-rst-docstrings
- "RUF", # ruff
- "S", # flake8-bandit
- "SLF", # flake8-self
- "SLOT", # flake8-slots
- "SIM", # flake8-simplify
- "T10", # flake8-debugger
- "T20", # flake8-print
- "TID", # flake8-tidy-imports
- "UP", # pyupgrade
- "W", # pycodestyle warning
- "YTT", # flake8-2020
-]
+select = [ "ALL",]
ignore = [
- "D100", # pydocstyle: Missing docstring in public module
- "D101", # pydocstyle: Missing docstring in public class
- "D102", # pydocstyle: Missing docstring in public method
- "D103", # pydocstyle: Missing docstring in public function
- "D104", # pydocstyle: Missing docstring in public package
- "D105", # pydocstyle: Missing docstring in magic method
- "D107", # pydocstyle: Missing docstring in __init__
- "D203", # Messes with the formatter
- "D213", # pydocstyle: Multi-line docstring summary should start at the second line (incompatible with D212)
- "D407", # pydocstyle: Missing-dashed-underline-after-section
+ "C90", # McCabe complexity
"COM812", # Messes with the formatter
+ "FIX002", # Line contains TODO
"ISC001", # Messes with the formatter
"PERF203", # Rarely useful
- "S112", # Rarely useful
+ "PLR09", # Too many something (arg, statements, etc)
"RUF012", # Doesn't play well with Pydantic
- "SLF001", # Private member access
- "UP007", # pyupgrade: non-pep604-annotation-union
- "UP045", # pyupgrade: non-pep604-annotation-optional
+ "TC001", # Doesn't play well with Pydantic
+ "TC002", # Doesn't play well with Pydantic
+ "TC003", # Doesn't play well with Pydantic
+ "TD002", # Missing author in TODO
+ "TD003", # Missing issue link in TODO
+]
+unfixable = [
+ "B028", # People should intentionally tune the stacklevel
+ "PLW1510", # People should intentionally set the check argument
]
-unfixable = ["B028"] # People should intentionally tune the stacklevel
+
+flake8-annotations.allow-star-arg-any = true
+flake8-annotations.mypy-init-return = true
+flake8-type-checking.runtime-evaluated-base-classes = ["pydantic.BaseModel","langchain_core.load.serializable.Serializable","langchain_core.runnables.base.RunnableSerializable"]
+pep8-naming.classmethod-decorators = [ "classmethod", "langchain_core.utils.pydantic.pre_init", "pydantic.field_validator", "pydantic.v1.root_validator",]
+pydocstyle.convention = "google"
pyupgrade.keep-runtime-typing = true
-[tool.ruff.lint.extend-per-file-ignores]
-"tests/**/*.py" = [
- "S101", # Tests need assertions
- "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
+[tool.ruff.lint.per-file-ignores]
+"scripts/**" = [
+ "D1", # Docstrings not mandatory in scripts
+ "INP001", # Not a package
+ "S311" # Standard pseudo-random generators are not suitable for cryptographic purposes
+]
+"tests/**" = [
+ "D1", # Docstrings not mandatory in tests
+ "PLR2004", # Magic value comparisons
+ "S101", # Tests need assertions
+ "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
+ "SLF001" # Private member access in tests
]
-
[tool.coverage.run]
omit = ["tests/*"]
@@ -142,9 +118,3 @@ markers = [
"compile: mark placeholder test used to compile integration tests without running them",
]
asyncio_mode = "auto"
-
-[tool.ruff.lint.pydocstyle]
-convention = "google"
-
-[tool.ruff.lint.per-file-ignores]
-"tests/**" = ["D"]
diff --git a/libs/text-splitters/scripts/check_imports.py b/libs/text-splitters/scripts/check_imports.py
index 00069f9f95..6242951b62 100644
--- a/libs/text-splitters/scripts/check_imports.py
+++ b/libs/text-splitters/scripts/check_imports.py
@@ -10,7 +10,7 @@ if __name__ == "__main__":
try:
module_name = f"test_module_{uuid.uuid4().hex[:20]}"
SourceFileLoader(module_name, file).load_module()
- except Exception:
+ except Exception: # noqa: BLE001
has_failure = True
print(file) # noqa: T201
traceback.print_exc()
diff --git a/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py b/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py
index b4d6499f98..82e55b0d9d 100644
--- a/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py
+++ b/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py
@@ -1,6 +1,6 @@
"""Test text splitting functionality using NLTK and Spacy based sentence splitters."""
-from typing import Any
+import re
import nltk
import pytest
@@ -15,11 +15,8 @@ def setup_module() -> None:
@pytest.fixture
-def spacy() -> Any:
- try:
- import spacy
- except ImportError:
- pytest.skip("Spacy not installed.")
+def spacy() -> None:
+ spacy = pytest.importorskip("spacy")
# Check if en_core_web_sm model is available
try:
@@ -32,18 +29,27 @@ def spacy() -> Any:
"en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"
)
- return spacy
-
def test_nltk_text_splitting_args() -> None:
"""Test invalid arguments."""
- with pytest.raises(ValueError):
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ "Got a larger chunk overlap (4) than chunk size (2), should be smaller."
+ ),
+ ):
NLTKTextSplitter(chunk_size=2, chunk_overlap=4)
-def test_spacy_text_splitting_args(spacy: Any) -> None:
+def test_spacy_text_splitting_args() -> None:
"""Test invalid arguments."""
- with pytest.raises(ValueError):
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ "Got a larger chunk overlap (4) than chunk size (2), should be smaller."
+ ),
+ ):
SpacyTextSplitter(chunk_size=2, chunk_overlap=4)
@@ -57,8 +63,9 @@ def test_nltk_text_splitter() -> None:
assert output == expected_output
@pytest.mark.parametrize("pipeline", ["sentencizer", "en_core_web_sm"])
-def test_spacy_text_splitter(pipeline: str, spacy: Any) -> None:
+def test_spacy_text_splitter(pipeline: str) -> None:
"""Test splitting by sentence using Spacy."""
text = "This is sentence one. And this is sentence two."
separator = "|||"
@@ -68,8 +75,9 @@ def test_spacy_text_splitter(pipeline: str, spacy: Any) -> None:
assert output == expected_output
@pytest.mark.parametrize("pipeline", ["sentencizer", "en_core_web_sm"])
-def test_spacy_text_splitter_strip_whitespace(pipeline: str, spacy: Any) -> None:
+def test_spacy_text_splitter_strip_whitespace(pipeline: str) -> None:
"""Test splitting by sentence using Spacy."""
text = "This is sentence one. And this is sentence two."
separator = "|||"
@@ -83,7 +91,9 @@ def test_spacy_text_splitter_strip_whitespace(pipeline: str, spacy: Any) -> None
def test_nltk_text_splitter_args() -> None:
"""Test invalid arguments for NLTKTextSplitter."""
- with pytest.raises(ValueError):
+ with pytest.raises(
+ ValueError, match="When use_span_tokenize is True, separator should be ''"
+ ):
NLTKTextSplitter(
chunk_size=80,
chunk_overlap=0,
diff --git a/libs/text-splitters/tests/integration_tests/test_text_splitter.py b/libs/text-splitters/tests/integration_tests/test_text_splitter.py
index ad0549e242..9ee1122011 100644
--- a/libs/text-splitters/tests/integration_tests/test_text_splitter.py
+++ b/libs/text-splitters/tests/integration_tests/test_text_splitter.py
@@ -1,8 +1,7 @@
"""Test text splitters that require an integration."""
-from typing import Any
-
import pytest
+from transformers import GPT2TokenizerFast
from langchain_text_splitters import (
TokenTextSplitter,
@@ -13,25 +12,17 @@ from langchain_text_splitters.sentence_transformers import (
)
-def sentence_transformers() -> Any:
- try:
- import sentence_transformers
- except ImportError:
- pytest.skip("SentenceTransformers not installed.")
- return sentence_transformers
-
-
def test_huggingface_type_check() -> None:
"""Test that type checks are done properly on input."""
- with pytest.raises(ValueError):
- CharacterTextSplitter.from_huggingface_tokenizer("foo")
+ with pytest.raises(
+ ValueError,
+ match="Tokenizer received was not an instance of PreTrainedTokenizerBase",
+ ):
+ CharacterTextSplitter.from_huggingface_tokenizer("foo") # type: ignore[arg-type]
def test_huggingface_tokenizer() -> None:
"""Test text splitter that uses a HuggingFace tokenizer."""
- from transformers import GPT2TokenizerFast
-
tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(
tokenizer, separator=" ", chunk_size=1, chunk_overlap=0
@@ -63,7 +54,8 @@ def test_token_text_splitter_from_tiktoken() -> None:
assert expected_tokenizer == actual_tokenizer
-def test_sentence_transformers_count_tokens(sentence_transformers: Any) -> None:
[email protected]("sentence_transformers")
+def test_sentence_transformers_count_tokens() -> None:
splitter = SentenceTransformersTokenTextSplitter(
model_name="sentence-transformers/paraphrase-albert-small-v2"
)
@@ -78,7 +70,8 @@ def test_sentence_transformers_count_tokens(sentence_transformers: Any) -> None:
assert expected_token_count == token_count
-def test_sentence_transformers_split_text(sentence_transformers: Any) -> None:
[email protected]("sentence_transformers")
+def test_sentence_transformers_split_text() -> None:
splitter = SentenceTransformersTokenTextSplitter(
model_name="sentence-transformers/paraphrase-albert-small-v2"
)
@@ -88,7 +81,8 @@ def test_sentence_transformers_split_text(sentence_transformers: Any) -> None:
assert expected_text_chunks == text_chunks
-def test_sentence_transformers_multiple_tokens(sentence_transformers: Any) -> None:
[email protected]("sentence_transformers")
+def test_sentence_transformers_multiple_tokens() -> None:
splitter = SentenceTransformersTokenTextSplitter(chunk_overlap=0)
text = "Lorem "
diff --git a/libs/text-splitters/tests/unit_tests/conftest.py b/libs/text-splitters/tests/unit_tests/conftest.py
index ad5ea6273c..d56ef0cedf 100644
--- a/libs/text-splitters/tests/unit_tests/conftest.py
+++ b/libs/text-splitters/tests/unit_tests/conftest.py
@@ -4,10 +4,9 @@ from collections.abc import Sequence
from importlib import util
import pytest
-from pytest import Config, Function, Parser
-def pytest_addoption(parser: Parser) -> None:
+def pytest_addoption(parser: pytest.Parser) -> None:
"""Add custom command line options to pytest."""
parser.addoption(
"--only-extended",
@@ -21,7 +20,9 @@ def pytest_addoption(parser: Parser) -> None:
)
-def pytest_collection_modifyitems(config: Config, items: Sequence[Function]) -> None:
+def pytest_collection_modifyitems(
+ config: pytest.Config, items: Sequence[pytest.Function]
+) -> None:
"""Add implementations for handling custom markers.
At the moment, this adds support for a custom `requires` marker.
@@ -64,7 +65,7 @@ def pytest_collection_modifyitems(config: Config, items: Sequence[Function]) ->
if pkg not in required_pkgs_info:
try:
installed = util.find_spec(pkg) is not None
- except Exception:
+ except (ImportError, ValueError):
installed = False
required_pkgs_info[pkg] = installed
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index 5ecca3bdf7..c366af9170 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -1,11 +1,14 @@
"""Test text splitting functionality."""
+from __future__ import annotations
+
import random
import re
import string
from typing import Any, Callable
import pytest
+from bs4 import Tag
from langchain_core.documents import Document
from langchain_text_splitters import (
@@ -103,7 +106,9 @@ def test_character_text_splitter_longer_words() -> None:
def test_character_text_splitter_keep_separator_regex(
*, separator: str, is_separator_regex: bool
) -> None:
- """Test splitting by characters while keeping the separator
+ """Test CharacterTextSplitter keep separator regex.
+
+ Test splitting by characters while keeping the separator
that is a regex special character.
"""
text = "foo.bar.baz.123"
@@ -125,7 +130,9 @@ def test_character_text_splitter_keep_separator_regex(
def test_character_text_splitter_keep_separator_regex_start(
*, separator: str, is_separator_regex: bool
) -> None:
- """Test splitting by characters while keeping the separator
+ """Test CharacterTextSplitter keep separator regex and put at start.
+
+ Test splitting by characters while keeping the separator
that is a regex special character and placing it at the start of each chunk.
"""
text = "foo.bar.baz.123"
@@ -147,7 +154,9 @@ def test_character_text_splitter_keep_separator_regex_start(
def test_character_text_splitter_keep_separator_regex_end(
*, separator: str, is_separator_regex: bool
) -> None:
- """Test splitting by characters while keeping the separator
+ """Test CharacterTextSplitter keep separator regex and put at end.
+
+ Test splitting by characters while keeping the separator
that is a regex special character and placing it at the end of each chunk.
"""
text = "foo.bar.baz.123"
@@ -169,8 +178,11 @@ def test_character_text_splitter_keep_separator_regex_end(
def test_character_text_splitter_discard_separator_regex(
*, separator: str, is_separator_regex: bool
) -> None:
- """Test splitting by characters discarding the separator
- that is a regex special character."""
+ """Test CharacterTextSplitter discard separator regex.
+
+ Test splitting by characters discarding the separator
+ that is a regex special character.
+ """
text = "foo.bar.baz.123"
splitter = CharacterTextSplitter(
separator=separator,
@@ -210,12 +222,17 @@ def test_recursive_character_text_splitter_keep_separators() -> None:
def test_character_text_splitting_args() -> None:
"""Test invalid arguments."""
- with pytest.raises(ValueError):
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ "Got a larger chunk overlap (4) than chunk size (2), should be smaller."
+ ),
+ ):
CharacterTextSplitter(chunk_size=2, chunk_overlap=4)
for invalid_size in (0, -1):
- with pytest.raises(ValueError):
+ with pytest.raises(ValueError, match="chunk_size must be > 0, got"):
CharacterTextSplitter(chunk_size=invalid_size)
- with pytest.raises(ValueError):
+ with pytest.raises(ValueError, match="chunk_overlap must be >= 0, got -1"):
CharacterTextSplitter(chunk_size=2, chunk_overlap=-1)
@@ -1164,7 +1181,6 @@ def test_html_code_splitter() -> None:
def test_md_header_text_splitter_1() -> None:
"""Test markdown splitter by header: Case 1."""
-
markdown_document = (
"# Foo\n\n"
" ## Bar\n\n"
@@ -1235,7 +1251,6 @@ def test_md_header_text_splitter_2() -> None:
def test_md_header_text_splitter_3() -> None:
"""Test markdown splitter by header: Case 3."""
-
markdown_document = (
"# Foo\n\n"
" ## Bar\n\n"
@@ -1290,7 +1305,6 @@ def test_md_header_text_splitter_3() -> None:
def test_md_header_text_splitter_preserve_headers_1() -> None:
"""Test markdown splitter by header: Preserve Headers."""
-
markdown_document = (
"# Foo\n\n"
" ## Bat\n\n"
@@ -1324,7 +1338,6 @@ def test_md_header_text_splitter_preserve_headers_1() -> None:
def test_md_header_text_splitter_preserve_headers_2() -> None:
"""Test markdown splitter by header: Preserve Headers."""
-
markdown_document = (
"# Foo\n\n"
" ## Bar\n\n"
@@ -1372,7 +1385,6 @@ def test_md_header_text_splitter_preserve_headers_2() -> None:
@pytest.mark.parametrize("fence", [("```"), ("~~~")])
def test_md_header_text_splitter_fenced_code_block(fence: str) -> None:
"""Test markdown splitter by header: Fenced code block."""
-
markdown_document = (
f"# This is a Header\n\n{fence}\nfoo()\n# Not a header\nbar()\n{fence}"
)
@@ -1402,7 +1414,6 @@ def test_md_header_text_splitter_fenced_code_block_interleaved(
fence: str, other_fence: str
) -> None:
"""Test markdown splitter by header: Interleaved fenced code block."""
-
markdown_document = (
"# This is a Header\n\n"
f"{fence}\n"
@@ -1438,7 +1449,6 @@ def test_md_header_text_splitter_fenced_code_block_interleaved(
@pytest.mark.parametrize("characters", ["\ufeff"])
def test_md_header_text_splitter_with_invisible_characters(characters: str) -> None:
"""Test markdown splitter by header: Fenced code block."""
-
markdown_document = f"{characters}# Foo\n\nfoo()\n{characters}## Bar\n\nbar()"
headers_to_split_on = [
@@ -1609,7 +1619,6 @@ EXPERIMENTAL_MARKDOWN_DOCUMENT = (
def test_experimental_markdown_syntax_text_splitter() -> None:
"""Test experimental markdown syntax splitter."""
-
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter()
output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)
@@ -1663,7 +1672,6 @@ def test_experimental_markdown_syntax_text_splitter() -> None:
def test_experimental_markdown_syntax_text_splitter_header_configuration() -> None:
"""Test experimental markdown syntax splitter."""
-
headers_to_split_on = [("#", "Encabezamiento 1")]
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(
@@ -1709,7 +1717,6 @@ def test_experimental_markdown_syntax_text_splitter_header_configuration() -> No
def test_experimental_markdown_syntax_text_splitter_with_headers() -> None:
"""Test experimental markdown syntax splitter."""
-
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(strip_headers=False)
output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)
@@ -1768,7 +1775,6 @@ def test_experimental_markdown_syntax_text_splitter_with_headers() -> None:
def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:
"""Test experimental markdown syntax splitter."""
-
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(return_each_line=True)
output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)
@@ -1876,8 +1882,11 @@ EXPERIMENTAL_MARKDOWN_DOCUMENTS = [
def test_experimental_markdown_syntax_text_splitter_on_multi_files() -> None:
- """Test experimental markdown syntax splitter split
- on default called consecutively on two files."""
+ """Test ExperimentalMarkdownSyntaxTextSplitter on multiple files.
+
+ Test experimental markdown syntax splitter split on default called consecutively
+ on two files.
+ """
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter()
output = []
for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:
@@ -1958,8 +1967,11 @@ def test_experimental_markdown_syntax_text_splitter_on_multi_files() -> None:
def test_experimental_markdown_syntax_text_splitter_split_lines_on_multi_files() -> (
None
):
- """Test experimental markdown syntax splitter split
- on each line called consecutively on two files."""
+ """Test ExperimentalMarkdownSyntaxTextSplitter split lines on multiple files.
+
+ Test experimental markdown syntax splitter split on each line called consecutively
+ on two files.
+ """
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(return_each_line=True)
output = []
for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:
@@ -2083,9 +2095,10 @@ def test_experimental_markdown_syntax_text_splitter_split_lines_on_multi_files()
def test_experimental_markdown_syntax_text_splitter_with_header_on_multi_files() -> (
None
):
- """Test experimental markdown splitter
- by header called consecutively on two files"""
+ """Test ExperimentalMarkdownSyntaxTextSplitter with header on multiple files.
+ Test experimental markdown splitter by header called consecutively on two files.
+ """
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(strip_headers=False)
output = []
for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:
@@ -2171,9 +2184,11 @@ def test_experimental_markdown_syntax_text_splitter_with_header_on_multi_files()
def test_experimental_markdown_syntax_text_splitter_header_config_on_multi_files() -> (
None
):
- """Test experimental markdown splitter
- by header configuration called consecutively on two files"""
+ """Test ExperimentalMarkdownSyntaxTextSplitter header config on multiple files.
+ Test experimental markdown splitter by header configuration called consecutively
+ on two files.
+ """
headers_to_split_on = [("#", "Encabezamiento 1")]
markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(
headers_to_split_on=headers_to_split_on
@@ -2354,8 +2369,8 @@ def test_haskell_code_splitter() -> None:
def html_header_splitter_splitter_factory() -> Callable[
[list[tuple[str, str]]], HTMLHeaderTextSplitter
]:
- """
- Fixture to create an HTMLHeaderTextSplitter instance with given headers.
+ """Fixture to create an HTMLHeaderTextSplitter instance with given headers.
+
This factory allows dynamic creation of splitters with different headers.
"""
@@ -2553,14 +2568,15 @@ def html_header_splitter_splitter_factory() -> Callable[
)
@pytest.mark.requires("bs4")
def test_html_header_text_splitter(
- html_header_splitter_splitter_factory: Any,
+ html_header_splitter_splitter_factory: Callable[
+ [list[tuple[str, str]]], HTMLHeaderTextSplitter
+ ],
headers_to_split_on: list[tuple[str, str]],
html_input: str,
expected_documents: list[Document],
test_case: str,
) -> None:
- """
- Test the HTML header text splitter.
+ """Test the HTML header text splitter.
Args:
html_header_splitter_splitter_factory (Any): Factory function to create
@@ -2574,10 +2590,7 @@ def test_html_header_text_splitter(
AssertionError: If the number of documents or their content/metadata
does not match the expected values.
"""
-
- splitter = html_header_splitter_splitter_factory(
- headers_to_split_on=headers_to_split_on
- )
+ splitter = html_header_splitter_splitter_factory(headers_to_split_on)
docs = splitter.split_text(html_input)
assert len(docs) == len(expected_documents), (
@@ -2709,14 +2722,15 @@ def test_html_header_text_splitter(
)
@pytest.mark.requires("bs4")
def test_additional_html_header_text_splitter(
- html_header_splitter_splitter_factory: Any,
+ html_header_splitter_splitter_factory: Callable[
+ [list[tuple[str, str]]], HTMLHeaderTextSplitter
+ ],
headers_to_split_on: list[tuple[str, str]],
html_content: str,
expected_output: list[Document],
test_case: str,
) -> None:
- """
- Test the HTML header text splitter.
+ """Test the HTML header text splitter.
Args:
html_header_splitter_splitter_factory (Any): Factory function to create
@@ -2730,9 +2744,7 @@ def test_additional_html_header_text_splitter(
AssertionError: If the number of documents or their content/metadata
does not match the expected output.
"""
- splitter = html_header_splitter_splitter_factory(
- headers_to_split_on=headers_to_split_on
- )
+ splitter = html_header_splitter_splitter_factory(headers_to_split_on)
docs = splitter.split_text(html_content)
assert len(docs) == len(expected_output), (
@@ -2780,14 +2792,16 @@ def test_additional_html_header_text_splitter(
)
@pytest.mark.requires("bs4")
def test_html_no_headers_with_multiple_splitters(
- html_header_splitter_splitter_factory: Any,
+ html_header_splitter_splitter_factory: Callable[
+ [list[tuple[str, str]]], HTMLHeaderTextSplitter
+ ],
headers_to_split_on: list[tuple[str, str]],
html_content: str,
expected_output: list[Document],
test_case: str,
) -> None:
- """
- Test HTML content splitting without headers using multiple splitters.
+ """Test HTML content splitting without headers using multiple splitters.
+
Args:
html_header_splitter_splitter_factory (Any): Factory to create the
HTML header splitter.
@@ -2796,13 +2810,12 @@ def test_html_no_headers_with_multiple_splitters(
expected_output (List[Document]): Expected list of Document objects
after splitting.
test_case (str): Description of the test case.
+
Raises:
AssertionError: If the number of documents or their content/metadata
does not match the expected output.
"""
- splitter = html_header_splitter_splitter_factory(
- headers_to_split_on=headers_to_split_on
- )
+ splitter = html_header_splitter_splitter_factory(headers_to_split_on)
docs = splitter.split_text(html_content)
assert len(docs) == len(expected_output), (
@@ -3046,7 +3059,7 @@ def test_happy_path_splitting_with_duplicate_header_tag() -> None:
def test_split_json() -> None:
- """Test json text splitter"""
+ """Test json text splitter."""
max_chunk = 800
splitter = RecursiveJsonSplitter(max_chunk_size=max_chunk)
@@ -3068,7 +3081,7 @@ def test_split_json() -> None:
def test_split_json_with_lists() -> None:
- """Test json text splitter with list conversion"""
+ """Test json text splitter with list conversion."""
max_chunk = 800
splitter = RecursiveJsonSplitter(max_chunk_size=max_chunk)
@@ -3240,7 +3253,7 @@ def test_visualbasic6_code_splitter() -> None:
]
-def custom_iframe_extractor(iframe_tag: Any) -> str:
+def custom_iframe_extractor(iframe_tag: Tag) -> str:
iframe_src = iframe_tag.get("src", "")
return f"[iframe:{iframe_src}]({iframe_src})"
@@ -3324,8 +3337,11 @@ def test_html_splitter_with_nested_elements() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_with_preserved_elements() -> None:
- """Test HTML splitting with preserved elements like <table>, <ul> with low chunk
- size."""
+ """Test HTML splitter with preserved elements.
+
+ Test HTML splitting with preserved elements like <table>, <ul> with low chunk
+ size.
+ """
html_content = """
<h1>Section 1</h1>
<table>
@@ -3563,8 +3579,11 @@ def test_html_splitter_with_no_headers() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_with_media_preservation() -> None:
- """Test HTML splitting with media elements preserved and converted to Markdown-like
- links."""
+ """Test HTML splitter with media preservation.
+
+ Test HTML splitting with media elements preserved and converted to Markdown-like
+ links.
+ """
html_content = """
<h1>Section 1</h1>
<p>This is an image:</p>
@@ -3600,7 +3619,7 @@ def test_html_splitter_with_media_preservation() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_keep_separator_true() -> None:
- """Test HTML splitting with keep_separator=True"""
+ """Test HTML splitting with keep_separator=True."""
html_content = """
<h1>Section 1</h1>
<p>This is some text. This is some other text.</p>
@@ -3629,7 +3648,7 @@ def test_html_splitter_keep_separator_true() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_keep_separator_false() -> None:
- """Test HTML splitting with keep_separator=False"""
+ """Test HTML splitting with keep_separator=False."""
html_content = """
<h1>Section 1</h1>
<p>This is some text. This is some other text.</p>
@@ -3658,7 +3677,7 @@ def test_html_splitter_keep_separator_false() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_keep_separator_start() -> None:
- """Test HTML splitting with keep_separator="start" """
+ """Test HTML splitting with keep_separator="start"."""
html_content = """
<h1>Section 1</h1>
<p>This is some text. This is some other text.</p>
@@ -3687,7 +3706,7 @@ def test_html_splitter_keep_separator_start() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_keep_separator_end() -> None:
- """Test HTML splitting with keep_separator="end" """
+ """Test HTML splitting with keep_separator="end"."""
html_content = """
<h1>Section 1</h1>
<p>This is some text. This is some other text.</p>
@@ -3716,7 +3735,7 @@ def test_html_splitter_keep_separator_end() -> None:
@pytest.mark.requires("bs4")
def test_html_splitter_keep_separator_default() -> None:
- """Test HTML splitting with keep_separator not set"""
+ """Test HTML splitting with keep_separator not set."""
html_content = """
<h1>Section 1</h1>
<p>This is some text. This is some other text.</p>
diff --git a/libs/text-splitters/uv.lock b/libs/text-splitters/uv.lock
index b5f8882b31..bf549a5ad2 100644
--- a/libs/text-splitters/uv.lock
+++ b/libs/text-splitters/uv.lock
@@ -129,15 +129,15 @@ wheels = [
[[package]]
name = "beautifulsoup4"
-version = "4.13.1"
+version = "4.13.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "soupsieve" },
{ name = "typing-extensions" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/58/f3/d90227cc52f7b8fcd0f2af804f56e55edf8dd07036b681a2809e3245318b/beautifulsoup4-4.13.1.tar.gz", hash = "sha256:741c8b6903a1e4ae8ba32b9c9ae7510dab7a197fdbadcf9fcdeb0891ef5ec66a", size = 618295, upload-time = "2025-02-03T12:15:13.622Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/85/2e/3e5079847e653b1f6dc647aa24549d68c6addb4c595cc0d902d1b19308ad/beautifulsoup4-4.13.5.tar.gz", hash = "sha256:5e70131382930e7c3de33450a2f54a63d5e4b19386eab43a5b34d594268f3695", size = 622954, upload-time = "2025-08-24T14:06:13.168Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/18/75/899bf9b6270b2ce5e8f01b8da121b29e4b88256feb2cf6c6418d4cc42130/beautifulsoup4-4.13.1-py3-none-any.whl", hash = "sha256:72465267014897bb10ca749bb632bde6c2d20f3254afd5458544bd74e6c2e6d8", size = 185056, upload-time = "2025-02-03T12:15:12.076Z" },
+ { url = "https://files.pythonhosted.org/packages/04/eb/f4151e0c7377a6e08a38108609ba5cede57986802757848688aeedd1b9e8/beautifulsoup4-4.13.5-py3-none-any.whl", hash = "sha256:642085eaa22233aceadff9c69651bc51e8bf3f874fb6d7104ece2beb24b47c4a", size = 105113, upload-time = "2025-08-24T14:06:14.884Z" },
]
[[package]]
@@ -1191,6 +1191,7 @@ test-integration = [
{ name = "transformers" },
]
typing = [
+ { name = "beautifulsoup4" },
{ name = "lxml-stubs" },
{ name = "mypy" },
{ name = "tiktoken" },
@@ -1229,9 +1230,10 @@ test-integration = [
{ name = "transformers", specifier = ">=4.51.3,<5.0.0" },
]
typing = [
+ { name = "beautifulsoup4", specifier = ">=4.13.5,<5.0.0" },
{ name = "lxml-stubs", specifier = ">=0.5.1,<1.0.0" },
{ name = "mypy", specifier = ">=1.17.1,<1.18" },
- { name = "tiktoken", specifier = ">=0.8.0,<1.0.0" },
+ { name = "tiktoken", specifier = ">=0.11.0,<1.0.0" },
{ name = "types-requests", specifier = ">=2.31.0.20240218,<3.0.0.0" },
]
@@ -3464,44 +3466,44 @@ wheels = [
[[package]]
name = "tiktoken"
-version = "0.8.0"
+version = "0.11.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "regex" },
{ name = "requests" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/37/02/576ff3a6639e755c4f70997b2d315f56d6d71e0d046f4fb64cb81a3fb099/tiktoken-0.8.0.tar.gz", hash = "sha256:9ccbb2740f24542534369c5635cfd9b2b3c2490754a78ac8831d99f89f94eeb2", size = 35107, upload-time = "2024-10-03T22:44:04.196Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/c9/ba/a35fad753bbca8ba0cc1b0f3402a70256a110ced7ac332cf84ba89fc87ab/tiktoken-0.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b07e33283463089c81ef1467180e3e00ab00d46c2c4bbcef0acab5f771d6695e", size = 1039905, upload-time = "2024-10-03T22:43:17.292Z" },
- { url = "https://files.pythonhosted.org/packages/91/05/13dab8fd7460391c387b3e69e14bf1e51ff71fe0a202cd2933cc3ea93fb6/tiktoken-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9269348cb650726f44dd3bbb3f9110ac19a8dcc8f54949ad3ef652ca22a38e21", size = 982417, upload-time = "2024-10-03T22:43:19.437Z" },
- { url = "https://files.pythonhosted.org/packages/e9/98/18ec4a8351a6cf4537e40cd6e19a422c10cce1ef00a2fcb716e0a96af58b/tiktoken-0.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e13f37bc4ef2d012731e93e0fef21dc3b7aea5bb9009618de9a4026844e560", size = 1144915, upload-time = "2024-10-03T22:43:21.385Z" },
- { url = "https://files.pythonhosted.org/packages/2e/28/cf3633018cbcc6deb7805b700ccd6085c9a5a7f72b38974ee0bffd56d311/tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f13d13c981511331eac0d01a59b5df7c0d4060a8be1e378672822213da51e0a2", size = 1177221, upload-time = "2024-10-03T22:43:23.325Z" },
- { url = "https://files.pythonhosted.org/packages/57/81/8a5be305cbd39d4e83a794f9e80c7f2c84b524587b7feb27c797b2046d51/tiktoken-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6b2ddbc79a22621ce8b1166afa9f9a888a664a579350dc7c09346a3b5de837d9", size = 1237398, upload-time = "2024-10-03T22:43:24.71Z" },
- { url = "https://files.pythonhosted.org/packages/dc/da/8d1cc3089a83f5cf11c2e489332752981435280285231924557350523a59/tiktoken-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d8c2d0e5ba6453a290b86cd65fc51fedf247e1ba170191715b049dac1f628005", size = 884215, upload-time = "2024-10-03T22:43:26.793Z" },
- { url = "https://files.pythonhosted.org/packages/f6/1e/ca48e7bfeeccaf76f3a501bd84db1fa28b3c22c9d1a1f41af9fb7579c5f6/tiktoken-0.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d622d8011e6d6f239297efa42a2657043aaed06c4f68833550cac9e9bc723ef1", size = 1039700, upload-time = "2024-10-03T22:43:28.315Z" },
- { url = "https://files.pythonhosted.org/packages/8c/f8/f0101d98d661b34534769c3818f5af631e59c36ac6d07268fbfc89e539ce/tiktoken-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2efaf6199717b4485031b4d6edb94075e4d79177a172f38dd934d911b588d54a", size = 982413, upload-time = "2024-10-03T22:43:29.807Z" },
- { url = "https://files.pythonhosted.org/packages/ac/3c/2b95391d9bd520a73830469f80a96e3790e6c0a5ac2444f80f20b4b31051/tiktoken-0.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5637e425ce1fc49cf716d88df3092048359a4b3bbb7da762840426e937ada06d", size = 1144242, upload-time = "2024-10-04T04:42:53.66Z" },
- { url = "https://files.pythonhosted.org/packages/01/c4/c4a4360de845217b6aa9709c15773484b50479f36bb50419c443204e5de9/tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fb0e352d1dbe15aba082883058b3cce9e48d33101bdaac1eccf66424feb5b47", size = 1176588, upload-time = "2024-10-03T22:43:31.136Z" },
- { url = "https://files.pythonhosted.org/packages/f8/a3/ef984e976822cd6c2227c854f74d2e60cf4cd6fbfca46251199914746f78/tiktoken-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56edfefe896c8f10aba372ab5706b9e3558e78db39dd497c940b47bf228bc419", size = 1237261, upload-time = "2024-10-03T22:43:32.75Z" },
- { url = "https://files.pythonhosted.org/packages/1e/86/eea2309dc258fb86c7d9b10db536434fc16420feaa3b6113df18b23db7c2/tiktoken-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:326624128590def898775b722ccc327e90b073714227175ea8febbc920ac0a99", size = 884537, upload-time = "2024-10-03T22:43:34.592Z" },
- { url = "https://files.pythonhosted.org/packages/c1/22/34b2e136a6f4af186b6640cbfd6f93400783c9ef6cd550d9eab80628d9de/tiktoken-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:881839cfeae051b3628d9823b2e56b5cc93a9e2efb435f4cf15f17dc45f21586", size = 1039357, upload-time = "2024-10-03T22:43:36.362Z" },
- { url = "https://files.pythonhosted.org/packages/04/d2/c793cf49c20f5855fd6ce05d080c0537d7418f22c58e71f392d5e8c8dbf7/tiktoken-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe9399bdc3f29d428f16a2f86c3c8ec20be3eac5f53693ce4980371c3245729b", size = 982616, upload-time = "2024-10-03T22:43:37.658Z" },
- { url = "https://files.pythonhosted.org/packages/b3/a1/79846e5ef911cd5d75c844de3fa496a10c91b4b5f550aad695c5df153d72/tiktoken-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a58deb7075d5b69237a3ff4bb51a726670419db6ea62bdcd8bd80c78497d7ab", size = 1144011, upload-time = "2024-10-03T22:43:39.092Z" },
- { url = "https://files.pythonhosted.org/packages/26/32/e0e3a859136e95c85a572e4806dc58bf1ddf651108ae8b97d5f3ebe1a244/tiktoken-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2908c0d043a7d03ebd80347266b0e58440bdef5564f84f4d29fb235b5df3b04", size = 1175432, upload-time = "2024-10-03T22:43:40.323Z" },
- { url = "https://files.pythonhosted.org/packages/c7/89/926b66e9025b97e9fbabeaa59048a736fe3c3e4530a204109571104f921c/tiktoken-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:294440d21a2a51e12d4238e68a5972095534fe9878be57d905c476017bff99fc", size = 1236576, upload-time = "2024-10-03T22:43:41.516Z" },
- { url = "https://files.pythonhosted.org/packages/45/e2/39d4aa02a52bba73b2cd21ba4533c84425ff8786cc63c511d68c8897376e/tiktoken-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d8f3192733ac4d77977432947d563d7e1b310b96497acd3c196c9bddb36ed9db", size = 883824, upload-time = "2024-10-03T22:43:43.33Z" },
- { url = "https://files.pythonhosted.org/packages/e3/38/802e79ba0ee5fcbf240cd624143f57744e5d411d2e9d9ad2db70d8395986/tiktoken-0.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:02be1666096aff7da6cbd7cdaa8e7917bfed3467cd64b38b1f112e96d3b06a24", size = 1039648, upload-time = "2024-10-03T22:43:45.22Z" },
- { url = "https://files.pythonhosted.org/packages/b1/da/24cdbfc302c98663fbea66f5866f7fa1048405c7564ab88483aea97c3b1a/tiktoken-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94ff53c5c74b535b2cbf431d907fc13c678bbd009ee633a2aca269a04389f9a", size = 982763, upload-time = "2024-10-03T22:43:46.571Z" },
- { url = "https://files.pythonhosted.org/packages/e4/f0/0ecf79a279dfa41fc97d00adccf976ecc2556d3c08ef3e25e45eb31f665b/tiktoken-0.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b231f5e8982c245ee3065cd84a4712d64692348bc609d84467c57b4b72dcbc5", size = 1144417, upload-time = "2024-10-03T22:43:48.633Z" },
- { url = "https://files.pythonhosted.org/packages/ab/d3/155d2d4514f3471a25dc1d6d20549ef254e2aa9bb5b1060809b1d3b03d3a/tiktoken-0.8.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4177faa809bd55f699e88c96d9bb4635d22e3f59d635ba6fd9ffedf7150b9953", size = 1175108, upload-time = "2024-10-03T22:43:50.568Z" },
- { url = "https://files.pythonhosted.org/packages/19/eb/5989e16821ee8300ef8ee13c16effc20dfc26c777d05fbb6825e3c037b81/tiktoken-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5376b6f8dc4753cd81ead935c5f518fa0fbe7e133d9e25f648d8c4dabdd4bad7", size = 1236520, upload-time = "2024-10-03T22:43:51.759Z" },
- { url = "https://files.pythonhosted.org/packages/40/59/14b20465f1d1cb89cfbc96ec27e5617b2d41c79da12b5e04e96d689be2a7/tiktoken-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:18228d624807d66c87acd8f25fc135665617cab220671eb65b50f5d70fa51f69", size = 883849, upload-time = "2024-10-03T22:43:53.999Z" },
- { url = "https://files.pythonhosted.org/packages/08/f3/8a8ba9329e6b426d822c974d58fc6477f3f7b3b8deef651813d275cbe75f/tiktoken-0.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e17807445f0cf1f25771c9d86496bd8b5c376f7419912519699f3cc4dc5c12e", size = 1040915, upload-time = "2024-10-03T22:43:55.177Z" },
- { url = "https://files.pythonhosted.org/packages/42/7a/914bd98100449422778f9222d00b3a4ee654211c40784e57541fa46311ab/tiktoken-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:886f80bd339578bbdba6ed6d0567a0d5c6cfe198d9e587ba6c447654c65b8edc", size = 983753, upload-time = "2024-10-03T22:43:56.303Z" },
- { url = "https://files.pythonhosted.org/packages/f7/01/1483856d84827c5fe541cb160f07914c6b063b8d961146e9c3557c4730c0/tiktoken-0.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6adc8323016d7758d6de7313527f755b0fc6c72985b7d9291be5d96d73ecd1e1", size = 1145913, upload-time = "2024-10-03T22:43:58.682Z" },
- { url = "https://files.pythonhosted.org/packages/c2/e1/6c7a772e0200131e960e3381f1d7b26406bc5612c70677989c1498af2a60/tiktoken-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b591fb2b30d6a72121a80be24ec7a0e9eb51c5500ddc7e4c2496516dd5e3816b", size = 1178505, upload-time = "2024-10-03T22:43:59.86Z" },
- { url = "https://files.pythonhosted.org/packages/3e/6b/3ae00f0bff5d0b6925bf6370cf0ff606f56daed76210c2b0a156017b78dc/tiktoken-0.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:845287b9798e476b4d762c3ebda5102be87ca26e5d2c9854002825d60cdb815d", size = 1239111, upload-time = "2024-10-03T22:44:01.094Z" },
- { url = "https://files.pythonhosted.org/packages/d5/3b/7c8812952ca55e1bab08afc1dda3c5991804c71b550b9402e82a082ab795/tiktoken-0.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:1473cfe584252dc3fa62adceb5b1c763c1874e04511b197da4e6de51d6ce5a02", size = 884803, upload-time = "2024-10-03T22:44:03.079Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/a7/86/ad0155a37c4f310935d5ac0b1ccf9bdb635dcb906e0a9a26b616dd55825a/tiktoken-0.11.0.tar.gz", hash = "sha256:3c518641aee1c52247c2b97e74d8d07d780092af79d5911a6ab5e79359d9b06a", size = 37648, upload-time = "2025-08-08T23:58:08.495Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8b/4d/c6a2e7dca2b4f2e9e0bfd62b3fe4f114322e2c028cfba905a72bc76ce479/tiktoken-0.11.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8a9b517d6331d7103f8bef29ef93b3cca95fa766e293147fe7bacddf310d5917", size = 1059937, upload-time = "2025-08-08T23:57:28.57Z" },
+ { url = "https://files.pythonhosted.org/packages/41/54/3739d35b9f94cb8dc7b0db2edca7192d5571606aa2369a664fa27e811804/tiktoken-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b4ddb1849e6bf0afa6cc1c5d809fb980ca240a5fffe585a04e119519758788c0", size = 999230, upload-time = "2025-08-08T23:57:30.241Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/f4/ec8d43338d28d53513004ebf4cd83732a135d11011433c58bf045890cc10/tiktoken-0.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10331d08b5ecf7a780b4fe4d0281328b23ab22cdb4ff65e68d56caeda9940ecc", size = 1130076, upload-time = "2025-08-08T23:57:31.706Z" },
+ { url = "https://files.pythonhosted.org/packages/94/80/fb0ada0a882cb453caf519a4bf0d117c2a3ee2e852c88775abff5413c176/tiktoken-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b062c82300341dc87e0258c69f79bed725f87e753c21887aea90d272816be882", size = 1183942, upload-time = "2025-08-08T23:57:33.142Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/e9/6c104355b463601719582823f3ea658bc3aa7c73d1b3b7553ebdc48468ce/tiktoken-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:195d84bec46169af3b1349a1495c151d37a0ff4cba73fd08282736be7f92cc6c", size = 1244705, upload-time = "2025-08-08T23:57:34.594Z" },
+ { url = "https://files.pythonhosted.org/packages/94/75/eaa6068f47e8b3f0aab9e05177cce2cf5aa2cc0ca93981792e620d4d4117/tiktoken-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe91581b0ecdd8783ce8cb6e3178f2260a3912e8724d2f2d49552b98714641a1", size = 884152, upload-time = "2025-08-08T23:57:36.18Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/91/912b459799a025d2842566fe1e902f7f50d54a1ce8a0f236ab36b5bd5846/tiktoken-0.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4ae374c46afadad0f501046db3da1b36cd4dfbfa52af23c998773682446097cf", size = 1059743, upload-time = "2025-08-08T23:57:37.516Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/e9/6faa6870489ce64f5f75dcf91512bf35af5864583aee8fcb0dcb593121f5/tiktoken-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25a512ff25dc6c85b58f5dd4f3d8c674dc05f96b02d66cdacf628d26a4e4866b", size = 999334, upload-time = "2025-08-08T23:57:38.595Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/3e/a05d1547cf7db9dc75d1461cfa7b556a3b48e0516ec29dfc81d984a145f6/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2130127471e293d385179c1f3f9cd445070c0772be73cdafb7cec9a3684c0458", size = 1129402, upload-time = "2025-08-08T23:57:39.627Z" },
+ { url = "https://files.pythonhosted.org/packages/34/9a/db7a86b829e05a01fd4daa492086f708e0a8b53952e1dbc9d380d2b03677/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e43022bf2c33f733ea9b54f6a3f6b4354b909f5a73388fb1b9347ca54a069c", size = 1184046, upload-time = "2025-08-08T23:57:40.689Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/bb/52edc8e078cf062ed749248f1454e9e5cfd09979baadb830b3940e522015/tiktoken-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:adb4e308eb64380dc70fa30493e21c93475eaa11669dea313b6bbf8210bfd013", size = 1244691, upload-time = "2025-08-08T23:57:42.251Z" },
+ { url = "https://files.pythonhosted.org/packages/60/d9/884b6cd7ae2570ecdcaffa02b528522b18fef1cbbfdbcaa73799807d0d3b/tiktoken-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:ece6b76bfeeb61a125c44bbefdfccc279b5288e6007fbedc0d32bfec602df2f2", size = 884392, upload-time = "2025-08-08T23:57:43.628Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/9e/eceddeffc169fc75fe0fd4f38471309f11cb1906f9b8aa39be4f5817df65/tiktoken-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fd9e6b23e860973cf9526544e220b223c60badf5b62e80a33509d6d40e6c8f5d", size = 1055199, upload-time = "2025-08-08T23:57:45.076Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/cf/5f02bfefffdc6b54e5094d2897bc80efd43050e5b09b576fd85936ee54bf/tiktoken-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76d53cee2da71ee2731c9caa747398762bda19d7f92665e882fef229cb0b5b", size = 996655, upload-time = "2025-08-08T23:57:46.304Z" },
+ { url = "https://files.pythonhosted.org/packages/65/8e/c769b45ef379bc360c9978c4f6914c79fd432400a6733a8afc7ed7b0726a/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef72aab3ea240646e642413cb363b73869fed4e604dcfd69eec63dc54d603e8", size = 1128867, upload-time = "2025-08-08T23:57:47.438Z" },
+ { url = "https://files.pythonhosted.org/packages/d5/2d/4d77f6feb9292bfdd23d5813e442b3bba883f42d0ac78ef5fdc56873f756/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f929255c705efec7a28bf515e29dc74220b2f07544a8c81b8d69e8efc4578bd", size = 1183308, upload-time = "2025-08-08T23:57:48.566Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/65/7ff0a65d3bb0fc5a1fb6cc71b03e0f6e71a68c5eea230d1ff1ba3fd6df49/tiktoken-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f1d15822e4404953d499fd1dcc62817a12ae9fb1e4898033ec8fe3915fdf8e", size = 1244301, upload-time = "2025-08-08T23:57:49.642Z" },
+ { url = "https://files.pythonhosted.org/packages/f5/6e/5b71578799b72e5bdcef206a214c3ce860d999d579a3b56e74a6c8989ee2/tiktoken-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:45927a71ab6643dfd3ef57d515a5db3d199137adf551f66453be098502838b0f", size = 884282, upload-time = "2025-08-08T23:57:50.759Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/cd/a9034bcee638716d9310443818d73c6387a6a96db93cbcb0819b77f5b206/tiktoken-0.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a5f3f25ffb152ee7fec78e90a5e5ea5b03b4ea240beed03305615847f7a6ace2", size = 1055339, upload-time = "2025-08-08T23:57:51.802Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/91/9922b345f611b4e92581f234e64e9661e1c524875c8eadd513c4b2088472/tiktoken-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7dc6e9ad16a2a75b4c4be7208055a1f707c9510541d94d9cc31f7fbdc8db41d8", size = 997080, upload-time = "2025-08-08T23:57:53.442Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/9d/49cd047c71336bc4b4af460ac213ec1c457da67712bde59b892e84f1859f/tiktoken-0.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a0517634d67a8a48fd4a4ad73930c3022629a85a217d256a6e9b8b47439d1e4", size = 1128501, upload-time = "2025-08-08T23:57:54.808Z" },
+ { url = "https://files.pythonhosted.org/packages/52/d5/a0dcdb40dd2ea357e83cb36258967f0ae96f5dd40c722d6e382ceee6bba9/tiktoken-0.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fb4effe60574675118b73c6fbfd3b5868e5d7a1f570d6cc0d18724b09ecf318", size = 1182743, upload-time = "2025-08-08T23:57:56.307Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/17/a0fc51aefb66b7b5261ca1314afa83df0106b033f783f9a7bcbe8e741494/tiktoken-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:94f984c9831fd32688aef4348803b0905d4ae9c432303087bae370dc1381a2b8", size = 1244057, upload-time = "2025-08-08T23:57:57.628Z" },
+ { url = "https://files.pythonhosted.org/packages/50/79/bcf350609f3a10f09fe4fc207f132085e497fdd3612f3925ab24d86a0ca0/tiktoken-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:2177ffda31dec4023356a441793fed82f7af5291120751dee4d696414f54db0c", size = 883901, upload-time = "2025-08-08T23:57:59.359Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/b6/81c5799ab77a9580c6d840cf77d4717e929193a42190fd623a080c647aa6/tiktoken-0.11.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:13220f12c9e82e399377e768640ddfe28bea962739cc3a869cad98f42c419a89", size = 1061648, upload-time = "2025-08-08T23:58:00.753Z" },
+ { url = "https://files.pythonhosted.org/packages/50/89/faa668066b2a4640534ef5797c09ecd0a48b43367502129b217339dfaa97/tiktoken-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f2db627f5c74477c0404b4089fd8a28ae22fa982a6f7d9c7d4c305c375218f3", size = 1000950, upload-time = "2025-08-08T23:58:01.855Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/7f/5f950528b54cb3025af4bc3522c23dbfb691afe8ffb292aa1e8dc2e6bddf/tiktoken-0.11.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2302772f035dceb2bcf8e55a735e4604a0b51a6dd50f38218ff664d46ec43807", size = 1130777, upload-time = "2025-08-08T23:58:03.256Z" },
+ { url = "https://files.pythonhosted.org/packages/27/a4/e82ddf0773835ba24536ac8c0dce561e697698ec020a93212a1e041d39b4/tiktoken-0.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20b977989afe44c94bcc50db1f76971bb26dca44218bd203ba95925ef56f8e7a", size = 1185692, upload-time = "2025-08-08T23:58:04.476Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/c2/06361e41d176e62797ae65fa678111cdd30553321cf4d83e7b84107ea95f/tiktoken-0.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:669a1aa1ad6ebf1b3c26b45deb346f345da7680f845b5ea700bba45c20dea24c", size = 1246518, upload-time = "2025-08-08T23:58:06.126Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/ad/ca37e15c46741ebb3904d562d03194e845539a08f7751a6df0f391757312/tiktoken-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:e363f33c720a055586f730c00e330df4c7ea0024bf1c83a8a9a9dbc054c4f304", size = 884702, upload-time = "2025-08-08T23:58:07.534Z" },
]
[[package]]