The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
chore(text-splitters): select ALL rules with exclusions (#32325)
Christophe Bornet0c3e8ccd0eba
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.pyindex 0ab553cefe..bce83f14fa 100644--- a/libs/text-splitters/langchain_text_splitters/base.py+++ b/libs/text-splitters/langchain_text_splitters/base.py@@ -1,3 +1,5 @@+"""Text splitter base interface."""+from __future__ import annotationsimport copy@@ -16,12 +18,27 @@ from typing import ()from langchain_core.documents import BaseDocumentTransformer, Document-from typing_extensions import Self+from typing_extensions import Self, overrideif TYPE_CHECKING:from collections.abc import Collection, Iterable, Sequencefrom collections.abc import Set as AbstractSet++try:+ import tiktoken++ _HAS_TIKTOKEN = True+except ImportError:+ _HAS_TIKTOKEN = False++try:+ from transformers.tokenization_utils_base import PreTrainedTokenizerBase++ _HAS_TRANSFORMERS = True+except ImportError:+ _HAS_TRANSFORMERS = False+logger = logging.getLogger(__name__)TS = TypeVar("TS", bound="TextSplitter")@@ -78,13 +95,13 @@ class TextSplitter(BaseDocumentTransformer, ABC):self, texts: list[str], metadatas: Optional[list[dict[Any, Any]]] = None) -> list[Document]:"""Create documents from a list of texts."""- _metadatas = metadatas or [{}] * len(texts)+ metadatas_ = metadatas or [{}] * len(texts)documents = []for i, text in enumerate(texts):index = 0previous_chunk_len = 0for chunk in self.split_text(text):- metadata = copy.deepcopy(_metadatas[i])+ metadata = copy.deepcopy(metadatas_[i])if self._add_start_index:offset = index + previous_chunk_len - self._chunk_overlapindex = text.find(chunk, max(0, offset))@@ -106,9 +123,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):text = separator.join(docs)if self._strip_whitespace:text = text.strip()- if text == "":- return None- return text+ return text or Nonedef _merge_splits(self, splits: Iterable[str], separator: str) -> list[str]:# We now want to combine these smaller pieces into medium size@@ -119,15 +134,17 @@ class TextSplitter(BaseDocumentTransformer, ABC):current_doc: list[str] = []total = 0for d in splits:- _len = self._length_function(d)+ len_ = self._length_function(d)if (- total + _len + (separator_len if len(current_doc) > 0 else 0)+ total + len_ + (separator_len if len(current_doc) > 0 else 0)> self._chunk_size):if total > self._chunk_size:logger.warning(- f"Created a chunk of size {total}, "- f"which is longer than the specified {self._chunk_size}"+ "Created a chunk of size %s, "+ "which is longer than the specified %s",+ total,+ self._chunk_size,)if len(current_doc) > 0:doc = self._join_docs(current_doc, separator)@@ -137,7 +154,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):# - we have a larger chunk than in the chunk overlap# - or if we still have any chunks and the length is longwhile total > self._chunk_overlap or (- total + _len + (separator_len if len(current_doc) > 0 else 0)+ total + len_ + (separator_len if len(current_doc) > 0 else 0)> self._chunk_sizeand total > 0):@@ -146,33 +163,31 @@ class TextSplitter(BaseDocumentTransformer, ABC):)current_doc = current_doc[1:]current_doc.append(d)- total += _len + (separator_len if len(current_doc) > 1 else 0)+ total += len_ + (separator_len if len(current_doc) > 1 else 0)doc = self._join_docs(current_doc, separator)if doc is not None:docs.append(doc)return docs@classmethod- def from_huggingface_tokenizer(cls, tokenizer: Any, **kwargs: Any) -> TextSplitter:+ def from_huggingface_tokenizer(+ cls, tokenizer: PreTrainedTokenizerBase, **kwargs: Any+ ) -> TextSplitter:"""Text splitter that uses HuggingFace tokenizer to count length."""- try:- from transformers.tokenization_utils_base import PreTrainedTokenizerBase-- if not isinstance(tokenizer, PreTrainedTokenizerBase):- msg = (- "Tokenizer received was not an instance of PreTrainedTokenizerBase"- )- raise ValueError(msg)-- def _huggingface_tokenizer_length(text: str) -> int:- return len(tokenizer.tokenize(text))-- except ImportError as err:+ if not _HAS_TRANSFORMERS:msg = ("Could not import transformers python package. ""Please install it with `pip install transformers`.")- raise ValueError(msg) from err+ raise ValueError(msg)++ if not isinstance(tokenizer, PreTrainedTokenizerBase):+ msg = "Tokenizer received was not an instance of PreTrainedTokenizerBase" # type: ignore[unreachable]+ raise ValueError(msg) # noqa: TRY004++ def _huggingface_tokenizer_length(text: str) -> int:+ return len(tokenizer.tokenize(text))+return cls(length_function=_huggingface_tokenizer_length, **kwargs)@classmethod@@ -185,15 +200,13 @@ class TextSplitter(BaseDocumentTransformer, ABC):**kwargs: Any,) -> Self:"""Text splitter that uses tiktoken encoder to count length."""- try:- import tiktoken- except ImportError as err:+ if not _HAS_TIKTOKEN:msg = ("Could not import tiktoken python package. ""This is needed in order to calculate max_tokens_for_prompt. ""Please install it with `pip install tiktoken`.")- raise ImportError(msg) from err+ raise ImportError(msg)if model_name is not None:enc = tiktoken.encoding_for_model(model_name)@@ -220,6 +233,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):return cls(length_function=_tiktoken_encoder, **kwargs)+ @overridedef transform_documents(self, documents: Sequence[Document], **kwargs: Any) -> Sequence[Document]:@@ -240,15 +254,13 @@ class TokenTextSplitter(TextSplitter):) -> None:"""Create a new TextSplitter."""super().__init__(**kwargs)- try:- import tiktoken- except ImportError as err:+ if not _HAS_TIKTOKEN:msg = ("Could not import tiktoken python package. ""This is needed in order to for TokenTextSplitter. ""Please install it with `pip install tiktoken`.")- raise ImportError(msg) from err+ raise ImportError(msg)if model_name is not None:enc = tiktoken.encoding_for_model(model_name)diff --git a/libs/text-splitters/langchain_text_splitters/character.py b/libs/text-splitters/langchain_text_splitters/character.pyindex 90366c5b51..3a6d77d42d 100644--- a/libs/text-splitters/langchain_text_splitters/character.py+++ b/libs/text-splitters/langchain_text_splitters/character.py@@ -1,3 +1,5 @@+"""Character text splitters."""+from __future__ import annotationsimport re@@ -56,24 +58,24 @@ def _split_text_with_regex(if separator:if keep_separator:# The parentheses in the pattern keep the delimiters in the result.- _splits = re.split(f"({separator})", text)+ splits_ = re.split(f"({separator})", text)splits = (- ([_splits[i] + _splits[i + 1] for i in range(0, len(_splits) - 1, 2)])+ ([splits_[i] + splits_[i + 1] for i in range(0, len(splits_) - 1, 2)])if keep_separator == "end"- else ([_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)])+ else ([splits_[i] + splits_[i + 1] for i in range(1, len(splits_), 2)]))- if len(_splits) % 2 == 0:- splits += _splits[-1:]+ if len(splits_) % 2 == 0:+ splits += splits_[-1:]splits = (- ([*splits, _splits[-1]])+ ([*splits, splits_[-1]])if keep_separator == "end"- else ([_splits[0], *splits])+ else ([splits_[0], *splits]))else:splits = re.split(separator, text)else:splits = list(text)- return [s for s in splits if s != ""]+ return [s for s in splits if s]class RecursiveCharacterTextSplitter(TextSplitter):@@ -102,38 +104,38 @@ class RecursiveCharacterTextSplitter(TextSplitter):separator = separators[-1]new_separators = []for i, _s in enumerate(separators):- _separator = _s if self._is_separator_regex else re.escape(_s)- if _s == "":+ separator_ = _s if self._is_separator_regex else re.escape(_s)+ if not _s:separator = _sbreak- if re.search(_separator, text):+ if re.search(separator_, text):separator = _snew_separators = separators[i + 1 :]break- _separator = separator if self._is_separator_regex else re.escape(separator)+ separator_ = separator if self._is_separator_regex else re.escape(separator)splits = _split_text_with_regex(- text, _separator, keep_separator=self._keep_separator+ text, separator_, keep_separator=self._keep_separator)# Now go merging things, recursively splitting longer texts.- _good_splits = []- _separator = "" if self._keep_separator else separator+ good_splits = []+ separator_ = "" if self._keep_separator else separatorfor s in splits:if self._length_function(s) < self._chunk_size:- _good_splits.append(s)+ good_splits.append(s)else:- if _good_splits:- merged_text = self._merge_splits(_good_splits, _separator)+ if good_splits:+ merged_text = self._merge_splits(good_splits, separator_)final_chunks.extend(merged_text)- _good_splits = []+ good_splits = []if not new_separators:final_chunks.append(s)else:other_info = self._split_text(s, new_separators)final_chunks.extend(other_info)- if _good_splits:- merged_text = self._merge_splits(_good_splits, _separator)+ if good_splits:+ merged_text = self._merge_splits(good_splits, separator_)final_chunks.extend(merged_text)return final_chunks@@ -177,7 +179,7 @@ class RecursiveCharacterTextSplitter(TextSplitter):Returns:List[str]: A list of separators appropriate for the specified language."""- if language in (Language.C, Language.CPP):+ if language in {Language.C, Language.CPP}:return [# Split along class definitions"\nclass ",diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.pyindex 4d78fbcc35..35471fb2ff 100644--- a/libs/text-splitters/langchain_text_splitters/html.py+++ b/libs/text-splitters/langchain_text_splitters/html.py@@ -1,3 +1,5 @@+"""HTML text splitters."""+from __future__ import annotationsimport copy@@ -5,6 +7,7 @@ import pathlibimport refrom io import StringIOfrom typing import (+ IO,TYPE_CHECKING,Any,Callable,@@ -18,13 +21,36 @@ from typing import (import requestsfrom langchain_core._api import betafrom langchain_core.documents import BaseDocumentTransformer, Document+from typing_extensions import overridefrom langchain_text_splitters.character import RecursiveCharacterTextSplitterif TYPE_CHECKING:- from collections.abc import Iterable, Sequence+ from collections.abc import Iterable, Iterator, Sequence++ from bs4.element import ResultSet++try:+ import nltk++ _HAS_NLTK = True+except ImportError:+ _HAS_NLTK = False++try:+ from bs4 import BeautifulSoup, Tag+ from bs4.element import NavigableString, PageElement- from bs4.element import PageElement+ _HAS_BS4 = True+except ImportError:+ _HAS_BS4 = False++try:+ from lxml import etree++ _HAS_LXML = True+except ImportError:+ _HAS_LXML = Falseclass ElementType(TypedDict):@@ -36,6 +62,29 @@ class ElementType(TypedDict):metadata: dict[str, str]+# Unfortunately, BeautifulSoup doesn't define overloads for Tag.find_all.+# So doing the type resolution ourselves.+++def _find_all_strings(+ tag: Tag,+ *,+ recursive: bool = True,+) -> ResultSet[NavigableString]:+ return cast(+ "ResultSet[NavigableString]", tag.find_all(string=True, recursive=recursive)+ )+++def _find_all_tags(+ tag: Tag,+ *,+ name: Union[bool, str, list[str], None] = None,+ recursive: bool = True,+) -> ResultSet[Tag]:+ return cast("ResultSet[Tag]", tag.find_all(name, recursive=recursive))++class HTMLHeaderTextSplitter:"""Split HTML content into structured Documents based on specified headers.@@ -157,16 +206,11 @@ class HTMLHeaderTextSplitter:Raises:requests.RequestException: If the HTTP request fails."""- kwargs.setdefault("timeout", timeout)- response = requests.get(- url,- timeout=kwargs.get("timeout", timeout),- **{k: v for k, v in kwargs.items() if k != "timeout"},- )+ response = requests.get(url, timeout=timeout, **kwargs)response.raise_for_status()return self.split_text(response.text)- def split_text_from_file(self, file: Any) -> list[Document]:+ def split_text_from_file(self, file: Union[str, IO[str]]) -> list[Document]:"""Split HTML content from a file into a list of Document objects.Args:@@ -178,13 +222,12 @@ class HTMLHeaderTextSplitter:the header hierarchy to their corresponding titles."""if isinstance(file, str):- with open(file, encoding="utf-8") as f:- html_content = f.read()+ html_content = pathlib.Path(file).read_text(encoding="utf-8")else:html_content = file.read()return list(self._generate_documents(html_content))- def _generate_documents(self, html_content: str) -> Any:+ def _generate_documents(self, html_content: str) -> Iterator[Document]:"""Private method that performs a DFS traversal over the DOM and yields.Document objects on-the-fly. This approach maintains the same splitting@@ -197,20 +240,18 @@ class HTMLHeaderTextSplitter:Yields:Document objects as they are created."""- try:- from bs4 import BeautifulSoup- except ImportError as e:+ if not _HAS_BS4:msg = ("Unable to import BeautifulSoup. Please install via `pip install bs4`.")- raise ImportError(msg) from e+ raise ImportError(msg)soup = BeautifulSoup(html_content, "html.parser")- body = soup.body if soup.body else soup+ body = soup.body or soup# Dictionary of active headers:# key = user-defined header name (e.g. "Header 1")- # value = (header_text, level, dom_depth)+ # value = tuple of header_text, level, dom_depthactive_headers: dict[str, tuple[str, int, int]] = {}current_chunk: list[str] = []@@ -232,7 +273,6 @@ class HTMLHeaderTextSplitter:while stack:node = stack.pop()children = list(node.children)- from bs4.element import Tagstack.extend(child for child in reversed(children) if isinstance(child, Tag)@@ -243,8 +283,7 @@ class HTMLHeaderTextSplitter:continuetext_elements = [- str(child).strip()- for child in node.find_all(string=True, recursive=False)+ str(child).strip() for child in _find_all_strings(node, recursive=False)]node_text = " ".join(elem for elem in text_elements if elem)if not node_text:@@ -353,11 +392,11 @@ class HTMLSectionSplitter:self, texts: list[str], metadatas: Optional[list[dict[Any, Any]]] = None) -> list[Document]:"""Create documents from a list of texts."""- _metadatas = metadatas or [{}] * len(texts)+ metadatas_ = metadatas or [{}] * len(texts)documents = []for i, text in enumerate(texts):for chunk in self.split_text(text):- metadata = copy.deepcopy(_metadatas[i])+ metadata = copy.deepcopy(metadatas_[i])for key in chunk.metadata:if chunk.metadata[key] == "#TITLE#":@@ -386,38 +425,35 @@ class HTMLSectionSplitter:* 'content': The content under the header.* 'tag_name': The name of the header tag (e.g., "h1", "h2")."""- try:- from bs4 import BeautifulSoup- except ImportError as e:+ if not _HAS_BS4:msg = "Unable to import BeautifulSoup/PageElement, \please install with `pip install \bs4`."- raise ImportError(msg) from e+ raise ImportError(msg)soup = BeautifulSoup(html_doc, "html.parser")- headers = list(self.headers_to_split_on.keys())+ header_names = list(self.headers_to_split_on.keys())sections: list[dict[str, str | None]] = []- headers = soup.find_all(["body", *headers]) # type: ignore[assignment]+ headers = _find_all_tags(soup, name=["body", *header_names])for i, header in enumerate(headers):- header_element = cast("PageElement", header)if i == 0:current_header = "#TITLE#"current_header_tag = "h1"section_content: list[str] = []else:- current_header = header_element.text.strip()- current_header_tag = header_element.name # type: ignore[attr-defined]+ current_header = header.text.strip()+ current_header_tag = header.namesection_content = []- for element in header_element.next_elements:- if i + 1 < len(headers) and element == headers[i + 1]: # type: ignore[comparison-overlap]+ for element in header.next_elements:+ if i + 1 < len(headers) and element == headers[i + 1]:breakif isinstance(element, str):section_content.append(element)content = " ".join(section_content).strip()- if content != "":+ if content:sections.append({"header": current_header,@@ -441,11 +477,9 @@ class HTMLSectionSplitter:Returns:str: The transformed HTML content as a string."""- try:- from lxml import etree- except ImportError as e:+ if not _HAS_LXML:msg = "Unable to import lxml, please install with `pip install lxml`."- raise ImportError(msg) from e+ raise ImportError(msg)# use lxml library to parse html document and return xml ElementTree# Create secure parsers to prevent XXE attackshtml_parser = etree.HTMLParser(no_network=True)@@ -464,7 +498,7 @@ class HTMLSectionSplitter:result = transform(tree)return str(result)- def split_text_from_file(self, file: Any) -> list[Document]:+ def split_text_from_file(self, file: StringIO) -> list[Document]:"""Split HTML content from a file into a list of Document objects.Args:@@ -546,7 +580,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):preserve_images: bool = False,preserve_videos: bool = False,preserve_audio: bool = False,- custom_handlers: Optional[dict[str, Callable[[Any], str]]] = None,+ custom_handlers: Optional[dict[str, Callable[[Tag], str]]] = None,stopword_removal: bool = False,stopword_lang: str = "english",normalize_text: bool = False,@@ -592,17 +626,12 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):keep_separator: Whether separatorsshould be at the beginning of a chunk, at the end, or not at all."""- try:- from bs4 import BeautifulSoup, Tag-- self._BeautifulSoup = BeautifulSoup- self._Tag = Tag- except ImportError as err:+ if not _HAS_BS4:msg = ("Could not import BeautifulSoup. ""Please install it with 'pip install bs4'.")- raise ImportError(msg) from err+ raise ImportError(msg)self._headers_to_split_on = sorted(headers_to_split_on)self._max_chunk_size = max_chunk_size@@ -645,16 +674,13 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):)if self._stopword_removal:- try:- import nltk-- nltk.download("stopwords")- self._stopwords = set(nltk.corpus.stopwords.words(self._stopword_lang))- except ImportError as err:+ if not _HAS_NLTK:msg = ("Could not import nltk. Please install it with 'pip install nltk'.")- raise ImportError(msg) from err+ raise ImportError(msg)+ nltk.download("stopwords")+ self._stopwords = set(nltk.corpus.stopwords.words(self._stopword_lang))def split_text(self, text: str) -> list[Document]:"""Splits the provided HTML text into smaller chunks based on the configuration.@@ -665,7 +691,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):Returns:List[Document]: A list of Document objects containing the split content."""- soup = self._BeautifulSoup(text, "html.parser")+ soup = BeautifulSoup(text, "html.parser")self._process_media(soup)@@ -677,6 +703,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):return self._process_html(soup)+ @overridedef transform_documents(self, documents: Sequence[Document], **kwargs: Any) -> list[Document]:@@ -695,17 +722,17 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):transformed.extend(splits)return transformed- def _process_media(self, soup: Any) -> None:+ def _process_media(self, soup: BeautifulSoup) -> None:"""Processes the media elements.Process elements in the HTML content by wrapping them in a <media-wrapper> tagand converting them to Markdown format.Args:- soup (Any): Parsed HTML content using BeautifulSoup.+ soup: Parsed HTML content using BeautifulSoup."""if self._preserve_images:- for img_tag in soup.find_all("img"):+ for img_tag in _find_all_tags(soup, name="img"):img_src = img_tag.get("src", "")markdown_img = f""wrapper = soup.new_tag("media-wrapper")@@ -713,7 +740,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):img_tag.replace_with(wrapper)if self._preserve_videos:- for video_tag in soup.find_all("video"):+ for video_tag in _find_all_tags(soup, name="video"):video_src = video_tag.get("src", "")markdown_video = f""wrapper = soup.new_tag("media-wrapper")@@ -721,40 +748,40 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):video_tag.replace_with(wrapper)if self._preserve_audio:- for audio_tag in soup.find_all("audio"):+ for audio_tag in _find_all_tags(soup, name="audio"):audio_src = audio_tag.get("src", "")markdown_audio = f""wrapper = soup.new_tag("media-wrapper")wrapper.string = markdown_audioaudio_tag.replace_with(wrapper)- def _process_links(self, soup: Any) -> None:+ def _process_links(self, soup: BeautifulSoup) -> None:"""Processes the links in the HTML content.Args:- soup (Any): Parsed HTML content using BeautifulSoup.+ soup: Parsed HTML content using BeautifulSoup."""- for a_tag in soup.find_all("a"):+ for a_tag in _find_all_tags(soup, name="a"):a_href = a_tag.get("href", "")a_text = a_tag.get_text(strip=True)markdown_link = f"[{a_text}]({a_href})"wrapper = soup.new_tag("link-wrapper")wrapper.string = markdown_link- a_tag.replace_with(markdown_link)+ a_tag.replace_with(NavigableString(markdown_link))- def _filter_tags(self, soup: Any) -> None:+ def _filter_tags(self, soup: BeautifulSoup) -> None:"""Filters the HTML content based on the allowlist and denylist tags.Args:- soup (Any): Parsed HTML content using BeautifulSoup.+ soup: Parsed HTML content using BeautifulSoup."""if self._allowlist_tags:- for tag in soup.find_all(name=True):+ for tag in _find_all_tags(soup, name=True):if tag.name not in self._allowlist_tags:tag.decompose()if self._denylist_tags:- for tag in soup.find_all(self._denylist_tags):+ for tag in _find_all_tags(soup, name=self._denylist_tags):tag.decompose()def _normalize_and_clean_text(self, text: str) -> str:@@ -778,14 +805,14 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):return text- def _process_html(self, soup: Any) -> list[Document]:+ def _process_html(self, soup: BeautifulSoup) -> list[Document]:"""Processes the HTML content using BeautifulSoup and splits it using headers.Args:- soup (Any): Parsed HTML content using BeautifulSoup.+ soup: Parsed HTML content using BeautifulSoup.Returns:- List[Document]: A list of Document objects containing the split content.+ A list of Document objects containing the split content."""documents: list[Document] = []current_headers: dict[str, str] = {}@@ -793,17 +820,18 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):preserved_elements: dict[str, str] = {}placeholder_count: int = 0- def _get_element_text(element: Any) -> str:+ def _get_element_text(element: PageElement) -> str:"""Recursively extracts and processes the text of an element.Applies custom handlers where applicable, and ensures correct spacing.Args:- element (Any): The HTML element to process.+ element: The HTML element to process.Returns:- str: The processed text of the element.+ The processed text of the element."""+ element = cast("Union[Tag, NavigableString]", element)if element.name in self._custom_handlers:return self._custom_handlers[element.name](element)@@ -820,10 +848,10 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):return self._normalize_and_clean_text(text)- elements = soup.find_all(recursive=False)+ elements = _find_all_tags(soup, recursive=False)def _process_element(- element: list[Any],+ element: ResultSet[Tag],documents: list[Document],current_headers: dict[str, str],current_content: list[str],@@ -831,8 +859,8 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):placeholder_count: int,) -> tuple[list[Document], dict[str, str], list[str], dict[str, str], int]:for elem in element:- if elem.name.lower() in ["html", "body", "div", "main"]:- children = elem.find_all(recursive=False)+ if elem.name.lower() in {"html", "body", "div", "main"}:+ children = _find_all_tags(elem, recursive=False)(documents,current_headers,@@ -847,7 +875,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):preserved_elements,placeholder_count,)- content = " ".join(elem.find_all(string=True, recursive=False))+ content = " ".join(_find_all_strings(elem, recursive=False))if content:content = self._normalize_and_clean_text(content)current_content.append(content)diff --git a/libs/text-splitters/langchain_text_splitters/json.py b/libs/text-splitters/langchain_text_splitters/json.pyindex c2fa8936d2..6e61bc33f9 100644--- a/libs/text-splitters/langchain_text_splitters/json.py+++ b/libs/text-splitters/langchain_text_splitters/json.py@@ -1,3 +1,5 @@+"""JSON text splitter."""+from __future__ import annotationsimport copy@@ -50,13 +52,17 @@ class RecursiveJsonSplitter:return len(json.dumps(data))@staticmethod- def _set_nested_dict(d: dict[str, Any], path: list[str], value: Any) -> None:+ def _set_nested_dict(+ d: dict[str, Any],+ path: list[str],+ value: Any, # noqa: ANN401+ ) -> None:"""Set a value in a nested dictionary based on the given path."""for key in path[:-1]:d = d.setdefault(key, {})d[path[-1]] = value- def _list_to_dict_preprocessing(self, data: Any) -> Any:+ def _list_to_dict_preprocessing(self, data: Any) -> Any: # noqa: ANN401if isinstance(data, dict):# Process each key-value pair in the dictionaryreturn {k: self._list_to_dict_preprocessing(v) for k, v in data.items()}@@ -71,7 +77,7 @@ class RecursiveJsonSplitter:def _json_split(self,- data: Any,+ data: Any, # noqa: ANN401current_path: Optional[list[str]] = None,chunks: Optional[list[dict[str, Any]]] = None,) -> list[dict[str, Any]]:@@ -136,13 +142,13 @@ class RecursiveJsonSplitter:metadatas: Optional[list[dict[Any, Any]]] = None,) -> list[Document]:"""Create documents from a list of json objects (Dict)."""- _metadatas = metadatas or [{}] * len(texts)+ metadatas_ = metadatas or [{}] * len(texts)documents = []for i, text in enumerate(texts):for chunk in self.split_text(json_data=text, convert_lists=convert_lists, ensure_ascii=ensure_ascii):- metadata = copy.deepcopy(_metadatas[i])+ metadata = copy.deepcopy(metadatas_[i])new_doc = Document(page_content=chunk, metadata=metadata)documents.append(new_doc)return documentsdiff --git a/libs/text-splitters/langchain_text_splitters/jsx.py b/libs/text-splitters/langchain_text_splitters/jsx.pyindex 4621670112..1333065a76 100644--- a/libs/text-splitters/langchain_text_splitters/jsx.py+++ b/libs/text-splitters/langchain_text_splitters/jsx.py@@ -1,3 +1,5 @@+"""JavaScript framework text splitter."""+import refrom typing import Any, Optionaldiff --git a/libs/text-splitters/langchain_text_splitters/konlpy.py b/libs/text-splitters/langchain_text_splitters/konlpy.pyindex fc1763790f..863447e883 100644--- a/libs/text-splitters/langchain_text_splitters/konlpy.py+++ b/libs/text-splitters/langchain_text_splitters/konlpy.py@@ -1,9 +1,18 @@+"""Konlpy text splitter."""+from __future__ import annotationsfrom typing import Anyfrom langchain_text_splitters.base import TextSplitter+try:+ import konlpy++ _HAS_KONLPY = True+except ImportError:+ _HAS_KONLPY = False+class KonlpyTextSplitter(TextSplitter):"""Splitting text using Konlpy package.@@ -19,14 +28,12 @@ class KonlpyTextSplitter(TextSplitter):"""Initialize the Konlpy text splitter."""super().__init__(**kwargs)self._separator = separator- try:- import konlpy- except ImportError as err:+ if not _HAS_KONLPY:msg = """Konlpy is not installed, please install it with`pip install konlpy`"""- raise ImportError(msg) from err+ raise ImportError(msg)self.kkma = konlpy.tag.Kkma()def split_text(self, text: str) -> list[str]:diff --git a/libs/text-splitters/langchain_text_splitters/latex.py b/libs/text-splitters/langchain_text_splitters/latex.pyindex 55d47f7851..fc9c2ff498 100644--- a/libs/text-splitters/langchain_text_splitters/latex.py+++ b/libs/text-splitters/langchain_text_splitters/latex.py@@ -1,3 +1,5 @@+"""Latex text splitter."""+from __future__ import annotationsfrom typing import Anydiff --git a/libs/text-splitters/langchain_text_splitters/markdown.py b/libs/text-splitters/langchain_text_splitters/markdown.pyindex b5fb63c5d7..a1d40a4322 100644--- a/libs/text-splitters/langchain_text_splitters/markdown.py+++ b/libs/text-splitters/langchain_text_splitters/markdown.py@@ -1,3 +1,5 @@+"""Markdown text splitters."""+from __future__ import annotationsimport re@@ -138,7 +140,6 @@ class MarkdownHeaderTextSplitter:current_content: list[str] = []current_metadata: dict[str, str] = {}# Keep track of the nested header structure- # header_stack: List[Dict[str, Union[int, str]]] = []header_stack: list[HeaderType] = []initial_metadata: dict[str, str] = {}diff --git a/libs/text-splitters/langchain_text_splitters/nltk.py b/libs/text-splitters/langchain_text_splitters/nltk.pyindex 4ac7b0dbf0..8041d47d49 100644--- a/libs/text-splitters/langchain_text_splitters/nltk.py+++ b/libs/text-splitters/langchain_text_splitters/nltk.py@@ -1,9 +1,18 @@+"""NLTK text splitter."""+from __future__ import annotationsfrom typing import Anyfrom langchain_text_splitters.base import TextSplitter+try:+ import nltk++ _HAS_NLTK = True+except ImportError:+ _HAS_NLTK = False+class NLTKTextSplitter(TextSplitter):"""Splitting text using NLTK package."""@@ -21,19 +30,16 @@ class NLTKTextSplitter(TextSplitter):self._separator = separatorself._language = languageself._use_span_tokenize = use_span_tokenize- if self._use_span_tokenize and self._separator != "":+ if self._use_span_tokenize and self._separator:msg = "When use_span_tokenize is True, separator should be ''"raise ValueError(msg)- try:- import nltk-- if self._use_span_tokenize:- self._tokenizer = nltk.tokenize._get_punkt_tokenizer(self._language)- else:- self._tokenizer = nltk.tokenize.sent_tokenize- except ImportError as err:+ if not _HAS_NLTK:msg = "NLTK is not installed, please install it with `pip install nltk`."- raise ImportError(msg) from err+ raise ImportError(msg)+ if self._use_span_tokenize:+ self._tokenizer = nltk.tokenize._get_punkt_tokenizer(self._language) # noqa: SLF001+ else:+ self._tokenizer = nltk.tokenize.sent_tokenizedef split_text(self, text: str) -> list[str]:"""Split incoming text and return chunks."""diff --git a/libs/text-splitters/langchain_text_splitters/python.py b/libs/text-splitters/langchain_text_splitters/python.pyindex ffd4a10f62..ccab44bcf6 100644--- a/libs/text-splitters/langchain_text_splitters/python.py+++ b/libs/text-splitters/langchain_text_splitters/python.py@@ -1,3 +1,5 @@+"""Python code text splitter."""+from __future__ import annotationsfrom typing import Anydiff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.pyindex 8cbe227495..de3cd27970 100644--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py@@ -1,9 +1,18 @@+"""Sentence transformers text splitter."""+from __future__ import annotationsfrom typing import Any, Optional, castfrom langchain_text_splitters.base import TextSplitter, Tokenizer, split_text_on_tokens+try:+ from sentence_transformers import SentenceTransformer++ _HAS_SENTENCE_TRANSFORMERS = True+except ImportError:+ _HAS_SENTENCE_TRANSFORMERS = False+class SentenceTransformersTokenTextSplitter(TextSplitter):"""Splitting text to tokens using sentence model tokenizer."""@@ -18,15 +27,13 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):"""Create a new TextSplitter."""super().__init__(**kwargs, chunk_overlap=chunk_overlap)- try:- from sentence_transformers import SentenceTransformer- except ImportError as err:+ if not _HAS_SENTENCE_TRANSFORMERS:msg = ("Could not import sentence_transformers python package. "- "This is needed in order to for SentenceTransformersTokenTextSplitter. "+ "This is needed in order to use SentenceTransformersTokenTextSplitter. ""Please install it with `pip install sentence-transformers`.")- raise ImportError(msg) from err+ raise ImportError(msg)self.model_name = model_nameself._model = SentenceTransformer(self.model_name)diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.pyindex af7c8e3e25..cfc6fa6eec 100644--- a/libs/text-splitters/langchain_text_splitters/spacy.py+++ b/libs/text-splitters/langchain_text_splitters/spacy.py@@ -1,9 +1,20 @@+"""Spacy text splitter."""+from __future__ import annotationsfrom typing import Anyfrom langchain_text_splitters.base import TextSplitter+try:+ import spacy+ from spacy.lang.en import English+ from spacy.language import Language++ _HAS_SPACY = True+except ImportError:+ _HAS_SPACY = False+class SpacyTextSplitter(TextSplitter):"""Splitting text using Spacy package.@@ -42,16 +53,12 @@ class SpacyTextSplitter(TextSplitter):def _make_spacy_pipeline_for_splitting(pipeline: str, *, max_length: int = 1_000_000-) -> Any: # avoid importing spacy- try:- import spacy- except ImportError as err:+) -> Language:+ if not _HAS_SPACY:msg = "Spacy is not installed, please install it with `pip install spacy`."- raise ImportError(msg) from err+ raise ImportError(msg)if pipeline == "sentencizer":- from spacy.lang.en import English-- sentencizer: Any = English()+ sentencizer: Language = English()sentencizer.add_pipe("sentencizer")else:sentencizer = spacy.load(pipeline, exclude=["ner", "tagger"])diff --git a/libs/text-splitters/pyproject.toml b/libs/text-splitters/pyproject.tomlindex 9d4dd95f39..35ced364e4 100644--- a/libs/text-splitters/pyproject.toml+++ b/libs/text-splitters/pyproject.toml@@ -25,7 +25,8 @@ typing = ["mypy<1.18,>=1.17.1","lxml-stubs<1.0.0,>=0.5.1","types-requests<3.0.0.0,>=2.31.0.20240218",- "tiktoken<1.0.0,>=0.8.0",+ "tiktoken<1.0.0,>=0.11.0",+ "beautifulsoup4<5.0.0,>=4.13.5",]dev = ["jupyter<2.0.0,>=1.0.0", "langchain-core"]test = [@@ -66,71 +67,46 @@ ignore_missing_imports = truetarget-version = "py39"[tool.ruff.lint]-select = [- "A", # flake8-builtins- "B", # flake8-bugbear- "ASYNC", # flake8-async- "C4", # flake8-comprehensions- "COM", # flake8-commas- "D", # pydocstyle- "E", # pycodestyle error- "EM", # flake8-errmsg- "F", # pyflakes- "FA", # flake8-future-annotations- "FBT", # flake8-boolean-trap- "FLY", # flake8-flynt- "I", # isort- "ICN", # flake8-import-conventions- "INT", # flake8-gettext- "ISC", # isort-comprehensions- "PGH", # pygrep-hooks- "PIE", # flake8-pie- "PERF", # flake8-perf- "PYI", # flake8-pyi- "Q", # flake8-quotes- "RET", # flake8-return- "RSE", # flake8-rst-docstrings- "RUF", # ruff- "S", # flake8-bandit- "SLF", # flake8-self- "SLOT", # flake8-slots- "SIM", # flake8-simplify- "T10", # flake8-debugger- "T20", # flake8-print- "TID", # flake8-tidy-imports- "UP", # pyupgrade- "W", # pycodestyle warning- "YTT", # flake8-2020-]+select = [ "ALL",]ignore = [- "D100", # pydocstyle: Missing docstring in public module- "D101", # pydocstyle: Missing docstring in public class- "D102", # pydocstyle: Missing docstring in public method- "D103", # pydocstyle: Missing docstring in public function- "D104", # pydocstyle: Missing docstring in public package- "D105", # pydocstyle: Missing docstring in magic method- "D107", # pydocstyle: Missing docstring in __init__- "D203", # Messes with the formatter- "D213", # pydocstyle: Multi-line docstring summary should start at the second line (incompatible with D212)- "D407", # pydocstyle: Missing-dashed-underline-after-section+ "C90", # McCabe complexity"COM812", # Messes with the formatter+ "FIX002", # Line contains TODO"ISC001", # Messes with the formatter"PERF203", # Rarely useful- "S112", # Rarely useful+ "PLR09", # Too many something (arg, statements, etc)"RUF012", # Doesn't play well with Pydantic- "SLF001", # Private member access- "UP007", # pyupgrade: non-pep604-annotation-union- "UP045", # pyupgrade: non-pep604-annotation-optional+ "TC001", # Doesn't play well with Pydantic+ "TC002", # Doesn't play well with Pydantic+ "TC003", # Doesn't play well with Pydantic+ "TD002", # Missing author in TODO+ "TD003", # Missing issue link in TODO+]+unfixable = [+ "B028", # People should intentionally tune the stacklevel+ "PLW1510", # People should intentionally set the check argument]-unfixable = ["B028"] # People should intentionally tune the stacklevel++flake8-annotations.allow-star-arg-any = true+flake8-annotations.mypy-init-return = true+flake8-type-checking.runtime-evaluated-base-classes = ["pydantic.BaseModel","langchain_core.load.serializable.Serializable","langchain_core.runnables.base.RunnableSerializable"]+pep8-naming.classmethod-decorators = [ "classmethod", "langchain_core.utils.pydantic.pre_init", "pydantic.field_validator", "pydantic.v1.root_validator",]+pydocstyle.convention = "google"pyupgrade.keep-runtime-typing = true-[tool.ruff.lint.extend-per-file-ignores]-"tests/**/*.py" = [- "S101", # Tests need assertions- "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes+[tool.ruff.lint.per-file-ignores]+"scripts/**" = [+ "D1", # Docstrings not mandatory in scripts+ "INP001", # Not a package+ "S311" # Standard pseudo-random generators are not suitable for cryptographic purposes+]+"tests/**" = [+ "D1", # Docstrings not mandatory in tests+ "PLR2004", # Magic value comparisons+ "S101", # Tests need assertions+ "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes+ "SLF001" # Private member access in tests]-[tool.coverage.run]omit = ["tests/*"]@@ -142,9 +118,3 @@ markers = ["compile: mark placeholder test used to compile integration tests without running them",]asyncio_mode = "auto"--[tool.ruff.lint.pydocstyle]-convention = "google"--[tool.ruff.lint.per-file-ignores]-"tests/**" = ["D"]diff --git a/libs/text-splitters/scripts/check_imports.py b/libs/text-splitters/scripts/check_imports.pyindex 00069f9f95..6242951b62 100644--- a/libs/text-splitters/scripts/check_imports.py+++ b/libs/text-splitters/scripts/check_imports.py@@ -10,7 +10,7 @@ if __name__ == "__main__":try:module_name = f"test_module_{uuid.uuid4().hex[:20]}"SourceFileLoader(module_name, file).load_module()- except Exception:+ except Exception: # noqa: BLE001has_failure = Trueprint(file) # noqa: T201traceback.print_exc()diff --git a/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py b/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.pyindex b4d6499f98..82e55b0d9d 100644--- a/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py+++ b/libs/text-splitters/tests/integration_tests/test_nlp_text_splitters.py@@ -1,6 +1,6 @@"""Test text splitting functionality using NLTK and Spacy based sentence splitters."""-from typing import Any+import reimport nltkimport pytest@@ -15,11 +15,8 @@ def setup_module() -> None:@pytest.fixture-def spacy() -> Any:- try:- import spacy- except ImportError:- pytest.skip("Spacy not installed.")+def spacy() -> None:+ spacy = pytest.importorskip("spacy")# Check if en_core_web_sm model is availabletry:@@ -32,18 +29,27 @@ def spacy() -> Any:"en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl")- return spacy-def test_nltk_text_splitting_args() -> None:"""Test invalid arguments."""- with pytest.raises(ValueError):+ with pytest.raises(+ ValueError,+ match=re.escape(+ "Got a larger chunk overlap (4) than chunk size (2), should be smaller."+ ),+ ):NLTKTextSplitter(chunk_size=2, chunk_overlap=4)-def test_spacy_text_splitting_args(spacy: Any) -> None:[email protected]("spacy")+def test_spacy_text_splitting_args() -> None:"""Test invalid arguments."""- with pytest.raises(ValueError):+ with pytest.raises(+ ValueError,+ match=re.escape(+ "Got a larger chunk overlap (4) than chunk size (2), should be smaller."+ ),+ ):SpacyTextSplitter(chunk_size=2, chunk_overlap=4)@@ -57,8 +63,9 @@ def test_nltk_text_splitter() -> None:assert output == expected_output[email protected]("spacy")@pytest.mark.parametrize("pipeline", ["sentencizer", "en_core_web_sm"])-def test_spacy_text_splitter(pipeline: str, spacy: Any) -> None:+def test_spacy_text_splitter(pipeline: str) -> None:"""Test splitting by sentence using Spacy."""text = "This is sentence one. And this is sentence two."separator = "|||"@@ -68,8 +75,9 @@ def test_spacy_text_splitter(pipeline: str, spacy: Any) -> None:assert output == expected_output[email protected]("spacy")@pytest.mark.parametrize("pipeline", ["sentencizer", "en_core_web_sm"])-def test_spacy_text_splitter_strip_whitespace(pipeline: str, spacy: Any) -> None:+def test_spacy_text_splitter_strip_whitespace(pipeline: str) -> None:"""Test splitting by sentence using Spacy."""text = "This is sentence one. And this is sentence two."separator = "|||"@@ -83,7 +91,9 @@ def test_spacy_text_splitter_strip_whitespace(pipeline: str, spacy: Any) -> Nonedef test_nltk_text_splitter_args() -> None:"""Test invalid arguments for NLTKTextSplitter."""- with pytest.raises(ValueError):+ with pytest.raises(+ ValueError, match="When use_span_tokenize is True, separator should be ''"+ ):NLTKTextSplitter(chunk_size=80,chunk_overlap=0,diff --git a/libs/text-splitters/tests/integration_tests/test_text_splitter.py b/libs/text-splitters/tests/integration_tests/test_text_splitter.pyindex ad0549e242..9ee1122011 100644--- a/libs/text-splitters/tests/integration_tests/test_text_splitter.py+++ b/libs/text-splitters/tests/integration_tests/test_text_splitter.py@@ -1,8 +1,7 @@"""Test text splitters that require an integration."""-from typing import Any-import pytest+from transformers import GPT2TokenizerFastfrom langchain_text_splitters import (TokenTextSplitter,@@ -13,25 +12,17 @@ from langchain_text_splitters.sentence_transformers import ()-def sentence_transformers() -> Any:- try:- import sentence_transformers- except ImportError:- pytest.skip("SentenceTransformers not installed.")- return sentence_transformers--def test_huggingface_type_check() -> None:"""Test that type checks are done properly on input."""- with pytest.raises(ValueError):- CharacterTextSplitter.from_huggingface_tokenizer("foo")+ with pytest.raises(+ ValueError,+ match="Tokenizer received was not an instance of PreTrainedTokenizerBase",+ ):+ CharacterTextSplitter.from_huggingface_tokenizer("foo") # type: ignore[arg-type]def test_huggingface_tokenizer() -> None:"""Test text splitter that uses a HuggingFace tokenizer."""- from transformers import GPT2TokenizerFast-tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")text_splitter = CharacterTextSplitter.from_huggingface_tokenizer(tokenizer, separator=" ", chunk_size=1, chunk_overlap=0@@ -63,7 +54,8 @@ def test_token_text_splitter_from_tiktoken() -> None:assert expected_tokenizer == actual_tokenizer-def test_sentence_transformers_count_tokens(sentence_transformers: Any) -> None:[email protected]("sentence_transformers")+def test_sentence_transformers_count_tokens() -> None:splitter = SentenceTransformersTokenTextSplitter(model_name="sentence-transformers/paraphrase-albert-small-v2")@@ -78,7 +70,8 @@ def test_sentence_transformers_count_tokens(sentence_transformers: Any) -> None:assert expected_token_count == token_count-def test_sentence_transformers_split_text(sentence_transformers: Any) -> None:[email protected]("sentence_transformers")+def test_sentence_transformers_split_text() -> None:splitter = SentenceTransformersTokenTextSplitter(model_name="sentence-transformers/paraphrase-albert-small-v2")@@ -88,7 +81,8 @@ def test_sentence_transformers_split_text(sentence_transformers: Any) -> None:assert expected_text_chunks == text_chunks-def test_sentence_transformers_multiple_tokens(sentence_transformers: Any) -> None:[email protected]("sentence_transformers")+def test_sentence_transformers_multiple_tokens() -> None:splitter = SentenceTransformersTokenTextSplitter(chunk_overlap=0)text = "Lorem "diff --git a/libs/text-splitters/tests/unit_tests/conftest.py b/libs/text-splitters/tests/unit_tests/conftest.pyindex ad5ea6273c..d56ef0cedf 100644--- a/libs/text-splitters/tests/unit_tests/conftest.py+++ b/libs/text-splitters/tests/unit_tests/conftest.py@@ -4,10 +4,9 @@ from collections.abc import Sequencefrom importlib import utilimport pytest-from pytest import Config, Function, Parser-def pytest_addoption(parser: Parser) -> None:+def pytest_addoption(parser: pytest.Parser) -> None:"""Add custom command line options to pytest."""parser.addoption("--only-extended",@@ -21,7 +20,9 @@ def pytest_addoption(parser: Parser) -> None:)-def pytest_collection_modifyitems(config: Config, items: Sequence[Function]) -> None:+def pytest_collection_modifyitems(+ config: pytest.Config, items: Sequence[pytest.Function]+) -> None:"""Add implementations for handling custom markers.At the moment, this adds support for a custom `requires` marker.@@ -64,7 +65,7 @@ def pytest_collection_modifyitems(config: Config, items: Sequence[Function]) ->if pkg not in required_pkgs_info:try:installed = util.find_spec(pkg) is not None- except Exception:+ except (ImportError, ValueError):installed = Falserequired_pkgs_info[pkg] = installeddiff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.pyindex 5ecca3bdf7..c366af9170 100644--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py@@ -1,11 +1,14 @@"""Test text splitting functionality."""+from __future__ import annotations+import randomimport reimport stringfrom typing import Any, Callableimport pytest+from bs4 import Tagfrom langchain_core.documents import Documentfrom langchain_text_splitters import (@@ -103,7 +106,9 @@ def test_character_text_splitter_longer_words() -> None:def test_character_text_splitter_keep_separator_regex(*, separator: str, is_separator_regex: bool) -> None:- """Test splitting by characters while keeping the separator+ """Test CharacterTextSplitter keep separator regex.++ Test splitting by characters while keeping the separatorthat is a regex special character."""text = "foo.bar.baz.123"@@ -125,7 +130,9 @@ def test_character_text_splitter_keep_separator_regex(def test_character_text_splitter_keep_separator_regex_start(*, separator: str, is_separator_regex: bool) -> None:- """Test splitting by characters while keeping the separator+ """Test CharacterTextSplitter keep separator regex and put at start.++ Test splitting by characters while keeping the separatorthat is a regex special character and placing it at the start of each chunk."""text = "foo.bar.baz.123"@@ -147,7 +154,9 @@ def test_character_text_splitter_keep_separator_regex_start(def test_character_text_splitter_keep_separator_regex_end(*, separator: str, is_separator_regex: bool) -> None:- """Test splitting by characters while keeping the separator+ """Test CharacterTextSplitter keep separator regex and put at end.++ Test splitting by characters while keeping the separatorthat is a regex special character and placing it at the end of each chunk."""text = "foo.bar.baz.123"@@ -169,8 +178,11 @@ def test_character_text_splitter_keep_separator_regex_end(def test_character_text_splitter_discard_separator_regex(*, separator: str, is_separator_regex: bool) -> None:- """Test splitting by characters discarding the separator- that is a regex special character."""+ """Test CharacterTextSplitter discard separator regex.++ Test splitting by characters discarding the separator+ that is a regex special character.+ """text = "foo.bar.baz.123"splitter = CharacterTextSplitter(separator=separator,@@ -210,12 +222,17 @@ def test_recursive_character_text_splitter_keep_separators() -> None:def test_character_text_splitting_args() -> None:"""Test invalid arguments."""- with pytest.raises(ValueError):+ with pytest.raises(+ ValueError,+ match=re.escape(+ "Got a larger chunk overlap (4) than chunk size (2), should be smaller."+ ),+ ):CharacterTextSplitter(chunk_size=2, chunk_overlap=4)for invalid_size in (0, -1):- with pytest.raises(ValueError):+ with pytest.raises(ValueError, match="chunk_size must be > 0, got"):CharacterTextSplitter(chunk_size=invalid_size)- with pytest.raises(ValueError):+ with pytest.raises(ValueError, match="chunk_overlap must be >= 0, got -1"):CharacterTextSplitter(chunk_size=2, chunk_overlap=-1)@@ -1164,7 +1181,6 @@ def test_html_code_splitter() -> None:def test_md_header_text_splitter_1() -> None:"""Test markdown splitter by header: Case 1."""-markdown_document = ("# Foo\n\n"" ## Bar\n\n"@@ -1235,7 +1251,6 @@ def test_md_header_text_splitter_2() -> None:def test_md_header_text_splitter_3() -> None:"""Test markdown splitter by header: Case 3."""-markdown_document = ("# Foo\n\n"" ## Bar\n\n"@@ -1290,7 +1305,6 @@ def test_md_header_text_splitter_3() -> None:def test_md_header_text_splitter_preserve_headers_1() -> None:"""Test markdown splitter by header: Preserve Headers."""-markdown_document = ("# Foo\n\n"" ## Bat\n\n"@@ -1324,7 +1338,6 @@ def test_md_header_text_splitter_preserve_headers_1() -> None:def test_md_header_text_splitter_preserve_headers_2() -> None:"""Test markdown splitter by header: Preserve Headers."""-markdown_document = ("# Foo\n\n"" ## Bar\n\n"@@ -1372,7 +1385,6 @@ def test_md_header_text_splitter_preserve_headers_2() -> None:@pytest.mark.parametrize("fence", [("```"), ("~~~")])def test_md_header_text_splitter_fenced_code_block(fence: str) -> None:"""Test markdown splitter by header: Fenced code block."""-markdown_document = (f"# This is a Header\n\n{fence}\nfoo()\n# Not a header\nbar()\n{fence}")@@ -1402,7 +1414,6 @@ def test_md_header_text_splitter_fenced_code_block_interleaved(fence: str, other_fence: str) -> None:"""Test markdown splitter by header: Interleaved fenced code block."""-markdown_document = ("# This is a Header\n\n"f"{fence}\n"@@ -1438,7 +1449,6 @@ def test_md_header_text_splitter_fenced_code_block_interleaved(@pytest.mark.parametrize("characters", ["\ufeff"])def test_md_header_text_splitter_with_invisible_characters(characters: str) -> None:"""Test markdown splitter by header: Fenced code block."""-markdown_document = f"{characters}# Foo\n\nfoo()\n{characters}## Bar\n\nbar()"headers_to_split_on = [@@ -1609,7 +1619,6 @@ EXPERIMENTAL_MARKDOWN_DOCUMENT = (def test_experimental_markdown_syntax_text_splitter() -> None:"""Test experimental markdown syntax splitter."""-markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter()output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)@@ -1663,7 +1672,6 @@ def test_experimental_markdown_syntax_text_splitter() -> None:def test_experimental_markdown_syntax_text_splitter_header_configuration() -> None:"""Test experimental markdown syntax splitter."""-headers_to_split_on = [("#", "Encabezamiento 1")]markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(@@ -1709,7 +1717,6 @@ def test_experimental_markdown_syntax_text_splitter_header_configuration() -> Nodef test_experimental_markdown_syntax_text_splitter_with_headers() -> None:"""Test experimental markdown syntax splitter."""-markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(strip_headers=False)output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)@@ -1768,7 +1775,6 @@ def test_experimental_markdown_syntax_text_splitter_with_headers() -> None:def test_experimental_markdown_syntax_text_splitter_split_lines() -> None:"""Test experimental markdown syntax splitter."""-markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(return_each_line=True)output = markdown_splitter.split_text(EXPERIMENTAL_MARKDOWN_DOCUMENT)@@ -1876,8 +1882,11 @@ EXPERIMENTAL_MARKDOWN_DOCUMENTS = [def test_experimental_markdown_syntax_text_splitter_on_multi_files() -> None:- """Test experimental markdown syntax splitter split- on default called consecutively on two files."""+ """Test ExperimentalMarkdownSyntaxTextSplitter on multiple files.++ Test experimental markdown syntax splitter split on default called consecutively+ on two files.+ """markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter()output = []for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:@@ -1958,8 +1967,11 @@ def test_experimental_markdown_syntax_text_splitter_on_multi_files() -> None:def test_experimental_markdown_syntax_text_splitter_split_lines_on_multi_files() -> (None):- """Test experimental markdown syntax splitter split- on each line called consecutively on two files."""+ """Test ExperimentalMarkdownSyntaxTextSplitter split lines on multiple files.++ Test experimental markdown syntax splitter split on each line called consecutively+ on two files.+ """markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(return_each_line=True)output = []for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:@@ -2083,9 +2095,10 @@ def test_experimental_markdown_syntax_text_splitter_split_lines_on_multi_files()def test_experimental_markdown_syntax_text_splitter_with_header_on_multi_files() -> (None):- """Test experimental markdown splitter- by header called consecutively on two files"""+ """Test ExperimentalMarkdownSyntaxTextSplitter with header on multiple files.+ Test experimental markdown splitter by header called consecutively on two files.+ """markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(strip_headers=False)output = []for experimental_markdown_document in EXPERIMENTAL_MARKDOWN_DOCUMENTS:@@ -2171,9 +2184,11 @@ def test_experimental_markdown_syntax_text_splitter_with_header_on_multi_files()def test_experimental_markdown_syntax_text_splitter_header_config_on_multi_files() -> (None):- """Test experimental markdown splitter- by header configuration called consecutively on two files"""+ """Test ExperimentalMarkdownSyntaxTextSplitter header config on multiple files.+ Test experimental markdown splitter by header configuration called consecutively+ on two files.+ """headers_to_split_on = [("#", "Encabezamiento 1")]markdown_splitter = ExperimentalMarkdownSyntaxTextSplitter(headers_to_split_on=headers_to_split_on@@ -2354,8 +2369,8 @@ def test_haskell_code_splitter() -> None:def html_header_splitter_splitter_factory() -> Callable[[list[tuple[str, str]]], HTMLHeaderTextSplitter]:- """- Fixture to create an HTMLHeaderTextSplitter instance with given headers.+ """Fixture to create an HTMLHeaderTextSplitter instance with given headers.+This factory allows dynamic creation of splitters with different headers."""@@ -2553,14 +2568,15 @@ def html_header_splitter_splitter_factory() -> Callable[)@pytest.mark.requires("bs4")def test_html_header_text_splitter(- html_header_splitter_splitter_factory: Any,+ html_header_splitter_splitter_factory: Callable[+ [list[tuple[str, str]]], HTMLHeaderTextSplitter+ ],headers_to_split_on: list[tuple[str, str]],html_input: str,expected_documents: list[Document],test_case: str,) -> None:- """- Test the HTML header text splitter.+ """Test the HTML header text splitter.Args:html_header_splitter_splitter_factory (Any): Factory function to create@@ -2574,10 +2590,7 @@ def test_html_header_text_splitter(AssertionError: If the number of documents or their content/metadatadoes not match the expected values."""-- splitter = html_header_splitter_splitter_factory(- headers_to_split_on=headers_to_split_on- )+ splitter = html_header_splitter_splitter_factory(headers_to_split_on)docs = splitter.split_text(html_input)assert len(docs) == len(expected_documents), (@@ -2709,14 +2722,15 @@ def test_html_header_text_splitter()@pytest.mark.requires("bs4")def test_additional_html_header_text_splitter(- html_header_splitter_splitter_factory: Any,+ html_header_splitter_splitter_factory: Callable[+ [list[tuple[str, str]]], HTMLHeaderTextSplitter+ ],headers_to_split_on: list[tuple[str, str]],html_content: str,expected_output: list[Document],test_case: str,) -> None:- """- Test the HTML header text splitter.+ """Test the HTML header text splitter.Args:html_header_splitter_splitter_factory (Any): Factory function to create@@ -2730,9 +2744,7 @@ def test_additional_html_header_text_splitter(AssertionError: If the number of documents or their content/metadatadoes not match the expected output."""- splitter = html_header_splitter_splitter_factory(- headers_to_split_on=headers_to_split_on- )+ splitter = html_header_splitter_splitter_factory(headers_to_split_on)docs = splitter.split_text(html_content)assert len(docs) == len(expected_output), (@@ -2780,14 +2792,16 @@ def test_additional_html_header_text_splitter()@pytest.mark.requires("bs4")def test_html_no_headers_with_multiple_splitters(- html_header_splitter_splitter_factory: Any,+ html_header_splitter_splitter_factory: Callable[+ [list[tuple[str, str]]], HTMLHeaderTextSplitter+ ],headers_to_split_on: list[tuple[str, str]],html_content: str,expected_output: list[Document],test_case: str,) -> None:- """- Test HTML content splitting without headers using multiple splitters.+ """Test HTML content splitting without headers using multiple splitters.+Args:html_header_splitter_splitter_factory (Any): Factory to create theHTML header splitter.@@ -2796,13 +2810,12 @@ def test_html_no_headers_with_multiple_splitters(expected_output (List[Document]): Expected list of Document objectsafter splitting.test_case (str): Description of the test case.+Raises:AssertionError: If the number of documents or their content/metadatadoes not match the expected output."""- splitter = html_header_splitter_splitter_factory(- headers_to_split_on=headers_to_split_on- )+ splitter = html_header_splitter_splitter_factory(headers_to_split_on)docs = splitter.split_text(html_content)assert len(docs) == len(expected_output), (@@ -3046,7 +3059,7 @@ def test_happy_path_splitting_with_duplicate_header_tag() -> None:def test_split_json() -> None:- """Test json text splitter"""+ """Test json text splitter."""max_chunk = 800splitter = RecursiveJsonSplitter(max_chunk_size=max_chunk)@@ -3068,7 +3081,7 @@ def test_split_json() -> None:def test_split_json_with_lists() -> None:- """Test json text splitter with list conversion"""+ """Test json text splitter with list conversion."""max_chunk = 800splitter = RecursiveJsonSplitter(max_chunk_size=max_chunk)@@ -3240,7 +3253,7 @@ def test_visualbasic6_code_splitter() -> None:]-def custom_iframe_extractor(iframe_tag: Any) -> str:+def custom_iframe_extractor(iframe_tag: Tag) -> str:iframe_src = iframe_tag.get("src", "")return f"[iframe:{iframe_src}]({iframe_src})"@@ -3324,8 +3337,11 @@ def test_html_splitter_with_nested_elements() -> None:@pytest.mark.requires("bs4")def test_html_splitter_with_preserved_elements() -> None:- """Test HTML splitting with preserved elements like <table>, <ul> with low chunk- size."""+ """Test HTML splitter with preserved elements.++ Test HTML splitting with preserved elements like <table>, <ul> with low chunk+ size.+ """html_content = """<h1>Section 1</h1><table>@@ -3563,8 +3579,11 @@ def test_html_splitter_with_no_headers() -> None:@pytest.mark.requires("bs4")def test_html_splitter_with_media_preservation() -> None:- """Test HTML splitting with media elements preserved and converted to Markdown-like- links."""+ """Test HTML splitter with media preservation.++ Test HTML splitting with media elements preserved and converted to Markdown-like+ links.+ """html_content = """<h1>Section 1</h1><p>This is an image:</p>@@ -3600,7 +3619,7 @@ def test_html_splitter_with_media_preservation() -> None:@pytest.mark.requires("bs4")def test_html_splitter_keep_separator_true() -> None:- """Test HTML splitting with keep_separator=True"""+ """Test HTML splitting with keep_separator=True."""html_content = """<h1>Section 1</h1><p>This is some text. This is some other text.</p>@@ -3629,7 +3648,7 @@ def test_html_splitter_keep_separator_true() -> None:@pytest.mark.requires("bs4")def test_html_splitter_keep_separator_false() -> None:- """Test HTML splitting with keep_separator=False"""+ """Test HTML splitting with keep_separator=False."""html_content = """<h1>Section 1</h1><p>This is some text. This is some other text.</p>@@ -3658,7 +3677,7 @@ def test_html_splitter_keep_separator_false() -> None:@pytest.mark.requires("bs4")def test_html_splitter_keep_separator_start() -> None:- """Test HTML splitting with keep_separator="start" """+ """Test HTML splitting with keep_separator="start"."""html_content = """<h1>Section 1</h1><p>This is some text. This is some other text.</p>@@ -3687,7 +3706,7 @@ def test_html_splitter_keep_separator_start() -> None:@pytest.mark.requires("bs4")def test_html_splitter_keep_separator_end() -> None:- """Test HTML splitting with keep_separator="end" """+ """Test HTML splitting with keep_separator="end"."""html_content = """<h1>Section 1</h1><p>This is some text. This is some other text.</p>@@ -3716,7 +3735,7 @@ def test_html_splitter_keep_separator_end() -> None:@pytest.mark.requires("bs4")def test_html_splitter_keep_separator_default() -> None:- """Test HTML splitting with keep_separator not set"""+ """Test HTML splitting with keep_separator not set."""html_content = """<h1>Section 1</h1><p>This is some text. This is some other text.</p>diff --git a/libs/text-splitters/uv.lock b/libs/text-splitters/uv.lockindex b5f8882b31..bf549a5ad2 100644--- a/libs/text-splitters/uv.lock+++ b/libs/text-splitters/uv.lock@@ -129,15 +129,15 @@ wheels = [[[package]]name = "beautifulsoup4"-version = "4.13.1"+version = "4.13.5"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "soupsieve" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/58/f3/d90227cc52f7b8fcd0f2af804f56e55edf8dd07036b681a2809e3245318b/beautifulsoup4-4.13.1.tar.gz", hash = "sha256:741c8b6903a1e4ae8ba32b9c9ae7510dab7a197fdbadcf9fcdeb0891ef5ec66a", size = 618295, upload-time = "2025-02-03T12:15:13.622Z" }+sdist = { url = "https://files.pythonhosted.org/packages/85/2e/3e5079847e653b1f6dc647aa24549d68c6addb4c595cc0d902d1b19308ad/beautifulsoup4-4.13.5.tar.gz", hash = "sha256:5e70131382930e7c3de33450a2f54a63d5e4b19386eab43a5b34d594268f3695", size = 622954, upload-time = "2025-08-24T14:06:13.168Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/18/75/899bf9b6270b2ce5e8f01b8da121b29e4b88256feb2cf6c6418d4cc42130/beautifulsoup4-4.13.1-py3-none-any.whl", hash = "sha256:72465267014897bb10ca749bb632bde6c2d20f3254afd5458544bd74e6c2e6d8", size = 185056, upload-time = "2025-02-03T12:15:12.076Z" },+ { url = "https://files.pythonhosted.org/packages/04/eb/f4151e0c7377a6e08a38108609ba5cede57986802757848688aeedd1b9e8/beautifulsoup4-4.13.5-py3-none-any.whl", hash = "sha256:642085eaa22233aceadff9c69651bc51e8bf3f874fb6d7104ece2beb24b47c4a", size = 105113, upload-time = "2025-08-24T14:06:14.884Z" },][[package]]@@ -1191,6 +1191,7 @@ test-integration = [{ name = "transformers" },]typing = [+ { name = "beautifulsoup4" },{ name = "lxml-stubs" },{ name = "mypy" },{ name = "tiktoken" },@@ -1229,9 +1230,10 @@ test-integration = [{ name = "transformers", specifier = ">=4.51.3,<5.0.0" },]typing = [+ { name = "beautifulsoup4", specifier = ">=4.13.5,<5.0.0" },{ name = "lxml-stubs", specifier = ">=0.5.1,<1.0.0" },{ name = "mypy", specifier = ">=1.17.1,<1.18" },- { name = "tiktoken", specifier = ">=0.8.0,<1.0.0" },+ { name = "tiktoken", specifier = ">=0.11.0,<1.0.0" },{ name = "types-requests", specifier = ">=2.31.0.20240218,<3.0.0.0" },]@@ -3464,44 +3466,44 @@ wheels = [[[package]]name = "tiktoken"-version = "0.8.0"+version = "0.11.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "regex" },{ name = "requests" },]-sdist = { url = "https://files.pythonhosted.org/packages/37/02/576ff3a6639e755c4f70997b2d315f56d6d71e0d046f4fb64cb81a3fb099/tiktoken-0.8.0.tar.gz", hash = "sha256:9ccbb2740f24542534369c5635cfd9b2b3c2490754a78ac8831d99f89f94eeb2", size = 35107, upload-time = "2024-10-03T22:44:04.196Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/c9/ba/a35fad753bbca8ba0cc1b0f3402a70256a110ced7ac332cf84ba89fc87ab/tiktoken-0.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b07e33283463089c81ef1467180e3e00ab00d46c2c4bbcef0acab5f771d6695e", size = 1039905, upload-time = "2024-10-03T22:43:17.292Z" },- { url = "https://files.pythonhosted.org/packages/91/05/13dab8fd7460391c387b3e69e14bf1e51ff71fe0a202cd2933cc3ea93fb6/tiktoken-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9269348cb650726f44dd3bbb3f9110ac19a8dcc8f54949ad3ef652ca22a38e21", size = 982417, upload-time = "2024-10-03T22:43:19.437Z" },- { url = "https://files.pythonhosted.org/packages/e9/98/18ec4a8351a6cf4537e40cd6e19a422c10cce1ef00a2fcb716e0a96af58b/tiktoken-0.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e13f37bc4ef2d012731e93e0fef21dc3b7aea5bb9009618de9a4026844e560", size = 1144915, upload-time = "2024-10-03T22:43:21.385Z" },- { url = "https://files.pythonhosted.org/packages/2e/28/cf3633018cbcc6deb7805b700ccd6085c9a5a7f72b38974ee0bffd56d311/tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f13d13c981511331eac0d01a59b5df7c0d4060a8be1e378672822213da51e0a2", size = 1177221, upload-time = "2024-10-03T22:43:23.325Z" },- { url = "https://files.pythonhosted.org/packages/57/81/8a5be305cbd39d4e83a794f9e80c7f2c84b524587b7feb27c797b2046d51/tiktoken-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6b2ddbc79a22621ce8b1166afa9f9a888a664a579350dc7c09346a3b5de837d9", size = 1237398, upload-time = "2024-10-03T22:43:24.71Z" },- { url = "https://files.pythonhosted.org/packages/dc/da/8d1cc3089a83f5cf11c2e489332752981435280285231924557350523a59/tiktoken-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d8c2d0e5ba6453a290b86cd65fc51fedf247e1ba170191715b049dac1f628005", size = 884215, upload-time = "2024-10-03T22:43:26.793Z" },- { url = "https://files.pythonhosted.org/packages/f6/1e/ca48e7bfeeccaf76f3a501bd84db1fa28b3c22c9d1a1f41af9fb7579c5f6/tiktoken-0.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d622d8011e6d6f239297efa42a2657043aaed06c4f68833550cac9e9bc723ef1", size = 1039700, upload-time = "2024-10-03T22:43:28.315Z" },- { url = "https://files.pythonhosted.org/packages/8c/f8/f0101d98d661b34534769c3818f5af631e59c36ac6d07268fbfc89e539ce/tiktoken-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2efaf6199717b4485031b4d6edb94075e4d79177a172f38dd934d911b588d54a", size = 982413, upload-time = "2024-10-03T22:43:29.807Z" },- { url = "https://files.pythonhosted.org/packages/ac/3c/2b95391d9bd520a73830469f80a96e3790e6c0a5ac2444f80f20b4b31051/tiktoken-0.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5637e425ce1fc49cf716d88df3092048359a4b3bbb7da762840426e937ada06d", size = 1144242, upload-time = "2024-10-04T04:42:53.66Z" },- { url = "https://files.pythonhosted.org/packages/01/c4/c4a4360de845217b6aa9709c15773484b50479f36bb50419c443204e5de9/tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fb0e352d1dbe15aba082883058b3cce9e48d33101bdaac1eccf66424feb5b47", size = 1176588, upload-time = "2024-10-03T22:43:31.136Z" },- { url = "https://files.pythonhosted.org/packages/f8/a3/ef984e976822cd6c2227c854f74d2e60cf4cd6fbfca46251199914746f78/tiktoken-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56edfefe896c8f10aba372ab5706b9e3558e78db39dd497c940b47bf228bc419", size = 1237261, upload-time = "2024-10-03T22:43:32.75Z" },- { url = "https://files.pythonhosted.org/packages/1e/86/eea2309dc258fb86c7d9b10db536434fc16420feaa3b6113df18b23db7c2/tiktoken-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:326624128590def898775b722ccc327e90b073714227175ea8febbc920ac0a99", size = 884537, upload-time = "2024-10-03T22:43:34.592Z" },- { url = "https://files.pythonhosted.org/packages/c1/22/34b2e136a6f4af186b6640cbfd6f93400783c9ef6cd550d9eab80628d9de/tiktoken-0.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:881839cfeae051b3628d9823b2e56b5cc93a9e2efb435f4cf15f17dc45f21586", size = 1039357, upload-time = "2024-10-03T22:43:36.362Z" },- { url = "https://files.pythonhosted.org/packages/04/d2/c793cf49c20f5855fd6ce05d080c0537d7418f22c58e71f392d5e8c8dbf7/tiktoken-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fe9399bdc3f29d428f16a2f86c3c8ec20be3eac5f53693ce4980371c3245729b", size = 982616, upload-time = "2024-10-03T22:43:37.658Z" },- { url = "https://files.pythonhosted.org/packages/b3/a1/79846e5ef911cd5d75c844de3fa496a10c91b4b5f550aad695c5df153d72/tiktoken-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a58deb7075d5b69237a3ff4bb51a726670419db6ea62bdcd8bd80c78497d7ab", size = 1144011, upload-time = "2024-10-03T22:43:39.092Z" },- { url = "https://files.pythonhosted.org/packages/26/32/e0e3a859136e95c85a572e4806dc58bf1ddf651108ae8b97d5f3ebe1a244/tiktoken-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2908c0d043a7d03ebd80347266b0e58440bdef5564f84f4d29fb235b5df3b04", size = 1175432, upload-time = "2024-10-03T22:43:40.323Z" },- { url = "https://files.pythonhosted.org/packages/c7/89/926b66e9025b97e9fbabeaa59048a736fe3c3e4530a204109571104f921c/tiktoken-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:294440d21a2a51e12d4238e68a5972095534fe9878be57d905c476017bff99fc", size = 1236576, upload-time = "2024-10-03T22:43:41.516Z" },- { url = "https://files.pythonhosted.org/packages/45/e2/39d4aa02a52bba73b2cd21ba4533c84425ff8786cc63c511d68c8897376e/tiktoken-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d8f3192733ac4d77977432947d563d7e1b310b96497acd3c196c9bddb36ed9db", size = 883824, upload-time = "2024-10-03T22:43:43.33Z" },- { url = "https://files.pythonhosted.org/packages/e3/38/802e79ba0ee5fcbf240cd624143f57744e5d411d2e9d9ad2db70d8395986/tiktoken-0.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:02be1666096aff7da6cbd7cdaa8e7917bfed3467cd64b38b1f112e96d3b06a24", size = 1039648, upload-time = "2024-10-03T22:43:45.22Z" },- { url = "https://files.pythonhosted.org/packages/b1/da/24cdbfc302c98663fbea66f5866f7fa1048405c7564ab88483aea97c3b1a/tiktoken-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94ff53c5c74b535b2cbf431d907fc13c678bbd009ee633a2aca269a04389f9a", size = 982763, upload-time = "2024-10-03T22:43:46.571Z" },- { url = "https://files.pythonhosted.org/packages/e4/f0/0ecf79a279dfa41fc97d00adccf976ecc2556d3c08ef3e25e45eb31f665b/tiktoken-0.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b231f5e8982c245ee3065cd84a4712d64692348bc609d84467c57b4b72dcbc5", size = 1144417, upload-time = "2024-10-03T22:43:48.633Z" },- { url = "https://files.pythonhosted.org/packages/ab/d3/155d2d4514f3471a25dc1d6d20549ef254e2aa9bb5b1060809b1d3b03d3a/tiktoken-0.8.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4177faa809bd55f699e88c96d9bb4635d22e3f59d635ba6fd9ffedf7150b9953", size = 1175108, upload-time = "2024-10-03T22:43:50.568Z" },- { url = "https://files.pythonhosted.org/packages/19/eb/5989e16821ee8300ef8ee13c16effc20dfc26c777d05fbb6825e3c037b81/tiktoken-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5376b6f8dc4753cd81ead935c5f518fa0fbe7e133d9e25f648d8c4dabdd4bad7", size = 1236520, upload-time = "2024-10-03T22:43:51.759Z" },- { url = "https://files.pythonhosted.org/packages/40/59/14b20465f1d1cb89cfbc96ec27e5617b2d41c79da12b5e04e96d689be2a7/tiktoken-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:18228d624807d66c87acd8f25fc135665617cab220671eb65b50f5d70fa51f69", size = 883849, upload-time = "2024-10-03T22:43:53.999Z" },- { url = "https://files.pythonhosted.org/packages/08/f3/8a8ba9329e6b426d822c974d58fc6477f3f7b3b8deef651813d275cbe75f/tiktoken-0.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e17807445f0cf1f25771c9d86496bd8b5c376f7419912519699f3cc4dc5c12e", size = 1040915, upload-time = "2024-10-03T22:43:55.177Z" },- { url = "https://files.pythonhosted.org/packages/42/7a/914bd98100449422778f9222d00b3a4ee654211c40784e57541fa46311ab/tiktoken-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:886f80bd339578bbdba6ed6d0567a0d5c6cfe198d9e587ba6c447654c65b8edc", size = 983753, upload-time = "2024-10-03T22:43:56.303Z" },- { url = "https://files.pythonhosted.org/packages/f7/01/1483856d84827c5fe541cb160f07914c6b063b8d961146e9c3557c4730c0/tiktoken-0.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6adc8323016d7758d6de7313527f755b0fc6c72985b7d9291be5d96d73ecd1e1", size = 1145913, upload-time = "2024-10-03T22:43:58.682Z" },- { url = "https://files.pythonhosted.org/packages/c2/e1/6c7a772e0200131e960e3381f1d7b26406bc5612c70677989c1498af2a60/tiktoken-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b591fb2b30d6a72121a80be24ec7a0e9eb51c5500ddc7e4c2496516dd5e3816b", size = 1178505, upload-time = "2024-10-03T22:43:59.86Z" },- { url = "https://files.pythonhosted.org/packages/3e/6b/3ae00f0bff5d0b6925bf6370cf0ff606f56daed76210c2b0a156017b78dc/tiktoken-0.8.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:845287b9798e476b4d762c3ebda5102be87ca26e5d2c9854002825d60cdb815d", size = 1239111, upload-time = "2024-10-03T22:44:01.094Z" },- { url = "https://files.pythonhosted.org/packages/d5/3b/7c8812952ca55e1bab08afc1dda3c5991804c71b550b9402e82a082ab795/tiktoken-0.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:1473cfe584252dc3fa62adceb5b1c763c1874e04511b197da4e6de51d6ce5a02", size = 884803, upload-time = "2024-10-03T22:44:03.079Z" },+sdist = { url = "https://files.pythonhosted.org/packages/a7/86/ad0155a37c4f310935d5ac0b1ccf9bdb635dcb906e0a9a26b616dd55825a/tiktoken-0.11.0.tar.gz", hash = "sha256:3c518641aee1c52247c2b97e74d8d07d780092af79d5911a6ab5e79359d9b06a", size = 37648, upload-time = "2025-08-08T23:58:08.495Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/8b/4d/c6a2e7dca2b4f2e9e0bfd62b3fe4f114322e2c028cfba905a72bc76ce479/tiktoken-0.11.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8a9b517d6331d7103f8bef29ef93b3cca95fa766e293147fe7bacddf310d5917", size = 1059937, upload-time = "2025-08-08T23:57:28.57Z" },+ { url = "https://files.pythonhosted.org/packages/41/54/3739d35b9f94cb8dc7b0db2edca7192d5571606aa2369a664fa27e811804/tiktoken-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b4ddb1849e6bf0afa6cc1c5d809fb980ca240a5fffe585a04e119519758788c0", size = 999230, upload-time = "2025-08-08T23:57:30.241Z" },+ { url = "https://files.pythonhosted.org/packages/dd/f4/ec8d43338d28d53513004ebf4cd83732a135d11011433c58bf045890cc10/tiktoken-0.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10331d08b5ecf7a780b4fe4d0281328b23ab22cdb4ff65e68d56caeda9940ecc", size = 1130076, upload-time = "2025-08-08T23:57:31.706Z" },+ { url = "https://files.pythonhosted.org/packages/94/80/fb0ada0a882cb453caf519a4bf0d117c2a3ee2e852c88775abff5413c176/tiktoken-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b062c82300341dc87e0258c69f79bed725f87e753c21887aea90d272816be882", size = 1183942, upload-time = "2025-08-08T23:57:33.142Z" },+ { url = "https://files.pythonhosted.org/packages/2f/e9/6c104355b463601719582823f3ea658bc3aa7c73d1b3b7553ebdc48468ce/tiktoken-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:195d84bec46169af3b1349a1495c151d37a0ff4cba73fd08282736be7f92cc6c", size = 1244705, upload-time = "2025-08-08T23:57:34.594Z" },+ { url = "https://files.pythonhosted.org/packages/94/75/eaa6068f47e8b3f0aab9e05177cce2cf5aa2cc0ca93981792e620d4d4117/tiktoken-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe91581b0ecdd8783ce8cb6e3178f2260a3912e8724d2f2d49552b98714641a1", size = 884152, upload-time = "2025-08-08T23:57:36.18Z" },+ { url = "https://files.pythonhosted.org/packages/8a/91/912b459799a025d2842566fe1e902f7f50d54a1ce8a0f236ab36b5bd5846/tiktoken-0.11.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4ae374c46afadad0f501046db3da1b36cd4dfbfa52af23c998773682446097cf", size = 1059743, upload-time = "2025-08-08T23:57:37.516Z" },+ { url = "https://files.pythonhosted.org/packages/8c/e9/6faa6870489ce64f5f75dcf91512bf35af5864583aee8fcb0dcb593121f5/tiktoken-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25a512ff25dc6c85b58f5dd4f3d8c674dc05f96b02d66cdacf628d26a4e4866b", size = 999334, upload-time = "2025-08-08T23:57:38.595Z" },+ { url = "https://files.pythonhosted.org/packages/a1/3e/a05d1547cf7db9dc75d1461cfa7b556a3b48e0516ec29dfc81d984a145f6/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2130127471e293d385179c1f3f9cd445070c0772be73cdafb7cec9a3684c0458", size = 1129402, upload-time = "2025-08-08T23:57:39.627Z" },+ { url = "https://files.pythonhosted.org/packages/34/9a/db7a86b829e05a01fd4daa492086f708e0a8b53952e1dbc9d380d2b03677/tiktoken-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e43022bf2c33f733ea9b54f6a3f6b4354b909f5a73388fb1b9347ca54a069c", size = 1184046, upload-time = "2025-08-08T23:57:40.689Z" },+ { url = "https://files.pythonhosted.org/packages/9d/bb/52edc8e078cf062ed749248f1454e9e5cfd09979baadb830b3940e522015/tiktoken-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:adb4e308eb64380dc70fa30493e21c93475eaa11669dea313b6bbf8210bfd013", size = 1244691, upload-time = "2025-08-08T23:57:42.251Z" },+ { url = "https://files.pythonhosted.org/packages/60/d9/884b6cd7ae2570ecdcaffa02b528522b18fef1cbbfdbcaa73799807d0d3b/tiktoken-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:ece6b76bfeeb61a125c44bbefdfccc279b5288e6007fbedc0d32bfec602df2f2", size = 884392, upload-time = "2025-08-08T23:57:43.628Z" },+ { url = "https://files.pythonhosted.org/packages/e7/9e/eceddeffc169fc75fe0fd4f38471309f11cb1906f9b8aa39be4f5817df65/tiktoken-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fd9e6b23e860973cf9526544e220b223c60badf5b62e80a33509d6d40e6c8f5d", size = 1055199, upload-time = "2025-08-08T23:57:45.076Z" },+ { url = "https://files.pythonhosted.org/packages/4f/cf/5f02bfefffdc6b54e5094d2897bc80efd43050e5b09b576fd85936ee54bf/tiktoken-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76d53cee2da71ee2731c9caa747398762bda19d7f92665e882fef229cb0b5b", size = 996655, upload-time = "2025-08-08T23:57:46.304Z" },+ { url = "https://files.pythonhosted.org/packages/65/8e/c769b45ef379bc360c9978c4f6914c79fd432400a6733a8afc7ed7b0726a/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef72aab3ea240646e642413cb363b73869fed4e604dcfd69eec63dc54d603e8", size = 1128867, upload-time = "2025-08-08T23:57:47.438Z" },+ { url = "https://files.pythonhosted.org/packages/d5/2d/4d77f6feb9292bfdd23d5813e442b3bba883f42d0ac78ef5fdc56873f756/tiktoken-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f929255c705efec7a28bf515e29dc74220b2f07544a8c81b8d69e8efc4578bd", size = 1183308, upload-time = "2025-08-08T23:57:48.566Z" },+ { url = "https://files.pythonhosted.org/packages/7a/65/7ff0a65d3bb0fc5a1fb6cc71b03e0f6e71a68c5eea230d1ff1ba3fd6df49/tiktoken-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:61f1d15822e4404953d499fd1dcc62817a12ae9fb1e4898033ec8fe3915fdf8e", size = 1244301, upload-time = "2025-08-08T23:57:49.642Z" },+ { url = "https://files.pythonhosted.org/packages/f5/6e/5b71578799b72e5bdcef206a214c3ce860d999d579a3b56e74a6c8989ee2/tiktoken-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:45927a71ab6643dfd3ef57d515a5db3d199137adf551f66453be098502838b0f", size = 884282, upload-time = "2025-08-08T23:57:50.759Z" },+ { url = "https://files.pythonhosted.org/packages/cc/cd/a9034bcee638716d9310443818d73c6387a6a96db93cbcb0819b77f5b206/tiktoken-0.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a5f3f25ffb152ee7fec78e90a5e5ea5b03b4ea240beed03305615847f7a6ace2", size = 1055339, upload-time = "2025-08-08T23:57:51.802Z" },+ { url = "https://files.pythonhosted.org/packages/f1/91/9922b345f611b4e92581f234e64e9661e1c524875c8eadd513c4b2088472/tiktoken-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7dc6e9ad16a2a75b4c4be7208055a1f707c9510541d94d9cc31f7fbdc8db41d8", size = 997080, upload-time = "2025-08-08T23:57:53.442Z" },+ { url = "https://files.pythonhosted.org/packages/d0/9d/49cd047c71336bc4b4af460ac213ec1c457da67712bde59b892e84f1859f/tiktoken-0.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a0517634d67a8a48fd4a4ad73930c3022629a85a217d256a6e9b8b47439d1e4", size = 1128501, upload-time = "2025-08-08T23:57:54.808Z" },+ { url = "https://files.pythonhosted.org/packages/52/d5/a0dcdb40dd2ea357e83cb36258967f0ae96f5dd40c722d6e382ceee6bba9/tiktoken-0.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fb4effe60574675118b73c6fbfd3b5868e5d7a1f570d6cc0d18724b09ecf318", size = 1182743, upload-time = "2025-08-08T23:57:56.307Z" },+ { url = "https://files.pythonhosted.org/packages/3b/17/a0fc51aefb66b7b5261ca1314afa83df0106b033f783f9a7bcbe8e741494/tiktoken-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:94f984c9831fd32688aef4348803b0905d4ae9c432303087bae370dc1381a2b8", size = 1244057, upload-time = "2025-08-08T23:57:57.628Z" },+ { url = "https://files.pythonhosted.org/packages/50/79/bcf350609f3a10f09fe4fc207f132085e497fdd3612f3925ab24d86a0ca0/tiktoken-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:2177ffda31dec4023356a441793fed82f7af5291120751dee4d696414f54db0c", size = 883901, upload-time = "2025-08-08T23:57:59.359Z" },+ { url = "https://files.pythonhosted.org/packages/aa/b6/81c5799ab77a9580c6d840cf77d4717e929193a42190fd623a080c647aa6/tiktoken-0.11.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:13220f12c9e82e399377e768640ddfe28bea962739cc3a869cad98f42c419a89", size = 1061648, upload-time = "2025-08-08T23:58:00.753Z" },+ { url = "https://files.pythonhosted.org/packages/50/89/faa668066b2a4640534ef5797c09ecd0a48b43367502129b217339dfaa97/tiktoken-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f2db627f5c74477c0404b4089fd8a28ae22fa982a6f7d9c7d4c305c375218f3", size = 1000950, upload-time = "2025-08-08T23:58:01.855Z" },+ { url = "https://files.pythonhosted.org/packages/aa/7f/5f950528b54cb3025af4bc3522c23dbfb691afe8ffb292aa1e8dc2e6bddf/tiktoken-0.11.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2302772f035dceb2bcf8e55a735e4604a0b51a6dd50f38218ff664d46ec43807", size = 1130777, upload-time = "2025-08-08T23:58:03.256Z" },+ { url = "https://files.pythonhosted.org/packages/27/a4/e82ddf0773835ba24536ac8c0dce561e697698ec020a93212a1e041d39b4/tiktoken-0.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20b977989afe44c94bcc50db1f76971bb26dca44218bd203ba95925ef56f8e7a", size = 1185692, upload-time = "2025-08-08T23:58:04.476Z" },+ { url = "https://files.pythonhosted.org/packages/1b/c2/06361e41d176e62797ae65fa678111cdd30553321cf4d83e7b84107ea95f/tiktoken-0.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:669a1aa1ad6ebf1b3c26b45deb346f345da7680f845b5ea700bba45c20dea24c", size = 1246518, upload-time = "2025-08-08T23:58:06.126Z" },+ { url = "https://files.pythonhosted.org/packages/bb/ad/ca37e15c46741ebb3904d562d03194e845539a08f7751a6df0f391757312/tiktoken-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:e363f33c720a055586f730c00e330df4c7ea0024bf1c83a8a9a9dbc054c4f304", size = 884702, upload-time = "2025-08-08T23:58:07.534Z" },][[package]]