Skip to content

The agent engineering platform.

fix(text-splitters): restore lazy imports for heavy optional dependencies (#35469)

Balaji Seshadrifd822b07c00e
diff --git a/libs/text-splitters/langchain_text_splitters/__init__.py b/libs/text-splitters/langchain_text_splitters/__init__.py
index 7dc07f50b2..fe36491845 100644
--- a/libs/text-splitters/langchain_text_splitters/__init__.py
+++ b/libs/text-splitters/langchain_text_splitters/__init__.py
@@ -6,6 +6,11 @@
`TextSplitter`.
"""
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING
+
from langchain_text_splitters.base import (
Language,
TextSplitter,
@@ -25,7 +30,6 @@ from langchain_text_splitters.html import (
)
from langchain_text_splitters.json import RecursiveJsonSplitter
from langchain_text_splitters.jsx import JSFrameworkTextSplitter
-from langchain_text_splitters.konlpy import KonlpyTextSplitter
from langchain_text_splitters.latex import LatexTextSplitter
from langchain_text_splitters.markdown import (
ExperimentalMarkdownSyntaxTextSplitter,
@@ -34,12 +38,15 @@ from langchain_text_splitters.markdown import (
MarkdownHeaderTextSplitter,
MarkdownTextSplitter,
)
-from langchain_text_splitters.nltk import NLTKTextSplitter
from langchain_text_splitters.python import PythonCodeTextSplitter
-from langchain_text_splitters.sentence_transformers import (
- SentenceTransformersTokenTextSplitter,
-)
-from langchain_text_splitters.spacy import SpacyTextSplitter
+
+if TYPE_CHECKING:
+ from langchain_text_splitters.konlpy import KonlpyTextSplitter
+ from langchain_text_splitters.nltk import NLTKTextSplitter
+ from langchain_text_splitters.sentence_transformers import (
+ SentenceTransformersTokenTextSplitter,
+ )
+ from langchain_text_splitters.spacy import SpacyTextSplitter
__all__ = [
"CharacterTextSplitter",
@@ -67,3 +74,26 @@ __all__ = [
"Tokenizer",
"split_text_on_tokens",
]
+
+# Splitters whose modules pull in heavy optional dependencies (konlpy, nltk,
+# spacy, sentence-transformers/torch). Deferring their import behind
+# `__getattr__` keeps `import langchain_text_splitters` lightweight even
+# though the classes remain in `__all__` and are fully accessible on first
+# access.
+_LAZY_SPLITTERS: dict[str, str] = {
+ "KonlpyTextSplitter": "konlpy",
+ "NLTKTextSplitter": "nltk",
+ "SentenceTransformersTokenTextSplitter": "sentence_transformers",
+ "SpacyTextSplitter": "spacy",
+}
+
+
+def __getattr__(attr_name: str) -> object:
+ module_name = _LAZY_SPLITTERS.get(attr_name)
+ if module_name is not None:
+ module = import_module(f".{module_name}", __name__)
+ result = getattr(module, attr_name)
+ globals()[attr_name] = result
+ return result
+ msg = f"module {__name__!r} has no attribute {attr_name!r}"
+ raise AttributeError(msg)
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py
index 320745dd55..cabe712f26 100644
--- a/libs/text-splitters/langchain_text_splitters/base.py
+++ b/libs/text-splitters/langchain_text_splitters/base.py
@@ -12,6 +12,7 @@ from typing import (
Any,
Literal,
TypeVar,
+ cast,
)
from langchain_core.documents import BaseDocumentTransformer, Document
@@ -21,24 +22,38 @@ if TYPE_CHECKING:
from collections.abc import Callable, Collection, Iterable, Sequence
from collections.abc import Set as AbstractSet
+ from transformers.tokenization_utils_base import PreTrainedTokenizerBase
-try:
- import tiktoken
+logger = logging.getLogger(__name__)
- _HAS_TIKTOKEN = True
-except ImportError:
- _HAS_TIKTOKEN = False
+TS = TypeVar("TS", bound="TextSplitter")
-try:
- from transformers.tokenization_utils_base import PreTrainedTokenizerBase
- _HAS_TRANSFORMERS = True
-except ImportError:
- _HAS_TRANSFORMERS = False
+def _import_tiktoken() -> object:
+ try:
+ import tiktoken # noqa: PLC0415
+ except ImportError as err:
+ msg = (
+ "Could not import tiktoken python package. "
+ "This is needed in order to calculate max_tokens_for_prompt. "
+ "Please install it with `pip install tiktoken`."
+ )
+ raise ImportError(msg) from err
+ return tiktoken
-logger = logging.getLogger(__name__)
-TS = TypeVar("TS", bound="TextSplitter")
+def _import_pretrained_tokenizer_base() -> type[PreTrainedTokenizerBase]:
+ try:
+ from transformers.tokenization_utils_base import ( # noqa: PLC0415
+ PreTrainedTokenizerBase,
+ )
+ except ImportError as err:
+ msg = (
+ "Could not import transformers python package. "
+ "Please install it with `pip install transformers`."
+ )
+ raise ValueError(msg) from err
+ return PreTrainedTokenizerBase
class TextSplitter(BaseDocumentTransformer, ABC):
@@ -206,14 +221,9 @@ class TextSplitter(BaseDocumentTransformer, ABC):
An instance of `TextSplitter` using the Hugging Face tokenizer for length
calculation.
"""
- if not _HAS_TRANSFORMERS:
- msg = (
- "Could not import transformers python package. "
- "Please install it with `pip install transformers`."
- )
- raise ValueError(msg)
+ pretrained_tokenizer_base = _import_pretrained_tokenizer_base()
- if not isinstance(tokenizer, PreTrainedTokenizerBase):
+ if not isinstance(tokenizer, pretrained_tokenizer_base):
msg = "Tokenizer received was not an instance of PreTrainedTokenizerBase"
raise ValueError(msg) # noqa: TRY004
@@ -250,13 +260,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
"""
if allowed_special is None:
allowed_special = set()
- if not _HAS_TIKTOKEN:
- msg = (
- "Could not import tiktoken python package. "
- "This is needed in order to calculate max_tokens_for_prompt. "
- "Please install it with `pip install tiktoken`."
- )
- raise ImportError(msg)
+ tiktoken = cast("Any", _import_tiktoken())
if model_name is not None:
enc = tiktoken.encoding_for_model(model_name)
@@ -344,13 +348,15 @@ class TokenTextSplitter(TextSplitter):
if allowed_special is None:
allowed_special = set()
super().__init__(**kwargs)
- if not _HAS_TIKTOKEN:
+ try:
+ tiktoken = cast("Any", _import_tiktoken())
+ except ImportError as err:
msg = (
"Could not import tiktoken python package. "
"This is needed in order to for TokenTextSplitter. "
"Please install it with `pip install tiktoken`."
)
- raise ImportError(msg)
+ raise ImportError(msg) from err
if model_name is not None:
enc = tiktoken.encoding_for_model(model_name)
@@ -419,10 +425,14 @@ class TokenTextSplitter(TextSplitter):
"""
def _encode(_text: str) -> list[int]:
- return self._tokenizer.encode(
- _text,
- allowed_special=self._allowed_special,
- disallowed_special=self._disallowed_special,
+ # `tiktoken` is lazy-imported, so mypy cannot infer the encoder return.
+ return cast(
+ "list[int]",
+ self._tokenizer.encode(
+ _text,
+ allowed_special=self._allowed_special,
+ disallowed_special=self._disallowed_special,
+ ),
)
tokenizer = Tokenizer(
diff --git a/libs/text-splitters/langchain_text_splitters/html.py b/libs/text-splitters/langchain_text_splitters/html.py
index 818d003751..e22977643f 100644
--- a/libs/text-splitters/langchain_text_splitters/html.py
+++ b/libs/text-splitters/langchain_text_splitters/html.py
@@ -24,29 +24,8 @@ from langchain_text_splitters.character import RecursiveCharacterTextSplitter
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Iterator, Sequence
- from bs4.element import ResultSet
-
-try:
- import nltk
-
- _HAS_NLTK = True
-except ImportError:
- _HAS_NLTK = False
-
-try:
from bs4 import BeautifulSoup, Tag
- from bs4.element import NavigableString, PageElement
-
- _HAS_BS4 = True
-except ImportError:
- _HAS_BS4 = False
-
-try:
- from lxml import etree
-
- _HAS_LXML = True
-except ImportError:
- _HAS_LXML = False
+ from bs4.element import NavigableString, PageElement, ResultSet
class ElementType(TypedDict):
@@ -58,6 +37,35 @@ class ElementType(TypedDict):
metadata: dict[str, str]
+def _import_bs4(
+ *, import_error_message: str
+) -> tuple[type[BeautifulSoup], type[Tag], type[NavigableString]]:
+ try:
+ from bs4 import BeautifulSoup, Tag # noqa: PLC0415
+ from bs4.element import NavigableString # noqa: PLC0415
+ except ImportError as err:
+ raise ImportError(import_error_message) from err
+ return BeautifulSoup, Tag, NavigableString
+
+
+def _import_lxml_etree() -> object:
+ try:
+ from lxml import etree # noqa: PLC0415
+ except ImportError as err:
+ msg = "Unable to import lxml, please install with `pip install lxml`."
+ raise ImportError(msg) from err
+ return etree
+
+
+def _import_nltk() -> object:
+ try:
+ import nltk # noqa: PLC0415
+ except ImportError as err:
+ msg = "Could not import nltk. Please install it with 'pip install nltk'."
+ raise ImportError(msg) from err
+ return nltk
+
+
# Unfortunately, BeautifulSoup doesn't define overloads for Tag.find_all.
# So doing the type resolution ourselves.
@@ -257,13 +265,13 @@ class HTMLHeaderTextSplitter:
Raises:
ImportError: If BeautifulSoup is not installed.
"""
- if not _HAS_BS4:
- msg = (
+ beautiful_soup, tag_cls, _ = _import_bs4(
+ import_error_message=(
"Unable to import BeautifulSoup. Please install via `pip install bs4`."
)
- raise ImportError(msg)
+ )
- soup = BeautifulSoup(html_content, "html.parser")
+ soup = beautiful_soup(html_content, "html.parser")
body = soup.body or soup
# Dictionary of active headers:
@@ -292,7 +300,7 @@ class HTMLHeaderTextSplitter:
children = list(node.children)
stack.extend(
- child for child in reversed(children) if isinstance(child, Tag)
+ child for child in reversed(children) if isinstance(child, tag_cls)
)
tag = getattr(node, "name", None)
@@ -465,13 +473,14 @@ class HTMLSectionSplitter:
Raises:
ImportError: If BeautifulSoup is not installed.
"""
- if not _HAS_BS4:
- msg = "Unable to import BeautifulSoup/PageElement, \
- please install with `pip install \
- bs4`."
- raise ImportError(msg)
+ beautiful_soup, _, _ = _import_bs4(
+ import_error_message=(
+ "Unable to import BeautifulSoup/PageElement, "
+ "please install with `pip install bs4`."
+ )
+ )
- soup = BeautifulSoup(html_doc, "html.parser")
+ soup = beautiful_soup(html_doc, "html.parser")
header_names = list(self.headers_to_split_on.keys())
sections: list[dict[str, str | None]] = []
@@ -520,9 +529,7 @@ class HTMLSectionSplitter:
Raises:
ImportError: If the `lxml` library is not installed.
"""
- if not _HAS_LXML:
- msg = "Unable to import lxml, please install with `pip install lxml`."
- raise ImportError(msg)
+ etree = cast("Any", _import_lxml_etree())
# use lxml library to parse html document and return xml ElementTree
# Create secure parsers to prevent XXE attacks
html_parser = etree.HTMLParser(no_network=True)
@@ -532,8 +539,7 @@ class HTMLSectionSplitter:
# Apply XSLT access control to prevent file/network access
# DENY_ALL is a predefined access control that blocks all file/network access
- # Type ignore needed due to incomplete lxml type stubs
- ac = etree.XSLTAccessControl.DENY_ALL # ty: ignore[unresolved-attribute]
+ ac = etree.XSLTAccessControl.DENY_ALL
tree = etree.parse(StringIO(html_content), html_parser)
xslt_tree = etree.parse(self.xslt_path, xslt_parser)
@@ -670,12 +676,12 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
ImportError: If BeautifulSoup or NLTK (when stopword removal is enabled)
is not installed.
"""
- if not _HAS_BS4:
- msg = (
+ _import_bs4(
+ import_error_message=(
"Could not import BeautifulSoup. "
"Please install it with 'pip install bs4'."
)
- raise ImportError(msg)
+ )
self._headers_to_split_on = sorted(headers_to_split_on)
self._max_chunk_size = max_chunk_size
@@ -718,11 +724,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
)
if self._stopword_removal:
- if not _HAS_NLTK:
- msg = (
- "Could not import nltk. Please install it with 'pip install nltk'."
- )
- raise ImportError(msg)
+ nltk = cast("Any", _import_nltk())
nltk.download("stopwords")
self._stopwords = set(nltk.corpus.stopwords.words(self._stopword_lang))
@@ -735,7 +737,13 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
Returns:
A list of `Document` objects containing the split content.
"""
- soup = BeautifulSoup(text, "html.parser")
+ beautiful_soup, _, _ = _import_bs4(
+ import_error_message=(
+ "Could not import BeautifulSoup. "
+ "Please install it with 'pip install bs4'."
+ )
+ )
+ soup = beautiful_soup(text, "html.parser")
self._process_media(soup)
@@ -813,13 +821,19 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
Args:
soup: Parsed HTML content using BeautifulSoup.
"""
+ _, _, navigable_string = _import_bs4(
+ import_error_message=(
+ "Could not import BeautifulSoup. "
+ "Please install it with 'pip install bs4'."
+ )
+ )
for a_tag in _find_all_tags(soup, name="a"):
a_href = a_tag.get("href", "")
a_text = a_tag.get_text(strip=True)
markdown_link = f"[{a_text}]({a_href})"
wrapper = soup.new_tag("link-wrapper")
wrapper.string = markdown_link
- a_tag.replace_with(NavigableString(markdown_link))
+ a_tag.replace_with(navigable_string(markdown_link))
def _filter_tags(self, soup: BeautifulSoup) -> None:
"""Filters the HTML content based on the allowlist and denylist tags.
@@ -866,6 +880,12 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
Returns:
A list of `Document` objects containing the split content.
"""
+ _, tag_cls, _ = _import_bs4(
+ import_error_message=(
+ "Could not import BeautifulSoup. "
+ "Please install it with 'pip install bs4'."
+ )
+ )
documents: list[Document] = []
current_headers: dict[str, str] = {}
current_content: list[str] = []
@@ -884,7 +904,7 @@ class HTMLSemanticPreservingSplitter(BaseDocumentTransformer):
The processed text of the element, or an empty string for
elements with no extractable text.
"""
- if isinstance(element, Tag):
+ if isinstance(element, tag_cls):
if element.name in self._custom_handlers:
return self._custom_handlers[element.name](element)
diff --git a/libs/text-splitters/langchain_text_splitters/konlpy.py b/libs/text-splitters/langchain_text_splitters/konlpy.py
index e9404431c9..6f999d6668 100644
--- a/libs/text-splitters/langchain_text_splitters/konlpy.py
+++ b/libs/text-splitters/langchain_text_splitters/konlpy.py
@@ -8,13 +8,6 @@ from typing_extensions import override
from langchain_text_splitters.base import TextSplitter
-try:
- import konlpy
-
- _HAS_KONLPY = True
-except ImportError:
- _HAS_KONLPY = False
-
class KonlpyTextSplitter(TextSplitter):
"""Splitting text using Konlpy package.
@@ -37,12 +30,13 @@ class KonlpyTextSplitter(TextSplitter):
"""
super().__init__(**kwargs)
self._separator = separator
- if not _HAS_KONLPY:
- msg = """
- Konlpy is not installed, please install it with
- `pip install konlpy`
- """
- raise ImportError(msg)
+ try:
+ import konlpy # noqa: PLC0415
+ except ImportError as err:
+ msg = (
+ "Konlpy is not installed, please install it with `pip install konlpy`."
+ )
+ raise ImportError(msg) from err
self.kkma = konlpy.tag.Kkma()
@override
diff --git a/libs/text-splitters/langchain_text_splitters/nltk.py b/libs/text-splitters/langchain_text_splitters/nltk.py
index 9951b5299e..a720b51ff1 100644
--- a/libs/text-splitters/langchain_text_splitters/nltk.py
+++ b/libs/text-splitters/langchain_text_splitters/nltk.py
@@ -11,13 +11,6 @@ from langchain_text_splitters.base import TextSplitter
if TYPE_CHECKING:
from collections.abc import Callable
-try:
- import nltk
-
- _HAS_NLTK = True
-except ImportError:
- _HAS_NLTK = False
-
class NLTKTextSplitter(TextSplitter):
"""Splitting text using NLTK package."""
@@ -47,9 +40,11 @@ class NLTKTextSplitter(TextSplitter):
if use_span_tokenize and self._separator:
msg = "When use_span_tokenize is True, separator should be ''"
raise ValueError(msg)
- if not _HAS_NLTK:
+ try:
+ import nltk # noqa: PLC0415,F401
+ except ImportError as err:
msg = "NLTK is not installed, please install it with `pip install nltk`."
- raise ImportError(msg)
+ raise ImportError(msg) from err
if use_span_tokenize:
self._tokenizer = self._span_tokenizer(language)
else:
@@ -57,10 +52,14 @@ class NLTKTextSplitter(TextSplitter):
@staticmethod
def _sent_tokenizer(language: str) -> Callable[[str], list[str]]:
+ import nltk # noqa: PLC0415
+
return lambda text: nltk.tokenize.sent_tokenize(text, language)
@staticmethod
def _span_tokenizer(language: str) -> Callable[[str], list[str]]:
+ import nltk # noqa: PLC0415
+
tokenizer = nltk.tokenize._get_punkt_tokenizer(language) # noqa: SLF001
def _tokenize(text: str) -> list[str]:
diff --git a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
index ae07bf5dd1..d77b804b3e 100644
--- a/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
+++ b/libs/text-splitters/langchain_text_splitters/sentence_transformers.py
@@ -2,21 +2,13 @@
from __future__ import annotations
+from importlib import import_module
from typing import Any, cast
from typing_extensions import override
from langchain_text_splitters.base import TextSplitter, Tokenizer, split_text_on_tokens
-try:
- from sentence_transformers import (
- SentenceTransformer,
- )
-
- _HAS_SENTENCE_TRANSFORMERS = True
-except ImportError:
- _HAS_SENTENCE_TRANSFORMERS = False
-
class SentenceTransformersTokenTextSplitter(TextSplitter):
"""Splitting text to tokens using sentence model tokenizer."""
@@ -42,19 +34,23 @@ class SentenceTransformersTokenTextSplitter(TextSplitter):
Raises:
ImportError: If the `sentence_transformers` package is not installed.
+ ValueError: If `tokens_per_chunk` exceeds the model's maximum token limit.
"""
super().__init__(**kwargs, chunk_overlap=chunk_overlap)
- if not _HAS_SENTENCE_TRANSFORMERS:
+ try:
+ sentence_transformers = cast("Any", import_module("sentence_transformers"))
+ sentence_transformer_cls = sentence_transformers.SentenceTransformer
+ except ImportError as err:
msg = (
"Could not import sentence_transformers python package. "
"This is needed in order to use SentenceTransformersTokenTextSplitter. "
"Please install it with `pip install sentence-transformers`."
)
- raise ImportError(msg)
+ raise ImportError(msg) from err
self.model_name = model_name
- self._model = SentenceTransformer(self.model_name, **(model_kwargs or {}))
+ self._model = sentence_transformer_cls(self.model_name, **(model_kwargs or {}))
self.tokenizer = self._model.tokenizer
self._initialize_chunk_configuration(tokens_per_chunk=tokens_per_chunk)
diff --git a/libs/text-splitters/langchain_text_splitters/spacy.py b/libs/text-splitters/langchain_text_splitters/spacy.py
index c61cf490ff..14bcdcc5e3 100644
--- a/libs/text-splitters/langchain_text_splitters/spacy.py
+++ b/libs/text-splitters/langchain_text_splitters/spacy.py
@@ -2,24 +2,18 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any
+from importlib import import_module
+from typing import TYPE_CHECKING, Any, cast
from typing_extensions import override
from langchain_text_splitters.base import TextSplitter
-try:
- import spacy
- from spacy.lang.en import English
-
- if TYPE_CHECKING:
- from spacy.language import (
- Language,
- )
-
- _HAS_SPACY = True
-except ImportError:
- _HAS_SPACY = False
+if TYPE_CHECKING:
+ # Type ignores needed as long as spacy doesn't support Python 3.14.
+ from spacy.language import ( # type: ignore[import-not-found, unused-ignore]
+ Language,
+ )
class SpacyTextSplitter(TextSplitter):
@@ -60,11 +54,14 @@ class SpacyTextSplitter(TextSplitter):
def _make_spacy_pipeline_for_splitting(
pipeline: str, *, max_length: int = 1_000_000
) -> Language:
- if not _HAS_SPACY:
+ try:
+ spacy = cast("Any", import_module("spacy"))
+ english_cls = cast("Any", import_module("spacy.lang.en")).English
+ except ImportError as err:
msg = "Spacy is not installed, please install it with `pip install spacy`."
- raise ImportError(msg)
+ raise ImportError(msg) from err
if pipeline == "sentencizer":
- sentencizer: Language = English()
+ sentencizer: Language = english_cls()
sentencizer.add_pipe("sentencizer")
else:
sentencizer = spacy.load(pipeline, exclude=["ner", "tagger"])
diff --git a/libs/text-splitters/tests/unit_tests/test_text_splitters.py b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
index cd680af6ce..e226501d73 100644
--- a/libs/text-splitters/tests/unit_tests/test_text_splitters.py
+++ b/libs/text-splitters/tests/unit_tests/test_text_splitters.py
@@ -54,6 +54,133 @@ def bar():
"""
+def test_no_heavy_imports_on_package_load() -> None:
+ """Ensure importing the package does not eagerly import heavy dependencies.
+
+ Runs in a fresh interpreter so the result is unaffected by modules the test
+ session already imported. A `sys.meta_path` finder records any *attempt* to
+ import a heavy optional dependency, so the guard holds whether or not those
+ packages are installed in the current environment (a plain `sys.modules` check
+ would pass vacuously when the packages are absent).
+ """
+ import subprocess # noqa: PLC0415
+ import sys # noqa: PLC0415
+
+ script = textwrap.dedent(
+ """
+ import sys
+
+ blocked = {
+ "nltk", "spacy", "sentence_transformers", "konlpy", "torch",
+ "transformers", "tiktoken",
+ }
+ attempted = []
+
+ class _Recorder:
+ def find_spec(self, name, path=None, target=None):
+ if name.split(".")[0] in blocked:
+ attempted.append(name.split(".")[0])
+ return None # defer to the real finders
+
+ sys.meta_path.insert(0, _Recorder())
+ import langchain_text_splitters # noqa: F401
+ print(",".join(sorted(set(attempted))))
+ """
+ )
+ result = subprocess.run( # noqa: S603 # list args, no shell; input is static
+ [sys.executable, "-c", script],
+ capture_output=True,
+ text=True,
+ check=False,
+ timeout=60,
+ )
+ assert result.returncode == 0, (
+ f"Importing langchain_text_splitters failed:\n{result.stderr}"
+ )
+ attempted = [p for p in result.stdout.strip().split(",") if p]
+ assert not attempted, (
+ f"Heavy packages imported at langchain_text_splitters load time: {attempted}"
+ )
+
+
+ ("module_name", "expected_message"),
+ [
+ ("konlpy", "pip install konlpy"),
+ ("nltk", "pip install nltk"),
+ ("spacy", "pip install spacy"),
+ ("sentence_transformers", "pip install sentence-transformers"),
+ ],
+)
+def test_missing_optional_dependency_raises_importerror(
+ module_name: str,
+ expected_message: str,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Each splitter raises a helpful ImportError when its optional dep is missing.
+
+ The missing dependency is simulated by forcing its import to fail, so the test
+ is independent of whether the optional package is actually installed.
+ """
+ import sys # noqa: PLC0415
+
+ from langchain_text_splitters.konlpy import KonlpyTextSplitter # noqa: PLC0415
+ from langchain_text_splitters.nltk import NLTKTextSplitter # noqa: PLC0415
+ from langchain_text_splitters.sentence_transformers import ( # noqa: PLC0415
+ SentenceTransformersTokenTextSplitter,
+ )
+ from langchain_text_splitters.spacy import SpacyTextSplitter # noqa: PLC0415
+
+ constructors: dict[str, Callable[[], TextSplitter]] = {
+ "konlpy": KonlpyTextSplitter,
+ "nltk": NLTKTextSplitter,
+ "spacy": SpacyTextSplitter,
+ "sentence_transformers": SentenceTransformersTokenTextSplitter,
+ }
+
+ # `None` in sys.modules makes both `import x` and `import_module(x)` raise
+ # ImportError, exercising the splitter's missing-dependency branch.
+ monkeypatch.setitem(sys.modules, module_name, None)
+ with pytest.raises(ImportError, match=re.escape(expected_message)):
+ constructors[module_name]()
+
+
+ "class_name",
+ [
+ "KonlpyTextSplitter",
+ "NLTKTextSplitter",
+ "SpacyTextSplitter",
+ "SentenceTransformersTokenTextSplitter",
+ ],
+)
+def test_lazy_getattr_resolves(class_name: str) -> None:
+ """`__getattr__` resolves lazy splitter classes from the package namespace."""
+ import langchain_text_splitters as lts # noqa: PLC0415
+
+ try:
+ cls = getattr(lts, class_name)
+ except ImportError:
+ pytest.skip(f"Optional dependency for {class_name} not installed")
+ assert isinstance(cls, type), f"{class_name} should be a class, got {type(cls)}"
+
+
+def test_lazy_getattr_raises_for_unknown() -> None:
+ """Accessing an unknown attribute raises `AttributeError`."""
+ import langchain_text_splitters as lts # noqa: PLC0415
+
+ with pytest.raises(AttributeError, match="no_such_thing"):
+ _ = lts.no_such_thing # type: ignore[attr-defined]
+
+
+def test_lightweight_splitters_remain_eagerly_accessible() -> None:
+ """Lightweight splitters are still directly importable from the package."""
+ import langchain_text_splitters as lts # noqa: PLC0415
+
+ assert issubclass(lts.RecursiveCharacterTextSplitter, lts.TextSplitter)
+ assert issubclass(lts.CharacterTextSplitter, lts.TextSplitter)
+
+
def test_character_text_splitter() -> None:
"""Test splitting by character count."""
text = "foo bar baz 123"