Skip to content

The agent engineering platform.

fix huggingface tokenizer default length function (#30185)

keshavshrikante8be3cca5c2d
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.py
index 10dd6903ba..c94c171c3c 100644
--- a/libs/text-splitters/langchain_text_splitters/base.py
+++ b/libs/text-splitters/langchain_text_splitters/base.py
@@ -158,7 +158,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
)
def _huggingface_tokenizer_length(text: str) -> int:
- return len(tokenizer.encode(text))
+ return len(tokenizer.tokenize(text))
except ImportError:
raise ValueError(