The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix huggingface tokenizer default length function (#30185)
keshavshrikante8be3cca5c2d
diff --git a/libs/text-splitters/langchain_text_splitters/base.py b/libs/text-splitters/langchain_text_splitters/base.pyindex 10dd6903ba..c94c171c3c 100644--- a/libs/text-splitters/langchain_text_splitters/base.py+++ b/libs/text-splitters/langchain_text_splitters/base.py@@ -158,7 +158,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):)def _huggingface_tokenizer_length(text: str) -> int:- return len(tokenizer.encode(text))+ return len(tokenizer.tokenize(text))except ImportError:raise ValueError(