Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Check tokenizer attribute exists, closes #204

davidmezzettid5436d2f4fbb
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.py
index 457e3d37..ea0358ea 100644
--- a/src/python/txtai/models/models.py
+++ b/src/python/txtai/models/models.py
@@ -32,7 +32,12 @@ class Models:
if hasattr(config, "config"):
config = config.config
- if hasattr(config, "max_position_embeddings") and tokenizer and tokenizer.model_max_length == int(1e30):
+ if (
+ hasattr(config, "max_position_embeddings")
+ and tokenizer
+ and hasattr(tokenizer, "model_max_length")
+ and tokenizer.model_max_length == int(1e30)
+ ):
tokenizer.model_max_length = config.max_position_embeddings
@staticmethod