💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Synchronize truncation/max length logic between embeddings and pipelines. Addresses #74 and #79
davidmezzettib2adcb9d6d24
diff --git a/src/python/txtai/models/__init__.py b/src/python/txtai/models/__init__.pynew file mode 100644index 00000000..eed0de22--- /dev/null+++ b/src/python/txtai/models/__init__.py@@ -0,0 +1,5 @@+"""+Models imports+"""++from .models import Modelsdiff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.pynew file mode 100644index 00000000..038872c2--- /dev/null+++ b/src/python/txtai/models/models.py@@ -0,0 +1,24 @@+"""+Models module+"""+++class Models:+ """+ Utility methods for working with machine learning models+ """++ @staticmethod+ def checklength(model, tokenizer):+ """+ Checks the length for a Hugging Face Transformers tokenizer using a Hugging Face Transformers model. Copies the+ max_position_embeddings parameter if the tokenizer has no max_length set. This helps with backwards compatibility+ with older tokenizers.++ Args:+ model: Transformers model+ tokenizer: Transformers tokenizer+ """++ if hasattr(model, "config") and hasattr(model.config, "max_position_embeddings") and tokenizer.model_max_length == int(1e30):+ tokenizer.model_max_length = model.config.max_position_embeddingsdiff --git a/src/python/txtai/pipeline/hfmodel.py b/src/python/txtai/pipeline/hfmodel.pyindex 50f01d31..b7eacb3c 100644--- a/src/python/txtai/pipeline/hfmodel.py+++ b/src/python/txtai/pipeline/hfmodel.py@@ -1,5 +1,5 @@"""-Hugging Face model wrapper module+Hugging Face Transformers model wrapper module"""from .tensors import Tensors@@ -7,7 +7,7 @@ from .tensors import Tensorsclass HFModel(Tensors):"""- Pipeline backed by a Hugging Face model.+ Pipeline backed by a Hugging Face Transformers model."""def __init__(self, path=None, quantize=False, gpu=False, batch=64):diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.pyindex 1fa1631a..cc7a375c 100644--- a/src/python/txtai/pipeline/hfpipeline.py+++ b/src/python/txtai/pipeline/hfpipeline.py@@ -1,15 +1,16 @@"""-Hugging Face pipeline wrapper module+Hugging Face Transformers pipeline wrapper module"""from transformers import pipeline+from ..models import Modelsfrom .tensors import Tensorsclass HFPipeline(Tensors):"""- Light wrapper around Hugging Face's pipeline component for selected tasks. Adds support for model+ Light wrapper around Hugging Face Transformers pipeline component for selected tasks. Adds support for modelquantization and minor interface changes."""@@ -40,3 +41,6 @@ class HFPipeline(Tensors):if deviceid == -1 and quantize:# pylint: disable=E1101self.pipeline.model = self.quantize(self.pipeline.model)++ # Detect unbounded tokenizer typically found in older models+ Models.checklength(self.pipeline.model, self.pipeline.tokenizer)diff --git a/src/python/txtai/pipeline/labels.py b/src/python/txtai/pipeline/labels.pyindex fe0743c1..79dc6180 100644--- a/src/python/txtai/pipeline/labels.py+++ b/src/python/txtai/pipeline/labels.py@@ -31,7 +31,7 @@ class Labels(HFPipeline):"""# Run ZSL pipeline- results = self.pipeline(text, labels, multi_label=multilabel)+ results = self.pipeline(text, labels, multi_label=multilabel, truncation=True)# Convert results to a list if necessaryif not isinstance(results, list):diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.pyindex 9ba28989..2f3f9fb2 100644--- a/src/python/txtai/vectors/transformers.py+++ b/src/python/txtai/vectors/transformers.py@@ -11,6 +11,7 @@ from sentence_transformers.models import Pooling, Transformerfrom ..pipeline.tokenizer import Tokenizerfrom .base import Vectors+from ..models import Modelsclass TransformersVectors(Vectors):@@ -23,9 +24,12 @@ class TransformersVectors(Vectors):# Download model from the model hub (default)if modelhub:- model = Transformer(path, max_seq_length=self.config.get("maxlength"))+ model = Transformer(path)pooling = Pooling(model.get_word_embedding_dimension())+ # Detect unbounded tokenizer typically found in older models+ Models.checklength(model.auto_model, model.tokenizer)+return SentenceTransformer(modules=[model, pooling])# Download model directly from sentence transformers if model hub disableddiff --git a/test/python/testlabels.py b/test/python/testlabels.pyindex c204f928..b7683f91 100644--- a/test/python/testlabels.py+++ b/test/python/testlabels.py@@ -60,3 +60,11 @@ class TestLabels(unittest.TestCase):results = [r[0][0] for r in self.similarity(["feel good story", "climate change"], self.data)]self.assertEqual(results, [4, 1])++ def testSimilarityLong(self):+ """+ Test similarity with long text+ """++ uid = self.similarity("other", ["Very long text " * 1000, "other text"])[0][0]+ self.assertEqual(uid, 1)