Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Synchronize truncation/max length logic between embeddings and pipelines. Addresses #74 and #79

davidmezzettib2adcb9d6d24
diff --git a/src/python/txtai/models/__init__.py b/src/python/txtai/models/__init__.py
new file mode 100644
index 00000000..eed0de22
--- /dev/null
+++ b/src/python/txtai/models/__init__.py
@@ -0,0 +1,5 @@
+"""
+Models imports
+"""
+
+from .models import Models
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.py
new file mode 100644
index 00000000..038872c2
--- /dev/null
+++ b/src/python/txtai/models/models.py
@@ -0,0 +1,24 @@
+"""
+Models module
+"""
+
+
+class Models:
+ """
+ Utility methods for working with machine learning models
+ """
+
+ @staticmethod
+ def checklength(model, tokenizer):
+ """
+ Checks the length for a Hugging Face Transformers tokenizer using a Hugging Face Transformers model. Copies the
+ max_position_embeddings parameter if the tokenizer has no max_length set. This helps with backwards compatibility
+ with older tokenizers.
+
+ Args:
+ model: Transformers model
+ tokenizer: Transformers tokenizer
+ """
+
+ if hasattr(model, "config") and hasattr(model.config, "max_position_embeddings") and tokenizer.model_max_length == int(1e30):
+ tokenizer.model_max_length = model.config.max_position_embeddings
diff --git a/src/python/txtai/pipeline/hfmodel.py b/src/python/txtai/pipeline/hfmodel.py
index 50f01d31..b7eacb3c 100644
--- a/src/python/txtai/pipeline/hfmodel.py
+++ b/src/python/txtai/pipeline/hfmodel.py
@@ -1,5 +1,5 @@
"""
-Hugging Face model wrapper module
+Hugging Face Transformers model wrapper module
"""
from .tensors import Tensors
@@ -7,7 +7,7 @@ from .tensors import Tensors
class HFModel(Tensors):
"""
- Pipeline backed by a Hugging Face model.
+ Pipeline backed by a Hugging Face Transformers model.
"""
def __init__(self, path=None, quantize=False, gpu=False, batch=64):
diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.py
index 1fa1631a..cc7a375c 100644
--- a/src/python/txtai/pipeline/hfpipeline.py
+++ b/src/python/txtai/pipeline/hfpipeline.py
@@ -1,15 +1,16 @@
"""
-Hugging Face pipeline wrapper module
+Hugging Face Transformers pipeline wrapper module
"""
from transformers import pipeline
+from ..models import Models
from .tensors import Tensors
class HFPipeline(Tensors):
"""
- Light wrapper around Hugging Face's pipeline component for selected tasks. Adds support for model
+ Light wrapper around Hugging Face Transformers pipeline component for selected tasks. Adds support for model
quantization and minor interface changes.
"""
@@ -40,3 +41,6 @@ class HFPipeline(Tensors):
if deviceid == -1 and quantize:
# pylint: disable=E1101
self.pipeline.model = self.quantize(self.pipeline.model)
+
+ # Detect unbounded tokenizer typically found in older models
+ Models.checklength(self.pipeline.model, self.pipeline.tokenizer)
diff --git a/src/python/txtai/pipeline/labels.py b/src/python/txtai/pipeline/labels.py
index fe0743c1..79dc6180 100644
--- a/src/python/txtai/pipeline/labels.py
+++ b/src/python/txtai/pipeline/labels.py
@@ -31,7 +31,7 @@ class Labels(HFPipeline):
"""
# Run ZSL pipeline
- results = self.pipeline(text, labels, multi_label=multilabel)
+ results = self.pipeline(text, labels, multi_label=multilabel, truncation=True)
# Convert results to a list if necessary
if not isinstance(results, list):
diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.py
index 9ba28989..2f3f9fb2 100644
--- a/src/python/txtai/vectors/transformers.py
+++ b/src/python/txtai/vectors/transformers.py
@@ -11,6 +11,7 @@ from sentence_transformers.models import Pooling, Transformer
from ..pipeline.tokenizer import Tokenizer
from .base import Vectors
+from ..models import Models
class TransformersVectors(Vectors):
@@ -23,9 +24,12 @@ class TransformersVectors(Vectors):
# Download model from the model hub (default)
if modelhub:
- model = Transformer(path, max_seq_length=self.config.get("maxlength"))
+ model = Transformer(path)
pooling = Pooling(model.get_word_embedding_dimension())
+ # Detect unbounded tokenizer typically found in older models
+ Models.checklength(model.auto_model, model.tokenizer)
+
return SentenceTransformer(modules=[model, pooling])
# Download model directly from sentence transformers if model hub disabled
diff --git a/test/python/testlabels.py b/test/python/testlabels.py
index c204f928..b7683f91 100644
--- a/test/python/testlabels.py
+++ b/test/python/testlabels.py
@@ -60,3 +60,11 @@ class TestLabels(unittest.TestCase):
results = [r[0][0] for r in self.similarity(["feel good story", "climate change"], self.data)]
self.assertEqual(results, [4, 1])
+
+ def testSimilarityLong(self):
+ """
+ Test similarity with long text
+ """
+
+ uid = self.similarity("other", ["Very long text " * 1000, "other text"])[0][0]
+ self.assertEqual(uid, 1)