Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Support max_seq_length parameter with model pooling, closes #746

davidmezzettief61741fcca5
diff --git a/docs/embeddings/configuration/vectors.md b/docs/embeddings/configuration/vectors.md
index d425039d..45f2a17d 100644
--- a/docs/embeddings/configuration/vectors.md
+++ b/docs/embeddings/configuration/vectors.md
@@ -34,6 +34,8 @@ Builds embeddings using a transformers model. While this can be any transformers
Both `mean` and `cls` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method
from `transformers` to `meanpooling` or `clspooling` respectively.
+Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.
+
### sentence-transformers
Same as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.
diff --git a/src/python/txtai/models/pooling/factory.py b/src/python/txtai/models/pooling/factory.py
index e02734e4..7d875801 100644
--- a/src/python/txtai/models/pooling/factory.py
+++ b/src/python/txtai/models/pooling/factory.py
@@ -31,12 +31,16 @@ class PoolingFactory:
"""
# Unpack parameters
- path, device, tokenizer, method = config["path"], config["device"], config.get("tokenizer"), config.get("method")
- modelargs = config.get("modelargs")
+ method, path, device, tokenizer, maxlength, modelargs = [
+ config.get(x) for x in ["method", "path", "device", "tokenizer", "maxlength", "modelargs"]
+ ]
+
+ # Derive maxlength, if applicable
+ maxlength = PoolingFactory.maxlength(path) if isinstance(maxlength, bool) and maxlength else maxlength
# Default pooling returns hidden state
if isinstance(path, bytes) or (isinstance(path, str) and os.path.isfile(path)) or method == "pooling":
- return Pooling(path, device=device, tokenizer=tokenizer, modelargs=modelargs)
+ return Pooling(path, device, tokenizer, maxlength, modelargs)
# Derive pooling method if it's not specified, path is a string and path is not a local path
if (not method or method not in ("clspooling", "meanpooling")) and (isinstance(path, str) and not os.path.exists(path)):
@@ -44,10 +48,10 @@ class PoolingFactory:
# Check for cls pooling
if method == "clspooling":
- return ClsPooling(path, device, tokenizer, modelargs=modelargs)
+ return ClsPooling(path, device, tokenizer, maxlength, modelargs)
# Default to mean pooling
- return MeanPooling(path, device, tokenizer, modelargs=modelargs)
+ return MeanPooling(path, device, tokenizer, maxlength, modelargs)
@staticmethod
def method(path):
@@ -64,19 +68,60 @@ class PoolingFactory:
# Default method
method = "meanpooling"
- # Load 1_Pooling/config.json file and read, if available
try:
- path = hf_hub_download(repo_id=path, filename="1_Pooling/config.json")
-
- with open(path, encoding="utf-8") as f:
- config = json.load(f)
+ # Load 1_Pooling/config.json file
+ config = PoolingFactory.load(path, "1_Pooling/config.json")
- # Set to CLS pooling if it's enabled and mean pooling is disabled
- if config["pooling_mode_cls_token"] and not config["pooling_mode_mean_tokens"]:
- method = "clspooling"
+ # Set to CLS pooling if it's enabled and mean pooling is disabled
+ if config["pooling_mode_cls_token"] and not config["pooling_mode_mean_tokens"]:
+ method = "clspooling"
# Ignore this error
except EntryNotFoundError:
pass
return method
+
+ @staticmethod
+ def maxlength(path):
+ """
+ Reads the max_seq_length parameter from sentence transformers config.
+
+ Args:
+ path: model path
+
+ Returns:
+ max sequence length
+ """
+
+ # Default length is unset
+ maxlength = None
+
+ # Read max_seq_length from sentence_bert_config.json
+ try:
+ config = PoolingFactory.load(path, "sentence_bert_config.json")
+ maxlength = config.get("max_seq_length")
+
+ # Ignore this error
+ except EntryNotFoundError:
+ pass
+
+ return maxlength
+
+ @staticmethod
+ def load(path, name):
+ """
+ Loads a JSON config file from the Hugging Face Hub.
+
+ Args:
+ path: model path
+ name: file to load
+
+ Returns:
+ config
+ """
+
+ # Download file and parse JSON
+ path = hf_hub_download(repo_id=path, filename=name)
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
diff --git a/src/python/txtai/vectors/huggingface.py b/src/python/txtai/vectors/huggingface.py
index 0d850cd6..e4582dad 100644
--- a/src/python/txtai/vectors/huggingface.py
+++ b/src/python/txtai/vectors/huggingface.py
@@ -48,7 +48,14 @@ class HFVectors(Vectors):
# Build embeddings with transformers (default)
if transformers:
return PoolingFactory.create(
- {"path": path, "device": deviceid, "tokenizer": self.config.get("tokenizer"), "method": method, "modelargs": modelargs}
+ {
+ "method": method,
+ "path": path,
+ "device": deviceid,
+ "tokenizer": self.config.get("tokenizer"),
+ "maxlength": self.config.get("maxlength"),
+ "modelargs": modelargs,
+ }
)
# Otherwise, use sentence-transformers library
diff --git a/test/python/testmodels/__init__.py b/test/python/testmodels/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/test/python/testmodels.py b/test/python/testmodels/testmodels.py
similarity index 51%
rename from test/python/testmodels.py
rename to test/python/testmodels/testmodels.py
index 44ab66c2..e04d3638 100644
--- a/test/python/testmodels.py
+++ b/test/python/testmodels/testmodels.py
@@ -8,7 +8,7 @@ from unittest.mock import patch
import torch
-from txtai.models import Models, ClsPooling, MeanPooling, PoolingFactory
+from txtai.models import Models
class TestModels(unittest.TestCase):
@@ -46,28 +46,3 @@ class TestModels(unittest.TestCase):
# pylint: disable=E1101
self.assertEqual(Models.device("cpu"), torch.device("cpu"))
self.assertEqual(Models.device(torch.device("cpu")), torch.device("cpu"))
-
- def testPooling(self):
- """
- Tests pooling methods
- """
-
- # Device id
- device = Models.deviceid(True)
-
- # Test mean pooling
- pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": device})
- self.assertEqual(type(pooling), MeanPooling)
-
- pooling = PoolingFactory.create({"method": "meanpooling", "path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": device})
- self.assertEqual(type(pooling), MeanPooling)
-
- # Test CLS pooling
- pooling = PoolingFactory.create({"path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": device})
- self.assertEqual(type(pooling), ClsPooling)
-
- pooling = PoolingFactory.create({"method": "clspooling", "path": "sentence-transformers/nli-mpnet-base-v2", "device": device})
- self.assertEqual(type(pooling), ClsPooling)
-
- # Test CLS pooling encoding
- self.assertEqual(pooling.encode(["test"])[0].shape, (768,))
diff --git a/test/python/testmodels/testpooling.py b/test/python/testmodels/testpooling.py
new file mode 100644
index 00000000..9cf6bb72
--- /dev/null
+++ b/test/python/testmodels/testpooling.py
@@ -0,0 +1,72 @@
+"""
+Pooling module tests
+"""
+
+import unittest
+
+from txtai.models import Models, ClsPooling, MeanPooling, PoolingFactory
+
+
+class TestPooling(unittest.TestCase):
+ """
+ Pooling tests.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Initialize device
+ """
+
+ # Device id
+ cls.device = Models.deviceid(True)
+
+ def testCLS(self):
+ """
+ Test CLS pooling
+ """
+
+ # Test CLS pooling
+ pooling = PoolingFactory.create({"path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": self.device})
+ self.assertEqual(type(pooling), ClsPooling)
+
+ pooling = PoolingFactory.create({"method": "clspooling", "path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})
+ self.assertEqual(type(pooling), ClsPooling)
+
+ # Test CLS pooling encoding
+ self.assertEqual(pooling.encode(["test"])[0].shape, (768,))
+
+ def testLength(self):
+ """
+ Test pooling with max_seq_length
+ """
+
+ # Test reading max_seq_length parmaeter
+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device, "maxlength": True})
+ self.assertEqual(pooling.maxlength, 75)
+
+ # Test specified maxlength
+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device, "maxlength": 256})
+ self.assertEqual(pooling.maxlength, 256)
+
+ # Test max_seq_length is ignored when parameter is omitted
+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})
+ self.assertEqual(pooling.maxlength, 512)
+
+ # Test maxlength when max_seq_length not present
+ pooling = PoolingFactory.create({"path": "hf-internal-testing/tiny-random-gpt2", "device": self.device, "maxlength": True})
+ self.assertEqual(pooling.maxlength, 1024)
+
+ def testMean(self):
+ """
+ Tests mean pooling
+ """
+
+ # Test mean pooling
+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})
+ self.assertEqual(type(pooling), MeanPooling)
+
+ pooling = PoolingFactory.create(
+ {"method": "meanpooling", "path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": self.device}
+ )
+ self.assertEqual(type(pooling), MeanPooling)