💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Support max_seq_length parameter with model pooling, closes #746
davidmezzettief61741fcca5
diff --git a/docs/embeddings/configuration/vectors.md b/docs/embeddings/configuration/vectors.mdindex d425039d..45f2a17d 100644--- a/docs/embeddings/configuration/vectors.md+++ b/docs/embeddings/configuration/vectors.md@@ -34,6 +34,8 @@ Builds embeddings using a transformers model. While this can be any transformersBoth `mean` and `cls` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the methodfrom `transformers` to `meanpooling` or `clspooling` respectively.+Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.+### sentence-transformersSame as transformers but loads models with the [sentence-transformers](https://github.com/UKPLab/sentence-transformers) library.diff --git a/src/python/txtai/models/pooling/factory.py b/src/python/txtai/models/pooling/factory.pyindex e02734e4..7d875801 100644--- a/src/python/txtai/models/pooling/factory.py+++ b/src/python/txtai/models/pooling/factory.py@@ -31,12 +31,16 @@ class PoolingFactory:"""# Unpack parameters- path, device, tokenizer, method = config["path"], config["device"], config.get("tokenizer"), config.get("method")- modelargs = config.get("modelargs")+ method, path, device, tokenizer, maxlength, modelargs = [+ config.get(x) for x in ["method", "path", "device", "tokenizer", "maxlength", "modelargs"]+ ]++ # Derive maxlength, if applicable+ maxlength = PoolingFactory.maxlength(path) if isinstance(maxlength, bool) and maxlength else maxlength# Default pooling returns hidden stateif isinstance(path, bytes) or (isinstance(path, str) and os.path.isfile(path)) or method == "pooling":- return Pooling(path, device=device, tokenizer=tokenizer, modelargs=modelargs)+ return Pooling(path, device, tokenizer, maxlength, modelargs)# Derive pooling method if it's not specified, path is a string and path is not a local pathif (not method or method not in ("clspooling", "meanpooling")) and (isinstance(path, str) and not os.path.exists(path)):@@ -44,10 +48,10 @@ class PoolingFactory:# Check for cls poolingif method == "clspooling":- return ClsPooling(path, device, tokenizer, modelargs=modelargs)+ return ClsPooling(path, device, tokenizer, maxlength, modelargs)# Default to mean pooling- return MeanPooling(path, device, tokenizer, modelargs=modelargs)+ return MeanPooling(path, device, tokenizer, maxlength, modelargs)@staticmethoddef method(path):@@ -64,19 +68,60 @@ class PoolingFactory:# Default methodmethod = "meanpooling"- # Load 1_Pooling/config.json file and read, if availabletry:- path = hf_hub_download(repo_id=path, filename="1_Pooling/config.json")-- with open(path, encoding="utf-8") as f:- config = json.load(f)+ # Load 1_Pooling/config.json file+ config = PoolingFactory.load(path, "1_Pooling/config.json")- # Set to CLS pooling if it's enabled and mean pooling is disabled- if config["pooling_mode_cls_token"] and not config["pooling_mode_mean_tokens"]:- method = "clspooling"+ # Set to CLS pooling if it's enabled and mean pooling is disabled+ if config["pooling_mode_cls_token"] and not config["pooling_mode_mean_tokens"]:+ method = "clspooling"# Ignore this errorexcept EntryNotFoundError:passreturn method++ @staticmethod+ def maxlength(path):+ """+ Reads the max_seq_length parameter from sentence transformers config.++ Args:+ path: model path++ Returns:+ max sequence length+ """++ # Default length is unset+ maxlength = None++ # Read max_seq_length from sentence_bert_config.json+ try:+ config = PoolingFactory.load(path, "sentence_bert_config.json")+ maxlength = config.get("max_seq_length")++ # Ignore this error+ except EntryNotFoundError:+ pass++ return maxlength++ @staticmethod+ def load(path, name):+ """+ Loads a JSON config file from the Hugging Face Hub.++ Args:+ path: model path+ name: file to load++ Returns:+ config+ """++ # Download file and parse JSON+ path = hf_hub_download(repo_id=path, filename=name)+ with open(path, encoding="utf-8") as f:+ return json.load(f)diff --git a/src/python/txtai/vectors/huggingface.py b/src/python/txtai/vectors/huggingface.pyindex 0d850cd6..e4582dad 100644--- a/src/python/txtai/vectors/huggingface.py+++ b/src/python/txtai/vectors/huggingface.py@@ -48,7 +48,14 @@ class HFVectors(Vectors):# Build embeddings with transformers (default)if transformers:return PoolingFactory.create(- {"path": path, "device": deviceid, "tokenizer": self.config.get("tokenizer"), "method": method, "modelargs": modelargs}+ {+ "method": method,+ "path": path,+ "device": deviceid,+ "tokenizer": self.config.get("tokenizer"),+ "maxlength": self.config.get("maxlength"),+ "modelargs": modelargs,+ })# Otherwise, use sentence-transformers librarydiff --git a/test/python/testmodels/__init__.py b/test/python/testmodels/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/test/python/testmodels.py b/test/python/testmodels/testmodels.pysimilarity index 51%rename from test/python/testmodels.pyrename to test/python/testmodels/testmodels.pyindex 44ab66c2..e04d3638 100644--- a/test/python/testmodels.py+++ b/test/python/testmodels/testmodels.py@@ -8,7 +8,7 @@ from unittest.mock import patchimport torch-from txtai.models import Models, ClsPooling, MeanPooling, PoolingFactory+from txtai.models import Modelsclass TestModels(unittest.TestCase):@@ -46,28 +46,3 @@ class TestModels(unittest.TestCase):# pylint: disable=E1101self.assertEqual(Models.device("cpu"), torch.device("cpu"))self.assertEqual(Models.device(torch.device("cpu")), torch.device("cpu"))-- def testPooling(self):- """- Tests pooling methods- """-- # Device id- device = Models.deviceid(True)-- # Test mean pooling- pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": device})- self.assertEqual(type(pooling), MeanPooling)-- pooling = PoolingFactory.create({"method": "meanpooling", "path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": device})- self.assertEqual(type(pooling), MeanPooling)-- # Test CLS pooling- pooling = PoolingFactory.create({"path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": device})- self.assertEqual(type(pooling), ClsPooling)-- pooling = PoolingFactory.create({"method": "clspooling", "path": "sentence-transformers/nli-mpnet-base-v2", "device": device})- self.assertEqual(type(pooling), ClsPooling)-- # Test CLS pooling encoding- self.assertEqual(pooling.encode(["test"])[0].shape, (768,))diff --git a/test/python/testmodels/testpooling.py b/test/python/testmodels/testpooling.pynew file mode 100644index 00000000..9cf6bb72--- /dev/null+++ b/test/python/testmodels/testpooling.py@@ -0,0 +1,72 @@+"""+Pooling module tests+"""++import unittest++from txtai.models import Models, ClsPooling, MeanPooling, PoolingFactory+++class TestPooling(unittest.TestCase):+ """+ Pooling tests.+ """++ @classmethod+ def setUpClass(cls):+ """+ Initialize device+ """++ # Device id+ cls.device = Models.deviceid(True)++ def testCLS(self):+ """+ Test CLS pooling+ """++ # Test CLS pooling+ pooling = PoolingFactory.create({"path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": self.device})+ self.assertEqual(type(pooling), ClsPooling)++ pooling = PoolingFactory.create({"method": "clspooling", "path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})+ self.assertEqual(type(pooling), ClsPooling)++ # Test CLS pooling encoding+ self.assertEqual(pooling.encode(["test"])[0].shape, (768,))++ def testLength(self):+ """+ Test pooling with max_seq_length+ """++ # Test reading max_seq_length parmaeter+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device, "maxlength": True})+ self.assertEqual(pooling.maxlength, 75)++ # Test specified maxlength+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device, "maxlength": 256})+ self.assertEqual(pooling.maxlength, 256)++ # Test max_seq_length is ignored when parameter is omitted+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})+ self.assertEqual(pooling.maxlength, 512)++ # Test maxlength when max_seq_length not present+ pooling = PoolingFactory.create({"path": "hf-internal-testing/tiny-random-gpt2", "device": self.device, "maxlength": True})+ self.assertEqual(pooling.maxlength, 1024)++ def testMean(self):+ """+ Tests mean pooling+ """++ # Test mean pooling+ pooling = PoolingFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})+ self.assertEqual(type(pooling), MeanPooling)++ pooling = PoolingFactory.create(+ {"method": "meanpooling", "path": "flax-sentence-embeddings/multi-qa_v1-MiniLM-L6-cls_dot", "device": self.device}+ )+ self.assertEqual(type(pooling), MeanPooling)