💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add support for last token pooling, closes #1072
davidmezzetti04153bd7378c
diff --git a/docs/embeddings/configuration/vectors.md b/docs/embeddings/configuration/vectors.mdindex 8e2e0ec9..da13dd88 100644--- a/docs/embeddings/configuration/vectors.md+++ b/docs/embeddings/configuration/vectors.md@@ -25,8 +25,8 @@ Embeddings method to use. If the method is not provided, it is inferred using thBuilds embeddings using a transformers model. While this can be any transformers model, it works best with[models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build embeddings.-`mean`, `cls` and `late` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method-from `transformers` to `meanpooling`, `clspooling` or `latepooling` respectively.+`mean`, `cls`, `last` and `late` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method+from `transformers` to `meanpooling`, `clspooling`, `lastpooling` or `latepooling` respectively.Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.diff --git a/src/python/txtai/models/pooling/__init__.py b/src/python/txtai/models/pooling/__init__.pyindex 9ba930e5..36fbab81 100644--- a/src/python/txtai/models/pooling/__init__.py+++ b/src/python/txtai/models/pooling/__init__.py@@ -5,5 +5,6 @@ Pooling importsfrom .base import Poolingfrom .cls import ClsPoolingfrom .factory import PoolingFactory+from .last import LastPoolingfrom .late import LatePoolingfrom .mean import MeanPoolingdiff --git a/src/python/txtai/models/pooling/factory.py b/src/python/txtai/models/pooling/factory.pyindex 172fef4b..15509908 100644--- a/src/python/txtai/models/pooling/factory.py+++ b/src/python/txtai/models/pooling/factory.py@@ -10,6 +10,7 @@ from transformers.utils import cached_filefrom .base import Poolingfrom .cls import ClsPooling+from .last import LastPoolingfrom .late import LatePoolingfrom .mean import MeanPooling@@ -44,13 +45,17 @@ class PoolingFactory:return Pooling(path, device, tokenizer, maxlength, loadprompts, modelargs)# Derive pooling method if it's not specified and path is a string- if (not method or method not in ("clspooling", "meanpooling", "latepooling")) and isinstance(path, str):+ if (not method or method not in ("clspooling", "meanpooling", "lastpooling", "latepooling")) and isinstance(path, str):method = PoolingFactory.method(path)# Check for cls poolingif method == "clspooling":return ClsPooling(path, device, tokenizer, maxlength, loadprompts, modelargs)+ # Check for last pooling+ if method == "lastpooling":+ return LastPooling(path, device, tokenizer, maxlength, loadprompts, modelargs)+# Check for late poolingif method == "latepooling":return LatePooling(path, device, tokenizer, maxlength, loadprompts, modelargs)@@ -77,9 +82,13 @@ class PoolingFactory:config = PoolingFactory.load(path, "1_Pooling/config.json")# Set to CLS pooling if it's enabled and mean pooling is disabled- if config and config["pooling_mode_cls_token"] and not config["pooling_mode_mean_tokens"]:+ if config and config.get("pooling_mode_cls_token") and not config["pooling_mode_mean_tokens"]:method = "clspooling"+ # Set to last token pooling if it's enabled and mean pooling is disabled+ if config and config.get("pooling_mode_lasttoken") and not config["pooling_mode_mean_tokens"]:+ method = "lastpooling"+# Check for late interaction poolingif not config:# Load 1_Dense/config.jsondiff --git a/src/python/txtai/models/pooling/last.py b/src/python/txtai/models/pooling/last.pynew file mode 100644index 00000000..3ee08a30--- /dev/null+++ b/src/python/txtai/models/pooling/last.py@@ -0,0 +1,49 @@+"""+Last module+"""++import torch++from .base import Pooling+++class LastPooling(Pooling):+ """+ Builds last token pooled vectors usings outputs from a transformers model.+ """++ def forward(self, **inputs):+ """+ Runs last pooling on token embeddings taking the input mask into account.++ Args:+ inputs: model inputs++ Returns:+ last pooled embeddings using output token embeddings (i.e. last hidden state)+ """++ # Run through transformers model+ tokens = super().forward(**inputs)+ mask = inputs["attention_mask"]++ # Last pooling logic from Sentence Transformers+ _, sequence, dimensions = tokens.shape++ # Avoid tracing the argmax with int64 input that can not be handled by ONNX Runtime+ mask = mask.to(torch.int32) if torch.jit.is_tracing() else mask++ # Use flip and max() to get the last index of 1 in the attention mask+ values, indices = mask.flip(1).max(1)+ indices = torch.where(values == 0, sequence - 1, indices)+ gather = sequence - indices - 1++ # Turn indices from shape [bs] --> [bs, 1, hidden_dim]+ gather = gather.unsqueeze(-1).repeat(1, dimensions)+ gather = gather.unsqueeze(1)++ # Expand mask to ignore 0 index attention masks+ mask = mask.unsqueeze(-1).expand(tokens.size()).to(tokens.dtype)++ # Return last pooled embeddings+ return torch.gather(tokens * mask, 1, gather).squeeze(dim=1)diff --git a/test/python/testmodels/testpooling.py b/test/python/testmodels/testpooling.pyindex 4c9b43e4..4dfac106 100644--- a/test/python/testmodels/testpooling.py+++ b/test/python/testmodels/testpooling.py@@ -4,7 +4,7 @@ Pooling module testsimport unittest-from txtai.models import Models, ClsPooling, MeanPooling, PoolingFactory+from txtai.models import Models, ClsPooling, LastPooling, MeanPooling, PoolingFactoryclass TestPooling(unittest.TestCase):@@ -36,6 +36,21 @@ class TestPooling(unittest.TestCase):# Test CLS pooling encodingself.assertEqual(pooling.encode(["test"])[0].shape, (768,))+ def testLast(self):+ """+ Test last pooling+ """++ # Test last pooling+ pooling = PoolingFactory.create({"path": "neuml/bert-tiny-sts-last-pooling", "device": self.device})+ self.assertEqual(type(pooling), LastPooling)++ pooling = PoolingFactory.create({"method": "lastpooling", "path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})+ self.assertEqual(type(pooling), LastPooling)++ # Test last pooling encoding+ self.assertEqual(pooling.encode(["test"])[0].shape, (768,))+def testLength(self):"""Test pooling with max_seq_length