Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add support for last token pooling, closes #1072

davidmezzetti04153bd7378c
diff --git a/docs/embeddings/configuration/vectors.md b/docs/embeddings/configuration/vectors.md
index 8e2e0ec9..da13dd88 100644
--- a/docs/embeddings/configuration/vectors.md
+++ b/docs/embeddings/configuration/vectors.md
@@ -25,8 +25,8 @@ Embeddings method to use. If the method is not provided, it is inferred using th
Builds embeddings using a transformers model. While this can be any transformers model, it works best with
[models trained](https://huggingface.co/models?pipeline_tag=sentence-similarity) to build embeddings.
-`mean`, `cls` and `late` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method
-from `transformers` to `meanpooling`, `clspooling` or `latepooling` respectively.
+`mean`, `cls`, `last` and `late` pooling are supported and automatically inferred from the model. The pooling method can be overwritten by changing the method
+from `transformers` to `meanpooling`, `clspooling`, `lastpooling` or `latepooling` respectively.
Setting `maxlength` to `True` enables truncating inputs to the `max_seq_length`. Setting `maxlength` to an integer will truncate inputs to that value. When omitted (default), the `maxlength` will be set to either the model or tokenizer maxlength.
diff --git a/src/python/txtai/models/pooling/__init__.py b/src/python/txtai/models/pooling/__init__.py
index 9ba930e5..36fbab81 100644
--- a/src/python/txtai/models/pooling/__init__.py
+++ b/src/python/txtai/models/pooling/__init__.py
@@ -5,5 +5,6 @@ Pooling imports
from .base import Pooling
from .cls import ClsPooling
from .factory import PoolingFactory
+from .last import LastPooling
from .late import LatePooling
from .mean import MeanPooling
diff --git a/src/python/txtai/models/pooling/factory.py b/src/python/txtai/models/pooling/factory.py
index 172fef4b..15509908 100644
--- a/src/python/txtai/models/pooling/factory.py
+++ b/src/python/txtai/models/pooling/factory.py
@@ -10,6 +10,7 @@ from transformers.utils import cached_file
from .base import Pooling
from .cls import ClsPooling
+from .last import LastPooling
from .late import LatePooling
from .mean import MeanPooling
@@ -44,13 +45,17 @@ class PoolingFactory:
return Pooling(path, device, tokenizer, maxlength, loadprompts, modelargs)
# Derive pooling method if it's not specified and path is a string
- if (not method or method not in ("clspooling", "meanpooling", "latepooling")) and isinstance(path, str):
+ if (not method or method not in ("clspooling", "meanpooling", "lastpooling", "latepooling")) and isinstance(path, str):
method = PoolingFactory.method(path)
# Check for cls pooling
if method == "clspooling":
return ClsPooling(path, device, tokenizer, maxlength, loadprompts, modelargs)
+ # Check for last pooling
+ if method == "lastpooling":
+ return LastPooling(path, device, tokenizer, maxlength, loadprompts, modelargs)
+
# Check for late pooling
if method == "latepooling":
return LatePooling(path, device, tokenizer, maxlength, loadprompts, modelargs)
@@ -77,9 +82,13 @@ class PoolingFactory:
config = PoolingFactory.load(path, "1_Pooling/config.json")
# Set to CLS pooling if it's enabled and mean pooling is disabled
- if config and config["pooling_mode_cls_token"] and not config["pooling_mode_mean_tokens"]:
+ if config and config.get("pooling_mode_cls_token") and not config["pooling_mode_mean_tokens"]:
method = "clspooling"
+ # Set to last token pooling if it's enabled and mean pooling is disabled
+ if config and config.get("pooling_mode_lasttoken") and not config["pooling_mode_mean_tokens"]:
+ method = "lastpooling"
+
# Check for late interaction pooling
if not config:
# Load 1_Dense/config.json
diff --git a/src/python/txtai/models/pooling/last.py b/src/python/txtai/models/pooling/last.py
new file mode 100644
index 00000000..3ee08a30
--- /dev/null
+++ b/src/python/txtai/models/pooling/last.py
@@ -0,0 +1,49 @@
+"""
+Last module
+"""
+
+import torch
+
+from .base import Pooling
+
+
+class LastPooling(Pooling):
+ """
+ Builds last token pooled vectors usings outputs from a transformers model.
+ """
+
+ def forward(self, **inputs):
+ """
+ Runs last pooling on token embeddings taking the input mask into account.
+
+ Args:
+ inputs: model inputs
+
+ Returns:
+ last pooled embeddings using output token embeddings (i.e. last hidden state)
+ """
+
+ # Run through transformers model
+ tokens = super().forward(**inputs)
+ mask = inputs["attention_mask"]
+
+ # Last pooling logic from Sentence Transformers
+ _, sequence, dimensions = tokens.shape
+
+ # Avoid tracing the argmax with int64 input that can not be handled by ONNX Runtime
+ mask = mask.to(torch.int32) if torch.jit.is_tracing() else mask
+
+ # Use flip and max() to get the last index of 1 in the attention mask
+ values, indices = mask.flip(1).max(1)
+ indices = torch.where(values == 0, sequence - 1, indices)
+ gather = sequence - indices - 1
+
+ # Turn indices from shape [bs] --> [bs, 1, hidden_dim]
+ gather = gather.unsqueeze(-1).repeat(1, dimensions)
+ gather = gather.unsqueeze(1)
+
+ # Expand mask to ignore 0 index attention masks
+ mask = mask.unsqueeze(-1).expand(tokens.size()).to(tokens.dtype)
+
+ # Return last pooled embeddings
+ return torch.gather(tokens * mask, 1, gather).squeeze(dim=1)
diff --git a/test/python/testmodels/testpooling.py b/test/python/testmodels/testpooling.py
index 4c9b43e4..4dfac106 100644
--- a/test/python/testmodels/testpooling.py
+++ b/test/python/testmodels/testpooling.py
@@ -4,7 +4,7 @@ Pooling module tests
import unittest
-from txtai.models import Models, ClsPooling, MeanPooling, PoolingFactory
+from txtai.models import Models, ClsPooling, LastPooling, MeanPooling, PoolingFactory
class TestPooling(unittest.TestCase):
@@ -36,6 +36,21 @@ class TestPooling(unittest.TestCase):
# Test CLS pooling encoding
self.assertEqual(pooling.encode(["test"])[0].shape, (768,))
+ def testLast(self):
+ """
+ Test last pooling
+ """
+
+ # Test last pooling
+ pooling = PoolingFactory.create({"path": "neuml/bert-tiny-sts-last-pooling", "device": self.device})
+ self.assertEqual(type(pooling), LastPooling)
+
+ pooling = PoolingFactory.create({"method": "lastpooling", "path": "sentence-transformers/nli-mpnet-base-v2", "device": self.device})
+ self.assertEqual(type(pooling), LastPooling)
+
+ # Test last pooling encoding
+ self.assertEqual(pooling.encode(["test"])[0].shape, (768,))
+
def testLength(self):
"""
Test pooling with max_seq_length