Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Integrate LEMUR late pooling

morgan-codedd8fa848f73dc
diff --git a/src/python/txtai/models/pooling/late.py b/src/python/txtai/models/pooling/late.py
index c46d7fa6..1bb94d1c 100644
--- a/src/python/txtai/models/pooling/late.py
+++ b/src/python/txtai/models/pooling/late.py
@@ -3,6 +3,7 @@ Late module
"""
from .base import Pooling
+from .lemur import Lemur
from .muvera import Muvera
# Core library imports
@@ -24,11 +25,15 @@ class LatePooling(Pooling):
# Check if fixed dimensional encoder is enabled
modelargs = modelargs.copy() if modelargs else {}
muvera = modelargs.pop("muvera", {})
- self.encoder = Muvera(**muvera) if muvera is not None else None
+ lemur = modelargs.pop("lemur", None)
# Call parent initialization
super().__init__(path, device, tokenizer, maxlength, loadprompts, modelargs)
+ # Create fixed dimensional encoder
+ self.encoder = Lemur(device=self.device, **lemur) if lemur is not None else Muvera(**muvera) if muvera is not None else None
+ self.lengths = None
+
# Get linear weights paths
config = self.load(path, "modules.json")
if config:
@@ -55,6 +60,10 @@ class LatePooling(Pooling):
Late pooled embeddings using output token embeddings (i.e. last hidden state)
"""
+ # Track true token counts for encoders that can't consume padding
+ if getattr(self.encoder, "unpadded", False):
+ self.lengths.extend(inputs["attention_mask"].sum(dim=1).cpu().tolist())
+
# Run through transformers model
tokens = super().forward(**inputs)
@@ -70,6 +79,10 @@ class LatePooling(Pooling):
category: embeddings category (query or data)
"""
+ # Reset true token counts for this encoding pass
+ if getattr(self.encoder, "unpadded", False):
+ self.lengths = []
+
results = []
# Apply prefix
@@ -98,6 +111,16 @@ class LatePooling(Pooling):
normalized results with padding
"""
+ # LEMUR operates on normalized true token rows before batch padding
+ if getattr(self.encoder, "unpadded", False):
+ data = []
+ for vectors, length in zip(results, self.lengths):
+ vectors = vectors[:length]
+ vectors /= np.linalg.norm(vectors, axis=1)[:, np.newaxis]
+ data.append(vectors)
+
+ return self.encoder(data, category)
+
length = 0
for vectors in results:
# Get max length
diff --git a/src/python/txtai/pipeline/text/lateencoder.py b/src/python/txtai/pipeline/text/lateencoder.py
index 93310077..e205cf7e 100644
--- a/src/python/txtai/pipeline/text/lateencoder.py
+++ b/src/python/txtai/pipeline/text/lateencoder.py
@@ -22,6 +22,9 @@ class LateEncoder(Pipeline):
# Get device
self.device = Models.device(Models.deviceid(kwargs.get("gpu", True)))
+ # Disable fixed dimensional encoders to preserve raw multi-vectors
+ encoders = {"muvera": None, "lemur": None}
+
# Load model
self.model = PoolingFactory.create(
{
@@ -30,7 +33,7 @@ class LateEncoder(Pipeline):
"device": self.device,
"tokenizer": kwargs.get("tokenizer"),
"maxlength": kwargs.get("maxlength"),
- "modelargs": {**kwargs.get("vectors", {}), **{"muvera": None}},
+ "modelargs": {**kwargs.get("vectors", {}), **encoders},
}
)