💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Integrate LEMUR late pooling
morgan-codedd8fa848f73dc
diff --git a/src/python/txtai/models/pooling/late.py b/src/python/txtai/models/pooling/late.pyindex c46d7fa6..1bb94d1c 100644--- a/src/python/txtai/models/pooling/late.py+++ b/src/python/txtai/models/pooling/late.py@@ -3,6 +3,7 @@ Late module"""from .base import Pooling+from .lemur import Lemurfrom .muvera import Muvera# Core library imports@@ -24,11 +25,15 @@ class LatePooling(Pooling):# Check if fixed dimensional encoder is enabledmodelargs = modelargs.copy() if modelargs else {}muvera = modelargs.pop("muvera", {})- self.encoder = Muvera(**muvera) if muvera is not None else None+ lemur = modelargs.pop("lemur", None)# Call parent initializationsuper().__init__(path, device, tokenizer, maxlength, loadprompts, modelargs)+ # Create fixed dimensional encoder+ self.encoder = Lemur(device=self.device, **lemur) if lemur is not None else Muvera(**muvera) if muvera is not None else None+ self.lengths = None+# Get linear weights pathsconfig = self.load(path, "modules.json")if config:@@ -55,6 +60,10 @@ class LatePooling(Pooling):Late pooled embeddings using output token embeddings (i.e. last hidden state)"""+ # Track true token counts for encoders that can't consume padding+ if getattr(self.encoder, "unpadded", False):+ self.lengths.extend(inputs["attention_mask"].sum(dim=1).cpu().tolist())+# Run through transformers modeltokens = super().forward(**inputs)@@ -70,6 +79,10 @@ class LatePooling(Pooling):category: embeddings category (query or data)"""+ # Reset true token counts for this encoding pass+ if getattr(self.encoder, "unpadded", False):+ self.lengths = []+results = []# Apply prefix@@ -98,6 +111,16 @@ class LatePooling(Pooling):normalized results with padding"""+ # LEMUR operates on normalized true token rows before batch padding+ if getattr(self.encoder, "unpadded", False):+ data = []+ for vectors, length in zip(results, self.lengths):+ vectors = vectors[:length]+ vectors /= np.linalg.norm(vectors, axis=1)[:, np.newaxis]+ data.append(vectors)++ return self.encoder(data, category)+length = 0for vectors in results:# Get max lengthdiff --git a/src/python/txtai/pipeline/text/lateencoder.py b/src/python/txtai/pipeline/text/lateencoder.pyindex 93310077..e205cf7e 100644--- a/src/python/txtai/pipeline/text/lateencoder.py+++ b/src/python/txtai/pipeline/text/lateencoder.py@@ -22,6 +22,9 @@ class LateEncoder(Pipeline):# Get deviceself.device = Models.device(Models.deviceid(kwargs.get("gpu", True)))+ # Disable fixed dimensional encoders to preserve raw multi-vectors+ encoders = {"muvera": None, "lemur": None}+# Load modelself.model = PoolingFactory.create({@@ -30,7 +33,7 @@ class LateEncoder(Pipeline):"device": self.device,"tokenizer": kwargs.get("tokenizer"),"maxlength": kwargs.get("maxlength"),- "modelargs": {**kwargs.get("vectors", {}), **{"muvera": None}},+ "modelargs": {**kwargs.get("vectors", {}), **encoders},})