Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Cache document token vectors in Reranker (#1321)

Morgan Carr893d0424bec9
diff --git a/src/python/txtai/pipeline/text/reranker.py b/src/python/txtai/pipeline/text/reranker.py
index b0ffd9f8..71d2253b 100644
--- a/src/python/txtai/pipeline/text/reranker.py
+++ b/src/python/txtai/pipeline/text/reranker.py
@@ -2,7 +2,18 @@
Reranker module
"""
+import hashlib
+import json
+import logging
+from collections import OrderedDict
+from threading import RLock
+
from ..base import Pipeline
+from ...util import Library
+
+library = Library()
+np, torch, safetensors = library.numpy(), library.torch(), library.safetensors()
+logger = logging.getLogger(__name__)
class Reranker(Pipeline):
@@ -11,16 +22,23 @@ class Reranker(Pipeline):
embeddings instance for this to work properly.
"""
- def __init__(self, embeddings, similarity):
+ # Cache lock shared by all instances, since rerankers can share one similarity pipeline
+ lock = RLock()
+
+ def __init__(self, embeddings, similarity, cache=None):
"""
Creates a Reranker pipeline.
Args:
embeddings: embeddings instance (content must be enabled)
similarity: similarity instance
+ cache: optional document token cache; True bounds at 1000, False/None disable; other values use int(cache), non-positive disables
"""
self.embeddings, self.similarity = embeddings, similarity
+ self.cachesize = 1000 if cache is True else 0 if cache is None or cache is False else int(cache)
+ self.cache = OrderedDict() if self.cachesize > 0 else None
+ self.namespace = None
# pylint: disable=W0222
def __call__(self, query, limit=3, factor=10, **kwargs):
@@ -47,11 +65,111 @@ class Reranker(Pipeline):
for x, result in enumerate(results):
texts = [row["text"] for row in result]
- # Score results and merge
- for uid, score in self.similarity(queries[x], texts):
- result[uid]["score"] = score
+ if self.cache is not None and (namespace := self.modelkey()) and all("id" in row for row in result):
+ with self.lock:
+ texts = self.vectors(result, namespace)
+ if not result:
+ ranked.append([])
+ continue
+ for uid, score in self.similarity(queries[x], texts):
+ result[uid]["score"] = score
+ else:
+ # Score results and merge
+ for uid, score in self.similarity(queries[x], texts):
+ result[uid]["score"] = score
# Sort and take top n sorted results
ranked.append(sorted(result, key=lambda row: row["score"], reverse=True)[:limit])
return ranked[0] if isinstance(query, str) else ranked
+
+ def modelkey(self):
+ """Compute a stable namespace for batch-independent document encoding."""
+
+ encoder = getattr(self.similarity, "lateencoder", None)
+ if not encoder:
+ return None
+
+ pooling = encoder.model
+ center = getattr(pooling, "center", None)
+ if center and center["scope"] == "batch":
+ return None
+
+ mean = center.get("mean") if center else None
+ identity = {
+ "pooling": type(pooling).__name__,
+ "model": getattr(pooling.model.config, "_name_or_path"),
+ "revision": getattr(pooling.model.config, "_commit_hash", None),
+ "tokenizer": pooling.tokenizer.name_or_path,
+ "scope": center["scope"] if center else None,
+ "mean": hashlib.sha256(np.asarray(mean).tobytes()).hexdigest() if mean is not None else None,
+ }
+ return hashlib.sha256(json.dumps(identity, sort_keys=True).encode("utf-8")).hexdigest()
+
+ def reset(self, namespace):
+ """Clear entries when the encoder namespace changes; caller holds the cache lock."""
+
+ if self.namespace != namespace:
+ self.cache.clear()
+ self.namespace = namespace
+
+ def vectors(self, rows, namespace):
+ """Gather cached vectors and encode misses once, preserving candidate positions."""
+
+ with self.lock:
+ self.reset(namespace)
+ if not rows:
+ return []
+ keys, vectors, misses = [], [], []
+ for row in rows:
+ key = (namespace, row["id"], hashlib.sha256(row["text"].encode("utf-8")).hexdigest())
+ vector = self.cache.get(key)
+ keys.append(key)
+ vectors.append(vector)
+ if vector is None:
+ misses.append(len(vectors) - 1)
+ else:
+ self.cache.move_to_end(key)
+
+ if misses:
+ encoded = self.similarity.encode([rows[x]["text"] for x in misses], "data")
+ for position, vector in zip(misses, encoded):
+ vectors[position] = vector[vector.abs().sum(dim=-1) > 0].detach().to(device="cpu", dtype=torch.float32)
+
+ # Keep local references through padding before LRU insertion can evict any current candidate.
+ batch = torch.zeros((len(vectors), max(vector.shape[0] for vector in vectors), vectors[0].shape[1]), dtype=torch.float32)
+ for position, vector in enumerate(vectors):
+ batch[position, : vector.shape[0]] = vector
+
+ for position in misses:
+ self.cache.update({keys[position]: vectors[position]})
+ while len(self.cache) > self.cachesize:
+ self.cache.popitem(last=False)
+
+ return batch.to(self.similarity.lateencoder.device)
+
+ def save(self, path):
+ """Save an independent safetensors snapshot; disabled or unsupported caches do nothing."""
+
+ if self.cache is not None and (namespace := self.modelkey()):
+ with self.lock:
+ self.reset(namespace)
+ metadata = {"version": "1", "namespace": namespace, "keys": json.dumps([list(key[1:]) for key in self.cache.keys()])}
+ safetensors.numpy.save_file({str(x): vector.numpy() for x, vector in enumerate(self.cache.values())}, path, metadata)
+
+ def load(self, path):
+ """Load a bounded snapshot, warning on model mismatch; disabled or unsupported caches do nothing."""
+
+ if self.cache is not None and (namespace := self.modelkey()):
+ with self.lock:
+ self.cache.clear()
+ self.namespace = namespace
+ with safetensors.safe_open(path, framework="np") as source:
+ metadata = source.metadata()
+ if metadata["namespace"] != namespace:
+ logger.warning("Ignoring reranker cache snapshot with a different encoder namespace")
+ return
+ for x, key in enumerate(json.loads(metadata["keys"])):
+ self.cache.update({(namespace, *key): torch.from_numpy(source.get_tensor(str(x)))})
+ while len(self.cache) > self.cachesize:
+ self.cache.popitem(last=False)
diff --git a/test/python/testpipeline/testtext/testreranker.py b/test/python/testpipeline/testtext/testreranker.py
index 58505702..55ab1eb6 100644
--- a/test/python/testpipeline/testtext/testreranker.py
+++ b/test/python/testpipeline/testtext/testreranker.py
@@ -2,7 +2,14 @@
Reranker module tests
"""
+import os
+import tempfile
+import threading
import unittest
+from unittest.mock import patch
+
+import numpy as np
+import torch
from txtai import Embeddings
from txtai.pipeline import Reranker, Similarity
@@ -28,6 +35,262 @@ class TestReranker(unittest.TestCase):
"Make huge profits without work, earn up to $100,000 a day",
]
+ cls.embeddings = Embeddings(content=True)
+ cls.embeddings.index(cls.data)
+ cls.similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True)
+
+ def assertScores(self, cached, uncached):
+ """Compare candidate order and scores at the supported precision."""
+
+ self.assertEqual([row.get("id") for row in cached], [row.get("id") for row in uncached])
+ self.assertEqual([row["text"] for row in cached], [row["text"] for row in uncached])
+ for actual, expected in zip(cached, uncached):
+ self.assertAlmostEqual(actual["score"], expected["score"], places=4)
+
+ @staticmethod
+ def dataCalls(encode):
+ """Return only document encoding calls."""
+
+ return [call for call in encode.call_args_list if call.args[1] == "data"]
+
+ @staticmethod
+ def lockRecorder(ranker, encode, calls):
+ """Record encoder lock ownership using a separate thread."""
+
+ def record(data, category):
+ held = []
+
+ def probe():
+ acquired = ranker.lock.acquire(blocking=False)
+ held.append(not acquired)
+ if acquired:
+ ranker.lock.release()
+
+ thread = threading.Thread(target=probe)
+ thread.start()
+ thread.join()
+ calls.append((category, held[0]))
+ return encode(data, category)
+
+ return record
+
+ def testCacheLock(self):
+ """Hold the cache lock across document and query encodes while leaving default calls unlocked."""
+
+ for enabled in (True, False):
+ ranker = Reranker(self.embeddings, self.similarity, cache=True) if enabled else Reranker(self.embeddings, self.similarity)
+ calls = []
+ record = self.lockRecorder(ranker, self.similarity.lateencoder.encode, calls)
+ with patch.object(self.similarity.lateencoder, "encode", side_effect=record):
+ ranker("lottery")
+ ranker("lottery")
+ self.assertEqual({category for category, _ in calls}, {"data", "query"})
+ self.assertTrue(all(held == enabled for _, held in calls), calls)
+
+ def testCacheShared(self):
+ """Serialize cached rerankers that share a similarity pipeline on one lock."""
+
+ first = Reranker(self.embeddings, self.similarity, cache=True)
+ second = Reranker(self.embeddings, self.similarity, cache=True)
+ calls = []
+ with patch.object(self.similarity.lateencoder, "encode", side_effect=self.lockRecorder(second, self.similarity.lateencoder.encode, calls)):
+ first("lottery")
+ self.assertEqual({category for category, _ in calls}, {"data", "query"})
+ self.assertTrue(all(held for _, held in calls), calls)
+
+ def testCacheArgument(self):
+ """Coerce cache bounds to integers and disable non-positive bounds."""
+
+ for cache, bound in ((True, 1000), (3, 3), ("2", 2), (2.5, 2)):
+ ranker = Reranker(self.embeddings, self.similarity, cache=cache)
+ self.assertEqual(ranker.cachesize, bound)
+ self.assertIsNotNone(ranker.cache)
+ for cache in (None, False, 0, -1):
+ self.assertIsNone(Reranker(self.embeddings, self.similarity, cache=cache).cache)
+
+ def testCacheHits(self):
+ """Reuse document vectors on both the second and third rescoring."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ expected = Reranker(self.embeddings, self.similarity)("lottery winner")
+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:
+ self.assertScores(ranker("lottery winner"), expected)
+ self.assertEqual(len(self.dataCalls(encode)), 1)
+ for _ in range(2):
+ encode.reset_mock()
+ self.assertScores(ranker("lottery winner"), expected)
+ self.assertEqual(len(self.dataCalls(encode)), 0)
+ self.assertEqual(ranker.cachesize, 1000)
+
+ def testCacheMixed(self):
+ """Pad mixed hits and misses in candidate order for batched queries."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ ranker("select id, text, score from txtai where id = '4'")
+ queries = ["select id, text, score from txtai order by id", "select id, text, score from txtai order by id desc"]
+ expected = Reranker(self.embeddings, self.similarity)(queries)
+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:
+ actual = ranker(queries)
+ self.assertEqual(len(self.dataCalls(encode)), 1)
+ self.assertEqual(len(self.dataCalls(encode)[0].args[0]), 5)
+ for cached, uncached in zip(actual, expected):
+ self.assertScores(cached, uncached)
+ self.assertGreater(len({value.shape[0] for value in ranker.cache.values()}), 1)
+ for value in ranker.cache.values():
+ self.assertEqual(value.dtype, torch.float32)
+ self.assertEqual(value.device.type, "cpu")
+ self.assertTrue((value.abs().sum(dim=-1) > 0).all())
+
+ def testCacheTextChanges(self):
+ """Re-encode changed text while rebuilds and embeddings swaps reuse unchanged text."""
+
+ embeddings = Embeddings(content=True)
+ embeddings.index([(0, self.data[4], None)])
+ ranker = Reranker(embeddings, self.similarity, cache=True)
+ ranker("lottery")
+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:
+ embeddings.upsert([(0, self.data[0], None)])
+ self.assertScores(ranker("lottery"), Reranker(embeddings, self.similarity)("lottery"))
+ self.assertEqual(len(self.dataCalls(encode)), 2)
+ embeddings.index([(0, self.data[0], None)])
+ replacement = Embeddings(content=True)
+ replacement.index([(0, self.data[0], None)])
+ ranker.embeddings = replacement
+ encode.reset_mock()
+ ranker("lottery")
+ self.assertEqual(len(self.dataCalls(encode)), 0)
+
+ def testDisabled(self):
+ """Keep the default string input path without building cache state."""
+
+ ranker = Reranker(self.embeddings, self.similarity)
+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:
+ ranker("lottery")
+ self.assertTrue(all(isinstance(text, str) for text in self.dataCalls(encode)[0].args[0]))
+ self.assertIsNone(getattr(ranker, "cache", None))
+
+ def testCacheSnapshot(self):
+ """Restore cached document vectors and preserve typed ids and LRU order."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ expected = ranker("lottery")
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "cache.safetensors")
+ ranker.save(path)
+ restored = Reranker(self.embeddings, self.similarity, cache=True)
+ restored.load(path)
+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:
+ self.assertScores(restored("lottery"), expected)
+ self.assertEqual(len(self.dataCalls(encode)), 0)
+ bounded = Reranker(self.embeddings, self.similarity, cache=2)
+ bounded.load(path)
+ self.assertEqual(list(bounded.cache), list(ranker.cache)[-2:])
+
+ # An external integer id and its string spelling must remain distinct.
+ rows = [[{"id": 1, "text": self.data[0]}, {"id": "1", "text": self.data[0]}]]
+ typed = Reranker(self.embeddings, self.similarity, cache=True)
+ with patch.object(self.embeddings, "batchsearch", return_value=rows):
+ typed("lottery")
+ typed.save(path)
+ restored.load(path)
+ self.assertEqual([key[1] for key in restored.cache.keys()], [1, "1"])
+
+ def testCacheMismatch(self):
+ """Discard mismatched snapshots with a warning and invalidate a replaced encoder."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ ranker("lottery")
+ centered = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": {"scope": "document"}})
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "cache.safetensors")
+ ranker.save(path)
+ restored = Reranker(self.embeddings, centered, cache=True)
+ with self.assertLogs("txtai.pipeline.text.reranker", level="WARNING"):
+ restored.load(path)
+ self.assertFalse(restored.cache)
+ with patch.object(centered.lateencoder, "encode", wraps=centered.lateencoder.encode) as encode:
+ restored("lottery")
+ self.assertEqual(len(self.dataCalls(encode)), 1)
+ encode.reset_mock()
+ ranker.similarity = centered
+ self.assertScores(ranker("lottery"), restored("lottery"))
+ self.assertEqual(len(self.dataCalls(encode)), 1)
+
+ def testCacheMean(self):
+ """Include a collection mean in the namespace and invalidate changed means."""
+
+ mean = np.zeros(128, dtype=np.float32)
+ similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": {"scope": "collection", "mean": mean}})
+ ranker = Reranker(self.embeddings, similarity, cache=True)
+ self.assertScores(ranker("lottery"), Reranker(self.embeddings, similarity)("lottery"))
+ namespace = ranker.namespace
+ similarity.lateencoder.model.center["mean"] = np.ones(128, dtype=np.float32) / 100
+ with patch.object(similarity.lateencoder, "encode", wraps=similarity.lateencoder.encode) as encode:
+ ranker("lottery")
+ self.assertEqual(len(self.dataCalls(encode)), 1)
+ self.assertNotEqual(namespace, ranker.namespace)
+
+ def testCacheEmpty(self):
+ """Skip encoding and scoring when the cached candidate batch is empty."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:
+ self.assertEqual(ranker("select id, text, score from txtai where id = 'missing'"), [])
+ self.assertEqual(encode.call_count, 0)
+
+ def testCacheBypass(self):
+ """Keep text scoring for missing ids, non-late encoders and batch centering."""
+
+ query = "select text, score from txtai where similar('lottery')"
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ self.assertScores(ranker(query), Reranker(self.embeddings, self.similarity)(query))
+ self.assertFalse(ranker.cache)
+ for similarity in [
+ Similarity("cross-encoder/ms-marco-MiniLM-L-2-v2", crossencode=True),
+ Similarity("prajjwal1/bert-medium-mnli"),
+ Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": True}),
+ ]:
+ ranker = Reranker(self.embeddings, similarity, cache=True)
+ self.assertScores(ranker("lottery"), Reranker(self.embeddings, similarity)("lottery"))
+ self.assertFalse(ranker.cache)
+
+ def testCacheBound(self):
+ """Evict the least recently used document without losing current batch vectors."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=2)
+ for uid in (0, 1, 0, 2):
+ ranker(f"select id, text, score from txtai where id = '{uid}'")
+ self.assertLessEqual(len(ranker.cache), 2)
+ self.assertEqual([key[1] for key in ranker.cache.keys()], ["0", "2"])
+ expected = Reranker(self.embeddings, self.similarity)("lottery")
+ self.assertScores(ranker("lottery"), expected)
+ self.assertEqual(len(ranker.cache), 2)
+
+ def testCacheDisabledSnapshot(self):
+ """Make disabled snapshots no-ops without accessing a file or an encoder."""
+
+ ranker = Reranker(self.embeddings, self.similarity)
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "cache.safetensors")
+ ranker.save(path)
+ ranker.load(path)
+ self.assertFalse(os.path.exists(path))
+
+ def testCacheEmptySnapshot(self):
+ """Round-trip an empty cache and bypass snapshots for unsupported encoders."""
+
+ ranker = Reranker(self.embeddings, self.similarity, cache=True)
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "cache.safetensors")
+ ranker.save(path)
+ ranker.load(path)
+ self.assertFalse(ranker.cache)
+ ranker.similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": True})
+ other = os.path.join(directory, "unused.safetensors")
+ ranker.save(other)
+ ranker.load(other)
+ self.assertFalse(os.path.exists(other))
+
def testRanker(self):
"""
Test re-ranking pipeline