💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Cache document token vectors in Reranker (#1321)
Morgan Carr893d0424bec9
diff --git a/src/python/txtai/pipeline/text/reranker.py b/src/python/txtai/pipeline/text/reranker.pyindex b0ffd9f8..71d2253b 100644--- a/src/python/txtai/pipeline/text/reranker.py+++ b/src/python/txtai/pipeline/text/reranker.py@@ -2,7 +2,18 @@Reranker module"""+import hashlib+import json+import logging+from collections import OrderedDict+from threading import RLock+from ..base import Pipeline+from ...util import Library++library = Library()+np, torch, safetensors = library.numpy(), library.torch(), library.safetensors()+logger = logging.getLogger(__name__)class Reranker(Pipeline):@@ -11,16 +22,23 @@ class Reranker(Pipeline):embeddings instance for this to work properly."""- def __init__(self, embeddings, similarity):+ # Cache lock shared by all instances, since rerankers can share one similarity pipeline+ lock = RLock()++ def __init__(self, embeddings, similarity, cache=None):"""Creates a Reranker pipeline.Args:embeddings: embeddings instance (content must be enabled)similarity: similarity instance+ cache: optional document token cache; True bounds at 1000, False/None disable; other values use int(cache), non-positive disables"""self.embeddings, self.similarity = embeddings, similarity+ self.cachesize = 1000 if cache is True else 0 if cache is None or cache is False else int(cache)+ self.cache = OrderedDict() if self.cachesize > 0 else None+ self.namespace = None# pylint: disable=W0222def __call__(self, query, limit=3, factor=10, **kwargs):@@ -47,11 +65,111 @@ class Reranker(Pipeline):for x, result in enumerate(results):texts = [row["text"] for row in result]- # Score results and merge- for uid, score in self.similarity(queries[x], texts):- result[uid]["score"] = score+ if self.cache is not None and (namespace := self.modelkey()) and all("id" in row for row in result):+ with self.lock:+ texts = self.vectors(result, namespace)+ if not result:+ ranked.append([])+ continue+ for uid, score in self.similarity(queries[x], texts):+ result[uid]["score"] = score+ else:+ # Score results and merge+ for uid, score in self.similarity(queries[x], texts):+ result[uid]["score"] = score# Sort and take top n sorted resultsranked.append(sorted(result, key=lambda row: row["score"], reverse=True)[:limit])return ranked[0] if isinstance(query, str) else ranked++ def modelkey(self):+ """Compute a stable namespace for batch-independent document encoding."""++ encoder = getattr(self.similarity, "lateencoder", None)+ if not encoder:+ return None++ pooling = encoder.model+ center = getattr(pooling, "center", None)+ if center and center["scope"] == "batch":+ return None++ mean = center.get("mean") if center else None+ identity = {+ "pooling": type(pooling).__name__,+ "model": getattr(pooling.model.config, "_name_or_path"),+ "revision": getattr(pooling.model.config, "_commit_hash", None),+ "tokenizer": pooling.tokenizer.name_or_path,+ "scope": center["scope"] if center else None,+ "mean": hashlib.sha256(np.asarray(mean).tobytes()).hexdigest() if mean is not None else None,+ }+ return hashlib.sha256(json.dumps(identity, sort_keys=True).encode("utf-8")).hexdigest()++ def reset(self, namespace):+ """Clear entries when the encoder namespace changes; caller holds the cache lock."""++ if self.namespace != namespace:+ self.cache.clear()+ self.namespace = namespace++ def vectors(self, rows, namespace):+ """Gather cached vectors and encode misses once, preserving candidate positions."""++ with self.lock:+ self.reset(namespace)+ if not rows:+ return []+ keys, vectors, misses = [], [], []+ for row in rows:+ key = (namespace, row["id"], hashlib.sha256(row["text"].encode("utf-8")).hexdigest())+ vector = self.cache.get(key)+ keys.append(key)+ vectors.append(vector)+ if vector is None:+ misses.append(len(vectors) - 1)+ else:+ self.cache.move_to_end(key)++ if misses:+ encoded = self.similarity.encode([rows[x]["text"] for x in misses], "data")+ for position, vector in zip(misses, encoded):+ vectors[position] = vector[vector.abs().sum(dim=-1) > 0].detach().to(device="cpu", dtype=torch.float32)++ # Keep local references through padding before LRU insertion can evict any current candidate.+ batch = torch.zeros((len(vectors), max(vector.shape[0] for vector in vectors), vectors[0].shape[1]), dtype=torch.float32)+ for position, vector in enumerate(vectors):+ batch[position, : vector.shape[0]] = vector++ for position in misses:+ self.cache.update({keys[position]: vectors[position]})+ while len(self.cache) > self.cachesize:+ self.cache.popitem(last=False)++ return batch.to(self.similarity.lateencoder.device)++ def save(self, path):+ """Save an independent safetensors snapshot; disabled or unsupported caches do nothing."""++ if self.cache is not None and (namespace := self.modelkey()):+ with self.lock:+ self.reset(namespace)+ metadata = {"version": "1", "namespace": namespace, "keys": json.dumps([list(key[1:]) for key in self.cache.keys()])}+ safetensors.numpy.save_file({str(x): vector.numpy() for x, vector in enumerate(self.cache.values())}, path, metadata)++ def load(self, path):+ """Load a bounded snapshot, warning on model mismatch; disabled or unsupported caches do nothing."""++ if self.cache is not None and (namespace := self.modelkey()):+ with self.lock:+ self.cache.clear()+ self.namespace = namespace+ with safetensors.safe_open(path, framework="np") as source:+ metadata = source.metadata()+ if metadata["namespace"] != namespace:+ logger.warning("Ignoring reranker cache snapshot with a different encoder namespace")+ return+ for x, key in enumerate(json.loads(metadata["keys"])):+ self.cache.update({(namespace, *key): torch.from_numpy(source.get_tensor(str(x)))})+ while len(self.cache) > self.cachesize:+ self.cache.popitem(last=False)diff --git a/test/python/testpipeline/testtext/testreranker.py b/test/python/testpipeline/testtext/testreranker.pyindex 58505702..55ab1eb6 100644--- a/test/python/testpipeline/testtext/testreranker.py+++ b/test/python/testpipeline/testtext/testreranker.py@@ -2,7 +2,14 @@Reranker module tests"""+import os+import tempfile+import threadingimport unittest+from unittest.mock import patch++import numpy as np+import torchfrom txtai import Embeddingsfrom txtai.pipeline import Reranker, Similarity@@ -28,6 +35,262 @@ class TestReranker(unittest.TestCase):"Make huge profits without work, earn up to $100,000 a day",]+ cls.embeddings = Embeddings(content=True)+ cls.embeddings.index(cls.data)+ cls.similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True)++ def assertScores(self, cached, uncached):+ """Compare candidate order and scores at the supported precision."""++ self.assertEqual([row.get("id") for row in cached], [row.get("id") for row in uncached])+ self.assertEqual([row["text"] for row in cached], [row["text"] for row in uncached])+ for actual, expected in zip(cached, uncached):+ self.assertAlmostEqual(actual["score"], expected["score"], places=4)++ @staticmethod+ def dataCalls(encode):+ """Return only document encoding calls."""++ return [call for call in encode.call_args_list if call.args[1] == "data"]++ @staticmethod+ def lockRecorder(ranker, encode, calls):+ """Record encoder lock ownership using a separate thread."""++ def record(data, category):+ held = []++ def probe():+ acquired = ranker.lock.acquire(blocking=False)+ held.append(not acquired)+ if acquired:+ ranker.lock.release()++ thread = threading.Thread(target=probe)+ thread.start()+ thread.join()+ calls.append((category, held[0]))+ return encode(data, category)++ return record++ def testCacheLock(self):+ """Hold the cache lock across document and query encodes while leaving default calls unlocked."""++ for enabled in (True, False):+ ranker = Reranker(self.embeddings, self.similarity, cache=True) if enabled else Reranker(self.embeddings, self.similarity)+ calls = []+ record = self.lockRecorder(ranker, self.similarity.lateencoder.encode, calls)+ with patch.object(self.similarity.lateencoder, "encode", side_effect=record):+ ranker("lottery")+ ranker("lottery")+ self.assertEqual({category for category, _ in calls}, {"data", "query"})+ self.assertTrue(all(held == enabled for _, held in calls), calls)++ def testCacheShared(self):+ """Serialize cached rerankers that share a similarity pipeline on one lock."""++ first = Reranker(self.embeddings, self.similarity, cache=True)+ second = Reranker(self.embeddings, self.similarity, cache=True)+ calls = []+ with patch.object(self.similarity.lateencoder, "encode", side_effect=self.lockRecorder(second, self.similarity.lateencoder.encode, calls)):+ first("lottery")+ self.assertEqual({category for category, _ in calls}, {"data", "query"})+ self.assertTrue(all(held for _, held in calls), calls)++ def testCacheArgument(self):+ """Coerce cache bounds to integers and disable non-positive bounds."""++ for cache, bound in ((True, 1000), (3, 3), ("2", 2), (2.5, 2)):+ ranker = Reranker(self.embeddings, self.similarity, cache=cache)+ self.assertEqual(ranker.cachesize, bound)+ self.assertIsNotNone(ranker.cache)+ for cache in (None, False, 0, -1):+ self.assertIsNone(Reranker(self.embeddings, self.similarity, cache=cache).cache)++ def testCacheHits(self):+ """Reuse document vectors on both the second and third rescoring."""++ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ expected = Reranker(self.embeddings, self.similarity)("lottery winner")+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:+ self.assertScores(ranker("lottery winner"), expected)+ self.assertEqual(len(self.dataCalls(encode)), 1)+ for _ in range(2):+ encode.reset_mock()+ self.assertScores(ranker("lottery winner"), expected)+ self.assertEqual(len(self.dataCalls(encode)), 0)+ self.assertEqual(ranker.cachesize, 1000)++ def testCacheMixed(self):+ """Pad mixed hits and misses in candidate order for batched queries."""++ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ ranker("select id, text, score from txtai where id = '4'")+ queries = ["select id, text, score from txtai order by id", "select id, text, score from txtai order by id desc"]+ expected = Reranker(self.embeddings, self.similarity)(queries)+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:+ actual = ranker(queries)+ self.assertEqual(len(self.dataCalls(encode)), 1)+ self.assertEqual(len(self.dataCalls(encode)[0].args[0]), 5)+ for cached, uncached in zip(actual, expected):+ self.assertScores(cached, uncached)+ self.assertGreater(len({value.shape[0] for value in ranker.cache.values()}), 1)+ for value in ranker.cache.values():+ self.assertEqual(value.dtype, torch.float32)+ self.assertEqual(value.device.type, "cpu")+ self.assertTrue((value.abs().sum(dim=-1) > 0).all())++ def testCacheTextChanges(self):+ """Re-encode changed text while rebuilds and embeddings swaps reuse unchanged text."""++ embeddings = Embeddings(content=True)+ embeddings.index([(0, self.data[4], None)])+ ranker = Reranker(embeddings, self.similarity, cache=True)+ ranker("lottery")+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:+ embeddings.upsert([(0, self.data[0], None)])+ self.assertScores(ranker("lottery"), Reranker(embeddings, self.similarity)("lottery"))+ self.assertEqual(len(self.dataCalls(encode)), 2)+ embeddings.index([(0, self.data[0], None)])+ replacement = Embeddings(content=True)+ replacement.index([(0, self.data[0], None)])+ ranker.embeddings = replacement+ encode.reset_mock()+ ranker("lottery")+ self.assertEqual(len(self.dataCalls(encode)), 0)++ def testDisabled(self):+ """Keep the default string input path without building cache state."""++ ranker = Reranker(self.embeddings, self.similarity)+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:+ ranker("lottery")+ self.assertTrue(all(isinstance(text, str) for text in self.dataCalls(encode)[0].args[0]))+ self.assertIsNone(getattr(ranker, "cache", None))++ def testCacheSnapshot(self):+ """Restore cached document vectors and preserve typed ids and LRU order."""++ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ expected = ranker("lottery")+ with tempfile.TemporaryDirectory() as directory:+ path = os.path.join(directory, "cache.safetensors")+ ranker.save(path)+ restored = Reranker(self.embeddings, self.similarity, cache=True)+ restored.load(path)+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:+ self.assertScores(restored("lottery"), expected)+ self.assertEqual(len(self.dataCalls(encode)), 0)+ bounded = Reranker(self.embeddings, self.similarity, cache=2)+ bounded.load(path)+ self.assertEqual(list(bounded.cache), list(ranker.cache)[-2:])++ # An external integer id and its string spelling must remain distinct.+ rows = [[{"id": 1, "text": self.data[0]}, {"id": "1", "text": self.data[0]}]]+ typed = Reranker(self.embeddings, self.similarity, cache=True)+ with patch.object(self.embeddings, "batchsearch", return_value=rows):+ typed("lottery")+ typed.save(path)+ restored.load(path)+ self.assertEqual([key[1] for key in restored.cache.keys()], [1, "1"])++ def testCacheMismatch(self):+ """Discard mismatched snapshots with a warning and invalidate a replaced encoder."""++ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ ranker("lottery")+ centered = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": {"scope": "document"}})+ with tempfile.TemporaryDirectory() as directory:+ path = os.path.join(directory, "cache.safetensors")+ ranker.save(path)+ restored = Reranker(self.embeddings, centered, cache=True)+ with self.assertLogs("txtai.pipeline.text.reranker", level="WARNING"):+ restored.load(path)+ self.assertFalse(restored.cache)+ with patch.object(centered.lateencoder, "encode", wraps=centered.lateencoder.encode) as encode:+ restored("lottery")+ self.assertEqual(len(self.dataCalls(encode)), 1)+ encode.reset_mock()+ ranker.similarity = centered+ self.assertScores(ranker("lottery"), restored("lottery"))+ self.assertEqual(len(self.dataCalls(encode)), 1)++ def testCacheMean(self):+ """Include a collection mean in the namespace and invalidate changed means."""++ mean = np.zeros(128, dtype=np.float32)+ similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": {"scope": "collection", "mean": mean}})+ ranker = Reranker(self.embeddings, similarity, cache=True)+ self.assertScores(ranker("lottery"), Reranker(self.embeddings, similarity)("lottery"))+ namespace = ranker.namespace+ similarity.lateencoder.model.center["mean"] = np.ones(128, dtype=np.float32) / 100+ with patch.object(similarity.lateencoder, "encode", wraps=similarity.lateencoder.encode) as encode:+ ranker("lottery")+ self.assertEqual(len(self.dataCalls(encode)), 1)+ self.assertNotEqual(namespace, ranker.namespace)++ def testCacheEmpty(self):+ """Skip encoding and scoring when the cached candidate batch is empty."""++ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ with patch.object(self.similarity.lateencoder, "encode", wraps=self.similarity.lateencoder.encode) as encode:+ self.assertEqual(ranker("select id, text, score from txtai where id = 'missing'"), [])+ self.assertEqual(encode.call_count, 0)++ def testCacheBypass(self):+ """Keep text scoring for missing ids, non-late encoders and batch centering."""++ query = "select text, score from txtai where similar('lottery')"+ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ self.assertScores(ranker(query), Reranker(self.embeddings, self.similarity)(query))+ self.assertFalse(ranker.cache)+ for similarity in [+ Similarity("cross-encoder/ms-marco-MiniLM-L-2-v2", crossencode=True),+ Similarity("prajjwal1/bert-medium-mnli"),+ Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": True}),+ ]:+ ranker = Reranker(self.embeddings, similarity, cache=True)+ self.assertScores(ranker("lottery"), Reranker(self.embeddings, similarity)("lottery"))+ self.assertFalse(ranker.cache)++ def testCacheBound(self):+ """Evict the least recently used document without losing current batch vectors."""++ ranker = Reranker(self.embeddings, self.similarity, cache=2)+ for uid in (0, 1, 0, 2):+ ranker(f"select id, text, score from txtai where id = '{uid}'")+ self.assertLessEqual(len(ranker.cache), 2)+ self.assertEqual([key[1] for key in ranker.cache.keys()], ["0", "2"])+ expected = Reranker(self.embeddings, self.similarity)("lottery")+ self.assertScores(ranker("lottery"), expected)+ self.assertEqual(len(ranker.cache), 2)++ def testCacheDisabledSnapshot(self):+ """Make disabled snapshots no-ops without accessing a file or an encoder."""++ ranker = Reranker(self.embeddings, self.similarity)+ with tempfile.TemporaryDirectory() as directory:+ path = os.path.join(directory, "cache.safetensors")+ ranker.save(path)+ ranker.load(path)+ self.assertFalse(os.path.exists(path))++ def testCacheEmptySnapshot(self):+ """Round-trip an empty cache and bypass snapshots for unsupported encoders."""++ ranker = Reranker(self.embeddings, self.similarity, cache=True)+ with tempfile.TemporaryDirectory() as directory:+ path = os.path.join(directory, "cache.safetensors")+ ranker.save(path)+ ranker.load(path)+ self.assertFalse(ranker.cache)+ ranker.similarity = Similarity("neuml/colbert-bert-tiny", lateencode=True, vectors={"center": True})+ other = os.path.join(directory, "unused.safetensors")+ ranker.save(other)+ ranker.load(other)+ self.assertFalse(os.path.exists(other))+def testRanker(self):"""Test re-ranking pipeline