💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add IVFFlat indexing for sparse vectors, closes #929
davidmezzettidb60bd76e6b1
diff --git a/docs/embeddings/configuration/scoring.md b/docs/embeddings/configuration/scoring.mdindex 945df33f..7e9c7618 100644--- a/docs/embeddings/configuration/scoring.md+++ b/docs/embeddings/configuration/scoring.md@@ -32,6 +32,10 @@ gpu: boolean|int|string|deviceencodebatch: intmodelargs: additional model init argsencodeargs: additional encode() args+ivf: #IVFFlat index configuration+ sample: percent of data to use for model training (0.0 - 1.0)+ nclusters: desired number of clusters (int)+ nprobe: search probe setting (int)```Sparse vector scoring options.diff --git a/src/python/txtai/scoring/__init__.py b/src/python/txtai/scoring/__init__.pyindex 4a6828a2..e1816992 100644--- a/src/python/txtai/scoring/__init__.py+++ b/src/python/txtai/scoring/__init__.py@@ -5,7 +5,9 @@ Scoring importsfrom .base import Scoringfrom .bm25 import BM25from .factory import ScoringFactory+from .ivf import IVFFlatfrom .pgtext import PGTextfrom .sif import SIF+from .sparse import Sparsefrom .terms import Termsfrom .tfidf import TFIDFdiff --git a/src/python/txtai/scoring/ivf.py b/src/python/txtai/scoring/ivf.pynew file mode 100644index 00000000..41c9f051--- /dev/null+++ b/src/python/txtai/scoring/ivf.py@@ -0,0 +1,390 @@+"""+IVF module+"""++import math++import numpy as np++# Conditional import+try:+ from scipy.sparse import csr_matrix, vstack+ from sklearn.cluster import MiniBatchKMeans+ from sklearn.metrics import pairwise_distances_argmin_min+ from sklearn.utils.extmath import safe_sparse_dot++ SKLEARN = True+except ImportError:+ SKLEARN = False++from ..serialize import SerializeFactory+++class IVFFlat:+ """+ Inverted file (IVF) index with flat vector storage and sparse array support.++ IVFFlat builds an IVF index and enables approximate nearest neighbor (ANN) search.++ This index is modeled after Faiss and supports many of the same parameters.++ See this link for more: https://github.com/facebookresearch/faiss/wiki/Faster-search+ """++ def __init__(self, config):+ """+ Create a new IVFFlat instance.++ Args:+ config: index configuration+ """++ if not SKLEARN:+ raise ImportError('IVFFlat is not available - install "scoring" extra to enable')++ # Index configuration+ self.config = config if isinstance(config, dict) else {}++ # Cluster centroids, if computed+ self.centroids = None++ # Cluster id mapping+ self.ids = None++ # Cluster data blocks - can be a single block with no computed centroids+ self.blocks = None++ # Deleted ids+ self.deletes = None++ def index(self, data):+ """+ Builds a new IVFFlat index.++ Args:+ data: input data+ config: index configuration+ """++ # Compute model training size+ train, sample = data, self.config.get("sample")+ if sample:+ # Get sample for training+ rng = np.random.default_rng(0)+ indices = sorted(rng.choice(train.shape[0], int(sample * train.shape[0]), replace=False, shuffle=False))+ train = train[indices]++ # Get number of clusters. Note that final number of clusters could be lower due to filtering duplicate centroids.+ clusters = self.nclusters(data.shape[0], train.shape[0])++ # A single cluster is an exact search+ if clusters > 1:+ # Calculate number of data points+ kmeans = MiniBatchKMeans(n_clusters=clusters, random_state=0, n_init="auto").fit(train)++ # Find closest points to each cluster center and use those as centroids+ # Filter out duplicate centroids+ indices, _ = pairwise_distances_argmin_min(kmeans.cluster_centers_, train, metric="cosine")+ self.centroids = data[np.unique(indices)]++ # Sort into clusters+ ids = self.aggregate(data)++ # Sort clusters by id+ self.ids = dict(sorted(ids.items(), key=lambda x: x[0]))++ # Create cluster data blocks+ self.blocks = {k: data[v] for k, v in self.ids.items()}++ # Initialize deletes+ self.deletes = []++ def append(self, data):+ """+ Appends elements to an existing index.++ Args:+ data: input data+ """++ # Get offset+ offset = self.size()++ # Sort into clusters and merge+ for cluster, ids in self.aggregate(data).items():+ # Add new ids+ self.ids[cluster].extend([x + offset for x in ids])++ # Add new data+ self.blocks[cluster] = vstack([self.blocks[cluster], data[ids]])++ def delete(self, ids):+ """+ Mark ids as deleted. This prevents deleted results from showing up in search results.+ The data is not removed from the underlying data structures.++ Args:+ ids: ids to delete+ """++ # Set index ids as deleted+ self.deletes.extend(ids)++ def search(self, query, limit):+ """+ Searches IVFFlat index for query. Returns topn results.++ Args:+ query: query array+ limit: maximum results++ Returns:+ query results+ """++ if self.centroids is not None:+ # Approximate search+ indices, _ = self.topn(query, self.centroids, self.nprobe())++ # Stack into single ids list+ ids = np.concatenate([self.ids[x] for x in indices if x in self.ids])++ # Stack data rows+ data = vstack([self.blocks[x] for x in indices if x in self.blocks])+ else:+ # Exact search+ ids, data = np.array(self.ids[0]), self.blocks[0]++ # Get deletes+ deletes = np.argwhere(np.isin(ids, self.deletes)).ravel()++ # Calculate similarity+ indices, scores = self.topn(query, data, limit, deletes)++ # Map data ids and return+ return list(zip(ids[indices].tolist(), scores.tolist()))++ def count(self):+ """+ Number of elements in the IVFFlat index.++ Returns:+ count+ """++ return self.size() - len(self.deletes)++ def load(self, path):+ """+ Loads an IVFFlat index from path.++ Args:+ path: file path+ """++ # Create streaming serializer and limit read size to a byte at a time to ensure+ # only msgpack data is consumed+ serializer = SerializeFactory.create("msgpack", streaming=True, read_size=1)++ with open(path, "rb") as f:+ # Read header+ unpacker = serializer.loadstream(f)+ header = next(unpacker)++ # Read cluster centroids, if available+ self.centroids = SparseArray().read(f) if header["centroids"] else None++ # Read cluster ids+ self.ids = dict(next(unpacker))++ # Read cluster data blocks+ self.blocks = {}+ for key in self.ids:+ self.blocks[key] = SparseArray().read(f)++ # Read deletes+ self.deletes = next(unpacker)++ def save(self, path):+ """+ Saves an IVFFlat index to path. This format uses a combination of msgpack and NumPy serialization.+ Sparse arrays are saved as groups of NumPy arrays.++ IVFFlat storage format:+ - header msgpack+ - centroids sparse array (optional based on header parameters)+ - cluster ids msgpack+ - cluster data blocks list of sparse arrays+ - deletes msgpack+ """++ # Create message pack serializer+ serializer = SerializeFactory.create("msgpack")++ with open(path, "wb") as f:+ # Write header+ serializer.savestream({"centroids": self.centroids is not None, "count": self.count(), "blocks": len(self.blocks)}, f)++ # Write cluster centroids, if available+ if self.centroids is not None:+ SparseArray().write(self.centroids, f)++ # Write cluster id mapping+ serializer.savestream(list(self.ids.items()), f)++ # Write cluster data blocks+ for block in self.blocks.values():+ SparseArray().write(block, f)++ # Write deletes+ serializer.savestream(self.deletes, f)++ def aggregate(self, data):+ """+ Aggregates input data array into clusters. This method sorts each data element into the+ cluster with the highest cosine similarity centroid.++ Args:+ data: input data++ Returns:+ {cluster, ids}+ """++ # Exact search when only a single cluster+ if self.centroids is None:+ return {0: list(range(data.shape[0]))}++ # Map data to closest centroids+ indices, _ = pairwise_distances_argmin_min(data, self.centroids, metric="cosine")++ # Sort into clusters+ ids = {}+ for x, cluster in enumerate(indices.tolist()):+ if cluster not in ids:+ ids[cluster] = []++ # Save id+ ids[cluster].append(x)++ return ids++ def topn(self, query, data, limit, deletes=None):+ """+ Gets the top n most similar data elements for query.++ Args:+ query: input query array+ data: data array+ limit: top n+ deletes: optional list of deletes to filter from results++ Returns:+ list of matching (indices, scores)+ """++ # Dot product similarity (assumes all data is normalized)+ scores = safe_sparse_dot(query, data.T, dense_output=True)++ # Clear deletes+ if deletes is not None:+ scores[:, deletes] = 0++ # Get top n matching indices and scores+ indices = np.argpartition(-scores, limit if limit < scores.shape[0] else scores.shape[0] - 1)[:, :limit]+ scores = np.clip(np.take_along_axis(scores, indices, axis=1), 0.0, 1.0)++ return indices[0], scores[0]++ def nclusters(self, count, train):+ """+ Calculates the number of clusters for this IVFFlat index. Note that the final number of clusters could be+ lower as duplicate cluster centroids are filtered out.++ Args:+ count: initial dataset size+ train: number of rows used to train++ Returns:+ number of clusters+ """++ # Get data size+ default = 1 if count <= 5000 else self.cells(train)++ # Number of clusters to create+ return self.config.get("nclusters", default)++ def nprobe(self):+ """+ Gets or derives the nprobe search parameter.++ Returns:+ nprobe setting+ """++ # Get size of embeddings index+ size = self.size()++ default = 6 if size <= 5000 else self.cells(size) // 64+ return self.config.get("nprobe", default)++ def cells(self, count):+ """+ Calculates the number of IVF cells for an IVFFlat index.++ Args:+ count: number of rows++ Returns:+ number of IVF cells+ """++ # Calculate number of IVF cells where x = min(4 * sqrt(count), count / 39)+ # Match faiss behavior that requires at least 39 * x data points+ return max(min(round(4 * math.sqrt(count)), int(count / 39)), 1)++ def size(self):+ """+ Gets the total size of this index including deletes.++ Returns:+ size+ """++ return sum(len(x) for x in self.ids.values())+++class SparseArray:+ """+ Methods to read and write sparse arrays to file.+ """++ def read(self, f):+ """+ Reads a sparse array from input file.++ Args:+ f: input file handle++ Returns:+ sparse array+ """++ data, indices, indptr, shape = np.load(f), np.load(f), np.load(f), np.load(f)++ # Read sparse array+ return csr_matrix((data, indices, indptr), shape=shape)++ def write(self, array, f):+ """+ Writes a sparse array to file.++ Args:+ array: sparse array+ f: output file handle+ """++ # Write sparse array to file+ for x in [array.data, array.indices, array.indptr, array.shape]:+ np.save(f, x, allow_pickle=False)diff --git a/src/python/txtai/scoring/sparse.py b/src/python/txtai/scoring/sparse.pyindex 3b1396cb..864186f0 100644--- a/src/python/txtai/scoring/sparse.py+++ b/src/python/txtai/scoring/sparse.py@@ -2,12 +2,11 @@Sparse module"""-import numpy as np-import torch-# Conditional importtry:+ from scipy.sparse import csr_matrix, vstackfrom sentence_transformers import SparseEncoder+ from sklearn.preprocessing import normalizeSPARSE = Trueexcept ImportError:@@ -16,6 +15,7 @@ except ImportError:from ..models import Modelsfrom .base import Scoring+from .ivf import IVFFlatclass Sparse(Scoring):@@ -41,11 +41,11 @@ class Sparse(Scoring):# Encode batch size - controls underlying model batch size when encoding vectorsself.encodebatch = config.get("encodebatch", 32)- # Sparse data backend+ # Index backendself.backend = None- # Deleted rows- self.deletes = []+ # Input queue - data to be indexed+ self.queue = Nonedef insert(self, documents, index=None):data = []@@ -58,63 +58,65 @@ class Sparse(Scoring):# Add datadata.append(" ".join(document) if isinstance(document, list) else document)- # Encode batch and append to data+ # Encode and normalize datadata = self.encode(data)- self.backend = torch.cat((self.backend, data)) if self.backend is not None else data++ # Create or add to existing sparse vectors+ self.queue = vstack((self.queue, data)) if self.queue is not None else datadef delete(self, ids):- self.deletes.append(ids)+ self.backend.delete(ids)++ def index(self, documents=None):+ # Insert documents, if provided+ if documents:+ self.insert(documents)++ # Create index+ if self.queue is not None:+ # Create a new index instance+ self.backend = IVFFlat(self.config.get("ivf"))+ self.backend.index(self.queue)++ # Clear queue+ self.queue = None++ def upsert(self, documents=None):+ # Insert documents, if provided+ if documents:+ self.insert(documents)++ # Upsert index+ if self.backend and self.queue is not None:+ self.backend.append(self.queue)+ else:+ self.index()++ # Clear queue+ self.queue = Nonedef weights(self, tokens):# Not supportedreturn Nonedef search(self, query, limit=3):- return self.batchsearch([query], limit)[0]+ return self.backend.search(self.encode([query]), limit)def batchsearch(self, queries, limit=3, threads=True):- queries = self.encode(queries)- scores = self.model.similarity(queries, self.backend).cpu().numpy()-- # Clear deletes- scores[:, self.deletes] = 0-- # Get top n scores- indices = np.argpartition(-scores, limit if limit < scores.shape[0] else scores.shape[0] - 1)[:, :limit]- scores = np.clip(np.take_along_axis(scores, indices, axis=1), 0.0, 1.0)-- # Get top n results- results = []- for x, index in enumerate(indices):- results.append(list(zip(index.tolist(), scores[x].tolist())))-- return results+ return [self.search(query, limit) for query in queries]def count(self):- return self.backend.shape[0] - len(self.deletes) if self.backend is not None else 0+ return self.backend.count()def load(self, path):- with open(path, "rb") as f:- # Load sparse index- indices, values, size = np.load(f), np.load(f), np.load(f)-- # Load deletes- self.deletes = np.load(f).tolist()-- # Create backend - load on same device as model- self.backend = torch.sparse_coo_tensor(indices, values, size=torch.Size(size), device=self.model.device)+ # Read IVFFlat index+ self.backend = IVFFlat(self.config.get("ivf"))+ self.backend.load(path)def save(self, path):+ # Write IVFFlat indexif self.backend is not None:- with open(path, "wb") as f:- # Save sparse index- data = self.backend.coalesce().cpu()- np.save(f, data.indices())- np.save(f, data.values())- np.save(f, data.size())-- # Save deletes- np.save(f, np.array(self.deletes))+ self.backend.save(path)def close(self):# Close pool before model is closed@@ -122,7 +124,7 @@ class Sparse(Scoring):self.model.stop_multi_process_pool(self.pool)self.pool = None- self.model, self.backend = None, None+ self.model, self.backend, self.queue = None, None, Nonedef hasterms(self):return True@@ -163,7 +165,7 @@ class Sparse(Scoring):modelargs = self.config.get("modelargs", {})# Build embeddings with sentence-transformers- model = SparseEncoder(path, device=Models.device(deviceid), similarity_fn_name="cosine", **modelargs)+ model = SparseEncoder(path, device=Models.device(deviceid), **modelargs)# Start process pool for multiple GPUsif pool:@@ -184,11 +186,22 @@ class Sparse(Scoring):data: input dataReturns:- encoded data+ encoded data as a SciPy CSR Matrix"""# Additional encoding argumentsencodeargs = self.config.get("encodeargs", {})# Encode data- return self.model.encode(data, pool=self.pool, batch_size=self.encodebatch, **encodeargs)+ data = self.model.encode(data, pool=self.pool, batch_size=self.encodebatch, **encodeargs)++ # Get data attributes+ data = data.cpu().coalesce()+ indices = data.indices().numpy()+ values = data.values().numpy()++ # Convert to CSR Matrix+ matrix = csr_matrix((values, indices), shape=data.size())++ # Normalize and return+ return normalize(matrix)diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 6934b1e0..54f42511 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -49,6 +49,7 @@ class TestOptional(unittest.TestCase):"PIL","rich","scipy",+ "scipy.sparse","sentence_transformers","sklearn.decomposition","smolagents",@@ -293,7 +294,7 @@ class TestOptional(unittest.TestCase):Test missing scoring dependencies"""- from txtai.scoring import ScoringFactory+ from txtai.scoring import IVFFlat, ScoringFactorywith self.assertRaises(ImportError):ScoringFactory.create({"method": "pgtext"})@@ -301,6 +302,9 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):ScoringFactory.create({"method": "sparse"})+ with self.assertRaises(ImportError):+ IVFFlat({})+def testVectors(self):"""Test missing vector dependenciesdiff --git a/test/python/testscoring.py b/test/python/testscoring.pyindex ace10806..9d390f23 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring.py@@ -123,15 +123,11 @@ class TestScoring(unittest.TestCase):self.runTests("sif")- @patch("torch.cuda.device_count")- def testSparse(self, count):+ def testSparse(self):"""Test sparse vectors"""- # Mock accelerator count- count.return_value = 1-# Models cachemodels = {}@@ -165,6 +161,26 @@ class TestScoring(unittest.TestCase):# Close scoringscoring.close()+ # Test model caching+ scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, models=models)+ self.assertIsNotNone(scoring.model)+ scoring.close()++ def testSparseEmpty(self):+ """+ Test empty sparse vectors+ """++ scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"})+ scoring.upsert((uid, {"text": text}, tags) for uid, text, tags in self.data)+ self.assertEqual(scoring.count(), len(self.data))++ @patch("torch.cuda.device_count")+ def testSparseGPU(self, count):+ """+ Test sparse vectors with GPU encoding+ """+# Mock accelerator countcount.return_value = 2@@ -173,9 +189,25 @@ class TestScoring(unittest.TestCase):self.assertIsNotNone(scoring)scoring.close()- # Test model caching- scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, models=models)- self.assertIsNotNone(scoring.model)+ def testSparseIVFFlat(self):+ """+ Test sparse vectors with IVFFlat clustering+ """++ # Expand dataset+ data = self.data * 1000++ # Test higher volume IVFFlat index with clustering+ config = {"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq", "ivf": {"sample": 1.0}}+ scoring = ScoringFactory.create(config)+ scoring.index((uid, {"text": text}, tags) for uid, text, tags in data)++ # Recreate index+ self.save(scoring, config, "scoring.sparse.index")++ # Run search and validate correct result returned+ index, _ = scoring.search("lottery ticket", 1)[0]+ self.assertEqual(data[index], self.data[4])scoring.close()def testTFIDF(self):