💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add support for update/delete embeddings index operations, closes #86
davidmezzetti75ec616e2a2f
diff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/annoy.pyindex 28540f95..cdb2bf1d 100644--- a/src/python/txtai/ann/annoy.py+++ b/src/python/txtai/ann/annoy.py@@ -6,7 +6,7 @@ from annoy import AnnoyIndexfrom .base import ANN-+# pylint: disable=W0223class Annoy(ANN):"""Builds an ANN model using the Annoy library.@@ -46,6 +46,10 @@ class Annoy(ANN):return results+ def count(self):+ # Number of items in index+ return self.model.get_n_items()+def save(self, path):# Write indexself.model.save(path)diff --git a/src/python/txtai/ann/base.py b/src/python/txtai/ann/base.pyindex 8386b68f..666d052f 100644--- a/src/python/txtai/ann/base.py+++ b/src/python/txtai/ann/base.py@@ -16,6 +16,9 @@ class ANN:# ANN indexself.model = None+ # Starting id offset+ self.offset = 0+# Model configurationself.config = config@@ -24,6 +27,8 @@ class ANN:Loads an ANN model at path."""+ raise NotImplementedError+def index(self, embeddings):"""Builds an ANN model.@@ -32,6 +37,28 @@ class ANN:embeddings: embeddings array"""+ raise NotImplementedError++ def append(self, embeddings):+ """+ Append elements to an existing model.++ Args:+ embeddings: embeddings array+ """++ raise NotImplementedError++ def delete(self, ids):+ """+ Deletes elements from existing model.++ Args:+ ids: ids to delete+ """++ raise NotImplementedError+def search(self, queries, limit):"""Searches ANN model for query. Returns topn results.@@ -44,11 +71,25 @@ class ANN:query results"""+ raise NotImplementedError++ def count(self):+ """+ Number of elements in the ANN model.++ Returns:+ count+ """++ raise NotImplementedError+def save(self, path):"""Saves an ANN model at path."""+ raise NotImplementedError+def setting(self, name, default=None):"""Looks up backend specific setting.diff --git a/src/python/txtai/ann/faiss.py b/src/python/txtai/ann/faiss.pyindex 714f111c..657626ef 100644--- a/src/python/txtai/ann/faiss.py+++ b/src/python/txtai/ann/faiss.py@@ -40,7 +40,23 @@ class Faiss(ANN):# Train modelself.model.train(embeddings)- self.model.add_with_ids(embeddings, np.array(range(embeddings.shape[0])))+ self.model.add_with_ids(embeddings, np.arange(embeddings.shape[0]))++ # Update id offset+ self.offset = embeddings.shape[0]++ def append(self, embeddings):+ new = embeddings.shape[0]++ # Append new ids+ self.model.add_with_ids(embeddings, np.arange(self.offset, self.offset + new))++ # Update id offset+ self.offset += new++ def delete(self, ids):+ # Remove specified ids+ self.model.remove_ids(np.array(ids))def search(self, queries, limit):# Run the query@@ -54,6 +70,9 @@ class Faiss(ANN):return results+ def count(self):+ return self.model.ntotal+def save(self, path):# Write indexwrite_index(self.model, path)diff --git a/src/python/txtai/ann/hnsw.py b/src/python/txtai/ann/hnsw.pyindex 2754c118..092680ab 100644--- a/src/python/txtai/ann/hnsw.py+++ b/src/python/txtai/ann/hnsw.py@@ -15,6 +15,13 @@ class HNSW(ANN):Builds an ANN model using the hnswlib library."""+ def __init__(self, config):+ # Parent constructor+ super().__init__(config)++ # Track number of successful deletes+ self.deletes = 0+def load(self, path):# Load indexself.model = Index(dim=self.config["dimensions"], space=self.config["metric"])@@ -34,7 +41,32 @@ class HNSW(ANN):self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)# Add items- self.model.add_items(embeddings, np.array(range(embeddings.shape[0])))+ self.model.add_items(embeddings, np.arange(embeddings.shape[0]))++ # Update id offset+ self.offset = embeddings.shape[0]++ def append(self, embeddings):+ new = embeddings.shape[0]++ # Resize index+ self.model.resize_index(self.offset + new)++ # Append new ids+ self.model.add_items(embeddings, np.arange(self.offset, self.offset + new))++ # Update id offset+ self.offset += new++ def delete(self, ids):+ # Mark elements as deleted to omit from search results+ for uid in ids:+ try:+ self.model.mark_deleted(uid)+ self.deletes += 1+ except RuntimeError:+ # Ignore label not found error+ continuedef search(self, queries, limit):# Set ef query param@@ -55,6 +87,9 @@ class HNSW(ANN):return results+ def count(self):+ return self.model.get_current_count() - self.deletes+def save(self, path):# Write indexself.model.save_index(path)diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 17e00244..e2d6fb48 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -49,16 +49,6 @@ class Embeddings:# Sentence vectors modelself.model = self.loadVectors() if self.config else None- def loadVectors(self):- """- Loads a vector model set in config.-- Returns:- vector model- """-- return VectorsFactory.create(self.config, self.scoring)-def score(self, documents):"""Builds a scoring index.@@ -73,7 +63,7 @@ class Embeddings:def index(self, documents):"""- Builds an embeddings index.+ Builds an embeddings index. This method overwrites an existing index.Args:documents: list of (id, text|tokens, tags)@@ -109,61 +99,78 @@ class Embeddings:# Build the indexself.embeddings.index(embeddings)- def buildLSA(self, embeddings, components):+ def upsert(self, documents):"""- Builds a LSA model. This model is used to remove the principal component within embeddings. This helps to- smooth out noisy embeddings (common words with less value).+ Runs an embeddings index upsert operation. If the index exists, new+ data is appended to the index, existing data is updated. If the index+ doesn't exist, this method runs a standard index operation.Args:- embeddings: input embeddings matrix- components: number of model components-- Returns:- LSA model+ documents: list of (id, text|tokens, tags)"""- svd = TruncatedSVD(n_components=components, random_state=0)- svd.fit(embeddings)+ # Run standard insert if index doesn't exist+ if not self.embeddings:+ self.index(documents)+ return- return svd+ # Transform documents to embeddings vectors+ ids, dimensions, stream = self.model.index(documents)- def removePC(self, embeddings):- """- Applies a LSA model to embeddings, removed the top n principal components. Operation applied- directly on array.+ # Load streamed embeddings back to memory+ embeddings = np.empty((len(ids), dimensions), dtype=np.float32)+ with open(stream, "rb") as queue:+ for x in range(embeddings.shape[0]):+ embeddings[x] = pickle.load(queue)- Args:- embeddings: input embeddings matrix- """+ # Remove temporary file+ os.remove(stream)- pc = self.lsa.components_- factor = embeddings.dot(pc.transpose())+ # Normalize embeddings+ self.normalize(embeddings)- # Apply LSA model- # Calculation is different if n_components = 1- if pc.shape[0] == 1:- embeddings -= factor * pc- elif len(embeddings.shape) > 1:- # Apply model on a row-wise basis to limit memory usage- for x in range(embeddings.shape[0]):- embeddings[x] -= factor[x].dot(pc)- else:- # Single embedding- embeddings -= factor.dot(pc)+ # Delete existing elements+ self.delete(documents)- def normalize(self, embeddings):+ # Append elements the index+ self.embeddings.append(embeddings)++ # Save embeddings metadata+ self.config["ids"] = self.config["ids"] + ids++ def delete(self, documents):"""- Normalizes embeddings using L2 normalization. Operation applied directly on array.+ Deletes from an embeddings index. Text and tags can be None. Returns list of ids deleted.Args:- embeddings: input embeddings matrix+ documents: list of (id, text|token, tags)++ Returns:+ ids"""- # Calculation is different for matrices vs vectors- if len(embeddings.shape) > 1:- embeddings /= np.linalg.norm(embeddings, axis=1)[:, np.newaxis]- else:- embeddings /= np.linalg.norm(embeddings)+ # List of internal ids to delete+ ids = []+ deletes = []++ # Get handle to config ids+ cids = self.config["ids"]++ # Find existing ids+ for uid, _, _ in documents:+ ids.extend([index for index, value in enumerate(cids) if uid == value])++ # Delete any found from config ids and embeddings+ if ids:+ # Clear config ids+ for index in ids:+ deletes.append(cids[index])+ cids[index] = None++ # Delete ids from index+ self.embeddings.delete(ids)++ return deletesdef transform(self, document):"""@@ -202,6 +209,16 @@ class Embeddings:return [self.transform(document) for document in documents]+ def count(self):+ """+ Total number of elements in this embeddings index.++ Returns:+ number of elements in embeddings index+ """++ return self.embeddings.count()+def search(self, query, limit=3):"""Finds documents in the embeddings model most similar to the input query. Returns@@ -356,3 +373,69 @@ class Embeddings:# Save embedding scoringif self.scoring:self.scoring.save(path)++ def loadVectors(self):+ """+ Loads a vector model set in config.++ Returns:+ vector model+ """++ return VectorsFactory.create(self.config, self.scoring)++ def buildLSA(self, embeddings, components):+ """+ Builds a LSA model. This model is used to remove the principal component within embeddings. This helps to+ smooth out noisy embeddings (common words with less value).++ Args:+ embeddings: input embeddings matrix+ components: number of model components++ Returns:+ LSA model+ """++ svd = TruncatedSVD(n_components=components, random_state=0)+ svd.fit(embeddings)++ return svd++ def removePC(self, embeddings):+ """+ Applies a LSA model to embeddings, removed the top n principal components. Operation applied+ directly on array.++ Args:+ embeddings: input embeddings matrix+ """++ pc = self.lsa.components_+ factor = embeddings.dot(pc.transpose())++ # Apply LSA model+ # Calculation is different if n_components = 1+ if pc.shape[0] == 1:+ embeddings -= factor * pc+ elif len(embeddings.shape) > 1:+ # Apply model on a row-wise basis to limit memory usage+ for x in range(embeddings.shape[0]):+ embeddings[x] -= factor[x].dot(pc)+ else:+ # Single embedding+ embeddings -= factor.dot(pc)++ def normalize(self, embeddings):+ """+ Normalizes embeddings using L2 normalization. Operation applied directly on array.++ Args:+ embeddings: input embeddings matrix+ """++ # Calculation is different for matrices vs vectors+ if len(embeddings.shape) > 1:+ embeddings /= np.linalg.norm(embeddings, axis=1)[:, np.newaxis]+ else:+ embeddings /= np.linalg.norm(embeddings)diff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.pyindex cbb101c0..45dc8cfd 100644--- a/src/python/txtai/vectors/base.py+++ b/src/python/txtai/vectors/base.py@@ -13,14 +13,15 @@ class Vectors:self.config = configself.scoring = scoring- # Detect if this is an initialized configuration- self.initialized = "dimensions" in config+ if config:+ # Detect if this is an initialized configuration+ self.initialized = "dimensions" in config- # Enables optional string tokenization- self.tokenize = "tokenize" not in config or config["tokenize"]+ # Enables optional string tokenization+ self.tokenize = "tokenize" not in config or config["tokenize"]- # pylint: disable=E1111- self.model = self.load(config["path"])+ # pylint: disable=E1111+ self.model = self.load(config["path"])def load(self, path):"""@@ -33,6 +34,8 @@ class Vectors:vector model"""+ raise NotImplementedError+def index(self, documents):"""Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,@@ -45,6 +48,8 @@ class Vectors:(ids, dimensions, stream)"""+ raise NotImplementedError+def transform(self, document):"""Transforms document into an embeddings vector.@@ -55,3 +60,5 @@ class Vectors:Returns:embeddings vector"""++ raise NotImplementedErrordiff --git a/test/python/testann.py b/test/python/testann.pyindex 526080ea..f82797b5 100644--- a/test/python/testann.py+++ b/test/python/testann.py@@ -8,7 +8,7 @@ import unittestimport numpy as np-from txtai.ann import ANNFactory+from txtai.ann import ANNFactory, ANNclass TestANN(unittest.TestCase):@@ -21,9 +21,7 @@ class TestANN(unittest.TestCase):Test Annoy backend"""- self.assertEqual(self.backend("annoy").config["backend"], "annoy")- self.assertIsNotNone(self.save("annoy"))- self.assertGreater(self.search("annoy"), 0)+ self.runTests("annoy", None, False)def testAnnoyCustom(self):"""@@ -31,11 +29,7 @@ class TestANN(unittest.TestCase):"""# Test with custom settings- config = {"annoy": {"ntrees": 2, "searchk": 1}}-- self.assertEqual(self.backend("annoy", config).config["backend"], "annoy")- self.assertIsNotNone(self.save("annoy", config))- self.assertGreater(self.search("annoy", config), 0)+ self.runTests("annoy", {"annoy": {"ntrees": 2, "searchk": 1}}, False)@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")def testFaiss(self):@@ -43,11 +37,7 @@ class TestANN(unittest.TestCase):Test Faiss backend"""- self.assertEqual(self.backend("faiss").config["backend"], "faiss")- self.assertEqual(self.backend("faiss", None, 5000).config["backend"], "faiss")-- self.assertIsNotNone(self.save("faiss"))- self.assertGreater(self.search("faiss"), 0)+ self.runTests("faiss")@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")def testFaissCustom(self):@@ -56,20 +46,14 @@ class TestANN(unittest.TestCase):"""# Test with custom settings- config = {"faiss": {"nprobe": 2, "components": "PCA16,IDMap,SQ8"}}-- self.assertEqual(self.backend("faiss", config).config["backend"], "faiss")- self.assertIsNotNone(self.save("faiss", config))- self.assertGreater(self.search("faiss", config), 0)+ self.runTests("faiss", {"faiss": {"nprobe": 2, "components": "PCA16,IDMap,SQ8"}}, False)def testHnsw(self):"""Test Hnswlib backend"""- self.assertEqual(self.backend("hnsw").config["backend"], "hnsw")- self.assertIsNotNone(self.save("hnsw"))- self.assertGreater(self.search("hnsw"), 0)+ self.runTests("hnsw")def testHnswCustom(self):"""@@ -77,18 +61,48 @@ class TestANN(unittest.TestCase):"""# Test with custom settings- config = {"hnsw": {"efconstruction": 100, "m": 4, "randomseed": 0, "efsearch": 5}}+ self.runTests("hnsw", {"hnsw": {"efconstruction": 100, "m": 4, "randomseed": 0, "efsearch": 5}})++ def testNotImplemented(self):+ """+ Tests exceptions for non-implemented methods+ """- self.assertEqual(self.backend("hnsw", config).config["backend"], "hnsw")- self.assertIsNotNone(self.save("hnsw", config))- self.assertGreater(self.search("hnsw", config), 0)+ ann = ANN({})++ self.assertRaises(NotImplementedError, ann.load, None)+ self.assertRaises(NotImplementedError, ann.index, None)+ self.assertRaises(NotImplementedError, ann.append, None)+ self.assertRaises(NotImplementedError, ann.delete, None)+ self.assertRaises(NotImplementedError, ann.search, None, None)+ self.assertRaises(NotImplementedError, ann.count)+ self.assertRaises(NotImplementedError, ann.save, None)++ def runTests(self, name, params=None, update=True):+ """+ Runs a series of standard backend tests.++ Args:+ name: backend name+ params: additional config parameters+ update: If append/delete options should be tested+ """++ self.assertEqual(self.backend(name, params).config["backend"], name)+ self.assertEqual(self.save(name, params).count(), 10000)++ if update:+ self.assertEqual(self.append(name, params, 500).count(), 10500)+ self.assertEqual(self.delete(name, params, [0, 1]).count(), 9998)++ self.assertGreater(self.search(name, params), 0)def backend(self, name, params=None, length=10000):"""Test a backend.Args:- backend: backend name+ name: backend nameparams: additional config parameterslength: number of rows to generate@@ -109,19 +123,62 @@ class TestANN(unittest.TestCase):return model- def save(self, backend, params=None):+ def append(self, name, params=None, length=500):+ """+ Appends new data to index++ Args:+ name: backend name+ params: additional config parameters+ length: number of rows to generate++ Returns:+ ANN model+ """++ # Initial model+ model = self.backend(name, params)++ # Generate test data+ data = np.random.rand(length, 300).astype(np.float32)+ self.normalize(data)++ model.append(data)++ return model++ def delete(self, name, params=None, ids=None):+ """+ Deletes data from index.++ Args:+ name: backend name+ params: additional config parameters+ ids: ids to delete++ Returns:+ ANN model+ """++ # Initial model+ model = self.backend(name, params)+ model.delete(ids)++ return model++ def save(self, name, params=None):"""Test save/load.Args:- backend: backend name+ name: backend nameparams: additional config parametersReturns:ANN model"""- model = self.backend(backend, params)+ model = self.backend(name, params)# Generate temp file pathindex = os.path.join(tempfile.gettempdir(), "ann")@@ -131,12 +188,12 @@ class TestANN(unittest.TestCase):return model- def search(self, backend, params=None):+ def search(self, name, params=None):"""Test ANN search.Args:- backend: backend name+ name: backend nameparams: additional config parametersReturns:@@ -144,7 +201,7 @@ class TestANN(unittest.TestCase):"""# Generate ANN index- model = self.backend(backend, params)+ model = self.backend(name, params)# Generate query vectorquery = np.random.rand(300).astype(np.float32)diff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex 7cf803a1..fdc2487d 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -46,6 +46,46 @@ class TestEmbeddings(unittest.TestCase):self.assertEqual(self.data[uid], self.data[4])+ def testUpsert(self):+ """+ Test upsert+ """++ # Build data array+ data = [(uid, text, None) for uid, text in enumerate(self.data)]++ # Reset embeddings for test+ self.embeddings.embeddings = None++ # Create an index for the list of text+ self.embeddings.upsert(data)++ # Update data+ data[0] = (0, "See it first: baby panda is born at the local zoo", None)+ self.embeddings.upsert([data[0]])++ # Search for best match+ uid = self.embeddings.search("feel good story", 1)[0][0]++ self.assertEqual(data[uid], data[4])++ def testDelete(self):+ """+ Test delete+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Delete best match+ self.embeddings.delete([(4, None, None)])++ # Search for best match+ uid = self.embeddings.search("feel good story", 1)[0][0]++ self.assertEqual(self.embeddings.count(), 5)+ self.assertEqual(self.data[uid], self.data[5])+def testSave(self):"""Test savediff --git a/test/python/testtransformers.py b/test/python/testtransformers.pyindex 3ae29c22..a46431ac 100644--- a/test/python/testtransformers.py+++ b/test/python/testtransformers.py@@ -21,9 +21,7 @@ class TestTransformersVectors(unittest.TestCase):Create single TransformersVectors instance."""- cls.model = VectorsFactory.create(- {"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens", "maxlength": 512}, None- )+ cls.model = VectorsFactory.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)def testIndex(self):"""diff --git a/test/python/testvectors.py b/test/python/testvectors.pynew file mode 100644index 00000000..d754463c--- /dev/null+++ b/test/python/testvectors.py@@ -0,0 +1,24 @@+"""+Vectors module tests+"""++import unittest++from txtai.vectors import Vectors+++class TestVectors(unittest.TestCase):+ """+ Vectors tests+ """++ def testNotImplemented(self):+ """+ Tests exceptions for non-implemented methods+ """++ vectors = Vectors(None, None)++ self.assertRaises(NotImplementedError, vectors.load, None)+ self.assertRaises(NotImplementedError, vectors.index, None)+ self.assertRaises(NotImplementedError, vectors.transform, None)