Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add support for update/delete embeddings index operations, closes #86

davidmezzetti75ec616e2a2f
diff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/annoy.py
index 28540f95..cdb2bf1d 100644
--- a/src/python/txtai/ann/annoy.py
+++ b/src/python/txtai/ann/annoy.py
@@ -6,7 +6,7 @@ from annoy import AnnoyIndex
from .base import ANN
-
+# pylint: disable=W0223
class Annoy(ANN):
"""
Builds an ANN model using the Annoy library.
@@ -46,6 +46,10 @@ class Annoy(ANN):
return results
+ def count(self):
+ # Number of items in index
+ return self.model.get_n_items()
+
def save(self, path):
# Write index
self.model.save(path)
diff --git a/src/python/txtai/ann/base.py b/src/python/txtai/ann/base.py
index 8386b68f..666d052f 100644
--- a/src/python/txtai/ann/base.py
+++ b/src/python/txtai/ann/base.py
@@ -16,6 +16,9 @@ class ANN:
# ANN index
self.model = None
+ # Starting id offset
+ self.offset = 0
+
# Model configuration
self.config = config
@@ -24,6 +27,8 @@ class ANN:
Loads an ANN model at path.
"""
+ raise NotImplementedError
+
def index(self, embeddings):
"""
Builds an ANN model.
@@ -32,6 +37,28 @@ class ANN:
embeddings: embeddings array
"""
+ raise NotImplementedError
+
+ def append(self, embeddings):
+ """
+ Append elements to an existing model.
+
+ Args:
+ embeddings: embeddings array
+ """
+
+ raise NotImplementedError
+
+ def delete(self, ids):
+ """
+ Deletes elements from existing model.
+
+ Args:
+ ids: ids to delete
+ """
+
+ raise NotImplementedError
+
def search(self, queries, limit):
"""
Searches ANN model for query. Returns topn results.
@@ -44,11 +71,25 @@ class ANN:
query results
"""
+ raise NotImplementedError
+
+ def count(self):
+ """
+ Number of elements in the ANN model.
+
+ Returns:
+ count
+ """
+
+ raise NotImplementedError
+
def save(self, path):
"""
Saves an ANN model at path.
"""
+ raise NotImplementedError
+
def setting(self, name, default=None):
"""
Looks up backend specific setting.
diff --git a/src/python/txtai/ann/faiss.py b/src/python/txtai/ann/faiss.py
index 714f111c..657626ef 100644
--- a/src/python/txtai/ann/faiss.py
+++ b/src/python/txtai/ann/faiss.py
@@ -40,7 +40,23 @@ class Faiss(ANN):
# Train model
self.model.train(embeddings)
- self.model.add_with_ids(embeddings, np.array(range(embeddings.shape[0])))
+ self.model.add_with_ids(embeddings, np.arange(embeddings.shape[0]))
+
+ # Update id offset
+ self.offset = embeddings.shape[0]
+
+ def append(self, embeddings):
+ new = embeddings.shape[0]
+
+ # Append new ids
+ self.model.add_with_ids(embeddings, np.arange(self.offset, self.offset + new))
+
+ # Update id offset
+ self.offset += new
+
+ def delete(self, ids):
+ # Remove specified ids
+ self.model.remove_ids(np.array(ids))
def search(self, queries, limit):
# Run the query
@@ -54,6 +70,9 @@ class Faiss(ANN):
return results
+ def count(self):
+ return self.model.ntotal
+
def save(self, path):
# Write index
write_index(self.model, path)
diff --git a/src/python/txtai/ann/hnsw.py b/src/python/txtai/ann/hnsw.py
index 2754c118..092680ab 100644
--- a/src/python/txtai/ann/hnsw.py
+++ b/src/python/txtai/ann/hnsw.py
@@ -15,6 +15,13 @@ class HNSW(ANN):
Builds an ANN model using the hnswlib library.
"""
+ def __init__(self, config):
+ # Parent constructor
+ super().__init__(config)
+
+ # Track number of successful deletes
+ self.deletes = 0
+
def load(self, path):
# Load index
self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])
@@ -34,7 +41,32 @@ class HNSW(ANN):
self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)
# Add items
- self.model.add_items(embeddings, np.array(range(embeddings.shape[0])))
+ self.model.add_items(embeddings, np.arange(embeddings.shape[0]))
+
+ # Update id offset
+ self.offset = embeddings.shape[0]
+
+ def append(self, embeddings):
+ new = embeddings.shape[0]
+
+ # Resize index
+ self.model.resize_index(self.offset + new)
+
+ # Append new ids
+ self.model.add_items(embeddings, np.arange(self.offset, self.offset + new))
+
+ # Update id offset
+ self.offset += new
+
+ def delete(self, ids):
+ # Mark elements as deleted to omit from search results
+ for uid in ids:
+ try:
+ self.model.mark_deleted(uid)
+ self.deletes += 1
+ except RuntimeError:
+ # Ignore label not found error
+ continue
def search(self, queries, limit):
# Set ef query param
@@ -55,6 +87,9 @@ class HNSW(ANN):
return results
+ def count(self):
+ return self.model.get_current_count() - self.deletes
+
def save(self, path):
# Write index
self.model.save_index(path)
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index 17e00244..e2d6fb48 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -49,16 +49,6 @@ class Embeddings:
# Sentence vectors model
self.model = self.loadVectors() if self.config else None
- def loadVectors(self):
- """
- Loads a vector model set in config.
-
- Returns:
- vector model
- """
-
- return VectorsFactory.create(self.config, self.scoring)
-
def score(self, documents):
"""
Builds a scoring index.
@@ -73,7 +63,7 @@ class Embeddings:
def index(self, documents):
"""
- Builds an embeddings index.
+ Builds an embeddings index. This method overwrites an existing index.
Args:
documents: list of (id, text|tokens, tags)
@@ -109,61 +99,78 @@ class Embeddings:
# Build the index
self.embeddings.index(embeddings)
- def buildLSA(self, embeddings, components):
+ def upsert(self, documents):
"""
- Builds a LSA model. This model is used to remove the principal component within embeddings. This helps to
- smooth out noisy embeddings (common words with less value).
+ Runs an embeddings index upsert operation. If the index exists, new
+ data is appended to the index, existing data is updated. If the index
+ doesn't exist, this method runs a standard index operation.
Args:
- embeddings: input embeddings matrix
- components: number of model components
-
- Returns:
- LSA model
+ documents: list of (id, text|tokens, tags)
"""
- svd = TruncatedSVD(n_components=components, random_state=0)
- svd.fit(embeddings)
+ # Run standard insert if index doesn't exist
+ if not self.embeddings:
+ self.index(documents)
+ return
- return svd
+ # Transform documents to embeddings vectors
+ ids, dimensions, stream = self.model.index(documents)
- def removePC(self, embeddings):
- """
- Applies a LSA model to embeddings, removed the top n principal components. Operation applied
- directly on array.
+ # Load streamed embeddings back to memory
+ embeddings = np.empty((len(ids), dimensions), dtype=np.float32)
+ with open(stream, "rb") as queue:
+ for x in range(embeddings.shape[0]):
+ embeddings[x] = pickle.load(queue)
- Args:
- embeddings: input embeddings matrix
- """
+ # Remove temporary file
+ os.remove(stream)
- pc = self.lsa.components_
- factor = embeddings.dot(pc.transpose())
+ # Normalize embeddings
+ self.normalize(embeddings)
- # Apply LSA model
- # Calculation is different if n_components = 1
- if pc.shape[0] == 1:
- embeddings -= factor * pc
- elif len(embeddings.shape) > 1:
- # Apply model on a row-wise basis to limit memory usage
- for x in range(embeddings.shape[0]):
- embeddings[x] -= factor[x].dot(pc)
- else:
- # Single embedding
- embeddings -= factor.dot(pc)
+ # Delete existing elements
+ self.delete(documents)
- def normalize(self, embeddings):
+ # Append elements the index
+ self.embeddings.append(embeddings)
+
+ # Save embeddings metadata
+ self.config["ids"] = self.config["ids"] + ids
+
+ def delete(self, documents):
"""
- Normalizes embeddings using L2 normalization. Operation applied directly on array.
+ Deletes from an embeddings index. Text and tags can be None. Returns list of ids deleted.
Args:
- embeddings: input embeddings matrix
+ documents: list of (id, text|token, tags)
+
+ Returns:
+ ids
"""
- # Calculation is different for matrices vs vectors
- if len(embeddings.shape) > 1:
- embeddings /= np.linalg.norm(embeddings, axis=1)[:, np.newaxis]
- else:
- embeddings /= np.linalg.norm(embeddings)
+ # List of internal ids to delete
+ ids = []
+ deletes = []
+
+ # Get handle to config ids
+ cids = self.config["ids"]
+
+ # Find existing ids
+ for uid, _, _ in documents:
+ ids.extend([index for index, value in enumerate(cids) if uid == value])
+
+ # Delete any found from config ids and embeddings
+ if ids:
+ # Clear config ids
+ for index in ids:
+ deletes.append(cids[index])
+ cids[index] = None
+
+ # Delete ids from index
+ self.embeddings.delete(ids)
+
+ return deletes
def transform(self, document):
"""
@@ -202,6 +209,16 @@ class Embeddings:
return [self.transform(document) for document in documents]
+ def count(self):
+ """
+ Total number of elements in this embeddings index.
+
+ Returns:
+ number of elements in embeddings index
+ """
+
+ return self.embeddings.count()
+
def search(self, query, limit=3):
"""
Finds documents in the embeddings model most similar to the input query. Returns
@@ -356,3 +373,69 @@ class Embeddings:
# Save embedding scoring
if self.scoring:
self.scoring.save(path)
+
+ def loadVectors(self):
+ """
+ Loads a vector model set in config.
+
+ Returns:
+ vector model
+ """
+
+ return VectorsFactory.create(self.config, self.scoring)
+
+ def buildLSA(self, embeddings, components):
+ """
+ Builds a LSA model. This model is used to remove the principal component within embeddings. This helps to
+ smooth out noisy embeddings (common words with less value).
+
+ Args:
+ embeddings: input embeddings matrix
+ components: number of model components
+
+ Returns:
+ LSA model
+ """
+
+ svd = TruncatedSVD(n_components=components, random_state=0)
+ svd.fit(embeddings)
+
+ return svd
+
+ def removePC(self, embeddings):
+ """
+ Applies a LSA model to embeddings, removed the top n principal components. Operation applied
+ directly on array.
+
+ Args:
+ embeddings: input embeddings matrix
+ """
+
+ pc = self.lsa.components_
+ factor = embeddings.dot(pc.transpose())
+
+ # Apply LSA model
+ # Calculation is different if n_components = 1
+ if pc.shape[0] == 1:
+ embeddings -= factor * pc
+ elif len(embeddings.shape) > 1:
+ # Apply model on a row-wise basis to limit memory usage
+ for x in range(embeddings.shape[0]):
+ embeddings[x] -= factor[x].dot(pc)
+ else:
+ # Single embedding
+ embeddings -= factor.dot(pc)
+
+ def normalize(self, embeddings):
+ """
+ Normalizes embeddings using L2 normalization. Operation applied directly on array.
+
+ Args:
+ embeddings: input embeddings matrix
+ """
+
+ # Calculation is different for matrices vs vectors
+ if len(embeddings.shape) > 1:
+ embeddings /= np.linalg.norm(embeddings, axis=1)[:, np.newaxis]
+ else:
+ embeddings /= np.linalg.norm(embeddings)
diff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.py
index cbb101c0..45dc8cfd 100644
--- a/src/python/txtai/vectors/base.py
+++ b/src/python/txtai/vectors/base.py
@@ -13,14 +13,15 @@ class Vectors:
self.config = config
self.scoring = scoring
- # Detect if this is an initialized configuration
- self.initialized = "dimensions" in config
+ if config:
+ # Detect if this is an initialized configuration
+ self.initialized = "dimensions" in config
- # Enables optional string tokenization
- self.tokenize = "tokenize" not in config or config["tokenize"]
+ # Enables optional string tokenization
+ self.tokenize = "tokenize" not in config or config["tokenize"]
- # pylint: disable=E1111
- self.model = self.load(config["path"])
+ # pylint: disable=E1111
+ self.model = self.load(config["path"])
def load(self, path):
"""
@@ -33,6 +34,8 @@ class Vectors:
vector model
"""
+ raise NotImplementedError
+
def index(self, documents):
"""
Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,
@@ -45,6 +48,8 @@ class Vectors:
(ids, dimensions, stream)
"""
+ raise NotImplementedError
+
def transform(self, document):
"""
Transforms document into an embeddings vector.
@@ -55,3 +60,5 @@ class Vectors:
Returns:
embeddings vector
"""
+
+ raise NotImplementedError
diff --git a/test/python/testann.py b/test/python/testann.py
index 526080ea..f82797b5 100644
--- a/test/python/testann.py
+++ b/test/python/testann.py
@@ -8,7 +8,7 @@ import unittest
import numpy as np
-from txtai.ann import ANNFactory
+from txtai.ann import ANNFactory, ANN
class TestANN(unittest.TestCase):
@@ -21,9 +21,7 @@ class TestANN(unittest.TestCase):
Test Annoy backend
"""
- self.assertEqual(self.backend("annoy").config["backend"], "annoy")
- self.assertIsNotNone(self.save("annoy"))
- self.assertGreater(self.search("annoy"), 0)
+ self.runTests("annoy", None, False)
def testAnnoyCustom(self):
"""
@@ -31,11 +29,7 @@ class TestANN(unittest.TestCase):
"""
# Test with custom settings
- config = {"annoy": {"ntrees": 2, "searchk": 1}}
-
- self.assertEqual(self.backend("annoy", config).config["backend"], "annoy")
- self.assertIsNotNone(self.save("annoy", config))
- self.assertGreater(self.search("annoy", config), 0)
+ self.runTests("annoy", {"annoy": {"ntrees": 2, "searchk": 1}}, False)
@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")
def testFaiss(self):
@@ -43,11 +37,7 @@ class TestANN(unittest.TestCase):
Test Faiss backend
"""
- self.assertEqual(self.backend("faiss").config["backend"], "faiss")
- self.assertEqual(self.backend("faiss", None, 5000).config["backend"], "faiss")
-
- self.assertIsNotNone(self.save("faiss"))
- self.assertGreater(self.search("faiss"), 0)
+ self.runTests("faiss")
@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")
def testFaissCustom(self):
@@ -56,20 +46,14 @@ class TestANN(unittest.TestCase):
"""
# Test with custom settings
- config = {"faiss": {"nprobe": 2, "components": "PCA16,IDMap,SQ8"}}
-
- self.assertEqual(self.backend("faiss", config).config["backend"], "faiss")
- self.assertIsNotNone(self.save("faiss", config))
- self.assertGreater(self.search("faiss", config), 0)
+ self.runTests("faiss", {"faiss": {"nprobe": 2, "components": "PCA16,IDMap,SQ8"}}, False)
def testHnsw(self):
"""
Test Hnswlib backend
"""
- self.assertEqual(self.backend("hnsw").config["backend"], "hnsw")
- self.assertIsNotNone(self.save("hnsw"))
- self.assertGreater(self.search("hnsw"), 0)
+ self.runTests("hnsw")
def testHnswCustom(self):
"""
@@ -77,18 +61,48 @@ class TestANN(unittest.TestCase):
"""
# Test with custom settings
- config = {"hnsw": {"efconstruction": 100, "m": 4, "randomseed": 0, "efsearch": 5}}
+ self.runTests("hnsw", {"hnsw": {"efconstruction": 100, "m": 4, "randomseed": 0, "efsearch": 5}})
+
+ def testNotImplemented(self):
+ """
+ Tests exceptions for non-implemented methods
+ """
- self.assertEqual(self.backend("hnsw", config).config["backend"], "hnsw")
- self.assertIsNotNone(self.save("hnsw", config))
- self.assertGreater(self.search("hnsw", config), 0)
+ ann = ANN({})
+
+ self.assertRaises(NotImplementedError, ann.load, None)
+ self.assertRaises(NotImplementedError, ann.index, None)
+ self.assertRaises(NotImplementedError, ann.append, None)
+ self.assertRaises(NotImplementedError, ann.delete, None)
+ self.assertRaises(NotImplementedError, ann.search, None, None)
+ self.assertRaises(NotImplementedError, ann.count)
+ self.assertRaises(NotImplementedError, ann.save, None)
+
+ def runTests(self, name, params=None, update=True):
+ """
+ Runs a series of standard backend tests.
+
+ Args:
+ name: backend name
+ params: additional config parameters
+ update: If append/delete options should be tested
+ """
+
+ self.assertEqual(self.backend(name, params).config["backend"], name)
+ self.assertEqual(self.save(name, params).count(), 10000)
+
+ if update:
+ self.assertEqual(self.append(name, params, 500).count(), 10500)
+ self.assertEqual(self.delete(name, params, [0, 1]).count(), 9998)
+
+ self.assertGreater(self.search(name, params), 0)
def backend(self, name, params=None, length=10000):
"""
Test a backend.
Args:
- backend: backend name
+ name: backend name
params: additional config parameters
length: number of rows to generate
@@ -109,19 +123,62 @@ class TestANN(unittest.TestCase):
return model
- def save(self, backend, params=None):
+ def append(self, name, params=None, length=500):
+ """
+ Appends new data to index
+
+ Args:
+ name: backend name
+ params: additional config parameters
+ length: number of rows to generate
+
+ Returns:
+ ANN model
+ """
+
+ # Initial model
+ model = self.backend(name, params)
+
+ # Generate test data
+ data = np.random.rand(length, 300).astype(np.float32)
+ self.normalize(data)
+
+ model.append(data)
+
+ return model
+
+ def delete(self, name, params=None, ids=None):
+ """
+ Deletes data from index.
+
+ Args:
+ name: backend name
+ params: additional config parameters
+ ids: ids to delete
+
+ Returns:
+ ANN model
+ """
+
+ # Initial model
+ model = self.backend(name, params)
+ model.delete(ids)
+
+ return model
+
+ def save(self, name, params=None):
"""
Test save/load.
Args:
- backend: backend name
+ name: backend name
params: additional config parameters
Returns:
ANN model
"""
- model = self.backend(backend, params)
+ model = self.backend(name, params)
# Generate temp file path
index = os.path.join(tempfile.gettempdir(), "ann")
@@ -131,12 +188,12 @@ class TestANN(unittest.TestCase):
return model
- def search(self, backend, params=None):
+ def search(self, name, params=None):
"""
Test ANN search.
Args:
- backend: backend name
+ name: backend name
params: additional config parameters
Returns:
@@ -144,7 +201,7 @@ class TestANN(unittest.TestCase):
"""
# Generate ANN index
- model = self.backend(backend, params)
+ model = self.backend(name, params)
# Generate query vector
query = np.random.rand(300).astype(np.float32)
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index 7cf803a1..fdc2487d 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -46,6 +46,46 @@ class TestEmbeddings(unittest.TestCase):
self.assertEqual(self.data[uid], self.data[4])
+ def testUpsert(self):
+ """
+ Test upsert
+ """
+
+ # Build data array
+ data = [(uid, text, None) for uid, text in enumerate(self.data)]
+
+ # Reset embeddings for test
+ self.embeddings.embeddings = None
+
+ # Create an index for the list of text
+ self.embeddings.upsert(data)
+
+ # Update data
+ data[0] = (0, "See it first: baby panda is born at the local zoo", None)
+ self.embeddings.upsert([data[0]])
+
+ # Search for best match
+ uid = self.embeddings.search("feel good story", 1)[0][0]
+
+ self.assertEqual(data[uid], data[4])
+
+ def testDelete(self):
+ """
+ Test delete
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Delete best match
+ self.embeddings.delete([(4, None, None)])
+
+ # Search for best match
+ uid = self.embeddings.search("feel good story", 1)[0][0]
+
+ self.assertEqual(self.embeddings.count(), 5)
+ self.assertEqual(self.data[uid], self.data[5])
+
def testSave(self):
"""
Test save
diff --git a/test/python/testtransformers.py b/test/python/testtransformers.py
index 3ae29c22..a46431ac 100644
--- a/test/python/testtransformers.py
+++ b/test/python/testtransformers.py
@@ -21,9 +21,7 @@ class TestTransformersVectors(unittest.TestCase):
Create single TransformersVectors instance.
"""
- cls.model = VectorsFactory.create(
- {"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens", "maxlength": 512}, None
- )
+ cls.model = VectorsFactory.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)
def testIndex(self):
"""
diff --git a/test/python/testvectors.py b/test/python/testvectors.py
new file mode 100644
index 00000000..d754463c
--- /dev/null
+++ b/test/python/testvectors.py
@@ -0,0 +1,24 @@
+"""
+Vectors module tests
+"""
+
+import unittest
+
+from txtai.vectors import Vectors
+
+
+class TestVectors(unittest.TestCase):
+ """
+ Vectors tests
+ """
+
+ def testNotImplemented(self):
+ """
+ Tests exceptions for non-implemented methods
+ """
+
+ vectors = Vectors(None, None)
+
+ self.assertRaises(NotImplementedError, vectors.load, None)
+ self.assertRaises(NotImplementedError, vectors.index, None)
+ self.assertRaises(NotImplementedError, vectors.transform, None)