Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update API to include support for embeddings upsert/delete/count (#86)

davidmezzetti368e822b6f4a
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index 96a023d9..e416158c 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -23,12 +23,13 @@ class API:
# Initialize member variables
self.config, self.documents, self.embeddings = config, None, None
- # Create/load embeddings index depending on writable flag
- if self.config.get("writable"):
- self.embeddings = Embeddings(self.config["embeddings"])
- elif self.config.get("path"):
+ if self.config.get("path") and Embeddings().exists(self.config["path"]):
+ # Load existing index if available
self.embeddings = Embeddings()
self.embeddings.load(self.config["path"])
+ elif self.config.get("embeddings"):
+ # Initialize empty embeddings
+ self.embeddings = Embeddings(self.config["embeddings"])
# Create pipelines
self.pipes()
@@ -159,8 +160,7 @@ class API:
def index(self):
"""
- Builds an embeddings index for previously batched documents. No further documents can be added
- after this call.
+ Builds an embeddings index for previously batched documents.
"""
if self.embeddings and self.config.get("writable") and self.documents:
@@ -171,13 +171,60 @@ class API:
# Build embeddings index
self.embeddings.index(self.documents)
- # Save index
- self.embeddings.save(self.config["path"])
+ # Save index if path available, otherwise this is an memory-only index
+ if self.config.get("path"):
+ self.embeddings.save(self.config["path"])
+
+ # Reset document stream
+ self.documents.close()
+ self.documents = None
+
+ def upsert(self):
+ """
+ Runs an embeddings upsert operation for previously batched documents.
+ """
+
+ if self.embeddings and self.config.get("writable") and self.documents:
+ # Run upsert
+ self.embeddings.upsert(self.documents)
+
+ # Save index if path available, otherwise this is an memory-only index
+ if self.config.get("path"):
+ self.embeddings.save(self.config["path"])
# Reset document stream
self.documents.close()
self.documents = None
+ def delete(self, ids):
+ """
+ Deletes from an embeddings index. Returns list of ids deleted.
+
+ Args:
+ ids: list of ids to delete
+
+ Returns:
+ ids deleted
+ """
+
+ if self.embeddings and self.config.get("writable"):
+ return self.embeddings.delete(ids)
+
+ return None
+
+ def count(self):
+ """
+ Total number of elements in this embeddings index.
+
+ Returns:
+ number of elements in embeddings index
+ """
+
+ if self.embeddings:
+ return self.embeddings.count()
+
+ return None
+
def similarity(self, query, texts):
"""
Computes the similarity between query and list of text. Returns a list of
diff --git a/src/python/txtai/api/routers/embeddings.py b/src/python/txtai/api/routers/embeddings.py
index 8bf98dc7..75df7ba0 100644
--- a/src/python/txtai/api/routers/embeddings.py
+++ b/src/python/txtai/api/routers/embeddings.py
@@ -62,13 +62,51 @@ def add(documents: List[dict] = Body(...)):
@router.get("/index")
def index():
"""
- Builds an embeddings index for previously batched documents. No further documents can be added
- after this call.
+ Builds an embeddings index for previously batched documents.
"""
application.get().index()
+def upsert():
+ """
+ Runs an embeddings upsert operation for previously batched documents.
+ """
+
+ application.get().upsert()
+
+
+def delete(ids: List = Body(...)):
+ """
+ Deletes from an embeddings index. Returns list of ids deleted.
+
+ Args:
+ ids: list of ids to delete
+
+ Returns:
+ ids deleted
+ """
+
+ application.get().delete(ids)
+
+
+def count():
+ """
+ Deletes from an embeddings index. Returns list of ids deleted.
+
+ Args:
+ ids: list of ids to delete
+
+ Returns:
+ ids deleted
+ """
+
+ return application.get().count()
+
+
@router.get("/transform")
def transform(text: str):
"""
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index ea9c3ee1..b008f2f2 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -70,16 +70,7 @@ class Embeddings:
"""
# Transform documents to embeddings vectors
- ids, dimensions, stream = self.model.index(documents)
-
- # Load streamed embeddings back to memory
- embeddings = np.empty((len(ids), dimensions), dtype=np.float32)
- with open(stream, "rb") as queue:
- for x in range(embeddings.shape[0]):
- embeddings[x] = pickle.load(queue)
-
- # Remove temporary file
- os.remove(stream)
+ ids, dimensions, embeddings = self.vectors(documents)
# Build LSA model (if enabled). Remove principal components from embeddings.
if self.config.get("pca"):
@@ -115,16 +106,7 @@ class Embeddings:
return
# Transform documents to embeddings vectors
- ids, dimensions, stream = self.model.index(documents)
-
- # Load streamed embeddings back to memory
- embeddings = np.empty((len(ids), dimensions), dtype=np.float32)
- with open(stream, "rb") as queue:
- for x in range(embeddings.shape[0]):
- embeddings[x] = pickle.load(queue)
-
- # Remove temporary file
- os.remove(stream)
+ ids, _, embeddings = self.vectors(documents)
# Normalize embeddings
self.normalize(embeddings)
@@ -342,6 +324,19 @@ class Embeddings:
# Sentence vectors model - transforms text into sentence embeddings
self.model = self.loadVectors()
+ def exists(self, path):
+ """
+ Checks if an index exists at path.
+
+ Args:
+ path: input directory path
+
+ Returns:
+ true if index exists, false otherwise
+ """
+
+ return os.path.exists("%s/config" % path) and os.path.exists("%s/embeddings" % path)
+
def save(self, path):
"""
Saves a model.
@@ -386,6 +381,31 @@ class Embeddings:
return VectorsFactory.create(self.config, self.scoring)
+ def vectors(self, documents):
+ """
+ Transforms documents into embeddings vectors.
+
+ Args:
+ documents: list of (id, text|tokens, tags)
+
+ Returns:
+ tuple of document ids, dimensions and embeddings
+ """
+
+ # Transform documents to embeddings vectors
+ ids, dimensions, stream = self.model.index(documents)
+
+ # Load streamed embeddings back to memory
+ embeddings = np.empty((len(ids), dimensions), dtype=np.float32)
+ with open(stream, "rb") as queue:
+ for x in range(embeddings.shape[0]):
+ embeddings[x] = pickle.load(queue)
+
+ # Remove temporary file
+ os.remove(stream)
+
+ return (ids, dimensions, embeddings)
+
def buildLSA(self, embeddings, components):
"""
Builds a LSA model. This model is used to remove the principal component within embeddings. This helps to
diff --git a/test/python/testapi/testembeddings.py b/test/python/testapi/testembeddings.py
index 5c504f55..f791c504 100644
--- a/test/python/testapi/testembeddings.py
+++ b/test/python/testapi/testembeddings.py
@@ -87,6 +87,24 @@ class TestEmbeddings(unittest.TestCase):
cls.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(cls.data)])
cls.client.get("index")
+ def testDelete(self):
+ """
+ Test delete via API
+ """
+
+ # Delete best match
+ self.client.post("delete", json=[4])
+
+ # Search for best match
+ uid = self.client.get("search?query=feel%20good%20story&limit=1").json()[0]["id"]
+
+ self.assertEqual(self.client.get("count").json(), 5)
+ self.assertEqual(uid, 5)
+
+ # Reset data
+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(self.data)])
+ self.client.get("index")
+
def testEmpty(self):
"""
Test empty API configuration
@@ -147,7 +165,7 @@ class TestEmbeddings(unittest.TestCase):
"""
uid = self.client.get("search?query=feel%20good%20story&limit=1").json()[0]["id"]
- self.assertEqual(self.data[uid], self.data[4])
+ self.assertEqual(uid, 4)
def testSearchBatch(self):
"""
@@ -166,7 +184,7 @@ class TestEmbeddings(unittest.TestCase):
uid = self.client.post("similarity", json={"query": "feel good story", "texts": self.data}).json()[0]["id"]
- self.assertEqual(self.data[uid], self.data[4])
+ self.assertEqual(uid, 4)
def testSimilarityBatch(self):
"""
@@ -195,6 +213,24 @@ class TestEmbeddings(unittest.TestCase):
self.assertEqual(len(embeddings), len(self.data))
self.assertEqual(len(embeddings[0]), 768)
+ def testUpsert(self):
+ """
+ Test upsert via API
+ """
+
+ # Update data
+ self.client.post("add", json=[{"id": 0, "text": "Feel good story: baby panda born"}])
+ self.client.get("upsert")
+
+ # Search for best match
+ uid = self.client.get("search?query=feel%20good%20story&limit=1").json()[0]["id"]
+
+ self.assertEqual(uid, 0)
+
+ # Reset data
+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(self.data)])
+ self.client.get("index")
+
def testViewOnly(self):
"""
Test read-only API instance
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index b25f96ab..3132d50e 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -33,58 +33,35 @@ class TestEmbeddings(unittest.TestCase):
# Create embeddings model, backed by sentence-transformers & transformers
cls.embeddings = Embeddings({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"})
- def testIndex(self):
+ def testDelete(self):
"""
- Test index
+ Test delete
"""
# Create an index for the list of text
self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
- # Search for best match
- uid = self.embeddings.search("feel good story", 1)[0][0]
-
- self.assertEqual(self.data[uid], self.data[4])
-
- def testUpsert(self):
- """
- Test upsert
- """
-
- # Build data array
- data = [(uid, text, None) for uid, text in enumerate(self.data)]
-
- # Reset embeddings for test
- self.embeddings.embeddings = None
-
- # Create an index for the list of text
- self.embeddings.upsert(data)
-
- # Update data
- data[0] = (0, "Feel good story: baby panda born", None)
- self.embeddings.upsert([data[0]])
+ # Delete best match
+ self.embeddings.delete([4])
# Search for best match
uid = self.embeddings.search("feel good story", 1)[0][0]
- self.assertEqual(data[uid], data[0])
+ self.assertEqual(self.embeddings.count(), 5)
+ self.assertEqual(uid, 5)
- def testDelete(self):
+ def testIndex(self):
"""
- Test delete
+ Test index
"""
# Create an index for the list of text
self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
- # Delete best match
- self.embeddings.delete([4])
-
# Search for best match
uid = self.embeddings.search("feel good story", 1)[0][0]
- self.assertEqual(self.embeddings.count(), 5)
- self.assertEqual(self.data[uid], self.data[5])
+ self.assertEqual(uid, 4)
def testSave(self):
"""
@@ -103,7 +80,7 @@ class TestEmbeddings(unittest.TestCase):
# Search for best match
uid = self.embeddings.search("feel good story", 1)[0][0]
- self.assertEqual(self.data[uid], self.data[4])
+ self.assertEqual(uid, 4)
# Test offsets still work after save/load
self.embeddings.upsert([(0, "Test insert", None)])
@@ -117,7 +94,30 @@ class TestEmbeddings(unittest.TestCase):
# Get best matching id
uid = self.embeddings.similarity("feel good story", self.data)[0][0]
- self.assertEqual(self.data[uid], self.data[4])
+ self.assertEqual(uid, 4)
+
+ def testUpsert(self):
+ """
+ Test upsert
+ """
+
+ # Build data array
+ data = [(uid, text, None) for uid, text in enumerate(self.data)]
+
+ # Reset embeddings for test
+ self.embeddings.embeddings = None
+
+ # Create an index for the list of text
+ self.embeddings.upsert(data)
+
+ # Update data
+ data[0] = (0, "Feel good story: baby panda born", None)
+ self.embeddings.upsert([data[0]])
+
+ # Search for best match
+ uid = self.embeddings.search("feel good story", 1)[0][0]
+
+ self.assertEqual(uid, 0)
def testWords(self):
"""
diff --git a/test/python/testsummary.py b/test/python/testsummary.py
index ec34b49b..144521c4 100644
--- a/test/python/testsummary.py
+++ b/test/python/testsummary.py
@@ -9,7 +9,7 @@ from txtai.pipeline import Summary
class TestSummary(unittest.TestCase):
"""
- Labels tests
+ Summary tests
"""
@classmethod
diff --git a/test/python/testtextractor.py b/test/python/testtextractor.py
index 757d87ba..1a3f3220 100644
--- a/test/python/testtextractor.py
+++ b/test/python/testtextractor.py
@@ -10,23 +10,23 @@ from txtai.pipeline import Textractor
from utils import Utils
-class TestSummary(unittest.TestCase):
+class TestTextractor(unittest.TestCase):
"""
- Labels tests
+ Textractor tests
"""
- def testSingle(self):
+ def testParagraphs(self):
"""
- Tests a single extraction with no tokenization of the results
+ Tests extraction to paragraphs
"""
- textractor = Textractor()
+ textractor = Textractor(paragraphs=True)
- # Extract text as a single block
- text = textractor(Utils.PATH + "/article.pdf")
+ # Extract text as sentences
+ paragraphs = textractor(Utils.PATH + "/article.pdf")
- # Check length of text is as expected
- self.assertEqual(len(text), 2301)
+ # Check number of paragraphs is as expected
+ self.assertEqual(len(paragraphs), 13)
def testSentences(self):
"""
@@ -41,15 +41,15 @@ class TestSummary(unittest.TestCase):
# Check number of sentences is as expected
self.assertEqual(len(sentences), 17)
- def testParagraphs(self):
+ def testSingle(self):
"""
- Tests extraction to paragraphs
+ Tests a single extraction with no tokenization of the results
"""
- textractor = Textractor(paragraphs=True)
+ textractor = Textractor()
- # Extract text as sentences
- paragraphs = textractor(Utils.PATH + "/article.pdf")
+ # Extract text as a single block
+ text = textractor(Utils.PATH + "/article.pdf")
- # Check number of paragraphs is as expected
- self.assertEqual(len(paragraphs), 13)
+ # Check length of text is as expected
+ self.assertEqual(len(text), 2301)
diff --git a/test/python/testtransformers.py b/test/python/testtransformers.py
index a46431ac..b3ea51b8 100644
--- a/test/python/testtransformers.py
+++ b/test/python/testtransformers.py
@@ -69,7 +69,7 @@ class TestTransformersVectors(unittest.TestCase):
self.assertFalse(np.array_equal(embeddings1[0], embeddings2[0]))
self.assertTrue(np.array_equal(embeddings1[1], embeddings2[1]))
- def testLongTransform(self):
+ def testTransformLong(self):
"""
Test transformers transform on long text
"""
diff --git a/test/python/testtranslation.py b/test/python/testtranslation.py
index c8142dbc..1a0cfc67 100644
--- a/test/python/testtranslation.py
+++ b/test/python/testtranslation.py
@@ -13,6 +13,19 @@ class TestTranslation(unittest.TestCase):
Translation tests
"""
+ @unittest.skipIf(os.name == "nt", "M2M100 skipped on Windows")
+ def testM2M100Translation(self):
+ """
+ Tests a translation using M2M100 models
+ """
+
+ translate = Translation()
+
+ text = translate("This is a test translation to Croatian", "hr")
+
+ # Validate translation text
+ self.assertEqual(text, "Ovo je testni prijevod na hrvatski")
+
def testMarianTranslation(self):
"""
Tests a translation using Marian models
@@ -30,18 +43,18 @@ class TestTranslation(unittest.TestCase):
translation = translate(translation, "en")
self.assertEqual(translation, text)
- @unittest.skipIf(os.name == "nt", "M2M100 skipped on Windows")
- def testM2M100Translation(self):
+ def testLongTranslation(self):
"""
- Tests a translation using M2M100 models
+ Tests a translation longer than max tokenization length.
"""
translate = Translation()
- text = translate("This is a test translation to Croatian", "hr")
+ text = "This is a test translation to Spanish. " * 100
+ translation = translate(text, "es")
# Validate translation text
- self.assertEqual(text, "Ovo je testni prijevod na hrvatski")
+ self.assertIsNotNone(translation)
def testNoTranslation(self):
"""
@@ -55,16 +68,3 @@ class TestTranslation(unittest.TestCase):
# Validate no translation
self.assertEqual(text, translation)
-
- def testLongTranslation(self):
- """
- Tests a translation longer than max tokenization length.
- """
-
- translate = Translation()
-
- text = "This is a test translation to Spanish. " * 100
- translation = translate(text, "es")
-
- # Validate translation text
- self.assertIsNotNone(translation)