💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update API to include support for embeddings upsert/delete/count (#86)
davidmezzetti368e822b6f4a
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.pyindex 96a023d9..e416158c 100644--- a/src/python/txtai/api/base.py+++ b/src/python/txtai/api/base.py@@ -23,12 +23,13 @@ class API:# Initialize member variablesself.config, self.documents, self.embeddings = config, None, None- # Create/load embeddings index depending on writable flag- if self.config.get("writable"):- self.embeddings = Embeddings(self.config["embeddings"])- elif self.config.get("path"):+ if self.config.get("path") and Embeddings().exists(self.config["path"]):+ # Load existing index if availableself.embeddings = Embeddings()self.embeddings.load(self.config["path"])+ elif self.config.get("embeddings"):+ # Initialize empty embeddings+ self.embeddings = Embeddings(self.config["embeddings"])# Create pipelinesself.pipes()@@ -159,8 +160,7 @@ class API:def index(self):"""- Builds an embeddings index for previously batched documents. No further documents can be added- after this call.+ Builds an embeddings index for previously batched documents."""if self.embeddings and self.config.get("writable") and self.documents:@@ -171,13 +171,60 @@ class API:# Build embeddings indexself.embeddings.index(self.documents)- # Save index- self.embeddings.save(self.config["path"])+ # Save index if path available, otherwise this is an memory-only index+ if self.config.get("path"):+ self.embeddings.save(self.config["path"])++ # Reset document stream+ self.documents.close()+ self.documents = None++ def upsert(self):+ """+ Runs an embeddings upsert operation for previously batched documents.+ """++ if self.embeddings and self.config.get("writable") and self.documents:+ # Run upsert+ self.embeddings.upsert(self.documents)++ # Save index if path available, otherwise this is an memory-only index+ if self.config.get("path"):+ self.embeddings.save(self.config["path"])# Reset document streamself.documents.close()self.documents = None+ def delete(self, ids):+ """+ Deletes from an embeddings index. Returns list of ids deleted.++ Args:+ ids: list of ids to delete++ Returns:+ ids deleted+ """++ if self.embeddings and self.config.get("writable"):+ return self.embeddings.delete(ids)++ return None++ def count(self):+ """+ Total number of elements in this embeddings index.++ Returns:+ number of elements in embeddings index+ """++ if self.embeddings:+ return self.embeddings.count()++ return None+def similarity(self, query, texts):"""Computes the similarity between query and list of text. Returns a list ofdiff --git a/src/python/txtai/api/routers/embeddings.py b/src/python/txtai/api/routers/embeddings.pyindex 8bf98dc7..75df7ba0 100644--- a/src/python/txtai/api/routers/embeddings.py+++ b/src/python/txtai/api/routers/embeddings.py@@ -62,13 +62,51 @@ def add(documents: List[dict] = Body(...)):@router.get("/index")def index():"""- Builds an embeddings index for previously batched documents. No further documents can be added- after this call.+ Builds an embeddings index for previously batched documents."""application.get().index()[email protected]("/upsert")+def upsert():+ """+ Runs an embeddings upsert operation for previously batched documents.+ """++ application.get().upsert()++[email protected]("/delete")+def delete(ids: List = Body(...)):+ """+ Deletes from an embeddings index. Returns list of ids deleted.++ Args:+ ids: list of ids to delete++ Returns:+ ids deleted+ """++ application.get().delete(ids)++[email protected]("/count")+def count():+ """+ Deletes from an embeddings index. Returns list of ids deleted.++ Args:+ ids: list of ids to delete++ Returns:+ ids deleted+ """++ return application.get().count()++@router.get("/transform")def transform(text: str):"""diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex ea9c3ee1..b008f2f2 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -70,16 +70,7 @@ class Embeddings:"""# Transform documents to embeddings vectors- ids, dimensions, stream = self.model.index(documents)-- # Load streamed embeddings back to memory- embeddings = np.empty((len(ids), dimensions), dtype=np.float32)- with open(stream, "rb") as queue:- for x in range(embeddings.shape[0]):- embeddings[x] = pickle.load(queue)-- # Remove temporary file- os.remove(stream)+ ids, dimensions, embeddings = self.vectors(documents)# Build LSA model (if enabled). Remove principal components from embeddings.if self.config.get("pca"):@@ -115,16 +106,7 @@ class Embeddings:return# Transform documents to embeddings vectors- ids, dimensions, stream = self.model.index(documents)-- # Load streamed embeddings back to memory- embeddings = np.empty((len(ids), dimensions), dtype=np.float32)- with open(stream, "rb") as queue:- for x in range(embeddings.shape[0]):- embeddings[x] = pickle.load(queue)-- # Remove temporary file- os.remove(stream)+ ids, _, embeddings = self.vectors(documents)# Normalize embeddingsself.normalize(embeddings)@@ -342,6 +324,19 @@ class Embeddings:# Sentence vectors model - transforms text into sentence embeddingsself.model = self.loadVectors()+ def exists(self, path):+ """+ Checks if an index exists at path.++ Args:+ path: input directory path++ Returns:+ true if index exists, false otherwise+ """++ return os.path.exists("%s/config" % path) and os.path.exists("%s/embeddings" % path)+def save(self, path):"""Saves a model.@@ -386,6 +381,31 @@ class Embeddings:return VectorsFactory.create(self.config, self.scoring)+ def vectors(self, documents):+ """+ Transforms documents into embeddings vectors.++ Args:+ documents: list of (id, text|tokens, tags)++ Returns:+ tuple of document ids, dimensions and embeddings+ """++ # Transform documents to embeddings vectors+ ids, dimensions, stream = self.model.index(documents)++ # Load streamed embeddings back to memory+ embeddings = np.empty((len(ids), dimensions), dtype=np.float32)+ with open(stream, "rb") as queue:+ for x in range(embeddings.shape[0]):+ embeddings[x] = pickle.load(queue)++ # Remove temporary file+ os.remove(stream)++ return (ids, dimensions, embeddings)+def buildLSA(self, embeddings, components):"""Builds a LSA model. This model is used to remove the principal component within embeddings. This helps todiff --git a/test/python/testapi/testembeddings.py b/test/python/testapi/testembeddings.pyindex 5c504f55..f791c504 100644--- a/test/python/testapi/testembeddings.py+++ b/test/python/testapi/testembeddings.py@@ -87,6 +87,24 @@ class TestEmbeddings(unittest.TestCase):cls.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(cls.data)])cls.client.get("index")+ def testDelete(self):+ """+ Test delete via API+ """++ # Delete best match+ self.client.post("delete", json=[4])++ # Search for best match+ uid = self.client.get("search?query=feel%20good%20story&limit=1").json()[0]["id"]++ self.assertEqual(self.client.get("count").json(), 5)+ self.assertEqual(uid, 5)++ # Reset data+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(self.data)])+ self.client.get("index")+def testEmpty(self):"""Test empty API configuration@@ -147,7 +165,7 @@ class TestEmbeddings(unittest.TestCase):"""uid = self.client.get("search?query=feel%20good%20story&limit=1").json()[0]["id"]- self.assertEqual(self.data[uid], self.data[4])+ self.assertEqual(uid, 4)def testSearchBatch(self):"""@@ -166,7 +184,7 @@ class TestEmbeddings(unittest.TestCase):uid = self.client.post("similarity", json={"query": "feel good story", "texts": self.data}).json()[0]["id"]- self.assertEqual(self.data[uid], self.data[4])+ self.assertEqual(uid, 4)def testSimilarityBatch(self):"""@@ -195,6 +213,24 @@ class TestEmbeddings(unittest.TestCase):self.assertEqual(len(embeddings), len(self.data))self.assertEqual(len(embeddings[0]), 768)+ def testUpsert(self):+ """+ Test upsert via API+ """++ # Update data+ self.client.post("add", json=[{"id": 0, "text": "Feel good story: baby panda born"}])+ self.client.get("upsert")++ # Search for best match+ uid = self.client.get("search?query=feel%20good%20story&limit=1").json()[0]["id"]++ self.assertEqual(uid, 0)++ # Reset data+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(self.data)])+ self.client.get("index")+def testViewOnly(self):"""Test read-only API instancediff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex b25f96ab..3132d50e 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -33,58 +33,35 @@ class TestEmbeddings(unittest.TestCase):# Create embeddings model, backed by sentence-transformers & transformerscls.embeddings = Embeddings({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"})- def testIndex(self):+ def testDelete(self):"""- Test index+ Test delete"""# Create an index for the list of textself.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])- # Search for best match- uid = self.embeddings.search("feel good story", 1)[0][0]-- self.assertEqual(self.data[uid], self.data[4])-- def testUpsert(self):- """- Test upsert- """-- # Build data array- data = [(uid, text, None) for uid, text in enumerate(self.data)]-- # Reset embeddings for test- self.embeddings.embeddings = None-- # Create an index for the list of text- self.embeddings.upsert(data)-- # Update data- data[0] = (0, "Feel good story: baby panda born", None)- self.embeddings.upsert([data[0]])+ # Delete best match+ self.embeddings.delete([4])# Search for best matchuid = self.embeddings.search("feel good story", 1)[0][0]- self.assertEqual(data[uid], data[0])+ self.assertEqual(self.embeddings.count(), 5)+ self.assertEqual(uid, 5)- def testDelete(self):+ def testIndex(self):"""- Test delete+ Test index"""# Create an index for the list of textself.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])- # Delete best match- self.embeddings.delete([4])-# Search for best matchuid = self.embeddings.search("feel good story", 1)[0][0]- self.assertEqual(self.embeddings.count(), 5)- self.assertEqual(self.data[uid], self.data[5])+ self.assertEqual(uid, 4)def testSave(self):"""@@ -103,7 +80,7 @@ class TestEmbeddings(unittest.TestCase):# Search for best matchuid = self.embeddings.search("feel good story", 1)[0][0]- self.assertEqual(self.data[uid], self.data[4])+ self.assertEqual(uid, 4)# Test offsets still work after save/loadself.embeddings.upsert([(0, "Test insert", None)])@@ -117,7 +94,30 @@ class TestEmbeddings(unittest.TestCase):# Get best matching iduid = self.embeddings.similarity("feel good story", self.data)[0][0]- self.assertEqual(self.data[uid], self.data[4])+ self.assertEqual(uid, 4)++ def testUpsert(self):+ """+ Test upsert+ """++ # Build data array+ data = [(uid, text, None) for uid, text in enumerate(self.data)]++ # Reset embeddings for test+ self.embeddings.embeddings = None++ # Create an index for the list of text+ self.embeddings.upsert(data)++ # Update data+ data[0] = (0, "Feel good story: baby panda born", None)+ self.embeddings.upsert([data[0]])++ # Search for best match+ uid = self.embeddings.search("feel good story", 1)[0][0]++ self.assertEqual(uid, 0)def testWords(self):"""diff --git a/test/python/testsummary.py b/test/python/testsummary.pyindex ec34b49b..144521c4 100644--- a/test/python/testsummary.py+++ b/test/python/testsummary.py@@ -9,7 +9,7 @@ from txtai.pipeline import Summaryclass TestSummary(unittest.TestCase):"""- Labels tests+ Summary tests"""@classmethoddiff --git a/test/python/testtextractor.py b/test/python/testtextractor.pyindex 757d87ba..1a3f3220 100644--- a/test/python/testtextractor.py+++ b/test/python/testtextractor.py@@ -10,23 +10,23 @@ from txtai.pipeline import Textractorfrom utils import Utils-class TestSummary(unittest.TestCase):+class TestTextractor(unittest.TestCase):"""- Labels tests+ Textractor tests"""- def testSingle(self):+ def testParagraphs(self):"""- Tests a single extraction with no tokenization of the results+ Tests extraction to paragraphs"""- textractor = Textractor()+ textractor = Textractor(paragraphs=True)- # Extract text as a single block- text = textractor(Utils.PATH + "/article.pdf")+ # Extract text as sentences+ paragraphs = textractor(Utils.PATH + "/article.pdf")- # Check length of text is as expected- self.assertEqual(len(text), 2301)+ # Check number of paragraphs is as expected+ self.assertEqual(len(paragraphs), 13)def testSentences(self):"""@@ -41,15 +41,15 @@ class TestSummary(unittest.TestCase):# Check number of sentences is as expectedself.assertEqual(len(sentences), 17)- def testParagraphs(self):+ def testSingle(self):"""- Tests extraction to paragraphs+ Tests a single extraction with no tokenization of the results"""- textractor = Textractor(paragraphs=True)+ textractor = Textractor()- # Extract text as sentences- paragraphs = textractor(Utils.PATH + "/article.pdf")+ # Extract text as a single block+ text = textractor(Utils.PATH + "/article.pdf")- # Check number of paragraphs is as expected- self.assertEqual(len(paragraphs), 13)+ # Check length of text is as expected+ self.assertEqual(len(text), 2301)diff --git a/test/python/testtransformers.py b/test/python/testtransformers.pyindex a46431ac..b3ea51b8 100644--- a/test/python/testtransformers.py+++ b/test/python/testtransformers.py@@ -69,7 +69,7 @@ class TestTransformersVectors(unittest.TestCase):self.assertFalse(np.array_equal(embeddings1[0], embeddings2[0]))self.assertTrue(np.array_equal(embeddings1[1], embeddings2[1]))- def testLongTransform(self):+ def testTransformLong(self):"""Test transformers transform on long text"""diff --git a/test/python/testtranslation.py b/test/python/testtranslation.pyindex c8142dbc..1a0cfc67 100644--- a/test/python/testtranslation.py+++ b/test/python/testtranslation.py@@ -13,6 +13,19 @@ class TestTranslation(unittest.TestCase):Translation tests"""+ @unittest.skipIf(os.name == "nt", "M2M100 skipped on Windows")+ def testM2M100Translation(self):+ """+ Tests a translation using M2M100 models+ """++ translate = Translation()++ text = translate("This is a test translation to Croatian", "hr")++ # Validate translation text+ self.assertEqual(text, "Ovo je testni prijevod na hrvatski")+def testMarianTranslation(self):"""Tests a translation using Marian models@@ -30,18 +43,18 @@ class TestTranslation(unittest.TestCase):translation = translate(translation, "en")self.assertEqual(translation, text)- @unittest.skipIf(os.name == "nt", "M2M100 skipped on Windows")- def testM2M100Translation(self):+ def testLongTranslation(self):"""- Tests a translation using M2M100 models+ Tests a translation longer than max tokenization length."""translate = Translation()- text = translate("This is a test translation to Croatian", "hr")+ text = "This is a test translation to Spanish. " * 100+ translation = translate(text, "es")# Validate translation text- self.assertEqual(text, "Ovo je testni prijevod na hrvatski")+ self.assertIsNotNone(translation)def testNoTranslation(self):"""@@ -55,16 +68,3 @@ class TestTranslation(unittest.TestCase):# Validate no translationself.assertEqual(text, translation)-- def testLongTranslation(self):- """- Tests a translation longer than max tokenization length.- """-- translate = Translation()-- text = "This is a test translation to Spanish. " * 100- translation = translate(text, "es")-- # Validate translation text- self.assertIsNotNone(translation)