💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Find ids to delete in a single pass when content is disabled
Tanishq5dc4f2dbf9d2
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex d78fd830..ab03f24e 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -227,9 +227,9 @@ class Embeddings:# Delete ids from databaseself.database.delete(deletes)elif self.ann or self.scoring:- # Find existing ids- for uid in ids:- indices.extend([index for index, value in enumerate(self.ids) if uid == value])+ # Find existing ids with a single pass over the index ids+ uids = set(ids)+ indices = [index for index, value in enumerate(self.ids) if value in uids]# Clear embeddings idsfor index in indices:diff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex 8398ea3c..425a3331 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -11,7 +11,7 @@ from unittest.mock import patchimport numpy as np-from txtai.embeddings import Embeddings, Reducer+from txtai.embeddings import Embeddings, IndexIds, Reducerfrom txtai.serialize import SerializeFactory@@ -162,6 +162,24 @@ class TestEmbeddings(unittest.TestCase):self.assertEqual(self.embeddings.count(), 5)self.assertEqual(uid, 5)+ def testDeleteIds(self):+ """+ Test delete with content disabled finds ids with a single pass over the index ids+ """++ # Create a keyword index with content disabled+ embeddings = Embeddings({"keyword": True})+ embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Delete a repeated id and an unknown id while counting the passes over the index ids+ with patch.object(IndexIds, "__iter__", autospec=True, side_effect=IndexIds.__iter__) as scan:+ deleted = embeddings.delete([4, 1, 4, 100])++ self.assertEqual(scan.call_count, 1)+ self.assertEqual(deleted, [1, 4])+ self.assertEqual(embeddings.count(), 4)+ self.assertEqual(embeddings.search("lottery ticket", 1), [])+def testDense(self):"""Test dense alias