Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Find ids to delete in a single pass when content is disabled

Tanishq5dc4f2dbf9d2
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index d78fd830..ab03f24e 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -227,9 +227,9 @@ class Embeddings:
# Delete ids from database
self.database.delete(deletes)
elif self.ann or self.scoring:
- # Find existing ids
- for uid in ids:
- indices.extend([index for index, value in enumerate(self.ids) if uid == value])
+ # Find existing ids with a single pass over the index ids
+ uids = set(ids)
+ indices = [index for index, value in enumerate(self.ids) if value in uids]
# Clear embeddings ids
for index in indices:
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index 8398ea3c..425a3331 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -11,7 +11,7 @@ from unittest.mock import patch
import numpy as np
-from txtai.embeddings import Embeddings, Reducer
+from txtai.embeddings import Embeddings, IndexIds, Reducer
from txtai.serialize import SerializeFactory
@@ -162,6 +162,24 @@ class TestEmbeddings(unittest.TestCase):
self.assertEqual(self.embeddings.count(), 5)
self.assertEqual(uid, 5)
+ def testDeleteIds(self):
+ """
+ Test delete with content disabled finds ids with a single pass over the index ids
+ """
+
+ # Create a keyword index with content disabled
+ embeddings = Embeddings({"keyword": True})
+ embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Delete a repeated id and an unknown id while counting the passes over the index ids
+ with patch.object(IndexIds, "__iter__", autospec=True, side_effect=IndexIds.__iter__) as scan:
+ deleted = embeddings.delete([4, 1, 4, 100])
+
+ self.assertEqual(scan.call_count, 1)
+ self.assertEqual(deleted, [1, 4])
+ self.assertEqual(embeddings.count(), 4)
+ self.assertEqual(embeddings.search("lottery ticket", 1), [])
+
def testDense(self):
"""
Test dense alias