Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Build the deletes set once in Terms.save (#1448)

Muneeb Ullahe7bf4190df2d
diff --git a/src/python/txtai/scoring/terms.py b/src/python/txtai/scoring/terms.py
index 26993e31..f6b8410b 100644
--- a/src/python/txtai/scoring/terms.py
+++ b/src/python/txtai/scoring/terms.py
@@ -306,8 +306,9 @@ class Terms:
# Serialize ids as BLOBs so SQLite TEXT affinity does not discard their types.
serializer = SerializeFactory.create()
+ deletes = set(self.deletes)
for i, uid in enumerate(self.ids):
- self.cursor.execute(Terms.INSERT_DOCUMENT, [i, serializer.savebytes(uid), 1 if i in self.deletes else 0, self.lengths[i]])
+ self.cursor.execute(Terms.INSERT_DOCUMENT, [i, serializer.savebytes(uid), 1 if i in deletes else 0, self.lengths[i]])
# Temporary database
if not self.path:
diff --git a/test/python/testscoring/testkeyword.py b/test/python/testscoring/testkeyword.py
index 856da201..5c47dab8 100644
--- a/test/python/testscoring/testkeyword.py
+++ b/test/python/testscoring/testkeyword.py
@@ -376,6 +376,43 @@ class TestKeyword(unittest.TestCase):
self.assertFalse(scoring.search("lunar eclipse", 1))
self.assertEqual(scoring.count(), total - 1)
+ def testDeleteSave(self):
+ """
+ Test that saving an index with deletes does not scan the deletes list for every document
+ """
+
+ class Deletes(list):
+ """
+ List that counts membership checks.
+ """
+
+ lookups = 0
+
+ def __contains__(self, item):
+ Deletes.lookups += 1
+ return super().__contains__(item)
+
+ # Terms.save checked "i in self.deletes" once per document, making save O(documents x deletes)
+ scoring = ScoringFactory.create({"method": "bm25", "terms": True})
+ scoring.index([(uid, f"word{uid} common", None) for uid in range(100)])
+ scoring.delete(list(range(50)))
+ scoring.terms.deletes = Deletes(scoring.terms.deletes)
+ Deletes.lookups = 0
+
+ with tempfile.TemporaryDirectory() as directory:
+ path = os.path.join(directory, "scoring")
+ scoring.save(path)
+ scoring.close()
+
+ scoring = ScoringFactory.create({"method": "bm25", "terms": True})
+ scoring.load(path)
+
+ self.assertEqual(scoring.count(), 50)
+ self.assertEqual(sorted(scoring.terms.deletes), list(range(50)))
+ self.assertEqual(Deletes.lookups, 0)
+
+ scoring.close()
+
def testTFIDF(self):
"""
Test tfidf