💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Build the deletes set once in Terms.save (#1448)
Muneeb Ullahe7bf4190df2d
diff --git a/src/python/txtai/scoring/terms.py b/src/python/txtai/scoring/terms.pyindex 26993e31..f6b8410b 100644--- a/src/python/txtai/scoring/terms.py+++ b/src/python/txtai/scoring/terms.py@@ -306,8 +306,9 @@ class Terms:# Serialize ids as BLOBs so SQLite TEXT affinity does not discard their types.serializer = SerializeFactory.create()+ deletes = set(self.deletes)for i, uid in enumerate(self.ids):- self.cursor.execute(Terms.INSERT_DOCUMENT, [i, serializer.savebytes(uid), 1 if i in self.deletes else 0, self.lengths[i]])+ self.cursor.execute(Terms.INSERT_DOCUMENT, [i, serializer.savebytes(uid), 1 if i in deletes else 0, self.lengths[i]])# Temporary databaseif not self.path:diff --git a/test/python/testscoring/testkeyword.py b/test/python/testscoring/testkeyword.pyindex 856da201..5c47dab8 100644--- a/test/python/testscoring/testkeyword.py+++ b/test/python/testscoring/testkeyword.py@@ -376,6 +376,43 @@ class TestKeyword(unittest.TestCase):self.assertFalse(scoring.search("lunar eclipse", 1))self.assertEqual(scoring.count(), total - 1)+ def testDeleteSave(self):+ """+ Test that saving an index with deletes does not scan the deletes list for every document+ """++ class Deletes(list):+ """+ List that counts membership checks.+ """++ lookups = 0++ def __contains__(self, item):+ Deletes.lookups += 1+ return super().__contains__(item)++ # Terms.save checked "i in self.deletes" once per document, making save O(documents x deletes)+ scoring = ScoringFactory.create({"method": "bm25", "terms": True})+ scoring.index([(uid, f"word{uid} common", None) for uid in range(100)])+ scoring.delete(list(range(50)))+ scoring.terms.deletes = Deletes(scoring.terms.deletes)+ Deletes.lookups = 0++ with tempfile.TemporaryDirectory() as directory:+ path = os.path.join(directory, "scoring")+ scoring.save(path)+ scoring.close()++ scoring = ScoringFactory.create({"method": "bm25", "terms": True})+ scoring.load(path)++ self.assertEqual(scoring.count(), 50)+ self.assertEqual(sorted(scoring.terms.deletes), list(range(50)))+ self.assertEqual(Deletes.lookups, 0)++ scoring.close()+def testTFIDF(self):"""Test tfidf