💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep deleted keyword results excluded after common-term scoring (#1247)
LimbC-C47d5f567681f
diff --git a/src/python/txtai/scoring/terms.py b/src/python/txtai/scoring/terms.pyindex b20d5db1..196b8f0b 100644--- a/src/python/txtai/scoring/terms.py+++ b/src/python/txtai/scoring/terms.py@@ -523,6 +523,9 @@ class Terms:if not hasscores:matches = self.candidates(scores, topn)+ # Common term scores can restore deleted candidates; clear them before ranking.+ scores[self.deletes] = 0+# Reorder matches using updated scoresmatches = matches[np.argsort(-scores[matches])]diff --git a/test/python/testscoring/testkeyword.py b/test/python/testscoring/testkeyword.pyindex 50ffbe46..1fa03113 100644--- a/test/python/testscoring/testkeyword.py+++ b/test/python/testscoring/testkeyword.py@@ -203,6 +203,43 @@ class TestKeyword(unittest.TestCase):scoring.delete([0])self.assertEqual(scoring.count(), total - 1)+ def testDeleteMixedTerms(self):+ """+ Test that common term scoring cannot restore deleted keyword results+ """++ for method in ["bm25", "tfidf", "sif"]:+ for content in [False, True]:+ with self.subTest(method=method, content=content), tempfile.TemporaryDirectory() as directory:+ config = {"method": method, "terms": True, "content": content}+ scoring = ScoringFactory.create(config)+ try:+ # rare occurs in one document; common occurs in every document.+ scoring.index([(uid, "rare common" if uid == 0 else "common", None) for uid in range(20)])+ scoring.search("rare common", 20)+ scoring.delete([1])++ for persisted in [False, True]:+ if persisted:+ path = os.path.join(directory, "scoring")+ scoring.save(path)+ scoring.close()+ scoring = ScoringFactory.create(config)+ scoring.load(path)++ for query in ["rare", "common", "rare common"]:+ for limit in [3, 20]:+ results = scoring.search(query, limit)+ ids = [result["id"] if content else result[0] for result in results]+ self.assertNotIn(1, ids)+ self.assertEqual(len(ids), 1 if query == "rare" else min(limit, 19))+ if query == "rare common":+ self.assertEqual(ids[0], 0)++ self.assertEqual(scoring.count(), 19)+ finally:+ scoring.close()+def testDeleteReinsert(self):"""Test that an id deleted, re-added and deleted again is removed from count() and search()