💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Store IVFSparse deletes as int (#1241)
Morgan Carrad77514b321d
diff --git a/src/python/txtai/ann/sparse/ivfsparse.py b/src/python/txtai/ann/sparse/ivfsparse.pyindex 716ecb58..fd10573a 100644--- a/src/python/txtai/ann/sparse/ivfsparse.py+++ b/src/python/txtai/ann/sparse/ivfsparse.py@@ -116,10 +116,11 @@ class IVFSparse(ANN):def delete(self, ids):# Set index ids as deleted, ignoring ids that were never indexed and ids already marked deleted+ # Ids are stored as int, the deletes list is serialized with msgpacksize = self.size()for x in ids:if x < size and x not in self.deletes:- self.deletes.append(x)+ self.deletes.append(int(x))def search(self, queries, limit):results = []diff --git a/test/python/testann/testsparse.py b/test/python/testann/testsparse.pyindex c2a4f973..b7b36f48 100644--- a/test/python/testann/testsparse.py+++ b/test/python/testann/testsparse.py@@ -8,6 +8,8 @@ import unittestfrom unittest.mock import patch+import numpy as np+from scipy.sparse import randomfrom sklearn.preprocessing import normalize@@ -100,6 +102,31 @@ class TestSparse(unittest.TestCase):self.assertLessEqual(len(ann.blocks), 15)ann.close()+ def testIVFSparseDeleteArray(self):+ """+ Test IVFSparse with ids deleted using a NumPy array+ """++ # Generate test record+ data = self.generate(50, 30522)++ # Create ANN+ path = os.path.join(tempfile.gettempdir(), "ivfsparse.deletes")+ ann = SparseANNFactory.create({"backend": "ivfsparse"})+ ann.index(data)++ # Test delete with ids passed in as a NumPy array+ ann.delete(np.array([0, 1]))+ self.assertEqual(ann.count(), 48)++ # Validate save/load+ ann.save(path)+ ann.load(path)+ self.assertEqual(ann.count(), 48)++ # Close ANN+ ann.close()+def testIVFSparseSortOrder(self):"""Test IVFSparse returns results sorted by score descending