Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Store IVFSparse deletes as int (#1241)

Morgan Carrad77514b321d
diff --git a/src/python/txtai/ann/sparse/ivfsparse.py b/src/python/txtai/ann/sparse/ivfsparse.py
index 716ecb58..fd10573a 100644
--- a/src/python/txtai/ann/sparse/ivfsparse.py
+++ b/src/python/txtai/ann/sparse/ivfsparse.py
@@ -116,10 +116,11 @@ class IVFSparse(ANN):
def delete(self, ids):
# Set index ids as deleted, ignoring ids that were never indexed and ids already marked deleted
+ # Ids are stored as int, the deletes list is serialized with msgpack
size = self.size()
for x in ids:
if x < size and x not in self.deletes:
- self.deletes.append(x)
+ self.deletes.append(int(x))
def search(self, queries, limit):
results = []
diff --git a/test/python/testann/testsparse.py b/test/python/testann/testsparse.py
index c2a4f973..b7b36f48 100644
--- a/test/python/testann/testsparse.py
+++ b/test/python/testann/testsparse.py
@@ -8,6 +8,8 @@ import unittest
from unittest.mock import patch
+import numpy as np
+
from scipy.sparse import random
from sklearn.preprocessing import normalize
@@ -100,6 +102,31 @@ class TestSparse(unittest.TestCase):
self.assertLessEqual(len(ann.blocks), 15)
ann.close()
+ def testIVFSparseDeleteArray(self):
+ """
+ Test IVFSparse with ids deleted using a NumPy array
+ """
+
+ # Generate test record
+ data = self.generate(50, 30522)
+
+ # Create ANN
+ path = os.path.join(tempfile.gettempdir(), "ivfsparse.deletes")
+ ann = SparseANNFactory.create({"backend": "ivfsparse"})
+ ann.index(data)
+
+ # Test delete with ids passed in as a NumPy array
+ ann.delete(np.array([0, 1]))
+ self.assertEqual(ann.count(), 48)
+
+ # Validate save/load
+ ann.save(path)
+ ann.load(path)
+ self.assertEqual(ann.count(), 48)
+
+ # Close ANN
+ ann.close()
+
def testIVFSparseSortOrder(self):
"""
Test IVFSparse returns results sorted by score descending