Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Support append and delete in the RaBitQ ivf mode (#1304)

Pablo Miguel Argudoc78fbd976ca8
diff --git a/docs/embeddings/configuration/ann.md b/docs/embeddings/configuration/ann.md
index e604500b..8f3f8b6b 100644
--- a/docs/embeddings/configuration/ann.md
+++ b/docs/embeddings/configuration/ann.md
@@ -109,7 +109,7 @@ rabitq:
randomseed: random-seed param for hnsw mode (int) - defaults to 100
```
-The [rabitq](https://github.com/VectorDB-NTU/RaBitQ-Library) backend is a quantized index powered by the [RaBitQ algorithm](https://doi.org/10.1145/3725413). It supports ivf and hnsw search modes. Vectors are stored with 1-bit quantization by default. More bits increase accuracy at the cost of memory. The index is stored as a single file. Note that rabitq indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
+The [rabitq](https://github.com/VectorDB-NTU/RaBitQ-Library) backend is a quantized index powered by the [RaBitQ algorithm](https://doi.org/10.1145/3725413). It supports ivf and hnsw search modes. Vectors are stored with 1-bit quantization by default. More bits increase accuracy at the cost of memory. The index is stored as a single file and no vectors are kept outside of it. With nbits set to 32, the vectors are stored in the index for reranking. The ivf mode supports upserts and deletes. Deleted rows keep their storage. Rows added after the first index are assigned to the clusters trained when the index was built, so recall can drop if the new data differs from the original data. Rebuild the index to train new clusters. Note that hnsw indexes can not be modified after creation, upserts/deletes and other modifications are not supported.
### torch
diff --git a/setup.py b/setup.py
index f99e8369..31b30418 100644
--- a/setup.py
+++ b/setup.py
@@ -62,7 +62,7 @@ extras["ann"] = [
"hnswlib>=0.5.0",
"milvus-lite>=3.1.1",
"pgvector>=0.4.1",
- "rabitqlib>=0.3.9",
+ "rabitqlib>=0.5.1",
"scikit-learn>=0.23.1",
"scipy>=1.4.1",
"sqlalchemy>=2.0.20",
diff --git a/src/python/txtai/ann/dense/rabitq.py b/src/python/txtai/ann/dense/rabitq.py
index 0fd72dc0..9d167cca 100644
--- a/src/python/txtai/ann/dense/rabitq.py
+++ b/src/python/txtai/ann/dense/rabitq.py
@@ -25,13 +25,13 @@ np = library.numpy()
SENTINEL = 0xFFFFFFFF
-# pylint: disable=W0223
class RabitQ(ANN):
"""
Builds an ANN index using the rabitqlib library (RaBitQ quantization).
- Only the quantized index is stored, as a single file. Native indexes can't be extended or have rows removed,
- so this backend doesn't support append and delete.
+ Only the native index is stored, as a single file. No vectors are kept outside of it. ivf indexes support append
+ and delete natively. The native id of a row is its index id. hnsw indexes can't be extended or have rows removed,
+ so append and delete are not supported in that mode.
"""
def __init__(self, config):
@@ -40,6 +40,9 @@ class RabitQ(ANN):
if not RABITQ:
raise ImportError('rabitqlib is not available - install "ann" extra to enable')
+ # Number of rows removed from the native index (ivf mode only)
+ self.config["deletes"] = self.config.get("deletes", 0)
+
def load(self, path):
self.close()
@@ -94,9 +97,34 @@ class RabitQ(ANN):
# Add id offset and index build metadata
self.config["offset"] = rows
+ self.config["deletes"] = 0
self.metadata(settings)
+ def append(self, embeddings):
+ if self.mode() != "ivf":
+ super().append(embeddings)
+
+ # Native ids continue from the last row ever added
+ embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)
+ self.backend.add(embeddings)
+
+ # Update id offset and index metadata
+ self.config["offset"] = self.backend.max_elements
+ self.metadata()
+
+ def delete(self, ids):
+ if self.mode() != "ivf":
+ super().delete(ids)
+
+ # Native remove hides rows from search. Unknown ids are silently ignored.
+ ids = [uid for uid in ids if 0 <= uid < self.backend.max_elements]
+ if ids:
+ self.config["deletes"] += self.backend.remove(np.array(ids, dtype=np.int64))
+
def search(self, queries, limit):
+ if not self.count():
+ return [[] for _ in queries]
+
queries = np.ascontiguousarray(queries, dtype=np.float32)
# The native index errors when k exceeds its row count
@@ -115,7 +143,7 @@ class RabitQ(ANN):
]
def count(self):
- return self.backend.max_elements if self.backend is not None else 0
+ return self.backend.max_elements - self.config["deletes"] if self.backend is not None else 0
def save(self, path):
self.backend.save(path)
diff --git a/test/python/testann/testdense/testrabitq.py b/test/python/testann/testdense/testrabitq.py
index 469b4264..aa04defe 100644
--- a/test/python/testann/testdense/testrabitq.py
+++ b/test/python/testann/testdense/testrabitq.py
@@ -25,19 +25,19 @@ class TestRaBitQ(DenseTest):
Test RaBitQ backend
"""
- self.runTests("rabitq", None, False)
+ self.runTests("rabitq")
def testRabitQCustom(self):
"""
Test RaBitQ backend with custom settings
"""
- # Test with custom settings
+ # Test with custom settings, hnsw indexes don't support append and delete
self.runTests("rabitq", {"rabitq": {"mode": "hnsw"}}, False)
self.runTests("rabitq", {"rabitq": {"clusters": 8, "nprobe": 2}}, False)
self.runTests("rabitq", {"rabitq": {"nbits": 4}}, False)
self.runTests("rabitq", {"rabitq": {"mode": "hnsw", "nbits": 4}}, False)
- self.runTests("rabitq", {"rabitq": {"nbits": 32}}, False)
+ self.runTests("rabitq", {"rabitq": {"nbits": 32}})
# Generate dummy data
data = np.random.rand(100, 240).astype(np.float32)
@@ -59,9 +59,58 @@ class TestRaBitQ(DenseTest):
ann.load(index)
self.assertEqual(ann.count(), 0)
+ def testRabitQAppendDelete(self):
+ """
+ Test RaBitQ ivf appends and deletes rows without retaining vectors
+ """
+
+ # Generate dummy data
+ data = np.random.rand(100, 240).astype(np.float32)
+ self.normalize(data)
+
+ # Raw vectors rank exactly, ivf probes every cluster
+ config = {"backend": "rabitq", "dimensions": 240, "rabitq": {"nbits": 32, "nprobe": 100}}
+ ann = ANNFactory.create(config)
+ ann.index(data)
+
+ # Appended rows continue the ids
+ ann.append(data[:10])
+ self.assertEqual(ann.count(), 110)
+ self.assertEqual(ann.config["offset"], 110)
+ self.assertEqual({uid for uid, _ in ann.search(data[5:6], 2)[0]}, {5, 105})
+
+ # Unknown and repeated ids are ignored, deleted rows are skipped by search and the limit is still filled
+ ann.delete([5, 5, 1000, -1])
+ self.assertEqual(ann.count(), 109)
+ for result in ann.search(data[:20], 10):
+ self.assertEqual(len(result), 10)
+ self.assertNotIn(5, {uid for uid, _ in result})
+
+ # Deletes are kept in a new instance created from the saved config, like Embeddings.load
+ index = os.path.join(tempfile.gettempdir(), f"rabitq.delete.{round(time.time() * 1000)}")
+ ann.save(index)
+ loaded = ANNFactory.create(dict(ann.config))
+ loaded.load(index)
+ self.assertEqual(loaded.count(), 109)
+ self.assertEqual(loaded.search(data[:3], 5), ann.search(data[:3], 5))
+
+ # A deleted row stays hidden after a reload, even for a query equal to its vector
+ self.assertNotIn(5, {uid for uid, _ in loaded.search(data[5:6], 10)[0]})
+
+ # Appends and deletes continue after a reload
+ loaded.append(data[:5])
+ self.assertEqual(loaded.count(), 114)
+ loaded.delete([0])
+ self.assertEqual(loaded.count(), 113)
+
+ # Deleting every row leaves an empty index
+ loaded.delete(list(range(200)))
+ self.assertEqual(loaded.count(), 0)
+ self.assertEqual(loaded.search(data[:1], 10), [[]])
+
def testRabitQUpdate(self):
"""
- Test RabitQ stores a single file and does not support append and delete
+ Test RaBitQ stores a single file and hnsw does not support append and delete
"""
# Generate dummy data
@@ -75,11 +124,12 @@ class TestRaBitQ(DenseTest):
# Offset marks the index as existing, see Embeddings.exists
self.assertEqual(ann.config["offset"], 100)
- with self.assertRaises(NotImplementedError):
- ann.append(data[:1])
+ if mode == "hnsw":
+ with self.assertRaises(NotImplementedError):
+ ann.append(data[:1])
- with self.assertRaises(NotImplementedError):
- ann.delete([0])
+ with self.assertRaises(NotImplementedError):
+ ann.delete([0])
# Limit above the row count returns every row
for result in ann.search(data[:2], 200):