Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix ANN delete on quantized Torch stores and tiny GGML indexes (#1228)

Morgan Carr21460347dd4a
diff --git a/src/python/txtai/ann/dense/ggml.py b/src/python/txtai/ann/dense/ggml.py
index 45e74f34..9e34e74b 100644
--- a/src/python/txtai/ann/dense/ggml.py
+++ b/src/python/txtai/ann/dense/ggml.py
@@ -155,7 +155,7 @@ class GGMLTensors:
gguf = ggml.gguf_init_from_file(path.encode("utf-8"), params)
# Load tensors from GGUF file
- self.loadtensors(context)
+ self.loadtensors(gguf, context)
# Create graph operation
self.graph, self.output = self.creategraph()
@@ -218,11 +218,11 @@ class GGMLTensors:
ids: ids to delete
"""
- shape = utils.get_shape(self.data)
+ rows = self.rows()
# Set index ids as deleted, ignoring ids outside the array and ids already marked deleted
for x in ids:
- if x < shape[1] and x not in self.deletes:
+ if x < rows and x not in self.deletes:
self.deletes.append(x)
def search(self, queries):
@@ -251,8 +251,7 @@ class GGMLTensors:
ggml.ggml_backend_graph_compute(self.backend, self.graph)
# Get size of embeddings data
- size = utils.get_shape(self.data)
- size = size[1] if len(size) > 1 else 1
+ size = self.rows()
# Get and return results
results = np.zeros((batch.shape[0], size), dtype=np.float32)
@@ -273,7 +272,7 @@ class GGMLTensors:
count
"""
- return utils.get_shape(self.data)[1] - len(self.deletes) if self.data else 0
+ return self.rows() - len(self.deletes) if self.data else 0
def save(self, path):
"""
@@ -289,9 +288,12 @@ class GGMLTensors:
# Init and save data tensor
gguf = ggml.gguf_init_empty()
- # Add the data tensor
- ggml.ggml_set_name(self.data, b"data")
- ggml.gguf_add_tensor(gguf, self.data)
+ # Add the data tensor. GGUF can't store a tensor with zero rows, only save the dimensions for an empty index.
+ if self.rows():
+ ggml.ggml_set_name(self.data, b"data")
+ ggml.gguf_add_tensor(gguf, self.data)
+ else:
+ ggml.gguf_set_val_u32(gguf, b"dimensions", utils.get_shape(self.data)[0])
# Optionally create and add the deletes tensor
if self.deletes:
@@ -414,11 +416,12 @@ class GGMLTensors:
# Copy embeddings data
self.copy(data, self.data, tensortype, 0)
- def loadtensors(self, context):
+ def loadtensors(self, gguf, context):
"""
Loads existing tensors from context.
Args:
+ gguf: gguf context
context: ggml context
"""
@@ -437,6 +440,10 @@ class GGMLTensors:
# Copy tensor data to backend
ggml.ggml_backend_tensor_set(self.data, ggml.ggml_get_data(data), 0, ggml.ggml_nbytes(data))
+ else:
+ # Empty index, create tensors using the saved dimensions
+ dimensions = ggml.gguf_get_val_u32(gguf, ggml.gguf_find_key(gguf, b"dimensions"))
+ self.createtensors(np.zeros((0, dimensions), dtype=np.float32))
# Load deletes tensor
data = ggml.ggml_get_tensor(context, b"deletes")
@@ -466,8 +473,7 @@ class GGMLTensors:
queries = ggml.ggml_new_tensor_2d(context, ggml.GGML_TYPE_F32, data.shape[1], self.querysize)
# Embeddings data with space for both existing and new data
- shape = utils.get_shape(self.data)
- merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + shape[1])
+ merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + self.rows())
# Create new buffer
buffer = ggml.ggml_backend_alloc_ctx_tensors(context, backend)
@@ -573,3 +579,13 @@ class GGMLTensors:
"""
return [queries[x : x + self.querysize] for x in range(0, len(queries), self.querysize)]
+
+ def rows(self):
+ """
+ Number of rows in the data tensor. Tensors with a single row or no rows report a 1-D shape, so the shape can't be used.
+
+ Returns:
+ row count
+ """
+
+ return ggml.ggml_nrows(self.data)
diff --git a/src/python/txtai/ann/dense/numpy.py b/src/python/txtai/ann/dense/numpy.py
index 97e6bbb4..0f109323 100644
--- a/src/python/txtai/ann/dense/numpy.py
+++ b/src/python/txtai/ann/dense/numpy.py
@@ -63,8 +63,8 @@ class NumPy(ANN):
# Filter any index greater than size of array
ids = [x for x in ids if x < self.backend.shape[0]]
- # Clear specified ids
- self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1])))
+ # Clear specified ids, zeros must match the array data type (e.g. uint8 for quantized data)
+ self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1]), dtype=self.backend.dtype))
def search(self, queries, limit):
if self.qbits:
diff --git a/test/python/testann/testdense.py b/test/python/testann/testdense.py
index 423f98eb..7284b527 100644
--- a/test/python/testann/testdense.py
+++ b/test/python/testann/testdense.py
@@ -213,6 +213,34 @@ class TestDense(unittest.TestCase):
ann.load(index)
self.assertEqual(ann.count(), 99)
+ def testGGMLEmpty(self):
+ """
+ Test GGML backend with an empty index
+ """
+
+ ann = ANNFactory.create({"backend": "ggml", "dimensions": 240})
+
+ # Index an empty array
+ ann.index(np.zeros((0, 240), dtype=np.float32))
+ self.assertEqual(ann.count(), 0)
+
+ # Test save and load
+ index = os.path.join(tempfile.gettempdir(), "ggml.empty")
+ ann.save(index)
+ ann.load(index)
+ self.assertEqual(ann.count(), 0)
+
+ # Append data to the loaded empty index
+ data = np.random.rand(10, 240).astype(np.float32)
+ self.normalize(data)
+ ann.append(data)
+ self.assertEqual(ann.count(), 10)
+
+ # Generate query vector and test search
+ query = np.random.rand(240).astype(np.float32)
+ self.normalize(query)
+ self.assertGreater(ann.search(np.array([query]), 1)[0][0][1], 0)
+
def testGGMLQuantization(self):
"""
Test GGML backend with quantization enabled
@@ -262,6 +290,34 @@ class TestDense(unittest.TestCase):
ann = ANNFactory.create({"backend": "ggml", "ggml": {"quantize": "Q4_K"}})
ann.index(data)
+ def testGGMLSingleRow(self):
+ """
+ Test GGML backend with a single row index
+ """
+
+ ann = ANNFactory.create({"backend": "ggml", "dimensions": 240})
+
+ # Generate and index a single row
+ data = np.random.rand(1, 240).astype(np.float32)
+ self.normalize(data)
+ ann.index(data)
+ self.assertEqual(ann.count(), 1)
+
+ # Test save and load
+ index = os.path.join(tempfile.gettempdir(), "ggml.single")
+ ann.save(index)
+ ann.load(index)
+ self.assertEqual(ann.count(), 1)
+ self.assertEqual(ann.search(data, 1)[0][0][0], 0)
+
+ # Append a row
+ ann.append(data)
+ self.assertEqual(ann.count(), 2)
+
+ # Test delete, including an out of range id
+ ann.delete([0, 5])
+ self.assertEqual(ann.count(), 1)
+
def testHnsw(self):
"""
Test Hnswlib backend
@@ -483,6 +539,28 @@ class TestDense(unittest.TestCase):
self.runTests("torch")
+ def testTorchDelete(self):
+ """
+ Test Torch backend delete keeps the stored data type for quantized (uint8) and float16 indexes
+ """
+
+ for dtype, config in [(np.uint8, {"quantize": 1}), (np.float16, {})]:
+ with self.subTest(dtype=dtype.__name__):
+ ann = ANNFactory.create({"backend": "torch", "dimensions": 4, **config})
+
+ # Index vectors with a single non-zero value per row
+ data = np.array([[1, 0, 0, 0], [0, 255, 0, 0], [0, 0, 255, 0], [0, 0, 0, 255]], dtype=dtype)
+ ann.index(data)
+
+ # Delete the first row and search with its (former) vector
+ ann.delete([0])
+ results = ann.search(np.array([data[0]]), 4)[0]
+
+ # Deleted row must score 0 so the caller's score > 0 filter drops it and the data type must be unchanged
+ self.assertEqual(dict(results).get(0), 0)
+ self.assertEqual(ann.count(), 3)
+ self.assertEqual(str(ann.backend.dtype), f"torch.{dtype.__name__}")
+
@unittest.skipIf(platform.system() == "Darwin", "Torch quantization not supported on macOS")
def testTorchQuantization(self):
"""