💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix ANN delete on quantized Torch stores and tiny GGML indexes (#1228)
Morgan Carr21460347dd4a
diff --git a/src/python/txtai/ann/dense/ggml.py b/src/python/txtai/ann/dense/ggml.pyindex 45e74f34..9e34e74b 100644--- a/src/python/txtai/ann/dense/ggml.py+++ b/src/python/txtai/ann/dense/ggml.py@@ -155,7 +155,7 @@ class GGMLTensors:gguf = ggml.gguf_init_from_file(path.encode("utf-8"), params)# Load tensors from GGUF file- self.loadtensors(context)+ self.loadtensors(gguf, context)# Create graph operationself.graph, self.output = self.creategraph()@@ -218,11 +218,11 @@ class GGMLTensors:ids: ids to delete"""- shape = utils.get_shape(self.data)+ rows = self.rows()# Set index ids as deleted, ignoring ids outside the array and ids already marked deletedfor x in ids:- if x < shape[1] and x not in self.deletes:+ if x < rows and x not in self.deletes:self.deletes.append(x)def search(self, queries):@@ -251,8 +251,7 @@ class GGMLTensors:ggml.ggml_backend_graph_compute(self.backend, self.graph)# Get size of embeddings data- size = utils.get_shape(self.data)- size = size[1] if len(size) > 1 else 1+ size = self.rows()# Get and return resultsresults = np.zeros((batch.shape[0], size), dtype=np.float32)@@ -273,7 +272,7 @@ class GGMLTensors:count"""- return utils.get_shape(self.data)[1] - len(self.deletes) if self.data else 0+ return self.rows() - len(self.deletes) if self.data else 0def save(self, path):"""@@ -289,9 +288,12 @@ class GGMLTensors:# Init and save data tensorgguf = ggml.gguf_init_empty()- # Add the data tensor- ggml.ggml_set_name(self.data, b"data")- ggml.gguf_add_tensor(gguf, self.data)+ # Add the data tensor. GGUF can't store a tensor with zero rows, only save the dimensions for an empty index.+ if self.rows():+ ggml.ggml_set_name(self.data, b"data")+ ggml.gguf_add_tensor(gguf, self.data)+ else:+ ggml.gguf_set_val_u32(gguf, b"dimensions", utils.get_shape(self.data)[0])# Optionally create and add the deletes tensorif self.deletes:@@ -414,11 +416,12 @@ class GGMLTensors:# Copy embeddings dataself.copy(data, self.data, tensortype, 0)- def loadtensors(self, context):+ def loadtensors(self, gguf, context):"""Loads existing tensors from context.Args:+ gguf: gguf contextcontext: ggml context"""@@ -437,6 +440,10 @@ class GGMLTensors:# Copy tensor data to backendggml.ggml_backend_tensor_set(self.data, ggml.ggml_get_data(data), 0, ggml.ggml_nbytes(data))+ else:+ # Empty index, create tensors using the saved dimensions+ dimensions = ggml.gguf_get_val_u32(gguf, ggml.gguf_find_key(gguf, b"dimensions"))+ self.createtensors(np.zeros((0, dimensions), dtype=np.float32))# Load deletes tensordata = ggml.ggml_get_tensor(context, b"deletes")@@ -466,8 +473,7 @@ class GGMLTensors:queries = ggml.ggml_new_tensor_2d(context, ggml.GGML_TYPE_F32, data.shape[1], self.querysize)# Embeddings data with space for both existing and new data- shape = utils.get_shape(self.data)- merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + shape[1])+ merge = ggml.ggml_new_tensor_2d(context, tensortype, data.shape[1], data.shape[0] + self.rows())# Create new bufferbuffer = ggml.ggml_backend_alloc_ctx_tensors(context, backend)@@ -573,3 +579,13 @@ class GGMLTensors:"""return [queries[x : x + self.querysize] for x in range(0, len(queries), self.querysize)]++ def rows(self):+ """+ Number of rows in the data tensor. Tensors with a single row or no rows report a 1-D shape, so the shape can't be used.++ Returns:+ row count+ """++ return ggml.ggml_nrows(self.data)diff --git a/src/python/txtai/ann/dense/numpy.py b/src/python/txtai/ann/dense/numpy.pyindex 97e6bbb4..0f109323 100644--- a/src/python/txtai/ann/dense/numpy.py+++ b/src/python/txtai/ann/dense/numpy.py@@ -63,8 +63,8 @@ class NumPy(ANN):# Filter any index greater than size of arrayids = [x for x in ids if x < self.backend.shape[0]]- # Clear specified ids- self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1])))+ # Clear specified ids, zeros must match the array data type (e.g. uint8 for quantized data)+ self.backend[ids] = self.tensor(self.zeros((len(ids), self.backend.shape[1]), dtype=self.backend.dtype))def search(self, queries, limit):if self.qbits:diff --git a/test/python/testann/testdense.py b/test/python/testann/testdense.pyindex 423f98eb..7284b527 100644--- a/test/python/testann/testdense.py+++ b/test/python/testann/testdense.py@@ -213,6 +213,34 @@ class TestDense(unittest.TestCase):ann.load(index)self.assertEqual(ann.count(), 99)+ def testGGMLEmpty(self):+ """+ Test GGML backend with an empty index+ """++ ann = ANNFactory.create({"backend": "ggml", "dimensions": 240})++ # Index an empty array+ ann.index(np.zeros((0, 240), dtype=np.float32))+ self.assertEqual(ann.count(), 0)++ # Test save and load+ index = os.path.join(tempfile.gettempdir(), "ggml.empty")+ ann.save(index)+ ann.load(index)+ self.assertEqual(ann.count(), 0)++ # Append data to the loaded empty index+ data = np.random.rand(10, 240).astype(np.float32)+ self.normalize(data)+ ann.append(data)+ self.assertEqual(ann.count(), 10)++ # Generate query vector and test search+ query = np.random.rand(240).astype(np.float32)+ self.normalize(query)+ self.assertGreater(ann.search(np.array([query]), 1)[0][0][1], 0)+def testGGMLQuantization(self):"""Test GGML backend with quantization enabled@@ -262,6 +290,34 @@ class TestDense(unittest.TestCase):ann = ANNFactory.create({"backend": "ggml", "ggml": {"quantize": "Q4_K"}})ann.index(data)+ def testGGMLSingleRow(self):+ """+ Test GGML backend with a single row index+ """++ ann = ANNFactory.create({"backend": "ggml", "dimensions": 240})++ # Generate and index a single row+ data = np.random.rand(1, 240).astype(np.float32)+ self.normalize(data)+ ann.index(data)+ self.assertEqual(ann.count(), 1)++ # Test save and load+ index = os.path.join(tempfile.gettempdir(), "ggml.single")+ ann.save(index)+ ann.load(index)+ self.assertEqual(ann.count(), 1)+ self.assertEqual(ann.search(data, 1)[0][0][0], 0)++ # Append a row+ ann.append(data)+ self.assertEqual(ann.count(), 2)++ # Test delete, including an out of range id+ ann.delete([0, 5])+ self.assertEqual(ann.count(), 1)+def testHnsw(self):"""Test Hnswlib backend@@ -483,6 +539,28 @@ class TestDense(unittest.TestCase):self.runTests("torch")+ def testTorchDelete(self):+ """+ Test Torch backend delete keeps the stored data type for quantized (uint8) and float16 indexes+ """++ for dtype, config in [(np.uint8, {"quantize": 1}), (np.float16, {})]:+ with self.subTest(dtype=dtype.__name__):+ ann = ANNFactory.create({"backend": "torch", "dimensions": 4, **config})++ # Index vectors with a single non-zero value per row+ data = np.array([[1, 0, 0, 0], [0, 255, 0, 0], [0, 0, 255, 0], [0, 0, 0, 255]], dtype=dtype)+ ann.index(data)++ # Delete the first row and search with its (former) vector+ ann.delete([0])+ results = ann.search(np.array([data[0]]), 4)[0]++ # Deleted row must score 0 so the caller's score > 0 filter drops it and the data type must be unchanged+ self.assertEqual(dict(results).get(0), 0)+ self.assertEqual(ann.count(), 3)+ self.assertEqual(str(ann.backend.dtype), f"torch.{dtype.__name__}")+@unittest.skipIf(platform.system() == "Darwin", "Torch quantization not supported on macOS")def testTorchQuantization(self):"""