💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Allow setting custom ANN settings, closes #55
davidmezzetti53ce647b18ec
diff --git a/src/python/txtai/ann.py b/src/python/txtai/ann.pyindex 7d7a9a60..c7cb314b 100644--- a/src/python/txtai/ann.py+++ b/src/python/txtai/ann.py@@ -98,6 +98,25 @@ class ANN(object):Saves an ANN model at path."""+ def setting(self, name, default=None):+ """+ Looks up backend specific setting.++ Args:+ name: setting name+ default: default value when setting not found++ Returns:+ setting value+ """++ # Get the backend-specific config object+ backend = self.config.get(self.config["backend"])++ # Get setting value, set default value if not found+ setting = backend.get(name) if backend else None+ return setting if setting else default+class Annoy(ANN):"""Builds an ANN model using the Annoy library.@@ -120,14 +139,18 @@ class Annoy(ANN):self.model.add_item(x, embeddings[x])# Build index- self.model.build(10)+ self.model.build(self.setting("ntrees", 10))def search(self, queries, limit):+ # Lookup search k setting+ searchk = self.setting("searchk", -1)+# Annoy doesn't have a built in batch query methodresults = []for query in queries:# Run the query- ids, scores = self.model.get_nns_by_vector(query, n=limit, include_distances=True)+ ids, scores = self.model.get_nns_by_vector(query, n=limit, search_k=searchk,+ include_distances=True)# Map results to [(id, score)]results.append(list(zip(ids, scores)))@@ -148,8 +171,13 @@ class Faiss(ANN):self.model = faiss.read_index(path)def index(self, embeddings):+ # Lookup components setting+ components = self.setting("components")+# Create embeddings index. Inner product is equal to cosine similarity on normalized vectors.- if self.config.get("quantize"):+ if components:+ params = components+ elif self.config.get("quantize"):params = "IVF100,SQ8" if embeddings.shape[0] >= 5000 else "IDMap,SQ8"else:params = "IVF100,Flat" if embeddings.shape[0] >= 5000 else "IDMap,Flat"@@ -162,7 +190,7 @@ class Faiss(ANN):def search(self, queries, limit):# Run the query- self.model.nprobe = 6+ self.model.nprobe = self.setting("nprobe", 6)scores, ids = self.model.search(queries, limit)# Map results to [(id, score)]@@ -190,14 +218,25 @@ class HNSW(ANN):# Inner product is equal to cosine similarity on normalized vectorsself.config["metric"] = "ip"+ # Lookup index settings+ efconstruction = self.setting("efconstruction", 200)+ m = self.setting("m", 16)+ seed = self.setting("randomseed", 100)+# Create indexself.model = Index(dim=self.config["dimensions"], space=self.config["metric"])- self.model.init_index(max_elements=embeddings.shape[0])+ self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction,+ M=m, random_seed=seed)# Add itemsself.model.add_items(embeddings, np.array(range(embeddings.shape[0])))def search(self, queries, limit):+ # Set ef query param+ ef = self.setting("efsearch")+ if ef:+ self.model.set_ef(ef)+# Run the queryids, distances = self.model.knn_query(queries, k=limit)diff --git a/test/python/testann.py b/test/python/testann.pyindex f877b323..2fa9066c 100644--- a/test/python/testann.py+++ b/test/python/testann.py@@ -24,6 +24,18 @@ class TestANN(unittest.TestCase):self.assertIsNotNone(self.save("annoy"))self.assertGreater(self.search("annoy"), 0)+ def testAnnoyCustom(self):+ """+ Test Annoy backend with custom settings+ """++ # Test with custom settings+ config = {"annoy": {"ntrees": 2, "searchk": 1}}++ self.assertEqual(self.backend("annoy", config).config["backend"], "annoy")+ self.assertIsNotNone(self.save("annoy", config))+ self.assertGreater(self.search("annoy", config), 0)+@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")def testFaiss(self):"""@@ -31,11 +43,24 @@ class TestANN(unittest.TestCase):"""self.assertEqual(self.backend("faiss").config["backend"], "faiss")- self.assertEqual(self.backend("faiss", 5000).config["backend"], "faiss")+ self.assertEqual(self.backend("faiss", None, 5000).config["backend"], "faiss")self.assertIsNotNone(self.save("faiss"))self.assertGreater(self.search("faiss"), 0)+ @unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")+ def testFaissCustom(self):+ """+ Test Faiss backend with custom settings+ """++ # Test with custom settings+ config = {"faiss": {"nprobe": 2, "components": "PCA16,IDMap,SQ8"}}++ self.assertEqual(self.backend("faiss", config).config["backend"], "faiss")+ self.assertIsNotNone(self.save("faiss", config))+ self.assertGreater(self.search("faiss", config), 0)+def testHnsw(self):"""Test Hnswlib backend@@ -45,26 +70,57 @@ class TestANN(unittest.TestCase):self.assertIsNotNone(self.save("hnsw"))self.assertGreater(self.search("hnsw"), 0)- def backend(self, name, length=100):+ def testHnswCustom(self):+ """+ Test Hnswlib backend with custom settings+ """++ # Test with custom settings+ config = {"hnsw": {"efconstruction": 100, "m": 4, "randomseed": 0, "efsearch": 5}}++ self.assertEqual(self.backend("hnsw", config).config["backend"], "hnsw")+ self.assertIsNotNone(self.save("hnsw", config))+ self.assertGreater(self.search("hnsw", config), 0)++ def backend(self, name, params=None, length=10000):"""- Test a backend+ Test a backend.++ Args:+ backend: backend name+ params: additional config parameters+ length: number of rows to generate++ Returns:+ ANN model"""# Generate test datadata = np.random.rand(length, 300).astype(np.float32)self.normalize(data)- model = ANN.create({"backend": name, "dimensions": data.shape[1]})+ config = {"backend": name, "dimensions": data.shape[1]}+ if params:+ config.update(params)++ model = ANN.create(config)model.index(data)return model- def save(self, backend):+ def save(self, backend, params=None):"""- Test save/load+ Test save/load.++ Args:+ backend: backend name+ params: additional config parameters++ Returns:+ ANN model"""- model = self.backend(backend)+ model = self.backend(backend, params)# Generate temp file pathindex = os.path.join(tempfile.gettempdir(), "ann")@@ -74,13 +130,20 @@ class TestANN(unittest.TestCase):return model- def search(self, backend):+ def search(self, backend, params=None):"""- Test ANN search+ Test ANN search.++ Args:+ backend: backend name+ params: additional config parameters++ Returns:+ search results"""# Generate ANN index- model = self.backend(backend)+ model = self.backend(backend, params)# Generate query vectorquery = np.random.rand(300).astype(np.float32)