💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update comments
davidmezzetti424eeb80f633
diff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/annoy.pyindex 58c29261..8af3293c 100644--- a/src/python/txtai/ann/annoy.py+++ b/src/python/txtai/ann/annoy.py@@ -30,7 +30,7 @@ class Annoy(ANN):# Create indexself.model = AnnoyIndex(self.config["dimensions"], self.config["metric"])- # Add items+ # Add items - position in embeddings is used as the idfor x in range(embeddings.shape[0]):self.model.add_item(x, embeddings[x])diff --git a/src/python/txtai/ann/faiss.py b/src/python/txtai/ann/faiss.pyindex 677e16c9..ad59a576 100644--- a/src/python/txtai/ann/faiss.py+++ b/src/python/txtai/ann/faiss.py@@ -34,6 +34,8 @@ class Faiss(ANN):# Train modelself.model.train(embeddings)++ # Add embeddings - position in embeddings is used as the idself.model.add_with_ids(embeddings, np.arange(embeddings.shape[0], dtype=np.int64))# Update id offset@@ -42,7 +44,7 @@ class Faiss(ANN):def append(self, embeddings):new = embeddings.shape[0]- # Append new ids+ # Append new ids - position in embeddings + existing offset is used as the idself.model.add_with_ids(embeddings, np.arange(self.config["offset"], self.config["offset"] + new, dtype=np.int64))# Update id offsetdiff --git a/src/python/txtai/ann/hnsw.py b/src/python/txtai/ann/hnsw.pyindex 10117cea..540e1d1f 100644--- a/src/python/txtai/ann/hnsw.py+++ b/src/python/txtai/ann/hnsw.py@@ -39,7 +39,7 @@ class HNSW(ANN):self.model = Index(dim=self.config["dimensions"], space=self.config["metric"])self.model.init_index(max_elements=embeddings.shape[0], ef_construction=efconstruction, M=m, random_seed=seed)- # Add items+ # Add items - position in embeddings is used as the idself.model.add_items(embeddings, np.arange(embeddings.shape[0]))# Update id offset and set delete counter@@ -52,7 +52,7 @@ class HNSW(ANN):# Resize indexself.model.resize_index(self.config["offset"] + new)- # Append new ids+ # Append new ids - position in embeddings + existing offset is used as the idself.model.add_items(embeddings, np.arange(self.config["offset"], self.config["offset"] + new))# Update id offset