💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Modify file delete
davidmezzettib25173a8650a
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 211fa337..684d3d4f 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -6,6 +6,7 @@ import jsonimport pickleimport osimport shutil+import tempfileimport typesimport numpy as np@@ -91,34 +92,30 @@ class Embeddings:# Create transform actiontransform = Transform(self, Action.REINDEX if reindex else Action.INDEX)- # Load documents into database and transform to vectors- ids, dimensions, embeddings = transform(documents)+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy") as buffer:+ # Load documents into database and transform to vectors+ ids, dimensions, embeddings = transform(documents, buffer)- # Build LSA model (if enabled). Remove principal components from embeddings.- if self.config.get("pca"):- self.reducer = Reducer(embeddings, self.config["pca"])- self.reducer(embeddings)-- # Normalize embeddings- self.normalize(embeddings)+ # Build LSA model (if enabled). Remove principal components from embeddings.+ if self.config.get("pca"):+ self.reducer = Reducer(embeddings, self.config["pca"])+ self.reducer(embeddings)- # Save index dimensions- self.config["dimensions"] = dimensions+ # Normalize embeddings+ self.normalize(embeddings)- # Create approximate nearest neighbor index- self.ann = ANNFactory.create(self.config)+ # Save index dimensions+ self.config["dimensions"] = dimensions- # Add embeddings to the index- self.ann.index(embeddings)+ # Create approximate nearest neighbor index+ self.ann = ANNFactory.create(self.config)- # Save indexids-ids mapping for indexes with no database, except when this is a reindex action- if not reindex and not self.database:- self.config["ids"] = ids+ # Add embeddings to the index+ self.ann.index(embeddings)- # Delete embeddings memmap buffer- path = embeddings.filename- del embeddings- os.remove(path)+ # Save indexids-ids mapping for indexes with no database, except when this is a reindex action+ if not reindex and not self.database:+ self.config["ids"] = idsdef upsert(self, documents):"""@@ -138,23 +135,19 @@ class Embeddings:# Create transform actiontransform = Transform(self, Action.UPSERT)- # Load documents into database and transform to vectors- ids, _, embeddings = transform(documents)-- # Normalize embeddings- self.normalize(embeddings)+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy") as buffer:+ # Load documents into database and transform to vectors+ ids, _, embeddings = transform(documents, buffer)- # Append embeddings to the index- self.ann.append(embeddings)+ # Normalize embeddings+ self.normalize(embeddings)- # Save indexids-ids mapping for indexes with no database- if not self.database:- self.config["ids"] = self.config["ids"] + ids+ # Append embeddings to the index+ self.ann.append(embeddings)- # Delete embeddings memmap buffer- path = embeddings.filename- del embeddings- os.remove(path)+ # Save indexids-ids mapping for indexes with no database+ if not self.database:+ self.config["ids"] = self.config["ids"] + idsdef delete(self, ids):"""diff --git a/src/python/txtai/embeddings/transform.py b/src/python/txtai/embeddings/transform.pyindex 3c074124..f38ab68d 100644--- a/src/python/txtai/embeddings/transform.py+++ b/src/python/txtai/embeddings/transform.py@@ -4,7 +4,6 @@ Transform moduleimport osimport pickle-import tempfilefrom enum import Enum@@ -40,7 +39,7 @@ class Transform:# List of deleted ids with this actionself.deletes = set()- def __call__(self, documents):+ def __call__(self, documents, buffer):"""Processes an iterable collection of documents, handles any iterable including generators.@@ -48,6 +47,7 @@ class Transform:Args:documents: iterable collection of (id, data, tags)+ buffer: file path used for memmap bufferReturns:(document ids, dimensions, embeddings)@@ -56,10 +56,6 @@ class Transform:# Transform documents to vectors and load into databaseids, dimensions, batches, stream = self.model.index(self.stream(documents))- # Generate buffer file name- # pylint: disable=R1732- buffer = tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False)-# Load streamed embeddings back to memoryembeddings = np.memmap(buffer, dtype=np.float32, shape=(len(ids), dimensions), mode="w+")with open(stream, "rb") as queue: