Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Modify file delete

davidmezzettib25173a8650a
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index 211fa337..684d3d4f 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -6,6 +6,7 @@ import json
import pickle
import os
import shutil
+import tempfile
import types
import numpy as np
@@ -91,34 +92,30 @@ class Embeddings:
# Create transform action
transform = Transform(self, Action.REINDEX if reindex else Action.INDEX)
- # Load documents into database and transform to vectors
- ids, dimensions, embeddings = transform(documents)
+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy") as buffer:
+ # Load documents into database and transform to vectors
+ ids, dimensions, embeddings = transform(documents, buffer)
- # Build LSA model (if enabled). Remove principal components from embeddings.
- if self.config.get("pca"):
- self.reducer = Reducer(embeddings, self.config["pca"])
- self.reducer(embeddings)
-
- # Normalize embeddings
- self.normalize(embeddings)
+ # Build LSA model (if enabled). Remove principal components from embeddings.
+ if self.config.get("pca"):
+ self.reducer = Reducer(embeddings, self.config["pca"])
+ self.reducer(embeddings)
- # Save index dimensions
- self.config["dimensions"] = dimensions
+ # Normalize embeddings
+ self.normalize(embeddings)
- # Create approximate nearest neighbor index
- self.ann = ANNFactory.create(self.config)
+ # Save index dimensions
+ self.config["dimensions"] = dimensions
- # Add embeddings to the index
- self.ann.index(embeddings)
+ # Create approximate nearest neighbor index
+ self.ann = ANNFactory.create(self.config)
- # Save indexids-ids mapping for indexes with no database, except when this is a reindex action
- if not reindex and not self.database:
- self.config["ids"] = ids
+ # Add embeddings to the index
+ self.ann.index(embeddings)
- # Delete embeddings memmap buffer
- path = embeddings.filename
- del embeddings
- os.remove(path)
+ # Save indexids-ids mapping for indexes with no database, except when this is a reindex action
+ if not reindex and not self.database:
+ self.config["ids"] = ids
def upsert(self, documents):
"""
@@ -138,23 +135,19 @@ class Embeddings:
# Create transform action
transform = Transform(self, Action.UPSERT)
- # Load documents into database and transform to vectors
- ids, _, embeddings = transform(documents)
-
- # Normalize embeddings
- self.normalize(embeddings)
+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy") as buffer:
+ # Load documents into database and transform to vectors
+ ids, _, embeddings = transform(documents, buffer)
- # Append embeddings to the index
- self.ann.append(embeddings)
+ # Normalize embeddings
+ self.normalize(embeddings)
- # Save indexids-ids mapping for indexes with no database
- if not self.database:
- self.config["ids"] = self.config["ids"] + ids
+ # Append embeddings to the index
+ self.ann.append(embeddings)
- # Delete embeddings memmap buffer
- path = embeddings.filename
- del embeddings
- os.remove(path)
+ # Save indexids-ids mapping for indexes with no database
+ if not self.database:
+ self.config["ids"] = self.config["ids"] + ids
def delete(self, ids):
"""
diff --git a/src/python/txtai/embeddings/transform.py b/src/python/txtai/embeddings/transform.py
index 3c074124..f38ab68d 100644
--- a/src/python/txtai/embeddings/transform.py
+++ b/src/python/txtai/embeddings/transform.py
@@ -4,7 +4,6 @@ Transform module
import os
import pickle
-import tempfile
from enum import Enum
@@ -40,7 +39,7 @@ class Transform:
# List of deleted ids with this action
self.deletes = set()
- def __call__(self, documents):
+ def __call__(self, documents, buffer):
"""
Processes an iterable collection of documents, handles any iterable including generators.
@@ -48,6 +47,7 @@ class Transform:
Args:
documents: iterable collection of (id, data, tags)
+ buffer: file path used for memmap buffer
Returns:
(document ids, dimensions, embeddings)
@@ -56,10 +56,6 @@ class Transform:
# Transform documents to vectors and load into database
ids, dimensions, batches, stream = self.model.index(self.stream(documents))
- # Generate buffer file name
- # pylint: disable=R1732
- buffer = tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False)
-
# Load streamed embeddings back to memory
embeddings = np.memmap(buffer, dtype=np.float32, shape=(len(ids), dimensions), mode="w+")
with open(stream, "rb") as queue: