Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Ensure output directory exists before writing index

davidmezzettib4d8b80e0b61
diff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings.py
index e0c0ba7b..3b5377c1 100644
--- a/src/python/txtai/embeddings.py
+++ b/src/python/txtai/embeddings.py
@@ -244,6 +244,7 @@ class Embeddings(object):
path: input directory path
"""
+ # Index configuration
with open("%s/config" % path, "rb") as handle:
self.config = pickle.load(handle)
@@ -260,7 +261,7 @@ class Embeddings(object):
self.scoring = Scoring.create(self.config["scoring"])
self.scoring.load(path)
- # Sentence vectors model
+ # Sentence vectors model - transforms text into sentence embeddings
self.model = self.loadVectors(self.config["path"])
def save(self, path):
@@ -272,12 +273,17 @@ class Embeddings(object):
"""
if self.config:
+ # Create output directory, if necessary
+ os.makedirs(path, exist_ok=True)
+
+ # Write index configuration
with open("%s/config" % path, "wb") as handle:
pickle.dump(self.config, handle, protocol=pickle.HIGHEST_PROTOCOL)
- # Write sentence embeddings
+ # Write sentence embeddings index
faiss.write_index(self.embeddings, "%s/embeddings" % path)
+ # Save dimensionality reduction
if self.lsa:
with open("%s/lsa" % path, "wb") as handle:
pickle.dump(self.lsa, handle, protocol=pickle.HIGHEST_PROTOCOL)