Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Word vector model improvements, closes #868

davidmezzettif52ce51413c5
diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.py
index 99496579..520c7dc6 100644
--- a/src/python/txtai/vectors/words.py
+++ b/src/python/txtai/vectors/words.py
@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
# Multiprocessing helper methods
# pylint: disable=W0603
-VECTORS = None
+PARAMETERS, VECTORS = None, None
def create(config, scoring):
@@ -42,10 +42,11 @@ def create(config, scoring):
scoring: scoring instance
"""
+ global PARAMETERS
global VECTORS
- # Create a global embedding object using configuration and saved
- VECTORS = WordVectors(config, scoring, None)
+ # Store model parameters for lazy loading
+ PARAMETERS, VECTORS = (config, scoring, None), None
def transform(document):
@@ -59,6 +60,11 @@ def transform(document):
(id, embedding)
"""
+ # Lazy load vectors model
+ global VECTORS
+ if not VECTORS:
+ VECTORS = WordVectors(*PARAMETERS)
+
return (document[0], VECTORS.transform(document))
@@ -145,9 +151,13 @@ class WordVectors(Vectors):
return np.array(embeddings, dtype=np.float32)
- def index(self, documents, batchsize=1):
+ def index(self, documents, batchsize=500):
+ # Derive number of parallel processes
+ parallel = self.config.get("parallel", True)
+ parallel = os.cpu_count() if parallel and isinstance(parallel, bool) else int(parallel)
+
# Use default single process indexing logic
- if "parallel" in self.config and not self.config["parallel"]:
+ if not parallel:
return super().index(documents, batchsize)
# Customize indexing logic with multiprocessing pool to efficiently build vectors
@@ -157,11 +167,11 @@ class WordVectors(Vectors):
args = (self.config, self.scoring)
# Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
- with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:
+ with Pool(parallel, initializer=create, initargs=args) as pool:
with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
stream = output.name
embeddings = []
- for uid, embedding in pool.imap(transform, documents):
+ for uid, embedding in pool.imap(transform, documents, self.encodebatch):
if not dimensions:
# Set number of dimensions for embeddings
dimensions = embedding.shape[0]
diff --git a/test/python/testvectors/testwordvectors.py b/test/python/testvectors/testwordvectors.py
index 8abf9adb..13eb35e3 100644
--- a/test/python/testvectors/testwordvectors.py
+++ b/test/python/testvectors/testwordvectors.py
@@ -12,6 +12,7 @@ import numpy as np
from huggingface_hub.errors import HFValidationError
from txtai.vectors import VectorsFactory
+from txtai.vectors.words import create, transform
class TestWordVectors(unittest.TestCase):
@@ -42,7 +43,7 @@ class TestWordVectors(unittest.TestCase):
model = VectorsFactory.create({"path": self.path, "parallel": True}, None)
- ids, dimension, batches, stream = model.index(documents)
+ ids, dimension, batches, stream = model.index(documents, 1)
self.assertEqual(len(ids), 1000)
self.assertEqual(dimension, 300)
@@ -89,7 +90,7 @@ class TestWordVectors(unittest.TestCase):
model = VectorsFactory.create({"path": self.path, "parallel": False}, None)
- ids, dimension, batches, stream = model.index(documents)
+ ids, dimension, batches, stream = model.index(documents, 1)
self.assertEqual(len(ids), 1000)
self.assertEqual(dimension, 300)
@@ -130,6 +131,17 @@ class TestWordVectors(unittest.TestCase):
model = VectorsFactory.create({"path": self.path}, None)
self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))
+ def testMultiprocess(self):
+ """
+ Test multiprocess helper methods
+ """
+
+ create({"path": self.path}, None)
+
+ uid, vector = transform((0, "test", None))
+ self.assertEqual(uid, 0)
+ self.assertEqual(vector.shape, (300,))
+
def testNoExist(self):
"""
Test loading model that doesn't exist