💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Word vector model improvements, closes #868
davidmezzettif52ce51413c5
diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.pyindex 99496579..520c7dc6 100644--- a/src/python/txtai/vectors/words.py+++ b/src/python/txtai/vectors/words.py@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)# Multiprocessing helper methods# pylint: disable=W0603-VECTORS = None+PARAMETERS, VECTORS = None, Nonedef create(config, scoring):@@ -42,10 +42,11 @@ def create(config, scoring):scoring: scoring instance"""+ global PARAMETERSglobal VECTORS- # Create a global embedding object using configuration and saved- VECTORS = WordVectors(config, scoring, None)+ # Store model parameters for lazy loading+ PARAMETERS, VECTORS = (config, scoring, None), Nonedef transform(document):@@ -59,6 +60,11 @@ def transform(document):(id, embedding)"""+ # Lazy load vectors model+ global VECTORS+ if not VECTORS:+ VECTORS = WordVectors(*PARAMETERS)+return (document[0], VECTORS.transform(document))@@ -145,9 +151,13 @@ class WordVectors(Vectors):return np.array(embeddings, dtype=np.float32)- def index(self, documents, batchsize=1):+ def index(self, documents, batchsize=500):+ # Derive number of parallel processes+ parallel = self.config.get("parallel", True)+ parallel = os.cpu_count() if parallel and isinstance(parallel, bool) else int(parallel)+# Use default single process indexing logic- if "parallel" in self.config and not self.config["parallel"]:+ if not parallel:return super().index(documents, batchsize)# Customize indexing logic with multiprocessing pool to efficiently build vectors@@ -157,11 +167,11 @@ class WordVectors(Vectors):args = (self.config, self.scoring)# Convert all documents to embedding arrays, stream embeddings to disk to control memory usage- with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:+ with Pool(parallel, initializer=create, initargs=args) as pool:with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:stream = output.nameembeddings = []- for uid, embedding in pool.imap(transform, documents):+ for uid, embedding in pool.imap(transform, documents, self.encodebatch):if not dimensions:# Set number of dimensions for embeddingsdimensions = embedding.shape[0]diff --git a/test/python/testvectors/testwordvectors.py b/test/python/testvectors/testwordvectors.pyindex 8abf9adb..13eb35e3 100644--- a/test/python/testvectors/testwordvectors.py+++ b/test/python/testvectors/testwordvectors.py@@ -12,6 +12,7 @@ import numpy as npfrom huggingface_hub.errors import HFValidationErrorfrom txtai.vectors import VectorsFactory+from txtai.vectors.words import create, transformclass TestWordVectors(unittest.TestCase):@@ -42,7 +43,7 @@ class TestWordVectors(unittest.TestCase):model = VectorsFactory.create({"path": self.path, "parallel": True}, None)- ids, dimension, batches, stream = model.index(documents)+ ids, dimension, batches, stream = model.index(documents, 1)self.assertEqual(len(ids), 1000)self.assertEqual(dimension, 300)@@ -89,7 +90,7 @@ class TestWordVectors(unittest.TestCase):model = VectorsFactory.create({"path": self.path, "parallel": False}, None)- ids, dimension, batches, stream = model.index(documents)+ ids, dimension, batches, stream = model.index(documents, 1)self.assertEqual(len(ids), 1000)self.assertEqual(dimension, 300)@@ -130,6 +131,17 @@ class TestWordVectors(unittest.TestCase):model = VectorsFactory.create({"path": self.path}, None)self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))+ def testMultiprocess(self):+ """+ Test multiprocess helper methods+ """++ create({"path": self.path}, None)++ uid, vector = transform((0, "test", None))+ self.assertEqual(uid, 0)+ self.assertEqual(vector.shape, (300,))+def testNoExist(self):"""Test loading model that doesn't exist