💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Refactor vectors package to improve code reuse, closes #276
davidmezzettib8c29666de7f
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.pyindex b8d6f2b9..754e50f8 100644--- a/src/python/txtai/scoring/base.py+++ b/src/python/txtai/scoring/base.py@@ -83,12 +83,12 @@ class Scoring:# Filter for tags that appear in at least 1% of the documentsself.tags = {tag: number for tag, number in self.tags.items() if number >= self.total * 0.005}- def weights(self, document):+ def weights(self, tokens):"""Builds weight vector for each token in the input token.Args:- document: (id, tokens, tags)+ tokens: input tokensReturns:list of weights for each token@@ -97,9 +97,6 @@ class Scoring:# Weights arrayweights = []- # Unpack document- _, tokens, _ = document-# Document lengthlength = len(tokens)diff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.pyindex 93f036dc..239e0101 100644--- a/src/python/txtai/vectors/base.py+++ b/src/python/txtai/vectors/base.py@@ -2,6 +2,9 @@Vectors module"""+import pickle+import tempfile+class Vectors:"""@@ -36,19 +39,56 @@ class Vectors:raise NotImplementedError- def index(self, documents):+ def encode(self, data):+ """+ Encodes a batch of data using vector model.++ Args:+ data: batch of data++ Return:+ transformed data+ """++ raise NotImplementedError++ def index(self, documents, batchsize=500):"""Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,number of dimensions and temporary file with embeddings.Args:documents: list of (id, data, tags)+ batchsize: index batch sizeReturns:(ids, dimensions, stream)"""- raise NotImplementedError+ ids, dimensions, batches, stream = [], None, 0, None++ # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:+ stream = output.name+ batch = []+ for document in documents:+ batch.append(document)++ if len(batch) == batchsize:+ # Convert batch to embeddings+ uids, dimensions = self.batch(batch, output)+ ids.extend(uids)+ batches += 1++ batch = []++ # Final batch+ if batch:+ uids, dimensions = self.batch(batch, output)+ ids.extend(uids)+ batches += 1++ return (ids, dimensions, batches, stream)def transform(self, document):"""@@ -61,4 +101,43 @@ class Vectors:embeddings vector"""- raise NotImplementedError+ # Prepare input document for transformers model and build embeddings+ return self.encode([self.prepare(document[1])])[0]++ def batch(self, documents, output):+ """+ Builds a batch of embeddings.++ Args:+ documents: list of documents used to build embeddings+ output: output temp file to store embeddings++ Returns:+ (ids, dimensions) list of ids and number of dimensions in embeddings+ """++ # Extract ids and prepare input documents for transformers model+ ids = [uid for uid, _, _ in documents]+ documents = [self.prepare(data) for _, data, _ in documents]+ dimensions = None++ # Build embeddings+ embeddings = self.encode(documents)+ if embeddings is not None:+ dimensions = embeddings.shape[1]+ pickle.dump(embeddings, output, protocol=4)++ return (ids, dimensions)++ def prepare(self, data):+ """+ Prepares input data for vector model.++ Args:+ data: input data++ Returns:+ data formatted for vector model+ """++ return datadiff --git a/src/python/txtai/vectors/external.py b/src/python/txtai/vectors/external.pyindex 79a19893..441e6472 100644--- a/src/python/txtai/vectors/external.py+++ b/src/python/txtai/vectors/external.py@@ -2,9 +2,6 @@External module"""-import pickle-import tempfile-import numpy as npfrom .base import Vectors@@ -19,72 +16,7 @@ class ExternalVectors(Vectors):def load(self, path):return None- def index(self, documents):- ids, dimensions, batches, stream = [], None, 0, None-- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:- stream = output.name- batch = []- for document in documents:- batch.append(document)-- if len(batch) == 500:- # Convert batch to embeddings- uids, dimensions = self.batch(batch, output)- ids.extend(uids)- batches += 1-- batch = []-- # Final batch- if batch:- uids, dimensions = self.batch(batch, output)- ids.extend(uids)- batches += 1-- return (ids, dimensions, batches, stream)-- def transform(self, document):- # Encode data and return- return self.encode([document[1]])[0]-- def batch(self, documents, output):- """- Builds a batch of embeddings.-- Args:- documents: list of documents used to build embeddings- output: output temp file to store embeddings-- Returns:- (ids, dimensions) list of ids and number of dimensions in embeddings- """-- # Extract ids and prepare input documents- ids = [uid for uid, _, _ in documents]- documents = [data for _, data, _ in documents]- dimensions = None-- # Build embeddings- embeddings = self.encode(documents)- if embeddings is not None:- dimensions = embeddings.shape[1]- pickle.dump(embeddings, output, protocol=4)-- return (ids, dimensions)-def encode(self, data):- """- Encodes a batch of data using the external transform function.-- Args:- data: batch of data-- Return:- transformed data- """-# Call external transform function, if available and data not already an arraytransform = self.config.get("transform")if transform and data and not isinstance(data[0], np.ndarray):diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.pyindex 2bb10b49..99980761 100644--- a/src/python/txtai/vectors/transformers.py+++ b/src/python/txtai/vectors/transformers.py@@ -3,8 +3,6 @@ Transformers module"""import os-import pickle-import tempfile# Conditional importtry:@@ -45,73 +43,7 @@ class TransformersVectors(Vectors):# Build embeddings with sentence-transformersreturn SentenceTransformer(path, device=Models.reference(deviceid))- def index(self, documents):- ids, dimensions, batches, stream = [], None, 0, None-- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:- stream = output.name- batch = []- for document in documents:- batch.append(document)-- if len(batch) == 500:- # Convert batch to embeddings- uids, dimensions = self.batch(batch, output)- ids.extend(uids)- batches += 1-- batch = []-- # Final batch- if batch:- uids, dimensions = self.batch(batch, output)- ids.extend(uids)- batches += 1-- return (ids, dimensions, batches, stream)-- def transform(self, document):- # Prepare input document for transformers model and build embeddings- return self.model.encode([self.prepare(document[1])])[0]-- def batch(self, documents, output):- """- Builds a batch of embeddings.-- Args:- documents: list of documents used to build embeddings- output: output temp file to store embeddings-- Returns:- (ids, dimensions) list of ids and number of dimensions in embeddings- """-- # Extract ids and prepare input documents for transformers model- ids = [uid for uid, _, _ in documents]- documents = [self.prepare(data) for _, data, _ in documents]- dimensions = None-- # Build embeddings- embeddings = self.model.encode(documents)- if embeddings is not None:- dimensions = embeddings.shape[1]- pickle.dump(embeddings, output, protocol=4)-- return (ids, dimensions)-def prepare(self, data):- """- Prepares input data for transformers model. This method supports optional string tokenization and- joins tokenized input into text.-- Args:- data: input data-- Returns:- data formatted for transformers model- """-# Optional string tokenizationif self.tokenize and isinstance(data, str):data = Tokenizer.tokenize(data)@@ -121,3 +53,7 @@ class TransformersVectors(Vectors):data = " ".join(data)return data++ def encode(self, data):+ # Encode data using vectors model+ return self.model.encode(data)diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.pyindex 378366ad..5aae632e 100644--- a/src/python/txtai/vectors/words.py+++ b/src/python/txtai/vectors/words.py@@ -57,34 +57,6 @@ def transform(document):return (document[0], VECTORS.transform(document))-class SerialPool:- """- Custom pool to execute vector transforms serially.- """-- def __init__(self, vectors):- global VECTORS- VECTORS = vectors-- def __enter__(self):- return self-- def __exit__(self, exc_type, exc_val, exc_tb):- pass-- def imap(self, func, iterable):- """- Single process version of imap.-- Args:- func: function to run- iterable: iterable to use- """-- for x in iterable:- yield func(x)--class WordVectors(Vectors):"""Builds sentence embeddings/vectors using weighted word embeddings.@@ -98,16 +70,42 @@ class WordVectors(Vectors):# Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loadedreturn Magnitude(path, case_insensitive=True, blocking=not self.initialized)- def index(self, documents):+ def encode(self, data):+ # Iterate over each data element, tokenize (if necessary) and build an aggregated embeddings vector+ embeddings = []+ for tokens in data:+ # Convert to tokens if necessary+ if isinstance(tokens, str):+ tokens = Tokenizer.tokenize(tokens)++ # Generate weights for each vector using a scoring method+ weights = self.scoring.weights(tokens) if self.scoring else None++ # pylint: disable=E1133+ if weights and [x for x in weights if x > 0]:+ # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.+ embedding = np.average(self.lookup(tokens), weights=np.array(weights, dtype=np.float32), axis=0)+ else:+ # If no weights, use mean+ embedding = np.mean(self.lookup(tokens), axis=0)++ embeddings.append(embedding)++ return np.array(embeddings, dtype=np.float32)++ def index(self, documents, batchsize=1):+ # Use default single process indexing logic+ if "parallel" in self.config and not self.config["parallel"]:+ return super().index(documents, batchsize)++ # Customize indexing logic with multiprocessing pool to efficiently build vectorsids, dimensions, stream = [], None, None# Shared objects with Poolargs = (self.config, self.scoring)# Convert all documents to embedding arrays, stream embeddings to disk to control memory usage- with SerialPool(self) if "parallel" in self.config and not self.config["parallel"] else Pool(- os.cpu_count(), initializer=create, initargs=args- ) as pool:+ with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:stream = output.namefor uid, embedding in pool.imap(transform, documents):@@ -120,24 +118,6 @@ class WordVectors(Vectors):return (ids, dimensions, len(ids), stream)- def transform(self, document):- # Convert to tokens if necessary- if isinstance(document[1], str):- document = (document[0], Tokenizer.tokenize(document[1]), document[2])-- # Generate weights for each vector using a scoring method- weights = self.scoring.weights(document) if self.scoring else None-- # pylint: disable=E1133- if weights and [x for x in weights if x > 0]:- # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.- embedding = np.average(self.lookup(document[1]), weights=np.array(weights, dtype=np.float32), axis=0)- else:- # If no weights, use mean- embedding = np.mean(self.lookup(document[1]), axis=0)-- return embedding-def lookup(self, tokens):"""Queries word vectors for given list of input tokens.diff --git a/test/python/testscoring.py b/test/python/testscoring.pyindex c1cadc8e..7435a0bd 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring.py@@ -122,7 +122,7 @@ class TestScoring(unittest.TestCase):document = (1, ["bear", "wins"], None)model = self.method(method)- weights = model.weights(document)+ weights = model.weights(document[1])# Default weightsself.assertNotEqual(weights[0], weights[1])@@ -133,7 +133,7 @@ class TestScoring(unittest.TestCase):data[3] = (uid, text, "wins")model = self.method(method, data)- weights = model.weights(document)+ weights = model.weights(document[1])# Modified weightsself.assertEqual(weights[0], weights[1])diff --git a/test/python/testvectors/testvectors.py b/test/python/testvectors/testvectors.pyindex 6b62d8aa..ebfa30de 100644--- a/test/python/testvectors/testvectors.py+++ b/test/python/testvectors/testvectors.py@@ -20,5 +20,4 @@ class TestVectors(unittest.TestCase):vectors = Vectors(None, None)self.assertRaises(NotImplementedError, vectors.load, None)- self.assertRaises(NotImplementedError, vectors.index, None)- self.assertRaises(NotImplementedError, vectors.transform, None)+ self.assertRaises(NotImplementedError, vectors.encode, None)