Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Refactor vectors package to improve code reuse, closes #276

davidmezzettib8c29666de7f
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.py
index b8d6f2b9..754e50f8 100644
--- a/src/python/txtai/scoring/base.py
+++ b/src/python/txtai/scoring/base.py
@@ -83,12 +83,12 @@ class Scoring:
# Filter for tags that appear in at least 1% of the documents
self.tags = {tag: number for tag, number in self.tags.items() if number >= self.total * 0.005}
- def weights(self, document):
+ def weights(self, tokens):
"""
Builds weight vector for each token in the input token.
Args:
- document: (id, tokens, tags)
+ tokens: input tokens
Returns:
list of weights for each token
@@ -97,9 +97,6 @@ class Scoring:
# Weights array
weights = []
- # Unpack document
- _, tokens, _ = document
-
# Document length
length = len(tokens)
diff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.py
index 93f036dc..239e0101 100644
--- a/src/python/txtai/vectors/base.py
+++ b/src/python/txtai/vectors/base.py
@@ -2,6 +2,9 @@
Vectors module
"""
+import pickle
+import tempfile
+
class Vectors:
"""
@@ -36,19 +39,56 @@ class Vectors:
raise NotImplementedError
- def index(self, documents):
+ def encode(self, data):
+ """
+ Encodes a batch of data using vector model.
+
+ Args:
+ data: batch of data
+
+ Return:
+ transformed data
+ """
+
+ raise NotImplementedError
+
+ def index(self, documents, batchsize=500):
"""
Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,
number of dimensions and temporary file with embeddings.
Args:
documents: list of (id, data, tags)
+ batchsize: index batch size
Returns:
(ids, dimensions, stream)
"""
- raise NotImplementedError
+ ids, dimensions, batches, stream = [], None, 0, None
+
+ # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
+ with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
+ stream = output.name
+ batch = []
+ for document in documents:
+ batch.append(document)
+
+ if len(batch) == batchsize:
+ # Convert batch to embeddings
+ uids, dimensions = self.batch(batch, output)
+ ids.extend(uids)
+ batches += 1
+
+ batch = []
+
+ # Final batch
+ if batch:
+ uids, dimensions = self.batch(batch, output)
+ ids.extend(uids)
+ batches += 1
+
+ return (ids, dimensions, batches, stream)
def transform(self, document):
"""
@@ -61,4 +101,43 @@ class Vectors:
embeddings vector
"""
- raise NotImplementedError
+ # Prepare input document for transformers model and build embeddings
+ return self.encode([self.prepare(document[1])])[0]
+
+ def batch(self, documents, output):
+ """
+ Builds a batch of embeddings.
+
+ Args:
+ documents: list of documents used to build embeddings
+ output: output temp file to store embeddings
+
+ Returns:
+ (ids, dimensions) list of ids and number of dimensions in embeddings
+ """
+
+ # Extract ids and prepare input documents for transformers model
+ ids = [uid for uid, _, _ in documents]
+ documents = [self.prepare(data) for _, data, _ in documents]
+ dimensions = None
+
+ # Build embeddings
+ embeddings = self.encode(documents)
+ if embeddings is not None:
+ dimensions = embeddings.shape[1]
+ pickle.dump(embeddings, output, protocol=4)
+
+ return (ids, dimensions)
+
+ def prepare(self, data):
+ """
+ Prepares input data for vector model.
+
+ Args:
+ data: input data
+
+ Returns:
+ data formatted for vector model
+ """
+
+ return data
diff --git a/src/python/txtai/vectors/external.py b/src/python/txtai/vectors/external.py
index 79a19893..441e6472 100644
--- a/src/python/txtai/vectors/external.py
+++ b/src/python/txtai/vectors/external.py
@@ -2,9 +2,6 @@
External module
"""
-import pickle
-import tempfile
-
import numpy as np
from .base import Vectors
@@ -19,72 +16,7 @@ class ExternalVectors(Vectors):
def load(self, path):
return None
- def index(self, documents):
- ids, dimensions, batches, stream = [], None, 0, None
-
- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
- stream = output.name
- batch = []
- for document in documents:
- batch.append(document)
-
- if len(batch) == 500:
- # Convert batch to embeddings
- uids, dimensions = self.batch(batch, output)
- ids.extend(uids)
- batches += 1
-
- batch = []
-
- # Final batch
- if batch:
- uids, dimensions = self.batch(batch, output)
- ids.extend(uids)
- batches += 1
-
- return (ids, dimensions, batches, stream)
-
- def transform(self, document):
- # Encode data and return
- return self.encode([document[1]])[0]
-
- def batch(self, documents, output):
- """
- Builds a batch of embeddings.
-
- Args:
- documents: list of documents used to build embeddings
- output: output temp file to store embeddings
-
- Returns:
- (ids, dimensions) list of ids and number of dimensions in embeddings
- """
-
- # Extract ids and prepare input documents
- ids = [uid for uid, _, _ in documents]
- documents = [data for _, data, _ in documents]
- dimensions = None
-
- # Build embeddings
- embeddings = self.encode(documents)
- if embeddings is not None:
- dimensions = embeddings.shape[1]
- pickle.dump(embeddings, output, protocol=4)
-
- return (ids, dimensions)
-
def encode(self, data):
- """
- Encodes a batch of data using the external transform function.
-
- Args:
- data: batch of data
-
- Return:
- transformed data
- """
-
# Call external transform function, if available and data not already an array
transform = self.config.get("transform")
if transform and data and not isinstance(data[0], np.ndarray):
diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.py
index 2bb10b49..99980761 100644
--- a/src/python/txtai/vectors/transformers.py
+++ b/src/python/txtai/vectors/transformers.py
@@ -3,8 +3,6 @@ Transformers module
"""
import os
-import pickle
-import tempfile
# Conditional import
try:
@@ -45,73 +43,7 @@ class TransformersVectors(Vectors):
# Build embeddings with sentence-transformers
return SentenceTransformer(path, device=Models.reference(deviceid))
- def index(self, documents):
- ids, dimensions, batches, stream = [], None, 0, None
-
- # Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
- with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
- stream = output.name
- batch = []
- for document in documents:
- batch.append(document)
-
- if len(batch) == 500:
- # Convert batch to embeddings
- uids, dimensions = self.batch(batch, output)
- ids.extend(uids)
- batches += 1
-
- batch = []
-
- # Final batch
- if batch:
- uids, dimensions = self.batch(batch, output)
- ids.extend(uids)
- batches += 1
-
- return (ids, dimensions, batches, stream)
-
- def transform(self, document):
- # Prepare input document for transformers model and build embeddings
- return self.model.encode([self.prepare(document[1])])[0]
-
- def batch(self, documents, output):
- """
- Builds a batch of embeddings.
-
- Args:
- documents: list of documents used to build embeddings
- output: output temp file to store embeddings
-
- Returns:
- (ids, dimensions) list of ids and number of dimensions in embeddings
- """
-
- # Extract ids and prepare input documents for transformers model
- ids = [uid for uid, _, _ in documents]
- documents = [self.prepare(data) for _, data, _ in documents]
- dimensions = None
-
- # Build embeddings
- embeddings = self.model.encode(documents)
- if embeddings is not None:
- dimensions = embeddings.shape[1]
- pickle.dump(embeddings, output, protocol=4)
-
- return (ids, dimensions)
-
def prepare(self, data):
- """
- Prepares input data for transformers model. This method supports optional string tokenization and
- joins tokenized input into text.
-
- Args:
- data: input data
-
- Returns:
- data formatted for transformers model
- """
-
# Optional string tokenization
if self.tokenize and isinstance(data, str):
data = Tokenizer.tokenize(data)
@@ -121,3 +53,7 @@ class TransformersVectors(Vectors):
data = " ".join(data)
return data
+
+ def encode(self, data):
+ # Encode data using vectors model
+ return self.model.encode(data)
diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.py
index 378366ad..5aae632e 100644
--- a/src/python/txtai/vectors/words.py
+++ b/src/python/txtai/vectors/words.py
@@ -57,34 +57,6 @@ def transform(document):
return (document[0], VECTORS.transform(document))
-class SerialPool:
- """
- Custom pool to execute vector transforms serially.
- """
-
- def __init__(self, vectors):
- global VECTORS
- VECTORS = vectors
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
- def imap(self, func, iterable):
- """
- Single process version of imap.
-
- Args:
- func: function to run
- iterable: iterable to use
- """
-
- for x in iterable:
- yield func(x)
-
-
class WordVectors(Vectors):
"""
Builds sentence embeddings/vectors using weighted word embeddings.
@@ -98,16 +70,42 @@ class WordVectors(Vectors):
# Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded
return Magnitude(path, case_insensitive=True, blocking=not self.initialized)
- def index(self, documents):
+ def encode(self, data):
+ # Iterate over each data element, tokenize (if necessary) and build an aggregated embeddings vector
+ embeddings = []
+ for tokens in data:
+ # Convert to tokens if necessary
+ if isinstance(tokens, str):
+ tokens = Tokenizer.tokenize(tokens)
+
+ # Generate weights for each vector using a scoring method
+ weights = self.scoring.weights(tokens) if self.scoring else None
+
+ # pylint: disable=E1133
+ if weights and [x for x in weights if x > 0]:
+ # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.
+ embedding = np.average(self.lookup(tokens), weights=np.array(weights, dtype=np.float32), axis=0)
+ else:
+ # If no weights, use mean
+ embedding = np.mean(self.lookup(tokens), axis=0)
+
+ embeddings.append(embedding)
+
+ return np.array(embeddings, dtype=np.float32)
+
+ def index(self, documents, batchsize=1):
+ # Use default single process indexing logic
+ if "parallel" in self.config and not self.config["parallel"]:
+ return super().index(documents, batchsize)
+
+ # Customize indexing logic with multiprocessing pool to efficiently build vectors
ids, dimensions, stream = [], None, None
# Shared objects with Pool
args = (self.config, self.scoring)
# Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
- with SerialPool(self) if "parallel" in self.config and not self.config["parallel"] else Pool(
- os.cpu_count(), initializer=create, initargs=args
- ) as pool:
+ with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:
with tempfile.NamedTemporaryFile(mode="wb", suffix=".npy", delete=False) as output:
stream = output.name
for uid, embedding in pool.imap(transform, documents):
@@ -120,24 +118,6 @@ class WordVectors(Vectors):
return (ids, dimensions, len(ids), stream)
- def transform(self, document):
- # Convert to tokens if necessary
- if isinstance(document[1], str):
- document = (document[0], Tokenizer.tokenize(document[1]), document[2])
-
- # Generate weights for each vector using a scoring method
- weights = self.scoring.weights(document) if self.scoring else None
-
- # pylint: disable=E1133
- if weights and [x for x in weights if x > 0]:
- # Build weighted average embeddings vector. Create weights array os float32 to match embeddings precision.
- embedding = np.average(self.lookup(document[1]), weights=np.array(weights, dtype=np.float32), axis=0)
- else:
- # If no weights, use mean
- embedding = np.mean(self.lookup(document[1]), axis=0)
-
- return embedding
-
def lookup(self, tokens):
"""
Queries word vectors for given list of input tokens.
diff --git a/test/python/testscoring.py b/test/python/testscoring.py
index c1cadc8e..7435a0bd 100644
--- a/test/python/testscoring.py
+++ b/test/python/testscoring.py
@@ -122,7 +122,7 @@ class TestScoring(unittest.TestCase):
document = (1, ["bear", "wins"], None)
model = self.method(method)
- weights = model.weights(document)
+ weights = model.weights(document[1])
# Default weights
self.assertNotEqual(weights[0], weights[1])
@@ -133,7 +133,7 @@ class TestScoring(unittest.TestCase):
data[3] = (uid, text, "wins")
model = self.method(method, data)
- weights = model.weights(document)
+ weights = model.weights(document[1])
# Modified weights
self.assertEqual(weights[0], weights[1])
diff --git a/test/python/testvectors/testvectors.py b/test/python/testvectors/testvectors.py
index 6b62d8aa..ebfa30de 100644
--- a/test/python/testvectors/testvectors.py
+++ b/test/python/testvectors/testvectors.py
@@ -20,5 +20,4 @@ class TestVectors(unittest.TestCase):
vectors = Vectors(None, None)
self.assertRaises(NotImplementedError, vectors.load, None)
- self.assertRaises(NotImplementedError, vectors.index, None)
- self.assertRaises(NotImplementedError, vectors.transform, None)
+ self.assertRaises(NotImplementedError, vectors.encode, None)