Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Added flag to disable default tokenization for vector models. This addresses language support issues in #39 and #43.

davidmezzetti172cd601bc83
diff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings.py
index 38ffb16d..d038ff87 100644
--- a/src/python/txtai/embeddings.py
+++ b/src/python/txtai/embeddings.py
@@ -56,14 +56,14 @@ class Embeddings(object):
vector model
"""
- return Vectors.create(self.config.get("method"), self.config["path"], True if not self.embeddings else False, self.scoring)
+ return Vectors.create(self.config, self.scoring)
def score(self, documents):
"""
- Builds a scoring index. Documents are tuples of (id, text|tokens, tags).
+ Builds a scoring index.
Args:
- documents: list of documents
+ documents: list of (id, text|tokens, tags)
"""
if self.scoring:
@@ -72,10 +72,10 @@ class Embeddings(object):
def index(self, documents):
"""
- Builds an embeddings index. Documents are tuples of (id, text|tokens, tags).
+ Builds an embeddings index.
Args:
- documents: list of documents
+ documents: list of (id, text|tokens, tags)
"""
# Transform documents to embeddings vectors
diff --git a/src/python/txtai/vectors.py b/src/python/txtai/vectors.py
index 019620aa..33640b7b 100644
--- a/src/python/txtai/vectors.py
+++ b/src/python/txtai/vectors.py
@@ -22,24 +22,23 @@ from .tokenizer import Tokenizer
# pylint: disable=W0603
VECTORS = None
-def create(path, scoring, blocking):
+def create(config, scoring):
"""
- Multiprocessing helper method. Creates a global embeddings object to be accessed in a new
- subprocess.
+ Multiprocessing helper method. Creates a global embeddings object to be accessed in a new subprocess.
Args:
- config: configuration
+ config: vector configuration
scoring: scoring instance
"""
global VECTORS
# Create a global embedding object using configuration and saved
- VECTORS = WordVectors(path, blocking, scoring)
+ VECTORS = WordVectors(config, scoring)
def transform(document):
"""
- Multiprocessing helper method. Transforms document tokens into an embedding.
+ Multiprocessing helper method. Transforms document into an embeddings vector.
Args:
document: (id, text|tokens, tags)
@@ -58,25 +57,41 @@ class Vectors(object):
"""
@staticmethod
- def create(method, path, blocking, scoring):
+ def create(config, scoring):
"""
Create a Vectors model instance.
Args:
- path: path to word vector model
- blocking: True if method should wait until vectors fully loaded before returning, false otherwise
+ config: vector configuration
+ scoring: scoring instance
"""
+ # Derive vector type
+ transformers = config.get("method") == "transformers"
+
# Create vector model instance
- return TransformersVectors(path) if method == "transformers" else WordVectors(path, blocking, scoring)
+ return TransformersVectors(config, scoring) if transformers else WordVectors(config, scoring)
+
+ def __init__(self, config, scoring):
+ # Store parameters
+ self.config = config
+ self.scoring = scoring
+
+ # Detect if this is an initialized configuration
+ self.initialized = "dimensions" in config
+
+ # Enables optional string tokenization
+ self.tokenize = "tokenize" not in config or config["tokenize"]
+
+ # pylint: disable=E1111
+ self.model = self.load(config["path"])
- def load(self, path, blocking):
+ def load(self, path):
"""
Loads vector model at path.
Args:
path: path to word vector model
- blocking: True if method should wait until vectors fully loaded before returning, false otherwise
Returns:
vector model
@@ -84,11 +99,11 @@ class Vectors(object):
def index(self, documents):
"""
- Converts a list of document tokens to a temporary file with embeddings arrays. Documents are tuples of (id, text|tokens, tags).
- Returns a tuple of document ids, number of dimensions and temporary file with embeddings.
+ Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,
+ number of dimensions and temporary file with embeddings.
Args:
- documents: list of documents
+ documents: list of (id, text|tokens, tags)
Returns:
(ids, dimensions, stream)
@@ -110,27 +125,19 @@ class WordVectors(Vectors):
Builds sentence embeddings/vectors using weighted word embeddings.
"""
- def __init__(self, path, blocking, scoring):
- self.model = self.load(path, blocking)
-
- # Store parameters
- self.path = path
- self.blocking = blocking
- self.scoring = scoring
-
- def load(self, path, blocking):
- # Require that vector path exists, if a path is provided and it's not found, Magnitude will try download from it's servers
+ def load(self, path):
+ # Ensure that vector path exists
if not path or not os.path.isfile(path):
raise IOError(ENOENT, "Vector model file not found", path)
- # Load magnitude model. If this is a training run (no embeddings yet), block until the vectors are fully loaded
- return Magnitude(path, case_insensitive=True, blocking=blocking)
+ # Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded
+ return Magnitude(path, case_insensitive=True, blocking=not self.initialized)
def index(self, documents):
ids, dimensions, stream = [], None, None
# Shared objects with Pool
- args = (self.path, self.scoring, self.blocking)
+ args = (self.config, self.scoring)
# Convert all documents to embedding arrays, stream embeddings to disk to control memory usage
with Pool(os.cpu_count(), initializer=create, initargs=args) as pool:
@@ -219,14 +226,7 @@ class TransformersVectors(Vectors):
Builds sentence embeddings/vectors using the transformers library.
"""
- def __init__(self, path):
- # Sentence transformer model
- self.model = self.load(path, True)
-
- # Store parameters
- self.path = path
-
- def load(self, path, blocking):
+ def load(self, path):
model = Transformer(path)
pooling = Pooling(model.get_word_embedding_dimension())
@@ -257,11 +257,8 @@ class TransformersVectors(Vectors):
return (ids, dimensions, stream)
def transform(self, document):
- # Convert to tokens if necessary
- if isinstance(document[1], str):
- document = (document[0], Tokenizer.tokenize(document[1]), document[2])
-
- return self.model.encode([" ".join(document[1])], show_progress_bar=False)[0]
+ # Convert input document to text and build embeddings
+ return self.model.encode([self.text(document[1])], show_progress_bar=False)[0]
def batch(self, documents, output):
"""
@@ -275,16 +272,12 @@ class TransformersVectors(Vectors):
(ids, dimensions) list of ids and number of dimensions in embeddings
"""
- # Convert to tokens if necessary
- if isinstance(documents[0][1], str):
- documents = [(d[0], Tokenizer.tokenize(d[1]), d[2]) for d in documents]
-
- # Get list of document texts
+ # Extract ids and convert input documents to text
ids = [uid for uid, _, _ in documents]
- documents = [" ".join(tokens) for _, tokens, _ in documents]
+ documents = [self.text(text) for _, text, _ in documents]
dimensions = None
- # Convert to embeddings
+ # Build embeddings
embeddings = self.model.encode(documents, show_progress_bar=False)
for embedding in embeddings:
if not dimensions:
@@ -294,3 +287,22 @@ class TransformersVectors(Vectors):
pickle.dump(embedding, output)
return (ids, dimensions)
+
+ def text(self, text):
+ """
+ Converts input into text that can be processed by transformer models. This method supports
+ optional string tokenization and joins tokenized input into text.
+
+ Args:
+ text: text|tokens
+ """
+
+ # Optional string tokenization
+ if self.tokenize and isinstance(text, str):
+ text = Tokenizer.tokenize(text)
+
+ # Transformer models require string input
+ if isinstance(text, list):
+ text = " ".join(text)
+
+ return text
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index fb433257..95980c43 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -16,21 +16,22 @@ class TestEmbeddings(unittest.TestCase):
Embeddings tests
"""
- def setUp(self):
+ @classmethod
+ def setUpClass(cls):
"""
Initialize test data.
"""
- self.data = ["US tops 5 million confirmed virus cases",
- "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
- "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
- "The National Park Service warns against sacrificing slower friends in a bear attack",
- "Maine man wins $1M from $25 lottery ticket",
- "Make huge profits without work, earn up to $100,000 a day"]
+ cls.data = ["US tops 5 million confirmed virus cases",
+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+ "The National Park Service warns against sacrificing slower friends in a bear attack",
+ "Maine man wins $1M from $25 lottery ticket",
+ "Make huge profits without work, earn up to $100,000 a day"]
# Create embeddings model, backed by sentence-transformers & transformers
- self.embeddings = Embeddings({"method": "transformers",
- "path": "sentence-transformers/bert-base-nli-mean-tokens"})
+ cls.embeddings = Embeddings({"method": "transformers",
+ "path": "sentence-transformers/bert-base-nli-mean-tokens"})
def testIndex(self):
"""
diff --git a/test/python/testscoring.py b/test/python/testscoring.py
index 46cc3c8c..4277cb3f 100644
--- a/test/python/testscoring.py
+++ b/test/python/testscoring.py
@@ -13,20 +13,21 @@ class TestScoring(unittest.TestCase):
Scoring tests
"""
- def setUp(self):
+ @classmethod
+ def setUpClass(cls):
"""
Initialize test data.
"""
- self.data = ["US tops 5 million confirmed virus cases",
- "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
- "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
- "The National Park Service warns against sacrificing slower friends in a bear attack",
- "Maine man wins $1M from $25 lottery ticket",
- "wins wins wins",
- "Make huge profits without work, earn up to $100,000 a day"]
+ cls.data = ["US tops 5 million confirmed virus cases",
+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+ "The National Park Service warns against sacrificing slower friends in a bear attack",
+ "Maine man wins $1M from $25 lottery ticket",
+ "wins wins wins",
+ "Make huge profits without work, earn up to $100,000 a day"]
- self.data = [(uid, x, None) for uid, x in enumerate(self.data)]
+ cls.data = [(uid, x, None) for uid, x in enumerate(cls.data)]
def testBM25(self):
"""
diff --git a/test/python/testtokenizer.py b/test/python/testtokenizer.py
index edb130b6..d5960d6d 100644
--- a/test/python/testtokenizer.py
+++ b/test/python/testtokenizer.py
@@ -16,5 +16,5 @@ class TestTokenizer(unittest.TestCase):
Test tokenize
"""
- self.assertEqual(Tokenizer.tokenize("this is a test"), ["test"])
- self.assertEqual(Tokenizer.tokenize("abc123 abc 123"), ["abc123", "abc"])
+ self.assertEqual(Tokenizer.tokenize("Y this is a test!"), ["test"])
+ self.assertEqual(Tokenizer.tokenize("abc123 ABC 123"), ["abc123", "abc"])
diff --git a/test/python/testtransformers.py b/test/python/testtransformers.py
new file mode 100644
index 00000000..2540615e
--- /dev/null
+++ b/test/python/testtransformers.py
@@ -0,0 +1,70 @@
+"""
+Vectors module tests
+"""
+
+import os
+import unittest
+
+import numpy as np
+
+from txtai.vectors import Vectors
+
+class TestTransformersVectors(unittest.TestCase):
+ """
+ TransformerVectors tests
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Create single TransformersVectors instance
+ """
+
+ cls.model = Vectors.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)
+
+ def testIndex(self):
+ """
+ Test transformers indexing
+ """
+
+ # Generate enough volume to test batching
+ documents = [(x, "This is a test", None) for x in range(1000)]
+
+ ids, dimension, stream = self.model.index(documents)
+
+ self.assertEqual(len(ids), 1000)
+ self.assertEqual(dimension, 768)
+ self.assertIsNotNone(os.path.exists(stream))
+
+ def testTransform(self):
+ """
+ Test transformers transform
+ """
+
+ # Sample documents: one where tokenizer changes text and one with no changes to text
+ documents = [(0, "This is a test and has no tokenization", None),
+ (1, "test tokenization", None)]
+
+ # Run with tokenization enabled
+ self.model.tokenize = True
+ embeddings1 = [self.model.transform(d) for d in documents]
+
+ # Run with tokenization disabled
+ self.model.tokenize = False
+ embeddings2 = [self.model.transform(d) for d in documents]
+
+ self.assertFalse(np.array_equal(embeddings1[0], embeddings2[0]))
+ self.assertTrue(np.array_equal(embeddings1[1], embeddings2[1]))
+
+ def testText(self):
+ """
+ Test transformers text conversion
+ """
+
+ self.model.tokenize = True
+ self.assertEqual(self.model.text("Y 123 This is a test!"), "test")
+ self.assertEqual(self.model.text(["This", "is", "a", "test"]), "This is a test")
+
+ self.model.tokenize = False
+ self.assertEqual(self.model.text("Y 123 This is a test!"), "Y 123 This is a test!")
+ self.assertEqual(self.model.text(["This", "is", "a", "test"]), "This is a test")
diff --git a/test/python/testvectors.py b/test/python/testwordvectors.py
similarity index 53%
rename from test/python/testvectors.py
rename to test/python/testwordvectors.py
index 1f6b508d..46df1b04 100644
--- a/test/python/testvectors.py
+++ b/test/python/testwordvectors.py
@@ -8,7 +8,7 @@ import unittest
from txtai.vectors import WordVectors, Vectors
-class TestVectors(unittest.TestCase):
+class TestWordVectors(unittest.TestCase):
"""
Vectors tests
"""
@@ -28,39 +28,59 @@ class TestVectors(unittest.TestCase):
# Build word vectors
WordVectors.build("README.md", 300, 3, path)
- def testLoad(self):
+ def testIndex(self):
"""
- Test loading word vectors
+ Test word vector indexing
"""
- # Test loading model
- model = Vectors.create("words", self.path, True, None)
+ # Generate data
+ documents = [(x, "This is a test", None) for x in range(1000)]
+
+ model = Vectors.create({"path": self.path}, None)
+
+ ids, dimension, stream = model.index(documents)
+
+ self.assertEqual(len(ids), 1000)
+ self.assertEqual(dimension, 300)
+ self.assertIsNotNone(os.path.exists(stream))
+
+ def testTransform(self):
+ """
+ Test word vector transform
+ """
+
+ model = Vectors.create({"path": self.path}, None)
self.assertEqual(len(model.transform((None, ["txtai"], None))), 300)
+ def testNoExist(self):
+ """
+ Test loading model that doesn't exist
+ """
+
# Test non-existent path raises an exception
with self.assertRaises(IOError):
- Vectors.create("words", os.path.join(tempfile.gettempdir(), "noexist"), True, None)
+ Vectors.create({"path": os.path.join(tempfile.gettempdir(), "noexist")}, None)
def testLookup(self):
"""
Test word vector lookup
"""
- model = Vectors.create("words", self.path, True, None)
+ model = Vectors.create({"path": self.path}, None)
self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))
- def testTransformers(self):
+ def testBlocking(self):
"""
- Test transformer vectors
+ Test blocking load of vector model
"""
- model = Vectors.create("transformers", "sentence-transformers/bert-base-nli-mean-tokens", False, None)
+ config = {"path": self.path}
+ model = Vectors.create(config, None)
- # Generate enough volume to test batching
- documents = [(x, "This is a test", None) for x in range(1000)]
+ self.assertFalse(model.initialized)
- ids, dimension, stream = model.index(documents)
+ config["ids"] = ["0", "1"]
+ config["dimensions"] = 300
+ model = Vectors.create(config, None)
- self.assertEqual(len(ids), 1000)
- self.assertEqual(dimension, 768)
- self.assertIsNotNone(os.path.exists(stream))
+ self.assertTrue(model.initialized)