💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Added flag to disable default tokenization for vector models. This addresses language support issues in #39 and #43.
davidmezzetti172cd601bc83
diff --git a/src/python/txtai/embeddings.py b/src/python/txtai/embeddings.pyindex 38ffb16d..d038ff87 100644--- a/src/python/txtai/embeddings.py+++ b/src/python/txtai/embeddings.py@@ -56,14 +56,14 @@ class Embeddings(object):vector model"""- return Vectors.create(self.config.get("method"), self.config["path"], True if not self.embeddings else False, self.scoring)+ return Vectors.create(self.config, self.scoring)def score(self, documents):"""- Builds a scoring index. Documents are tuples of (id, text|tokens, tags).+ Builds a scoring index.Args:- documents: list of documents+ documents: list of (id, text|tokens, tags)"""if self.scoring:@@ -72,10 +72,10 @@ class Embeddings(object):def index(self, documents):"""- Builds an embeddings index. Documents are tuples of (id, text|tokens, tags).+ Builds an embeddings index.Args:- documents: list of documents+ documents: list of (id, text|tokens, tags)"""# Transform documents to embeddings vectorsdiff --git a/src/python/txtai/vectors.py b/src/python/txtai/vectors.pyindex 019620aa..33640b7b 100644--- a/src/python/txtai/vectors.py+++ b/src/python/txtai/vectors.py@@ -22,24 +22,23 @@ from .tokenizer import Tokenizer# pylint: disable=W0603VECTORS = None-def create(path, scoring, blocking):+def create(config, scoring):"""- Multiprocessing helper method. Creates a global embeddings object to be accessed in a new- subprocess.+ Multiprocessing helper method. Creates a global embeddings object to be accessed in a new subprocess.Args:- config: configuration+ config: vector configurationscoring: scoring instance"""global VECTORS# Create a global embedding object using configuration and saved- VECTORS = WordVectors(path, blocking, scoring)+ VECTORS = WordVectors(config, scoring)def transform(document):"""- Multiprocessing helper method. Transforms document tokens into an embedding.+ Multiprocessing helper method. Transforms document into an embeddings vector.Args:document: (id, text|tokens, tags)@@ -58,25 +57,41 @@ class Vectors(object):"""@staticmethod- def create(method, path, blocking, scoring):+ def create(config, scoring):"""Create a Vectors model instance.Args:- path: path to word vector model- blocking: True if method should wait until vectors fully loaded before returning, false otherwise+ config: vector configuration+ scoring: scoring instance"""+ # Derive vector type+ transformers = config.get("method") == "transformers"+# Create vector model instance- return TransformersVectors(path) if method == "transformers" else WordVectors(path, blocking, scoring)+ return TransformersVectors(config, scoring) if transformers else WordVectors(config, scoring)++ def __init__(self, config, scoring):+ # Store parameters+ self.config = config+ self.scoring = scoring++ # Detect if this is an initialized configuration+ self.initialized = "dimensions" in config++ # Enables optional string tokenization+ self.tokenize = "tokenize" not in config or config["tokenize"]++ # pylint: disable=E1111+ self.model = self.load(config["path"])- def load(self, path, blocking):+ def load(self, path):"""Loads vector model at path.Args:path: path to word vector model- blocking: True if method should wait until vectors fully loaded before returning, false otherwiseReturns:vector model@@ -84,11 +99,11 @@ class Vectors(object):def index(self, documents):"""- Converts a list of document tokens to a temporary file with embeddings arrays. Documents are tuples of (id, text|tokens, tags).- Returns a tuple of document ids, number of dimensions and temporary file with embeddings.+ Converts a list of documents to a temporary file with embeddings arrays. Returns a tuple of document ids,+ number of dimensions and temporary file with embeddings.Args:- documents: list of documents+ documents: list of (id, text|tokens, tags)Returns:(ids, dimensions, stream)@@ -110,27 +125,19 @@ class WordVectors(Vectors):Builds sentence embeddings/vectors using weighted word embeddings."""- def __init__(self, path, blocking, scoring):- self.model = self.load(path, blocking)-- # Store parameters- self.path = path- self.blocking = blocking- self.scoring = scoring-- def load(self, path, blocking):- # Require that vector path exists, if a path is provided and it's not found, Magnitude will try download from it's servers+ def load(self, path):+ # Ensure that vector path existsif not path or not os.path.isfile(path):raise IOError(ENOENT, "Vector model file not found", path)- # Load magnitude model. If this is a training run (no embeddings yet), block until the vectors are fully loaded- return Magnitude(path, case_insensitive=True, blocking=blocking)+ # Load magnitude model. If this is a training run (uninitialized config), block until vectors are fully loaded+ return Magnitude(path, case_insensitive=True, blocking=not self.initialized)def index(self, documents):ids, dimensions, stream = [], None, None# Shared objects with Pool- args = (self.path, self.scoring, self.blocking)+ args = (self.config, self.scoring)# Convert all documents to embedding arrays, stream embeddings to disk to control memory usagewith Pool(os.cpu_count(), initializer=create, initargs=args) as pool:@@ -219,14 +226,7 @@ class TransformersVectors(Vectors):Builds sentence embeddings/vectors using the transformers library."""- def __init__(self, path):- # Sentence transformer model- self.model = self.load(path, True)-- # Store parameters- self.path = path-- def load(self, path, blocking):+ def load(self, path):model = Transformer(path)pooling = Pooling(model.get_word_embedding_dimension())@@ -257,11 +257,8 @@ class TransformersVectors(Vectors):return (ids, dimensions, stream)def transform(self, document):- # Convert to tokens if necessary- if isinstance(document[1], str):- document = (document[0], Tokenizer.tokenize(document[1]), document[2])-- return self.model.encode([" ".join(document[1])], show_progress_bar=False)[0]+ # Convert input document to text and build embeddings+ return self.model.encode([self.text(document[1])], show_progress_bar=False)[0]def batch(self, documents, output):"""@@ -275,16 +272,12 @@ class TransformersVectors(Vectors):(ids, dimensions) list of ids and number of dimensions in embeddings"""- # Convert to tokens if necessary- if isinstance(documents[0][1], str):- documents = [(d[0], Tokenizer.tokenize(d[1]), d[2]) for d in documents]-- # Get list of document texts+ # Extract ids and convert input documents to textids = [uid for uid, _, _ in documents]- documents = [" ".join(tokens) for _, tokens, _ in documents]+ documents = [self.text(text) for _, text, _ in documents]dimensions = None- # Convert to embeddings+ # Build embeddingsembeddings = self.model.encode(documents, show_progress_bar=False)for embedding in embeddings:if not dimensions:@@ -294,3 +287,22 @@ class TransformersVectors(Vectors):pickle.dump(embedding, output)return (ids, dimensions)++ def text(self, text):+ """+ Converts input into text that can be processed by transformer models. This method supports+ optional string tokenization and joins tokenized input into text.++ Args:+ text: text|tokens+ """++ # Optional string tokenization+ if self.tokenize and isinstance(text, str):+ text = Tokenizer.tokenize(text)++ # Transformer models require string input+ if isinstance(text, list):+ text = " ".join(text)++ return textdiff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex fb433257..95980c43 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -16,21 +16,22 @@ class TestEmbeddings(unittest.TestCase):Embeddings tests"""- def setUp(self):+ @classmethod+ def setUpClass(cls):"""Initialize test data."""- self.data = ["US tops 5 million confirmed virus cases",- "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",- "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",- "The National Park Service warns against sacrificing slower friends in a bear attack",- "Maine man wins $1M from $25 lottery ticket",- "Make huge profits without work, earn up to $100,000 a day"]+ cls.data = ["US tops 5 million confirmed virus cases",+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",+ "The National Park Service warns against sacrificing slower friends in a bear attack",+ "Maine man wins $1M from $25 lottery ticket",+ "Make huge profits without work, earn up to $100,000 a day"]# Create embeddings model, backed by sentence-transformers & transformers- self.embeddings = Embeddings({"method": "transformers",- "path": "sentence-transformers/bert-base-nli-mean-tokens"})+ cls.embeddings = Embeddings({"method": "transformers",+ "path": "sentence-transformers/bert-base-nli-mean-tokens"})def testIndex(self):"""diff --git a/test/python/testscoring.py b/test/python/testscoring.pyindex 46cc3c8c..4277cb3f 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring.py@@ -13,20 +13,21 @@ class TestScoring(unittest.TestCase):Scoring tests"""- def setUp(self):+ @classmethod+ def setUpClass(cls):"""Initialize test data."""- self.data = ["US tops 5 million confirmed virus cases",- "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",- "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",- "The National Park Service warns against sacrificing slower friends in a bear attack",- "Maine man wins $1M from $25 lottery ticket",- "wins wins wins",- "Make huge profits without work, earn up to $100,000 a day"]+ cls.data = ["US tops 5 million confirmed virus cases",+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",+ "The National Park Service warns against sacrificing slower friends in a bear attack",+ "Maine man wins $1M from $25 lottery ticket",+ "wins wins wins",+ "Make huge profits without work, earn up to $100,000 a day"]- self.data = [(uid, x, None) for uid, x in enumerate(self.data)]+ cls.data = [(uid, x, None) for uid, x in enumerate(cls.data)]def testBM25(self):"""diff --git a/test/python/testtokenizer.py b/test/python/testtokenizer.pyindex edb130b6..d5960d6d 100644--- a/test/python/testtokenizer.py+++ b/test/python/testtokenizer.py@@ -16,5 +16,5 @@ class TestTokenizer(unittest.TestCase):Test tokenize"""- self.assertEqual(Tokenizer.tokenize("this is a test"), ["test"])- self.assertEqual(Tokenizer.tokenize("abc123 abc 123"), ["abc123", "abc"])+ self.assertEqual(Tokenizer.tokenize("Y this is a test!"), ["test"])+ self.assertEqual(Tokenizer.tokenize("abc123 ABC 123"), ["abc123", "abc"])diff --git a/test/python/testtransformers.py b/test/python/testtransformers.pynew file mode 100644index 00000000..2540615e--- /dev/null+++ b/test/python/testtransformers.py@@ -0,0 +1,70 @@+"""+Vectors module tests+"""++import os+import unittest++import numpy as np++from txtai.vectors import Vectors++class TestTransformersVectors(unittest.TestCase):+ """+ TransformerVectors tests+ """++ @classmethod+ def setUpClass(cls):+ """+ Create single TransformersVectors instance+ """++ cls.model = Vectors.create({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"}, None)++ def testIndex(self):+ """+ Test transformers indexing+ """++ # Generate enough volume to test batching+ documents = [(x, "This is a test", None) for x in range(1000)]++ ids, dimension, stream = self.model.index(documents)++ self.assertEqual(len(ids), 1000)+ self.assertEqual(dimension, 768)+ self.assertIsNotNone(os.path.exists(stream))++ def testTransform(self):+ """+ Test transformers transform+ """++ # Sample documents: one where tokenizer changes text and one with no changes to text+ documents = [(0, "This is a test and has no tokenization", None),+ (1, "test tokenization", None)]++ # Run with tokenization enabled+ self.model.tokenize = True+ embeddings1 = [self.model.transform(d) for d in documents]++ # Run with tokenization disabled+ self.model.tokenize = False+ embeddings2 = [self.model.transform(d) for d in documents]++ self.assertFalse(np.array_equal(embeddings1[0], embeddings2[0]))+ self.assertTrue(np.array_equal(embeddings1[1], embeddings2[1]))++ def testText(self):+ """+ Test transformers text conversion+ """++ self.model.tokenize = True+ self.assertEqual(self.model.text("Y 123 This is a test!"), "test")+ self.assertEqual(self.model.text(["This", "is", "a", "test"]), "This is a test")++ self.model.tokenize = False+ self.assertEqual(self.model.text("Y 123 This is a test!"), "Y 123 This is a test!")+ self.assertEqual(self.model.text(["This", "is", "a", "test"]), "This is a test")diff --git a/test/python/testvectors.py b/test/python/testwordvectors.pysimilarity index 53%rename from test/python/testvectors.pyrename to test/python/testwordvectors.pyindex 1f6b508d..46df1b04 100644--- a/test/python/testvectors.py+++ b/test/python/testwordvectors.py@@ -8,7 +8,7 @@ import unittestfrom txtai.vectors import WordVectors, Vectors-class TestVectors(unittest.TestCase):+class TestWordVectors(unittest.TestCase):"""Vectors tests"""@@ -28,39 +28,59 @@ class TestVectors(unittest.TestCase):# Build word vectorsWordVectors.build("README.md", 300, 3, path)- def testLoad(self):+ def testIndex(self):"""- Test loading word vectors+ Test word vector indexing"""- # Test loading model- model = Vectors.create("words", self.path, True, None)+ # Generate data+ documents = [(x, "This is a test", None) for x in range(1000)]++ model = Vectors.create({"path": self.path}, None)++ ids, dimension, stream = model.index(documents)++ self.assertEqual(len(ids), 1000)+ self.assertEqual(dimension, 300)+ self.assertIsNotNone(os.path.exists(stream))++ def testTransform(self):+ """+ Test word vector transform+ """++ model = Vectors.create({"path": self.path}, None)self.assertEqual(len(model.transform((None, ["txtai"], None))), 300)+ def testNoExist(self):+ """+ Test loading model that doesn't exist+ """+# Test non-existent path raises an exceptionwith self.assertRaises(IOError):- Vectors.create("words", os.path.join(tempfile.gettempdir(), "noexist"), True, None)+ Vectors.create({"path": os.path.join(tempfile.gettempdir(), "noexist")}, None)def testLookup(self):"""Test word vector lookup"""- model = Vectors.create("words", self.path, True, None)+ model = Vectors.create({"path": self.path}, None)self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))- def testTransformers(self):+ def testBlocking(self):"""- Test transformer vectors+ Test blocking load of vector model"""- model = Vectors.create("transformers", "sentence-transformers/bert-base-nli-mean-tokens", False, None)+ config = {"path": self.path}+ model = Vectors.create(config, None)- # Generate enough volume to test batching- documents = [(x, "This is a test", None) for x in range(1000)]+ self.assertFalse(model.initialized)- ids, dimension, stream = model.index(documents)+ config["ids"] = ["0", "1"]+ config["dimensions"] = 300+ model = Vectors.create(config, None)- self.assertEqual(len(ids), 1000)- self.assertEqual(dimension, 768)- self.assertIsNotNone(os.path.exists(stream))+ self.assertTrue(model.initialized)