💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
"""
Huggingface module tests
"""
import os
import unittest
import numpy as np
from txtai.embeddings import Embeddings
from txtai.vectors import VectorsFactory
class TestHFVectors(unittest.TestCase):
"""
HFVectors tests
"""
@classmethod
def setUpClass(cls):
"""
Create HFVectors instance.
"""
cls.model = VectorsFactory.create({"path": "sentence-transformers/nli-mpnet-base-v2"}, None)
def testIndex(self):
"""
Test transformers indexing
"""
# Generate enough volume to test batching
documents = [(x, "This is a test", None) for x in range(1000)]
ids, dimension, batches, stream = self.model.index(documents)
self.assertEqual(len(ids), 1000)
self.assertEqual(dimension, 768)
self.assertEqual(batches, 2)
self.assertIsNotNone(os.path.exists(stream))
# Test shape of serialized embeddings
with open(stream, "rb") as queue:
self.assertEqual(np.load(queue).shape, (500, 768))
def testLateInteraction(self):
"""
Test late interaction models produce 2 dimensional vectors
"""
embeddings = Embeddings({"path": "neuml/colbert-bert-tiny", "backend": "numpy", "content": False, "gpu": False, "vectors": {"muvera": False}})
with self.assertRaisesRegex(ValueError, "Expected 2 dimensional vectors"):
embeddings.index(["Short text.", "A considerably longer text exercises padding behavior.", "Third text."])
def testText(self):
"""
Test transformers text conversion
"""
self.model.tokenize = True
self.assertEqual(self.model.prepare("Y 123 This is a test!"), "test")
self.assertEqual(self.model.prepare(["This", "is", "a", "test"]), "This is a test")
self.model.tokenize = False
self.assertEqual(self.model.prepare("Y 123 This is a test!"), "Y 123 This is a test!")
self.assertEqual(self.model.prepare(["This", "is", "a", "test"]), "This is a test")
def testTransform(self):
"""
Test transformers transform
"""
# Sample documents: one where tokenizer changes text and one with no changes to text
documents = [(0, "This is a test and has no tokenization", None), (1, "test tokenization", None)]
# Run with tokenization enabled
self.model.tokenize = True
embeddings1 = [self.model.transform(d) for d in documents]
# Run with tokenization disabled
self.model.tokenize = False
embeddings2 = [self.model.transform(d) for d in documents]
self.assertFalse(np.array_equal(embeddings1[0], embeddings2[0]))
self.assertTrue(np.array_equal(embeddings1[1], embeddings2[1]))
def testTransformArray(self):
"""
Test transformers skips transforming NumPy arrays
"""
# Generate data and run through vector model
data1 = np.random.rand(5, 5).astype(np.float32)
data2 = self.model.transform((0, data1, None))
# Test transform method returns original data
self.assertTrue(np.array_equal(data1, data2))
def testTransformLong(self):
"""
Test transformers transform on long text
"""
# Sample documents: short text and longer text
documents = [(0, "This is long text " * 512, None), (1, "This is short text", None)]
# Run transform and ensure it completes without errors
embeddings = [self.model.transform(d) for d in documents]
self.assertIsNotNone(embeddings)