Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Added API tests (closes #42). Increased overall test coverage.

davidmezzettief8226da8fc7
diff --git a/test/python/testann.py b/test/python/testann.py
index 8d763637..12f8eb49 100644
--- a/test/python/testann.py
+++ b/test/python/testann.py
@@ -21,6 +21,8 @@ class TestANN(unittest.TestCase):
"""
self.assertEqual(self.backend("annoy").config["backend"], "annoy")
+ self.assertIsNotNone(self.save("annoy"))
+ self.assertGreater(self.search("annoy"), 0)
@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")
def testFaiss(self):
@@ -31,53 +33,61 @@ class TestANN(unittest.TestCase):
self.assertEqual(self.backend("faiss").config["backend"], "faiss")
self.assertEqual(self.backend("faiss", 5000).config["backend"], "faiss")
+ self.assertIsNotNone(self.save("faiss"))
+ self.assertGreater(self.search("faiss"), 0)
+
def testHnsw(self):
"""
- Test Hnswlib
+ Test Hnswlib backend
"""
self.assertEqual(self.backend("hnsw").config["backend"], "hnsw")
+ self.assertIsNotNone(self.save("hnsw"))
+ self.assertGreater(self.search("hnsw"), 0)
- def testSave(self):
+ def backend(self, name, length=100):
"""
- Tests ANN save/load.
+ Test a backend
"""
+ # Generate test data
+ data = np.random.rand(length, 300).astype(np.float32)
+ self.normalize(data)
+
+ model = ANN.create({"backend": name, "dimensions": data.shape[1]})
+ model.index(data)
+
+ return model
+
+ def save(self, backend):
+ """
+ Test save/load
+ """
+
+ model = self.backend(backend)
+
# Generate temp file path
index = os.path.join(tempfile.gettempdir(), "ann")
- model = self.backend("annoy")
model.save(index)
model.load(index)
- def testSearch(self):
+ return model
+
+ def search(self, backend):
"""
- Tests ANN search
+ Test ANN search
"""
# Generate ANN index
- model = self.backend("annoy")
+ model = self.backend(backend)
# Generate query vector
query = np.random.rand(300).astype(np.float32)
self.normalize(query)
# Ensure top result has similarity > 0
- self.assertGreater(model.search(query, 1)[0][1], 0)
-
- def backend(self, name, length=100):
- """
- Tests a backend
- """
-
- # Generate test data
- data = np.random.rand(length, 300).astype(np.float32)
- self.normalize(data)
-
- model = ANN.create({"backend": name, "dimensions": data.shape[1]})
- model.index(data)
-
- return model
+ return model.search(query, 1)[0][1]
def normalize(self, embeddings):
"""
diff --git a/test/python/testapi.py b/test/python/testapi.py
new file mode 100644
index 00000000..f4f37ada
--- /dev/null
+++ b/test/python/testapi.py
@@ -0,0 +1,186 @@
+"""
+API module tests
+"""
+
+import os
+import tempfile
+import unittest
+
+import numpy as np
+
+from fastapi.testclient import TestClient
+
+from txtai.api import API, app, start
+
+# Full API configuration with similarity search, extractive QA and zero-shot labeling.
+FULL = """
+# Index file path
+path: %s
+
+# Allow indexing of documents
+writable: True
+
+# Embeddings settings
+embeddings:
+ method: transformers
+ path: sentence-transformers/bert-base-nli-mean-tokens
+
+# Extractor settings
+extractor:
+ path: distilbert-base-cased-distilled-squad
+
+# Labels settings
+labels:
+"""
+
+# Configuration that reads an existing similarity search index
+READONLY = """
+# Index file path
+path: %s
+
+# Allow indexing of documents
+writable: False
+"""
+
+class TestAPI(unittest.TestCase):
+ """
+ API tests
+ """
+
+ @staticmethod
+ @unittest.mock.patch.dict(os.environ, {"CONFIG": "/tmp/testapi.yml", "API_CLASS": "txtai.api.API"})
+ def start(full):
+ """
+ Starts a mock FastAPI client.
+
+ Args:
+ full: true if full api configuration should be loaded, otherwise a read-only configuration is used
+ """
+
+ config = os.path.join(tempfile.gettempdir(), "testapi.yml")
+ index = os.path.join(tempfile.gettempdir(), "testapi")
+
+ with open(config, "w") as output:
+ output.write((FULL if full else READONLY) % (index))
+
+ client = TestClient(app)
+ start()
+
+ return client
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Create API client on creation of class
+ """
+
+ cls.client = TestAPI.start(True)
+
+ def testEmbeddings(self):
+ """
+ Test embeddings via API
+ """
+
+ data = ["US tops 5 million confirmed virus cases",
+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+ "The National Park Service warns against sacrificing slower friends in a bear attack",
+ "Maine man wins $1M from $25 lottery ticket",
+ "Make huge profits without work, earn up to $100,000 a day"]
+
+ # Test similarity
+ uid = np.argmax(self.client.post("similarity", json={
+ "search": "feel good story",
+ "data": data
+ }).json())
+
+ self.assertEqual(data[uid], data[4])
+
+ # Test indexing
+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(data)])
+ self.client.get("index")
+
+ # Test search
+ uid = self.client.get("search?q=feel%20good%20story&n=1").json()[0][0]
+ self.assertEqual(data[uid], data[4])
+
+ # Test embeddings
+ self.assertIsNotNone(self.client.get("embeddings?t=testembed").json())
+
+ def testEmpty(self):
+ """
+ Test empty API configuration
+ """
+
+ api = API({})
+
+ self.assertIsNone(api.search("test", None))
+ self.assertIsNone(api.similarity("test", ["test"]))
+ self.assertIsNone(api.transform("test"))
+ self.assertIsNone(api.extract(["test"], ["test"]))
+ self.assertIsNone(api.label("test", ["test"]))
+
+ def testExtractor(self):
+ """
+ Test qa extraction via API
+ """
+
+ sections = ["Giants hit 3 HRs to down Dodgers",
+ "Giants 5 Dodgers 4 final",
+ "Dodgers drop Game 2 against the Giants, 5-4",
+ "Blue Jays 2 Red Sox 1 final",
+ "Red Sox lost to the Blue Jays, 2-1",
+ "Blue Jays at Red Sox is over. Score: 2-1",
+ "Phillies win over the Braves, 5-0",
+ "Phillies 5 Braves 0 final",
+ "Final: Braves lose to the Phillies in the series opener, 5-0",
+ "Final score: Flyers 4 Lightning 1",
+ "Flyers 4 Lightning 1 final",
+ "Flyers win 4-1"]
+
+ # Add unique id to each section to assist with qa extraction
+ sections = [{"id": uid, "text": section} for uid, section in enumerate(sections)]
+
+ questions = ["What team won the game?", "What was score?"]
+
+ execute = lambda query: self.client.post("extract", json={
+ "documents": sections,
+ "queue": [{"name": question, "query": query, "question": question, "snippet": False} for question in questions]
+ }).json()
+
+ answers = execute("Red Sox - Blue Jays")
+ self.assertEqual("Blue Jays", answers[0][1])
+ self.assertEqual("2-1", answers[1][1])
+
+ # Ad-hoc questions
+ question = "What hockey team won?"
+
+ answers = self.client.post("extract", json={
+ "documents": sections,
+ "queue": [{"name": question, "query": question, "question": question, "snippet": False}]
+ }).json()
+ self.assertEqual("Flyers", answers[0][1])
+
+ def testLabels(self):
+ """
+ Test labels via API
+ """
+
+ labels = self.client.post("label", json={
+ "text": "this is the best sentence ever",
+ "labels": ["positive", "negative"]
+ }).json()
+
+ self.assertEqual(labels[0][0], "positive")
+
+ def testReadOnly(self):
+ """
+ Test read-only API instance
+ """
+
+ # Re-create read-only model
+ self.client = TestAPI.start(False)
+
+ # Test search
+ uid = self.client.get("search?q=feel%20good%20story&n=1").json()[0][0]
+ self.assertEqual(uid, 4)
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index e04bf2df..fb433257 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -34,7 +34,7 @@ class TestEmbeddings(unittest.TestCase):
def testIndex(self):
"""
- Test embeddings.index
+ Test index
"""
# Create an index for the list of sections
@@ -47,7 +47,7 @@ class TestEmbeddings(unittest.TestCase):
def testSave(self):
"""
- Test embeddings.save
+ Test save
"""
# Create an index for the list of sections
@@ -66,7 +66,7 @@ class TestEmbeddings(unittest.TestCase):
def testSimilarity(self):
"""
- Test embeddings.similarity
+ Test similarity
"""
# Get best matching id
@@ -111,3 +111,13 @@ class TestEmbeddings(unittest.TestCase):
# Test search
self.assertIsNotNone(embeddings.search("win", 1))
+
+ # Generate temp file path
+ index = os.path.join(tempfile.gettempdir(), "embeddings")
+
+ # Test save/load
+ embeddings.save(index)
+ embeddings.load(index)
+
+ # Test search
+ self.assertIsNotNone(embeddings.search("win", 1))
diff --git a/test/python/testextractor.py b/test/python/testextractor.py
index 3dc1b1c2..4522df27 100644
--- a/test/python/testextractor.py
+++ b/test/python/testextractor.py
@@ -12,17 +12,12 @@ class TestExtractor(unittest.TestCase):
Extractor tests
"""
- def testQA(self):
+ @classmethod
+ def setUpClass(cls):
"""
- Test qa extraction
+ Create single extractor instance.
"""
- # Create embeddings model, backed by sentence-transformers & transformers
- embeddings = Embeddings({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"})
-
- # Create extractor instance
- extractor = Extractor(embeddings, "distilbert-base-cased-distilled-squad")
-
sections = ["Giants hit 3 HRs to down Dodgers",
"Giants 5 Dodgers 4 final",
"Dodgers drop Game 2 against the Giants, 5-4",
@@ -37,11 +32,22 @@ class TestExtractor(unittest.TestCase):
"Flyers win 4-1"]
# Add unique id to each section to assist with qa extraction
- sections = [(uid, section) for uid, section in enumerate(sections)]
+ cls.sections = [(uid, section) for uid, section in enumerate(sections)]
+
+ # Create embeddings model, backed by sentence-transformers & transformers
+ cls.embeddings = Embeddings({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"})
+
+ # Create extractor instance
+ cls.extractor = Extractor(cls.embeddings, "distilbert-base-cased-distilled-squad")
+
+ def testAnswer(self):
+ """
+ Test qa extraction with an answer
+ """
questions = ["What team won the game?", "What was score?"]
- execute = lambda query: extractor(sections, [(question, query, question, False) for question in questions])
+ execute = lambda query: self.extractor(self.sections, [(question, query, question, False) for question in questions])
answers = execute("Red Sox - Blue Jays")
self.assertEqual("Blue Jays", answers[0][1])
@@ -50,5 +56,37 @@ class TestExtractor(unittest.TestCase):
# Ad-hoc questions
question = "What hockey team won?"
- answers = extractor(sections, [(question, question, question, False)])
+ answers = self.extractor(self.sections, [(question, question, question, False)])
self.assertEqual("Flyers", answers[0][1])
+
+ def testNoAnswer(self):
+ """
+ Test qa extraction with no answer
+ """
+
+ question = ""
+
+ answers = self.extractor(self.sections, [(question, question, question, False)])
+ self.assertIsNone(answers[0][1])
+
+ def testQuantize(self):
+ """
+ Test qa extraction backed by a quantized model
+ """
+
+ extractor = Extractor(self.embeddings, "distilbert-base-cased-distilled-squad", True)
+
+ question = "How many home runs?"
+
+ answers = extractor(self.sections, [(question, question, question, True)])
+ self.assertTrue(answers[0][1].startswith("Giants hit 3 HRs"))
+
+ def testSnippet(self):
+ """
+ Test qa extraction with a full answer snippet
+ """
+
+ question = "How many home runs?"
+
+ answers = self.extractor(self.sections, [(question, question, question, True)])
+ self.assertTrue(answers[0][1].startswith("Giants hit 3 HRs"))
diff --git a/test/python/testlabels.py b/test/python/testlabels.py
index 6c836fd9..5df843fc 100644
--- a/test/python/testlabels.py
+++ b/test/python/testlabels.py
@@ -13,8 +13,8 @@ class TestLabels(unittest.TestCase):
def testLabel(self):
"""
- Tests labeling
+ Test labels
"""
labels = Labels()
- self.assertIsNotNone(labels("This is a test", "test"))
+ self.assertEqual(labels("This is the best sentence ever", ["positive", "negative"])[0][0], "positive")
diff --git a/test/python/testscoring.py b/test/python/testscoring.py
index 6ccc780d..46cc3c8c 100644
--- a/test/python/testscoring.py
+++ b/test/python/testscoring.py
@@ -10,7 +10,7 @@ from txtai.scoring import Scoring
class TestScoring(unittest.TestCase):
"""
- Scoring tests.
+ Scoring tests
"""
def setUp(self):
@@ -30,17 +30,19 @@ class TestScoring(unittest.TestCase):
def testBM25(self):
"""
- Test BM25
+ Test bm25
"""
self.method("bm25")
+ self.weights("bm25")
def testSIF(self):
"""
- Test SIF
+ Test sif
"""
self.method("sif")
+ self.weights("sif")
def testTFIDF(self):
"""
@@ -48,31 +50,70 @@ class TestScoring(unittest.TestCase):
"""
self.method("tfidf")
+ self.weights("tfidf")
- def testSave(self):
+ def testUnknown(self):
"""
- Test scoring index save/load
+ Test unknown method
"""
- # Generate temp file path
- index = os.path.join(tempfile.gettempdir(), "bm25")
- os.makedirs(index, exist_ok=True)
-
- model = self.method("bm25")
- model.save(index)
- model.load(index)
+ self.assertIsNone(Scoring.create("unknown"))
- def method(self, method):
+ def method(self, method, data=None):
"""
Runs scoring method
"""
+ # Derive input data
+ data = data if data else self.data
+
model = Scoring.create(method)
- model.index(self.data)
+ model.index(data)
keys = [k for k, v in sorted(model.idf.items(), key=lambda x: x[1])]
# Win should be lowest score for all models
self.assertEqual(keys[0], "wins")
+ # Test save/load
+ self.assertIsNotNone(self.save(model))
+
+ return model
+
+ def save(self, model):
+ """
+ Test scoring index save/load
+ """
+
+ # Generate temp file path
+ index = os.path.join(tempfile.gettempdir(), "scoring")
+ os.makedirs(index, exist_ok=True)
+
+ model.save(index)
+ model.load(index)
+
return model
+
+ def weights(self, method):
+ """
+ Test standard and tag weighted scores
+ """
+
+ document = (1, ["bear", "wins"], None)
+
+ model = self.method(method)
+ weights = model.weights(document)
+
+ # Default weights
+ self.assertNotEqual(weights[0], weights[1])
+
+ data = self.data[:]
+
+ uid, text, _ = data[3]
+ data[3] = (uid, text, "wins")
+
+ model = self.method(method, data)
+ weights = model.weights(document)
+
+ # Modified weights
+ self.assertEqual(weights[0], weights[1])
diff --git a/test/python/testtokenizer.py b/test/python/testtokenizer.py
index 97cc3b9e..edb130b6 100644
--- a/test/python/testtokenizer.py
+++ b/test/python/testtokenizer.py
@@ -13,7 +13,7 @@ class TestTokenizer(unittest.TestCase):
def testTokenize(self):
"""
- Test Tokenizer.tokenize
+ Test tokenize
"""
self.assertEqual(Tokenizer.tokenize("this is a test"), ["test"])
diff --git a/test/python/testvectors.py b/test/python/testvectors.py
index 09d61a72..36280279 100644
--- a/test/python/testvectors.py
+++ b/test/python/testvectors.py
@@ -13,29 +13,34 @@ class TestVectors(unittest.TestCase):
Vectors tests
"""
- def setUp(self):
+ @classmethod
+ def setUpClass(cls):
"""
- Test a WordVectors build.
+ Test a WordVectors build
"""
# Word vectors path
path = os.path.join(tempfile.gettempdir(), "vectors")
# Save model path
- self.path = path + ".magnitude"
+ cls.path = path + ".magnitude"
- # Build word vectors, if they don't already exist
- if not os.path.exists(self.path):
- WordVectors.build("README.md", 300, 3, path)
+ # Build word vectors
+ WordVectors.build("README.md", 300, 3, path)
def testLoad(self):
"""
Test loading word vectors
"""
+ # Test loading model
model = Vectors.create("words", self.path, True, None)
self.assertEqual(len(model.transform((None, ["txtai"], None))), 300)
+ # Test non-existent path raises an exception
+ with self.assertRaises(IOError):
+ Vectors.create("words", "/tmp/noexist", True, None)
+
def testLookup(self):
"""
Test word vector lookup
@@ -43,3 +48,19 @@ class TestVectors(unittest.TestCase):
model = Vectors.create("words", self.path, True, None)
self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))
+
+ def testTransformers(self):
+ """
+ Test transformer vectors
+ """
+
+ model = Vectors.create("transformers", "sentence-transformers/bert-base-nli-mean-tokens", False, None)
+
+ # Generate enough volume to test batching
+ documents = [(x, "This is a test", None) for x in range(1000)]
+
+ ids, dimension, stream = model.index(documents)
+
+ self.assertEqual(len(ids), 1000)
+ self.assertEqual(dimension, 768)
+ self.assertIsNotNone(os.path.exists(stream))