💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Added API tests (closes #42). Increased overall test coverage.
davidmezzettief8226da8fc7
diff --git a/test/python/testann.py b/test/python/testann.pyindex 8d763637..12f8eb49 100644--- a/test/python/testann.py+++ b/test/python/testann.py@@ -21,6 +21,8 @@ class TestANN(unittest.TestCase):"""self.assertEqual(self.backend("annoy").config["backend"], "annoy")+ self.assertIsNotNone(self.save("annoy"))+ self.assertGreater(self.search("annoy"), 0)@unittest.skipIf(os.name == "nt", "Faiss not installed on Windows")def testFaiss(self):@@ -31,53 +33,61 @@ class TestANN(unittest.TestCase):self.assertEqual(self.backend("faiss").config["backend"], "faiss")self.assertEqual(self.backend("faiss", 5000).config["backend"], "faiss")+ self.assertIsNotNone(self.save("faiss"))+ self.assertGreater(self.search("faiss"), 0)+def testHnsw(self):"""- Test Hnswlib+ Test Hnswlib backend"""self.assertEqual(self.backend("hnsw").config["backend"], "hnsw")+ self.assertIsNotNone(self.save("hnsw"))+ self.assertGreater(self.search("hnsw"), 0)- def testSave(self):+ def backend(self, name, length=100):"""- Tests ANN save/load.+ Test a backend"""+ # Generate test data+ data = np.random.rand(length, 300).astype(np.float32)+ self.normalize(data)++ model = ANN.create({"backend": name, "dimensions": data.shape[1]})+ model.index(data)++ return model++ def save(self, backend):+ """+ Test save/load+ """++ model = self.backend(backend)+# Generate temp file pathindex = os.path.join(tempfile.gettempdir(), "ann")- model = self.backend("annoy")model.save(index)model.load(index)- def testSearch(self):+ return model++ def search(self, backend):"""- Tests ANN search+ Test ANN search"""# Generate ANN index- model = self.backend("annoy")+ model = self.backend(backend)# Generate query vectorquery = np.random.rand(300).astype(np.float32)self.normalize(query)# Ensure top result has similarity > 0- self.assertGreater(model.search(query, 1)[0][1], 0)-- def backend(self, name, length=100):- """- Tests a backend- """-- # Generate test data- data = np.random.rand(length, 300).astype(np.float32)- self.normalize(data)-- model = ANN.create({"backend": name, "dimensions": data.shape[1]})- model.index(data)-- return model+ return model.search(query, 1)[0][1]def normalize(self, embeddings):"""diff --git a/test/python/testapi.py b/test/python/testapi.pynew file mode 100644index 00000000..f4f37ada--- /dev/null+++ b/test/python/testapi.py@@ -0,0 +1,186 @@+"""+API module tests+"""++import os+import tempfile+import unittest++import numpy as np++from fastapi.testclient import TestClient++from txtai.api import API, app, start++# Full API configuration with similarity search, extractive QA and zero-shot labeling.+FULL = """+# Index file path+path: %s++# Allow indexing of documents+writable: True++# Embeddings settings+embeddings:+ method: transformers+ path: sentence-transformers/bert-base-nli-mean-tokens++# Extractor settings+extractor:+ path: distilbert-base-cased-distilled-squad++# Labels settings+labels:+"""++# Configuration that reads an existing similarity search index+READONLY = """+# Index file path+path: %s++# Allow indexing of documents+writable: False+"""++class TestAPI(unittest.TestCase):+ """+ API tests+ """++ @staticmethod+ @unittest.mock.patch.dict(os.environ, {"CONFIG": "/tmp/testapi.yml", "API_CLASS": "txtai.api.API"})+ def start(full):+ """+ Starts a mock FastAPI client.++ Args:+ full: true if full api configuration should be loaded, otherwise a read-only configuration is used+ """++ config = os.path.join(tempfile.gettempdir(), "testapi.yml")+ index = os.path.join(tempfile.gettempdir(), "testapi")++ with open(config, "w") as output:+ output.write((FULL if full else READONLY) % (index))++ client = TestClient(app)+ start()++ return client++ @classmethod+ def setUpClass(cls):+ """+ Create API client on creation of class+ """++ cls.client = TestAPI.start(True)++ def testEmbeddings(self):+ """+ Test embeddings via API+ """++ data = ["US tops 5 million confirmed virus cases",+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",+ "The National Park Service warns against sacrificing slower friends in a bear attack",+ "Maine man wins $1M from $25 lottery ticket",+ "Make huge profits without work, earn up to $100,000 a day"]++ # Test similarity+ uid = np.argmax(self.client.post("similarity", json={+ "search": "feel good story",+ "data": data+ }).json())++ self.assertEqual(data[uid], data[4])++ # Test indexing+ self.client.post("add", json=[{"id": x, "text": row} for x, row in enumerate(data)])+ self.client.get("index")++ # Test search+ uid = self.client.get("search?q=feel%20good%20story&n=1").json()[0][0]+ self.assertEqual(data[uid], data[4])++ # Test embeddings+ self.assertIsNotNone(self.client.get("embeddings?t=testembed").json())++ def testEmpty(self):+ """+ Test empty API configuration+ """++ api = API({})++ self.assertIsNone(api.search("test", None))+ self.assertIsNone(api.similarity("test", ["test"]))+ self.assertIsNone(api.transform("test"))+ self.assertIsNone(api.extract(["test"], ["test"]))+ self.assertIsNone(api.label("test", ["test"]))++ def testExtractor(self):+ """+ Test qa extraction via API+ """++ sections = ["Giants hit 3 HRs to down Dodgers",+ "Giants 5 Dodgers 4 final",+ "Dodgers drop Game 2 against the Giants, 5-4",+ "Blue Jays 2 Red Sox 1 final",+ "Red Sox lost to the Blue Jays, 2-1",+ "Blue Jays at Red Sox is over. Score: 2-1",+ "Phillies win over the Braves, 5-0",+ "Phillies 5 Braves 0 final",+ "Final: Braves lose to the Phillies in the series opener, 5-0",+ "Final score: Flyers 4 Lightning 1",+ "Flyers 4 Lightning 1 final",+ "Flyers win 4-1"]++ # Add unique id to each section to assist with qa extraction+ sections = [{"id": uid, "text": section} for uid, section in enumerate(sections)]++ questions = ["What team won the game?", "What was score?"]++ execute = lambda query: self.client.post("extract", json={+ "documents": sections,+ "queue": [{"name": question, "query": query, "question": question, "snippet": False} for question in questions]+ }).json()++ answers = execute("Red Sox - Blue Jays")+ self.assertEqual("Blue Jays", answers[0][1])+ self.assertEqual("2-1", answers[1][1])++ # Ad-hoc questions+ question = "What hockey team won?"++ answers = self.client.post("extract", json={+ "documents": sections,+ "queue": [{"name": question, "query": question, "question": question, "snippet": False}]+ }).json()+ self.assertEqual("Flyers", answers[0][1])++ def testLabels(self):+ """+ Test labels via API+ """++ labels = self.client.post("label", json={+ "text": "this is the best sentence ever",+ "labels": ["positive", "negative"]+ }).json()++ self.assertEqual(labels[0][0], "positive")++ def testReadOnly(self):+ """+ Test read-only API instance+ """++ # Re-create read-only model+ self.client = TestAPI.start(False)++ # Test search+ uid = self.client.get("search?q=feel%20good%20story&n=1").json()[0][0]+ self.assertEqual(uid, 4)diff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex e04bf2df..fb433257 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -34,7 +34,7 @@ class TestEmbeddings(unittest.TestCase):def testIndex(self):"""- Test embeddings.index+ Test index"""# Create an index for the list of sections@@ -47,7 +47,7 @@ class TestEmbeddings(unittest.TestCase):def testSave(self):"""- Test embeddings.save+ Test save"""# Create an index for the list of sections@@ -66,7 +66,7 @@ class TestEmbeddings(unittest.TestCase):def testSimilarity(self):"""- Test embeddings.similarity+ Test similarity"""# Get best matching id@@ -111,3 +111,13 @@ class TestEmbeddings(unittest.TestCase):# Test searchself.assertIsNotNone(embeddings.search("win", 1))++ # Generate temp file path+ index = os.path.join(tempfile.gettempdir(), "embeddings")++ # Test save/load+ embeddings.save(index)+ embeddings.load(index)++ # Test search+ self.assertIsNotNone(embeddings.search("win", 1))diff --git a/test/python/testextractor.py b/test/python/testextractor.pyindex 3dc1b1c2..4522df27 100644--- a/test/python/testextractor.py+++ b/test/python/testextractor.py@@ -12,17 +12,12 @@ class TestExtractor(unittest.TestCase):Extractor tests"""- def testQA(self):+ @classmethod+ def setUpClass(cls):"""- Test qa extraction+ Create single extractor instance."""- # Create embeddings model, backed by sentence-transformers & transformers- embeddings = Embeddings({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"})-- # Create extractor instance- extractor = Extractor(embeddings, "distilbert-base-cased-distilled-squad")-sections = ["Giants hit 3 HRs to down Dodgers","Giants 5 Dodgers 4 final","Dodgers drop Game 2 against the Giants, 5-4",@@ -37,11 +32,22 @@ class TestExtractor(unittest.TestCase):"Flyers win 4-1"]# Add unique id to each section to assist with qa extraction- sections = [(uid, section) for uid, section in enumerate(sections)]+ cls.sections = [(uid, section) for uid, section in enumerate(sections)]++ # Create embeddings model, backed by sentence-transformers & transformers+ cls.embeddings = Embeddings({"method": "transformers", "path": "sentence-transformers/bert-base-nli-mean-tokens"})++ # Create extractor instance+ cls.extractor = Extractor(cls.embeddings, "distilbert-base-cased-distilled-squad")++ def testAnswer(self):+ """+ Test qa extraction with an answer+ """questions = ["What team won the game?", "What was score?"]- execute = lambda query: extractor(sections, [(question, query, question, False) for question in questions])+ execute = lambda query: self.extractor(self.sections, [(question, query, question, False) for question in questions])answers = execute("Red Sox - Blue Jays")self.assertEqual("Blue Jays", answers[0][1])@@ -50,5 +56,37 @@ class TestExtractor(unittest.TestCase):# Ad-hoc questionsquestion = "What hockey team won?"- answers = extractor(sections, [(question, question, question, False)])+ answers = self.extractor(self.sections, [(question, question, question, False)])self.assertEqual("Flyers", answers[0][1])++ def testNoAnswer(self):+ """+ Test qa extraction with no answer+ """++ question = ""++ answers = self.extractor(self.sections, [(question, question, question, False)])+ self.assertIsNone(answers[0][1])++ def testQuantize(self):+ """+ Test qa extraction backed by a quantized model+ """++ extractor = Extractor(self.embeddings, "distilbert-base-cased-distilled-squad", True)++ question = "How many home runs?"++ answers = extractor(self.sections, [(question, question, question, True)])+ self.assertTrue(answers[0][1].startswith("Giants hit 3 HRs"))++ def testSnippet(self):+ """+ Test qa extraction with a full answer snippet+ """++ question = "How many home runs?"++ answers = self.extractor(self.sections, [(question, question, question, True)])+ self.assertTrue(answers[0][1].startswith("Giants hit 3 HRs"))diff --git a/test/python/testlabels.py b/test/python/testlabels.pyindex 6c836fd9..5df843fc 100644--- a/test/python/testlabels.py+++ b/test/python/testlabels.py@@ -13,8 +13,8 @@ class TestLabels(unittest.TestCase):def testLabel(self):"""- Tests labeling+ Test labels"""labels = Labels()- self.assertIsNotNone(labels("This is a test", "test"))+ self.assertEqual(labels("This is the best sentence ever", ["positive", "negative"])[0][0], "positive")diff --git a/test/python/testscoring.py b/test/python/testscoring.pyindex 6ccc780d..46cc3c8c 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring.py@@ -10,7 +10,7 @@ from txtai.scoring import Scoringclass TestScoring(unittest.TestCase):"""- Scoring tests.+ Scoring tests"""def setUp(self):@@ -30,17 +30,19 @@ class TestScoring(unittest.TestCase):def testBM25(self):"""- Test BM25+ Test bm25"""self.method("bm25")+ self.weights("bm25")def testSIF(self):"""- Test SIF+ Test sif"""self.method("sif")+ self.weights("sif")def testTFIDF(self):"""@@ -48,31 +50,70 @@ class TestScoring(unittest.TestCase):"""self.method("tfidf")+ self.weights("tfidf")- def testSave(self):+ def testUnknown(self):"""- Test scoring index save/load+ Test unknown method"""- # Generate temp file path- index = os.path.join(tempfile.gettempdir(), "bm25")- os.makedirs(index, exist_ok=True)-- model = self.method("bm25")- model.save(index)- model.load(index)+ self.assertIsNone(Scoring.create("unknown"))- def method(self, method):+ def method(self, method, data=None):"""Runs scoring method"""+ # Derive input data+ data = data if data else self.data+model = Scoring.create(method)- model.index(self.data)+ model.index(data)keys = [k for k, v in sorted(model.idf.items(), key=lambda x: x[1])]# Win should be lowest score for all modelsself.assertEqual(keys[0], "wins")+ # Test save/load+ self.assertIsNotNone(self.save(model))++ return model++ def save(self, model):+ """+ Test scoring index save/load+ """++ # Generate temp file path+ index = os.path.join(tempfile.gettempdir(), "scoring")+ os.makedirs(index, exist_ok=True)++ model.save(index)+ model.load(index)+return model++ def weights(self, method):+ """+ Test standard and tag weighted scores+ """++ document = (1, ["bear", "wins"], None)++ model = self.method(method)+ weights = model.weights(document)++ # Default weights+ self.assertNotEqual(weights[0], weights[1])++ data = self.data[:]++ uid, text, _ = data[3]+ data[3] = (uid, text, "wins")++ model = self.method(method, data)+ weights = model.weights(document)++ # Modified weights+ self.assertEqual(weights[0], weights[1])diff --git a/test/python/testtokenizer.py b/test/python/testtokenizer.pyindex 97cc3b9e..edb130b6 100644--- a/test/python/testtokenizer.py+++ b/test/python/testtokenizer.py@@ -13,7 +13,7 @@ class TestTokenizer(unittest.TestCase):def testTokenize(self):"""- Test Tokenizer.tokenize+ Test tokenize"""self.assertEqual(Tokenizer.tokenize("this is a test"), ["test"])diff --git a/test/python/testvectors.py b/test/python/testvectors.pyindex 09d61a72..36280279 100644--- a/test/python/testvectors.py+++ b/test/python/testvectors.py@@ -13,29 +13,34 @@ class TestVectors(unittest.TestCase):Vectors tests"""- def setUp(self):+ @classmethod+ def setUpClass(cls):"""- Test a WordVectors build.+ Test a WordVectors build"""# Word vectors pathpath = os.path.join(tempfile.gettempdir(), "vectors")# Save model path- self.path = path + ".magnitude"+ cls.path = path + ".magnitude"- # Build word vectors, if they don't already exist- if not os.path.exists(self.path):- WordVectors.build("README.md", 300, 3, path)+ # Build word vectors+ WordVectors.build("README.md", 300, 3, path)def testLoad(self):"""Test loading word vectors"""+ # Test loading modelmodel = Vectors.create("words", self.path, True, None)self.assertEqual(len(model.transform((None, ["txtai"], None))), 300)+ # Test non-existent path raises an exception+ with self.assertRaises(IOError):+ Vectors.create("words", "/tmp/noexist", True, None)+def testLookup(self):"""Test word vector lookup@@ -43,3 +48,19 @@ class TestVectors(unittest.TestCase):model = Vectors.create("words", self.path, True, None)self.assertEqual(model.lookup(["txtai", "embeddings", "sentence"]).shape, (3, 300))++ def testTransformers(self):+ """+ Test transformer vectors+ """++ model = Vectors.create("transformers", "sentence-transformers/bert-base-nli-mean-tokens", False, None)++ # Generate enough volume to test batching+ documents = [(x, "This is a test", None) for x in range(1000)]++ ids, dimension, stream = model.index(documents)++ self.assertEqual(len(ids), 1000)+ self.assertEqual(dimension, 768)+ self.assertIsNotNone(os.path.exists(stream))