💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update optional package tests to better simulate missing packages, closes #154
davidmezzetti1b80f7674f4a
diff --git a/src/python/txtai/embeddings/reducer.py b/src/python/txtai/embeddings/reducer.pyindex 2b691eaf..79dae5da 100644--- a/src/python/txtai/embeddings/reducer.py+++ b/src/python/txtai/embeddings/reducer.py@@ -27,6 +27,9 @@ class Reducer:components: number of model components"""+ if not REDUCER:+ raise ImportError('Dimensionality reduction is not available - install "similarity" extra to enable')+self.model = self.build(embeddings, components) if embeddings is not None and components else Nonedef __call__(self, embeddings):diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 1e79219c..891e6074 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -2,82 +2,95 @@Optional module tests"""+import sysimport unittest-import txtai.ann.factory--from txtai.ann import ANNFactory-from txtai.models import OnnxModel-from txtai.pipeline import HFOnnx, MLOnnx, Segmentation, Tabular, Textractor, Transcription, Translation-from txtai.vectors import VectorsFactory-from txtai.workflow import ImageTask, ServiceTask, StorageTask+# pylint: disable=C0415,W0611+import transformers+from transformers import Trainer, set_seed, ViTFeatureExtractorclass TestOptional(unittest.TestCase):"""- Optional tests+ Optional tests. Simulates optional dependencies not being installed."""- @staticmethod- def toggle():- """- Toggles parameters used to determine the presence of optional libraries- """-- txtai.ann.factory.ANNOY = not txtai.ann.factory.ANNOY- txtai.ann.factory.HNSWLIB = not txtai.ann.factory.HNSWLIB-- txtai.models.onnx.ONNX_RUNTIME = not txtai.models.onnx.ONNX_RUNTIME-- txtai.pipeline.audio.transcription.SOUNDFILE = not txtai.pipeline.audio.transcription.SOUNDFILE- txtai.pipeline.segment.segmentation.NLTK = not txtai.pipeline.segment.segmentation.NLTK- txtai.pipeline.segment.tabular.PANDAS = not txtai.pipeline.segment.tabular.PANDAS- txtai.pipeline.segment.textractor.TIKA = not txtai.pipeline.segment.textractor.TIKA- txtai.pipeline.text.translation.FASTTEXT = not txtai.pipeline.text.translation.FASTTEXT- txtai.pipeline.train.hfonnx.ONNX_RUNTIME = not txtai.pipeline.train.hfonnx.ONNX_RUNTIME- txtai.pipeline.train.mlonnx.ONNX_MLTOOLS = not txtai.pipeline.train.mlonnx.ONNX_MLTOOLS-- txtai.vectors.factory.WORDS = not txtai.vectors.factory.WORDS- txtai.vectors.transformers.SENTENCE_TRANSFORMERS = not txtai.vectors.transformers.SENTENCE_TRANSFORMERS-- txtai.workflow.task.image.PIL = not txtai.workflow.task.image.PIL- txtai.workflow.task.service.XML_TO_DICT = not txtai.workflow.task.service.XML_TO_DICT- txtai.workflow.task.storage.LIBCLOUD = not txtai.workflow.task.storage.LIBCLOUD-@classmethoddef setUpClass(cls):"""Simulate optional packages not being installed"""- # Toggle parameters- TestOptional.toggle()+ modules = [+ "annoy",+ "fasttext",+ "hnswlib",+ "nltk",+ "libcloud.storage.providers",+ "onnxmltools",+ "onnxruntime",+ "pandas",+ "PIL",+ "sklearn.decomposition",+ "sentence_transformers",+ "soundfile",+ "tika",+ "xmltodict",+ ]++ # Get handle to all currently loaded txtai modules+ modules = modules + [key for key in sys.modules if key.startswith("txtai")]+ cls.modules = {module: None for module in modules}++ # Replace loaded modules with stubs. Save modules for later reloading+ for module in cls.modules:+ if module in sys.modules:+ cls.modules[module] = sys.modules[module]++ # Remove txtai modules. Set optional dependencies to None to prevent reloading.+ if "txtai" in module:+ if module in sys.modules:+ del sys.modules[module]+ else:+ sys.modules[module] = None@classmethoddef tearDownClass(cls):"""- Reset global parameters+ Resets modules environment back to initial state"""- # Toggle parameters back- TestOptional.toggle()+ # Reset replaced modules in setup+ for key, value in cls.modules.items():+ if value:+ sys.modules[key] = value+ else:+ del sys.modules[key]def testAnn(self):"""Test missing ann dependencies"""+ from txtai.ann import ANNFactory+with self.assertRaises(ImportError):ANNFactory.create({"backend": "annoy"})with self.assertRaises(ImportError):ANNFactory.create({"backend": "hnsw"})- def testModel(self):+ def testModels(self):"""Test missing model dependencies"""+ from txtai.embeddings import Reducer+ from txtai.models import OnnxModel++ with self.assertRaises(ImportError):+ Reducer()+with self.assertRaises(ImportError):OnnxModel(None)@@ -86,6 +99,8 @@ class TestOptional(unittest.TestCase):Test missing pipeline dependencies"""+ from txtai.pipeline import HFOnnx, MLOnnx, Segmentation, Tabular, Textractor, Transcription, Translation+with self.assertRaises(ImportError):HFOnnx()("google/bert_uncased_L-2_H-128_A-2", quantize=True)@@ -112,6 +127,8 @@ class TestOptional(unittest.TestCase):Test missing vectors dependencies"""+ from txtai.vectors import VectorsFactory+with self.assertRaises(ImportError):VectorsFactory.create({"method": "words"}, None)@@ -123,6 +140,8 @@ class TestOptional(unittest.TestCase):Test missing workflow dependencies"""+ from txtai.workflow import ImageTask, ServiceTask, StorageTask+with self.assertRaises(ImportError):ImageTask()