💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Derive and set ORT provider in OnnxModel and HFOnnx pipeline #195
davidmezzettibe1b020ed4be
diff --git a/src/python/txtai/models/onnx.py b/src/python/txtai/models/onnx.pyindex fdd6c58f..5be90433 100644--- a/src/python/txtai/models/onnx.py+++ b/src/python/txtai/models/onnx.py@@ -4,7 +4,7 @@ ONNX module# Conditional importtry:- from onnxruntime import InferenceSession, SessionOptions+ import onnxruntime as ortONNX_RUNTIME = Trueexcept ImportError:@@ -42,11 +42,27 @@ class OnnxModel(PreTrainedModel):super().__init__(AutoConfig.from_pretrained(config) if config else OnnxConfig())# Create ONNX session- self.model = InferenceSession(model, SessionOptions())+ self.model = ort.InferenceSession(model, ort.SessionOptions(), self.providers())# Add references for this class to supported AutoModel classesRegistry.register(self)+ def providers(self):+ """+ Returns a list of available and usable providers.++ Returns:+ list of available and usable providers+ """++ # Create list of providers, prefer CUDA provider if available+ # CUDA provider only available if GPU is available and onnxruntime-gpu installed+ if torch.cuda.is_available() and "CUDAExecutionProvider" in ort.get_available_providers():+ return ["CUDAExecutionProvider", "CPUExecutionProvider"]++ # Default when CUDA provider isn't available+ return ["CPUExecutionProvider"]+def forward(self, **inputs):"""Runs inputs through an ONNX model and returns outputs. This method handles casting inputsdiff --git a/src/python/txtai/pipeline/train/hfonnx.py b/src/python/txtai/pipeline/train/hfonnx.pyindex 048b5ae8..ee49e476 100644--- a/src/python/txtai/pipeline/train/hfonnx.py+++ b/src/python/txtai/pipeline/train/hfonnx.py@@ -106,11 +106,11 @@ class HFOnnx(Tensors):output = temp- # Optimize model first+ # Optimize model - only need CPU providersess_option = SessionOptions()sess_option.optimized_model_filepath = outputsess_option.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_BASIC- _ = InferenceSession(output, sess_option)+ _ = InferenceSession(output, sess_option, ["CPUExecutionProvider"])# Quantize optimized modelquantize_dynamic(output, output, optimize_model=False)diff --git a/test/python/testpipeline/testonnx.py b/test/python/testpipeline/testonnx.pyindex c72f5410..af1e8590 100644--- a/test/python/testpipeline/testonnx.py+++ b/test/python/testpipeline/testonnx.py@@ -6,6 +6,8 @@ import osimport tempfileimport unittest+from unittest.mock import patch+from sklearn.feature_extraction.text import TfidfVectorizerfrom sklearn.linear_model import LogisticRegressionfrom sklearn.pipeline import Pipeline@@ -60,7 +62,9 @@ class TestOnnx(unittest.TestCase):labels = Labels((model, path), dynamic=False)self.assertEqual(labels("cat")[0][0], 1)- def testPooling(self):+ @patch("onnxruntime.get_available_providers")+ @patch("torch.cuda.is_available")+ def testPooling(self, cuda, providers):"""Test exporting a pooling model to ONNX and running inference"""@@ -71,6 +75,31 @@ class TestOnnx(unittest.TestCase):onnx = HFOnnx()model = onnx(path, "pooling", quantize=True)+ # Test no CUDA and onnxruntime installed+ cuda.return_value = False+ providers.return_value = ["CPUExecutionProvider"]++ embeddings = Embeddings({"path": model, "tokenizer": path})+ self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)++ # Test no CUDA and onnxruntime-gpu installed+ cuda.return_value = False+ providers.return_value = ["CUDAExecutionProvider", "CPUExecutionProvider"]++ embeddings = Embeddings({"path": model, "tokenizer": path})+ self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)++ # Test CUDA and only onnxruntime installed+ cuda.return_value = True+ providers.return_value = ["CPUExecutionProvider"]++ embeddings = Embeddings({"path": model, "tokenizer": path})+ self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)++ # Test CUDA and onnxruntime-gpu installed+ cuda.return_value = True+ providers.return_value = ["CUDAExecutionProvider", "CPUExecutionProvider"]+embeddings = Embeddings({"path": model, "tokenizer": path})self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)