Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Derive and set ORT provider in OnnxModel and HFOnnx pipeline #195

davidmezzettibe1b020ed4be
diff --git a/src/python/txtai/models/onnx.py b/src/python/txtai/models/onnx.py
index fdd6c58f..5be90433 100644
--- a/src/python/txtai/models/onnx.py
+++ b/src/python/txtai/models/onnx.py
@@ -4,7 +4,7 @@ ONNX module
# Conditional import
try:
- from onnxruntime import InferenceSession, SessionOptions
+ import onnxruntime as ort
ONNX_RUNTIME = True
except ImportError:
@@ -42,11 +42,27 @@ class OnnxModel(PreTrainedModel):
super().__init__(AutoConfig.from_pretrained(config) if config else OnnxConfig())
# Create ONNX session
- self.model = InferenceSession(model, SessionOptions())
+ self.model = ort.InferenceSession(model, ort.SessionOptions(), self.providers())
# Add references for this class to supported AutoModel classes
Registry.register(self)
+ def providers(self):
+ """
+ Returns a list of available and usable providers.
+
+ Returns:
+ list of available and usable providers
+ """
+
+ # Create list of providers, prefer CUDA provider if available
+ # CUDA provider only available if GPU is available and onnxruntime-gpu installed
+ if torch.cuda.is_available() and "CUDAExecutionProvider" in ort.get_available_providers():
+ return ["CUDAExecutionProvider", "CPUExecutionProvider"]
+
+ # Default when CUDA provider isn't available
+ return ["CPUExecutionProvider"]
+
def forward(self, **inputs):
"""
Runs inputs through an ONNX model and returns outputs. This method handles casting inputs
diff --git a/src/python/txtai/pipeline/train/hfonnx.py b/src/python/txtai/pipeline/train/hfonnx.py
index 048b5ae8..ee49e476 100644
--- a/src/python/txtai/pipeline/train/hfonnx.py
+++ b/src/python/txtai/pipeline/train/hfonnx.py
@@ -106,11 +106,11 @@ class HFOnnx(Tensors):
output = temp
- # Optimize model first
+ # Optimize model - only need CPU provider
sess_option = SessionOptions()
sess_option.optimized_model_filepath = output
sess_option.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_BASIC
- _ = InferenceSession(output, sess_option)
+ _ = InferenceSession(output, sess_option, ["CPUExecutionProvider"])
# Quantize optimized model
quantize_dynamic(output, output, optimize_model=False)
diff --git a/test/python/testpipeline/testonnx.py b/test/python/testpipeline/testonnx.py
index c72f5410..af1e8590 100644
--- a/test/python/testpipeline/testonnx.py
+++ b/test/python/testpipeline/testonnx.py
@@ -6,6 +6,8 @@ import os
import tempfile
import unittest
+from unittest.mock import patch
+
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
@@ -60,7 +62,9 @@ class TestOnnx(unittest.TestCase):
labels = Labels((model, path), dynamic=False)
self.assertEqual(labels("cat")[0][0], 1)
- def testPooling(self):
+ @patch("onnxruntime.get_available_providers")
+ @patch("torch.cuda.is_available")
+ def testPooling(self, cuda, providers):
"""
Test exporting a pooling model to ONNX and running inference
"""
@@ -71,6 +75,31 @@ class TestOnnx(unittest.TestCase):
onnx = HFOnnx()
model = onnx(path, "pooling", quantize=True)
+ # Test no CUDA and onnxruntime installed
+ cuda.return_value = False
+ providers.return_value = ["CPUExecutionProvider"]
+
+ embeddings = Embeddings({"path": model, "tokenizer": path})
+ self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)
+
+ # Test no CUDA and onnxruntime-gpu installed
+ cuda.return_value = False
+ providers.return_value = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+
+ embeddings = Embeddings({"path": model, "tokenizer": path})
+ self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)
+
+ # Test CUDA and only onnxruntime installed
+ cuda.return_value = True
+ providers.return_value = ["CPUExecutionProvider"]
+
+ embeddings = Embeddings({"path": model, "tokenizer": path})
+ self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)
+
+ # Test CUDA and onnxruntime-gpu installed
+ cuda.return_value = True
+ providers.return_value = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+
embeddings = Embeddings({"path": model, "tokenizer": path})
self.assertEqual(embeddings.similarity("animal", ["dog", "book", "rug"])[0][0], 0)