Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Infer configuration for ONNX models

davidmezzetti378bcd4175c0
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.py
index 459a7877..2d1e6f6b 100644
--- a/src/python/txtai/models/models.py
+++ b/src/python/txtai/models/models.py
@@ -89,13 +89,14 @@ class Models:
return "cpu" if deviceid < 0 else "cuda:{}".format(deviceid)
@staticmethod
- def load(path, task="default"):
+ def load(path, config=None, task="default"):
"""
Loads a machine learning model. Handles multiple model frameworks (ONNX, Transformers).
Args:
path: path to model
- task: task name used to lookup model configuration
+ config: path to model configuration
+ task: task name used to lookup model type
Returns:
machine learning model
@@ -103,14 +104,14 @@ class Models:
# Detect ONNX models
if isinstance(path, bytes) or (isinstance(path, str) and os.path.isfile(path)):
- return OnnxModel(path)
+ return OnnxModel(path, config)
# Return path, if path isn't a string
if not isinstance(path, str):
return path
# Transformer models
- config = {
+ models = {
"default": AutoModel.from_pretrained,
"question-answering": AutoModelForQuestionAnswering.from_pretrained,
"summarization": AutoModelForSeq2SeqLM.from_pretrained,
@@ -119,4 +120,4 @@ class Models:
}
# Load model for supported tasks. Return path for unsupported tasks.
- return config[task](path) if task in config else path
+ return models[task](path) if task in models else path
diff --git a/src/python/txtai/models/onnx.py b/src/python/txtai/models/onnx.py
index 71ce1804..57d81e86 100644
--- a/src/python/txtai/models/onnx.py
+++ b/src/python/txtai/models/onnx.py
@@ -15,6 +15,7 @@ except ImportError:
import numpy as np
import torch
+from transformers import AutoConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.models.auto.modeling_auto import (
MODEL_MAPPING,
@@ -32,18 +33,19 @@ class OnnxModel(PreTrainedModel):
and outputs with minimal to no copying of data.
"""
- def __init__(self, model):
+ def __init__(self, model, config=None):
"""
Creates a new OnnxModel.
Args:
model: path to model or InferenceSession
+ config: path to model configuration
"""
if not ONNX_RUNTIME:
raise ImportError('onnxruntime is not available - install "model" extra to enable')
- super().__init__(OnnxConfig())
+ super().__init__(AutoConfig.from_pretrained(config) if config else OnnxConfig())
# Create ONNX session
self.model = InferenceSession(model, SessionOptions())
diff --git a/src/python/txtai/pipeline/hfonnx.py b/src/python/txtai/pipeline/hfonnx.py
index d97a307a..99899506 100644
--- a/src/python/txtai/pipeline/hfonnx.py
+++ b/src/python/txtai/pipeline/hfonnx.py
@@ -46,7 +46,7 @@ class HFOnnx(Tensors):
inputs, outputs, model = self.parameters(task)
- if isinstance(path, tuple):
+ if isinstance(path, (list, tuple)):
model, tokenizer = path
model = model.cpu()
else:
diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.py
index 61be8e99..88a59df0 100644
--- a/src/python/txtai/pipeline/hfpipeline.py
+++ b/src/python/txtai/pipeline/hfpipeline.py
@@ -35,8 +35,11 @@ class HFPipeline(Tensors):
deviceid = Models.deviceid(gpu)
# Transformer pipeline task
- if isinstance(path, tuple):
- self.pipeline = pipeline(task, model=Models.load(path[0], task), tokenizer=path[1], device=deviceid)
+ if isinstance(path, (list, tuple)):
+ # Derive configuration, if possible
+ config = path[1] if path[1] and isinstance(path[1], str) else None
+
+ self.pipeline = pipeline(task, model=Models.load(path[0], config, task), tokenizer=path[1], device=deviceid)
else:
self.pipeline = pipeline(task, model=path, tokenizer=path, device=deviceid)
diff --git a/src/python/txtai/pipeline/hftrainer.py b/src/python/txtai/pipeline/hftrainer.py
index 21736b7f..7785d34b 100644
--- a/src/python/txtai/pipeline/hftrainer.py
+++ b/src/python/txtai/pipeline/hftrainer.py
@@ -111,7 +111,7 @@ class HFTrainer(Tensors):
(config, tokenizer, maxlength)
"""
- if isinstance(base, tuple):
+ if isinstance(base, (list, tuple)):
# Unpack existing config and tokenizer
model, tokenizer = base
config = model.config
@@ -150,7 +150,7 @@ class HFTrainer(Tensors):
# pylint: disable=E1120
# Unpack existing model or create new model from config
- if isinstance(base, tuple):
+ if isinstance(base, (list, tuple)):
return base[0]
if task == "question-answering":
return AutoModelForQuestionAnswering.from_pretrained(base, config=config)