💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Infer configuration for ONNX models
davidmezzetti378bcd4175c0
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.pyindex 459a7877..2d1e6f6b 100644--- a/src/python/txtai/models/models.py+++ b/src/python/txtai/models/models.py@@ -89,13 +89,14 @@ class Models:return "cpu" if deviceid < 0 else "cuda:{}".format(deviceid)@staticmethod- def load(path, task="default"):+ def load(path, config=None, task="default"):"""Loads a machine learning model. Handles multiple model frameworks (ONNX, Transformers).Args:path: path to model- task: task name used to lookup model configuration+ config: path to model configuration+ task: task name used to lookup model typeReturns:machine learning model@@ -103,14 +104,14 @@ class Models:# Detect ONNX modelsif isinstance(path, bytes) or (isinstance(path, str) and os.path.isfile(path)):- return OnnxModel(path)+ return OnnxModel(path, config)# Return path, if path isn't a stringif not isinstance(path, str):return path# Transformer models- config = {+ models = {"default": AutoModel.from_pretrained,"question-answering": AutoModelForQuestionAnswering.from_pretrained,"summarization": AutoModelForSeq2SeqLM.from_pretrained,@@ -119,4 +120,4 @@ class Models:}# Load model for supported tasks. Return path for unsupported tasks.- return config[task](path) if task in config else path+ return models[task](path) if task in models else pathdiff --git a/src/python/txtai/models/onnx.py b/src/python/txtai/models/onnx.pyindex 71ce1804..57d81e86 100644--- a/src/python/txtai/models/onnx.py+++ b/src/python/txtai/models/onnx.py@@ -15,6 +15,7 @@ except ImportError:import numpy as npimport torch+from transformers import AutoConfigfrom transformers.configuration_utils import PretrainedConfigfrom transformers.models.auto.modeling_auto import (MODEL_MAPPING,@@ -32,18 +33,19 @@ class OnnxModel(PreTrainedModel):and outputs with minimal to no copying of data."""- def __init__(self, model):+ def __init__(self, model, config=None):"""Creates a new OnnxModel.Args:model: path to model or InferenceSession+ config: path to model configuration"""if not ONNX_RUNTIME:raise ImportError('onnxruntime is not available - install "model" extra to enable')- super().__init__(OnnxConfig())+ super().__init__(AutoConfig.from_pretrained(config) if config else OnnxConfig())# Create ONNX sessionself.model = InferenceSession(model, SessionOptions())diff --git a/src/python/txtai/pipeline/hfonnx.py b/src/python/txtai/pipeline/hfonnx.pyindex d97a307a..99899506 100644--- a/src/python/txtai/pipeline/hfonnx.py+++ b/src/python/txtai/pipeline/hfonnx.py@@ -46,7 +46,7 @@ class HFOnnx(Tensors):inputs, outputs, model = self.parameters(task)- if isinstance(path, tuple):+ if isinstance(path, (list, tuple)):model, tokenizer = pathmodel = model.cpu()else:diff --git a/src/python/txtai/pipeline/hfpipeline.py b/src/python/txtai/pipeline/hfpipeline.pyindex 61be8e99..88a59df0 100644--- a/src/python/txtai/pipeline/hfpipeline.py+++ b/src/python/txtai/pipeline/hfpipeline.py@@ -35,8 +35,11 @@ class HFPipeline(Tensors):deviceid = Models.deviceid(gpu)# Transformer pipeline task- if isinstance(path, tuple):- self.pipeline = pipeline(task, model=Models.load(path[0], task), tokenizer=path[1], device=deviceid)+ if isinstance(path, (list, tuple)):+ # Derive configuration, if possible+ config = path[1] if path[1] and isinstance(path[1], str) else None++ self.pipeline = pipeline(task, model=Models.load(path[0], config, task), tokenizer=path[1], device=deviceid)else:self.pipeline = pipeline(task, model=path, tokenizer=path, device=deviceid)diff --git a/src/python/txtai/pipeline/hftrainer.py b/src/python/txtai/pipeline/hftrainer.pyindex 21736b7f..7785d34b 100644--- a/src/python/txtai/pipeline/hftrainer.py+++ b/src/python/txtai/pipeline/hftrainer.py@@ -111,7 +111,7 @@ class HFTrainer(Tensors):(config, tokenizer, maxlength)"""- if isinstance(base, tuple):+ if isinstance(base, (list, tuple)):# Unpack existing config and tokenizermodel, tokenizer = baseconfig = model.config@@ -150,7 +150,7 @@ class HFTrainer(Tensors):# pylint: disable=E1120# Unpack existing model or create new model from config- if isinstance(base, tuple):+ if isinstance(base, (list, tuple)):return base[0]if task == "question-answering":return AutoModelForQuestionAnswering.from_pretrained(base, config=config)