💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add ONNX export pipeline, closes #107
davidmezzettide6da9daba81
diff --git a/docs/pipelines/onnx.md b/docs/pipelines/onnx.mdnew file mode 100644index 00000000..08f28b27--- /dev/null+++ b/docs/pipelines/onnx.md@@ -0,0 +1,34 @@+# HFOnnx++Exports a Hugging Face Transformer model to ONNX.++Example on how to use the pipeline below.++```python+from onnxruntime import InferenceSession, SessionOptions+from transformers import AutoTokenizer++from txtai.pipeline import HFOnnx++# Normalize logits using sigmoid function+sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x))++# Export model to ONNX+onnx = HFOnnx()+model = onnx("distilbert-base-uncased-finetuned-sst-2-english", "sequence-classification", "model.onnx", True)++# Build ONNX session+options = SessionOptions()+session = InferenceSession(model, options)++# Tokenize+tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")+tokens = tokenizer(["I am happy"], return_tensors="np")++# Run inference and validate+outputs = session.run(None, dict(tokens))+outputs = sigmoid(outputs[0])+```++::: txtai.pipeline.HFOnnx.__init__+::: txtai.pipeline.HFOnnx.__call__diff --git a/docs/pipelines/overview.md b/docs/pipelines/overview.mdindex eeb6b228..c6bcff20 100644--- a/docs/pipelines/overview.md+++ b/docs/pipelines/overview.md@@ -11,6 +11,7 @@ The following pipeline types are currently supported.- [Similarity](../similarity)- [Summary](../summary)- Hugging Face Transformers models+ - [ONNX](../onnx)- [Trainer](../trainer)- [Transcription](../transcription)- [Translation](../translation)diff --git a/mkdocs.yml b/mkdocs.ymlindex 62a24b49..235388a4 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -36,6 +36,7 @@ nav:- Overview: pipelines/overview.md- Extractor: pipelines/extractor.md- Labels: pipelines/labels.md+ - ONNX: pipelines/onnx.md- Segmentation: pipelines/segmentation.md- Similarity: pipelines/similarity.md- Summary: pipelines/summary.mddiff --git a/setup.py b/setup.pyindex f8f859b4..cfba4b2f 100644--- a/setup.py+++ b/setup.py@@ -19,12 +19,21 @@ extras["api"] = ["uvicorn>=0.12.1",]-extras["pipeline"] = ["fasttext>=0.9.2", "nltk>=3.5", "sentencepiece>=0.1.91", "soundfile>=0.10.3.post1", "tika>=1.24"]+extras["pipeline"] = [+ "fasttext>=0.9.2",+ "nltk>=3.5",+ "onnx>=1.10.1",+ "onnxruntime>=1.8.1",+ "sentencepiece>=0.1.91",+ "soundfile>=0.10.3.post1",+ "tika>=1.24",+]extras["similarity"] = ["annoy>=1.16.3","fasttext>=0.9.2","hnswlib>=0.5.0",+ "onnxruntime>=1.8.1","pymagnitude-lite>=0.1.43","scikit-learn>=0.23.1","sentence-transformers>=2.0.0",diff --git a/src/python/txtai/models/__init__.py b/src/python/txtai/models/__init__.pyindex eed0de22..9b881a9f 100644--- a/src/python/txtai/models/__init__.py+++ b/src/python/txtai/models/__init__.py@@ -3,3 +3,4 @@ Models imports"""from .models import Models+from .pooling import MeanPooling, Poolingdiff --git a/src/python/txtai/vectors/pooling.py b/src/python/txtai/models/pooling.pysimilarity index 95%rename from src/python/txtai/vectors/pooling.pyrename to src/python/txtai/models/pooling.pyindex 7164990c..072fce5a 100644--- a/src/python/txtai/vectors/pooling.py+++ b/src/python/txtai/models/pooling.py@@ -9,7 +9,7 @@ from torch import nnfrom transformers import AutoModel, AutoTokenizer-from ..models import Models+from .models import Modelsclass Pooling(nn.Module):@@ -72,7 +72,7 @@ class Pooling(nn.Module):# Run inputs through modelwith torch.no_grad():- outputs = self.forward(inputs)+ outputs = self.forward(**inputs)# Add batch resultresults.extend(outputs.cpu().numpy())@@ -94,7 +94,7 @@ class Pooling(nn.Module):return [texts[x : x + size] for x in range(0, len(texts), size)]- def forward(self, inputs):+ def forward(self, **inputs):"""Runs inputs through transformers model and returns outputs.@@ -113,7 +113,7 @@ class MeanPooling(Pooling):Builds mean pooled vectors usings outputs from a transformers model."""- def forward(self, inputs):+ def forward(self, **inputs):"""Runs mean pooling on token embeddings taking the input mask into account.@@ -126,7 +126,7 @@ class MeanPooling(Pooling):"""# Run through transformers model- tokens = super().forward(inputs)+ tokens = super().forward(**inputs)mask = inputs["attention_mask"]# Mean poolingdiff --git a/src/python/txtai/pipeline/__init__.py b/src/python/txtai/pipeline/__init__.pyindex 24d76a56..da922fa7 100644--- a/src/python/txtai/pipeline/__init__.py+++ b/src/python/txtai/pipeline/__init__.py@@ -6,6 +6,7 @@ from .base import Pipelinefrom .extractor import Extractorfrom .factory import PipelineFactoryfrom .hfmodel import HFModel+from .hfonnx import HFOnnxfrom .hfpipeline import HFPipelinefrom .hftrainer import HFTrainerfrom .labels import Labelsdiff --git a/src/python/txtai/pipeline/hfonnx.py b/src/python/txtai/pipeline/hfonnx.pynew file mode 100644index 00000000..6729bab1--- /dev/null+++ b/src/python/txtai/pipeline/hfonnx.py@@ -0,0 +1,179 @@+"""+Hugging Face Transformers ONNX export module+"""++from collections import OrderedDict+from io import BytesIO+from itertools import chain+from tempfile import NamedTemporaryFile++# Conditional import+try:+ from onnxruntime import GraphOptimizationLevel, InferenceSession, SessionOptions+ from onnxruntime.quantization import quantize_dynamic++ ONNX_RUNTIME = True+except ImportError:+ ONNX_RUNTIME = False++from torch.onnx import export++from transformers import AutoModel, AutoModelForCausalLM, AutoModelForMultipleChoice, AutoModelForQuestionAnswering+from transformers import AutoModelForSeq2SeqLM, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoTokenizer++from ..models import MeanPooling+from .tensors import Tensors+++class HFOnnx(Tensors):+ """+ Exports a Hugging Face Transformer model to ONNX.+ """++ def __call__(self, path, task="default", output=None, quantize=False, opset=12):+ """+ Exports a Hugging Face Transformer model to ONNX.++ Args:+ path: path to model, accepts Hugging Face model hub id, local path or (model, tokenizer) tuple+ task: optional model task or category, determines the model type and outputs, defaults to export hidden state+ output: optional output model path+ """++ inputs, outputs, model = self.parameters(task)++ if isinstance(path, tuple):+ model, tokenizer = path+ model = model.cpu()+ else:+ model = model(path)+ tokenizer = AutoTokenizer.from_pretrained(path)++ # Generate dummy inputs+ dummy = dict(tokenizer(["test inputs"], return_tensors="pt"))++ # Default to BytesIO if no output file provided+ output = output if output else BytesIO()++ # Export model to ONNX+ export(+ model,+ (dummy,),+ output,+ opset_version=opset,+ do_constant_folding=True,+ input_names=list(inputs.keys()),+ output_names=list(outputs.keys()),+ dynamic_axes=dict(chain(inputs.items(), outputs.items())),+ )++ # Quantize model+ if quantize:+ if not ONNX_RUNTIME:+ raise ImportError('onnxruntime is not available - install "pipeline" extra to enable')++ output = self.quantization(output)++ if isinstance(output, BytesIO):+ # Reset stream and return bytes+ output.seek(0)+ output = output.read()++ return output++ def quantization(self, output):+ """+ Quantizes an ONNX model.++ Args:+ data: path to ONNX model or BytesIO with model data++ Returns:+ quantized model as file path or bytes+ """++ temp = None+ if isinstance(output, BytesIO):+ with NamedTemporaryFile(suffix=".quant", delete=False) as tmpfile:+ temp = tmpfile.name++ with open(temp, "wb") as f:+ f.write(output.getbuffer())++ output = temp++ # Optimize model first+ sess_option = SessionOptions()+ sess_option.optimized_model_filepath = output+ sess_option.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_BASIC+ _ = InferenceSession(output, sess_option)++ # Quantize optimized model+ quantize_dynamic(output, output, optimize_model=False)++ # Read file back to bytes if temp file was created+ if temp:+ with open(temp, "rb") as f:+ output = f.read()++ return output++ def parameters(self, task):+ """+ Defines inputs and outputs for an ONNX model.++ Args:+ task: task name used to lookup model configuration++ Returns:+ (inputs, outputs, model function)+ """++ inputs = OrderedDict(+ [+ ("input_ids", {0: "batch", 1: "sequence"}),+ ("attention_mask", {0: "batch", 1: "sequence"}),+ ("token_type_ids", {0: "batch", 1: "sequence"}),+ ]+ )++ config = {+ "default": (+ OrderedDict({"last_hidden_state": {0: "batch", 1: "sequence"}}),+ AutoModel.from_pretrained,+ ),+ "causal-lm": (OrderedDict({"logits": {0: "batch", 1: "sequence"}}), AutoModelForCausalLM.from_pretrained),+ "pooling": (OrderedDict({"embeddings": {0: "batch", 1: "sequence"}}), lambda x: MeanPoolingOnnx(x, -1)),+ "seq2seq-lm": (OrderedDict({"logits": {0: "batch", 1: "decoder_sequence"}}), AutoModelForSeq2SeqLM.from_pretrained),+ "sequence-classification": (OrderedDict({"logits": {0: "batch"}}), AutoModelForSequenceClassification.from_pretrained),+ "token-classification": (OrderedDict({"logits": {0: "batch", 1: "sequence"}}), AutoModelForTokenClassification.from_pretrained),+ "multiple-choice": (OrderedDict({"logits": {0: "batch"}}), AutoModelForMultipleChoice.from_pretrained),+ "question-answering": (+ OrderedDict(+ {+ "start_logits": {0: "batch", 1: "sequence"},+ "end_logits": {0: "batch", 1: "sequence"},+ }+ ),+ AutoModelForQuestionAnswering.from_pretrained,+ ),+ }++ return (inputs,) + config[task]+++class MeanPoolingOnnx(MeanPooling):+ """+ Extends MeanPooling class to explicitly name inputs to model, which is required+ to export to ONNX.+ """++ # pylint: disable=W0221+ def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):+ # Build list of arguments dynamically since some models take token_type_ids+ # and others don't+ inputs = {"input_ids": input_ids, "attention_mask": attention_mask}+ if token_type_ids is not None:+ inputs["token_type_ids"] = token_type_ids++ return super().forward(**inputs)diff --git a/src/python/txtai/pipeline/hftrainer.py b/src/python/txtai/pipeline/hftrainer.pyindex c84e0206..3b458dd6 100644--- a/src/python/txtai/pipeline/hftrainer.py+++ b/src/python/txtai/pipeline/hftrainer.py@@ -25,7 +25,7 @@ class HFTrainer(Tensors):Builds a new model using arguments.Args:- base: base model - supports a file path or (model, tokenizer) tuple+ base: path to base model, accepts Hugging Face model hub id, local path or (model, tokenizer) tupletrain: training datavalidation: validation datacolumns: tuple of columns to use for text/label, defaults to (text, None, label)diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.pyindex 137001ac..f72071bd 100644--- a/src/python/txtai/vectors/transformers.py+++ b/src/python/txtai/vectors/transformers.py@@ -14,9 +14,8 @@ except ImportError:SENTENCE_TRANSFORMERS = Falsefrom .base import Vectors-from ..models import Models+from ..models import MeanPooling, Modelsfrom ..pipeline.tokenizer import Tokenizer-from .pooling import MeanPoolingclass TransformersVectors(Vectors):diff --git a/test/python/testonnx.py b/test/python/testonnx.pynew file mode 100644index 00000000..5d3e580c--- /dev/null+++ b/test/python/testonnx.py@@ -0,0 +1,92 @@+"""+ONNX module tests+"""++import os+import tempfile+import unittest++import numpy as np++from onnxruntime import InferenceSession, SessionOptions++from transformers import AutoTokenizer++from txtai.pipeline import HFOnnx, HFTrainer+++class TestOnnx(unittest.TestCase):+ """+ ONNX tests+ """++ @classmethod+ def setUpClass(cls):+ """+ Create default datasets+ """++ cls.data = [{"text": "Dogs", "label": 0}, {"text": "dog", "label": 0}, {"text": "Cats", "label": 1}, {"text": "cat", "label": 1}] * 100++ def testDefault(self):+ """+ Test exporting an ONNX model with default parameters+ """++ # Export model to ONNX, use default parameters+ onnx = HFOnnx()+ model = onnx("google/bert_uncased_L-2_H-128_A-2")++ # Validate model has data+ self.assertGreater(len(model), 0)++ def testClassification(self):+ """+ Test exporting a classification model to ONNX and running inference+ """++ # Normalize logits using sigmoid function+ sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x))++ trainer = HFTrainer()+ model, tokenizer = trainer("google/bert_uncased_L-2_H-128_A-2", self.data)++ # Output file path+ output = os.path.join(tempfile.gettempdir(), "onnx")++ # Export model to ONNX+ onnx = HFOnnx()+ model = onnx((model, tokenizer), "sequence-classification", output, True)++ # Build ONNX session+ options = SessionOptions()+ session = InferenceSession(model, options)++ # Tokenize+ tokens = tokenizer(["cat"], return_tensors="np")++ # Run inference and validate+ outputs = session.run(None, dict(tokens))+ outputs = sigmoid(outputs[0])+ self.assertEqual(np.argmax(outputs[0]), 1)++ def testPooling(self):+ """+ Test exporting a pooling model to ONNX and running inference+ """++ # Export model to ONNX+ onnx = HFOnnx()+ model = onnx("sentence-transformers/paraphrase-MiniLM-L3-v2", "pooling", quantize=True)+ tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2")++ # Build ONNX session+ options = SessionOptions()+ session = InferenceSession(model, options)++ # Tokenize+ tokens = tokenizer(["cat"], return_tensors="np")++ # Run inference and validate+ outputs = session.run(None, dict(tokens))+ self.assertEqual(outputs[0].shape, (1, 384))diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 9e316fbe..2c46f52a 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -7,7 +7,7 @@ import unittestimport txtai.ann.factoryfrom txtai.ann import ANNFactory-from txtai.pipeline import Segmentation, Textractor, Transcription, Translation+from txtai.pipeline import HFOnnx, Segmentation, Textractor, Transcription, Translationfrom txtai.vectors import VectorsFactoryfrom txtai.workflow.task.image import ImageTaskfrom txtai.workflow.task.storage import StorageTask@@ -27,6 +27,7 @@ class TestOptional(unittest.TestCase):txtai.ann.factory.ANNOY = not txtai.ann.factory.ANNOYtxtai.ann.factory.HNSWLIB = not txtai.ann.factory.HNSWLIB+ txtai.pipeline.hfonnx.ONNX_RUNTIME = not txtai.pipeline.hfonnx.ONNX_RUNTIMEtxtai.pipeline.segmentation.NLTK = not txtai.pipeline.segmentation.NLTKtxtai.pipeline.textractor.TIKA = not txtai.pipeline.textractor.TIKAtxtai.pipeline.transcription.SOUNDFILE = not txtai.pipeline.transcription.SOUNDFILE@@ -72,6 +73,9 @@ class TestOptional(unittest.TestCase):Test missing pipeline dependencies"""+ with self.assertRaises(ImportError):+ HFOnnx()("google/bert_uncased_L-2_H-128_A-2", quantize=True)+with self.assertRaises(ImportError):Segmentation()diff --git a/test/python/testtransformers.py b/test/python/testtransformers.pyindex 343dfd5c..9400f71c 100644--- a/test/python/testtransformers.py+++ b/test/python/testtransformers.py@@ -43,7 +43,7 @@ class TestTransformersVectors(unittest.TestCase):"""model = VectorsFactory.create({"method": "sentence-transformers", "path": "paraphrase-MiniLM-L3-v2"}, None)- self.assertIsNotNone(model.transform("This is a test"))+ self.assertEqual(model.transform("This is a test").shape, (384,))def testText(self):"""