Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add ONNX export pipeline, closes #107

davidmezzettide6da9daba81
diff --git a/docs/pipelines/onnx.md b/docs/pipelines/onnx.md
new file mode 100644
index 00000000..08f28b27
--- /dev/null
+++ b/docs/pipelines/onnx.md
@@ -0,0 +1,34 @@
+# HFOnnx
+
+Exports a Hugging Face Transformer model to ONNX.
+
+Example on how to use the pipeline below.
+
+```python
+from onnxruntime import InferenceSession, SessionOptions
+from transformers import AutoTokenizer
+
+from txtai.pipeline import HFOnnx
+
+# Normalize logits using sigmoid function
+sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x))
+
+# Export model to ONNX
+onnx = HFOnnx()
+model = onnx("distilbert-base-uncased-finetuned-sst-2-english", "sequence-classification", "model.onnx", True)
+
+# Build ONNX session
+options = SessionOptions()
+session = InferenceSession(model, options)
+
+# Tokenize
+tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
+tokens = tokenizer(["I am happy"], return_tensors="np")
+
+# Run inference and validate
+outputs = session.run(None, dict(tokens))
+outputs = sigmoid(outputs[0])
+```
+
+::: txtai.pipeline.HFOnnx.__init__
+::: txtai.pipeline.HFOnnx.__call__
diff --git a/docs/pipelines/overview.md b/docs/pipelines/overview.md
index eeb6b228..c6bcff20 100644
--- a/docs/pipelines/overview.md
+++ b/docs/pipelines/overview.md
@@ -11,6 +11,7 @@ The following pipeline types are currently supported.
- [Similarity](../similarity)
- [Summary](../summary)
- Hugging Face Transformers models
+ - [ONNX](../onnx)
- [Trainer](../trainer)
- [Transcription](../transcription)
- [Translation](../translation)
diff --git a/mkdocs.yml b/mkdocs.yml
index 62a24b49..235388a4 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -36,6 +36,7 @@ nav:
- Overview: pipelines/overview.md
- Extractor: pipelines/extractor.md
- Labels: pipelines/labels.md
+ - ONNX: pipelines/onnx.md
- Segmentation: pipelines/segmentation.md
- Similarity: pipelines/similarity.md
- Summary: pipelines/summary.md
diff --git a/setup.py b/setup.py
index f8f859b4..cfba4b2f 100644
--- a/setup.py
+++ b/setup.py
@@ -19,12 +19,21 @@ extras["api"] = [
"uvicorn>=0.12.1",
]
-extras["pipeline"] = ["fasttext>=0.9.2", "nltk>=3.5", "sentencepiece>=0.1.91", "soundfile>=0.10.3.post1", "tika>=1.24"]
+extras["pipeline"] = [
+ "fasttext>=0.9.2",
+ "nltk>=3.5",
+ "onnx>=1.10.1",
+ "onnxruntime>=1.8.1",
+ "sentencepiece>=0.1.91",
+ "soundfile>=0.10.3.post1",
+ "tika>=1.24",
+]
extras["similarity"] = [
"annoy>=1.16.3",
"fasttext>=0.9.2",
"hnswlib>=0.5.0",
+ "onnxruntime>=1.8.1",
"pymagnitude-lite>=0.1.43",
"scikit-learn>=0.23.1",
"sentence-transformers>=2.0.0",
diff --git a/src/python/txtai/models/__init__.py b/src/python/txtai/models/__init__.py
index eed0de22..9b881a9f 100644
--- a/src/python/txtai/models/__init__.py
+++ b/src/python/txtai/models/__init__.py
@@ -3,3 +3,4 @@ Models imports
"""
from .models import Models
+from .pooling import MeanPooling, Pooling
diff --git a/src/python/txtai/vectors/pooling.py b/src/python/txtai/models/pooling.py
similarity index 95%
rename from src/python/txtai/vectors/pooling.py
rename to src/python/txtai/models/pooling.py
index 7164990c..072fce5a 100644
--- a/src/python/txtai/vectors/pooling.py
+++ b/src/python/txtai/models/pooling.py
@@ -9,7 +9,7 @@ from torch import nn
from transformers import AutoModel, AutoTokenizer
-from ..models import Models
+from .models import Models
class Pooling(nn.Module):
@@ -72,7 +72,7 @@ class Pooling(nn.Module):
# Run inputs through model
with torch.no_grad():
- outputs = self.forward(inputs)
+ outputs = self.forward(**inputs)
# Add batch result
results.extend(outputs.cpu().numpy())
@@ -94,7 +94,7 @@ class Pooling(nn.Module):
return [texts[x : x + size] for x in range(0, len(texts), size)]
- def forward(self, inputs):
+ def forward(self, **inputs):
"""
Runs inputs through transformers model and returns outputs.
@@ -113,7 +113,7 @@ class MeanPooling(Pooling):
Builds mean pooled vectors usings outputs from a transformers model.
"""
- def forward(self, inputs):
+ def forward(self, **inputs):
"""
Runs mean pooling on token embeddings taking the input mask into account.
@@ -126,7 +126,7 @@ class MeanPooling(Pooling):
"""
# Run through transformers model
- tokens = super().forward(inputs)
+ tokens = super().forward(**inputs)
mask = inputs["attention_mask"]
# Mean pooling
diff --git a/src/python/txtai/pipeline/__init__.py b/src/python/txtai/pipeline/__init__.py
index 24d76a56..da922fa7 100644
--- a/src/python/txtai/pipeline/__init__.py
+++ b/src/python/txtai/pipeline/__init__.py
@@ -6,6 +6,7 @@ from .base import Pipeline
from .extractor import Extractor
from .factory import PipelineFactory
from .hfmodel import HFModel
+from .hfonnx import HFOnnx
from .hfpipeline import HFPipeline
from .hftrainer import HFTrainer
from .labels import Labels
diff --git a/src/python/txtai/pipeline/hfonnx.py b/src/python/txtai/pipeline/hfonnx.py
new file mode 100644
index 00000000..6729bab1
--- /dev/null
+++ b/src/python/txtai/pipeline/hfonnx.py
@@ -0,0 +1,179 @@
+"""
+Hugging Face Transformers ONNX export module
+"""
+
+from collections import OrderedDict
+from io import BytesIO
+from itertools import chain
+from tempfile import NamedTemporaryFile
+
+# Conditional import
+try:
+ from onnxruntime import GraphOptimizationLevel, InferenceSession, SessionOptions
+ from onnxruntime.quantization import quantize_dynamic
+
+ ONNX_RUNTIME = True
+except ImportError:
+ ONNX_RUNTIME = False
+
+from torch.onnx import export
+
+from transformers import AutoModel, AutoModelForCausalLM, AutoModelForMultipleChoice, AutoModelForQuestionAnswering
+from transformers import AutoModelForSeq2SeqLM, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoTokenizer
+
+from ..models import MeanPooling
+from .tensors import Tensors
+
+
+class HFOnnx(Tensors):
+ """
+ Exports a Hugging Face Transformer model to ONNX.
+ """
+
+ def __call__(self, path, task="default", output=None, quantize=False, opset=12):
+ """
+ Exports a Hugging Face Transformer model to ONNX.
+
+ Args:
+ path: path to model, accepts Hugging Face model hub id, local path or (model, tokenizer) tuple
+ task: optional model task or category, determines the model type and outputs, defaults to export hidden state
+ output: optional output model path
+ """
+
+ inputs, outputs, model = self.parameters(task)
+
+ if isinstance(path, tuple):
+ model, tokenizer = path
+ model = model.cpu()
+ else:
+ model = model(path)
+ tokenizer = AutoTokenizer.from_pretrained(path)
+
+ # Generate dummy inputs
+ dummy = dict(tokenizer(["test inputs"], return_tensors="pt"))
+
+ # Default to BytesIO if no output file provided
+ output = output if output else BytesIO()
+
+ # Export model to ONNX
+ export(
+ model,
+ (dummy,),
+ output,
+ opset_version=opset,
+ do_constant_folding=True,
+ input_names=list(inputs.keys()),
+ output_names=list(outputs.keys()),
+ dynamic_axes=dict(chain(inputs.items(), outputs.items())),
+ )
+
+ # Quantize model
+ if quantize:
+ if not ONNX_RUNTIME:
+ raise ImportError('onnxruntime is not available - install "pipeline" extra to enable')
+
+ output = self.quantization(output)
+
+ if isinstance(output, BytesIO):
+ # Reset stream and return bytes
+ output.seek(0)
+ output = output.read()
+
+ return output
+
+ def quantization(self, output):
+ """
+ Quantizes an ONNX model.
+
+ Args:
+ data: path to ONNX model or BytesIO with model data
+
+ Returns:
+ quantized model as file path or bytes
+ """
+
+ temp = None
+ if isinstance(output, BytesIO):
+ with NamedTemporaryFile(suffix=".quant", delete=False) as tmpfile:
+ temp = tmpfile.name
+
+ with open(temp, "wb") as f:
+ f.write(output.getbuffer())
+
+ output = temp
+
+ # Optimize model first
+ sess_option = SessionOptions()
+ sess_option.optimized_model_filepath = output
+ sess_option.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_BASIC
+ _ = InferenceSession(output, sess_option)
+
+ # Quantize optimized model
+ quantize_dynamic(output, output, optimize_model=False)
+
+ # Read file back to bytes if temp file was created
+ if temp:
+ with open(temp, "rb") as f:
+ output = f.read()
+
+ return output
+
+ def parameters(self, task):
+ """
+ Defines inputs and outputs for an ONNX model.
+
+ Args:
+ task: task name used to lookup model configuration
+
+ Returns:
+ (inputs, outputs, model function)
+ """
+
+ inputs = OrderedDict(
+ [
+ ("input_ids", {0: "batch", 1: "sequence"}),
+ ("attention_mask", {0: "batch", 1: "sequence"}),
+ ("token_type_ids", {0: "batch", 1: "sequence"}),
+ ]
+ )
+
+ config = {
+ "default": (
+ OrderedDict({"last_hidden_state": {0: "batch", 1: "sequence"}}),
+ AutoModel.from_pretrained,
+ ),
+ "causal-lm": (OrderedDict({"logits": {0: "batch", 1: "sequence"}}), AutoModelForCausalLM.from_pretrained),
+ "pooling": (OrderedDict({"embeddings": {0: "batch", 1: "sequence"}}), lambda x: MeanPoolingOnnx(x, -1)),
+ "seq2seq-lm": (OrderedDict({"logits": {0: "batch", 1: "decoder_sequence"}}), AutoModelForSeq2SeqLM.from_pretrained),
+ "sequence-classification": (OrderedDict({"logits": {0: "batch"}}), AutoModelForSequenceClassification.from_pretrained),
+ "token-classification": (OrderedDict({"logits": {0: "batch", 1: "sequence"}}), AutoModelForTokenClassification.from_pretrained),
+ "multiple-choice": (OrderedDict({"logits": {0: "batch"}}), AutoModelForMultipleChoice.from_pretrained),
+ "question-answering": (
+ OrderedDict(
+ {
+ "start_logits": {0: "batch", 1: "sequence"},
+ "end_logits": {0: "batch", 1: "sequence"},
+ }
+ ),
+ AutoModelForQuestionAnswering.from_pretrained,
+ ),
+ }
+
+ return (inputs,) + config[task]
+
+
+class MeanPoolingOnnx(MeanPooling):
+ """
+ Extends MeanPooling class to explicitly name inputs to model, which is required
+ to export to ONNX.
+ """
+
+ # pylint: disable=W0221
+ def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):
+ # Build list of arguments dynamically since some models take token_type_ids
+ # and others don't
+ inputs = {"input_ids": input_ids, "attention_mask": attention_mask}
+ if token_type_ids is not None:
+ inputs["token_type_ids"] = token_type_ids
+
+ return super().forward(**inputs)
diff --git a/src/python/txtai/pipeline/hftrainer.py b/src/python/txtai/pipeline/hftrainer.py
index c84e0206..3b458dd6 100644
--- a/src/python/txtai/pipeline/hftrainer.py
+++ b/src/python/txtai/pipeline/hftrainer.py
@@ -25,7 +25,7 @@ class HFTrainer(Tensors):
Builds a new model using arguments.
Args:
- base: base model - supports a file path or (model, tokenizer) tuple
+ base: path to base model, accepts Hugging Face model hub id, local path or (model, tokenizer) tuple
train: training data
validation: validation data
columns: tuple of columns to use for text/label, defaults to (text, None, label)
diff --git a/src/python/txtai/vectors/transformers.py b/src/python/txtai/vectors/transformers.py
index 137001ac..f72071bd 100644
--- a/src/python/txtai/vectors/transformers.py
+++ b/src/python/txtai/vectors/transformers.py
@@ -14,9 +14,8 @@ except ImportError:
SENTENCE_TRANSFORMERS = False
from .base import Vectors
-from ..models import Models
+from ..models import MeanPooling, Models
from ..pipeline.tokenizer import Tokenizer
-from .pooling import MeanPooling
class TransformersVectors(Vectors):
diff --git a/test/python/testonnx.py b/test/python/testonnx.py
new file mode 100644
index 00000000..5d3e580c
--- /dev/null
+++ b/test/python/testonnx.py
@@ -0,0 +1,92 @@
+"""
+ONNX module tests
+"""
+
+import os
+import tempfile
+import unittest
+
+import numpy as np
+
+from onnxruntime import InferenceSession, SessionOptions
+
+from transformers import AutoTokenizer
+
+from txtai.pipeline import HFOnnx, HFTrainer
+
+
+class TestOnnx(unittest.TestCase):
+ """
+ ONNX tests
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Create default datasets
+ """
+
+ cls.data = [{"text": "Dogs", "label": 0}, {"text": "dog", "label": 0}, {"text": "Cats", "label": 1}, {"text": "cat", "label": 1}] * 100
+
+ def testDefault(self):
+ """
+ Test exporting an ONNX model with default parameters
+ """
+
+ # Export model to ONNX, use default parameters
+ onnx = HFOnnx()
+ model = onnx("google/bert_uncased_L-2_H-128_A-2")
+
+ # Validate model has data
+ self.assertGreater(len(model), 0)
+
+ def testClassification(self):
+ """
+ Test exporting a classification model to ONNX and running inference
+ """
+
+ # Normalize logits using sigmoid function
+ sigmoid = lambda x: 1.0 / (1.0 + np.exp(-x))
+
+ trainer = HFTrainer()
+ model, tokenizer = trainer("google/bert_uncased_L-2_H-128_A-2", self.data)
+
+ # Output file path
+ output = os.path.join(tempfile.gettempdir(), "onnx")
+
+ # Export model to ONNX
+ onnx = HFOnnx()
+ model = onnx((model, tokenizer), "sequence-classification", output, True)
+
+ # Build ONNX session
+ options = SessionOptions()
+ session = InferenceSession(model, options)
+
+ # Tokenize
+ tokens = tokenizer(["cat"], return_tensors="np")
+
+ # Run inference and validate
+ outputs = session.run(None, dict(tokens))
+ outputs = sigmoid(outputs[0])
+ self.assertEqual(np.argmax(outputs[0]), 1)
+
+ def testPooling(self):
+ """
+ Test exporting a pooling model to ONNX and running inference
+ """
+
+ # Export model to ONNX
+ onnx = HFOnnx()
+ model = onnx("sentence-transformers/paraphrase-MiniLM-L3-v2", "pooling", quantize=True)
+ tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2")
+
+ # Build ONNX session
+ options = SessionOptions()
+ session = InferenceSession(model, options)
+
+ # Tokenize
+ tokens = tokenizer(["cat"], return_tensors="np")
+
+ # Run inference and validate
+ outputs = session.run(None, dict(tokens))
+ self.assertEqual(outputs[0].shape, (1, 384))
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index 9e316fbe..2c46f52a 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -7,7 +7,7 @@ import unittest
import txtai.ann.factory
from txtai.ann import ANNFactory
-from txtai.pipeline import Segmentation, Textractor, Transcription, Translation
+from txtai.pipeline import HFOnnx, Segmentation, Textractor, Transcription, Translation
from txtai.vectors import VectorsFactory
from txtai.workflow.task.image import ImageTask
from txtai.workflow.task.storage import StorageTask
@@ -27,6 +27,7 @@ class TestOptional(unittest.TestCase):
txtai.ann.factory.ANNOY = not txtai.ann.factory.ANNOY
txtai.ann.factory.HNSWLIB = not txtai.ann.factory.HNSWLIB
+ txtai.pipeline.hfonnx.ONNX_RUNTIME = not txtai.pipeline.hfonnx.ONNX_RUNTIME
txtai.pipeline.segmentation.NLTK = not txtai.pipeline.segmentation.NLTK
txtai.pipeline.textractor.TIKA = not txtai.pipeline.textractor.TIKA
txtai.pipeline.transcription.SOUNDFILE = not txtai.pipeline.transcription.SOUNDFILE
@@ -72,6 +73,9 @@ class TestOptional(unittest.TestCase):
Test missing pipeline dependencies
"""
+ with self.assertRaises(ImportError):
+ HFOnnx()("google/bert_uncased_L-2_H-128_A-2", quantize=True)
+
with self.assertRaises(ImportError):
Segmentation()
diff --git a/test/python/testtransformers.py b/test/python/testtransformers.py
index 343dfd5c..9400f71c 100644
--- a/test/python/testtransformers.py
+++ b/test/python/testtransformers.py
@@ -43,7 +43,7 @@ class TestTransformersVectors(unittest.TestCase):
"""
model = VectorsFactory.create({"method": "sentence-transformers", "path": "paraphrase-MiniLM-L3-v2"}, None)
- self.assertIsNotNone(model.transform("This is a test"))
+ self.assertEqual(model.transform("This is a test").shape, (384,))
def testText(self):
"""