Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update notebook

davidmezzettia0ab014e9608
diff --git a/examples/18_Export_and_run_models_with_ONNX.ipynb b/examples/18_Export_and_run_models_with_ONNX.ipynb
index 2a9b88c6..af7eba82 100644
--- a/examples/18_Export_and_run_models_with_ONNX.ipynb
+++ b/examples/18_Export_and_run_models_with_ONNX.ipynb
@@ -51,7 +51,7 @@
"%%capture\n",
"!pip install datasets git+https://github.com/neuml/txtai#egg=txtai[pipeline]"
],
- "execution_count": 23,
+ "execution_count": 31,
"outputs": []
},
{
@@ -73,7 +73,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "367c3baf-5e9d-4992-8e12-b517fff6688d"
+ "outputId": "b32db17b-2563-4960-c1bc-fc1193172ea4"
},
"source": [
"import numpy as np\n",
@@ -87,7 +87,7 @@
"\n",
"# Export to ONNX\n",
"onnx = HFOnnx()\n",
- "model = onnx(\"distilbert-base-uncased-finetuned-sst-2-english\", \"sequence-classification\")\n",
+ "model = onnx(\"distilbert-base-uncased-finetuned-sst-2-english\", \"text-classification\")\n",
"\n",
"# Start inference session\n",
"options = SessionOptions()\n",
@@ -101,15 +101,15 @@
"outputs = session.run(None, dict(tokens))\n",
"print(sigmoid(outputs[0]))"
],
- "execution_count": 24,
+ "execution_count": 32,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"[[0.01295124 0.9909526 ]\n",
" [0.9874723 0.0297817 ]]\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -143,7 +143,7 @@
"height": 398
},
"id": "Wh8TkszumlIe",
- "outputId": "c1556a18-7ba6-44ec-b80f-217d115c90b1"
+ "outputId": "f1fd81e5-5e4b-4a46-9ed7-33aacd20b862"
},
"source": [
"from datasets import load_dataset\n",
@@ -159,7 +159,7 @@
"model, tokenizer = trainer(\"google/electra-base-discriminator\", data, columns=(\"sentence\", \"label\"))\n",
"\n",
"# Export model trained in-memory to ONNX (still in-memory)\n",
- "output = onnx((model, tokenizer), \"sequence-classification\", quantize=True)\n",
+ "output = onnx((model, tokenizer), \"text-classification\", quantize=True)\n",
"\n",
"# Start inference session\n",
"options = SessionOptions()\n",
@@ -172,21 +172,21 @@
"outputs = session.run(None, dict(tokens))\n",
"print(sigmoid(outputs[0]))"
],
- "execution_count": 25,
+ "execution_count": 33,
"outputs": [
{
"output_type": "stream",
+ "name": "stderr",
"text": [
"Reusing dataset glue (/root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
"Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-0ad799e068ae6c27.arrow\n",
"Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-422ba3363aae2656.arrow\n",
- "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight']\n",
+ "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight']\n",
"- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
- "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
+ "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.bias']\n",
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
- ],
- "name": "stderr"
+ ]
},
{
"output_type": "display_data",
@@ -196,7 +196,7 @@
" <div>\n",
" \n",
" <progress value='1875' max='1875' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [1875/1875 05:48, Epoch 3/3]\n",
+ " [1875/1875 05:47, Epoch 3/3]\n",
" </div>\n",
" <table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
@@ -229,11 +229,11 @@
},
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"[[0.02424305 0.9557785 ]\n",
" [0.95884305 0.05541185]]\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -252,7 +252,7 @@
"id": "Q_kAFYd_s_Bi"
},
"source": [
- "text = onnx((model, tokenizer), \"sequence-classification\", \"text-classify.onnx\", quantize=True)"
+ "text = onnx((model, tokenizer), \"text-classification\", \"text-classify.onnx\", quantize=True)"
],
"execution_count": null,
"outputs": []
@@ -291,11 +291,11 @@
{
"cell_type": "code",
"metadata": {
+ "id": "6MBraENcu8Oz",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "6MBraENcu8Oz",
- "outputId": "8399f575-fc1d-4f0f-81a3-977ec8040ac0"
+ "outputId": "96f5e4ad-bb7d-4afe-fb7e-0ae2c76faed5"
},
"source": [
"from sklearn.metrics.pairwise import cosine_similarity\n",
@@ -309,15 +309,15 @@
"\n",
"print(cosine_similarity(outputs))"
],
- "execution_count": 28,
+ "execution_count": 36,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"[[1.0000002 0.8430618]\n",
" [0.8430618 1. ]]\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -332,6 +332,48 @@
"This outputs two embeddings arrays and those arrays are compared using cosine similarity. As we can see, the two text snippets have close semantic meaning."
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "t_OQaQeIb7UB"
+ },
+ "source": [
+ "# Load an ONNX model with txtai\n",
+ "\n",
+ "txtai has built-in support for ONNX models. Loading an ONNX model is seamless and Embeddings and Pipelines support it. The following section shows how to load a classification pipeline and embeddings model backed by ONNX."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "vhsFzCRBby-h",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "54d336ed-e9d6-4f65-969e-1f318cde4757"
+ },
+ "source": [
+ "from txtai.embeddings import Embeddings\n",
+ "from txtai.pipeline import Labels\n",
+ "\n",
+ "labels = Labels((\"text-classify.onnx\", \"google/electra-base-discriminator\"), dynamic=False)\n",
+ "print(labels([\"I am happy\", \"I am mad\"]))\n",
+ "\n",
+ "embeddings = Embeddings({\"path\": \"embeddings.onnx\", \"tokenizer\": \"sentence-transformers/paraphrase-MiniLM-L6-v2\"})\n",
+ "print(embeddings.similarity(\"I am happy\", [\"I am glad\"]))"
+ ],
+ "execution_count": 37,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[[(1, 0.9988517761230469), (0, 0.0011482156114652753)], [(0, 0.997488260269165), (1, 0.0025116782635450363)]]\n",
+ "[(0, 0.8581848740577698)]\n"
+ ]
+ }
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {
@@ -357,17 +399,17 @@
"!mkdir js\n",
"os.chdir(\"/content/js\")"
],
- "execution_count": 29,
+ "execution_count": 38,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
+ "id": "0HtVEl74xrZ7",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "0HtVEl74xrZ7",
- "outputId": "95252125-a83b-4ed4-b02a-3e763ee821fd"
+ "outputId": "b205fb46-b5cd-4461-e81a-d4504767c793"
},
"source": [
"%%writefile package.json\n",
@@ -383,14 +425,14 @@
" }\n",
"}"
],
- "execution_count": 30,
+ "execution_count": 39,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Overwriting package.json\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -424,7 +466,7 @@
"os.chdir(\"/content/js\")\n",
"!npm install"
],
- "execution_count": 31,
+ "execution_count": 40,
"outputs": []
},
{
@@ -439,12 +481,12 @@
{
"cell_type": "code",
"metadata": {
+ "id": "RImohEnFyFg0",
+ "cellView": "form",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "cellView": "form",
- "id": "RImohEnFyFg0",
- "outputId": "2cf7b23a-6625-4859-a193-4cd139c61d71"
+ "outputId": "7409644e-5c18-4596-9197-bc5ff6491a2a"
},
"source": [
"#@title\n",
@@ -537,14 +579,14 @@
"\n",
"main();"
],
- "execution_count": 32,
+ "execution_count": 41,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Overwriting index.js\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -560,25 +602,25 @@
{
"cell_type": "code",
"metadata": {
+ "id": "bdz68KZT1Jfm",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "bdz68KZT1Jfm",
- "outputId": "c4d78358-45cf-4d97-baaa-86906816c882"
+ "outputId": "3b388d33-56c2-4e4e-a872-f484d9e1b847"
},
"source": [
"!node . \"I am happy\"\n",
"!node . \"I am mad\""
],
- "execution_count": 33,
+ "execution_count": 42,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Float32Array(2) [ \u001b[33m0.001104647060856223\u001b[39m, \u001b[33m0.9988954067230225\u001b[39m ]\n",
"Float32Array(2) [ \u001b[33m0.9976443648338318\u001b[39m, \u001b[33m0.00235558208078146\u001b[39m ]\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -607,23 +649,23 @@
{
"cell_type": "code",
"metadata": {
+ "id": "10jcUbUx6MAI",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "10jcUbUx6MAI",
- "outputId": "297af5ff-3ff7-4847-8388-066ae98f7008"
+ "outputId": "aa4ca007-159e-4206-fcc4-af0530959f37"
},
"source": [
"!node . \"I am happy\", \"I am glad\""
],
- "execution_count": 34,
+ "execution_count": 43,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"\u001b[33m0.8414919420066624\u001b[39m\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -676,17 +718,17 @@
"!unzip -o gradle-7.2-bin.zip\n",
"!gradle-7.2/bin/gradle wrapper"
],
- "execution_count": 35,
+ "execution_count": 44,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
+ "id": "gjZ2p7Jf9mOV",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "gjZ2p7Jf9mOV",
- "outputId": "e30eae02-d679-4fa0-aaf8-aeb086ffae50"
+ "outputId": "a5e6483e-322a-42e1-eba8-2cee9c94a39b"
},
"source": [
"%%writefile build.gradle\n",
@@ -717,26 +759,26 @@
" main = \"OnnxDemo\"\n",
"}"
],
- "execution_count": 36,
+ "execution_count": 45,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Overwriting build.gradle\n"
- ],
- "name": "stdout"
+ ]
}
]
},
{
"cell_type": "code",
"metadata": {
+ "cellView": "form",
+ "id": "vnxKGSuz_fnj",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "cellView": "form",
- "id": "vnxKGSuz_fnj",
- "outputId": "df79ecd8-726a-417a-8836-b4b73693a5ad"
+ "outputId": "ae79fd32-c6f3-49ec-fcd3-7d09a5d2997a"
},
"source": [
"#@title\n",
@@ -879,14 +921,14 @@
" }\n",
"}"
],
- "execution_count": 37,
+ "execution_count": 46,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Overwriting src/main/java/OnnxDemo.java\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -902,26 +944,26 @@
{
"cell_type": "code",
"metadata": {
+ "id": "hFXyH96gAZpu",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "hFXyH96gAZpu",
- "outputId": "c58d464b-c40b-44b4-eea0-b2593d466abb"
+ "outputId": "78701d6e-e4e7-4e5c-da2f-a29695030362"
},
"source": [
"!./gradlew -q --console=plain onnx --args='\"I am happy\"' 2> /dev/null\n",
"!./gradlew -q --console=plain onnx --args='\"I am mad\"' 2> /dev/null"
],
- "execution_count": 38,
+ "execution_count": 47,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"[0.0011046471, 0.99889535]\n",
"\u001b[m[0.9976444, 0.002355582]\n",
"\u001b[m"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -948,24 +990,24 @@
{
"cell_type": "code",
"metadata": {
+ "id": "f6zE9VrwCcUa",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "f6zE9VrwCcUa",
- "outputId": "ae83bb5f-4f36-4f6f-cc6b-2b2538dddeb5"
+ "outputId": "d6b304f3-151d-4ef0-978a-19e51e2b3fe0"
},
"source": [
"!./gradlew -q --console=plain onnx --args='\"I am happy\" \"I am glad\"' 2> /dev/null"
],
- "execution_count": 39,
+ "execution_count": 48,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"0.8581848568615768\n",
"\u001b[m"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -1016,17 +1058,17 @@
"\n",
"!mkdir -p src"
],
- "execution_count": 40,
+ "execution_count": 49,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
+ "id": "c7hz--Gne6Oa",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "c7hz--Gne6Oa",
- "outputId": "e50ea1d0-e105-4d8b-cd50-3f7b6b6048c1"
+ "outputId": "39635f79-2071-403c-f833-9152907c2d1a"
},
"source": [
"%%writefile Cargo.toml\n",
@@ -1042,26 +1084,26 @@
"onnxruntime = { version = \"0.0.14\"}\n",
"tokenizers = { version = \"0.10.1\"}"
],
- "execution_count": 41,
+ "execution_count": 50,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Overwriting Cargo.toml\n"
- ],
- "name": "stdout"
+ ]
}
]
},
{
"cell_type": "code",
"metadata": {
+ "cellView": "form",
+ "id": "_8fdRvO1fFBm",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "cellView": "form",
- "id": "_8fdRvO1fFBm",
- "outputId": "b4eec92e-e9fc-4a4f-f7ce-41a2c8f0d7e0"
+ "outputId": "79407b9d-8934-4cf2-c3cf-21e0a68954ca"
},
"source": [
"#@title\n",
@@ -1186,14 +1228,14 @@
" Ok(())\n",
"}"
],
- "execution_count": 42,
+ "execution_count": 51,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"Overwriting src/main.rs\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -1209,25 +1251,25 @@
{
"cell_type": "code",
"metadata": {
+ "id": "b0ymX4ftgWcT",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "b0ymX4ftgWcT",
- "outputId": "e7995455-1d89-4af5-a502-a2d4e41f27b2"
+ "outputId": "3446c3e7-a5d4-4572-ad9e-f463a1d4a7d7"
},
"source": [
"!cargo run \"I am happy\" 2> /dev/null\n",
"!cargo run \"I am mad\" 2> /dev/null"
],
- "execution_count": 43,
+ "execution_count": 52,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"[0.0011003953, 0.99889964]\n",
"[0.9976444, 0.0023555849]\n"
- ],
- "name": "stdout"
+ ]
}
]
},
@@ -1254,23 +1296,23 @@
{
"cell_type": "code",
"metadata": {
+ "id": "A9p6F_ODhenH",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "id": "A9p6F_ODhenH",
- "outputId": "486e358c-955f-44c6-8d7f-d22b12b1417d"
+ "outputId": "3dfff62e-d910-433f-c8de-c0b2311141a6"
},
"source": [
"!cargo run \"I am happy\" \"I am glad\" 2> /dev/null"
],
- "execution_count": 44,
+ "execution_count": 53,
"outputs": [
{
"output_type": "stream",
+ "name": "stdout",
"text": [
"0.8583641740656903\n"
- ],
- "name": "stdout"
+ ]
}
]
},