💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update notebook
davidmezzettia0ab014e9608
diff --git a/examples/18_Export_and_run_models_with_ONNX.ipynb b/examples/18_Export_and_run_models_with_ONNX.ipynbindex 2a9b88c6..af7eba82 100644--- a/examples/18_Export_and_run_models_with_ONNX.ipynb+++ b/examples/18_Export_and_run_models_with_ONNX.ipynb@@ -51,7 +51,7 @@"%%capture\n","!pip install datasets git+https://github.com/neuml/txtai#egg=txtai[pipeline]"],- "execution_count": 23,+ "execution_count": 31,"outputs": []},{@@ -73,7 +73,7 @@"colab": {"base_uri": "https://localhost:8080/"},- "outputId": "367c3baf-5e9d-4992-8e12-b517fff6688d"+ "outputId": "b32db17b-2563-4960-c1bc-fc1193172ea4"},"source": ["import numpy as np\n",@@ -87,7 +87,7 @@"\n","# Export to ONNX\n","onnx = HFOnnx()\n",- "model = onnx(\"distilbert-base-uncased-finetuned-sst-2-english\", \"sequence-classification\")\n",+ "model = onnx(\"distilbert-base-uncased-finetuned-sst-2-english\", \"text-classification\")\n","\n","# Start inference session\n","options = SessionOptions()\n",@@ -101,15 +101,15 @@"outputs = session.run(None, dict(tokens))\n","print(sigmoid(outputs[0]))"],- "execution_count": 24,+ "execution_count": 32,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["[[0.01295124 0.9909526 ]\n"," [0.9874723 0.0297817 ]]\n"- ],- "name": "stdout"+ ]}]},@@ -143,7 +143,7 @@"height": 398},"id": "Wh8TkszumlIe",- "outputId": "c1556a18-7ba6-44ec-b80f-217d115c90b1"+ "outputId": "f1fd81e5-5e4b-4a46-9ed7-33aacd20b862"},"source": ["from datasets import load_dataset\n",@@ -159,7 +159,7 @@"model, tokenizer = trainer(\"google/electra-base-discriminator\", data, columns=(\"sentence\", \"label\"))\n","\n","# Export model trained in-memory to ONNX (still in-memory)\n",- "output = onnx((model, tokenizer), \"sequence-classification\", quantize=True)\n",+ "output = onnx((model, tokenizer), \"text-classification\", quantize=True)\n","\n","# Start inference session\n","options = SessionOptions()\n",@@ -172,21 +172,21 @@"outputs = session.run(None, dict(tokens))\n","print(sigmoid(outputs[0]))"],- "execution_count": 25,+ "execution_count": 33,"outputs": [{"output_type": "stream",+ "name": "stderr","text": ["Reusing dataset glue (/root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n","Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-0ad799e068ae6c27.arrow\n","Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-422ba3363aae2656.arrow\n",- "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight']\n",+ "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight']\n","- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",- "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",+ "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.bias']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"- ],- "name": "stderr"+ ]},{"output_type": "display_data",@@ -196,7 +196,7 @@" <div>\n"," \n"," <progress value='1875' max='1875' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",- " [1875/1875 05:48, Epoch 3/3]\n",+ " [1875/1875 05:47, Epoch 3/3]\n"," </div>\n"," <table border=\"1\" class=\"dataframe\">\n"," <thead>\n",@@ -229,11 +229,11 @@},{"output_type": "stream",+ "name": "stdout","text": ["[[0.02424305 0.9557785 ]\n"," [0.95884305 0.05541185]]\n"- ],- "name": "stdout"+ ]}]},@@ -252,7 +252,7 @@"id": "Q_kAFYd_s_Bi"},"source": [- "text = onnx((model, tokenizer), \"sequence-classification\", \"text-classify.onnx\", quantize=True)"+ "text = onnx((model, tokenizer), \"text-classification\", \"text-classify.onnx\", quantize=True)"],"execution_count": null,"outputs": []@@ -291,11 +291,11 @@{"cell_type": "code","metadata": {+ "id": "6MBraENcu8Oz","colab": {"base_uri": "https://localhost:8080/"},- "id": "6MBraENcu8Oz",- "outputId": "8399f575-fc1d-4f0f-81a3-977ec8040ac0"+ "outputId": "96f5e4ad-bb7d-4afe-fb7e-0ae2c76faed5"},"source": ["from sklearn.metrics.pairwise import cosine_similarity\n",@@ -309,15 +309,15 @@"\n","print(cosine_similarity(outputs))"],- "execution_count": 28,+ "execution_count": 36,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["[[1.0000002 0.8430618]\n"," [0.8430618 1. ]]\n"- ],- "name": "stdout"+ ]}]},@@ -332,6 +332,48 @@"This outputs two embeddings arrays and those arrays are compared using cosine similarity. As we can see, the two text snippets have close semantic meaning."]},+ {+ "cell_type": "markdown",+ "metadata": {+ "id": "t_OQaQeIb7UB"+ },+ "source": [+ "# Load an ONNX model with txtai\n",+ "\n",+ "txtai has built-in support for ONNX models. Loading an ONNX model is seamless and Embeddings and Pipelines support it. The following section shows how to load a classification pipeline and embeddings model backed by ONNX."+ ]+ },+ {+ "cell_type": "code",+ "metadata": {+ "id": "vhsFzCRBby-h",+ "colab": {+ "base_uri": "https://localhost:8080/"+ },+ "outputId": "54d336ed-e9d6-4f65-969e-1f318cde4757"+ },+ "source": [+ "from txtai.embeddings import Embeddings\n",+ "from txtai.pipeline import Labels\n",+ "\n",+ "labels = Labels((\"text-classify.onnx\", \"google/electra-base-discriminator\"), dynamic=False)\n",+ "print(labels([\"I am happy\", \"I am mad\"]))\n",+ "\n",+ "embeddings = Embeddings({\"path\": \"embeddings.onnx\", \"tokenizer\": \"sentence-transformers/paraphrase-MiniLM-L6-v2\"})\n",+ "print(embeddings.similarity(\"I am happy\", [\"I am glad\"]))"+ ],+ "execution_count": 37,+ "outputs": [+ {+ "output_type": "stream",+ "name": "stdout",+ "text": [+ "[[(1, 0.9988517761230469), (0, 0.0011482156114652753)], [(0, 0.997488260269165), (1, 0.0025116782635450363)]]\n",+ "[(0, 0.8581848740577698)]\n"+ ]+ }+ ]+ },{"cell_type": "markdown","metadata": {@@ -357,17 +399,17 @@"!mkdir js\n","os.chdir(\"/content/js\")"],- "execution_count": 29,+ "execution_count": 38,"outputs": []},{"cell_type": "code","metadata": {+ "id": "0HtVEl74xrZ7","colab": {"base_uri": "https://localhost:8080/"},- "id": "0HtVEl74xrZ7",- "outputId": "95252125-a83b-4ed4-b02a-3e763ee821fd"+ "outputId": "b205fb46-b5cd-4461-e81a-d4504767c793"},"source": ["%%writefile package.json\n",@@ -383,14 +425,14 @@" }\n","}"],- "execution_count": 30,+ "execution_count": 39,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Overwriting package.json\n"- ],- "name": "stdout"+ ]}]},@@ -424,7 +466,7 @@"os.chdir(\"/content/js\")\n","!npm install"],- "execution_count": 31,+ "execution_count": 40,"outputs": []},{@@ -439,12 +481,12 @@{"cell_type": "code","metadata": {+ "id": "RImohEnFyFg0",+ "cellView": "form","colab": {"base_uri": "https://localhost:8080/"},- "cellView": "form",- "id": "RImohEnFyFg0",- "outputId": "2cf7b23a-6625-4859-a193-4cd139c61d71"+ "outputId": "7409644e-5c18-4596-9197-bc5ff6491a2a"},"source": ["#@title\n",@@ -537,14 +579,14 @@"\n","main();"],- "execution_count": 32,+ "execution_count": 41,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Overwriting index.js\n"- ],- "name": "stdout"+ ]}]},@@ -560,25 +602,25 @@{"cell_type": "code","metadata": {+ "id": "bdz68KZT1Jfm","colab": {"base_uri": "https://localhost:8080/"},- "id": "bdz68KZT1Jfm",- "outputId": "c4d78358-45cf-4d97-baaa-86906816c882"+ "outputId": "3b388d33-56c2-4e4e-a872-f484d9e1b847"},"source": ["!node . \"I am happy\"\n","!node . \"I am mad\""],- "execution_count": 33,+ "execution_count": 42,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Float32Array(2) [ \u001b[33m0.001104647060856223\u001b[39m, \u001b[33m0.9988954067230225\u001b[39m ]\n","Float32Array(2) [ \u001b[33m0.9976443648338318\u001b[39m, \u001b[33m0.00235558208078146\u001b[39m ]\n"- ],- "name": "stdout"+ ]}]},@@ -607,23 +649,23 @@{"cell_type": "code","metadata": {+ "id": "10jcUbUx6MAI","colab": {"base_uri": "https://localhost:8080/"},- "id": "10jcUbUx6MAI",- "outputId": "297af5ff-3ff7-4847-8388-066ae98f7008"+ "outputId": "aa4ca007-159e-4206-fcc4-af0530959f37"},"source": ["!node . \"I am happy\", \"I am glad\""],- "execution_count": 34,+ "execution_count": 43,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["\u001b[33m0.8414919420066624\u001b[39m\n"- ],- "name": "stdout"+ ]}]},@@ -676,17 +718,17 @@"!unzip -o gradle-7.2-bin.zip\n","!gradle-7.2/bin/gradle wrapper"],- "execution_count": 35,+ "execution_count": 44,"outputs": []},{"cell_type": "code","metadata": {+ "id": "gjZ2p7Jf9mOV","colab": {"base_uri": "https://localhost:8080/"},- "id": "gjZ2p7Jf9mOV",- "outputId": "e30eae02-d679-4fa0-aaf8-aeb086ffae50"+ "outputId": "a5e6483e-322a-42e1-eba8-2cee9c94a39b"},"source": ["%%writefile build.gradle\n",@@ -717,26 +759,26 @@" main = \"OnnxDemo\"\n","}"],- "execution_count": 36,+ "execution_count": 45,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Overwriting build.gradle\n"- ],- "name": "stdout"+ ]}]},{"cell_type": "code","metadata": {+ "cellView": "form",+ "id": "vnxKGSuz_fnj","colab": {"base_uri": "https://localhost:8080/"},- "cellView": "form",- "id": "vnxKGSuz_fnj",- "outputId": "df79ecd8-726a-417a-8836-b4b73693a5ad"+ "outputId": "ae79fd32-c6f3-49ec-fcd3-7d09a5d2997a"},"source": ["#@title\n",@@ -879,14 +921,14 @@" }\n","}"],- "execution_count": 37,+ "execution_count": 46,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Overwriting src/main/java/OnnxDemo.java\n"- ],- "name": "stdout"+ ]}]},@@ -902,26 +944,26 @@{"cell_type": "code","metadata": {+ "id": "hFXyH96gAZpu","colab": {"base_uri": "https://localhost:8080/"},- "id": "hFXyH96gAZpu",- "outputId": "c58d464b-c40b-44b4-eea0-b2593d466abb"+ "outputId": "78701d6e-e4e7-4e5c-da2f-a29695030362"},"source": ["!./gradlew -q --console=plain onnx --args='\"I am happy\"' 2> /dev/null\n","!./gradlew -q --console=plain onnx --args='\"I am mad\"' 2> /dev/null"],- "execution_count": 38,+ "execution_count": 47,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["[0.0011046471, 0.99889535]\n","\u001b[m[0.9976444, 0.002355582]\n","\u001b[m"- ],- "name": "stdout"+ ]}]},@@ -948,24 +990,24 @@{"cell_type": "code","metadata": {+ "id": "f6zE9VrwCcUa","colab": {"base_uri": "https://localhost:8080/"},- "id": "f6zE9VrwCcUa",- "outputId": "ae83bb5f-4f36-4f6f-cc6b-2b2538dddeb5"+ "outputId": "d6b304f3-151d-4ef0-978a-19e51e2b3fe0"},"source": ["!./gradlew -q --console=plain onnx --args='\"I am happy\" \"I am glad\"' 2> /dev/null"],- "execution_count": 39,+ "execution_count": 48,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["0.8581848568615768\n","\u001b[m"- ],- "name": "stdout"+ ]}]},@@ -1016,17 +1058,17 @@"\n","!mkdir -p src"],- "execution_count": 40,+ "execution_count": 49,"outputs": []},{"cell_type": "code","metadata": {+ "id": "c7hz--Gne6Oa","colab": {"base_uri": "https://localhost:8080/"},- "id": "c7hz--Gne6Oa",- "outputId": "e50ea1d0-e105-4d8b-cd50-3f7b6b6048c1"+ "outputId": "39635f79-2071-403c-f833-9152907c2d1a"},"source": ["%%writefile Cargo.toml\n",@@ -1042,26 +1084,26 @@"onnxruntime = { version = \"0.0.14\"}\n","tokenizers = { version = \"0.10.1\"}"],- "execution_count": 41,+ "execution_count": 50,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Overwriting Cargo.toml\n"- ],- "name": "stdout"+ ]}]},{"cell_type": "code","metadata": {+ "cellView": "form",+ "id": "_8fdRvO1fFBm","colab": {"base_uri": "https://localhost:8080/"},- "cellView": "form",- "id": "_8fdRvO1fFBm",- "outputId": "b4eec92e-e9fc-4a4f-f7ce-41a2c8f0d7e0"+ "outputId": "79407b9d-8934-4cf2-c3cf-21e0a68954ca"},"source": ["#@title\n",@@ -1186,14 +1228,14 @@" Ok(())\n","}"],- "execution_count": 42,+ "execution_count": 51,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["Overwriting src/main.rs\n"- ],- "name": "stdout"+ ]}]},@@ -1209,25 +1251,25 @@{"cell_type": "code","metadata": {+ "id": "b0ymX4ftgWcT","colab": {"base_uri": "https://localhost:8080/"},- "id": "b0ymX4ftgWcT",- "outputId": "e7995455-1d89-4af5-a502-a2d4e41f27b2"+ "outputId": "3446c3e7-a5d4-4572-ad9e-f463a1d4a7d7"},"source": ["!cargo run \"I am happy\" 2> /dev/null\n","!cargo run \"I am mad\" 2> /dev/null"],- "execution_count": 43,+ "execution_count": 52,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["[0.0011003953, 0.99889964]\n","[0.9976444, 0.0023555849]\n"- ],- "name": "stdout"+ ]}]},@@ -1254,23 +1296,23 @@{"cell_type": "code","metadata": {+ "id": "A9p6F_ODhenH","colab": {"base_uri": "https://localhost:8080/"},- "id": "A9p6F_ODhenH",- "outputId": "486e358c-955f-44c6-8d7f-d22b12b1417d"+ "outputId": "3dfff62e-d910-433f-c8de-c0b2311141a6"},"source": ["!cargo run \"I am happy\" \"I am glad\" 2> /dev/null"],- "execution_count": 44,+ "execution_count": 53,"outputs": [{"output_type": "stream",+ "name": "stdout","text": ["0.8583641740656903\n"- ],- "name": "stdout"+ ]}]},