Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix various issues with notebook, closes #369

davidmezzettibbd16ede2bec
diff --git a/examples/18_Export_and_run_models_with_ONNX.ipynb b/examples/18_Export_and_run_models_with_ONNX.ipynb
index 0b2c4a39..21fbaf92 100644
--- a/examples/18_Export_and_run_models_with_ONNX.ipynb
+++ b/examples/18_Export_and_run_models_with_ONNX.ipynb
@@ -3,7 +3,6 @@
"nbformat_minor": 0,
"metadata": {
"colab": {
- "name": "18 - Export and run models with ONNX",
"provenance": [],
"collapsed_sections": []
},
@@ -53,7 +52,7 @@
"%%capture\n",
"!pip install datasets git+https://github.com/neuml/txtai#egg=txtai[pipeline]"
],
- "execution_count": null,
+ "execution_count": 25,
"outputs": []
},
{
@@ -75,7 +74,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "b32db17b-2563-4960-c1bc-fc1193172ea4"
+ "outputId": "28d3e70e-efa9-4b07-a602-6ffd89d1279f"
},
"source": [
"import numpy as np\n",
@@ -103,7 +102,7 @@
"outputs = session.run(None, dict(tokens))\n",
"print(sigmoid(outputs[0]))"
],
- "execution_count": null,
+ "execution_count": 26,
"outputs": [
{
"output_type": "stream",
@@ -142,10 +141,10 @@
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
- "height": 398
+ "height": 579
},
"id": "Wh8TkszumlIe",
- "outputId": "f1fd81e5-5e4b-4a46-9ed7-33aacd20b862"
+ "outputId": "864f2074-ae50-40d6-bc34-2b1d86a71488"
},
"source": [
"from datasets import load_dataset\n",
@@ -155,9 +154,9 @@
"\n",
"# Hugging Face dataset\n",
"ds = load_dataset(\"glue\", \"sst2\")\n",
- "data = ds[\"train\"].select(range(5000)).flatten_indices()\n",
+ "data = ds[\"train\"].select(range(10000)).flatten_indices()\n",
"\n",
- "# Train new model using 5,000 SST2 records (in-memory)\n",
+ "# Train new model using 10,000 SST2 records (in-memory)\n",
"model, tokenizer = trainer(\"google/electra-base-discriminator\", data, columns=(\"sentence\", \"label\"))\n",
"\n",
"# Export model trained in-memory to ONNX (still in-memory)\n",
@@ -174,20 +173,22 @@
"outputs = session.run(None, dict(tokens))\n",
"print(sigmoid(outputs[0]))"
],
- "execution_count": null,
+ "execution_count": 27,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
- "Reusing dataset glue (/root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n",
- "Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-0ad799e068ae6c27.arrow\n",
- "Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-422ba3363aae2656.arrow\n",
- "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight']\n",
+ "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-e28d0e20a676bad0.arrow\n",
+ "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-d7b5d80ca22204f9.arrow\n",
+ "Some weights of the model checkpoint at google/electra-base-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense_prediction.bias', 'discriminator_predictions.dense.weight', 'discriminator_predictions.dense.bias']\n",
"- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
- "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.bias']\n",
- "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+ "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-base-discriminator and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.dense.bias']\n",
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+ "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+ " FutureWarning,\n",
+ "You're using a ElectraTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
]
},
{
@@ -197,12 +198,12 @@
"\n",
" <div>\n",
" \n",
- " <progress value='1875' max='1875' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
- " [1875/1875 05:47, Epoch 3/3]\n",
+ " <progress value='3750' max='3750' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+ " [3750/3750 07:56, Epoch 3/3]\n",
" </div>\n",
" <table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
- " <tr style=\"text-align: left;\">\n",
+ " <tr style=\"text-align: left;\">\n",
" <th>Step</th>\n",
" <th>Training Loss</th>\n",
" </tr>\n",
@@ -210,21 +211,34 @@
" <tbody>\n",
" <tr>\n",
" <td>500</td>\n",
- " <td>0.409200</td>\n",
+ " <td>0.396800</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1000</td>\n",
- " <td>0.269900</td>\n",
+ " <td>0.330900</td>\n",
" </tr>\n",
" <tr>\n",
" <td>1500</td>\n",
- " <td>0.153200</td>\n",
+ " <td>0.232400</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2000</td>\n",
+ " <td>0.188200</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2500</td>\n",
+ " <td>0.173600</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3000</td>\n",
+ " <td>0.068600</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3500</td>\n",
+ " <td>0.069800</td>\n",
" </tr>\n",
" </tbody>\n",
"</table><p>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
]
},
"metadata": {}
@@ -233,8 +247,8 @@
"output_type": "stream",
"name": "stdout",
"text": [
- "[[0.02424305 0.9557785 ]\n",
- " [0.95884305 0.05541185]]\n"
+ "[[0.01525715 0.975399 ]\n",
+ " [0.97395283 0.04432926]]\n"
]
}
]
@@ -254,9 +268,10 @@
"id": "Q_kAFYd_s_Bi"
},
"source": [
+ "onnx = HFOnnx()\n",
"text = onnx((model, tokenizer), \"text-classification\", \"text-classify.onnx\", quantize=True)"
],
- "execution_count": null,
+ "execution_count": 29,
"outputs": []
},
{
@@ -278,7 +293,7 @@
"source": [
"embeddings = onnx(\"sentence-transformers/paraphrase-MiniLM-L6-v2\", \"pooling\", \"embeddings.onnx\", quantize=True)"
],
- "execution_count": null,
+ "execution_count": 30,
"outputs": []
},
{
@@ -297,7 +312,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "96f5e4ad-bb7d-4afe-fb7e-0ae2c76faed5"
+ "outputId": "ba4528d2-6d6a-4181-e9c4-3d2a98d6663a"
},
"source": [
"from sklearn.metrics.pairwise import cosine_similarity\n",
@@ -311,14 +326,14 @@
"\n",
"print(cosine_similarity(outputs))"
],
- "execution_count": null,
+ "execution_count": 31,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[[1.0000002 0.8430618]\n",
- " [0.8430618 1. ]]\n"
+ "[[0.99999994 0.8474637 ]\n",
+ " [0.8474637 0.9999997 ]]\n"
]
}
]
@@ -352,7 +367,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "54d336ed-e9d6-4f65-969e-1f318cde4757"
+ "outputId": "745d69b0-035b-4e44-a881-57f9ece171ab"
},
"source": [
"from txtai.embeddings import Embeddings\n",
@@ -364,14 +379,14 @@
"embeddings = Embeddings({\"path\": \"embeddings.onnx\", \"tokenizer\": \"sentence-transformers/paraphrase-MiniLM-L6-v2\"})\n",
"print(embeddings.similarity(\"I am happy\", [\"I am glad\"]))"
],
- "execution_count": null,
+ "execution_count": 32,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[[(1, 0.9988517761230469), (0, 0.0011482156114652753)], [(0, 0.997488260269165), (1, 0.0025116782635450363)]]\n",
- "[(0, 0.8581848740577698)]\n"
+ "[[(1, 0.999687910079956), (0, 0.0003121310146525502)], [(0, 0.9991233944892883), (1, 0.0008765518432483077)]]\n",
+ "[(0, 0.8298245072364807)]\n"
]
}
]
@@ -398,10 +413,29 @@
"%%capture\n",
"import os\n",
"\n",
+ "os.chdir(\"/content\")\n",
"!mkdir js\n",
+ "os.chdir(\"/content/js\")\n",
+ "\n",
+ "# Copy ONNX models\n",
+ "!cp ../text-classify.onnx .\n",
+ "!cp ../embeddings.onnx .\n",
+ "\n",
+ "# Get tokenizers project\n",
+ "!git clone https://github.com/huggingface/tokenizers.git\n",
+ "\n",
+ "os.chdir(\"/content/js/tokenizers/bindings/node\")\n",
+ "\n",
+ "# Install Rust to compile tokenizer bindings\n",
+ "!apt-get install rustc cargo\n",
+ "\n",
+ "# Build tokenizers package locally as binary version on npm doesn't work for latest version of Node.js\n",
+ "!npm install --also=dev\n",
+ "!npm run dev\n",
+ "\n",
"os.chdir(\"/content/js\")"
],
- "execution_count": null,
+ "execution_count": 33,
"outputs": []
},
{
@@ -411,7 +445,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "b205fb46-b5cd-4461-e81a-d4504767c793"
+ "outputId": "7ffea868-6dd7-4603-e04c-ce8d4c557ff6"
},
"source": [
"%%writefile package.json\n",
@@ -422,53 +456,34 @@
" \"description\": \"ONNX Runtime Node.js test\",\n",
" \"main\": \"index.js\",\n",
" \"dependencies\": {\n",
- " \"onnxruntime-node\": \">=1.8.0\",\n",
+ " \"onnxruntime-node\": \">=1.12.1\",\n",
" \"tokenizers\": \"file:tokenizers/bindings/node\"\n",
" }\n",
"}"
],
- "execution_count": null,
+ "execution_count": 34,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Overwriting package.json\n"
+ "Writing package.json\n"
]
}
]
},
{
"cell_type": "code",
- "metadata": {
- "id": "NCfV3GL0wt65"
- },
"source": [
"%%capture\n",
- "# Copy ONNX models\n",
- "!cp ../text-classify.onnx .\n",
- "!cp ../embeddings.onnx .\n",
- "\n",
- "# Save copy of Bert Tokenizer\n",
- "tokenizer.save_pretrained(\"bert\")\n",
- "\n",
- "# Get tokenizers project\n",
- "!git clone https://github.com/huggingface/tokenizers.git\n",
- "\n",
- "os.chdir(\"/content/js/tokenizers/bindings/node\")\n",
- "\n",
- "# Install Rust\n",
- "!apt-get install rustc\n",
- "\n",
- "# Build tokenizers project locally as version on NPM isn't working properly for latest version of Node.js\n",
- "!npm install --also=dev\n",
- "!npm run dev\n",
"\n",
"# Install all dependencies\n",
- "os.chdir(\"/content/js\")\n",
"!npm install"
],
- "execution_count": null,
+ "metadata": {
+ "id": "4naPtk-iBI-g"
+ },
+ "execution_count": 35,
"outputs": []
},
{
@@ -484,11 +499,11 @@
"cell_type": "code",
"metadata": {
"id": "RImohEnFyFg0",
- "cellView": "form",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "7409644e-5c18-4596-9197-bc5ff6491a2a"
+ "outputId": "094ef937-0650-4d5b-f4e3-79f114bd9807",
+ "cellView": "form"
},
"source": [
"#@title\n",
@@ -519,15 +534,15 @@
" return dot / (Math.sqrt(norm1) * Math.sqrt(norm2));\n",
"}\n",
"\n",
- "function tokenizer(path) {\n",
- " let tokenizer = Tokenizer.fromFile(path);\n",
+ "function tokenizer() {\n",
+ " let tokenizer = Tokenizer.fromPretrained(\"bert-base-uncased\");\n",
" return promisify(tokenizer.encode.bind(tokenizer));\n",
"}\n",
"\n",
"async function predict(session, text) {\n",
" try {\n",
" // Tokenize input\n",
- " let encode = tokenizer(\"bert/tokenizer.json\");\n",
+ " let encode = tokenizer();\n",
" let output = await encode(text);\n",
"\n",
" let ids = output.getIds().map(x => BigInt(x))\n",
@@ -581,13 +596,13 @@
"\n",
"main();"
],
- "execution_count": null,
+ "execution_count": 36,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Overwriting index.js\n"
+ "Writing index.js\n"
]
}
]
@@ -608,20 +623,20 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "3b388d33-56c2-4e4e-a872-f484d9e1b847"
+ "outputId": "48c4a427-3108-436c-eac9-564835ea061c"
},
"source": [
"!node . \"I am happy\"\n",
"!node . \"I am mad\""
],
- "execution_count": null,
+ "execution_count": 37,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Float32Array(2) [ \u001b[33m0.001104647060856223\u001b[39m, \u001b[33m0.9988954067230225\u001b[39m ]\n",
- "Float32Array(2) [ \u001b[33m0.9976443648338318\u001b[39m, \u001b[33m0.00235558208078146\u001b[39m ]\n"
+ "Float32Array(2) [ \u001b[33m0.0003121308400295675\u001b[39m, \u001b[33m0.9996878504753113\u001b[39m ]\n",
+ "Float32Array(2) [ \u001b[33m0.9991234540939331\u001b[39m, \u001b[33m0.0008765519596636295\u001b[39m ]\n"
]
}
]
@@ -655,18 +670,18 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "aa4ca007-159e-4206-fcc4-af0530959f37"
+ "outputId": "ac751cee-5f44-4dad-c164-5d124be75ec3"
},
"source": [
"!node . \"I am happy\", \"I am glad\""
],
- "execution_count": null,
+ "execution_count": 38,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "\u001b[33m0.8414919420066624\u001b[39m\n"
+ "\u001b[33m0.8285076844387538\u001b[39m\n"
]
}
]
@@ -695,9 +710,6 @@
},
{
"cell_type": "code",
- "metadata": {
- "id": "9wlVWVky9NZ3"
- },
"source": [
"%%capture\n",
"import os\n",
@@ -716,11 +728,13 @@
"!mkdir -p src/main/java\n",
"\n",
"# Install gradle\n",
- "!wget https://services.gradle.org/distributions/gradle-7.2-bin.zip\n",
- "!unzip -o gradle-7.2-bin.zip\n",
- "!gradle-7.2/bin/gradle wrapper"
+ "!wget https://services.gradle.org/distributions/gradle-7.5.1-bin.zip\n",
+ "!unzip -o gradle-7.5.1-bin.zip"
],
- "execution_count": null,
+ "metadata": {
+ "id": "L1YMoO7WwkEk"
+ },
+ "execution_count": 39,
"outputs": []
},
{
@@ -730,7 +744,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "a5e6483e-322a-42e1-eba8-2cee9c94a39b"
+ "outputId": "0cc3f4ef-bac6-4c13-dc6b-8482353bb741"
},
"source": [
"%%writefile build.gradle\n",
@@ -742,7 +756,7 @@
"\n",
"dependencies {\n",
" implementation \"com.robrua.nlp:easy-bert:1.0.3\"\n",
- " implementation \"com.microsoft.onnxruntime:onnxruntime:1.8.1\"\n",
+ " implementation \"com.microsoft.onnxruntime:onnxruntime:1.12.1\"\n",
"}\n",
"\n",
"java {\n",
@@ -761,13 +775,13 @@
" main = \"OnnxDemo\"\n",
"}"
],
- "execution_count": null,
+ "execution_count": 40,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Overwriting build.gradle\n"
+ "Writing build.gradle\n"
]
}
]
@@ -775,12 +789,26 @@
{
"cell_type": "code",
"metadata": {
- "cellView": "form",
+ "id": "9wlVWVky9NZ3"
+ },
+ "source": [
+ "%%capture\n",
+ "\n",
+ "# Create environment\n",
+ "!gradle-7.5.1/bin/gradle wrapper"
+ ],
+ "execution_count": 41,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
"id": "vnxKGSuz_fnj",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "ae79fd32-c6f3-49ec-fcd3-7d09a5d2997a"
+ "outputId": "c1b73bf8-c5f4-46fa-df90-29bd60507287",
+ "cellView": "form"
},
"source": [
"#@title\n",
@@ -923,13 +951,13 @@
" }\n",
"}"
],
- "execution_count": null,
+ "execution_count": 42,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Overwriting src/main/java/OnnxDemo.java\n"
+ "Writing src/main/java/OnnxDemo.java\n"
]
}
]
@@ -950,20 +978,20 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "78701d6e-e4e7-4e5c-da2f-a29695030362"
+ "outputId": "efd5b783-b23a-407c-8577-c18e3a6cb984"
},
"source": [
"!./gradlew -q --console=plain onnx --args='\"I am happy\"' 2> /dev/null\n",
"!./gradlew -q --console=plain onnx --args='\"I am mad\"' 2> /dev/null"
],
- "execution_count": null,
+ "execution_count": 43,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[0.0011046471, 0.99889535]\n",
- "\u001b[m[0.9976444, 0.002355582]\n",
+ "[3.1213084E-4, 0.99968785]\n",
+ "\u001b[m[0.99912345, 8.7655196E-4]\n",
"\u001b[m"
]
}
@@ -996,18 +1024,18 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "d6b304f3-151d-4ef0-978a-19e51e2b3fe0"
+ "outputId": "988e59d0-943f-45b6-d37e-5fc1ebbbcefe"
},
"source": [
"!./gradlew -q --console=plain onnx --args='\"I am happy\" \"I am glad\"' 2> /dev/null"
],
- "execution_count": null,
+ "execution_count": 44,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "0.8581848568615768\n",
+ "0.8298244656285757\n",
"\u001b[m"
]
}
@@ -1052,15 +1080,12 @@
"!cp ../text-classify.onnx .\n",
"!cp ../embeddings.onnx .\n",
"\n",
- "# Save copy of Bert Tokenizer\n",
- "tokenizer.save_pretrained(\"bert\")\n",
- "\n",
"# Install Rust\n",
- "!apt-get install rustc\n",
+ "!apt-get install rustc cargo\n",
"\n",
"!mkdir -p src"
],
- "execution_count": null,
+ "execution_count": 45,
"outputs": []
},
{
@@ -1070,7 +1095,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "39635f79-2071-403c-f833-9152907c2d1a"
+ "outputId": "d98ad709-5675-4193-e598-e6cbe12edda3"
},
"source": [
"%%writefile Cargo.toml\n",
@@ -1084,15 +1109,15 @@
"\n",
"[dependencies]\n",
"onnxruntime = { version = \"0.0.14\"}\n",
- "tokenizers = { version = \"0.10.1\"}"
+ "tokenizers = { version = \"0.13.1\"}"
],
- "execution_count": null,
+ "execution_count": 46,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Overwriting Cargo.toml\n"
+ "Writing Cargo.toml\n"
]
}
]
@@ -1100,12 +1125,12 @@
{
"cell_type": "code",
"metadata": {
- "cellView": "form",
"id": "_8fdRvO1fFBm",
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "79407b9d-8934-4cf2-c3cf-21e0a68954ca"
+ "outputId": "53168684-12c2-46fc-e0e6-54c4c6d03cb1",
+ "cellView": "form"
},
"source": [
"#@title\n",
@@ -1117,37 +1142,14 @@
"\n",
"use std::env;\n",
"\n",
- "use tokenizers::decoders::wordpiece::WordPiece as WordPieceDecoder;\n",
- "use tokenizers::models::wordpiece::WordPiece;\n",
- "use tokenizers::normalizers::bert::BertNormalizer;\n",
- "use tokenizers::pre_tokenizers::bert::BertPreTokenizer;\n",
- "use tokenizers::processors::bert::BertProcessing;\n",
- "use tokenizers::tokenizer::{Result, Tokenizer, EncodeInput};\n",
+ "use tokenizers::tokenizer::{Result, Tokenizer};\n",
"\n",
"fn tokenize(text: String, inputs: usize) -> Vec<Array2<i64>> {\n",
- " // Load tokenizer\n",
- " let mut tokenizer = Tokenizer::new(Box::new(\n",
- " WordPiece::from_files(\"bert/vocab.txt\")\n",
- " .build()\n",
- " .expect(\"Vocab file not found\"),\n",
- " ));\n",
- "\n",
- " tokenizer.with_normalizer(Box::new(BertNormalizer::default()));\n",
- " tokenizer.with_pre_tokenizer(Box::new(BertPreTokenizer));\n",
- " tokenizer.with_decoder(Box::new(WordPieceDecoder::default()));\n",
- " tokenizer.with_post_processor(Box::new(BertProcessing::new(\n",
- " (\n",
- " String::from(\"[SEP]\"),\n",
- " tokenizer.get_model().token_to_id(\"[SEP]\").unwrap(),\n",
- " ),\n",
- " (\n",
- " String::from(\"[CLS]\"),\n",
- " tokenizer.get_model().token_to_id(\"[CLS]\").unwrap(),\n",
- " ),\n",
- " )));\n",
+ " // Load tokenizer from HF Hub\n",
+ " let tokenizer = Tokenizer::from_pretrained(\"bert-base-uncased\", None).unwrap();\n",
"\n",
" // Encode input text\n",
- " let encoding = tokenizer.encode(EncodeInput::Single(text), true).unwrap();\n",
+ " let encoding = tokenizer.encode(text, true).unwrap();\n",
"\n",
" let v1: Vec<i64> = encoding.get_ids().to_vec().into_iter().map(|x| x as i64).collect();\n",
" let v2: Vec<i64> = encoding.get_attention_mask().to_vec().into_iter().map(|x| x as i64).collect();\n",
@@ -1230,13 +1232,13 @@
" Ok(())\n",
"}"
],
- "execution_count": null,
+ "execution_count": 47,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "Overwriting src/main.rs\n"
+ "Writing src/main.rs\n"
]
}
]
@@ -1257,20 +1259,20 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "3446c3e7-a5d4-4572-ad9e-f463a1d4a7d7"
+ "outputId": "84b42d3c-82d4-46fc-fb84-94967bd5330f"
},
"source": [
"!cargo run \"I am happy\" 2> /dev/null\n",
"!cargo run \"I am mad\" 2> /dev/null"
],
- "execution_count": null,
+ "execution_count": 48,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[0.0011003953, 0.99889964]\n",
- "[0.9976444, 0.0023555849]\n"
+ "[0.00030939875, 0.9996906]\n",
+ "[0.99912345, 0.0008765513]\n"
]
}
]
@@ -1302,18 +1304,18 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "3dfff62e-d910-433f-c8de-c0b2311141a6"
+ "outputId": "b43ad47e-e1f3-4748-d854-a0dc2024b780"
},
"source": [
"!cargo run \"I am happy\" \"I am glad\" 2> /dev/null"
],
- "execution_count": null,
+ "execution_count": 49,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "0.8583641740656903\n"
+ "0.8298246060854143\n"
]
}
]
@@ -1349,4 +1351,4 @@
]
}
]
-}
+}
\ No newline at end of file