Skip to content

armgit / txtai

Public Imported from GitHub
0

πŸ’‘ All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Updated notebooks with 2.0.0 changes

davidmezzetti563ba7f77c10
diff --git a/examples/01_Introducing_txtai.ipynb b/examples/01_Introducing_txtai.ipynb
index b8218f72..9b473011 100644
--- a/examples/01_Introducing_txtai.ipynb
+++ b/examples/01_Introducing_txtai.ipynb
@@ -21,7 +21,8 @@
},
"colab": {
"name": "01 - Introducing txtai",
- "provenance": []
+ "provenance": [],
+ "collapsed_sections": []
}
},
"cells": [
@@ -60,7 +61,7 @@
"source": [
"# Install dependencies\n",
"\n",
- "Install txtai and all dependencies"
+ "Install `txtai` and all dependencies."
]
},
{
@@ -128,26 +129,24 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "86661189-5aec-43a9-8577-17017c1e99c9"
+ "outputId": "420366d1-3215-4817-9269-461d43ba493b"
},
"source": [
- "import numpy as np\n",
- "\n",
- "sections = [\"US tops 5 million confirmed virus cases\",\n",
- " \"Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg\",\n",
- " \"Beijing mobilises invasion craft along coast as Taiwan tensions escalate\",\n",
- " \"The National Park Service warns against sacrificing slower friends in a bear attack\",\n",
- " \"Maine man wins $1M from $25 lottery ticket\",\n",
- " \"Make huge profits without work, earn up to $100,000 a day\"]\n",
+ "data = [\"US tops 5 million confirmed virus cases\",\n",
+ " \"Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg\",\n",
+ " \"Beijing mobilises invasion craft along coast as Taiwan tensions escalate\",\n",
+ " \"The National Park Service warns against sacrificing slower friends in a bear attack\",\n",
+ " \"Maine man wins $1M from $25 lottery ticket\",\n",
+ " \"Make huge profits without work, earn up to $100,000 a day\"]\n",
"\n",
"print(\"%-20s %s\" % (\"Query\", \"Best Match\"))\n",
"print(\"-\" * 50)\n",
"\n",
"for query in (\"feel good story\", \"climate change\", \"health\", \"war\", \"wildlife\", \"asia\", \"north america\", \"dishonest junk\"):\n",
" # Get index of best section that best matches query\n",
- " uid = np.argmax(embeddings.similarity(query, sections))\n",
+ " uid = embeddings.similarity(query, data)[0][0]\n",
"\n",
- " print(\"%-20s %s\" % (query, sections[uid]))"
+ " print(\"%-20s %s\" % (query, data[uid]))"
],
"execution_count": 3,
"outputs": [
@@ -199,11 +198,11 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "f5fddb55-ff37-42bf-d85f-cac81304a6ad"
+ "outputId": "89c727e8-bc46-40e4-d411-7afc83d678da"
},
"source": [
- "# Create an index for the list of sections\n",
- "embeddings.index([(uid, text, None) for uid, text in enumerate(sections)])\n",
+ "# Create an index for the list of text\n",
+ "embeddings.index([(uid, text, None) for uid, text in enumerate(data)])\n",
"\n",
"print(\"%-20s %s\" % (\"Query\", \"Best Match\"))\n",
"print(\"-\" * 50)\n",
@@ -214,8 +213,8 @@
" # search result format: (uid, score)\n",
" uid = embeddings.search(query, 1)[0][0]\n",
"\n",
- " # Print section\n",
- " print(\"%-20s %s\" % (query, sections[uid]))"
+ " # Print text\n",
+ " print(\"%-20s %s\" % (query, data[uid]))"
],
"execution_count": 4,
"outputs": [
@@ -256,7 +255,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "4c1d124d-4ee2-4f2f-e79a-2a859a224e4d"
+ "outputId": "8af2eaf8-de58-49c4-8218-8b3beca090c4"
},
"source": [
"embeddings.save(\"index\")\n",
@@ -265,7 +264,7 @@
"embeddings.load(\"index\")\n",
"\n",
"uid = embeddings.search(\"climate change\", 1)[0][0]\n",
- "print(sections[uid])"
+ "print(data[uid])"
],
"execution_count": 5,
"outputs": [
@@ -286,7 +285,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "3c35204b-eea7-4d5a-c27d-9eb1001996c8"
+ "outputId": "1dc1b488-a9e8-472d-b348-9d7a13b72296"
},
"source": [
"!ls index"
diff --git a/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb b/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb
index 7114c415..da90ee66 100644
--- a/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb
+++ b/examples/02_Build_an_Embeddings_index_with_Hugging_Face_Datasets.ipynb
@@ -24,7 +24,9 @@
"\n",
"This notebook shows how txtai can index and search with Hugging Face's [Datasets](https://github.com/huggingface/datasets) library. Datasets opens access to a large and growing list of publicly available datasets. Datasets has functionality to select, transform and filter data stored in each dataset.\n",
"\n",
- "In this example, txtai will be used to index and query a dataset."
+ "In this example, txtai will be used to index and query a dataset.\n",
+ "\n",
+ "**Make sure to select a GPU runtime when running this notebook**"
]
},
{
@@ -99,8 +101,7 @@
"\n",
"def ranksearch(query):\n",
" results = [text for _, text in search(query)]\n",
- " scores = similarity(query, results)\n",
- " return [(scores[x], results[x]) for x in reversed(np.argsort(scores))]\n",
+ " return [(score, results[x]) for x, score in similarity(query, results)]\n",
"\n",
"# Load HF dataset\n",
"dataset = load_dataset(\"ag_news\", split=\"train\")\n",
@@ -136,7 +137,7 @@
"height": 1000
},
"id": "YVmbiY92vxEO",
- "outputId": "46b94655-b9bf-4982-c7ed-2d6a0a251e39"
+ "outputId": "1034faa8-2cf1-4392-eaff-ab457c88e58f"
},
"source": [
"from IPython.core.display import display, HTML\n",
diff --git a/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb b/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb
index 8ffbf6f1..bc137451 100644
--- a/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb
+++ b/examples/03_Build_an_Embeddings_index_from_a_data_source.ipynb
@@ -36,7 +36,7 @@
"source": [
"# Install dependencies\n",
"\n",
- "Install txtai and all dependencies"
+ "Install `txtai` and all dependencies."
]
},
{
@@ -48,7 +48,7 @@
"%%capture\n",
"!pip install git+https://github.com/neuml/txtai"
],
- "execution_count": 1,
+ "execution_count": null,
"outputs": []
},
{
@@ -75,7 +75,7 @@
"!gunzip tests.gz\n",
"!mv tests articles.sqlite"
],
- "execution_count": 2,
+ "execution_count": null,
"outputs": []
},
{
@@ -101,7 +101,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "0360df2f-9c4b-4c1a-8495-a250ba9ae39f"
+ "outputId": "f59ae9dd-76e4-40b4-f170-ccbd6a57416e"
},
"source": [
"import os\n",
@@ -137,7 +137,7 @@
"# Show files\n",
"!ls -l"
],
- "execution_count": 3,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -147,9 +147,9 @@
"Converting vectors to magnitude format\n",
"total 78948\n",
"-rw-r--r-- 1 root root 8065024 Aug 25 01:44 articles.sqlite\n",
- "-rw-r--r-- 1 root root 24145920 Jan 7 01:39 cord19-300d.magnitude\n",
- "-rw-r--r-- 1 root root 48625387 Jan 7 01:38 cord19-300d.txt\n",
- "drwxr-xr-x 1 root root 4096 Dec 21 17:29 sample_data\n"
+ "-rw-r--r-- 1 root root 24145920 Jan 9 20:45 cord19-300d.magnitude\n",
+ "-rw-r--r-- 1 root root 48625387 Jan 9 20:45 cord19-300d.txt\n",
+ "drwxr-xr-x 1 root root 4096 Jan 6 18:10 sample_data\n"
],
"name": "stdout"
}
@@ -173,7 +173,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "2f464dec-4c68-4079-d8a0-b62070a846c0"
+ "outputId": "50a0fa76-be8d-4571-8c1a-5b911ad9b726"
},
"source": [
"import sqlite3\n",
@@ -228,7 +228,7 @@
"# Build embeddings index\n",
"embeddings.index(stream())\n"
],
- "execution_count": 4,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -259,7 +259,7 @@
"base_uri": "https://localhost:8080/",
"height": 293
},
- "outputId": "7c3ae07a-74a3-4603-89ef-cce0a6c51769"
+ "outputId": "59153112-aca0-43ac-bdd9-ebb524fe51e1"
},
"source": [
"import pandas as pd\n",
@@ -294,7 +294,7 @@
"\n",
"display(HTML(df.to_html(index=False)))"
],
- "execution_count": 5,
+ "execution_count": null,
"outputs": [
{
"output_type": "display_data",
@@ -361,7 +361,7 @@
"source": [
"# Extracting additional columns from query results\n",
"\n",
- "The example above uses the Embeddings index to find the top 5 best matches. In addition to this, an Extractor instance (this will be explained further in part 4) is used to ask additional questions over the search results, creating a richer query response."
+ "The example above uses the Embeddings index to find the top 5 best matches. In addition to this, an Extractor instance (this will be explained further in part 5) is used to ask additional questions over the search results, creating a richer query response."
]
},
{
@@ -376,7 +376,7 @@
"# Create extractor instance using qa model designed for the CORD-19 dataset\n",
"extractor = Extractor(embeddings, \"NeuML/bert-small-cord19qa\")"
],
- "execution_count": 6,
+ "execution_count": null,
"outputs": []
},
{
@@ -387,7 +387,7 @@
"base_uri": "https://localhost:8080/",
"height": 293
},
- "outputId": "61f60aad-e6a3-4572-f5f3-f44d7c520ec8"
+ "outputId": "bd02174a-193b-41c0-8e1d-bf658ea932ee"
},
"source": [
"db = sqlite3.connect(\"articles.sqlite\")\n",
@@ -399,18 +399,18 @@
" uid, text = cur.fetchone()\n",
"\n",
" # Get list of document text sections to use for the context\n",
- " cur.execute(\"SELECT Id, Name, Text FROM sections WHERE (labels is null or labels NOT IN ('FRAGMENT', 'QUESTION')) AND article = ?\", [uid])\n",
- " sections = []\n",
- " for sid, name, txt in cur.fetchall():\n",
+ " cur.execute(\"SELECT Name, Text FROM sections WHERE (labels is null or labels NOT IN ('FRAGMENT', 'QUESTION')) AND article = ? ORDER BY Id\", [uid])\n",
+ " texts = []\n",
+ " for name, txt in cur.fetchall():\n",
" if not name or not re.search(r\"background|(?<!.*?results.*?)discussion|introduction|reference\", name.lower()):\n",
- " sections.append((sid, txt))\n",
+ " texts.append(txt)\n",
"\n",
" cur.execute(\"SELECT Title, Published, Reference from articles where id = ?\", [uid])\n",
" article = cur.fetchone()\n",
"\n",
" # Use QA extractor to derive additional columns\n",
- " answers = extractor(sections, [(\"Risk Factors\", \"risk factors\", \"What risk factors?\", False),\n",
- " (\"Locations\", \"hospital country\", \"What locations?\", False)])\n",
+ " answers = extractor([(\"Risk Factors\", \"risk factors\", \"What risk factors?\", False),\n",
+ " (\"Locations\", \"hospital country\", \"What locations?\", False)], texts)\n",
"\n",
" results.append(article + (text,) + tuple([answer[1] for answer in answers]))\n",
"\n",
@@ -420,7 +420,7 @@
"df = pd.DataFrame(results, columns=[\"Title\", \"Published\", \"Reference\", \"Match\", \"Risk Factors\", \"Locations\"])\n",
"display(HTML(df.to_html(index=False)))"
],
- "execution_count": 7,
+ "execution_count": null,
"outputs": [
{
"output_type": "display_data",
@@ -507,7 +507,7 @@
},
"source": [
"# Next\n",
- "In part 4 of this series, we'll look at how to use txtai to run extractive searches"
+ "In part 4 of this series, we'll look at adding semantic search to an existing search system."
]
}
]
diff --git a/examples/04_Add_semantic_search_to_Elasticsearch.ipynb b/examples/04_Add_semantic_search_to_Elasticsearch.ipynb
new file mode 100644
index 00000000..a4c21939
--- /dev/null
+++ b/examples/04_Add_semantic_search_to_Elasticsearch.ipynb
@@ -0,0 +1,535 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "name": "04 - Add semantic search to Elasticsearch",
+ "provenance": [],
+ "collapsed_sections": []
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "accelerator": "GPU"
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "zzZbP0LM6m5z"
+ },
+ "source": [
+ "# Part 4: Add semantic search to Elasticsearch\n",
+ "\n",
+ "Part 2 and Part 3 of this series showed how to index and search data in txtai. Part 2 indexed and searched a Hugging Face Dataset, Part 3 indexed and searched an external data source. \n",
+ "\n",
+ "txtai is modular in design, it's components can be individually used. txtai has a similarity function that works on lists of text. This method can be integrated with any external search service, such as a REST API, a SQL query or anything else that returns text search results. \n",
+ "\n",
+ "In this notebook, we'll take the same Hugging Face Dataset used in Part 2, index it in Elasticsearch and rank the search results using a semantic similarity function from txtai.\n",
+ "\n",
+ "**Make sure to select a GPU runtime when running this notebook**"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "xk7t5Jcd6reO"
+ },
+ "source": [
+ "# Install dependencies\n",
+ "\n",
+ "Install `txtai`, `datasets` and `Elasticsearch`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "0y1UA4-q-YdA"
+ },
+ "source": [
+ "%%capture\n",
+ "\n",
+ "# Install txtai, datasets and elasticsearch python client\n",
+ "!pip install git+https://github.com/neuml/txtai datasets elasticsearch\n",
+ "\n",
+ "# Download and extract elasticsearch\n",
+ "!wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.1-linux-x86_64.tar.gz\n",
+ "!tar -xzf elasticsearch-7.10.1-linux-x86_64.tar.gz\n",
+ "!chown -R daemon:daemon elasticsearch-7.10.1"
+ ],
+ "execution_count": 1,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "M5c_WxmxCShQ"
+ },
+ "source": [
+ "Start an instance of Elasticsearch directly within this notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "3ZfJeWbM6wmj"
+ },
+ "source": [
+ "import os\n",
+ "from subprocess import Popen, PIPE, STDOUT\n",
+ "\n",
+ "# If issues are encountered with this section, ES can be manually started as follows:\n",
+ "# ./elasticsearch-7.10.1/bin/elasticsearch\n",
+ "\n",
+ "# Start and wait for server\n",
+ "server = Popen(['elasticsearch-7.10.1/bin/elasticsearch'], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1))\n",
+ "!sleep 30"
+ ],
+ "execution_count": 2,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hSWFzkCn61tM"
+ },
+ "source": [
+ "# Load data into Elasticsearch\n",
+ "\n",
+ "The following block loads the dataset into Elasticsearch."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "So-OBvUT61QD",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "d5b6fa1e-4ba5-4e38-ed51-f3fc734904fd"
+ },
+ "source": [
+ "from datasets import load_dataset\n",
+ "\n",
+ "from elasticsearch import Elasticsearch, helpers\n",
+ "\n",
+ "# Connect to ES instance\n",
+ "es = Elasticsearch(hosts=[\"http://localhost:9200\"], timeout=60, retry_on_timeout=True)\n",
+ "\n",
+ "# Load HF dataset\n",
+ "dataset = load_dataset(\"ag_news\", split=\"train\")[\"text\"][:50000]\n",
+ "\n",
+ "# Elasticsearch bulk buffer\n",
+ "buffer = []\n",
+ "rows = 0\n",
+ "\n",
+ "for x, text in enumerate(dataset):\n",
+ " # Article record\n",
+ " article = {\"_id\": x, \"_index\": \"articles\", \"title\": text}\n",
+ "\n",
+ " # Buffer article\n",
+ " buffer.append(article)\n",
+ "\n",
+ " # Increment number of articles processed\n",
+ " rows += 1\n",
+ "\n",
+ " # Bulk load every 1000 records\n",
+ " if rows % 1000 == 0:\n",
+ " helpers.bulk(es, buffer)\n",
+ " buffer = []\n",
+ "\n",
+ " print(\"Inserted {} articles\".format(rows), end=\"\\r\")\n",
+ "\n",
+ "if buffer:\n",
+ " helpers.bulk(es, buffer)\n",
+ "\n",
+ "print(\"Total articles inserted: {}\".format(rows))"
+ ],
+ "execution_count": 3,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "text": [
+ "Using custom data configuration default\n",
+ "Reusing dataset ag_news (/root/.cache/huggingface/datasets/ag_news/default/0.0.0/fb5c5e74a110037311ef5e904583ce9f8b9fbc1354290f97b4929f01b3f48b1a)\n"
+ ],
+ "name": "stderr"
+ },
+ {
+ "output_type": "stream",
+ "text": [
+ "Total articles inserted: 50000\n"
+ ],
+ "name": "stdout"
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "X5RO-VNwzMAo"
+ },
+ "source": [
+ "# Query data with Elasticsearch\n",
+ "\n",
+ "Elasticsearch is a token-based search system. Queries and documents are parsed into tokens and the most relevant query-document matches are calculated using a scoring algorithm. The default scoring algorithm is [BM25](https://www.elastic.co/blog/practical-bm25-part-2-the-bm25-algorithm-and-its-variables). Powerful queries can be built using a [rich query syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax) and [Query DSL](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/query-dsl.html). \n",
+ "\n",
+ "The following section runs a query against Elasticsearch, finds the top 5 matches and returns the corresponding documents associated with each match.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "ucd9mwSfFTMm",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 297
+ },
+ "outputId": "3eb5cc21-bf4e-4273-a212-f89b87a022e3"
+ },
+ "source": [
+ "from IPython.display import display, HTML\n",
+ "\n",
+ "def table(category, query, rows):\n",
+ " html = \"\"\"\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " \"\"\"\n",
+ "\n",
+ " html += \"<h3>[%s] %s</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead>\" % (category, query)\n",
+ " for score, text in rows:\n",
+ " html += \"<tr><td>%.4f</td><td>%s</td></tr>\" % (score, text)\n",
+ " html += \"</table>\"\n",
+ "\n",
+ " display(HTML(html))\n",
+ "\n",
+ "def search(query, limit):\n",
+ " query = {\n",
+ " \"size\": limit,\n",
+ " \"query\": {\n",
+ " \"query_string\": {\"query\": query}\n",
+ " }\n",
+ " }\n",
+ "\n",
+ " results = []\n",
+ " for result in es.search(index=\"articles\", body=query)[\"hits\"][\"hits\"]:\n",
+ " source = result[\"_source\"]\n",
+ " results.append((min(result[\"_score\"], 18) / 18, source[\"title\"]))\n",
+ "\n",
+ " return results\n",
+ "\n",
+ "limit = 3\n",
+ "query= \"+yankees lose\"\n",
+ "table(\"Elasticsearch\", query, search(query, limit))"
+ ],
+ "execution_count": 4,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": [
+ "\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " <h3>[Elasticsearch] +yankees lose</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead><tr><td>0.5817</td><td>El Duque adds to gloomy NY forecast The Yankees #39; staff infection has spread to the one man the team can #39;t afford to lose. Orlando Hernandez was scratched from last night #39;s scheduled start because </td></tr><tr><td>0.5697</td><td>Rangers Derail Red Sox The Red Sox lose for the first time in 11 games, falling to the Rangers 8-6 Saturday and missing a chance to pull within 1 1/2 games of the Yankees in the AL East.</td></tr><tr><td>0.5069</td><td>Rout leaves Yanks #39; lead at 3 Royals gain control with 10-run 5th Against a nothing-to-lose team such as the Kansas City Royals, the Yankees #39; manager wanted his team to put down the hammer early and not let baseball #39;s second worst team believe it had a chance.</td></tr></table>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "W1DkpcIob5kt"
+ },
+ "source": [
+ "The table above shows the results for the query `+yankees lose`. This query requires the token `yankees`. The search doesn't understand the semantic meaning of the query. It returns the most relevant results with those two tokens.\n",
+ "\n",
+ "We can see in this case, the results aren't capturing the meaning of the search. Let's try adding semantic similarity to the search!"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hMro47KedzJq"
+ },
+ "source": [
+ "# Ranking search results with txtai\n",
+ "\n",
+ "txtai has a similarity module that computes the similarity between a query and a list of strings. Of course, txtai can also build a full index as shown in the previous notebooks but in this case we'll just use the ad-hoc similarity function.\n",
+ "\n",
+ "The code below creates a Similarity instance and defines a ranking function to order search results based on the computed similarity.\n",
+ "\n",
+ "`ranksearch` queries Elasticsearch for a larger set of results, ranks the results using the similarity instance and returns the top n results. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "id": "RUOj5zhFFK8N"
+ },
+ "source": [
+ "%%capture\n",
+ "from txtai.pipeline import Similarity\n",
+ "\n",
+ "def ranksearch(query, limit):\n",
+ " results = [text for _, text in search(query, limit * 10)]\n",
+ " return [(score, results[x]) for x, score in similarity(query, results)][:limit]\n",
+ "\n",
+ "# Create similarity instance for re-ranking\n",
+ "similarity = Similarity(\"valhalla/distilbart-mnli-12-3\")"
+ ],
+ "execution_count": 5,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UMFuv5-Hedfc"
+ },
+ "source": [
+ "Now let's re-run the previous search."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 297
+ },
+ "id": "3jJI9OxU0dZk",
+ "outputId": "d8828af3-7873-454b-ce5b-e6ab79900e62"
+ },
+ "source": [
+ "# Run the search\n",
+ "table(\"Elasticsearch + txtai\", query, ranksearch(query, limit))"
+ ],
+ "execution_count": 6,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": [
+ "\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " <h3>[Elasticsearch + txtai] +yankees lose</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead><tr><td>0.9929</td><td>Ouch! Yankees hit new low INDIANS 22, YANKEES 0---At New York, Omar Vizquel went 6-for-7 to tie the American League record for hits as Cleveland handed the Yankees the largest loss in their history last night.</td></tr><tr><td>0.9874</td><td>Vazquez and Yankees Buckle Early Because Javier Vazquez fizzled while Brad Radke flourished, the Yankees sustained their first regular-season defeat by the Minnesota Twins since 2001.</td></tr><tr><td>0.9542</td><td>Slide of the Yankees: Pinstripes Punished George Steinbrenner watched from his box as his Yankees suffered the most one-sided loss in the franchise's long history.</td></tr></table>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RXB2PaDZfd8o"
+ },
+ "source": [
+ "The results above do a much better job of finding results semantically similar in meaning to the query. Instead of just finding matches with `yankees` and `lose`, it finds matches where the `yankees lose`. \n",
+ "\n",
+ "This combination is effective and powerful. It takes advantage of the high performance of Elasticsearch while adding a semantic search capability. We may already have a large Elasticsearch cluster with TBs (or PBs)+ of data and years of engineering investment that solves most use cases. Semantically ranking search results is a practical approach."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XBVL56fsRI86"
+ },
+ "source": [
+ "# More examples\n",
+ "\n",
+ "Now for some more examples comparing the results from Elasticsearch vs Elasticsearch + txtai."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 1000
+ },
+ "id": "7IHS38SERRpQ",
+ "outputId": "9fd23084-48f5-4aff-edd2-2f9b347260a5"
+ },
+ "source": [
+ "for query in [\"good news +economy\", \"bad news +economy\"]:\n",
+ " table(\"Elasticsearch\", query, search(query, limit))\n",
+ " table(\"Elasticsearch + txtai\", query, ranksearch(query, limit))"
+ ],
+ "execution_count": 7,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": [
+ "\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " <h3>[Elasticsearch] good news +economy</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead><tr><td>0.8756</td><td>Surprise drop US wholesale prices is mixed news for economy (AFP) AFP - A surprise drop in US wholesale prices in August showed inflation apparently in check, but analysts said this was good and bad news for the US economy.</td></tr><tr><td>0.7379</td><td>China investment slows Good news for officials who are trying to cool an overheated economy; austerity measures to remain. BEIJING (Reuters) - China reported a marked slowdown in investment and money supply growth Monday, but stubbornly </td></tr><tr><td>0.7145</td><td>Spending Rebounds, Good News for Growth WASHINGTON (Reuters) - U.S. consumer spending rebounded sharply July, government data showed on Monday, erasing the disappointment of June and bolstering hopes that the U.S. economy has recovered from its recent soft spot.</td></tr></table>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": [
+ "\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " <h3>[Elasticsearch + txtai] good news +economy</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead><tr><td>0.9996</td><td>Spending Rebounds, Good News for Growth WASHINGTON (Reuters) - U.S. consumer spending rebounded sharply in July, the government said on Monday, erasing the disappointment of June and bolstering hopes that the U.S. economy has recovered from its recent soft spot.</td></tr><tr><td>0.9996</td><td>Spending Rebounds, Good News for Growth WASHINGTON (Reuters) - U.S. consumer spending rebounded sharply July, government data showed on Monday, erasing the disappointment of June and bolstering hopes that the U.S. economy has recovered from its recent soft spot.</td></tr><tr><td>0.9993</td><td>Home building surges Housing construction in August jumped to its highest level in five months, a dose of encouraging news for the economy #39;s expansion.</td></tr></table>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": [
+ "\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " <h3>[Elasticsearch] bad news +economy</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead><tr><td>0.9228</td><td>Surprise drop US wholesale prices is mixed news for economy (AFP) AFP - A surprise drop in US wholesale prices in August showed inflation apparently in check, but analysts said this was good and bad news for the US economy.</td></tr><tr><td>0.6405</td><td>Field Poll: Californians liking economy Bee Staff Writer. Californians are slowly growing more optimistic about the health of the economy, but a majority still feels the state is in bad economic times, according to a new Field Poll.</td></tr><tr><td>0.6188</td><td>ADB says China should raise rates to cool economy China should raise interest rates to cool the economy and prevent a future buildup of bad loans in the banking system, the Asian Development Bank #39;s (ADB) Bei-jing representative Bruce Murray said.</td></tr></table>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ },
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": [
+ "\n",
+ " <style type='text/css'>\n",
+ " @import url('https://fonts.googleapis.com/css?family=Oswald&display=swap');\n",
+ " table {\n",
+ " border-collapse: collapse;\n",
+ " width: 900px;\n",
+ " }\n",
+ " th, td {\n",
+ " border: 1px solid #9e9e9e;\n",
+ " padding: 10px;\n",
+ " font: 15px Oswald;\n",
+ " }\n",
+ " </style>\n",
+ " <h3>[Elasticsearch + txtai] bad news +economy</h3><table><thead><tr><th>Score</th><th>Text</th></tr></thead><tr><td>0.9977</td><td>Aging society hits Japan #39;s economy Japan #39;s economy will be the most severely affected among industrialized nations by population aging, Kyodo News said Thursday.</td></tr><tr><td>0.9963</td><td>Funds: Fund Mergers Can Hurt Investors (Reuters) Reuters - Mergers and acquisitions have\\played an enormous role in the U.S. economy during the past\\several decades, but sometimes the results have been bad for\\consumers. Similarly, consolidation in the mutual fund\\business has sometimes hurt fund investors.</td></tr><tr><td>0.9958</td><td>Signs of listless economy persist In a sign of persistent weakness in the US economy, a widely watched measure of business activity declined in August for the third consecutive month.</td></tr></table>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "h-Bk3KLjZMpF"
+ },
+ "source": [
+ "Once again while Elasticsearch usually returns quality results, occasionally it will match results that aren't semantically relevant. The power of semantic search is that not only will it find direct matches but matches with the same meaning. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "PpV4qxIPT6XO"
+ },
+ "source": [
+ "# Next\n",
+ "In part 5 of this series, we'll look at how to use txtai to run extractive searches.\n"
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/examples/04_Extractive_QA_with_txtai.ipynb b/examples/05_Extractive_QA_with_txtai.ipynb
similarity index 72%
rename from examples/04_Extractive_QA_with_txtai.ipynb
rename to examples/05_Extractive_QA_with_txtai.ipynb
index 6b5d3980..3c770e05 100644
--- a/examples/04_Extractive_QA_with_txtai.ipynb
+++ b/examples/05_Extractive_QA_with_txtai.ipynb
@@ -3,8 +3,9 @@
"nbformat_minor": 0,
"metadata": {
"colab": {
- "name": "04 - Extractive QA with txtai",
- "provenance": []
+ "name": "05 - Extractive QA with txtai",
+ "provenance": [],
+ "collapsed_sections": []
},
"kernelspec": {
"name": "python3",
@@ -18,9 +19,9 @@
"id": "vwELCooy4ljr"
},
"source": [
- "# Part 4: Extractive QA with txtai\n",
+ "# Part 5: Extractive QA with txtai\n",
"\n",
- "In Parts 1 through 3, we gave a general overview of txtai, the backing technology and examples of how to use it for similarity searches. This notebook builds on that and extends to building extractive question-answering systems."
+ "In Parts 1 through 4, we gave a general overview of txtai, the backing technology and examples of how to use it for similarity searches. This notebook builds on that and extends to building extractive question-answering systems."
]
},
{
@@ -31,7 +32,7 @@
"source": [
"# Install dependencies\n",
"\n",
- "Install txtai and all dependencies"
+ "Install `txtai` and all dependencies."
]
},
{
@@ -88,28 +89,25 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "5c1ef8f8-3957-497e-ecee-2b574404292d"
+ "outputId": "21461baa-7625-4b7a-e7a6-1c23cd7f33c2"
},
"source": [
- "sections = [\"Giants hit 3 HRs to down Dodgers\",\n",
- " \"Giants 5 Dodgers 4 final\",\n",
- " \"Dodgers drop Game 2 against the Giants, 5-4\",\n",
- " \"Blue Jays 2 Red Sox 1 final\",\n",
- " \"Red Sox lost to the Blue Jays, 2-1\",\n",
- " \"Blue Jays at Red Sox is over. Score: 2-1\",\n",
- " \"Phillies win over the Braves, 5-0\",\n",
- " \"Phillies 5 Braves 0 final\",\n",
- " \"Final: Braves lose to the Phillies in the series opener, 5-0\",\n",
- " \"Final score: Flyers 4 Lightning 1\",\n",
- " \"Flyers 4 Lightning 1 final\",\n",
- " \"Flyers win 4-1\"]\n",
- "\n",
- "# Add unique id to each section to assist with qa extraction\n",
- "sections = [(uid, section) for uid, section in enumerate(sections)]\n",
+ "data = [\"Giants hit 3 HRs to down Dodgers\",\n",
+ " \"Giants 5 Dodgers 4 final\",\n",
+ " \"Dodgers drop Game 2 against the Giants, 5-4\",\n",
+ " \"Blue Jays 2 Red Sox 1 final\",\n",
+ " \"Red Sox lost to the Blue Jays, 2-1\",\n",
+ " \"Blue Jays at Red Sox is over. Score: 2-1\",\n",
+ " \"Phillies win over the Braves, 5-0\",\n",
+ " \"Phillies 5 Braves 0 final\",\n",
+ " \"Final: Braves lose to the Phillies in the series opener, 5-0\",\n",
+ " \"Final score: Flyers 4 Lightning 1\",\n",
+ " \"Flyers 4 Lightning 1 final\",\n",
+ " \"Flyers win 4-1\"]\n",
"\n",
"questions = [\"What team won the game?\", \"What was score?\"]\n",
"\n",
- "execute = lambda query: extractor(sections, [(question, query, question, False) for question in questions])\n",
+ "execute = lambda query: extractor([(question, query, question, False) for question in questions], data)\n",
"\n",
"for query in [\"Red Sox - Blue Jays\", \"Phillies - Braves\", \"Dodgers - Giants\", \"Flyers - Lightning\"]:\n",
" print(\"----\", query, \"----\")\n",
@@ -121,7 +119,7 @@
"question = \"What hockey team won?\"\n",
"\n",
"print(\"----\", question, \"----\")\n",
- "print(extractor(sections, [(question, question, question, False)]))"
+ "print(extractor([(question, question, question, False)], data))"
],
"execution_count": 3,
"outputs": [
@@ -158,7 +156,7 @@
},
"source": [
"# Next\n",
- "In part 5 of this series, we'll use combine the power of Elasticsearch with Extractive QA to build a large-scale, advanced search system."
+ "In part 6 of this series, we'll combine the power of Elasticsearch with Extractive QA to build a large-scale, advanced search system."
]
}
]
diff --git a/examples/05_Extractive_QA_with_Elasticsearch.ipynb b/examples/06_Extractive_QA_with_Elasticsearch.ipynb
similarity index 88%
rename from examples/05_Extractive_QA_with_Elasticsearch.ipynb
rename to examples/06_Extractive_QA_with_Elasticsearch.ipynb
index 32f51a6e..0a10360c 100644
--- a/examples/05_Extractive_QA_with_Elasticsearch.ipynb
+++ b/examples/06_Extractive_QA_with_Elasticsearch.ipynb
@@ -3,7 +3,7 @@
"nbformat_minor": 0,
"metadata": {
"colab": {
- "name": "05 - Extractive QA with Elasticsearch",
+ "name": "06 - Extractive QA with Elasticsearch",
"provenance": [],
"collapsed_sections": []
},
@@ -19,7 +19,7 @@
"id": "zzZbP0LM6m5z"
},
"source": [
- "# Part 5: Extractive QA with Elasticsearch\n",
+ "# Part 6: Extractive QA with Elasticsearch\n",
"\n",
"txtai is datastore agnostic, the library analyzes sets of text. The following example shows how extractive question-answering can be added on top of an Elasticsearch system."
]
@@ -30,9 +30,9 @@
"id": "xk7t5Jcd6reO"
},
"source": [
- "# Install txtai and Elasticsearch\n",
+ "# Install dependencies\n",
"\n",
- "Install txtai and Elasticsearch. Start an instance of Elasticsearch directly within this notebook. "
+ "Install `txtai` and `Elasticsearch`."
]
},
{
@@ -47,13 +47,22 @@
"!pip install git+https://github.com/neuml/txtai elasticsearch\n",
"\n",
"# Download and extract elasticsearch\n",
- "!wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.8.1-linux-x86_64.tar.gz\n",
- "!tar -xzf elasticsearch-7.8.1-linux-x86_64.tar.gz\n",
- "!chown -R daemon:daemon elasticsearch-7.8.1"
+ "!wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.1-linux-x86_64.tar.gz\n",
+ "!tar -xzf elasticsearch-7.10.1-linux-x86_64.tar.gz\n",
+ "!chown -R daemon:daemon elasticsearch-7.10.1"
],
- "execution_count": 8,
+ "execution_count": 1,
"outputs": []
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "nKWz-C5gCJy8"
+ },
+ "source": [
+ "Start an instance of Elasticsearch directly within this notebook. "
+ ]
+ },
{
"cell_type": "code",
"metadata": {
@@ -64,13 +73,13 @@
"from subprocess import Popen, PIPE, STDOUT\n",
"\n",
"# If issues are encountered with this section, ES can be manually started as follows:\n",
- "# ./elasticsearch-7.8.1/bin/elasticsearch\n",
+ "# ./elasticsearch-7.10.1/bin/elasticsearch\n",
"\n",
"# Start and wait for server\n",
- "server = Popen(['elasticsearch-7.8.1/bin/elasticsearch'], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1))\n",
+ "server = Popen(['elasticsearch-7.10.1/bin/elasticsearch'], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1))\n",
"!sleep 30"
],
- "execution_count": 9,
+ "execution_count": 2,
"outputs": []
},
{
@@ -97,7 +106,7 @@
"!gunzip tests.gz\n",
"!mv tests articles.sqlite"
],
- "execution_count": 10,
+ "execution_count": 3,
"outputs": []
},
{
@@ -118,7 +127,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "96399343-0a96-4c4c-aacc-a55b73df8c08"
+ "outputId": "07023f3c-78ac-407b-a1fb-e88a63365589"
},
"source": [
"import sqlite3\n",
@@ -169,7 +178,7 @@
"\n",
"print(\"Total articles inserted: {}\".format(rows))\n"
],
- "execution_count": 11,
+ "execution_count": 4,
"outputs": [
{
"output_type": "stream",
@@ -200,7 +209,7 @@
"base_uri": "https://localhost:8080/",
"height": 293
},
- "outputId": "27953588-c372-4dcb-cf24-b5e664624d5a"
+ "outputId": "85371ffd-55a9-4fbc-e612-2474336e53e8"
},
"source": [
"import pandas as pd\n",
@@ -226,7 +235,7 @@
"\n",
"display(HTML(df.to_html(index=False)))"
],
- "execution_count": 12,
+ "execution_count": 5,
"outputs": [
{
"output_type": "display_data",
@@ -285,6 +294,17 @@
}
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ylxOKji1-9_K"
+ },
+ "source": [
+ "# Derive columns with Extractive QA\n",
+ "\n",
+ "The next section uses Extractive QA to derive additional columns. For each article, the full text is retrieved and a series of questions are asked of the document. The answers are added as a derived column per article."
+ ]
+ },
{
"cell_type": "code",
"metadata": {
@@ -301,7 +321,7 @@
"# Create extractor instance using qa model designed for the CORD-19 dataset\n",
"extractor = Extractor(embeddings, \"NeuML/bert-small-cord19qa\")"
],
- "execution_count": 13,
+ "execution_count": 6,
"outputs": []
},
{
@@ -312,7 +332,7 @@
"base_uri": "https://localhost:8080/",
"height": 328
},
- "outputId": "6461efc2-cbed-4e05-c5d4-13eb9c9f60bb"
+ "outputId": "3ddc0b05-fb98-45a2-9d11-e0f4220d3026"
},
"source": [
"document = {\n",
@@ -332,10 +352,10 @@
"\n",
" for result in es.search(index=\"articles\", body=search)[\"hits\"][\"hits\"]:\n",
" source = result[\"_source\"]\n",
- " sid, name, text = source[\"id\"], source[\"name\"], source[\"text\"]\n",
+ " name, text = source[\"name\"], source[\"text\"]\n",
"\n",
" if not name or not re.search(r\"background|(?<!.*?results.*?)discussion|introduction|reference\", name.lower()):\n",
- " rows.append((sid, text))\n",
+ " rows.append(text)\n",
" \n",
" return rows\n",
"\n",
@@ -344,8 +364,8 @@
" source = result[\"_source\"]\n",
"\n",
" # Use QA extractor to derive additional columns\n",
- " answers = extractor(sections(source[\"article\"]), [(\"Risk factors\", \"risk factor\", \"What are names of risk factors?\", False),\n",
- " (\"Locations\", \"city country state\", \"What are names of locations?\", False)])\n",
+ " answers = extractor([(\"Risk factors\", \"risk factor\", \"What are names of risk factors?\", False),\n",
+ " (\"Locations\", \"city country state\", \"What are names of locations?\", False)], sections(source[\"article\"]))\n",
"\n",
" results.append((source[\"title\"], source[\"published\"], source[\"reference\"], source[\"text\"]) + tuple([answer[1] for answer in answers]))\n",
"\n",
@@ -353,7 +373,7 @@
"\n",
"display(HTML(df.to_html(index=False)))"
],
- "execution_count": 14,
+ "execution_count": 7,
"outputs": [
{
"output_type": "display_data",
@@ -431,7 +451,7 @@
},
"source": [
"# Next\n",
- "Part 6 of this series uses zero-shot classification to apply labels to input text.\n"
+ "In part 7 of this series, we'll use a zero-shot classification to apply labels to input text.\n"
]
}
]
diff --git a/examples/06_Labeling_with_zero_shot_classification.ipynb b/examples/07_Apply_labels_with_zero_shot_classification.ipynb
similarity index 81%
rename from examples/06_Labeling_with_zero_shot_classification.ipynb
rename to examples/07_Apply_labels_with_zero_shot_classification.ipynb
index cd46c252..7a86118b 100644
--- a/examples/06_Labeling_with_zero_shot_classification.ipynb
+++ b/examples/07_Apply_labels_with_zero_shot_classification.ipynb
@@ -3,7 +3,7 @@
"nbformat_minor": 0,
"metadata": {
"colab": {
- "name": "06 - Labeling with zero-shot classification",
+ "name": "07 - Apply labels with zero-shot classification",
"provenance": [],
"collapsed_sections": []
},
@@ -19,9 +19,9 @@
"id": "4Pjmz-RORV8E"
},
"source": [
- "# Part 6: Labeling with zero-shot classification\n",
+ "# Part 7: Apply labels with zero-shot classification\n",
"\n",
- "This notebook shows how zero-shot classification can be used to perform text classification/labeling. txtai provides a light-weight wrapper around the zero-shot-classification pipeline in Hugging Face Transformers. This method works impressively well out of the box. Kudos to the Hugging Face team for the phenomenal work on zero-shot classification!\n",
+ "This notebook shows how zero-shot classification can be used to perform text classification, labeling and topic modeling. txtai provides a light-weight wrapper around the zero-shot-classification pipeline in Hugging Face Transformers. This method works impressively well out of the box. Kudos to the Hugging Face team for the phenomenal work on zero-shot classification!\n",
"\n",
"The examples in this notebook pick the best matching label using a list of labels for a snippet of text.\n",
"\n",
@@ -36,7 +36,7 @@
"source": [
"# Install dependencies\n",
"\n",
- "Install txtai and all dependencies"
+ "Install `txtai` and all dependencies."
]
},
{
@@ -103,21 +103,21 @@
"base_uri": "https://localhost:8080/"
},
"id": "-K2YJJzsVtfq",
- "outputId": "f7185ce7-f4b4-4290-bf2c-44210e7f21c0"
+ "outputId": "63938a37-47f1-4838-8fc2-e445c87d97e4"
},
"source": [
- "sections = [\"Dodgers lose again, give up 3 HRs in a loss to the Giants\",\n",
- " \"Giants 5 Cardinals 4 final in extra innings\",\n",
- " \"Dodgers drop Game 2 against the Giants, 5-4\",\n",
- " \"Flyers 4 Lightning 1 final. 45 saves for the Lightning.\",\n",
- " \"Slashing, penalty, 2 minute power play coming up\",\n",
- " \"What a stick save!\",\n",
- " \"Leads the NFL in sacks with 9.5\",\n",
- " \"UCF 38 Temple 13\",\n",
- " \"With the 30 yard completion, down to the 10 yard line\",\n",
- " \"Drains the 3pt shot!!, 0:15 remaining in the game\",\n",
- " \"Intercepted! Drives down the court and shoots for the win\",\n",
- " \"Massive dunk!!! they are now up by 15 with 2 minutes to go\"]\n",
+ "data = [\"Dodgers lose again, give up 3 HRs in a loss to the Giants\",\n",
+ " \"Giants 5 Cardinals 4 final in extra innings\",\n",
+ " \"Dodgers drop Game 2 against the Giants, 5-4\",\n",
+ " \"Flyers 4 Lightning 1 final. 45 saves for the Lightning.\",\n",
+ " \"Slashing, penalty, 2 minute power play coming up\",\n",
+ " \"What a stick save!\",\n",
+ " \"Leads the NFL in sacks with 9.5\",\n",
+ " \"UCF 38 Temple 13\",\n",
+ " \"With the 30 yard completion, down to the 10 yard line\",\n",
+ " \"Drains the 3pt shot!!, 0:15 remaining in the game\",\n",
+ " \"Intercepted! Drives down the court and shoots for the win\",\n",
+ " \"Massive dunk!!! they are now up by 15 with 2 minutes to go\"]\n",
"\n",
"# List of labels\n",
"tags = [\"Baseball\", \"Football\", \"Hockey\", \"Basketball\"]\n",
@@ -125,8 +125,8 @@
"print(\"%-75s %s\" % (\"Text\", \"Label\"))\n",
"print(\"-\" * 100)\n",
"\n",
- "for text in sections:\n",
- " print(\"%-75s %s\" % (text, labels(text, tags)[0][0]))"
+ "for text in data:\n",
+ " print(\"%-75s %s\" % (text, tags[labels(text, tags)[0][0]]))"
],
"execution_count": 3,
"outputs": [
@@ -170,7 +170,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "uIf064M9pbjn",
- "outputId": "ccb8c2ee-cc06-424b-caf2-1e8162e7a0c1"
+ "outputId": "527dc810-5439-4a13-bea9-6f0b5220e673"
},
"source": [
"tags = [\"πŸ˜€\", \"😑\"]\n",
@@ -178,8 +178,8 @@
"print(\"%-75s %s\" % (\"Text\", \"Label\"))\n",
"print(\"-\" * 100)\n",
"\n",
- "for text in sections:\n",
- " print(\"%-75s %s\" % (text, labels(text, tags)[0][0]))"
+ "for text in data:\n",
+ " print(\"%-75s %s\" % (text, tags[labels(text, tags)[0][0]]))"
],
"execution_count": 4,
"outputs": [
@@ -212,7 +212,7 @@
},
"source": [
"# Next\n",
- "Part 7 of this series shows how to use the txtai API."
+ "In part 8 of this series, we'll explore the txtai API."
]
}
]
diff --git a/examples/07_API_Gallery.ipynb b/examples/08_API_Gallery.ipynb
similarity index 90%
rename from examples/07_API_Gallery.ipynb
rename to examples/08_API_Gallery.ipynb
index e710793c..110edf71 100644
--- a/examples/07_API_Gallery.ipynb
+++ b/examples/08_API_Gallery.ipynb
@@ -3,7 +3,7 @@
"nbformat_minor": 0,
"metadata": {
"colab": {
- "name": "07 - API Gallery",
+ "name": "08 - API Gallery",
"provenance": [],
"collapsed_sections": []
},
@@ -19,7 +19,7 @@
"id": "4Pjmz-RORV8E"
},
"source": [
- "# Part 7: API Gallery\n",
+ "# Part 8: API Gallery\n",
"\n",
"The txtai API is a web-based service backed by [FastAPI](https://fastapi.tiangolo.com/). All txtai functionality including similarity search, extractive QA and zero-shot labeling is available via the API.\n",
"\n",
@@ -34,7 +34,7 @@
"source": [
"# Install dependencies\n",
"\n",
- "Install txtai and all dependencies"
+ "Install `txtai` and all dependencies."
]
},
{
@@ -46,7 +46,7 @@
"%%capture\n",
"!pip install git+https://github.com/neuml/txtai"
],
- "execution_count": 1,
+ "execution_count": null,
"outputs": []
},
{
@@ -57,7 +57,7 @@
"source": [
"# Python\n",
"\n",
- "The first method we'll try is direct access via Python. We'll use zero-shot labeling for all the examples here. See [this notebook](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/05_Labeling_with_zero_shot_classification.ipynb) for more details on zero-shot classification. "
+ "The first method we'll try is direct access via Python. We'll use zero-shot labeling for all the examples here. See [this notebook](https://colab.research.google.com/github/neuml/txtai/blob/master/examples/07_Apply_labels_with_zero_shot_classification.ipynb) for more details on zero-shot classification. "
]
},
{
@@ -106,7 +106,7 @@
"# Create labels model\n",
"labels = Labels()"
],
- "execution_count": 2,
+ "execution_count": null,
"outputs": []
},
{
@@ -129,19 +129,19 @@
"outputId": "8f740a3a-c380-474a-f1aa-8c26adfb3c85"
},
"source": [
- "sections = [\"Wears a red suit and says ho ho\",\n",
- " \"Pulls a flying sleigh\",\n",
- " \"This is cut down and decorated\",\n",
- " \"Santa puts these under the tree\",\n",
- " \"Best way to spend the holidays\"]\n",
+ "data = [\"Wears a red suit and says ho ho\",\n",
+ " \"Pulls a flying sleigh\",\n",
+ " \"This is cut down and decorated\",\n",
+ " \"Santa puts these under the tree\",\n",
+ " \"Best way to spend the holidays\"]\n",
"\n",
"# List of labels\n",
"tags = [\"πŸŽ… Santa Clause\", \"🦌 Reindeer\", \"πŸͺ Cookies\", \"πŸŽ„ Christmas Tree\", \"🎁 Gifts\", \"πŸ‘ͺ Family\"]\n",
"\n",
"# Render output to table\n",
- "table([(text, labels(text, tags)[0][0]) for text in sections])"
+ "table([(text, tags[labels(text, tags)[0][0]]) for text in data])"
],
- "execution_count": 3,
+ "execution_count": null,
"outputs": [
{
"output_type": "display_data",
@@ -209,7 +209,7 @@
"# Labels settings\n",
"labels:"
],
- "execution_count": 4,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -229,7 +229,7 @@
"!CONFIG=index.yml nohup uvicorn \"txtai.api:app\" &> api.log &\n",
"!sleep 90"
],
- "execution_count": 5,
+ "execution_count": null,
"outputs": []
},
{
@@ -258,7 +258,7 @@
"%%capture\n",
"!git clone https://github.com/neuml/txtai.js"
],
- "execution_count": 6,
+ "execution_count": null,
"outputs": []
},
{
@@ -290,11 +290,11 @@
" try {\n",
" let labels = new Labels(\"http://localhost:8000\");\n",
"\n",
- " let sections = [\"Wears a red suit and says ho ho\",\n",
- " \"Pulls a flying sleigh\",\n",
- " \"This is cut down and decorated\",\n",
- " \"Santa puts these under the tree\",\n",
- " \"Best way to spend the holidays\"];\n",
+ " let data = [\"Wears a red suit and says ho ho\",\n",
+ " \"Pulls a flying sleigh\",\n",
+ " \"This is cut down and decorated\",\n",
+ " \"Santa puts these under the tree\",\n",
+ " \"Best way to spend the holidays\"];\n",
"\n",
" // List of labels\n",
" let tags = [\"πŸŽ… Santa Clause\", \"🦌 Reindeer\", \"πŸͺ Cookies\", \"πŸŽ„ Christmas Tree\", \"🎁 Gifts\", \"πŸ‘ͺ Family\"];\n",
@@ -302,9 +302,9 @@
" console.log(sprintf(\"%-40s %s\", \"Text\", \"Label\"));\n",
" console.log(\"-\".repeat(75))\n",
"\n",
- " for (let text of sections) {\n",
+ " for (let text of data) {\n",
" let label = await labels.label(text, tags);\n",
- " label = label[0][0];\n",
+ " label = tags[label[0][0]];\n",
"\n",
" console.log(sprintf(\"%-40s %s\", text, label));\n",
" }\n",
@@ -316,7 +316,7 @@
"\n",
"run();\n"
],
- "execution_count": 7,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -350,7 +350,7 @@
"!npm install\n",
"!npm run build"
],
- "execution_count": 8,
+ "execution_count": null,
"outputs": []
},
{
@@ -365,7 +365,7 @@
"source": [
"!node dist/labels.js"
],
- "execution_count": 9,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -418,7 +418,7 @@
"os.chdir(\"/content\")\n",
"!git clone https://github.com/neuml/txtai.java"
],
- "execution_count": 10,
+ "execution_count": null,
"outputs": []
},
{
@@ -455,7 +455,7 @@
" try {\n",
" Labels labels = new Labels(\"http://localhost:8000\");\n",
"\n",
- " List <String> sections = \n",
+ " List <String> data = \n",
" Arrays.asList(\"Wears a red suit and says ho ho\",\n",
" \"Pulls a flying sleigh\",\n",
" \"This is cut down and decorated\",\n",
@@ -468,9 +468,9 @@
" System.out.printf(\"%-40s %s%n\", \"Text\", \"Label\");\n",
" System.out.println(new String(new char[75]).replace(\"\\0\", \"-\"));\n",
"\n",
- " for (String text: sections) {\n",
+ " for (String text: data) {\n",
" List<Score> scores = labels.label(text, tags);\n",
- " System.out.printf(\"%-40s %s%n\", text, scores.get(0).id);\n",
+ " System.out.printf(\"%-40s %s%n\", text, tags.get(scores.get(0).id));\n",
" }\n",
" }\n",
" catch (Exception ex) {\n",
@@ -479,7 +479,7 @@
" }\n",
"}\n"
],
- "execution_count": 11,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -512,7 +512,7 @@
"os.chdir(\"txtai.java/examples\")\n",
"!../gradlew -q --console=plain labels 2> /dev/null"
],
- "execution_count": 12,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -569,7 +569,7 @@
"!apt-get install rustc\n",
"!git clone https://github.com/neuml/txtai.rs"
],
- "execution_count": 13,
+ "execution_count": null,
"outputs": []
},
{
@@ -601,18 +601,18 @@
"pub async fn labels() -> Result<(), Box<dyn Error>> {\n",
" let labels = Labels::new(\"http://localhost:8000\");\n",
"\n",
- " let sections = [\"Wears a red suit and says ho ho\",\n",
- " \"Pulls a flying sleigh\",\n",
- " \"This is cut down and decorated\",\n",
- " \"Santa puts these under the tree\",\n",
- " \"Best way to spend the holidays\"];\n",
+ " let data = [\"Wears a red suit and says ho ho\",\n",
+ " \"Pulls a flying sleigh\",\n",
+ " \"This is cut down and decorated\",\n",
+ " \"Santa puts these under the tree\",\n",
+ " \"Best way to spend the holidays\"];\n",
"\n",
" println!(\"{:<40} {}\", \"Text\", \"Label\");\n",
" println!(\"{}\", \"-\".repeat(75));\n",
"\n",
- " for text in sections.iter() {\n",
+ " for text in data.iter() {\n",
" let tags = vec![\"πŸŽ… Santa Clause\", \"🦌 Reindeer\", \"πŸͺ Cookies\", \"πŸŽ„ Christmas Tree\", \"🎁 Gifts\", \"πŸ‘ͺ Family\"];\n",
- " let label = &labels.label(text, &tags.to_vec()).await?[0].id;\n",
+ " let label = tags[&labels.label(text, &tags.to_vec()).await?[0].id];\n",
"\n",
" println!(\"{:<40} {}\", text, label);\n",
" }\n",
@@ -620,7 +620,7 @@
" Ok(())\n",
"}"
],
- "execution_count": 14,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -653,7 +653,7 @@
"os.chdir(\"txtai.rs/examples/demo\")\n",
"!cargo build"
],
- "execution_count": 15,
+ "execution_count": null,
"outputs": []
},
{
@@ -668,7 +668,7 @@
"source": [
"!cargo run labels"
],
- "execution_count": 16,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -724,7 +724,7 @@
"!apt install golang-go\n",
"!go get \"github.com/neuml/txtai.go\""
],
- "execution_count": 17,
+ "execution_count": null,
"outputs": []
},
{
@@ -760,11 +760,11 @@
"func main() {\n",
"\tlabels := txtai.Labels(\"http://localhost:8000\")\n",
"\n",
- "\tsections := []string{\"Wears a red suit and says ho ho\",\n",
- " \"Pulls a flying sleigh\",\n",
- " \"This is cut down and decorated\",\n",
- " \"Santa puts these under the tree\",\n",
- " \"Best way to spend the holidays\"}\n",
+ "\tdata := []string{\"Wears a red suit and says ho ho\",\n",
+ " \"Pulls a flying sleigh\",\n",
+ " \"This is cut down and decorated\",\n",
+ " \"Santa puts these under the tree\",\n",
+ " \"Best way to spend the holidays\"}\n",
"\n",
"\t// List of labels\n",
"\ttags := []string{\"πŸŽ… Santa Clause\", \"🦌 Reindeer\", \"πŸͺ Cookies\", \"πŸŽ„ Christmas Tree\", \"🎁 Gifts\", \"πŸ‘ͺ Family\"}\n",
@@ -774,11 +774,11 @@
"\n",
"\tfor _, text := range sections {\n",
"\t\tscores := labels.Label(text, tags)\n",
- "\t\tfmt.Printf(\"%-40s %s\\n\", text, scores[0].Id)\n",
+ "\t\tfmt.Printf(\"%-40s %s\\n\", text, tags[scores[0].Id])\n",
"\t}\n",
"}"
],
- "execution_count": 18,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",
@@ -810,7 +810,7 @@
"source": [
"!go run labels.go"
],
- "execution_count": 19,
+ "execution_count": null,
"outputs": [
{
"output_type": "stream",