💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update notebook
davidmezzettib43cdc1f4354
diff --git a/examples/24_Whats_new_in_txtai_4_0.ipynb b/examples/24_Whats_new_in_txtai_4_0.ipynbindex 424aef69..13de0b14 100644--- a/examples/24_Whats_new_in_txtai_4_0.ipynb+++ b/examples/24_Whats_new_in_txtai_4_0.ipynb@@ -63,7 +63,7 @@"%%capture\n","!pip install git+https://github.com/neuml/txtai"],- "execution_count": null,+ "execution_count": 21,"outputs": []},{@@ -90,7 +90,7 @@"colab": {"base_uri": "https://localhost:8080/"},- "outputId": "8c7ff0df-e447-47fc-a5b5-480fcd4df01d"+ "outputId": "c37d3b1d-1096-4518-92fa-1950cb69b183"},"source": ["from txtai.embeddings import Embeddings\n",@@ -119,7 +119,7 @@" # Print text\n"," print(\"%-20s %s\" % (query, text))"],- "execution_count": null,+ "execution_count": 22,"outputs": [{"output_type": "stream",@@ -182,9 +182,9 @@"base_uri": "https://localhost:8080/"},"id": "aPH-dnV2ZuL1",- "outputId": "13b58387-0a76-466e-84bc-b51c8f0427dd"+ "outputId": "7946a782-8b9c-4ee5-abcc-ce0f45858349"},- "execution_count": null,+ "execution_count": 23,"outputs": [{"output_type": "stream",@@ -251,9 +251,9 @@"base_uri": "https://localhost:8080/","height": 420},- "outputId": "2eeec5fc-260b-42f0-eae5-4957da37684c"+ "outputId": "3dfe3593-6a07-4277-b388-4983e81c5e8a"},- "execution_count": null,+ "execution_count": 24,"outputs": [{"output_type": "execute_result",@@ -268,7 +268,7 @@"width": 600}},- "execution_count": 40+ "execution_count": 24}]},@@ -302,9 +302,9 @@"base_uri": "https://localhost:8080/"},"id": "fAz7Jv6SjR6a",- "outputId": "d9b6f387-3c5b-4fd2-968d-bcee43d91ae2"+ "outputId": "0ea984bf-a9ab-42fc-fa18-3f59bd8ef2b9"},- "execution_count": null,+ "execution_count": 25,"outputs": [{"output_type": "stream",@@ -313,7 +313,7 @@"{\n"," \"backend\": \"faiss\",\n"," \"build\": {\n",- " \"create\": \"2021-12-28T14:29:24Z\",\n",+ " \"create\": \"2021-12-29T00:42:17Z\",\n"," \"python\": \"3.7.12\",\n"," \"settings\": {\n"," \"components\": \"IDMap,Flat\"\n",@@ -331,7 +331,7 @@"{\n"," \"backend\": \"faiss\",\n"," \"build\": {\n",- " \"create\": \"2021-12-28T14:29:30Z\",\n",+ " \"create\": \"2021-12-29T00:42:29Z\",\n"," \"python\": \"3.7.12\",\n"," \"settings\": {\n"," \"components\": \"IDMap,Flat\"\n",@@ -383,18 +383,18 @@"base_uri": "https://localhost:8080/"},"id": "0oOC8ToG1pyn",- "outputId": "bba1aa74-ef50-4137-bc5f-38f1961dd794"+ "outputId": "bd42dbbe-0de1-4fe3-97b9-602992bdca5a"},- "execution_count": null,+ "execution_count": 26,"outputs": [{"output_type": "stream","name": "stdout","text": [- "drwx------ root/root 0 2021-12-28 14:29 ./\n",- "-rw-r--r-- root/root 290 2021-12-28 14:29 ./config\n",- "-rw-r--r-- root/root 77824 2021-12-28 14:29 ./documents\n",- "-rw-r--r-- root/root 10898 2021-12-28 14:29 ./embeddings\n",+ "drwx------ root/root 0 2021-12-29 00:42 ./\n",+ "-rw-r--r-- root/root 290 2021-12-29 00:42 ./config\n",+ "-rw-r--r-- root/root 77824 2021-12-29 00:42 ./documents\n",+ "-rw-r--r-- root/root 10898 2021-12-29 00:42 ./embeddings\n","\n","Strms Blocks Compressed Uncompressed Ratio Check Filename\n"," 1 1 45.8 KiB 100.0 KiB 0.458 CRC64 index.tar.xz\n",@@ -411,7 +411,7 @@]},"metadata": {},- "execution_count": 42+ "execution_count": 26}]},@@ -424,6 +424,68 @@"id": "x5iSRKZVYfok"}},+ {+ "cell_type": "markdown",+ "source": [+ "# External vector models\n",+ "\n",+ "txtai supports generating vectors with [Hugging Face Transformers](https://github.com/huggingface/transformers), [PyTorch](https://github.com/pytorch/pytorch), [ONNX](https://github.com/microsoft/onnxruntime) and [Word Vector](https://github.com/neuml/magnitude) models.\n",+ "\n",+ "This release adds support for pre-computed vectors using external models. External models may be an API, custom library and/or another way to vectorize data. The example below uses [spaCy](https://github.com/explosion/spaCy) to build vectors and then loads them into txtai. "+ ],+ "metadata": {+ "id": "oAISaBYVg60i"+ }+ },+ {+ "cell_type": "code",+ "source": [+ "%%capture\n",+ "!pip install spacy --upgrade\n",+ "!python -m spacy download en_core_web_md"+ ],+ "metadata": {+ "id": "IIGlel3ShyGP"+ },+ "execution_count": 27,+ "outputs": []+ },+ {+ "cell_type": "code",+ "source": [+ "import spacy\n",+ "\n",+ "# Load spacy\n",+ "nlp = spacy.load(\"en_core_web_md\")\n",+ "\n",+ "def transform(document):\n",+ " return nlp(document[1]).vector\n",+ "\n",+ "# Index data with spacy pipeline\n",+ "embeddings = Embeddings({\"method\": \"external\", \"transform\": transform, \"content\": True})\n",+ "embeddings.index([(x, {\"object\": nlp(row).vector, \"row\": row}, None) for x, row in enumerate(data)])\n",+ "\n",+ "# Run search\n",+ "print(embeddings.search(\"select id, row as text, score from txtai where similar('hiking danger')\", 1))"+ ],+ "metadata": {+ "colab": {+ "base_uri": "https://localhost:8080/"+ },+ "id": "nikKybxEkhjC",+ "outputId": "84950459-8d2b-4570-b7ed-ad468c783901"+ },+ "execution_count": 28,+ "outputs": [+ {+ "output_type": "stream",+ "name": "stdout",+ "text": [+ "[{'id': '3', 'text': 'The National Park Service warns against sacrificing slower friends in a bear attack', 'score': 0.5822835564613342}]\n"+ ]+ }+ ]+ },{"cell_type": "markdown","metadata": {