Skip to content

armgit / txtai

Public Imported from GitHub
0

đź’ˇ All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update notebook to show how to run workflow yaml in python

davidmezzetti8a1636b8c916
diff --git a/examples/22_Transform_tabular_data_with_composable_workflows.ipynb b/examples/22_Transform_tabular_data_with_composable_workflows.ipynb
index f62073e3..037ef0c8 100644
--- a/examples/22_Transform_tabular_data_with_composable_workflows.ipynb
+++ b/examples/22_Transform_tabular_data_with_composable_workflows.ipynb
@@ -47,7 +47,7 @@
"%%capture\n",
"!pip install git+https://github.com/neuml/txtai#egg=txtai[api,pipeline,workflow]"
],
- "execution_count": 61,
+ "execution_count": 93,
"outputs": []
},
{
@@ -75,7 +75,7 @@
"!head -1 metadata.csv > input.csv\n",
"!tail -10000 metadata.csv >> input.csv"
],
- "execution_count": 78,
+ "execution_count": 94,
"outputs": []
},
{
@@ -96,7 +96,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "8f7b3807-fa98-46b3-b1c3-ca1de6e2de55"
+ "outputId": "813cded8-b3fd-4600-a3ad-c75137a8c79b"
},
"source": [
"from txtai.pipeline import Tabular\n",
@@ -111,7 +111,7 @@
"# Print 5 rows of input.csv via workflow\n",
"list(workflow([\"input.csv\"]))[:5]"
],
- "execution_count": 63,
+ "execution_count": 95,
"outputs": [
{
"output_type": "execute_result",
@@ -135,7 +135,7 @@
]
},
"metadata": {},
- "execution_count": 63
+ "execution_count": 95
}
]
},
@@ -155,7 +155,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "G7M34puLWeZm",
- "outputId": "3ce7f13c-3c87-4595-d41c-f972fb37379c"
+ "outputId": "bea8d46e-36ba-4091-bee2-f411940eb7eb"
},
"source": [
"from txtai.embeddings import Embeddings\n",
@@ -171,7 +171,7 @@
" title = [text for url, text, _ in data if url == uid][0]\n",
" print(title, uid)"
],
- "execution_count": 64,
+ "execution_count": 96,
"outputs": [
{
"output_type": "stream",
@@ -213,7 +213,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "bA8SihkeZqbJ",
- "outputId": "353814c6-a1f3-4952-df35-a601e4b12dd7"
+ "outputId": "af921a0c-99d5-4d17-a0ea-ffd443704dcb"
},
"source": [
"from txtai.workflow import ServiceTask\n",
@@ -221,42 +221,42 @@
"service = ServiceTask(url=\"https://hn.algolia.com/api/v1/search\", method=\"get\", params={\"tags\": None}, batch=False, extract=\"hits\")\n",
"workflow = Workflow([service])\n",
"\n",
- "list(workflow([\"front_page\"]))[:1]"
+ "list(workflow([\"front_page\"]))[4]"
],
- "execution_count": 65,
+ "execution_count": 97,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
- "[{'_highlightResult': {'author': {'matchLevel': 'none',\n",
- " 'matchedWords': [],\n",
- " 'value': 'withzombies'},\n",
- " 'title': {'matchLevel': 'none',\n",
- " 'matchedWords': [],\n",
- " 'value': 'An opinionated guide on how to reverse engineer software'},\n",
- " 'url': {'matchLevel': 'none',\n",
- " 'matchedWords': [],\n",
- " 'value': 'https://margin.re/media/an-opinionated-guide-on-how-to-reverse-engineer-software-part-1.aspx'}},\n",
- " '_tags': ['story', 'author_withzombies', 'story_29084716', 'front_page'],\n",
- " 'author': 'withzombies',\n",
- " 'comment_text': None,\n",
- " 'created_at': '2021-11-02T17:43:47.000Z',\n",
- " 'created_at_i': 1635875027,\n",
- " 'num_comments': 25,\n",
- " 'objectID': '29084716',\n",
- " 'parent_id': None,\n",
- " 'points': 254,\n",
- " 'story_id': None,\n",
- " 'story_text': None,\n",
- " 'story_title': None,\n",
- " 'story_url': None,\n",
- " 'title': 'An opinionated guide on how to reverse engineer software',\n",
- " 'url': 'https://margin.re/media/an-opinionated-guide-on-how-to-reverse-engineer-software-part-1.aspx'}]"
+ "{'_highlightResult': {'author': {'matchLevel': 'none',\n",
+ " 'matchedWords': [],\n",
+ " 'value': 'makerdiety'},\n",
+ " 'title': {'matchLevel': 'none',\n",
+ " 'matchedWords': [],\n",
+ " 'value': 'Tips For Making a Popular Open Source Project in 2021'},\n",
+ " 'url': {'matchLevel': 'none',\n",
+ " 'matchedWords': [],\n",
+ " 'value': 'https://skerritt.blog/make-popular-open-source-projects/'}},\n",
+ " '_tags': ['story', 'author_makerdiety', 'story_29197806', 'front_page'],\n",
+ " 'author': 'makerdiety',\n",
+ " 'comment_text': None,\n",
+ " 'created_at': '2021-11-12T10:06:45.000Z',\n",
+ " 'created_at_i': 1636711605,\n",
+ " 'num_comments': 53,\n",
+ " 'objectID': '29197806',\n",
+ " 'parent_id': None,\n",
+ " 'points': 138,\n",
+ " 'story_id': None,\n",
+ " 'story_text': None,\n",
+ " 'story_title': None,\n",
+ " 'story_url': None,\n",
+ " 'title': 'Tips For Making a Popular Open Source Project in 2021',\n",
+ " 'url': 'https://skerritt.blog/make-popular-open-source-projects/'}"
]
},
"metadata": {},
- "execution_count": 65
+ "execution_count": 97
}
]
},
@@ -276,7 +276,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "YAbwhsaveKo1",
- "outputId": "286b19b4-3008-4a09-df95-87f3c1c9b44e"
+ "outputId": "d7f8eecc-566f-4233-e348-52ed32bff1ed"
},
"source": [
"from txtai.workflow import Task\n",
@@ -285,21 +285,21 @@
"service = ServiceTask(action=tabular, url=\"https://hn.algolia.com/api/v1/search\", method=\"get\", params={\"tags\": None}, batch=False, extract=\"hits\")\n",
"workflow = Workflow([service])\n",
"\n",
- "list(workflow([\"front_page\"]))[:1]"
+ "list(workflow([\"front_page\"]))[4]"
],
- "execution_count": 66,
+ "execution_count": 98,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
- "[('https://margin.re/media/an-opinionated-guide-on-how-to-reverse-engineer-software-part-1.aspx',\n",
- " 'An opinionated guide on how to reverse engineer software',\n",
- " None)]"
+ "('https://skerritt.blog/make-popular-open-source-projects/',\n",
+ " 'Tips For Making a Popular Open Source Project in 2021',\n",
+ " None)"
]
},
"metadata": {},
- "execution_count": 66
+ "execution_count": 98
}
]
},
@@ -319,7 +319,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "036949d0-d2e0-45db-cad0-b5e5605682fd"
+ "outputId": "58a2b1bc-e431-43cf-8938-9e662bd038f2"
},
"source": [
"# Embeddings with sentence-transformers backend\n",
@@ -329,19 +329,19 @@
"data = list(workflow([\"front_page\"]))\n",
"embeddings.index(data)\n",
"\n",
- "for uid, _ in embeddings.search(\"cloud\"):\n",
+ "for uid, _ in embeddings.search(\"programming\"):\n",
" title = [text for url, text, _ in data if url == uid][0]\n",
" print(title, uid)"
],
- "execution_count": 67,
+ "execution_count": 99,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "How Temporal Workload Shifting Can Reduce Carbon Emissions in the Cloud https://arxiv.org/abs/2110.13234\n",
- "Hetzner Cloud Goes US (Ashburn/VA) https://twitter.com/Hetzner_Online/status/1455837994259210242\n",
- "Hetzner cloud servers are now available in Ashburn USA https://www.hetzner.com/news/11-21-usa-cloud\n"
+ "A guide to organizing settings in Django https://apibakery.com/blog/django-settings-howto/\n",
+ "Useful sed scripts and patterns https://github.com/adrianscheff/useful-sed\n",
+ "Tips For Making a Popular Open Source Project in 2021 https://skerritt.blog/make-popular-open-source-projects/\n"
]
}
]
@@ -366,7 +366,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "K6CbS2QwltGi",
- "outputId": "5734d3a4-83eb-4a69-eb10-d1243809fbdd"
+ "outputId": "2836a8d1-cc7b-46b6-9fc9-08aef2701ecb"
},
"source": [
"service = ServiceTask(url=\"http://export.arxiv.org/api/query\", method=\"get\", params={\"search_query\": None, \"max_results\": 25}, batch=False, extract=[\"feed\", \"entry\"])\n",
@@ -374,7 +374,7 @@
"\n",
"list(workflow([\"all:aliens\"]))[:1]"
],
- "execution_count": 68,
+ "execution_count": 100,
"outputs": [
{
"output_type": "execute_result",
@@ -416,7 +416,7 @@
]
},
"metadata": {},
- "execution_count": 68
+ "execution_count": 100
}
]
},
@@ -436,7 +436,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "DyIetJ7OmJjP",
- "outputId": "fe98dbed-a7a1-4c36-f1b7-d2f81d632239"
+ "outputId": "5355c37c-f335-4278-c579-fd2fcdd6cea6"
},
"source": [
"from txtai.workflow import Task\n",
@@ -450,7 +450,7 @@
"\n",
"list(workflow([\"all:aliens\"]))[:1]"
],
- "execution_count": 69,
+ "execution_count": 101,
"outputs": [
{
"output_type": "execute_result",
@@ -462,7 +462,7 @@
]
},
"metadata": {},
- "execution_count": 69
+ "execution_count": 101
}
]
},
@@ -482,7 +482,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "NX2oR5dhm_99",
- "outputId": "2763cf1b-0c0f-4d46-9ae4-0646a8b178be"
+ "outputId": "cabfebcd-e63b-4a50-a182-112e09bc2d51"
},
"source": [
"# Embeddings with sentence-transformers backend\n",
@@ -496,7 +496,7 @@
" title = [text for url, text, _ in data if url == uid][0]\n",
" print(title, uid)"
],
- "execution_count": 70,
+ "execution_count": 102,
"outputs": [
{
"output_type": "stream",
@@ -505,7 +505,7 @@
"Calculating the probability of detecting radio signals from alien\n",
" civilizations http://arxiv.org/abs/0707.0011v2\n",
"Field Trial of Alien Wavelengths on GARR Optical Network http://arxiv.org/abs/1805.04278v1\n",
- "Aliens on Earth. Are reports of close encounters correct? http://arxiv.org/abs/1203.6805v2\n"
+ "Do alien particles exist, and can they be detected? http://arxiv.org/abs/1606.07403v1\n"
]
}
]
@@ -528,7 +528,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "1eF5IJlzpNbw",
- "outputId": "1814a2b8-855a-4945-dd3c-ee95a44c79a4"
+ "outputId": "10e8be56-6456-46b8-8fe7-2b192422da9c"
},
"source": [
"%%writefile workflow.yml\n",
@@ -539,7 +539,7 @@
"\n",
"# Tabular pipeline\n",
"tabular:\n",
- " idcolumn: url\n",
+ " idcolumn: id\n",
" textcolumns: \n",
" - title\n",
"\n",
@@ -549,15 +549,15 @@
" tasks:\n",
" - task: service\n",
" action: tabular\n",
- " url: https://hn.algolia.com/api/v1/search\n",
+ " url: http://export.arxiv.org/api/query?max_results=25\n",
" method: get\n",
" params:\n",
- " tags: null\n",
+ " search_query: null\n",
" batch: false\n",
- " extract: hits\n",
+ " extract: [feed, entry]\n",
" - action: upsert"
],
- "execution_count": 71,
+ "execution_count": 103,
"outputs": [
{
"output_type": "stream",
@@ -574,7 +574,7 @@
"id": "dxi5w3IezR7Q"
},
"source": [
- "This workflow connects to the Hacker News API, gets the data for the front page and indexes it. The workflow configures the same actions that were configured in Python previously. \n",
+ "This workflow once again runs an arXiv query and indexes article titles. The workflow configures the same actions that were configured in Python previously. \n",
"\n",
"Let's start an API instance "
]
@@ -586,7 +586,7 @@
"colab": {
"base_uri": "https://localhost:8080/"
},
- "outputId": "20e62c95-4d85-45b8-e622-98ad200b3cb3"
+ "outputId": "6c0b76f9-f73d-4de7-bc6f-c857bd2e4724"
},
"source": [
"!killall -9 uvicorn\n",
@@ -594,13 +594,13 @@
"!sleep 30\n",
"!cat api.log"
],
- "execution_count": 72,
+ "execution_count": 104,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "INFO: Started server process [911]\n",
+ "INFO: Started server process [921]\n",
"INFO: Waiting for application startup.\n",
"INFO: Application startup complete.\n",
"INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)\n"
@@ -624,19 +624,19 @@
"base_uri": "https://localhost:8080/"
},
"id": "xt_qL6eA0SrS",
- "outputId": "b7d0c4cd-d612-461a-ddb1-c104882fa9fd"
+ "outputId": "bfc5cda3-2c99-40c0-90f8-dbaf470ae745"
},
"source": [
"# Execute workflow via API call\n",
- "!curl -X POST \"http://localhost:8000/workflow\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -d \"{\\\"name\\\":\\\"index\\\",\\\"elements\\\":[\\\"front_page\\\"]}\""
+ "!curl -X POST \"http://localhost:8000/workflow\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -d \"{\\\"name\\\":\\\"index\\\",\\\"elements\\\":[\\\"all:aliens\\\"]}\""
],
- "execution_count": 73,
+ "execution_count": 105,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[[\"https://margin.re/media/an-opinionated-guide-on-how-to-reverse-engineer-software-part-1.aspx\",\"An opinionated guide on how to reverse engineer software\",null],[\"https://jichu4n.com/posts/how-x-window-managers-work-and-how-to-write-one-part-i/\",\"How X Window Managers Work, and How to Write One (2014)\",null],[\"https://mattferraro.dev/posts/geometric-algebra\",\"What is the inverse of a vector?\",null],[\"https://www.hetzner.com/news/11-21-usa-cloud\",\"Hetzner cloud servers are now available in Ashburn USA\",null],[\"https://zigmonthly.org/letters/2021/october/\",\"Zig monthly, October 2021: Games, gamedev, Elixir, tools and more\",null],[\"https://twitter.com/Hetzner_Online/status/1455837994259210242\",\"Hetzner Cloud Goes US (Ashburn/VA)\",null],[\"https://zig.news/sobeston/using-zig-and-translate-c-to-understand-weird-c-code-4f8\",\"Using Zig's translate-C to understand weird C code\",null],[\"https://real-italian-coffee.herokuapp.com/home\",\"Show HN: A generator of Fake Italian Coffee names\",null],[\"https://www.sciencealert.com/useless-specks-of-dust-turn-out-to-be-ancient-building-blocks-of-all-vertebrate-genomes\",\"'Useless Specks of Dust' Turn Out to Be Building Blocks of All Vertebrate Genome\",null],[\"https://olivierlacan.com/posts/high-fidelity-remote-communication/\",\"High-fidelity remote communication\",null],[\"https://github.com/Qinti/nazca\",\"Nazca – New GUI for the Web\",null],[\"https://susam.in/blog/microcentury.html\",\"Microcentury (2020)\",null],[\"https://addons.mozilla.org/de/firefox/addon/downthemall/\",\"DownThemAll\",null],[\"https://arxiv.org/abs/2110.13234\",\"How Temporal Workload Shifting Can Reduce Carbon Emissions in the Cloud\",null],[\"https://ideas.krishnan.ca/are-daily-standups-hurting-your-team\",\"Are daily standups hurting your team?\",null],[\"https://sethkim.me/l/thesolutionspace/\",\"Automation != Leverage\",null],[\"https://www.listennotes.com/blog/the-boring-technology-behind-a-one-person-23/\",\"The boring technology behind a one-person Internet company\",null],[\"https://www.vox.com/the-goods/22725031/buying-less-supply-chain-holiday-shopping\",\"It’s time for Americans to buy less stuff\",null],[\"https://blog.ilograph.com/posts/beyond-whiteboarding-creating-expert-system-architecture-diagrams/\",\"Beyond Whiteboarding: Creating Expert System Architecture Diagrams\",null],[\"https://www.sciencemuseumgroup.org.uk/blog/covid19-effect-on-the-vaccinated/\",\"Covid-19: How the virus spreads among vaccinated people\",null]]"
+ "[[\"http://arxiv.org/abs/2102.01522v3\",\"If Loud Aliens Explain Human Earliness, Quiet Aliens Are Also Rare\",null],[\"http://arxiv.org/abs/cs/0306071v1\",\"AliEnFS - a Linux File System for the AliEn Grid Services\",null],[\"http://arxiv.org/abs/physics/0306103v1\",\"AliEn - EDG Interoperability in ALICE\",null],[\"http://arxiv.org/abs/2103.05559v1\",\"Oumuamua Is Not a Probe Sent to our Solar System by an Alien\\n Civilization\",null],[\"http://arxiv.org/abs/1403.3979v1\",\"Robust transitivity and density of periodic points of partially\\n hyperbolic diffeomorphisms\",null],[\"http://arxiv.org/abs/1712.09210v1\",\"Sampling alien species inside and outside protected areas: does it\\n matter?\",null],[\"http://arxiv.org/abs/cs/0306067v1\",\"The AliEn system, status and perspectives\",null],[\"http://arxiv.org/abs/0707.0011v2\",\"Calculating the probability of detecting radio signals from alien\\n civilizations\",null],[\"http://arxiv.org/abs/1805.04278v1\",\"Field Trial of Alien Wavelengths on GARR Optical Network\",null],[\"http://arxiv.org/abs/1808.00529v1\",\"Open Category Detection with PAC Guarantees\",null],[\"http://arxiv.org/abs/1206.3640v1\",\"The Study of Climate on Alien Worlds\",null],[\"http://arxiv.org/abs/1203.6805v2\",\"Aliens on Earth. Are reports of close encounters correct?\",null],[\"http://arxiv.org/abs/1604.05078v1\",\"The Imprecise Search for Habitability\",null],[\"http://arxiv.org/abs/1006.2613v1\",\"Resurgence, Stokes phenomenon and alien derivatives for level-one linear\\n differential systems\",null],[\"http://arxiv.org/abs/1705.03394v1\",\"That is not dead which can eternal lie: the aestivation hypothesis for\\n resolving Fermi's paradox\",null],[\"http://arxiv.org/abs/1701.02294v1\",\"Alien Calculus and non perturbative effects in Quantum Field Theory\",null],[\"http://arxiv.org/abs/1307.0653v1\",\"General and alien solutions of a functional equation and of a functional\\n inequality\",null],[\"http://arxiv.org/abs/1801.06180v1\",\"Are Alien Civilizations Technologically Advanced?\",null],[\"http://arxiv.org/abs/1902.05387v1\",\"Simultaneous x, y Pixel Estimation and Feature Extraction for Multiple\\n Small Objects in a Scene: A Description of the ALIEN Network\",null],[\"http://arxiv.org/abs/0711.4034v1\",\"The q-analogue of the wild fundamental group (II)\",null],[\"http://arxiv.org/abs/astro-ph/0501119v1\",\"Expanding advanced civilizations in the universe\",null],[\"http://arxiv.org/abs/cs/0306068v1\",\"AliEn Resource Brokers\",null],[\"http://arxiv.org/abs/hep-ph/9403231v2\",\"The Renormalization of Composite Operators in Yang-Mills Theories Using\\n General Covariant Gauge\",null],[\"http://arxiv.org/abs/1410.0501v1\",\"Alienation in Italian cities. Social network fragmentation from\\n collective data\",null],[\"http://arxiv.org/abs/1606.07403v1\",\"Do alien particles exist, and can they be detected?\",null]]"
]
}
]
@@ -647,7 +647,7 @@
"id": "_bwn4KBt1Cos"
},
"source": [
- "The data is now indexed. Note that the index configuration has an `upsert` action. Each workflow call will insert new rows or update existing rows. This call could be scheduled with a system cron to execute periodically and build an index of top stories on Hacker News over time. \n",
+ "The data is now indexed. Note that the index configuration has an `upsert` action. Each workflow call will insert new rows or update existing rows. This call could be scheduled with a system cron to execute periodically and build an index of arXiv article titles. \n",
"\n",
"Now that the index is ready, let's run a search."
]
@@ -659,19 +659,19 @@
"base_uri": "https://localhost:8080/"
},
"id": "qbteIueJ1Fds",
- "outputId": "c7dcbbc9-a9a9-4640-ea7e-d7dfcbefcc1a"
+ "outputId": "97eb4fa7-e660-417e-c562-b1937e8e4f99"
},
"source": [
"# Run a search\n",
- "!curl -X GET \"http://localhost:8000/search?query=cloud&limit=3\" -H \"accept: application/json\""
+ "!curl -X GET \"http://localhost:8000/search?query=radio&limit=3\" -H \"accept: application/json\""
],
- "execution_count": 74,
+ "execution_count": 106,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[{\"id\":\"https://twitter.com/Hetzner_Online/status/1455837994259210242\",\"score\":0.475553035736084},{\"id\":\"https://arxiv.org/abs/2110.13234\",\"score\":0.3434651494026184},{\"id\":\"https://olivierlacan.com/posts/high-fidelity-remote-communication/\",\"score\":0.2550020217895508}]"
+ "[{\"id\":\"http://arxiv.org/abs/0707.0011v2\",\"score\":0.40350067615509033},{\"id\":\"http://arxiv.org/abs/1805.04278v1\",\"score\":0.34062114357948303},{\"id\":\"http://arxiv.org/abs/1902.05387v1\",\"score\":0.22262515127658844}]"
]
}
]
@@ -684,7 +684,7 @@
"source": [
"# Add a translation step to workflow\n",
"\n",
- "Next we'll recreate the workflow, adding one additional step, translating the text into French before indexing. This workflow gets the top results from Hacker News, translates the results and builds an semantic index of titles in French. "
+ "Next we'll recreate the workflow, adding one additional step, translating the text into French before indexing. This workflow runs an arXiv query, translates the results and builds an semantic index of titles in French. "
]
},
{
@@ -694,7 +694,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "j8rBVl17293q",
- "outputId": "20ce444a-eb9c-4ac3-8edf-1d14c4b7b528"
+ "outputId": "03858923-19aa-417b-cf8b-b7e78056c51e"
},
"source": [
"%%writefile workflow.yml\n",
@@ -705,7 +705,7 @@
"\n",
"# Tabular pipeline\n",
"tabular:\n",
- " idcolumn: url\n",
+ " idcolumn: id\n",
" textcolumns: \n",
" - title\n",
"\n",
@@ -718,17 +718,17 @@
" tasks:\n",
" - task: service\n",
" action: tabular\n",
- " url: https://hn.algolia.com/api/v1/search\n",
+ " url: http://export.arxiv.org/api/query?max_results=25\n",
" method: get\n",
" params:\n",
- " tags: null\n",
+ " search_query: null\n",
" batch: false\n",
- " extract: hits\n",
+ " extract: [feed, entry]\n",
" - action: translation\n",
" args: [fr]\n",
" - action: upsert"
],
- "execution_count": 75,
+ "execution_count": 107,
"outputs": [
{
"output_type": "stream",
@@ -746,7 +746,7 @@
"base_uri": "https://localhost:8080/"
},
"id": "UQWvvgb2CwgG",
- "outputId": "52a051a8-b3ae-4d59-8e37-034cc62b0c09"
+ "outputId": "90c8be97-716b-4dcd-be6a-3fc5cfd105fc"
},
"source": [
"!killall -9 uvicorn\n",
@@ -754,13 +754,13 @@
"!sleep 30\n",
"!cat api.log"
],
- "execution_count": 76,
+ "execution_count": 108,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "INFO: Started server process [935]\n",
+ "INFO: Started server process [945]\n",
"INFO: Waiting for application startup.\n",
"INFO: Application startup complete.\n",
"INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)\n"
@@ -784,22 +784,80 @@
"base_uri": "https://localhost:8080/"
},
"id": "npW3rjCw6_nx",
- "outputId": "b73ec9db-b42c-40fa-c5ec-84d2a1bcb674"
+ "outputId": "8bfa2cb0-579d-442f-9f6d-cd5df4531dbe"
},
"source": [
"# Execute workflow via API call\n",
- "!curl -s -X POST \"http://localhost:8000/workflow\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -d \"{\\\"name\\\":\\\"index\\\",\\\"elements\\\":[\\\"front_page\\\"]}\" > /dev/null\n",
+ "!curl -s -X POST \"http://localhost:8000/workflow\" -H \"accept: application/json\" -H \"Content-Type: application/json\" -d \"{\\\"name\\\":\\\"index\\\",\\\"elements\\\":[\\\"all:aliens\\\"]}\" > /dev/null\n",
+ "\n",
+ "# Run a search\n",
+ "!curl -X GET \"http://localhost:8000/search?query=radio&limit=3\" -H \"accept: application/json\""
+ ],
+ "execution_count": 109,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[{\"id\":\"http://arxiv.org/abs/0707.0011v2\",\"score\":0.5328004956245422},{\"id\":\"http://arxiv.org/abs/0711.4034v1\",\"score\":0.2441330999135971},{\"id\":\"http://arxiv.org/abs/2102.01522v3\",\"score\":0.22881504893302917}]"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "OsQO5DG9zBQF"
+ },
+ "source": [
+ "# Run YAML workflow in Python\n",
+ "\n",
+ "Workflow YAML files can also be directly executed in Python. In this case, all input data is passed locally in Python and not through network interfaces. The following section shows how to do this!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "iXmkLDTlzPT3",
+ "outputId": "7446952f-3116-4bca-85e4-6553866827b0"
+ },
+ "source": [
+ "import yaml\n",
+ "\n",
+ "from txtai.api import API\n",
+ "\n",
+ "with open(\"workflow.yml\") as config:\n",
+ " workflow = yaml.safe_load(config)\n",
+ "\n",
+ "app = API(workflow)\n",
+ "\n",
+ "# Run the workflow\n",
+ "data = app.workflow(\"index\", [\"all:aliens\"])\n",
"\n",
"# Run a search\n",
- "!curl -X GET \"http://localhost:8000/search?query=nuage&limit=3\" -H \"accept: application/json\""
+ "for result in app.search(\"radio\", None):\n",
+ " text = [row[1] for row in data if row[0] == result[\"id\"]][0]\n",
+ " print(result[\"id\"], result[\"score\"], text)"
],
- "execution_count": 77,
+ "execution_count": 113,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "[{\"id\":\"https://arxiv.org/abs/2110.13234\",\"score\":0.46083423495292664},{\"id\":\"https://twitter.com/Hetzner_Online/status/1455837994259210242\",\"score\":0.37807518243789673},{\"id\":\"https://jichu4n.com/posts/how-x-window-managers-work-and-how-to-write-one-part-i/\",\"score\":0.36641401052474976}]"
+ "http://arxiv.org/abs/0707.0011v2 0.5328004956245422 Calcul de la probabilité de détection des signaux radio de l'étrangercivilisations\n",
+ "http://arxiv.org/abs/0711.4034v1 0.2441330999135971 Le q-analogue du groupe fondamental sauvage (II)\n",
+ "http://arxiv.org/abs/2102.01522v3 0.22881504893302917 Si les étrangers louds expliquent le début de l'humanité, les étrangers tranquilles sont aussi rares\n",
+ "http://arxiv.org/abs/physics/0306103v1 0.21307508647441864 Alien - EDG Interopérabilité en ALICE\n",
+ "http://arxiv.org/abs/1006.2613v1 0.19786792993545532 Résurgence, phénomène Stokes et dérivés extraterrestres pour le niveau 1 linéairesystèmes différentiels\n",
+ "http://arxiv.org/abs/1403.3979v1 0.1915999799966812 Transitivité robuste et densité des points périodiques en partiedifféomorphismes hyperboliques\n",
+ "http://arxiv.org/abs/1805.04278v1 0.19029255211353302 Essai sur le terrain des longueurs d'onde aliens sur le réseau optique GARR\n",
+ "http://arxiv.org/abs/1606.07403v1 0.17492449283599854 Les particules exotiques existent-elles et peuvent-elles être détectées?\n",
+ "http://arxiv.org/abs/1902.05387v1 0.17426751554012299 Simultanée x, y Pixel Estimation et Extraction de Caractéristiques pour MultiplePetits objets dans une scène : une description du réseau ALIEN\n",
+ "http://arxiv.org/abs/cs/0306071v1 0.17286795377731323 AliEnFS - un système de fichiers Linux pour les services AliEn Grid\n"
]
}
]