Convert any text to a graph of knowledge. This can be used for Graph Augmented Generation or Knowledge Graph based QnA
HTML
cleanup
Nayak0977b9bf77be
diff --git a/.gitignore b/.gitignoreindex 652953f..a9334ac 100644--- a/.gitignore+++ b/.gitignore@@ -1,3 +1,4 @@*/__pycache__*/.DS_Store-.DS_Store\ No newline at end of file+.DS_Store+old_notebooks\ No newline at end of filediff --git a/concept_graph.ipynb b/concept_graph.ipynbdeleted file mode 100644index 14eb0f8..0000000--- a/concept_graph.ipynb+++ /dev/null@@ -1,1043 +0,0 @@-{- "cells": [- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# Create Network Graph\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Basic setup\n"- ]- },- {- "cell_type": "code",- "execution_count": 268,- "metadata": {},- "outputs": [],- "source": [- "import pandas as pd\n",- "import numpy as np\n",- "from pyvis.network import Network\n",- "import networkx as nx\n",- "import seaborn as sns"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Loading previously calculated dataframes\n"- ]- },- {- "cell_type": "code",- "execution_count": 269,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "(359, 5)\n"- ]- },- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity</th>\n",- " <th>importance</th>\n",- " <th>category</th>\n",- " <th>chunk_id</th>\n",- " <th>type</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>interoperability</td>\n",- " <td>5</td>\n",- " <td>concept</td>\n",- " <td>bb894d4a987241c6b7827d4acabac6dc</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>introduction</td>\n",- " <td>3</td>\n",- " <td>event</td>\n",- " <td>bb894d4a987241c6b7827d4acabac6dc</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>unique charging port design</td>\n",- " <td>4</td>\n",- " <td>misc</td>\n",- " <td>ebcfb9727a42482d986fa2b0a64fa8ac</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>usb c</td>\n",- " <td>5</td>\n",- " <td>object</td>\n",- " <td>ebcfb9727a42482d986fa2b0a64fa8ac</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>mobile phone model</td>\n",- " <td>3</td>\n",- " <td>object</td>\n",- " <td>ebcfb9727a42482d986fa2b0a64fa8ac</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " entity importance category \\\n",- "0 interoperability 5 concept \n",- "1 introduction 3 event \n",- "2 unique charging port design 4 misc \n",- "3 usb c 5 object \n",- "4 mobile phone model 3 object \n",- "\n",- " chunk_id type \n",- "0 bb894d4a987241c6b7827d4acabac6dc concept \n",- "1 bb894d4a987241c6b7827d4acabac6dc concept \n",- "2 ebcfb9727a42482d986fa2b0a64fa8ac concept \n",- "3 ebcfb9727a42482d986fa2b0a64fa8ac concept \n",- "4 ebcfb9727a42482d986fa2b0a64fa8ac concept "- ]- },- "execution_count": 269,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "dataframe_dir = \"MediumArticles\"\n",- "# df = pd.read_csv(f\"./data_output/{dataframe_dir}/chunks.csv\", sep=\"|\")\n",- "df_concepts = pd.read_csv(f\"./data_output/{dataframe_dir}/concepts.csv\", sep=\"|\")\n",- "print(df_concepts.shape)\n",- "df_concepts.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Calculate Graph Dataframe\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Graph dataframe is a dataframe where every row is a connection between two nodes.\n",- "\n",- "It is basically an inner self join of the nodes dataframe\n"- ]- },- {- "cell_type": "code",- "execution_count": 240,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Total number of nodes = 2570\n"- ]- },- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>index</th>\n",- " <th>entity_L</th>\n",- " <th>importance_L</th>\n",- " <th>category_L</th>\n",- " <th>chunk_id</th>\n",- " <th>type_L</th>\n",- " <th>entity_R</th>\n",- " <th>importance_R</th>\n",- " <th>category_R</th>\n",- " <th>type_R</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>1</td>\n",- " <td>interoperability</td>\n",- " <td>5</td>\n",- " <td>concept</td>\n",- " <td>bb894d4a987241c6b7827d4acabac6dc</td>\n",- " <td>concept</td>\n",- " <td>introduction</td>\n",- " <td>3</td>\n",- " <td>event</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>2</td>\n",- " <td>introduction</td>\n",- " <td>3</td>\n",- " <td>event</td>\n",- " <td>bb894d4a987241c6b7827d4acabac6dc</td>\n",- " <td>concept</td>\n",- " <td>interoperability</td>\n",- " <td>5</td>\n",- " <td>concept</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>5</td>\n",- " <td>unique charging port design</td>\n",- " <td>4</td>\n",- " <td>misc</td>\n",- " <td>ebcfb9727a42482d986fa2b0a64fa8ac</td>\n",- " <td>concept</td>\n",- " <td>usb c</td>\n",- " <td>5</td>\n",- " <td>object</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>6</td>\n",- " <td>unique charging port design</td>\n",- " <td>4</td>\n",- " <td>misc</td>\n",- " <td>ebcfb9727a42482d986fa2b0a64fa8ac</td>\n",- " <td>concept</td>\n",- " <td>mobile phone model</td>\n",- " <td>3</td>\n",- " <td>object</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>7</td>\n",- " <td>unique charging port design</td>\n",- " <td>4</td>\n",- " <td>misc</td>\n",- " <td>ebcfb9727a42482d986fa2b0a64fa8ac</td>\n",- " <td>concept</td>\n",- " <td>phone companies</td>\n",- " <td>4</td>\n",- " <td>organisation</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " index entity_L importance_L category_L \\\n",- "0 1 interoperability 5 concept \n",- "1 2 introduction 3 event \n",- "2 5 unique charging port design 4 misc \n",- "3 6 unique charging port design 4 misc \n",- "4 7 unique charging port design 4 misc \n",- "\n",- " chunk_id type_L entity_R \\\n",- "0 bb894d4a987241c6b7827d4acabac6dc concept introduction \n",- "1 bb894d4a987241c6b7827d4acabac6dc concept interoperability \n",- "2 ebcfb9727a42482d986fa2b0a64fa8ac concept usb c \n",- "3 ebcfb9727a42482d986fa2b0a64fa8ac concept mobile phone model \n",- "4 ebcfb9727a42482d986fa2b0a64fa8ac concept phone companies \n",- "\n",- " importance_R category_R type_R \n",- "0 3 event concept \n",- "1 5 concept concept \n",- "2 5 object concept \n",- "3 3 object concept \n",- "4 4 organisation concept "- ]- },- "execution_count": 240,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "dfne_join = pd.merge(\n",- " df_concepts, df_concepts, how=\"inner\", on=\"chunk_id\", suffixes=(\"_L\", \"_R\")\n",- ")\n",- "\n",- "## Remove self Loops\n",- "self_loops_drop = dfne_join[dfne_join[\"entity_L\"] == dfne_join[\"entity_R\"]].index\n",- "dfg = dfne_join.drop(index=self_loops_drop).reset_index()\n",- "\n",- "## This is our graph dataframe\n",- "print(\"Total number of nodes = \", dfg.shape[0])\n",- "dfg.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "#### Clean the graph dataframe\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "The original graph dataframe is too big to visualise. So we will make another dataframe for visualisation purpose.\n",- "\n",- "- remove the less important nodes\n",- "- remove less important edges\n"- ]- },- {- "cell_type": "code",- "execution_count": 259,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Less important Nodes = 29 \n",- "Less Important Edges = 2\n"- ]- }- ],- "source": [- "## Drop nodes which are less important\n",- "less_important_nodes = dfg[(dfg[\"importance_L\"] < 2)].index\n",- "## Drop edges where both the nodes are less important than 5\n",- "less_important_edges = dfg[(dfg[\"importance_L\"] < 2) & (dfg[\"importance_R\"] < 2)].index\n",- "drops = less_important_nodes.union(less_important_edges)\n",- "\n",- "print(\n",- " \"Less important Nodes = \",\n",- " less_important_nodes.shape[0],\n",- " \"\\nLess Important Edges = \",\n",- " less_important_edges.shape[0],\n",- ")\n",- "\n",- "## Remove these rows from the graph dataframe\n",- "dfg_vis = dfg.drop(index=drops).reset_index()\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "### Combine similar edges\n",- "\n",- "Group the edges between the same nodes and combine them into single edge with its weight equal to the count. "- ]- },- {- "cell_type": "code",- "execution_count": 260,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Final Number of Edges in the Visualisation Graph = 2481\n"- ]- },- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity_L</th>\n",- " <th>entity_R</th>\n",- " <th>importance_L</th>\n",- " <th>importance_R</th>\n",- " <th>chunks</th>\n",- " <th>count</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>35200</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>35209</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>99def</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>adt_a01</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>al</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " entity_L entity_R importance_L importance_R \\\n",- "0 10000 w 100th ave 35200 2.0 2.0 \n",- "1 10000 w 100th ave 35209 2.0 2.0 \n",- "2 10000 w 100th ave 99def 2.0 2.0 \n",- "3 10000 w 100th ave adt_a01 2.0 2.0 \n",- "4 10000 w 100th ave al 2.0 2.0 \n",- "\n",- " chunks count \n",- "0 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "1 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "2 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "3 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "4 45a6201014f84cc7bcb7aea24a7f7af6 1 "- ]- },- "execution_count": 260,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "\n",- "## Group and aggregate edges.\n",- "dfg_vis = (\n",- " dfg_vis.groupby([\"entity_L\", \"entity_R\"])\n",- " .agg(\n",- " {\n",- " \"importance_L\": \"mean\",\n",- " \"importance_R\": \"mean\",\n",- " \"chunk_id\": [\",\".join, \"count\"],\n",- " }\n",- " )\n",- " .reset_index()\n",- ")\n",- "dfg_vis.columns = [\n",- " \"entity_L\",\n",- " \"entity_R\",\n",- " \"importance_L\",\n",- " \"importance_R\",\n",- " \"chunks\",\n",- " \"count\",\n",- "]\n",- "\n",- "print(\"Final Number of Edges in the Visualisation Graph = \", dfg_vis.shape[0])\n",- "dfg_vis.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "### Removing overconnected nodes\n",- "\n",- "These are featured in the header and the footer of the pdf file, so they are a little too connected.\n"- ]- },- {- "cell_type": "code",- "execution_count": 261,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Final Number of Edges = 2481 \n",- "Dropped edges: 0\n"- ]- },- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity_L</th>\n",- " <th>entity_R</th>\n",- " <th>importance_L</th>\n",- " <th>importance_R</th>\n",- " <th>chunks</th>\n",- " <th>count</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>35200</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>35209</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>99def</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>adt_a01</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>al</td>\n",- " <td>2.0</td>\n",- " <td>2.0</td>\n",- " <td>45a6201014f84cc7bcb7aea24a7f7af6</td>\n",- " <td>1</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " entity_L entity_R importance_L importance_R \\\n",- "0 10000 w 100th ave 35200 2.0 2.0 \n",- "1 10000 w 100th ave 35209 2.0 2.0 \n",- "2 10000 w 100th ave 99def 2.0 2.0 \n",- "3 10000 w 100th ave adt_a01 2.0 2.0 \n",- "4 10000 w 100th ave al 2.0 2.0 \n",- "\n",- " chunks count \n",- "0 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "1 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "2 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "3 45a6201014f84cc7bcb7aea24a7f7af6 1 \n",- "4 45a6201014f84cc7bcb7aea24a7f7af6 1 "- ]- },- "execution_count": 261,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "ind = dfg_vis[\n",- " dfg_vis[\"entity_L\"].isin(\n",- " [\"Pathways to Health Equity for the G20\", \"Accelerating Global Health\"]\n",- " )\n",- " | dfg_vis[\"entity_R\"].isin(\n",- " [\"Pathways to Health Equity for the G20\", \"Accelerating Global Health\"]\n",- " )\n",- "].index\n",- "dfg_vis.drop(index=ind, axis=1, inplace=True)\n",- "print(\"Final Number of Edges = \", dfg_vis.shape[0], \"\\nDropped edges:\", len(ind))\n",- "dfg_vis.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Creating a NetworkX Graph\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Calculate nodes\n",- "\n",- "Here I am grouping the graph dataframe by left node and calculating the mean importance. This way we will end up with only the unique nodes from the graph dataframe along with their weights.\n"- ]- },- {- "cell_type": "code",- "execution_count": 262,- "metadata": {},- "outputs": [- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity_L</th>\n",- " <th>importance_L</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>2.0</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>2010</td>\n",- " <td>4.0</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>30-year-old standard</td>\n",- " <td>4.0</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>35200</td>\n",- " <td>2.0</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>35209</td>\n",- " <td>2.0</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " entity_L importance_L\n",- "0 10000 w 100th ave 2.0\n",- "1 2010 4.0\n",- "2 30-year-old standard 4.0\n",- "3 35200 2.0\n",- "4 35209 2.0"- ]- },- "execution_count": 262,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "# nodes = df_graph[\"entity_L\"].unique()\n",- "nodes = dfg_vis.groupby([\"entity_L\"]).agg({\"importance_L\": \"mean\"}).reset_index()\n",- "nodes.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Build a NetworkX object with nodes and edges\n"- ]- },- {- "cell_type": "code",- "execution_count": 263,- "metadata": {},- "outputs": [],- "source": [- "G = nx.Graph()\n",- "for index, row in nodes.iterrows():\n",- " G.add_node(row[\"entity_L\"])\n",- "\n",- "for index, row in dfg_vis.iterrows():\n",- " G.add_edge(str(row[\"entity_L\"]), str(row[\"entity_R\"]))"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "### Community Detection\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Detect communities using the Girvan Newman algorithm\n"- ]- },- {- "cell_type": "code",- "execution_count": 264,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Number of Communities = 27\n"- ]- }- ],- "source": [- "communities_generator = nx.community.girvan_newman(G)\n",- "top_level_communities = next(communities_generator)\n",- "next_level_communities = next(communities_generator)\n",- "communities = sorted(map(sorted, next_level_communities))\n",- "print(\"Number of Communities = \", len(communities))"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Add colors to nodes based on community\n"- ]- },- {- "cell_type": "code",- "execution_count": 265,- "metadata": {},- "outputs": [- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity_L</th>\n",- " <th>importance_L</th>\n",- " <th>color</th>\n",- " <th>group</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>10000 w 100th ave</td>\n",- " <td>2.0</td>\n",- " <td>#db576c</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>2010</td>\n",- " <td>4.0</td>\n",- " <td>#db578a</td>\n",- " <td>2</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>30-year-old standard</td>\n",- " <td>4.0</td>\n",- " <td>#db57a7</td>\n",- " <td>3</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>35200</td>\n",- " <td>2.0</td>\n",- " <td>#db576c</td>\n",- " <td>1</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>35209</td>\n",- " <td>2.0</td>\n",- " <td>#db576c</td>\n",- " <td>1</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " entity_L importance_L color group\n",- "0 10000 w 100th ave 2.0 #db576c 1\n",- "1 2010 4.0 #db578a 2\n",- "2 30-year-old standard 4.0 #db57a7 3\n",- "3 35200 2.0 #db576c 1\n",- "4 35209 2.0 #db576c 1"- ]- },- "execution_count": 265,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "palette = \"hls\"\n",- "\n",- "\n",- "## Now add these colors to communities and make another dataframe\n",- "def colors2Community(communities) -> pd.DataFrame:\n",- " ## Define a color palette\n",- " p = sns.color_palette(palette, len(communities)).as_hex()\n",- " rows = []\n",- " group = 0\n",- " for community in communities:\n",- " color = p.pop()\n",- " group += 1\n",- " for node in community:\n",- " rows += [{\"entity_L\": node, \"color\": color, \"group\": group}]\n",- " df_colors = pd.DataFrame(rows)\n",- " return df_colors\n",- "\n",- "\n",- "colors = colors2Community(communities)\n",- "\n",- "df_nodes_colors = pd.merge(\n",- " nodes, colors, how=\"left\", on=\"entity_L\", suffixes=(\"_N\", \"_C\")\n",- ")\n",- "# nodes.head()\n",- "df_nodes_colors.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "So now we have a nodes dataframe with colors and sizes of each node.\n",- "\n",- "lets recreate our graph.\n"- ]- },- {- "cell_type": "code",- "execution_count": 266,- "metadata": {},- "outputs": [],- "source": [- "G = nx.Graph()\n",- "node_size_multiple = 6\n",- "\n",- "for index, row in df_nodes_colors.iterrows():\n",- " G.add_node(\n",- " row[\"entity_L\"],\n",- " size=row[\"importance_L\"] * node_size_multiple,\n",- " title=row[\"entity_L\"],\n",- " color=row[\"color\"],\n",- " )\n",- "\n",- "for index, row in dfg_vis.iterrows():\n",- " G.add_edge(\n",- " str(row[\"entity_L\"]),\n",- " str(row[\"entity_R\"]),\n",- " weight=row[\"count\"],\n",- " name=row[\"chunks\"],\n",- " )"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Visualisation\n"- ]- },- {- "cell_type": "code",- "execution_count": 267,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "./docs/index.html\n"- ]- }- ],- "source": [- "graph_output_directory = \"./docs/index.html\"\n",- "\n",- "net = Network(\n",- " notebook=False,\n",- " bgcolor=\"#1a1a1a\",\n",- " cdn_resources=\"remote\",\n",- " height=\"900px\",\n",- " width=\"100%\",\n",- " select_menu=True,\n",- " font_color=\"#cccccc\",\n",- " # filter_menu=True,\n",- ")\n",- "\n",- "net.from_nx(G)\n",- "net.repulsion(node_distance=150, spring_length=400)\n",- "# net.barnes_hut(gravity=-18100, central_gravity=5.05, spring_length=380)\n",- "net.show_buttons(filter_=[\"physics\"])\n",- "\n",- "net.show(graph_output_directory, notebook=False)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": []- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": []- }- ],- "metadata": {- "kernelspec": {- "display_name": "OpenAI@3111",- "language": "python",- "name": "python3"- },- "language_info": {- "codemirror_mode": {- "name": "ipython",- "version": 3- },- "file_extension": ".py",- "mimetype": "text/x-python",- "name": "python",- "nbconvert_exporter": "python",- "pygments_lexer": "ipython3",- "version": "3.11.1"- }- },- "nbformat": 4,- "nbformat_minor": 2-}diff --git a/extract_concepts.ipynb b/extract_concepts.ipynbdeleted file mode 100644index c7f8291..0000000--- a/extract_concepts.ipynb+++ /dev/null@@ -1,698 +0,0 @@-{- "cells": [- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Setup"- ]- },- {- "cell_type": "code",- "execution_count": 1,- "metadata": {},- "outputs": [],- "source": [- "import pandas as pd\n",- "import numpy as np\n",- "import os\n",- "from langchain.document_loaders import PyPDFLoader, UnstructuredPDFLoader, PyPDFium2Loader\n",- "from langchain.document_loaders import PyPDFDirectoryLoader\n",- "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",- "from pathlib import Path\n",- "\n",- "## Input data directory\n",- "data_dir = \"OrfPathHealth\"\n",- "inputdirectory = Path(f\"./data_input/{data_dir}\")\n",- "## This is where the output csv files will be written\n",- "out_dir = data_dir\n",- "outputdirectory = Path(f\"./data_output/{out_dir}\")"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Load Documents"- ]- },- {- "cell_type": "code",- "execution_count": 2,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Number of chunks = 226\n"- ]- }- ],- "source": [- "## Dir Loader\n",- "documents = PyPDFDirectoryLoader(inputdirectory).load()\n",- "## File Loader\n",- "# documents = PyPDFLoader(\"./data/MedicalDocuments/orf-path_health-n1.pdf\").load()\n",- "\n",- "splitter = RecursiveCharacterTextSplitter(\n",- " chunk_size=1000,\n",- " chunk_overlap=50,\n",- " length_function=len,\n",- " is_separator_regex=False,\n",- ")\n",- "\n",- "pages = splitter.split_documents(documents)\n",- "print(\"Number of chunks = \", len(pages))\n"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Create a dataframe of all the chunks"- ]- },- {- "cell_type": "code",- "execution_count": 3,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "(226, 4)\n"- ]- },- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>text</th>\n",- " <th>source</th>\n",- " <th>page</th>\n",- " <th>chunk_id</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>147\\n147Mental Health as a \\nPathway to Health...</td>\n",- " <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",- " <td>0</td>\n",- " <td>6a13f1b883a641d698148be3652fae52</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>sense, mental health encompasses promotive and...</td>\n",- " <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",- " <td>0</td>\n",- " <td>9867158fca1e4e2e9e6c9a70472fea37</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>148Accelerating Global Health: Pathways to Hea...</td>\n",- " <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",- " <td>1</td>\n",- " <td>6c16b5fb1bdf4e7481a58a41f3935d01</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>adversity is a recognised risk factor in condi...</td>\n",- " <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",- " <td>1</td>\n",- " <td>310128b7580441d99c7ac8466a60c4df</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>both in healthcare service delivery.\\nThe chal...</td>\n",- " <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",- " <td>1</td>\n",- " <td>995ff2fe191a41a2ae6e99b87beb01fc</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " text \\\n",- "0 147\\n147Mental Health as a \\nPathway to Health... \n",- "1 sense, mental health encompasses promotive and... \n",- "2 148Accelerating Global Health: Pathways to Hea... \n",- "3 adversity is a recognised risk factor in condi... \n",- "4 both in healthcare service delivery.\\nThe chal... \n",- "\n",- " source page \\\n",- "0 data_input/OrfPathHealth/MentalHealthPathwayTo... 0 \n",- "1 data_input/OrfPathHealth/MentalHealthPathwayTo... 0 \n",- "2 data_input/OrfPathHealth/MentalHealthPathwayTo... 1 \n",- "3 data_input/OrfPathHealth/MentalHealthPathwayTo... 1 \n",- "4 data_input/OrfPathHealth/MentalHealthPathwayTo... 1 \n",- "\n",- " chunk_id \n",- "0 6a13f1b883a641d698148be3652fae52 \n",- "1 9867158fca1e4e2e9e6c9a70472fea37 \n",- "2 6c16b5fb1bdf4e7481a58a41f3935d01 \n",- "3 310128b7580441d99c7ac8466a60c4df \n",- "4 995ff2fe191a41a2ae6e99b87beb01fc "- ]- },- "execution_count": 3,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "from helpers.df_helpers import documents2Dataframe\n",- "df = documents2Dataframe(pages)\n",- "print(df.shape)\n",- "df.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Extract Concepts"- ]- },- {- "cell_type": "code",- "execution_count": 4,- "metadata": {},- "outputs": [],- "source": [- "## This function uses the helpers/prompt function to extract concepts from text\n",- "from helpers.df_helpers import df2ConceptsList\n",- "from helpers.df_helpers import concepts2Df"- ]- },- {- "cell_type": "code",- "execution_count": 9,- "metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- " [\n",- " {\n",- " \"entity\": \"income\",\n",- " \"importance\": 4,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"mental illness\",\n",- " \"importance\": 5,\n",- " \"category\": \"concept\"\n",- " },\n",- " {\n",- " \"entity\": \"middle-aged\",\n",- " \"importance\": 3,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"mental healthcare\",\n",- " \"importance\": 4,\n",- " \"category\": \"object\"\n",- " },\n",- " {\n",- " \"entity\": \"mental health professionals\",\n",- " \"importance\": 3,\n",- " \"category\": \"organisation\"\n",- " },\n",- " {\n",- " \"entity\": \"mental health literacy\",\n",- " \"importance\": 3,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"stigma and discrimination\",\n",- " \"importance\": 2,\n",- " \"category\": \"misc\"\n",- " }\n",- "] [\n",- " {\n",- " \"entity\": \"mental health equity\",\n",- " \"importance\": 4,\n",- " \"category\": \"concept\"\n",- " },\n",- " {\n",- " \"entity\": \"social inequality\",\n",- " \"importance\": 3,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"gender issues\",\n",- " \"importance\": 2,\n",- " \"category\": \"misc\"\n",- " },\n",- " {\n",- " \"entity\": \"lack of community participation\",\n",- " \"importance\": 3,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"India's National Program for Mental Health (NMHP)\",\n",- " \"importance\": 4,\n",- " \"category\": \"organisation\"\n",- " },\n",- " {\n",- " \"entity\": \"District Mental Health Program (DMHP)\",\n",- " \"importance\": 3,\n",- " \"category\": \"organisation\"\n",- " },\n",- " {\n",- " \"entity\": \"mental healthcare\",\n",- " \"importance\": 4,\n",- " \"category\": \"concept\"\n",- " },\n",- " {\n",- " \"entity\": \"integration of mental health in general healthcare\",\n",- " \"importance\": 3,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"community participation in mental health development\",\n",- " \"importance\": 3,\n",- " \"category\": \"condition\"\n",- " },\n",- " {\n",- " \"entity\": \"National Mental Health Programme – District Mental Health Programme\",\n",- " \"importance\": 4,\n",- " \"category\": \"organisation\"\n",- " },\n",- " {\n",- " \"entity\": \"Taluk Mental Health programme (e.g., Karnataka)\",\n",- " \"importance\": 2,\n",- " \"category\": \"misc\"\n",- " }\n",- "]"- ]- }- ],- "source": [- "concepts_list = df2ConceptsList(df[10:12])"- ]- },- {- "cell_type": "code",- "execution_count": 26,- "metadata": {},- "outputs": [- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity</th>\n",- " <th>importance</th>\n",- " <th>category</th>\n",- " <th>chunk_id</th>\n",- " <th>type</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>Mental Health</td>\n",- " <td>5</td>\n",- " <td>concept</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>Health Equity</td>\n",- " <td>4</td>\n",- " <td>concept</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>World Health Organization (WHO)</td>\n",- " <td>3</td>\n",- " <td>organisation</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>United Nations (UN)</td>\n",- " <td>3</td>\n",- " <td>organisation</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>Sustainable Development Goals (SDGs)</td>\n",- " <td>4</td>\n",- " <td>document</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>concept</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "</div>"- ],- "text/plain": [- " entity importance category \\\n",- "0 Mental Health 5 concept \n",- "1 Health Equity 4 concept \n",- "2 World Health Organization (WHO) 3 organisation \n",- "3 United Nations (UN) 3 organisation \n",- "4 Sustainable Development Goals (SDGs) 4 document \n",- "\n",- " chunk_id type \n",- "0 83d4d0367bb0467e811782a4ada3bbb9 concept \n",- "1 83d4d0367bb0467e811782a4ada3bbb9 concept \n",- "2 83d4d0367bb0467e811782a4ada3bbb9 concept \n",- "3 83d4d0367bb0467e811782a4ada3bbb9 concept \n",- "4 83d4d0367bb0467e811782a4ada3bbb9 concept "- ]- },- "execution_count": 26,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "dfne = concepts2Df(concepts_list)\n",- "dfne.head()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "### Write CSV to an output directory"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "Both the dataframes are written in the csv format so we dont have to calculate them again. \n",- "\n",- " dfne = dataframe of names entities\n",- "\n",- " df = dataframe of chunks"- ]- },- {- "cell_type": "code",- "execution_count": 28,- "metadata": {},- "outputs": [],- "source": [- "if not os.path.exists(outputdirectory):\n",- " os.makedirs(outputdirectory)\n",- " \n",- "dfne.to_csv(outputdirectory/\"concepts.csv\", sep=\"|\", index=False)\n",- "df.to_csv(outputdirectory/\"chunks.csv\", sep=\"|\", index=False)"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## Named Entities from Concepts"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "**Not using this right now**\n",- "\n",- "Extracting named entities our of concepts. \n"- ]- },- {- "cell_type": "code",- "execution_count": 3,- "metadata": {},- "outputs": [],- "source": [- "from transformers import pipeline\n",- "\n",- "ner = pipeline(\"token-classification\", model=\"2rtl3/mn-xlm-roberta-base-named-entity\", aggregation_strategy=\"simple\")\n",- "# ner = pipeline(\"token-classification\", model=\"dslim/bert-large-NER\", aggregation_strategy=\"simple\")\n",- "\n",- "def row2NamedEntities(row):\n",- " ner_results = ner(row['entity'])\n",- " metadata = {'chunk_id': row['chunk_id'], 'type': 'entity'}\n",- " entities = []\n",- " for result in ner_results:\n",- " entities = entities + [{'entity': result['word'], 'catetory': result['entity_group'], **metadata}]\n",- " \n",- " return entities\n",- "\n",- "\n",- "\n",- "def dfText2DfNE(dataframe: pd.DataFrame):\n",- " ## Takes a dataframe from the parsed data and returns dataframe with named entities. \n",- " ## The input dataframe must have a entity and a chunk_id column. \n",- "\n",- " ## 1. Calculate named entities for each row of the dataframe. \n",- " results = dataframe.apply(row2NamedEntities, axis=1).reset_index(drop=True)\n",- "\n",- " ## Flatten the list of lists to one single list of entities. \n",- " entities_list = np.concatenate(results).ravel().tolist()\n",- "\n",- " ## Remove all NaN entities\n",- " entities_dataframe = pd.DataFrame(entities_list).replace(' ', np.nan)\n",- " entities_dataframe = entities_dataframe.dropna(subset=['entity'])\n",- "\n",- " ## Count the number of occurances per chunk id\n",- " # entities_dataframe = entities_dataframe.groupby(['entity', 'category', 'chunk_id']).size().reset_index(name='count')\n",- "\n",- " return entities_dataframe"- ]- },- {- "cell_type": "code",- "execution_count": 5,- "metadata": {},- "outputs": [- {- "data": {- "text/html": [- "<div>\n",- "<style scoped>\n",- " .dataframe tbody tr th:only-of-type {\n",- " vertical-align: middle;\n",- " }\n",- "\n",- " .dataframe tbody tr th {\n",- " vertical-align: top;\n",- " }\n",- "\n",- " .dataframe thead th {\n",- " text-align: right;\n",- " }\n",- "</style>\n",- "<table border=\"1\" class=\"dataframe\">\n",- " <thead>\n",- " <tr style=\"text-align: right;\">\n",- " <th></th>\n",- " <th>entity</th>\n",- " <th>catetory</th>\n",- " <th>chunk_id</th>\n",- " <th>type</th>\n",- " </tr>\n",- " </thead>\n",- " <tbody>\n",- " <tr>\n",- " <th>0</th>\n",- " <td>Mental Health</td>\n",- " <td>MISC</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>1</th>\n",- " <td>Health Equity</td>\n",- " <td>ORG</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>2</th>\n",- " <td>World Health Organization (WHO)</td>\n",- " <td>ORG</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>3</th>\n",- " <td>United Nations (UN)</td>\n",- " <td>ORG</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>4</th>\n",- " <td>Sustainable Development Goals (SDGs)</td>\n",- " <td>MISC</td>\n",- " <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>...</th>\n",- " <td>...</td>\n",- " <td>...</td>\n",- " <td>...</td>\n",- " <td>...</td>\n",- " </tr>\n",- " <tr>\n",- " <th>967</th>\n",- " <td>National University of Singapore's Saw Swee Ho...</td>\n",- " <td>ORG</td>\n",- " <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>968</th>\n",- " <td>Associate Professor</td>\n",- " <td>MISC</td>\n",- " <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>969</th>\n",- " <td>Jeremy Lim</td>\n",- " <td>PER</td>\n",- " <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>970</th>\n",- " <td>Janice Tan</td>\n",- " <td>PER</td>\n",- " <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " <tr>\n",- " <th>971</th>\n",- " <td>singapore</td>\n",- " <td>LOC</td>\n",- " <td>2f3c4ce941c14624bfbd3da82ff8def7</td>\n",- " <td>entity</td>\n",- " </tr>\n",- " </tbody>\n",- "</table>\n",- "<p>972 rows × 4 columns</p>\n",- "</div>"- ],- "text/plain": [- " entity catetory \\\n",- "0 Mental Health MISC \n",- "1 Health Equity ORG \n",- "2 World Health Organization (WHO) ORG \n",- "3 United Nations (UN) ORG \n",- "4 Sustainable Development Goals (SDGs) MISC \n",- ".. ... ... \n",- "967 National University of Singapore's Saw Swee Ho... ORG \n",- "968 Associate Professor MISC \n",- "969 Jeremy Lim PER \n",- "970 Janice Tan PER \n",- "971 singapore LOC \n",- "\n",- " chunk_id type \n",- "0 83d4d0367bb0467e811782a4ada3bbb9 entity \n",- "1 83d4d0367bb0467e811782a4ada3bbb9 entity \n",- "2 83d4d0367bb0467e811782a4ada3bbb9 entity \n",- "3 83d4d0367bb0467e811782a4ada3bbb9 entity \n",- "4 83d4d0367bb0467e811782a4ada3bbb9 entity \n",- ".. ... ... \n",- "967 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",- "968 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",- "969 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",- "970 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",- "971 2f3c4ce941c14624bfbd3da82ff8def7 entity \n",- "\n",- "[972 rows x 4 columns]"- ]- },- "execution_count": 5,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "\n",- "dataframe_dir = 'OrfPathHealth'\n",- "df_concepts = pd.read_csv(f\"./data/output/{dataframe_dir}/concepts.csv\", sep=\"|\")\n",- "\n",- "dfc_split = dfText2DfNE(df_concepts)\n",- "dfc_split"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": []- }- ],- "metadata": {- "kernelspec": {- "display_name": "OpenAI@3111",- "language": "python",- "name": "python3"- },- "language_info": {- "codemirror_mode": {- "name": "ipython",- "version": 3- },- "file_extension": ".py",- "mimetype": "text/x-python",- "name": "python",- "nbconvert_exporter": "python",- "pygments_lexer": "ipython3",- "version": "3.11.1"- }- },- "nbformat": 4,- "nbformat_minor": 2-}diff --git a/extract_graph.ipynb b/extract_graph.ipynbindex 30ac2d1..8ae3497 100644--- a/extract_graph.ipynb+++ b/extract_graph.ipynb@@ -20,6 +20,7 @@"from langchain.document_loaders import PyPDFDirectoryLoader, DirectoryLoader\n","from langchain.text_splitter import RecursiveCharacterTextSplitter\n","from pathlib import Path\n",+ "import random\n","\n","## Input data directory\n","data_dir = \"cureus\"\n",@@ -377,7 +378,7 @@},{"cell_type": "code",- "execution_count": 167,+ "execution_count": 205,"metadata": {},"outputs": [{@@ -558,34 +559,39 @@"[700 rows x 5 columns]"]},- "execution_count": 167,+ "execution_count": 205,"metadata": {},"output_type": "execute_result"}],"source": [- "## Melt the dataframe into a list of nodes\n",- "dfg_long = pd.melt(\n",- " dfg1, id_vars=[\"chunk_id\"], value_vars=[\"node_1\", \"node_2\"], value_name=\"node\"\n",- ")\n",- "dfg_long.drop(columns=[\"variable\"], inplace=True)\n",- "# Self join with chunk id as the key will create a link between terms occuring in the same text chunk. \n",- "dfg_wide = pd.merge(dfg_long, dfg_long, on=\"chunk_id\", suffixes=(\"_1\", \"_2\"))\n",- "# drop self loops\n",- "self_loops_drop = dfg_wide[dfg_wide[\"node_1\"] == dfg_wide[\"node_2\"]].index\n",- "dfg2 = dfg_wide.drop(index=self_loops_drop).reset_index(drop=True)\n",- "## Group and count edges.\n",- "dfg2 = (\n",- " dfg2.groupby([\"node_1\", \"node_2\"])\n",- " .agg({\"chunk_id\": [\",\".join, \"count\"]})\n",- " .reset_index()\n",- ")\n",- "dfg2.columns = [\"node_1\", \"node_2\", \"chunk_id\", \"count\"]\n",- "dfg2.replace(\"\", np.nan, inplace=True)\n",- "dfg2.dropna(subset=[\"node_1\", \"node_2\"], inplace=True)\n",- "# Drop edges with 1 count\n",- "dfg2 = dfg2[dfg2[\"count\"] != 1]\n",- "dfg2[\"edge\"] = \"contextual proximity\"\n",+ "def contextual_proximity(df: pd.DataFrame) -> pd.DataFrame:\n",+ " ## Melt the dataframe into a list of nodes\n",+ " dfg_long = pd.melt(\n",+ " df, id_vars=[\"chunk_id\"], value_vars=[\"node_1\", \"node_2\"], value_name=\"node\"\n",+ " )\n",+ " dfg_long.drop(columns=[\"variable\"], inplace=True)\n",+ " # Self join with chunk id as the key will create a link between terms occuring in the same text chunk.\n",+ " dfg_wide = pd.merge(dfg_long, dfg_long, on=\"chunk_id\", suffixes=(\"_1\", \"_2\"))\n",+ " # drop self loops\n",+ " self_loops_drop = dfg_wide[dfg_wide[\"node_1\"] == dfg_wide[\"node_2\"]].index\n",+ " dfg2 = dfg_wide.drop(index=self_loops_drop).reset_index(drop=True)\n",+ " ## Group and count edges.\n",+ " dfg2 = (\n",+ " dfg2.groupby([\"node_1\", \"node_2\"])\n",+ " .agg({\"chunk_id\": [\",\".join, \"count\"]})\n",+ " .reset_index()\n",+ " )\n",+ " dfg2.columns = [\"node_1\", \"node_2\", \"chunk_id\", \"count\"]\n",+ " dfg2.replace(\"\", np.nan, inplace=True)\n",+ " dfg2.dropna(subset=[\"node_1\", \"node_2\"], inplace=True)\n",+ " # Drop edges with 1 count\n",+ " dfg2 = dfg2[dfg2[\"count\"] != 1]\n",+ " dfg2[\"edge\"] = \"contextual proximity\"\n",+ " return dfg2\n",+ "\n",+ "\n",+ "dfg2 = contextual_proximity(dfg1)\n","dfg2"]},@@ -884,7 +890,7 @@},{"cell_type": "code",- "execution_count": 184,+ "execution_count": 192,"metadata": {},"outputs": [{@@ -917,31 +923,31 @@" <tr>\n"," <th>0</th>\n"," <td>56 articles</td>\n",- " <td>#db577e</td>\n",+ " <td>#57db9d</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>1</th>\n"," <td>analysis</td>\n",- " <td>#db577e</td>\n",+ " <td>#57db9d</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>2</th>\n"," <td>corresponding authors' experiential knowledge</td>\n",- " <td>#db577e</td>\n",+ " <td>#57db9d</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>3</th>\n"," <td>extensive literature search</td>\n",- " <td>#db577e</td>\n",+ " <td>#57db9d</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n"," <th>4</th>\n"," <td>peer-reviewed journals</td>\n",- " <td>#db577e</td>\n",+ " <td>#57db9d</td>\n"," <td>1</td>\n"," </tr>\n"," <tr>\n",@@ -953,31 +959,31 @@" <tr>\n"," <th>210</th>\n"," <td>rural medical assistants (rmas)</td>\n",- " <td>#dbbc57</td>\n",+ " <td>#575edb</td>\n"," <td>15</td>\n"," </tr>\n"," <tr>\n"," <th>211</th>\n"," <td>limited uptake</td>\n",- " <td>#db8d57</td>\n",+ " <td>#dbbc57</td>\n"," <td>16</td>\n"," </tr>\n"," <tr>\n"," <th>212</th>\n"," <td>national health protection mission</td>\n",- " <td>#db8d57</td>\n",+ " <td>#dbbc57</td>\n"," <td>16</td>\n"," </tr>\n"," <tr>\n"," <th>213</th>\n"," <td>private health sector systems</td>\n",- " <td>#db5f57</td>\n",+ " <td>#57db6e</td>\n"," <td>17</td>\n"," </tr>\n"," <tr>\n"," <th>214</th>\n"," <td>public</td>\n",- " <td>#db5f57</td>\n",+ " <td>#57db6e</td>\n"," <td>17</td>\n"," </tr>\n"," </tbody>\n",@@ -987,22 +993,22 @@],"text/plain": [" node color group\n",- "0 56 articles #db577e 1\n",- "1 analysis #db577e 1\n",- "2 corresponding authors' experiential knowledge #db577e 1\n",- "3 extensive literature search #db577e 1\n",- "4 peer-reviewed journals #db577e 1\n",+ "0 56 articles #57db9d 1\n",+ "1 analysis #57db9d 1\n",+ "2 corresponding authors' experiential knowledge #57db9d 1\n",+ "3 extensive literature search #57db9d 1\n",+ "4 peer-reviewed journals #57db9d 1\n",".. ... ... ...\n",- "210 rural medical assistants (rmas) #dbbc57 15\n",- "211 limited uptake #db8d57 16\n",- "212 national health protection mission #db8d57 16\n",- "213 private health sector systems #db5f57 17\n",- "214 public #db5f57 17\n",+ "210 rural medical assistants (rmas) #575edb 15\n",+ "211 limited uptake #dbbc57 16\n",+ "212 national health protection mission #dbbc57 16\n",+ "213 private health sector systems #57db6e 17\n",+ "214 public #57db6e 17\n","\n","[215 rows x 3 columns]"]},- "execution_count": 184,+ "execution_count": 192,"metadata": {},"output_type": "execute_result"}@@ -1015,6 +1021,7 @@"def colors2Community(communities) -> pd.DataFrame:\n"," ## Define a color palette\n"," p = sns.color_palette(palette, len(communities)).as_hex()\n",+ " random.shuffle(p)\n"," rows = []\n"," group = 0\n"," for community in communities:\n",@@ -1039,7 +1046,7 @@},{"cell_type": "code",- "execution_count": 185,+ "execution_count": 193,"metadata": {},"outputs": [],"source": [@@ -1051,21 +1058,21 @@},{"cell_type": "code",- "execution_count": 186,+ "execution_count": 195,"metadata": {},"outputs": [{"name": "stdout","output_type": "stream","text": [- "./docs/graph.html\n"+ "./docs/index.html\n"]}],"source": ["from pyvis.network import Network\n","\n",- "graph_output_directory = \"./docs/graph.html\"\n",+ "graph_output_directory = \"./docs/index.html\"\n","\n","net = Network(\n"," notebook=False,\n",@@ -1086,39 +1093,6 @@"\n","net.show(graph_output_directory, notebook=False)"]- },- {- "cell_type": "code",- "execution_count": 94,- "metadata": {},- "outputs": [- {- "data": {- "text/plain": [- "count 748.000000\n",- "mean 3.157754\n",- "std 2.841504\n",- "min 2.000000\n",- "25% 2.000000\n",- "50% 2.000000\n",- "75% 3.000000\n",- "max 30.000000\n",- "Name: count, dtype: float64"- ]- },- "execution_count": 94,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": []- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": []}],"metadata": {diff --git a/zefyr.ipynb b/zefyr.ipynbindex 50cda6e..be8c4b6 100644--- a/zefyr.ipynb+++ b/zefyr.ipynb@@ -33,7 +33,7 @@},{"cell_type": "code",- "execution_count": 20,+ "execution_count": 22,"metadata": {},"outputs": [],"source": [@@ -106,50 +106,20 @@},{"cell_type": "code",- "execution_count": 21,+ "execution_count": 23,"metadata": {},"outputs": [{- "name": "stdout",- "output_type": "stream",- "text": [- "{'modelfile': '# Modelfile generated by \"ollama show\"\\n# To build a new Modelfile based on this one, replace the FROM line with:\\n# FROM zephyr:latest\\n\\nFROM /Users/rahulnayak/.ollama/models/blobs/sha256:0e655574a746a702b8faa09ea5bd779801371cec9e34f69760d0ebb1aec1a6ce\\nTEMPLATE \"\"\"{{- if .System }}\\n<|system|>\\n{{ .System }}\\n</s>\\n{{- end }}\\n<|user|>\\n{{ .Prompt }}\\n</s>\\n<|assistant|>\\n\"\"\"\\nPARAMETER stop \"<|system|>\"\\nPARAMETER stop \"<|user|>\"\\nPARAMETER stop \"<|assistant|>\"\\nPARAMETER stop \"</s>\"', 'parameters': 'stop <|system|>\\nstop <|user|>\\nstop <|assistant|>\\nstop </s>', 'template': '{{- if .System }}\\n<|system|>\\n{{ .System }}\\n</s>\\n{{- end }}\\n<|user|>\\n{{ .Prompt }}\\n</s>\\n<|assistant|>\\n'}\n",- "[\n",- " {\n",- " \"node_1\": \"Mental Health\",\n",- " \"node_2\": \"Health Equity\",\n",- " \"edge\": \"Investing in mental health is important from the point of view improving population health and reducing social inequality, promoting human rights, and improving economic efficiency.\"\n",- " }, {\n",- " \"node_1\": \"Mental Health\",\n",- " \"node_2\": \"Economic Loss\",\n",- " \"edge\": \"The global economic loss due to common mental disorders is an estimated US$ 1 trillion per year.\"\n",- " }, {\n",- " \"node_1\": \"Treatment of Mental Illness\",\n",- " \"node_2\": \"Economic Productivity Gains\",\n",- " \"edge\": \"Investment in treatment of mental illness can result in large economic productivity gains.\"\n",- " }, {\n",- " \"node_1\": \"Mental Health\",\n",- " \"node_2\": \"Prevalence of Mental Illness\",\n",- " \"edge\": \"The National Mental Health Survey (NMHS), 2015-16, has reported a 13.6-percent lifetime prevalence of any mental/substance-abuse disorder morbid"- ]- },- {- "ename": "KeyboardInterrupt",- "evalue": "",+ "ename": "TypeError",+ "evalue": "the JSON object must be str, bytes or bytearray, not dict","output_type": "error","traceback": ["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",- "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",+ "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)","\u001b[1;32m/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb Cell 3\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m r \u001b[39m=\u001b[39m graphPrompt(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39m \u001b[39;49m\u001b[39m\"\"\"\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39mtext delimites bweteen ``` \\n\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39mtext: ```\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m \u001b[39mMental Health as a Pathway to Health Equityargued that investing in mental health is important from the point of view \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m \u001b[39mof improving population health and reducing social inequality, promoting \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m \u001b[39mhuman rights, and improving economic efficiency.8 The economic implications \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39mof mental and substance disorders can be enormous: for example, the \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39mglobal economic loss due to common mental disorders is an estimated US$ \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m1 trillion per year.9 Given the increasing prevalence of mental illness, the \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39mcosts are likely to only further mount. Investment in treatment of mental \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39millness can result in large economic productivity gains.10\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m \u001b[39mThe Burden of Mental Health, Substance Use Disorders and \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m \u001b[39mMultimorbidity\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m \u001b[39mMental and substance-abuse disorders are leading contributors to premature \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m \u001b[39mmorbidity and mortality globally. In India, for example, the National Mental \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m \u001b[39mHealth Survey (NMHS), 2015-16, has reported a 13.6-percent lifetime \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m \u001b[39mprevalence of any mental/substance-abuse disorder morbidity.11 The overall \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m \u001b[39mnumber of Disability-Adjusted-Life-Years (DALYs) for mental disorders has \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=19'>20</a>\u001b[0m \u001b[39mincreased by more than 50 percent between 1990 and 2019.12 This growth \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=20'>21</a>\u001b[0m \u001b[39mpattern is expected to continue, necessitating a proportionate response from \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=21'>22</a>\u001b[0m \u001b[39mthe health systems. Most of the mental and substance abuse disorders \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=22'>23</a>\u001b[0m \u001b[39mare chronic and disabling conditions. The treatment gap for mental and \u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=23'>24</a>\u001b[0m \u001b[39msubstance abuse disorders (defined as proportions of those needing care\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=24'>25</a>\u001b[0m \u001b[39m```\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=25'>26</a>\u001b[0m \u001b[39m\"\"\"\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=26'>27</a>\u001b[0m )\n",- "\u001b[1;32m/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb Cell 3\u001b[0m line \u001b[0;36m5\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=29'>30</a>\u001b[0m \u001b[39mprint\u001b[39m(model_info)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=30'>31</a>\u001b[0m SYS_PROMPT \u001b[39m=\u001b[39m (\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=31'>32</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mYou are a network graph maker who extracts terms and their relations from a given text\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=32'>33</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mYour task is to extract the ontology of terms mentioned in the given text. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=49'>50</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39m]\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=50'>51</a>\u001b[0m )\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=51'>52</a>\u001b[0m response, _ \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49mgenerate(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=52'>53</a>\u001b[0m model_name\u001b[39m=\u001b[39;49mmodel, \n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=53'>54</a>\u001b[0m system\u001b[39m=\u001b[39;49mSYS_PROMPT, \n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=54'>55</a>\u001b[0m \u001b[39m# prompt=SYS_PROMPT + \"\\n\\n\" + input\u001b[39;49;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=55'>56</a>\u001b[0m prompt\u001b[39m=\u001b[39;49m\u001b[39minput\u001b[39;49m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=56'>57</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=57'>58</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=58'>59</a>\u001b[0m result \u001b[39m=\u001b[39m json\u001b[39m.\u001b[39mloads(response)\n",- "File \u001b[0;32m~/TechWork/AIProjects/knowlege_graph/ollama/client.py:35\u001b[0m, in \u001b[0;36mgenerate\u001b[0;34m(model_name, prompt, system, template, context, options, callback)\u001b[0m\n\u001b[1;32m 32\u001b[0m full_response \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 34\u001b[0m \u001b[39m# Iterating over the response line by line and displaying the details\u001b[39;00m\n\u001b[0;32m---> 35\u001b[0m \u001b[39mfor\u001b[39;00m line \u001b[39min\u001b[39;00m response\u001b[39m.\u001b[39miter_lines():\n\u001b[1;32m 36\u001b[0m \u001b[39mif\u001b[39;00m line:\n\u001b[1;32m 37\u001b[0m \u001b[39m# Parsing each line (JSON chunk) and extracting the details\u001b[39;00m\n\u001b[1;32m 38\u001b[0m chunk \u001b[39m=\u001b[39m json\u001b[39m.\u001b[39mloads(line)\n",- "File \u001b[0;32m~/.pyenv/versions/3.11.1/envs/OpenAI@3111/lib/python3.11/site-packages/requests/models.py:865\u001b[0m, in \u001b[0;36mResponse.iter_lines\u001b[0;34m(self, chunk_size, decode_unicode, delimiter)\u001b[0m\n\u001b[1;32m 856\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Iterates over the response data, one line at a time. When\u001b[39;00m\n\u001b[1;32m 857\u001b[0m \u001b[39mstream=True is set on the request, this avoids reading the\u001b[39;00m\n\u001b[1;32m 858\u001b[0m \u001b[39mcontent at once into memory for large responses.\u001b[39;00m\n\u001b[1;32m 859\u001b[0m \n\u001b[1;32m 860\u001b[0m \u001b[39m.. note:: This method is not reentrant safe.\u001b[39;00m\n\u001b[1;32m 861\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 863\u001b[0m pending \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 865\u001b[0m \u001b[39mfor\u001b[39;00m chunk \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39miter_content(\n\u001b[1;32m 866\u001b[0m chunk_size\u001b[39m=\u001b[39mchunk_size, decode_unicode\u001b[39m=\u001b[39mdecode_unicode\n\u001b[1;32m 867\u001b[0m ):\n\u001b[1;32m 869\u001b[0m \u001b[39mif\u001b[39;00m pending \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 870\u001b[0m chunk \u001b[39m=\u001b[39m pending \u001b[39m+\u001b[39m chunk\n",- "File \u001b[0;32m~/.pyenv/versions/3.11.1/envs/OpenAI@3111/lib/python3.11/site-packages/requests/models.py:816\u001b[0m, in \u001b[0;36mResponse.iter_content.<locals>.generate\u001b[0;34m()\u001b[0m\n\u001b[1;32m 814\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mraw, \u001b[39m\"\u001b[39m\u001b[39mstream\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m 815\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 816\u001b[0m \u001b[39myield from\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mraw\u001b[39m.\u001b[39mstream(chunk_size, decode_content\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m 817\u001b[0m \u001b[39mexcept\u001b[39;00m ProtocolError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 818\u001b[0m \u001b[39mraise\u001b[39;00m ChunkedEncodingError(e)\n",- "File \u001b[0;32m~/.pyenv/versions/3.11.1/envs/OpenAI@3111/lib/python3.11/site-packages/urllib3/response.py:933\u001b[0m, in \u001b[0;36mHTTPResponse.stream\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m 917\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 918\u001b[0m \u001b[39mA generator wrapper for the read() method. A call will block until\u001b[39;00m\n\u001b[1;32m 919\u001b[0m \u001b[39m``amt`` bytes have been read from the connection or until the\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 930\u001b[0m \u001b[39m 'content-encoding' header.\u001b[39;00m\n\u001b[1;32m 931\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 932\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mchunked \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msupports_chunked_reads():\n\u001b[0;32m--> 933\u001b[0m \u001b[39myield from\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mread_chunked(amt, decode_content\u001b[39m=\u001b[39mdecode_content)\n\u001b[1;32m 934\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 935\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mnot\u001b[39;00m is_fp_closed(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_fp) \u001b[39mor\u001b[39;00m \u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_decoded_buffer) \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m:\n",- "File \u001b[0;32m~/.pyenv/versions/3.11.1/envs/OpenAI@3111/lib/python3.11/site-packages/urllib3/response.py:1073\u001b[0m, in \u001b[0;36mHTTPResponse.read_chunked\u001b[0;34m(self, amt, decode_content)\u001b[0m\n\u001b[1;32m 1070\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 1072\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m-> 1073\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_update_chunk_length()\n\u001b[1;32m 1074\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mchunk_left \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[1;32m 1075\u001b[0m \u001b[39mbreak\u001b[39;00m\n",- "File \u001b[0;32m~/.pyenv/versions/3.11.1/envs/OpenAI@3111/lib/python3.11/site-packages/urllib3/response.py:1001\u001b[0m, in \u001b[0;36mHTTPResponse._update_chunk_length\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 999\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mchunk_left \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 1000\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m-> 1001\u001b[0m line \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_fp\u001b[39m.\u001b[39mfp\u001b[39m.\u001b[39mreadline() \u001b[39m# type: ignore[union-attr]\u001b[39;00m\n\u001b[1;32m 1002\u001b[0m line \u001b[39m=\u001b[39m line\u001b[39m.\u001b[39msplit(\u001b[39mb\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m;\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m1\u001b[39m)[\u001b[39m0\u001b[39m]\n\u001b[1;32m 1003\u001b[0m \u001b[39mtry\u001b[39;00m:\n",- "File \u001b[0;32m~/.pyenv/versions/3.11.1/lib/python3.11/socket.py:706\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m 705\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 706\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv_into(b)\n\u001b[1;32m 707\u001b[0m \u001b[39mexcept\u001b[39;00m timeout:\n\u001b[1;32m 708\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_timeout_occurred \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n",- "\u001b[0;31mKeyboardInterrupt\u001b[0m: "+ "\u001b[1;32m/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb Cell 3\u001b[0m line \u001b[0;36m3\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=27'>28</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mgraphPrompt\u001b[39m(\u001b[39minput\u001b[39m: \u001b[39mstr\u001b[39m, metadata\u001b[39m=\u001b[39m{}, model\u001b[39m=\u001b[39mmodel):\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=28'>29</a>\u001b[0m model_info \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39mshow(model_name\u001b[39m=\u001b[39mmodel)\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=29'>30</a>\u001b[0m \u001b[39mprint\u001b[39m(json\u001b[39m.\u001b[39;49mloads(model_info), \u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=30'>31</a>\u001b[0m SYS_PROMPT \u001b[39m=\u001b[39m (\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=31'>32</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mYou are a network graph maker who extracts terms and their relations from a given text\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=32'>33</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mYour task is to extract the ontology of terms mentioned in the given text. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=49'>50</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39m]\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=50'>51</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=51'>52</a>\u001b[0m response, _ \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39mgenerate(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=52'>53</a>\u001b[0m model_name\u001b[39m=\u001b[39mmodel, \n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=53'>54</a>\u001b[0m system\u001b[39m=\u001b[39mSYS_PROMPT, \n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=54'>55</a>\u001b[0m \u001b[39m# prompt=SYS_PROMPT + \"\\n\\n\" + input\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=55'>56</a>\u001b[0m prompt\u001b[39m=\u001b[39m\u001b[39minput\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/rahulnayak/TechWork/AIProjects/knowlege_graph/zefyr.ipynb#W2sZmlsZQ%3D%3D?line=56'>57</a>\u001b[0m )\n",+ "File \u001b[0;32m~/.pyenv/versions/3.11.1/lib/python3.11/json/__init__.py:339\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 338\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(s, (\u001b[39mbytes\u001b[39m, \u001b[39mbytearray\u001b[39m)):\n\u001b[0;32m--> 339\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mTypeError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mthe JSON object must be str, bytes or bytearray, \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 340\u001b[0m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mnot \u001b[39m\u001b[39m{\u001b[39;00ms\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m)\n\u001b[1;32m 341\u001b[0m s \u001b[39m=\u001b[39m s\u001b[39m.\u001b[39mdecode(detect_encoding(s), \u001b[39m'\u001b[39m\u001b[39msurrogatepass\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 343\u001b[0m \u001b[39mif\u001b[39;00m (\u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m parse_float \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_pairs_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m kw):\n",+ "\u001b[0;31mTypeError\u001b[0m: the JSON object must be str, bytes or bytearray, not dict"]}],@@ -183,6 +153,26 @@")"]},+ {+ "cell_type": "code",+ "execution_count": 28,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "{'modelfile': '# Modelfile generated by \"ollama show\"\\n# To build a new Modelfile based on this one, replace the FROM line with:\\n# FROM zephyr:latest\\n\\nFROM /Users/rahulnayak/.ollama/models/blobs/sha256:0e655574a746a702b8faa09ea5bd779801371cec9e34f69760d0ebb1aec1a6ce\\nTEMPLATE \"\"\"{{- if .System }}\\n<|system|>\\n{{ .System }}\\n</s>\\n{{- end }}\\n<|user|>\\n{{ .Prompt }}\\n</s>\\n<|assistant|>\\n\"\"\"\\nPARAMETER stop \"<|system|>\"\\nPARAMETER stop \"<|user|>\"\\nPARAMETER stop \"<|assistant|>\"\\nPARAMETER stop \"</s>\"',\n",+ " 'parameters': 'stop <|system|>\\nstop <|user|>\\nstop <|assistant|>\\nstop </s>',\n",+ " 'template': '{{- if .System }}\\n<|system|>\\n{{ .System }}\\n</s>\\n{{- end }}\\n<|user|>\\n{{ .Prompt }}\\n</s>\\n<|assistant|>\\n'}"+ ]+ },+ "execution_count": 28,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": []+ },{"cell_type": "code","execution_count": null,