Convert any text to a graph of knowledge. This can be used for Graph Augmented Generation or Knowledge Graph based QnA
HTML
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"from langchain.document_loaders import PyPDFLoader, UnstructuredPDFLoader, PyPDFium2Loader\n",
"from langchain.document_loaders import PyPDFDirectoryLoader\n",
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from pathlib import Path\n",
"\n",
"## Input data directory\n",
"data_dir = \"OrfPathHealth\"\n",
"inputdirectory = Path(f\"./data_input/{data_dir}\")\n",
"## This is where the output csv files will be written\n",
"out_dir = data_dir\n",
"outputdirectory = Path(f\"./data_output/{out_dir}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load Documents"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of chunks = 226\n"
]
}
],
"source": [
"## Dir Loader\n",
"documents = PyPDFDirectoryLoader(inputdirectory).load()\n",
"## File Loader\n",
"# documents = PyPDFLoader(\"./data/MedicalDocuments/orf-path_health-n1.pdf\").load()\n",
"\n",
"splitter = RecursiveCharacterTextSplitter(\n",
" chunk_size=1000,\n",
" chunk_overlap=50,\n",
" length_function=len,\n",
" is_separator_regex=False,\n",
")\n",
"\n",
"pages = splitter.split_documents(documents)\n",
"print(\"Number of chunks = \", len(pages))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Create a dataframe of all the chunks"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(226, 4)\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>text</th>\n",
" <th>source</th>\n",
" <th>page</th>\n",
" <th>chunk_id</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>147\\n147Mental Health as a \\nPathway to Health...</td>\n",
" <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",
" <td>0</td>\n",
" <td>6a13f1b883a641d698148be3652fae52</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>sense, mental health encompasses promotive and...</td>\n",
" <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",
" <td>0</td>\n",
" <td>9867158fca1e4e2e9e6c9a70472fea37</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>148Accelerating Global Health: Pathways to Hea...</td>\n",
" <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",
" <td>1</td>\n",
" <td>6c16b5fb1bdf4e7481a58a41f3935d01</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>adversity is a recognised risk factor in condi...</td>\n",
" <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",
" <td>1</td>\n",
" <td>310128b7580441d99c7ac8466a60c4df</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>both in healthcare service delivery.\\nThe chal...</td>\n",
" <td>data_input/OrfPathHealth/MentalHealthPathwayTo...</td>\n",
" <td>1</td>\n",
" <td>995ff2fe191a41a2ae6e99b87beb01fc</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" text \\\n",
"0 147\\n147Mental Health as a \\nPathway to Health... \n",
"1 sense, mental health encompasses promotive and... \n",
"2 148Accelerating Global Health: Pathways to Hea... \n",
"3 adversity is a recognised risk factor in condi... \n",
"4 both in healthcare service delivery.\\nThe chal... \n",
"\n",
" source page \\\n",
"0 data_input/OrfPathHealth/MentalHealthPathwayTo... 0 \n",
"1 data_input/OrfPathHealth/MentalHealthPathwayTo... 0 \n",
"2 data_input/OrfPathHealth/MentalHealthPathwayTo... 1 \n",
"3 data_input/OrfPathHealth/MentalHealthPathwayTo... 1 \n",
"4 data_input/OrfPathHealth/MentalHealthPathwayTo... 1 \n",
"\n",
" chunk_id \n",
"0 6a13f1b883a641d698148be3652fae52 \n",
"1 9867158fca1e4e2e9e6c9a70472fea37 \n",
"2 6c16b5fb1bdf4e7481a58a41f3935d01 \n",
"3 310128b7580441d99c7ac8466a60c4df \n",
"4 995ff2fe191a41a2ae6e99b87beb01fc "
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from helpers.df_helpers import documents2Dataframe\n",
"df = documents2Dataframe(pages)\n",
"print(df.shape)\n",
"df.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Extract Concepts"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"## This function uses the helpers/prompt function to extract concepts from text\n",
"from helpers.df_helpers import df2ConceptsList\n",
"from helpers.df_helpers import concepts2Df"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" [\n",
" {\n",
" \"entity\": \"income\",\n",
" \"importance\": 4,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"mental illness\",\n",
" \"importance\": 5,\n",
" \"category\": \"concept\"\n",
" },\n",
" {\n",
" \"entity\": \"middle-aged\",\n",
" \"importance\": 3,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"mental healthcare\",\n",
" \"importance\": 4,\n",
" \"category\": \"object\"\n",
" },\n",
" {\n",
" \"entity\": \"mental health professionals\",\n",
" \"importance\": 3,\n",
" \"category\": \"organisation\"\n",
" },\n",
" {\n",
" \"entity\": \"mental health literacy\",\n",
" \"importance\": 3,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"stigma and discrimination\",\n",
" \"importance\": 2,\n",
" \"category\": \"misc\"\n",
" }\n",
"] [\n",
" {\n",
" \"entity\": \"mental health equity\",\n",
" \"importance\": 4,\n",
" \"category\": \"concept\"\n",
" },\n",
" {\n",
" \"entity\": \"social inequality\",\n",
" \"importance\": 3,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"gender issues\",\n",
" \"importance\": 2,\n",
" \"category\": \"misc\"\n",
" },\n",
" {\n",
" \"entity\": \"lack of community participation\",\n",
" \"importance\": 3,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"India's National Program for Mental Health (NMHP)\",\n",
" \"importance\": 4,\n",
" \"category\": \"organisation\"\n",
" },\n",
" {\n",
" \"entity\": \"District Mental Health Program (DMHP)\",\n",
" \"importance\": 3,\n",
" \"category\": \"organisation\"\n",
" },\n",
" {\n",
" \"entity\": \"mental healthcare\",\n",
" \"importance\": 4,\n",
" \"category\": \"concept\"\n",
" },\n",
" {\n",
" \"entity\": \"integration of mental health in general healthcare\",\n",
" \"importance\": 3,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"community participation in mental health development\",\n",
" \"importance\": 3,\n",
" \"category\": \"condition\"\n",
" },\n",
" {\n",
" \"entity\": \"National Mental Health Programme – District Mental Health Programme\",\n",
" \"importance\": 4,\n",
" \"category\": \"organisation\"\n",
" },\n",
" {\n",
" \"entity\": \"Taluk Mental Health programme (e.g., Karnataka)\",\n",
" \"importance\": 2,\n",
" \"category\": \"misc\"\n",
" }\n",
"]"
]
}
],
"source": [
"concepts_list = df2ConceptsList(df[10:12])"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>entity</th>\n",
" <th>importance</th>\n",
" <th>category</th>\n",
" <th>chunk_id</th>\n",
" <th>type</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Mental Health</td>\n",
" <td>5</td>\n",
" <td>concept</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>concept</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Health Equity</td>\n",
" <td>4</td>\n",
" <td>concept</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>concept</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>World Health Organization (WHO)</td>\n",
" <td>3</td>\n",
" <td>organisation</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>concept</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>United Nations (UN)</td>\n",
" <td>3</td>\n",
" <td>organisation</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>concept</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Sustainable Development Goals (SDGs)</td>\n",
" <td>4</td>\n",
" <td>document</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>concept</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" entity importance category \\\n",
"0 Mental Health 5 concept \n",
"1 Health Equity 4 concept \n",
"2 World Health Organization (WHO) 3 organisation \n",
"3 United Nations (UN) 3 organisation \n",
"4 Sustainable Development Goals (SDGs) 4 document \n",
"\n",
" chunk_id type \n",
"0 83d4d0367bb0467e811782a4ada3bbb9 concept \n",
"1 83d4d0367bb0467e811782a4ada3bbb9 concept \n",
"2 83d4d0367bb0467e811782a4ada3bbb9 concept \n",
"3 83d4d0367bb0467e811782a4ada3bbb9 concept \n",
"4 83d4d0367bb0467e811782a4ada3bbb9 concept "
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfne = concepts2Df(concepts_list)\n",
"dfne.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Write CSV to an output directory"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Both the dataframes are written in the csv format so we dont have to calculate them again. \n",
"\n",
" dfne = dataframe of names entities\n",
"\n",
" df = dataframe of chunks"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"if not os.path.exists(outputdirectory):\n",
" os.makedirs(outputdirectory)\n",
" \n",
"dfne.to_csv(outputdirectory/\"concepts.csv\", sep=\"|\", index=False)\n",
"df.to_csv(outputdirectory/\"chunks.csv\", sep=\"|\", index=False)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Named Entities from Concepts"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Not using this right now**\n",
"\n",
"Extracting named entities our of concepts. \n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from transformers import pipeline\n",
"\n",
"ner = pipeline(\"token-classification\", model=\"2rtl3/mn-xlm-roberta-base-named-entity\", aggregation_strategy=\"simple\")\n",
"# ner = pipeline(\"token-classification\", model=\"dslim/bert-large-NER\", aggregation_strategy=\"simple\")\n",
"\n",
"def row2NamedEntities(row):\n",
" ner_results = ner(row['entity'])\n",
" metadata = {'chunk_id': row['chunk_id'], 'type': 'entity'}\n",
" entities = []\n",
" for result in ner_results:\n",
" entities = entities + [{'entity': result['word'], 'catetory': result['entity_group'], **metadata}]\n",
" \n",
" return entities\n",
"\n",
"\n",
"\n",
"def dfText2DfNE(dataframe: pd.DataFrame):\n",
" ## Takes a dataframe from the parsed data and returns dataframe with named entities. \n",
" ## The input dataframe must have a entity and a chunk_id column. \n",
"\n",
" ## 1. Calculate named entities for each row of the dataframe. \n",
" results = dataframe.apply(row2NamedEntities, axis=1).reset_index(drop=True)\n",
"\n",
" ## Flatten the list of lists to one single list of entities. \n",
" entities_list = np.concatenate(results).ravel().tolist()\n",
"\n",
" ## Remove all NaN entities\n",
" entities_dataframe = pd.DataFrame(entities_list).replace(' ', np.nan)\n",
" entities_dataframe = entities_dataframe.dropna(subset=['entity'])\n",
"\n",
" ## Count the number of occurances per chunk id\n",
" # entities_dataframe = entities_dataframe.groupby(['entity', 'category', 'chunk_id']).size().reset_index(name='count')\n",
"\n",
" return entities_dataframe"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>entity</th>\n",
" <th>catetory</th>\n",
" <th>chunk_id</th>\n",
" <th>type</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Mental Health</td>\n",
" <td>MISC</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Health Equity</td>\n",
" <td>ORG</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>World Health Organization (WHO)</td>\n",
" <td>ORG</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>United Nations (UN)</td>\n",
" <td>ORG</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Sustainable Development Goals (SDGs)</td>\n",
" <td>MISC</td>\n",
" <td>83d4d0367bb0467e811782a4ada3bbb9</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>967</th>\n",
" <td>National University of Singapore's Saw Swee Ho...</td>\n",
" <td>ORG</td>\n",
" <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>968</th>\n",
" <td>Associate Professor</td>\n",
" <td>MISC</td>\n",
" <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>969</th>\n",
" <td>Jeremy Lim</td>\n",
" <td>PER</td>\n",
" <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>970</th>\n",
" <td>Janice Tan</td>\n",
" <td>PER</td>\n",
" <td>4dbae5e4a3ee45bdbf9d55dec8921c2c</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" <tr>\n",
" <th>971</th>\n",
" <td>singapore</td>\n",
" <td>LOC</td>\n",
" <td>2f3c4ce941c14624bfbd3da82ff8def7</td>\n",
" <td>entity</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>972 rows × 4 columns</p>\n",
"</div>"
],
"text/plain": [
" entity catetory \\\n",
"0 Mental Health MISC \n",
"1 Health Equity ORG \n",
"2 World Health Organization (WHO) ORG \n",
"3 United Nations (UN) ORG \n",
"4 Sustainable Development Goals (SDGs) MISC \n",
".. ... ... \n",
"967 National University of Singapore's Saw Swee Ho... ORG \n",
"968 Associate Professor MISC \n",
"969 Jeremy Lim PER \n",
"970 Janice Tan PER \n",
"971 singapore LOC \n",
"\n",
" chunk_id type \n",
"0 83d4d0367bb0467e811782a4ada3bbb9 entity \n",
"1 83d4d0367bb0467e811782a4ada3bbb9 entity \n",
"2 83d4d0367bb0467e811782a4ada3bbb9 entity \n",
"3 83d4d0367bb0467e811782a4ada3bbb9 entity \n",
"4 83d4d0367bb0467e811782a4ada3bbb9 entity \n",
".. ... ... \n",
"967 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",
"968 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",
"969 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",
"970 4dbae5e4a3ee45bdbf9d55dec8921c2c entity \n",
"971 2f3c4ce941c14624bfbd3da82ff8def7 entity \n",
"\n",
"[972 rows x 4 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"dataframe_dir = 'OrfPathHealth'\n",
"df_concepts = pd.read_csv(f\"./data/output/{dataframe_dir}/concepts.csv\", sep=\"|\")\n",
"\n",
"dfc_split = dfText2DfNE(df_concepts)\n",
"dfc_split"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "OpenAI@3111",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}