An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
moved the function that creates the vector store from rm.py to utils.py
AMMAS1498aa6427860
diff --git a/examples/run_storm_wiki_gpt_with_VectorRM.py b/examples/run_storm_wiki_gpt_with_VectorRM.pyindex 863b95d..320610e 100644--- a/examples/run_storm_wiki_gpt_with_VectorRM.py+++ b/examples/run_storm_wiki_gpt_with_VectorRM.py@@ -30,10 +30,11 @@ import osimport sysfrom argparse import ArgumentParser+sys.path.append('./')from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigsfrom knowledge_storm.rm import VectorRMfrom knowledge_storm.lm import OpenAIModel, AzureOpenAIModel-from knowledge_storm.utils import load_api_key+from knowledge_storm.utils import load_api_key, VectorStoreManagerdef main(args):@@ -83,6 +84,31 @@ def main(args):max_thread_num=args.max_thread_num,)+ # Create / update the vector store with the documents in the csv file+ if args.csv_file_path:+ kwargs = {+ 'file_path': args.csv_file_path,+ 'content_column': 'content',+ 'title_column': 'title',+ 'url_column': 'url',+ 'desc_column': 'description',+ 'batch_size': args.embed_batch_size,+ 'vector_db_mode': args.vector_db_mode,+ 'collection_name': args.collection_name,+ 'device': args.device,+ }+ if args.vector_db_mode == 'offline':+ VectorStoreManager.create_or_update_vector_store(+ vector_store_path=args.offline_vector_db_dir,+ **kwargs+ )+ elif args.vector_db_mode == 'online':+ VectorStoreManager.create_or_update_vector_store(+ url=args.online_vector_db_url,+ api_key=os.getenv('QDRANT_API_KEY'),+ **kwargs+ )+# Setup VectorRM to retrieve information from your own datarm = VectorRM(collection_name=args.collection_name, device=args.device, k=engine_args.search_top_k)@@ -92,17 +118,6 @@ def main(args):elif args.vector_db_mode == 'online':rm.init_online_vector_db(url=args.online_vector_db_url, api_key=os.getenv('QDRANT_API_KEY'))- # Update the vector store with the documents in the csv file- if args.csv_file_path:- rm.update_vector_store(- file_path=args.csv_file_path,- content_column='content',- title_column='title',- url_column='url',- desc_column='description',- batch_size=args.embed_batch_size- )-# Initialize the STORM Wiki Runnerrunner = STORMWikiRunner(engine_args, engine_lm_configs, rm)diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex 0951ef3..723b125 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -5,11 +5,10 @@ from typing import Callable, Union, Listimport dspyimport pandas as pdimport requests-from langchain_core.documents import Document+from langchain_huggingface import HuggingFaceEmbeddingsfrom langchain_qdrant import Qdrant-from qdrant_client import QdrantClient, models-from tqdm import tqdm+from qdrant_client import QdrantClientfrom .utils import WebPageHelper@@ -182,16 +181,13 @@ class VectorRM(dspy.Retrieve):embedding_model: str = 'BAAI/bge-m3',device: str = "mps",k: int = 3,- chunk_size: int = 500,- chunk_overlap: int = 100):+ ):"""Params:collection_name: Name of the Qdrant collection.embedding_model: Name of the Hugging Face embedding model.device: Device to run the embeddings model on, can be "mps", "cuda", "cpu".k: Number of top chunks to retrieve.- chunk_size: Size of each chunk if you need to build the vector store from documents.- chunk_overlap: Overlap between chunks if you need to build the vector store from documents."""super().__init__(k=k)self.usage = 0@@ -202,14 +198,11 @@ class VectorRM(dspy.Retrieve):model_name=embedding_model, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs)- self.chunk_size = chunk_size- self.chunk_overlap = chunk_overlap-self.collection_name = collection_nameself.client = Noneself.qdrant = None- def _check_create_collection(self):+ def _check_collection(self):"""Check if the Qdrant collection exists and create it if it does not."""@@ -223,17 +216,7 @@ class VectorRM(dspy.Retrieve):embeddings=self.model,)else:- print(f"Collection {self.collection_name} does not exist. Creating the collection...")- # create the collection- self.client.create_collection(- collection_name=f"{self.collection_name}",- vectors_config=models.VectorParams(size=1024, distance=models.Distance.COSINE),- )- self.qdrant = Qdrant(- client=self.client,- collection_name=self.collection_name,- embeddings=self.model,- )+ raise ValueError(f"Collection {self.collection_name} does not exist. Please create the collection first.")def init_online_vector_db(self, url: str, api_key: str):"""@@ -252,7 +235,7 @@ class VectorRM(dspy.Retrieve):try:self.client = QdrantClient(url=url, api_key=api_key)- self._check_create_collection()+ self._check_collection()except Exception as e:raise ValueError(f"Error occurs when connecting to the server: {e}")@@ -268,97 +251,10 @@ class VectorRM(dspy.Retrieve):try:self.client = QdrantClient(path=vector_store_path)- self._check_create_collection()+ self._check_collection()except Exception as e:raise ValueError(f"Error occurs when loading the vector store: {e}")- def update_vector_store(- self,- file_path: str,- content_column: str,- title_column: str = "title",- url_column: str = "url",- desc_column: str = "description",- batch_size: int = 64- ):- """- Takes a CSV file where each row is a document and has columns for content, title, url, and description.- Then it converts all these documents in the content column to vectors and add them the Qdrant collection.-- Args:- file_path (str): Path to the CSV file.- content_column (str): Name of the column containing the content.- title_column (str): Name of the column containing the title. Default is "title".- url_column (str): Name of the column containing the URL. Default is "url".- desc_column (str): Name of the column containing the description. Default is "description".- batch_size (int): Batch size for adding documents to the collection.- """- if file_path is None:- raise ValueError("Please provide a file path.")- # check if the file is a csv file- if not file_path.endswith('.csv'):- raise ValueError(f"Not valid file format. Please provide a csv file.")- if content_column is None:- raise ValueError("Please provide the name of the content column.")- if url_column is None:- raise ValueError("Please provide the name of the url column.")-- if self.qdrant is None:- raise ValueError("Qdrant client is not initialized.")-- # read the csv file- df = pd.read_csv(file_path)- # check that content column exists and url column exists- if content_column not in df.columns:- raise ValueError(f"Content column {content_column} not found in the csv file.")- if url_column not in df.columns:- raise ValueError(f"URL column {url_column} not found in the csv file.")-- documents = [- Document(- page_content=row[content_column],- metadata={- "title": row.get(title_column, ''),- "url": row[url_column],- "description": row.get(desc_column, ''),- }- )- for row in df.to_dict(orient='records')- ]-- # split the documents- from langchain_text_splitters import RecursiveCharacterTextSplitter- text_splitter = RecursiveCharacterTextSplitter(- chunk_size=self.chunk_size,- chunk_overlap=self.chunk_overlap,- length_function=len,- add_start_index=True,- separators=[- "\n\n",- "\n",- ".",- "\uff0e", # Fullwidth full stop- "\u3002", # Ideographic full stop- ",",- "\uff0c", # Fullwidth comma- "\u3001", # Ideographic comma- " ",- "\u200B", # Zero-width space- "",- ]- )- split_documents = text_splitter.split_documents(documents)-- # update and save the vector store- num_batches = (len(split_documents) + batch_size - 1) // batch_size- for i in tqdm(range(num_batches)):- start_idx = i * batch_size- end_idx = min((i + 1) * batch_size, len(split_documents))- self.qdrant.add_documents(- documents=split_documents[start_idx:end_idx],- batch_size=batch_size,- )-def get_usage_and_reset(self):usage = self.usageself.usage = 0diff --git a/knowledge_storm/utils.py b/knowledge_storm/utils.pyindex 5cf6f45..b14b535 100644--- a/knowledge_storm/utils.py+++ b/knowledge_storm/utils.py@@ -10,7 +10,13 @@ from typing import List, Dictimport httpximport tomlfrom langchain_text_splitters import RecursiveCharacterTextSplitter+from langchain_core.documents import Document+from langchain_huggingface import HuggingFaceEmbeddings+from langchain_qdrant import Qdrant+from qdrant_client import QdrantClient, modelsfrom trafilatura import extract+from tqdm import tqdm+import pandas as pdlogging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.@@ -33,6 +39,198 @@ def load_api_key(toml_file_path):def makeStringRed(message):return f"\033[91m {message}\033[00m"+class VectorStoreManager:+ @staticmethod+ def _check_create_collection(client: QdrantClient, collection_name: str, model: HuggingFaceEmbeddings):+ """+ Check if the Qdrant collection exists and create it if it does not.+ """+ if client is None:+ raise ValueError("Qdrant client is not initialized.")+ if client.collection_exists(collection_name=f"{collection_name}"):+ print(f"Collection {collection_name} exists. Loading the collection...")+ return Qdrant(+ client=client,+ collection_name=collection_name,+ embeddings=model,+ )+ else:+ print(f"Collection {collection_name} does not exist. Creating the collection...")+ # create the collection+ client.create_collection(+ collection_name=f"{collection_name}",+ vectors_config=models.VectorParams(size=1024, distance=models.Distance.COSINE),+ )+ return Qdrant(+ client=client,+ collection_name=collection_name,+ embeddings=model,+ )+ @staticmethod+ def _init_online_vector_db(url: str, api_key: str, collection_name: str, model: HuggingFaceEmbeddings):+ """+ Initialize the Qdrant client that is connected to an online vector store with the given URL and API key.++ Args:+ url (str): URL of the Qdrant server.+ api_key (str): API key for the Qdrant server.+ """+ if api_key is None:+ if not os.getenv("QDRANT_API_KEY"):+ raise ValueError("Please provide an api key.")+ api_key = os.getenv("QDRANT_API_KEY")+ if url is None:+ raise ValueError("Please provide a url for the Qdrant server.")++ try:+ client = QdrantClient(url=url, api_key=api_key)+ return VectorStoreManager._check_create_collection(client=client, collection_name=collection_name, model=model)+ except Exception as e:+ raise ValueError(f"Error occurs when connecting to the server: {e}")++ def _init_offline_vector_db(vector_store_path: str, collection_name: str, model: HuggingFaceEmbeddings):+ """+ Initialize the Qdrant client that is connected to an offline vector store with the given vector store folder path.++ Args:+ vector_store_path (str): Path to the vector store.+ """+ if vector_store_path is None:+ raise ValueError("Please provide a folder path.")++ try:+ client = QdrantClient(path=vector_store_path)+ return VectorStoreManager._check_create_collection(client=client, collection_name=collection_name, model=model)+ except Exception as e:+ raise ValueError(f"Error occurs when loading the vector store: {e}")++ def create_or_update_vector_store(++ vector_db_mode: str,+ file_path: str,+ content_column: str,+ title_column: str = "title",+ url_column: str = "url",+ desc_column: str = "description",+ batch_size: int = 64,+ chunk_size: int = 500,+ chunk_overlap: int = 100,+ collection_name: str = "my_documents",+ vector_store_path: str = None,+ url: str = None,+ qdrant_api_key: str = None,+ embedding_model: str = 'BAAI/bge-m3',+ device: str = "mps",+ ):+ """+ Takes a CSV file where each row is a document and has columns for content, title, url, and description.+ Then it converts all these documents in the content column to vectors and add them the Qdrant collection.++ Args:+ vector_store_path (str): Path to the directory where the vector store is stored or will be stored.+ vector_db_mode (str): Mode of the Qdrant vector store (offline or online).+ file_path (str): Path to the CSV file.+ content_column (str): Name of the column containing the content.+ title_column (str): Name of the column containing the title. Default is "title".+ url_column (str): Name of the column containing the URL. Default is "url".+ desc_column (str): Name of the column containing the description. Default is "description".+ batch_size (int): Batch size for adding documents to the collection.+ chunk_size: Size of each chunk if you need to build the vector store from documents.+ chunk_overlap: Overlap between chunks if you need to build the vector store from documents.+ collection_name: Name of the Qdrant collection.+ embedding_model: Name of the Hugging Face embedding model.+ device: Device to run the embeddings model on, can be "mps", "cuda", "cpu".+ qdrant_api_key: API key for the Qdrant server (Only required if the Qdrant server is online).+ """++ model_kwargs = {"device": device}+ encode_kwargs = {"normalize_embeddings": True}+ model = HuggingFaceEmbeddings(+ model_name=embedding_model, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs+ )++ if file_path is None:+ raise ValueError("Please provide a file path.")+ # check if the file is a csv file+ if not file_path.endswith('.csv'):+ raise ValueError(f"Not valid file format. Please provide a csv file.")+ if content_column is None:+ raise ValueError("Please provide the name of the content column.")+ if url_column is None:+ raise ValueError("Please provide the name of the url column.")++ # try to initialize the Qdrant client+ qdrant = None+ if vector_db_mode == 'online':+ qdrant = VectorStoreManager._init_online_vector_db(+ url=url,+ api_key=qdrant_api_key,+ collection_name=collection_name,+ model=model,+ )+ elif vector_db_mode == 'offline':+ qdrant = VectorStoreManager._init_offline_vector_db(vector_store_path=vector_store_path, collection_name=collection_name, model=model)+ else:+ raise ValueError("Invalid vector_db_mode. Please provide either 'online' or 'offline'.")+ if qdrant is None:+ raise ValueError("Qdrant client is not initialized.")++ # read the csv file+ df = pd.read_csv(file_path)+ # check that content column exists and url column exists+ if content_column not in df.columns:+ raise ValueError(f"Content column {content_column} not found in the csv file.")+ if url_column not in df.columns:+ raise ValueError(f"URL column {url_column} not found in the csv file.")++ documents = [+ Document(+ page_content=row[content_column],+ metadata={+ "title": row.get(title_column, ''),+ "url": row[url_column],+ "description": row.get(desc_column, ''),+ }+ )+ for row in df.to_dict(orient='records')+ ]++ # split the documents+ from langchain_text_splitters import RecursiveCharacterTextSplitter+ text_splitter = RecursiveCharacterTextSplitter(+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ length_function=len,+ add_start_index=True,+ separators=[+ "\n\n",+ "\n",+ ".",+ "\uff0e", # Fullwidth full stop+ "\u3002", # Ideographic full stop+ ",",+ "\uff0c", # Fullwidth comma+ "\u3001", # Ideographic comma+ " ",+ "\u200B", # Zero-width space+ "",+ ]+ )+ split_documents = text_splitter.split_documents(documents)++ # update and save the vector store+ num_batches = (len(split_documents) + batch_size - 1) // batch_size+ for i in tqdm(range(num_batches)):+ start_idx = i * batch_size+ end_idx = min((i + 1) * batch_size, len(split_documents))+ qdrant.add_documents(+ documents=split_documents[start_idx:end_idx],+ batch_size=batch_size,+ )++ # close the qdrant client+ qdrant.client.close()+class ArticleTextProcessing:@staticmethod