Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

moved the function that creates the vector store from rm.py to utils.py

AMMAS1498aa6427860
diff --git a/examples/run_storm_wiki_gpt_with_VectorRM.py b/examples/run_storm_wiki_gpt_with_VectorRM.py
index 863b95d..320610e 100644
--- a/examples/run_storm_wiki_gpt_with_VectorRM.py
+++ b/examples/run_storm_wiki_gpt_with_VectorRM.py
@@ -30,10 +30,11 @@ import os
import sys
from argparse import ArgumentParser
+sys.path.append('./')
from knowledge_storm import STORMWikiRunnerArguments, STORMWikiRunner, STORMWikiLMConfigs
from knowledge_storm.rm import VectorRM
from knowledge_storm.lm import OpenAIModel, AzureOpenAIModel
-from knowledge_storm.utils import load_api_key
+from knowledge_storm.utils import load_api_key, VectorStoreManager
def main(args):
@@ -83,6 +84,31 @@ def main(args):
max_thread_num=args.max_thread_num,
)
+ # Create / update the vector store with the documents in the csv file
+ if args.csv_file_path:
+ kwargs = {
+ 'file_path': args.csv_file_path,
+ 'content_column': 'content',
+ 'title_column': 'title',
+ 'url_column': 'url',
+ 'desc_column': 'description',
+ 'batch_size': args.embed_batch_size,
+ 'vector_db_mode': args.vector_db_mode,
+ 'collection_name': args.collection_name,
+ 'device': args.device,
+ }
+ if args.vector_db_mode == 'offline':
+ VectorStoreManager.create_or_update_vector_store(
+ vector_store_path=args.offline_vector_db_dir,
+ **kwargs
+ )
+ elif args.vector_db_mode == 'online':
+ VectorStoreManager.create_or_update_vector_store(
+ url=args.online_vector_db_url,
+ api_key=os.getenv('QDRANT_API_KEY'),
+ **kwargs
+ )
+
# Setup VectorRM to retrieve information from your own data
rm = VectorRM(collection_name=args.collection_name, device=args.device, k=engine_args.search_top_k)
@@ -92,17 +118,6 @@ def main(args):
elif args.vector_db_mode == 'online':
rm.init_online_vector_db(url=args.online_vector_db_url, api_key=os.getenv('QDRANT_API_KEY'))
- # Update the vector store with the documents in the csv file
- if args.csv_file_path:
- rm.update_vector_store(
- file_path=args.csv_file_path,
- content_column='content',
- title_column='title',
- url_column='url',
- desc_column='description',
- batch_size=args.embed_batch_size
- )
-
# Initialize the STORM Wiki Runner
runner = STORMWikiRunner(engine_args, engine_lm_configs, rm)
diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.py
index 0951ef3..723b125 100644
--- a/knowledge_storm/rm.py
+++ b/knowledge_storm/rm.py
@@ -5,11 +5,10 @@ from typing import Callable, Union, List
import dspy
import pandas as pd
import requests
-from langchain_core.documents import Document
+
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_qdrant import Qdrant
-from qdrant_client import QdrantClient, models
-from tqdm import tqdm
+from qdrant_client import QdrantClient
from .utils import WebPageHelper
@@ -182,16 +181,13 @@ class VectorRM(dspy.Retrieve):
embedding_model: str = 'BAAI/bge-m3',
device: str = "mps",
k: int = 3,
- chunk_size: int = 500,
- chunk_overlap: int = 100):
+ ):
"""
Params:
collection_name: Name of the Qdrant collection.
embedding_model: Name of the Hugging Face embedding model.
device: Device to run the embeddings model on, can be "mps", "cuda", "cpu".
k: Number of top chunks to retrieve.
- chunk_size: Size of each chunk if you need to build the vector store from documents.
- chunk_overlap: Overlap between chunks if you need to build the vector store from documents.
"""
super().__init__(k=k)
self.usage = 0
@@ -202,14 +198,11 @@ class VectorRM(dspy.Retrieve):
model_name=embedding_model, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs
)
- self.chunk_size = chunk_size
- self.chunk_overlap = chunk_overlap
-
self.collection_name = collection_name
self.client = None
self.qdrant = None
- def _check_create_collection(self):
+ def _check_collection(self):
"""
Check if the Qdrant collection exists and create it if it does not.
"""
@@ -223,17 +216,7 @@ class VectorRM(dspy.Retrieve):
embeddings=self.model,
)
else:
- print(f"Collection {self.collection_name} does not exist. Creating the collection...")
- # create the collection
- self.client.create_collection(
- collection_name=f"{self.collection_name}",
- vectors_config=models.VectorParams(size=1024, distance=models.Distance.COSINE),
- )
- self.qdrant = Qdrant(
- client=self.client,
- collection_name=self.collection_name,
- embeddings=self.model,
- )
+ raise ValueError(f"Collection {self.collection_name} does not exist. Please create the collection first.")
def init_online_vector_db(self, url: str, api_key: str):
"""
@@ -252,7 +235,7 @@ class VectorRM(dspy.Retrieve):
try:
self.client = QdrantClient(url=url, api_key=api_key)
- self._check_create_collection()
+ self._check_collection()
except Exception as e:
raise ValueError(f"Error occurs when connecting to the server: {e}")
@@ -268,97 +251,10 @@ class VectorRM(dspy.Retrieve):
try:
self.client = QdrantClient(path=vector_store_path)
- self._check_create_collection()
+ self._check_collection()
except Exception as e:
raise ValueError(f"Error occurs when loading the vector store: {e}")
- def update_vector_store(
- self,
- file_path: str,
- content_column: str,
- title_column: str = "title",
- url_column: str = "url",
- desc_column: str = "description",
- batch_size: int = 64
- ):
- """
- Takes a CSV file where each row is a document and has columns for content, title, url, and description.
- Then it converts all these documents in the content column to vectors and add them the Qdrant collection.
-
- Args:
- file_path (str): Path to the CSV file.
- content_column (str): Name of the column containing the content.
- title_column (str): Name of the column containing the title. Default is "title".
- url_column (str): Name of the column containing the URL. Default is "url".
- desc_column (str): Name of the column containing the description. Default is "description".
- batch_size (int): Batch size for adding documents to the collection.
- """
- if file_path is None:
- raise ValueError("Please provide a file path.")
- # check if the file is a csv file
- if not file_path.endswith('.csv'):
- raise ValueError(f"Not valid file format. Please provide a csv file.")
- if content_column is None:
- raise ValueError("Please provide the name of the content column.")
- if url_column is None:
- raise ValueError("Please provide the name of the url column.")
-
- if self.qdrant is None:
- raise ValueError("Qdrant client is not initialized.")
-
- # read the csv file
- df = pd.read_csv(file_path)
- # check that content column exists and url column exists
- if content_column not in df.columns:
- raise ValueError(f"Content column {content_column} not found in the csv file.")
- if url_column not in df.columns:
- raise ValueError(f"URL column {url_column} not found in the csv file.")
-
- documents = [
- Document(
- page_content=row[content_column],
- metadata={
- "title": row.get(title_column, ''),
- "url": row[url_column],
- "description": row.get(desc_column, ''),
- }
- )
- for row in df.to_dict(orient='records')
- ]
-
- # split the documents
- from langchain_text_splitters import RecursiveCharacterTextSplitter
- text_splitter = RecursiveCharacterTextSplitter(
- chunk_size=self.chunk_size,
- chunk_overlap=self.chunk_overlap,
- length_function=len,
- add_start_index=True,
- separators=[
- "\n\n",
- "\n",
- ".",
- "\uff0e", # Fullwidth full stop
- "\u3002", # Ideographic full stop
- ",",
- "\uff0c", # Fullwidth comma
- "\u3001", # Ideographic comma
- " ",
- "\u200B", # Zero-width space
- "",
- ]
- )
- split_documents = text_splitter.split_documents(documents)
-
- # update and save the vector store
- num_batches = (len(split_documents) + batch_size - 1) // batch_size
- for i in tqdm(range(num_batches)):
- start_idx = i * batch_size
- end_idx = min((i + 1) * batch_size, len(split_documents))
- self.qdrant.add_documents(
- documents=split_documents[start_idx:end_idx],
- batch_size=batch_size,
- )
-
def get_usage_and_reset(self):
usage = self.usage
self.usage = 0
diff --git a/knowledge_storm/utils.py b/knowledge_storm/utils.py
index 5cf6f45..b14b535 100644
--- a/knowledge_storm/utils.py
+++ b/knowledge_storm/utils.py
@@ -10,7 +10,13 @@ from typing import List, Dict
import httpx
import toml
from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_qdrant import Qdrant
+from qdrant_client import QdrantClient, models
from trafilatura import extract
+from tqdm import tqdm
+import pandas as pd
logging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.
@@ -33,6 +39,198 @@ def load_api_key(toml_file_path):
def makeStringRed(message):
return f"\033[91m {message}\033[00m"
+class VectorStoreManager:
+ @staticmethod
+ def _check_create_collection(client: QdrantClient, collection_name: str, model: HuggingFaceEmbeddings):
+ """
+ Check if the Qdrant collection exists and create it if it does not.
+ """
+ if client is None:
+ raise ValueError("Qdrant client is not initialized.")
+ if client.collection_exists(collection_name=f"{collection_name}"):
+ print(f"Collection {collection_name} exists. Loading the collection...")
+ return Qdrant(
+ client=client,
+ collection_name=collection_name,
+ embeddings=model,
+ )
+ else:
+ print(f"Collection {collection_name} does not exist. Creating the collection...")
+ # create the collection
+ client.create_collection(
+ collection_name=f"{collection_name}",
+ vectors_config=models.VectorParams(size=1024, distance=models.Distance.COSINE),
+ )
+ return Qdrant(
+ client=client,
+ collection_name=collection_name,
+ embeddings=model,
+ )
+ @staticmethod
+ def _init_online_vector_db(url: str, api_key: str, collection_name: str, model: HuggingFaceEmbeddings):
+ """
+ Initialize the Qdrant client that is connected to an online vector store with the given URL and API key.
+
+ Args:
+ url (str): URL of the Qdrant server.
+ api_key (str): API key for the Qdrant server.
+ """
+ if api_key is None:
+ if not os.getenv("QDRANT_API_KEY"):
+ raise ValueError("Please provide an api key.")
+ api_key = os.getenv("QDRANT_API_KEY")
+ if url is None:
+ raise ValueError("Please provide a url for the Qdrant server.")
+
+ try:
+ client = QdrantClient(url=url, api_key=api_key)
+ return VectorStoreManager._check_create_collection(client=client, collection_name=collection_name, model=model)
+ except Exception as e:
+ raise ValueError(f"Error occurs when connecting to the server: {e}")
+
+ def _init_offline_vector_db(vector_store_path: str, collection_name: str, model: HuggingFaceEmbeddings):
+ """
+ Initialize the Qdrant client that is connected to an offline vector store with the given vector store folder path.
+
+ Args:
+ vector_store_path (str): Path to the vector store.
+ """
+ if vector_store_path is None:
+ raise ValueError("Please provide a folder path.")
+
+ try:
+ client = QdrantClient(path=vector_store_path)
+ return VectorStoreManager._check_create_collection(client=client, collection_name=collection_name, model=model)
+ except Exception as e:
+ raise ValueError(f"Error occurs when loading the vector store: {e}")
+
+ def create_or_update_vector_store(
+
+ vector_db_mode: str,
+ file_path: str,
+ content_column: str,
+ title_column: str = "title",
+ url_column: str = "url",
+ desc_column: str = "description",
+ batch_size: int = 64,
+ chunk_size: int = 500,
+ chunk_overlap: int = 100,
+ collection_name: str = "my_documents",
+ vector_store_path: str = None,
+ url: str = None,
+ qdrant_api_key: str = None,
+ embedding_model: str = 'BAAI/bge-m3',
+ device: str = "mps",
+ ):
+ """
+ Takes a CSV file where each row is a document and has columns for content, title, url, and description.
+ Then it converts all these documents in the content column to vectors and add them the Qdrant collection.
+
+ Args:
+ vector_store_path (str): Path to the directory where the vector store is stored or will be stored.
+ vector_db_mode (str): Mode of the Qdrant vector store (offline or online).
+ file_path (str): Path to the CSV file.
+ content_column (str): Name of the column containing the content.
+ title_column (str): Name of the column containing the title. Default is "title".
+ url_column (str): Name of the column containing the URL. Default is "url".
+ desc_column (str): Name of the column containing the description. Default is "description".
+ batch_size (int): Batch size for adding documents to the collection.
+ chunk_size: Size of each chunk if you need to build the vector store from documents.
+ chunk_overlap: Overlap between chunks if you need to build the vector store from documents.
+ collection_name: Name of the Qdrant collection.
+ embedding_model: Name of the Hugging Face embedding model.
+ device: Device to run the embeddings model on, can be "mps", "cuda", "cpu".
+ qdrant_api_key: API key for the Qdrant server (Only required if the Qdrant server is online).
+ """
+
+ model_kwargs = {"device": device}
+ encode_kwargs = {"normalize_embeddings": True}
+ model = HuggingFaceEmbeddings(
+ model_name=embedding_model, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs
+ )
+
+ if file_path is None:
+ raise ValueError("Please provide a file path.")
+ # check if the file is a csv file
+ if not file_path.endswith('.csv'):
+ raise ValueError(f"Not valid file format. Please provide a csv file.")
+ if content_column is None:
+ raise ValueError("Please provide the name of the content column.")
+ if url_column is None:
+ raise ValueError("Please provide the name of the url column.")
+
+ # try to initialize the Qdrant client
+ qdrant = None
+ if vector_db_mode == 'online':
+ qdrant = VectorStoreManager._init_online_vector_db(
+ url=url,
+ api_key=qdrant_api_key,
+ collection_name=collection_name,
+ model=model,
+ )
+ elif vector_db_mode == 'offline':
+ qdrant = VectorStoreManager._init_offline_vector_db(vector_store_path=vector_store_path, collection_name=collection_name, model=model)
+ else:
+ raise ValueError("Invalid vector_db_mode. Please provide either 'online' or 'offline'.")
+ if qdrant is None:
+ raise ValueError("Qdrant client is not initialized.")
+
+ # read the csv file
+ df = pd.read_csv(file_path)
+ # check that content column exists and url column exists
+ if content_column not in df.columns:
+ raise ValueError(f"Content column {content_column} not found in the csv file.")
+ if url_column not in df.columns:
+ raise ValueError(f"URL column {url_column} not found in the csv file.")
+
+ documents = [
+ Document(
+ page_content=row[content_column],
+ metadata={
+ "title": row.get(title_column, ''),
+ "url": row[url_column],
+ "description": row.get(desc_column, ''),
+ }
+ )
+ for row in df.to_dict(orient='records')
+ ]
+
+ # split the documents
+ from langchain_text_splitters import RecursiveCharacterTextSplitter
+ text_splitter = RecursiveCharacterTextSplitter(
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ length_function=len,
+ add_start_index=True,
+ separators=[
+ "\n\n",
+ "\n",
+ ".",
+ "\uff0e", # Fullwidth full stop
+ "\u3002", # Ideographic full stop
+ ",",
+ "\uff0c", # Fullwidth comma
+ "\u3001", # Ideographic comma
+ " ",
+ "\u200B", # Zero-width space
+ "",
+ ]
+ )
+ split_documents = text_splitter.split_documents(documents)
+
+ # update and save the vector store
+ num_batches = (len(split_documents) + batch_size - 1) // batch_size
+ for i in tqdm(range(num_batches)):
+ start_idx = i * batch_size
+ end_idx = min((i + 1) * batch_size, len(split_documents))
+ qdrant.add_documents(
+ documents=split_documents[start_idx:end_idx],
+ batch_size=batch_size,
+ )
+
+ # close the qdrant client
+ qdrant.client.close()
+
class ArticleTextProcessing:
@staticmethod