An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Fix import order and refine code comments.
Yijia Shaof9c973877a81
diff --git a/knowledge_storm/utils.py b/knowledge_storm/utils.pyindex efeade7..84f8976 100644--- a/knowledge_storm/utils.py+++ b/knowledge_storm/utils.py@@ -8,15 +8,15 @@ import sysfrom typing import List, Dictimport httpx+import pandas as pdimport toml-from langchain_text_splitters import RecursiveCharacterTextSplitterfrom langchain_core.documents import Documentfrom langchain_huggingface import HuggingFaceEmbeddingsfrom langchain_qdrant import Qdrant+from langchain_text_splitters import RecursiveCharacterTextSplitterfrom qdrant_client import QdrantClient, models-from trafilatura import extractfrom tqdm import tqdm-import pandas as pd+from trafilatura import extractlogging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.@@ -39,18 +39,17 @@ def load_api_key(toml_file_path):def makeStringRed(message):return f"\033[91m {message}\033[00m"+class QdrantVectorStoreManager:"""- This is a helper class to manage the Qdrant vector store. It is related to VectorRM retrieval model in rm.py.- It provides methods to create or update the vector store from a CSV file, before you initialize the VectorRM.- Use the function create_or_update_vector_store to create or update the vector store, then once you have the- vector store, you can initialize the VectorRM with the vector store path or the Qdrant server URL.+ Helper class for managing the Qdrant vector store, can be used with `VectorRM` in rm.py.++ Before you initialize `VectorRM`, call `create_or_update_vector_store` to create or update the vector store.+ Once you have the vector store, you can initialize `VectorRM` with the vector store path or the Qdrant server URL."""@staticmethoddef _check_create_collection(client: QdrantClient, collection_name: str, model: HuggingFaceEmbeddings):- """- Check if the Qdrant collection exists and create it if it does not.- """+ """Check if the Qdrant collection exists and create it if it does not."""if client is None:raise ValueError("Qdrant client is not initialized.")if client.collection_exists(collection_name=f"{collection_name}"):@@ -75,8 +74,7 @@ class QdrantVectorStoreManager:@staticmethoddef _init_online_vector_db(url: str, api_key: str, collection_name: str, model: HuggingFaceEmbeddings):- """- Initialize the Qdrant client that is connected to an online vector store with the given URL and API key.+ """Initialize the Qdrant client that is connected to an online vector store with the given URL and API key.Args:url (str): URL of the Qdrant server.@@ -97,8 +95,7 @@ class QdrantVectorStoreManager:@staticmethoddef _init_offline_vector_db(vector_store_path: str, collection_name: str, model: HuggingFaceEmbeddings):- """- Initialize the Qdrant client that is connected to an offline vector store with the given vector store folder path.+ """Initialize the Qdrant client that is connected to an offline vector store with the given vector store folder path.Args:vector_store_path (str): Path to the vector store.@@ -131,10 +128,13 @@ class QdrantVectorStoreManager:device: str = "mps",):"""- Takes a CSV file where each row is a document and has columns for content, title, url, and description.- Then it converts all these documents in the content column to vectors and add them the Qdrant collection.+ Takes a CSV file and adds each row in the CSV file to the Qdrant collection.++ This function expects each row of the CSV file as a document.+ The CSV file should have columns for "content", "title", "URL", and "description".Args:+ collection_name: Name of the Qdrant collection.vector_store_path (str): Path to the directory where the vector store is stored or will be stored.vector_db_mode (str): Mode of the Qdrant vector store (offline or online).file_path (str): Path to the CSV file.@@ -145,7 +145,6 @@ class QdrantVectorStoreManager:batch_size (int): Batch size for adding documents to the collection.chunk_size: Size of each chunk if you need to build the vector store from documents.chunk_overlap: Overlap between chunks if you need to build the vector store from documents.- collection_name: Name of the Qdrant collection.embedding_model: Name of the Hugging Face embedding model.device: Device to run the embeddings model on, can be "mps", "cuda", "cpu".qdrant_api_key: API key for the Qdrant server (Only required if the Qdrant server is online).