An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
Made changes based off pr feedback
zenith110a1844fa4aa6f
diff --git a/knowledge_storm/rm.py b/knowledge_storm/rm.pyindex 441ea57..3d6092d 100644--- a/knowledge_storm/rm.py+++ b/knowledge_storm/rm.py@@ -1,7 +1,6 @@import loggingimport os-from typing import Callable, Union, List, Dict-from typing_extensions import Dict+from typing import Callable, Union, Listimport dspyimport pandas as pd@@ -11,18 +10,15 @@ from langchain_huggingface import HuggingFaceEmbeddingsfrom langchain_qdrant import Qdrantfrom qdrant_client import QdrantClient, modelsfrom tqdm import tqdm-import requests-import json-from utils import WebPageHelper++from .utils import WebPageHelperclass YouRM(dspy.Retrieve):def __init__(self, ydc_api_key=None, k=3, is_valid_source: Callable = None):super().__init__(k=k)if not ydc_api_key and not os.environ.get("YDC_API_KEY"):- raise RuntimeError(- "You must supply ydc_api_key or set environment variable YDC_API_KEY"- )+ raise RuntimeError("You must supply ydc_api_key or set environment variable YDC_API_KEY")elif ydc_api_key:self.ydc_api_key = ydc_api_keyelse:@@ -39,11 +35,9 @@ class YouRM(dspy.Retrieve):usage = self.usageself.usage = 0- return {"YouRM": usage}+ return {'YouRM': usage}- def forward(- self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []- ):+ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):"""Search with You.com for self.k top passages for query or queriesArgs:@@ -69,30 +63,21 @@ class YouRM(dspy.Retrieve):).json()authoritative_results = []- for r in results["hits"]:- if self.is_valid_source(r["url"]) and r["url"] not in exclude_urls:+ for r in results['hits']:+ if self.is_valid_source(r['url']) and r['url'] not in exclude_urls:authoritative_results.append(r)- if "hits" in results:- collected_results.extend(authoritative_results[: self.k])+ if 'hits' in results:+ collected_results.extend(authoritative_results[:self.k])except Exception as e:- logging.error(f"Error occurs when searching query {query}: {e}")+ logging.error(f'Error occurs when searching query {query}: {e}')return collected_resultsclass BingSearch(dspy.Retrieve):- def __init__(- self,- bing_search_api_key=None,- k=3,- is_valid_source: Callable = None,- min_char_count: int = 150,- snippet_chunk_size: int = 1000,- webpage_helper_max_threads=10,- mkt="en-US",- language="en",- **kwargs,- ):+ def __init__(self, bing_search_api_key=None, k=3, is_valid_source: Callable = None,+ min_char_count: int = 150, snippet_chunk_size: int = 1000, webpage_helper_max_threads=10,+ mkt='en-US', language='en', **kwargs):"""Params:min_char_count: Minimum character count for the article to be considered valid.@@ -104,18 +89,22 @@ class BingSearch(dspy.Retrieve):super().__init__(k=k)if not bing_search_api_key and not os.environ.get("BING_SEARCH_API_KEY"):raise RuntimeError(- "You must supply bing_search_subscription_key or set environment variable BING_SEARCH_API_KEY"- )+ "You must supply bing_search_subscription_key or set environment variable BING_SEARCH_API_KEY")elif bing_search_api_key:self.bing_api_key = bing_search_api_keyelse:self.bing_api_key = os.environ["BING_SEARCH_API_KEY"]self.endpoint = "https://api.bing.microsoft.com/v7.0/search"- self.params = {"mkt": mkt, "setLang": language, "count": k, **kwargs}+ self.params = {+ 'mkt': mkt,+ "setLang": language,+ "count": k,+ **kwargs+ }self.webpage_helper = WebPageHelper(min_char_count=min_char_count,snippet_chunk_size=snippet_chunk_size,- max_thread_num=webpage_helper_max_threads,+ max_thread_num=webpage_helper_max_threads)self.usage = 0@@ -129,11 +118,9 @@ class BingSearch(dspy.Retrieve):usage = self.usageself.usage = 0- return {"BingSearch": usage}+ return {'BingSearch': usage}- def forward(- self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []- ):+ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str] = []):"""Search with Bing for self.k top passages for query or queriesArgs:@@ -157,26 +144,22 @@ class BingSearch(dspy.Retrieve):for query in queries:try:results = requests.get(- self.endpoint, headers=headers, params={**self.params, "q": query}+ self.endpoint,+ headers=headers,+ params={**self.params, 'q': query}).json()- for d in results["webPages"]["value"]:- if self.is_valid_source(d["url"]) and d["url"] not in exclude_urls:- url_to_results[d["url"]] = {- "url": d["url"],- "title": d["name"],- "description": d["snippet"],- }+ for d in results['webPages']['value']:+ if self.is_valid_source(d['url']) and d['url'] not in exclude_urls:+ url_to_results[d['url']] = {'url': d['url'], 'title': d['name'], 'description': d['snippet']}except Exception as e:- logging.error(f"Error occurs when searching query {query}: {e}")+ logging.error(f'Error occurs when searching query {query}: {e}')- valid_url_to_snippets = self.webpage_helper.urls_to_snippets(- list(url_to_results.keys())- )+ valid_url_to_snippets = self.webpage_helper.urls_to_snippets(list(url_to_results.keys()))collected_results = []for url in valid_url_to_snippets:r = url_to_results[url]- r["snippets"] = valid_url_to_snippets[url]["snippets"]+ r['snippets'] = valid_url_to_snippets[url]['snippets']collected_results.append(r)return collected_results@@ -194,15 +177,13 @@ class VectorRM(dspy.Retrieve):The documents should be stored in a CSV file."""- def __init__(- self,- collection_name: str = "my_documents",- embedding_model: str = "BAAI/bge-m3",- device: str = "mps",- k: int = 3,- chunk_size: int = 500,- chunk_overlap: int = 100,- ):+ def __init__(self,+ collection_name: str = "my_documents",+ embedding_model: str = 'BAAI/bge-m3',+ device: str = "mps",+ k: int = 3,+ chunk_size: int = 500,+ chunk_overlap: int = 100):"""Params:collection_name: Name of the Qdrant collection.@@ -218,9 +199,7 @@ class VectorRM(dspy.Retrieve):model_kwargs = {"device": device}encode_kwargs = {"normalize_embeddings": True}self.model = HuggingFaceEmbeddings(- model_name=embedding_model,- model_kwargs=model_kwargs,- encode_kwargs=encode_kwargs,+ model_name=embedding_model, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs)self.chunk_size = chunk_size@@ -237,24 +216,18 @@ class VectorRM(dspy.Retrieve):if self.client is None:raise ValueError("Qdrant client is not initialized.")if self.client.collection_exists(collection_name=f"{self.collection_name}"):- print(- f"Collection {self.collection_name} exists. Loading the collection..."- )+ print(f"Collection {self.collection_name} exists. Loading the collection...")self.qdrant = Qdrant(client=self.client,collection_name=self.collection_name,embeddings=self.model,)else:- print(- f"Collection {self.collection_name} does not exist. Creating the collection..."- )+ print(f"Collection {self.collection_name} does not exist. Creating the collection...")# create the collectionself.client.create_collection(collection_name=f"{self.collection_name}",- vectors_config=models.VectorParams(- size=1024, distance=models.Distance.COSINE- ),+ vectors_config=models.VectorParams(size=1024, distance=models.Distance.COSINE),)self.qdrant = Qdrant(client=self.client,@@ -300,13 +273,13 @@ class VectorRM(dspy.Retrieve):raise ValueError(f"Error occurs when loading the vector store: {e}")def update_vector_store(- self,- file_path: str,- content_column: str,- title_column: str = "title",- url_column: str = "url",- desc_column: str = "description",- batch_size: int = 64,+ self,+ file_path: str,+ content_column: str,+ title_column: str = "title",+ url_column: str = "url",+ desc_column: str = "description",+ batch_size: int = 64):"""Takes a CSV file where each row is a document and has columns for content, title, url, and description.@@ -323,7 +296,7 @@ class VectorRM(dspy.Retrieve):if file_path is None:raise ValueError("Please provide a file path.")# check if the file is a csv file- if not file_path.endswith(".csv"):+ if not file_path.endswith('.csv'):raise ValueError(f"Not valid file format. Please provide a csv file.")if content_column is None:raise ValueError("Please provide the name of the content column.")@@ -337,9 +310,7 @@ class VectorRM(dspy.Retrieve):df = pd.read_csv(file_path)# check that content column exists and url column existsif content_column not in df.columns:- raise ValueError(- f"Content column {content_column} not found in the csv file."- )+ raise ValueError(f"Content column {content_column} not found in the csv file.")if url_column not in df.columns:raise ValueError(f"URL column {url_column} not found in the csv file.")@@ -347,17 +318,16 @@ class VectorRM(dspy.Retrieve):Document(page_content=row[content_column],metadata={- "title": row.get(title_column, ""),+ "title": row.get(title_column, ''),"url": row[url_column],- "description": row.get(desc_column, ""),- },+ "description": row.get(desc_column, ''),+ })- for row in df.to_dict(orient="records")+ for row in df.to_dict(orient='records')]# split the documentsfrom langchain_text_splitters import RecursiveCharacterTextSplitter-text_splitter = RecursiveCharacterTextSplitter(chunk_size=self.chunk_size,chunk_overlap=self.chunk_overlap,@@ -375,7 +345,7 @@ class VectorRM(dspy.Retrieve):" ","\u200B", # Zero-width space"",- ],+ ])split_documents = text_splitter.split_documents(documents)@@ -393,7 +363,7 @@ class VectorRM(dspy.Retrieve):usage = self.usageself.usage = 0- return {"VectorRM": usage}+ return {'VectorRM': usage}def get_vector_count(self):"""@@ -426,63 +396,71 @@ class VectorRM(dspy.Retrieve):related_docs = self.qdrant.similarity_search_with_score(query, k=self.k)for i in range(len(related_docs)):doc = related_docs[i][0]- collected_results.append(- {- "description": doc.metadata["description"],- "snippets": [doc.page_content],- "title": doc.metadata["title"],- "url": doc.metadata["url"],- }- )+ collected_results.append({+ 'description': doc.metadata['description'],+ 'snippets': [doc.page_content],+ 'title': doc.metadata['title'],+ 'url': doc.metadata['url'],+ })return collected_results-class SerperRM(dspy.Retrieve):"""Retrieve information from custom queries using Serper.dev.-- To be compatible with STORM, the results should have the following fields:- - snippet: Snippets that will be used for the document- - title: The title of the document.- - url: The URL of the document. STORM use url as the unique identifier of the document, so ensure different- documents have different urls.- - description (optional): The description of the document.-"""+ """+ Args:+ serper_search_api_key str: Api key to run serper, can be found by creating an account on https://serper.dev/+ query_params (dict or list of dict): paramaters in dictionary or list of dictionaries that has a max size of 100 that will be used to query.+ values:+ q str: query that will be used with google search+ type str: type that will be used for browsing google. Types are search, images, video, maps, places and many more. Refer to the playground for the types.+ gl str: Country that will be focused on for the search+ location str: Country where the search will originate from. All locates can be found here: https://api.serper.dev/locations.+ autocorrect bool: Enable autocorrect on the queries while searching, if query is misspelled, will be updated.+ results int: Max number of results per page.+ page int: Max number of pages per call.+ tbs str: date time range, automically set to any time by default.+ qdr:h str: Date time range for past hour.+ qdr:d str: Date time range for past 24 hours.+ qdr:w str: Date time range for past week.+ qdr:m str: Date time range for past month.+ qdr:y str: Date time range for past year.+ """def __init__(self, serper_search_api_key=None, query_params=None):super().__init__()self.usage = 0self.query_params = query_paramsself.serper_search_api_key = serper_search_api_key- if not self.serper_search_api_key and not os.environ.get("SERPER_API_KEY"):+ if not self.serper_search_api_key and not os.environ.get('SERPER_API_KEY'):raise RuntimeError(- "You must supply a serper_search_api_key param or set environment variable SERPER_API_KEY"+ 'You must supply a serper_search_api_key param or set environment variable SERPER_API_KEY')elif self.serper_search_api_key:self.serper_search_api_key = serper_search_api_keyelse:- self.serper_search_api_key = os.environ["SERPER_API_KEY"]+ self.serper_search_api_key = os.environ['SERPER_API_KEY']- self.base_url = "https://google.serper.dev"+ self.base_url = 'https://google.serper.dev'def serper_runner(self, query_params):- self.search_url = f"{self.base_url}/search"+ self.search_url = f'{self.base_url}/search'headers = {- "X-API-KEY": self.serper_search_api_key,- "Content-Type": "application/json",+ 'X-API-KEY': self.serper_search_api_key,+ 'Content-Type': 'application/json',}response = requests.request(- "POST", self.search_url, headers=headers, json=query_params+ 'POST', self.search_url, headers=headers, json=query_params)if response == None:raise RuntimeError(- f"Error had occured while running the process {process_name}.\n Error is {response.reason}, had failed with status code {response.status_code}"+ f'Error had occured while running the search process.\n Error is {response.reason}, had failed with status code {response.status_code}')return response.json()@@ -490,18 +468,19 @@ class SerperRM(dspy.Retrieve):def get_usage_and_reset(self):usage = self.usageself.usage = 0- return {"SerperRM": usage}+ return {'SerperRM': usage}def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str]):"""Calls the API and searches for the query passed in.-++Args:query_or_queries (Union[str, List[str]]): The query or queries to search for.exclude_urls (List[str]): Dummy parameter to match the interface. Does not have any effect.Returns:- a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ a list of dictionaries, each dictionary has keys of 'description', 'snippets' (list of strings), 'title', 'url'"""queries = ([query_or_queries]@@ -513,43 +492,53 @@ class SerperRM(dspy.Retrieve):self.results = []collected_results = []for query in queries:- if query == "Queries:":+ if query == 'Queries:':continuequery_params = self.query_params- query_params["q"] = query- query_params["type"] = "search"+ """+ All paramaters can be found in the playground: https://serper.dev/playground+ """+ # Sets the json value for query to be the query that is being parsed.+ query_params['q'] = query++ # Sets the type to be search, can be images, video, places, maps etc that Google provides.+ query_params['type'] = 'search'+self.result = self.serper_runner(query_params)self.results.append(self.result)+ # Array of dictionaries that will be used by Storm to create the jsonscollected_results = []for result in self.results:try:- organic_results = result.get("organic")+ # An array of dictionaries that contains the snippets, title of the document and url that will be used.+ organic_results = result.get('organic')- knowledge_graph = result.get("knowledgeGraph")+ knowledge_graph = result.get('knowledgeGraph')for organic in organic_results:snippets = []- snippets.append(organic.get("snippet"))+ snippets.append(organic.get('snippet'))if knowledge_graph != None:collected_results.append({- "snippets": snippets,- "title": organic.get("title"),- "url": organic.get("link"),- "description": knowledge_graph.get("description"),+ 'snippets': snippets,+ 'title': organic.get('title'),+ 'url': organic.get('link'),+ 'description': knowledge_graph.get('description'),})else:+ # Common for knowledge graph to be None, set description to empty stringcollected_results.append({- "snippets": snippets,- "title": result.get("title"),- "url": result.get("link"),- "description": "",+ 'snippets': snippets,+ 'title': result.get('title'),+ 'url': result.get('link'),+ 'description': '',})except:continue- return collected_results+ return collected_results\ No newline at end of file