An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
encapsulate encoding function and Encoder class
Yucheng-Jiang0764f542b516
diff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.pyindex 2e281f6..7977143 100644--- a/knowledge_storm/collaborative_storm/engine.py+++ b/knowledge_storm/collaborative_storm/engine.py@@ -14,6 +14,7 @@ from .modules.co_storm_agents import (from .modules.expert_generation import GenerateExpertModulefrom .modules.warmstart_hierarchical_chat import WarmStartModulefrom ..dataclass import ConversationTurn, KnowledgeBase+from ..encoder import Encoderfrom ..interface import LMConfigs, Agentfrom ..logging_wrapper import LoggingWrapperfrom ..lm import OpenAIModel, AzureOpenAIModel, TogetherClient@@ -323,6 +324,7 @@ class DiscourseManager:lm_config: CollaborativeStormLMConfigs,runner_argument: RunnerArgument,rm: dspy.Retrieve,+ encoder: Encoder,callback_handler: BaseCallbackHandler,):# parameter management@@ -331,6 +333,7 @@ class DiscourseManager:self.logging_wrapper = logging_wrapperself.callback_handler = callback_handlerself.rm = rm+ self.encoder = encoder# role managementself.experts: List[CoStormExpert] = []self.simulated_user: SimulatedUser = SimulatedUser(@@ -360,6 +363,7 @@ class DiscourseManager:lm_config=self.lm_config,runner_argument=self.runner_argument,logging_wrapper=self.logging_wrapper,+ encoder=self.encoder,callback_handler=self.callback_handler,)self.general_knowledge_provider = CoStormExpert(@@ -516,18 +520,21 @@ class CoStormRunner:self.rm = BingSearch(k=runner_argument.retrieve_top_k)else:self.rm = rm+ self.encoder = Encoder()self.conversation_history = []self.warmstart_conv_archive = []self.knowledge_base = KnowledgeBase(topic=self.runner_argument.topic,knowledge_base_lm=self.lm_config.knowledge_base_lm,node_expansion_trigger_count=self.runner_argument.node_expansion_trigger_count,+ encoder=self.encoder,)self.discourse_manager = DiscourseManager(lm_config=self.lm_config,runner_argument=self.runner_argument,logging_wrapper=self.logging_wrapper,rm=self.rm,+ encoder=self.encoder,callback_handler=callback_handler,)@@ -555,6 +562,7 @@ class CoStormRunner:runner_argument=RunnerArgument.from_dict(data["runner_argument"]),logging_wrapper=LoggingWrapper(lm_config),)+ costorm_runner.encoder = Encoder()costorm_runner.conversation_history = [ConversationTurn.from_dict(turn) for turn in data["conversation_history"]]@@ -567,6 +575,7 @@ class CoStormRunner:data=data["knowledge_base"],knowledge_base_lm=costorm_runner.lm_config.knowledge_base_lm,node_expansion_trigger_count=costorm_runner.runner_argument.node_expansion_trigger_count,+ encoder=costorm_runner.encoder,)return costorm_runnerdiff --git a/knowledge_storm/collaborative_storm/modules/co_storm_agents.py b/knowledge_storm/collaborative_storm/modules/co_storm_agents.pyindex e7f6029..dd42cb8 100644--- a/knowledge_storm/collaborative_storm/modules/co_storm_agents.py+++ b/knowledge_storm/collaborative_storm/modules/co_storm_agents.py@@ -13,7 +13,7 @@ from .costorm_expert_utterance_generator import CoStormExpertUtteranceGenerationfrom .grounded_question_generation import GroundedQuestionGenerationModulefrom .simulate_user import GenSimulatedUserUtterancefrom ...dataclass import ConversationTurn, KnowledgeBase-from ...encoder import get_text_embeddings+from ...encoder import Encoderfrom ...interface import Agent, Information, LMConfigsfrom ...logging_wrapper import LoggingWrapper@@ -174,6 +174,7 @@ class Moderator(Agent):lm_config: LMConfigs,runner_argument: "RunnerArgument",logging_wrapper: LoggingWrapper,+ encoder: Encoder,callback_handler: BaseCallbackHandler = None,):super().__init__(topic, role_name, role_description)@@ -184,6 +185,7 @@ class Moderator(Agent):engine=self.lm_config.question_asking_lm)self.callback_handler = callback_handler+ self.encoder = encoderdef _get_conv_turn_unused_information(self, conv_turn: ConversationTurn, knowledge_base: KnowledgeBase@@ -211,19 +213,12 @@ class Moderator(Agent):# extract snippets to get embeddingsunused_information_snippets = [info.snippets[0] for info in unused_information]# get embeddings- cache = knowledge_base.embedding_cache- unused_snippets_embeddings, _ = get_text_embeddings(- unused_information_snippets, embedding_cache=cache, max_workers=100- )- claim_embedding, _ = get_text_embeddings(- conv_turn.claim_to_make, embedding_cache=cache- )- query_embedding, _ = get_text_embeddings(- conv_turn.queries, embedding_cache=cache- )- cited_snippets_embedding, _ = get_text_embeddings(- cited_snippets, embedding_cache=cache+ unused_snippets_embeddings = self.encoder.encode(+ unused_information_snippets, max_workers=20)+ claim_embedding = self.encoder.encode(conv_turn.claim_to_make)+ query_embedding = self.encoder.encode(conv_turn.queries)+ cited_snippets_embedding = self.encoder.encode(cited_snippets)# calculate similarityquery_similarities = cosine_similarity(unused_snippets_embeddings, query_embedding@@ -270,8 +265,7 @@ class Moderator(Agent):)batch_snippets.append(conv_turn.claim_to_make)batch_snippets.extend(conv_turn.queries)- cache = knowledge_base.embedding_cache- get_text_embeddings(batch_snippets, embedding_cache=cache, max_workers=300)+ self.encoder.encode(batch_snippets, max_workers=20)# get sorted unused snippets for each turnsorted_snippets = []diff --git a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py b/knowledge_storm/collaborative_storm/modules/information_insertion_module.pyindex c858671..47daafb 100644--- a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py+++ b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py@@ -9,7 +9,7 @@ from typing import List, Union, Dict, Optionalfrom .collaborative_storm_utils import trim_output_after_hintfrom ...dataclass import KnowledgeNode, KnowledgeBase-from ...encoder import get_text_embeddings+from ...encoder import Encoderfrom ...interface import Information@@ -51,8 +51,9 @@ class InsertInformationCandidateChoice(dspy.Signature):class InsertInformationModule(dspy.Module):- def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel], encoder: Encoder):self.engine = engine+ self.encoder = encoderself.insert_info = dspy.ChainOfThought(InsertInformation)self.candidate_choosing = dspy.Predict(InsertInformationCandidateChoice)@@ -153,7 +154,7 @@ class InsertInformationModule(dspy.Module):query: str,):if encoded_outline is not None and encoded_outline.size > 0:- encoded_query, token_usage = get_text_embeddings(f"{question}, {query}")+ encoded_query = self.encoder.encode(f"{question}, {query}")sim = cosine_similarity([encoded_query], encoded_outline)[0]sorted_indices = np.argsort(sim)sorted_outlines = np.array(outlines)[sorted_indices[::-1]]@@ -226,7 +227,6 @@ class InsertInformationModule(dspy.Module):insert_root: Optional[KnowledgeNode] = None,skip_candidate_from_embedding: bool = False,):-if not isinstance(information, List):information = [information]intent_to_placement_dict: Dict = self._info_list_to_intent_mapping(diff --git a/knowledge_storm/dataclass.py b/knowledge_storm/dataclass.pyindex 1eff9a6..80cae47 100644--- a/knowledge_storm/dataclass.py+++ b/knowledge_storm/dataclass.py@@ -4,7 +4,7 @@ import reimport threadingfrom typing import Set, Dict, List, Optional, Union, Tuple-from .encoder import get_text_embeddings+from .encoder import Encoderfrom .interface import Information@@ -310,6 +310,7 @@ class KnowledgeBase:topic: str,knowledge_base_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],node_expansion_trigger_count: int,+ encoder: Encoder,):"""Initializes a KnowledgeBase instance.@@ -333,9 +334,10 @@ class KnowledgeBase:)self.topic: str = topic+ self.encoder: Encoder = encoderself.information_insert_module = InsertInformationModule(- engine=knowledge_base_lm+ engine=knowledge_base_lm, encoder=self.encoder)self.expand_node_module = ExpandNodeModule(engine=knowledge_base_lm,@@ -353,7 +355,6 @@ class KnowledgeBase:"encoded_structure": np.array([[]]),"structure_string": "",}- self.embedding_cache: Dict[str, np.ndarray] = {}self.info_uuid_to_info_dict: Dict[int, Information] = {}self.info_hash_to_uuid_dict: Dict[int, int] = {}self._lock = threading.Lock()@@ -375,11 +376,13 @@ class KnowledgeBase:data: Dict,knowledge_base_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],node_expansion_trigger_count: int,+ encoder: Encoder,):knowledge_base = cls(topic=data["topic"],knowledge_base_lm=knowledge_base_lm,node_expansion_trigger_count=node_expansion_trigger_count,+ encoder=encoder,)knowledge_base.root = KnowledgeNode.from_dict(data["tree"])knowledge_base.info_hash_to_uuid_dict = {@@ -408,9 +411,7 @@ class KnowledgeBase:cleaned_outline_strings = [outline.replace(" -> ", ", ") for outline in outline_strings]- encoded_outline, _ = get_text_embeddings(- cleaned_outline_strings, embedding_cache=self.embedding_cache- )+ encoded_outline = self.encoder.encode(cleaned_outline_strings)self.kb_embedding = {"hash": outline_string_hash,"encoded_structure": encoded_outline,@@ -545,7 +546,6 @@ class KnowledgeBase:cited_indices: Optional[List[int]] = None,root: Optional[KnowledgeNode] = None,) -> str:-def find_node_contain_index(node, index):"""Traverses the tree downward from the given node.diff --git a/knowledge_storm/encoder.py b/knowledge_storm/encoder.pyindex 01fc972..0283a70 100644--- a/knowledge_storm/encoder.py+++ b/knowledge_storm/encoder.py@@ -1,144 +1,158 @@-import requestsimport os-from typing import List, Tuple, Union, Optional, Dict, Literalimport numpy as npfrom concurrent.futures import ThreadPoolExecutor, as_completed---class EmbeddingModel:- def __init__(self):- pass-- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:- raise Exception("Not implemented")---class OpenAIEmbeddingModel(EmbeddingModel):- def __init__(self, model: str = "text-embedding-3-small", api_key: str = None):- if not api_key:- api_key = os.getenv("OPENAI_API_KEY")-- self.url = "https://api.openai.com/v1/embeddings"- self.headers = {- "Content-Type": "application/json",- "Authorization": f"Bearer {api_key}",- }- self.model = model-- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:- data = {"input": text, "model": self.model}-- response = requests.post(self.url, headers=self.headers, json=data)- if response.status_code == 200:- data = response.json()- embedding = np.array(data["data"][0]["embedding"])- token = data["usage"]["prompt_tokens"]- return embedding, token+from typing import List, Tuple, Union, Optional, Dict, Literal+from pathlib import Path++try:+ import warnings++ with warnings.catch_warnings():+ warnings.filterwarnings("ignore", category=UserWarning)+ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:+ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"+ import litellm++ litellm.drop_params = True+ litellm.telemetry = False++ from litellm.caching.caching import Cache++ disk_cache_dir = os.path.join(Path.home(), ".storm_local_cache")+ litellm.cache = Cache(disk_cache_dir=disk_cache_dir, type="disk")++except ImportError:++ class LitellmPlaceholder:+ def __getattr__(self, _):+ raise ImportError(+ "The LiteLLM package is not installed. Run `pip install litellm`."+ )++ litellm = LitellmPlaceholder()+++class Encoder:+ def __init__(+ self,+ encoder_type: Optional[str] = None,+ api_key: Optional[str] = None,+ api_base: Optional[str] = None,+ api_version: Optional[str] = None,+ ):+ """+ Initializes the Encoder with the appropriate embedding model.++ Args:+ encoder_type (Optional[str]): Type of encoder ('openai', 'azure', etc.).+ api_key (Optional[str]): API key for the encoder service.+ api_base (Optional[str]): API base URL for the encoder service.+ api_version (Optional[str]): API version for the encoder service.+ """+ self.embedding_model_name = None+ self.kargs = {}+ self.total_token_usage = 0++ # Initialize the appropriate embedding model+ encoder_type = encoder_type or os.getenv("ENCODER_API_TYPE")+ if not encoder_type:+ raise ValueError("ENCODER_API_TYPE environment variable is not set.")++ if encoder_type.lower() == "openai":+ self.embedding_model_name = "text-embedding-3-small"+ self.kargs = {"api_key": api_key or os.getenv("OPENAI_API_KEY")}+ elif encoder_type.lower() == "azure":+ self.embedding_model_name = "azure/text-embedding-3-small"+ self.kargs = {+ "api_key": api_key or os.getenv("AZURE_API_KEY"),+ "api_base": api_base or os.getenv("AZURE_API_BASE"),+ "api_version": api_version or os.getenv("AZURE_API_VERSION"),+ }else:- response.raise_for_status()---class TogetherEmbeddingModel:- def __init__(self, model: str = "BAAI/bge-large-en-v1.5", api_key: str = None):- import together-- self.model = model- if not api_key:- api_key = os.getenv("TOGETHER_API_KEY")- self.together_client = together.Together(api_key=api_key)-- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:- response = self.together_client.embeddings.create(input=text, model=self.model)- return response.data[0].embedding, -1---class AzureOpenAIEmbeddingModel:- def __init__(self, model: str = "text-embedding-3-small", api_key: str = None):- from openai import AzureOpenAI-- self.model = model- if not api_key:- api_key = os.getenv("AZURE_API_KEY")-- self.client = AzureOpenAI(- api_key=api_key,- api_version=os.getenv("AZURE_API_VERSION"),- azure_endpoint=os.getenv("AZURE_API_BASE"),+ raise ValueError(+ f"Unsupported ENCODER_API_TYPE '{encoder_type}'. Supported types are 'openai', 'azure', 'together'."+ )++ def get_total_token_usage(self, reset: bool = False) -> int:+ """+ Retrieves the total token usage.++ Args:+ reset (bool): If True, resets the total token usage counter after retrieval.++ Returns:+ int: The total number of tokens used.+ """+ token_usage = self.total_token_usage+ if reset:+ self.total_token_usage = 0+ return token_usage++ def encode(self, texts: Union[str, List[str]], max_workers: int = 5) -> np.ndarray:+ """+ Public method to get embeddings for the given texts.++ Args:+ texts (Union[str, List[str]]): A single text string or a list of text strings to embed.++ Returns:+ np.ndarray: The array of embeddings.+ """+ return self._get_text_embeddings(texts, max_workers=max_workers)++ def _get_single_text_embedding(self, text):+ response = litellm.embedding(+ model=self.embedding_model_name, input=text, caching=True, **self.kargs)-- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:- response = self.client.embeddings.create(input=text, model=self.model)-- embedding = np.array(response.data[0].embedding)- token = response.usage.prompt_tokens- return embedding, token---def get_text_embeddings(- texts: Union[str, List[str]],- max_workers: int = 5,- embedding_cache: Optional[Dict[str, np.ndarray]] = None,-) -> Tuple[np.ndarray, int]:- """- Get text embeddings using OpenAI's text-embedding-3-small model.-- Args:- texts (Union[str, List[str]]): A single text string or a list of text strings to embed.- max_workers (int): The maximum number of workers for parallel processing.- api_key (str): The API key for accessing OpenAI's services.- embedding_cache (Optional[Dict[str, np.ndarray]]): A cache to store previously computed embeddings.-- Returns:- Tuple[np.ndarray, int]: The 2D array of embeddings and the total token usage.- """- embedding_model = None- encoder_type = os.getenv("ENCODER_API_TYPE")- if encoder_type and encoder_type == "openai":- embedding_model = OpenAIEmbeddingModel()- elif encoder_type and encoder_type == "azure":- embedding_model = AzureOpenAIEmbeddingModel()- elif encoder_type == encoder_type == "together":- embedding_model = TogetherEmbeddingModel()- else:- raise Exception(- "No valid encoder type is provided. Check <repo root>/secrets.toml for the field ENCODER_API_TYPE"- )-- def fetch_embedding(text: str) -> Tuple[str, np.ndarray, int]:- if embedding_cache is not None and text in embedding_cache:- return (- text,- embedding_cache[text],- 0,- ) # Returning 0 tokens since no API call is made- embedding, token_usage = embedding_model.get_embedding(text)+ embedding = response.data[0]["embedding"]+ token_usage = response.get("usage", {}).get("total_tokens", 0)return text, embedding, token_usage- if isinstance(texts, str):- _, embedding, tokens = fetch_embedding(texts)- return np.array(embedding), tokens-- embeddings = []- total_tokens = 0-- with ThreadPoolExecutor(max_workers=max_workers) as executor:- futures = {executor.submit(fetch_embedding, text): text for text in texts}-- for future in as_completed(futures):- try:- text, embedding, tokens = future.result()- embeddings.append((text, embedding, tokens))- total_tokens += tokens- except Exception as e:- print(f"An error occurred for text: {futures[future]}")- print(e)-- # Sort results to match the order of the input texts- embeddings.sort(key=lambda x: texts.index(x[0]))- if embedding_cache is not None:- for text, embedding, _ in embeddings:- embedding_cache[text] = embedding- embeddings = [result[1] for result in embeddings]-- return np.array(embeddings), total_tokens+ def _get_text_embeddings(+ self,+ texts: Union[str, List[str]],+ max_workers: int = 5,+ ) -> Tuple[np.ndarray, int]:+ """+ Get text embeddings using OpenAI's text-embedding-3-small model.++ Args:+ texts (Union[str, List[str]]): A single text string or a list of text strings to embed.+ max_workers (int): The maximum number of workers for parallel processing.+ api_key (str): The API key for accessing OpenAI's services.+ embedding_cache (Optional[Dict[str, np.ndarray]]): A cache to store previously computed embeddings.++ Returns:+ Tuple[np.ndarray, int]: The 2D array of embeddings and the total token usage.+ """++ if isinstance(texts, str):+ _, embedding, tokens = self._get_single_text_embedding(texts)+ self.total_token_usage += tokens+ return np.array(embedding)++ embeddings = []+ total_tokens = 0++ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ futures = {+ executor.submit(self._get_single_text_embedding, text): text+ for text in texts+ }++ for future in as_completed(futures):+ try:+ text, embedding, tokens = future.result()+ embeddings.append((text, embedding, tokens))+ total_tokens += tokens+ except Exception as e:+ print(f"An error occurred for text: {futures[future]}")+ print(e)++ # Sort results to match the order of the input texts+ embeddings.sort(key=lambda x: texts.index(x[0]))+ embeddings = [result[1] for result in embeddings]+ self.total_token_usage += total_tokens++ return np.array(embeddings)diff --git a/knowledge_storm/storm_wiki/modules/storm_dataclass.py b/knowledge_storm/storm_wiki/modules/storm_dataclass.pyindex 75812d9..119869c 100644--- a/knowledge_storm/storm_wiki/modules/storm_dataclass.py+++ b/knowledge_storm/storm_wiki/modules/storm_dataclass.py@@ -114,9 +114,7 @@ class StormInformationTable(InformationTable):for snippet in information.snippets:self.collected_urls.append(url)self.collected_snippets.append(snippet)- self.encoded_snippets = self.encoder.encode(- self.collected_snippets, show_progress_bar=False- )+ self.encoded_snippets = self.encoder.encode(self.collected_snippets)def retrieve_information(self, queries: Union[List[str], str], search_top_k@@ -126,7 +124,7 @@ class StormInformationTable(InformationTable):if type(queries) is str:queries = [queries]for query in queries:- encoded_query = self.encoder.encode(query, show_progress_bar=False)+ encoded_query = self.encoder.encode(query)sim = cosine_similarity([encoded_query], self.encoded_snippets)[0]sorted_indices = np.argsort(sim)for i in sorted_indices[-search_top_k:][::-1]: