Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

encapsulate encoding function and Encoder class

Yucheng-Jiang0764f542b516
diff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.py
index 2e281f6..7977143 100644
--- a/knowledge_storm/collaborative_storm/engine.py
+++ b/knowledge_storm/collaborative_storm/engine.py
@@ -14,6 +14,7 @@ from .modules.co_storm_agents import (
from .modules.expert_generation import GenerateExpertModule
from .modules.warmstart_hierarchical_chat import WarmStartModule
from ..dataclass import ConversationTurn, KnowledgeBase
+from ..encoder import Encoder
from ..interface import LMConfigs, Agent
from ..logging_wrapper import LoggingWrapper
from ..lm import OpenAIModel, AzureOpenAIModel, TogetherClient
@@ -323,6 +324,7 @@ class DiscourseManager:
lm_config: CollaborativeStormLMConfigs,
runner_argument: RunnerArgument,
rm: dspy.Retrieve,
+ encoder: Encoder,
callback_handler: BaseCallbackHandler,
):
# parameter management
@@ -331,6 +333,7 @@ class DiscourseManager:
self.logging_wrapper = logging_wrapper
self.callback_handler = callback_handler
self.rm = rm
+ self.encoder = encoder
# role management
self.experts: List[CoStormExpert] = []
self.simulated_user: SimulatedUser = SimulatedUser(
@@ -360,6 +363,7 @@ class DiscourseManager:
lm_config=self.lm_config,
runner_argument=self.runner_argument,
logging_wrapper=self.logging_wrapper,
+ encoder=self.encoder,
callback_handler=self.callback_handler,
)
self.general_knowledge_provider = CoStormExpert(
@@ -516,18 +520,21 @@ class CoStormRunner:
self.rm = BingSearch(k=runner_argument.retrieve_top_k)
else:
self.rm = rm
+ self.encoder = Encoder()
self.conversation_history = []
self.warmstart_conv_archive = []
self.knowledge_base = KnowledgeBase(
topic=self.runner_argument.topic,
knowledge_base_lm=self.lm_config.knowledge_base_lm,
node_expansion_trigger_count=self.runner_argument.node_expansion_trigger_count,
+ encoder=self.encoder,
)
self.discourse_manager = DiscourseManager(
lm_config=self.lm_config,
runner_argument=self.runner_argument,
logging_wrapper=self.logging_wrapper,
rm=self.rm,
+ encoder=self.encoder,
callback_handler=callback_handler,
)
@@ -555,6 +562,7 @@ class CoStormRunner:
runner_argument=RunnerArgument.from_dict(data["runner_argument"]),
logging_wrapper=LoggingWrapper(lm_config),
)
+ costorm_runner.encoder = Encoder()
costorm_runner.conversation_history = [
ConversationTurn.from_dict(turn) for turn in data["conversation_history"]
]
@@ -567,6 +575,7 @@ class CoStormRunner:
data=data["knowledge_base"],
knowledge_base_lm=costorm_runner.lm_config.knowledge_base_lm,
node_expansion_trigger_count=costorm_runner.runner_argument.node_expansion_trigger_count,
+ encoder=costorm_runner.encoder,
)
return costorm_runner
diff --git a/knowledge_storm/collaborative_storm/modules/co_storm_agents.py b/knowledge_storm/collaborative_storm/modules/co_storm_agents.py
index e7f6029..dd42cb8 100644
--- a/knowledge_storm/collaborative_storm/modules/co_storm_agents.py
+++ b/knowledge_storm/collaborative_storm/modules/co_storm_agents.py
@@ -13,7 +13,7 @@ from .costorm_expert_utterance_generator import CoStormExpertUtteranceGeneration
from .grounded_question_generation import GroundedQuestionGenerationModule
from .simulate_user import GenSimulatedUserUtterance
from ...dataclass import ConversationTurn, KnowledgeBase
-from ...encoder import get_text_embeddings
+from ...encoder import Encoder
from ...interface import Agent, Information, LMConfigs
from ...logging_wrapper import LoggingWrapper
@@ -174,6 +174,7 @@ class Moderator(Agent):
lm_config: LMConfigs,
runner_argument: "RunnerArgument",
logging_wrapper: LoggingWrapper,
+ encoder: Encoder,
callback_handler: BaseCallbackHandler = None,
):
super().__init__(topic, role_name, role_description)
@@ -184,6 +185,7 @@ class Moderator(Agent):
engine=self.lm_config.question_asking_lm
)
self.callback_handler = callback_handler
+ self.encoder = encoder
def _get_conv_turn_unused_information(
self, conv_turn: ConversationTurn, knowledge_base: KnowledgeBase
@@ -211,19 +213,12 @@ class Moderator(Agent):
# extract snippets to get embeddings
unused_information_snippets = [info.snippets[0] for info in unused_information]
# get embeddings
- cache = knowledge_base.embedding_cache
- unused_snippets_embeddings, _ = get_text_embeddings(
- unused_information_snippets, embedding_cache=cache, max_workers=100
- )
- claim_embedding, _ = get_text_embeddings(
- conv_turn.claim_to_make, embedding_cache=cache
- )
- query_embedding, _ = get_text_embeddings(
- conv_turn.queries, embedding_cache=cache
- )
- cited_snippets_embedding, _ = get_text_embeddings(
- cited_snippets, embedding_cache=cache
+ unused_snippets_embeddings = self.encoder.encode(
+ unused_information_snippets, max_workers=20
)
+ claim_embedding = self.encoder.encode(conv_turn.claim_to_make)
+ query_embedding = self.encoder.encode(conv_turn.queries)
+ cited_snippets_embedding = self.encoder.encode(cited_snippets)
# calculate similarity
query_similarities = cosine_similarity(
unused_snippets_embeddings, query_embedding
@@ -270,8 +265,7 @@ class Moderator(Agent):
)
batch_snippets.append(conv_turn.claim_to_make)
batch_snippets.extend(conv_turn.queries)
- cache = knowledge_base.embedding_cache
- get_text_embeddings(batch_snippets, embedding_cache=cache, max_workers=300)
+ self.encoder.encode(batch_snippets, max_workers=20)
# get sorted unused snippets for each turn
sorted_snippets = []
diff --git a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py
index c858671..47daafb 100644
--- a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py
+++ b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py
@@ -9,7 +9,7 @@ from typing import List, Union, Dict, Optional
from .collaborative_storm_utils import trim_output_after_hint
from ...dataclass import KnowledgeNode, KnowledgeBase
-from ...encoder import get_text_embeddings
+from ...encoder import Encoder
from ...interface import Information
@@ -51,8 +51,9 @@ class InsertInformationCandidateChoice(dspy.Signature):
class InsertInformationModule(dspy.Module):
- def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):
+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel], encoder: Encoder):
self.engine = engine
+ self.encoder = encoder
self.insert_info = dspy.ChainOfThought(InsertInformation)
self.candidate_choosing = dspy.Predict(InsertInformationCandidateChoice)
@@ -153,7 +154,7 @@ class InsertInformationModule(dspy.Module):
query: str,
):
if encoded_outline is not None and encoded_outline.size > 0:
- encoded_query, token_usage = get_text_embeddings(f"{question}, {query}")
+ encoded_query = self.encoder.encode(f"{question}, {query}")
sim = cosine_similarity([encoded_query], encoded_outline)[0]
sorted_indices = np.argsort(sim)
sorted_outlines = np.array(outlines)[sorted_indices[::-1]]
@@ -226,7 +227,6 @@ class InsertInformationModule(dspy.Module):
insert_root: Optional[KnowledgeNode] = None,
skip_candidate_from_embedding: bool = False,
):
-
if not isinstance(information, List):
information = [information]
intent_to_placement_dict: Dict = self._info_list_to_intent_mapping(
diff --git a/knowledge_storm/dataclass.py b/knowledge_storm/dataclass.py
index 1eff9a6..80cae47 100644
--- a/knowledge_storm/dataclass.py
+++ b/knowledge_storm/dataclass.py
@@ -4,7 +4,7 @@ import re
import threading
from typing import Set, Dict, List, Optional, Union, Tuple
-from .encoder import get_text_embeddings
+from .encoder import Encoder
from .interface import Information
@@ -310,6 +310,7 @@ class KnowledgeBase:
topic: str,
knowledge_base_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],
node_expansion_trigger_count: int,
+ encoder: Encoder,
):
"""
Initializes a KnowledgeBase instance.
@@ -333,9 +334,10 @@ class KnowledgeBase:
)
self.topic: str = topic
+ self.encoder: Encoder = encoder
self.information_insert_module = InsertInformationModule(
- engine=knowledge_base_lm
+ engine=knowledge_base_lm, encoder=self.encoder
)
self.expand_node_module = ExpandNodeModule(
engine=knowledge_base_lm,
@@ -353,7 +355,6 @@ class KnowledgeBase:
"encoded_structure": np.array([[]]),
"structure_string": "",
}
- self.embedding_cache: Dict[str, np.ndarray] = {}
self.info_uuid_to_info_dict: Dict[int, Information] = {}
self.info_hash_to_uuid_dict: Dict[int, int] = {}
self._lock = threading.Lock()
@@ -375,11 +376,13 @@ class KnowledgeBase:
data: Dict,
knowledge_base_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],
node_expansion_trigger_count: int,
+ encoder: Encoder,
):
knowledge_base = cls(
topic=data["topic"],
knowledge_base_lm=knowledge_base_lm,
node_expansion_trigger_count=node_expansion_trigger_count,
+ encoder=encoder,
)
knowledge_base.root = KnowledgeNode.from_dict(data["tree"])
knowledge_base.info_hash_to_uuid_dict = {
@@ -408,9 +411,7 @@ class KnowledgeBase:
cleaned_outline_strings = [
outline.replace(" -> ", ", ") for outline in outline_strings
]
- encoded_outline, _ = get_text_embeddings(
- cleaned_outline_strings, embedding_cache=self.embedding_cache
- )
+ encoded_outline = self.encoder.encode(cleaned_outline_strings)
self.kb_embedding = {
"hash": outline_string_hash,
"encoded_structure": encoded_outline,
@@ -545,7 +546,6 @@ class KnowledgeBase:
cited_indices: Optional[List[int]] = None,
root: Optional[KnowledgeNode] = None,
) -> str:
-
def find_node_contain_index(node, index):
"""
Traverses the tree downward from the given node.
diff --git a/knowledge_storm/encoder.py b/knowledge_storm/encoder.py
index 01fc972..0283a70 100644
--- a/knowledge_storm/encoder.py
+++ b/knowledge_storm/encoder.py
@@ -1,144 +1,158 @@
-import requests
import os
-from typing import List, Tuple, Union, Optional, Dict, Literal
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
-
-
-class EmbeddingModel:
- def __init__(self):
- pass
-
- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:
- raise Exception("Not implemented")
-
-
-class OpenAIEmbeddingModel(EmbeddingModel):
- def __init__(self, model: str = "text-embedding-3-small", api_key: str = None):
- if not api_key:
- api_key = os.getenv("OPENAI_API_KEY")
-
- self.url = "https://api.openai.com/v1/embeddings"
- self.headers = {
- "Content-Type": "application/json",
- "Authorization": f"Bearer {api_key}",
- }
- self.model = model
-
- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:
- data = {"input": text, "model": self.model}
-
- response = requests.post(self.url, headers=self.headers, json=data)
- if response.status_code == 200:
- data = response.json()
- embedding = np.array(data["data"][0]["embedding"])
- token = data["usage"]["prompt_tokens"]
- return embedding, token
+from typing import List, Tuple, Union, Optional, Dict, Literal
+from pathlib import Path
+
+try:
+ import warnings
+
+ with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=UserWarning)
+ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
+ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
+ import litellm
+
+ litellm.drop_params = True
+ litellm.telemetry = False
+
+ from litellm.caching.caching import Cache
+
+ disk_cache_dir = os.path.join(Path.home(), ".storm_local_cache")
+ litellm.cache = Cache(disk_cache_dir=disk_cache_dir, type="disk")
+
+except ImportError:
+
+ class LitellmPlaceholder:
+ def __getattr__(self, _):
+ raise ImportError(
+ "The LiteLLM package is not installed. Run `pip install litellm`."
+ )
+
+ litellm = LitellmPlaceholder()
+
+
+class Encoder:
+ def __init__(
+ self,
+ encoder_type: Optional[str] = None,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
+ ):
+ """
+ Initializes the Encoder with the appropriate embedding model.
+
+ Args:
+ encoder_type (Optional[str]): Type of encoder ('openai', 'azure', etc.).
+ api_key (Optional[str]): API key for the encoder service.
+ api_base (Optional[str]): API base URL for the encoder service.
+ api_version (Optional[str]): API version for the encoder service.
+ """
+ self.embedding_model_name = None
+ self.kargs = {}
+ self.total_token_usage = 0
+
+ # Initialize the appropriate embedding model
+ encoder_type = encoder_type or os.getenv("ENCODER_API_TYPE")
+ if not encoder_type:
+ raise ValueError("ENCODER_API_TYPE environment variable is not set.")
+
+ if encoder_type.lower() == "openai":
+ self.embedding_model_name = "text-embedding-3-small"
+ self.kargs = {"api_key": api_key or os.getenv("OPENAI_API_KEY")}
+ elif encoder_type.lower() == "azure":
+ self.embedding_model_name = "azure/text-embedding-3-small"
+ self.kargs = {
+ "api_key": api_key or os.getenv("AZURE_API_KEY"),
+ "api_base": api_base or os.getenv("AZURE_API_BASE"),
+ "api_version": api_version or os.getenv("AZURE_API_VERSION"),
+ }
else:
- response.raise_for_status()
-
-
-class TogetherEmbeddingModel:
- def __init__(self, model: str = "BAAI/bge-large-en-v1.5", api_key: str = None):
- import together
-
- self.model = model
- if not api_key:
- api_key = os.getenv("TOGETHER_API_KEY")
- self.together_client = together.Together(api_key=api_key)
-
- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:
- response = self.together_client.embeddings.create(input=text, model=self.model)
- return response.data[0].embedding, -1
-
-
-class AzureOpenAIEmbeddingModel:
- def __init__(self, model: str = "text-embedding-3-small", api_key: str = None):
- from openai import AzureOpenAI
-
- self.model = model
- if not api_key:
- api_key = os.getenv("AZURE_API_KEY")
-
- self.client = AzureOpenAI(
- api_key=api_key,
- api_version=os.getenv("AZURE_API_VERSION"),
- azure_endpoint=os.getenv("AZURE_API_BASE"),
+ raise ValueError(
+ f"Unsupported ENCODER_API_TYPE '{encoder_type}'. Supported types are 'openai', 'azure', 'together'."
+ )
+
+ def get_total_token_usage(self, reset: bool = False) -> int:
+ """
+ Retrieves the total token usage.
+
+ Args:
+ reset (bool): If True, resets the total token usage counter after retrieval.
+
+ Returns:
+ int: The total number of tokens used.
+ """
+ token_usage = self.total_token_usage
+ if reset:
+ self.total_token_usage = 0
+ return token_usage
+
+ def encode(self, texts: Union[str, List[str]], max_workers: int = 5) -> np.ndarray:
+ """
+ Public method to get embeddings for the given texts.
+
+ Args:
+ texts (Union[str, List[str]]): A single text string or a list of text strings to embed.
+
+ Returns:
+ np.ndarray: The array of embeddings.
+ """
+ return self._get_text_embeddings(texts, max_workers=max_workers)
+
+ def _get_single_text_embedding(self, text):
+ response = litellm.embedding(
+ model=self.embedding_model_name, input=text, caching=True, **self.kargs
)
-
- def get_embedding(self, text: str) -> Tuple[np.ndarray, int]:
- response = self.client.embeddings.create(input=text, model=self.model)
-
- embedding = np.array(response.data[0].embedding)
- token = response.usage.prompt_tokens
- return embedding, token
-
-
-def get_text_embeddings(
- texts: Union[str, List[str]],
- max_workers: int = 5,
- embedding_cache: Optional[Dict[str, np.ndarray]] = None,
-) -> Tuple[np.ndarray, int]:
- """
- Get text embeddings using OpenAI's text-embedding-3-small model.
-
- Args:
- texts (Union[str, List[str]]): A single text string or a list of text strings to embed.
- max_workers (int): The maximum number of workers for parallel processing.
- api_key (str): The API key for accessing OpenAI's services.
- embedding_cache (Optional[Dict[str, np.ndarray]]): A cache to store previously computed embeddings.
-
- Returns:
- Tuple[np.ndarray, int]: The 2D array of embeddings and the total token usage.
- """
- embedding_model = None
- encoder_type = os.getenv("ENCODER_API_TYPE")
- if encoder_type and encoder_type == "openai":
- embedding_model = OpenAIEmbeddingModel()
- elif encoder_type and encoder_type == "azure":
- embedding_model = AzureOpenAIEmbeddingModel()
- elif encoder_type == encoder_type == "together":
- embedding_model = TogetherEmbeddingModel()
- else:
- raise Exception(
- "No valid encoder type is provided. Check <repo root>/secrets.toml for the field ENCODER_API_TYPE"
- )
-
- def fetch_embedding(text: str) -> Tuple[str, np.ndarray, int]:
- if embedding_cache is not None and text in embedding_cache:
- return (
- text,
- embedding_cache[text],
- 0,
- ) # Returning 0 tokens since no API call is made
- embedding, token_usage = embedding_model.get_embedding(text)
+ embedding = response.data[0]["embedding"]
+ token_usage = response.get("usage", {}).get("total_tokens", 0)
return text, embedding, token_usage
- if isinstance(texts, str):
- _, embedding, tokens = fetch_embedding(texts)
- return np.array(embedding), tokens
-
- embeddings = []
- total_tokens = 0
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- futures = {executor.submit(fetch_embedding, text): text for text in texts}
-
- for future in as_completed(futures):
- try:
- text, embedding, tokens = future.result()
- embeddings.append((text, embedding, tokens))
- total_tokens += tokens
- except Exception as e:
- print(f"An error occurred for text: {futures[future]}")
- print(e)
-
- # Sort results to match the order of the input texts
- embeddings.sort(key=lambda x: texts.index(x[0]))
- if embedding_cache is not None:
- for text, embedding, _ in embeddings:
- embedding_cache[text] = embedding
- embeddings = [result[1] for result in embeddings]
-
- return np.array(embeddings), total_tokens
+ def _get_text_embeddings(
+ self,
+ texts: Union[str, List[str]],
+ max_workers: int = 5,
+ ) -> Tuple[np.ndarray, int]:
+ """
+ Get text embeddings using OpenAI's text-embedding-3-small model.
+
+ Args:
+ texts (Union[str, List[str]]): A single text string or a list of text strings to embed.
+ max_workers (int): The maximum number of workers for parallel processing.
+ api_key (str): The API key for accessing OpenAI's services.
+ embedding_cache (Optional[Dict[str, np.ndarray]]): A cache to store previously computed embeddings.
+
+ Returns:
+ Tuple[np.ndarray, int]: The 2D array of embeddings and the total token usage.
+ """
+
+ if isinstance(texts, str):
+ _, embedding, tokens = self._get_single_text_embedding(texts)
+ self.total_token_usage += tokens
+ return np.array(embedding)
+
+ embeddings = []
+ total_tokens = 0
+
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ futures = {
+ executor.submit(self._get_single_text_embedding, text): text
+ for text in texts
+ }
+
+ for future in as_completed(futures):
+ try:
+ text, embedding, tokens = future.result()
+ embeddings.append((text, embedding, tokens))
+ total_tokens += tokens
+ except Exception as e:
+ print(f"An error occurred for text: {futures[future]}")
+ print(e)
+
+ # Sort results to match the order of the input texts
+ embeddings.sort(key=lambda x: texts.index(x[0]))
+ embeddings = [result[1] for result in embeddings]
+ self.total_token_usage += total_tokens
+
+ return np.array(embeddings)
diff --git a/knowledge_storm/storm_wiki/modules/storm_dataclass.py b/knowledge_storm/storm_wiki/modules/storm_dataclass.py
index 75812d9..119869c 100644
--- a/knowledge_storm/storm_wiki/modules/storm_dataclass.py
+++ b/knowledge_storm/storm_wiki/modules/storm_dataclass.py
@@ -114,9 +114,7 @@ class StormInformationTable(InformationTable):
for snippet in information.snippets:
self.collected_urls.append(url)
self.collected_snippets.append(snippet)
- self.encoded_snippets = self.encoder.encode(
- self.collected_snippets, show_progress_bar=False
- )
+ self.encoded_snippets = self.encoder.encode(self.collected_snippets)
def retrieve_information(
self, queries: Union[List[str], str], search_top_k
@@ -126,7 +124,7 @@ class StormInformationTable(InformationTable):
if type(queries) is str:
queries = [queries]
for query in queries:
- encoded_query = self.encoder.encode(query, show_progress_bar=False)
+ encoded_query = self.encoder.encode(query)
sim = cosine_similarity([encoded_query], self.encoded_snippets)[0]
sorted_indices = np.argsort(sim)
for i in sorted_indices[-search_top_k:][::-1]: