An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
integrate Litellm into lm interface
Yucheng-Jiang46b472b979d0
diff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.pyindex 7977143..51b6374 100644--- a/knowledge_storm/collaborative_storm/engine.py+++ b/knowledge_storm/collaborative_storm/engine.py@@ -17,7 +17,7 @@ from ..dataclass import ConversationTurn, KnowledgeBasefrom ..encoder import Encoderfrom ..interface import LMConfigs, Agentfrom ..logging_wrapper import LoggingWrapper-from ..lm import OpenAIModel, AzureOpenAIModel, TogetherClient+from ..lm import LitellmModelfrom ..rm import BingSearch@@ -46,27 +46,26 @@ class CollaborativeStormLMConfigs(LMConfigs):if lm_type and lm_type == "openai":openai_kwargs = {"api_key": os.getenv("OPENAI_API_KEY"),- "api_provider": "openai","temperature": temperature,"top_p": top_p,"api_base": None,}- self.question_answering_lm = OpenAIModel(+ self.question_answering_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=1000, **openai_kwargs)- self.discourse_manage_lm = OpenAIModel(+ self.discourse_manage_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=500, **openai_kwargs)- self.utterance_polishing_lm = OpenAIModel(+ self.utterance_polishing_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=2000, **openai_kwargs)- self.warmstart_outline_gen_lm = OpenAIModel(+ self.warmstart_outline_gen_lm = LitellmModel(model="gpt-4-1106-preview", max_tokens=500, **openai_kwargs)- self.question_asking_lm = OpenAIModel(+ self.question_asking_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=300, **openai_kwargs)- self.knowledge_base_lm = OpenAIModel(+ self.knowledge_base_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=1000, **openai_kwargs)elif lm_type and lm_type == "azure":@@ -77,23 +76,23 @@ class CollaborativeStormLMConfigs(LMConfigs):"api_base": os.getenv("AZURE_API_BASE"),"api_version": os.getenv("AZURE_API_VERSION"),}- self.question_answering_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"+ self.question_answering_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat")- self.discourse_manage_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=500, **azure_kwargs, model_type="chat"+ self.discourse_manage_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=500, **azure_kwargs, model_type="chat")- self.utterance_polishing_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=2000, **azure_kwargs, model_type="chat"+ self.utterance_polishing_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=2000, **azure_kwargs, model_type="chat")- self.warmstart_outline_gen_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"+ self.warmstart_outline_gen_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat")- self.question_asking_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"+ self.question_asking_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat")- self.knowledge_base_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"+ self.knowledge_base_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat")elif lm_type and lm_type == "together":together_kwargs = {@@ -101,38 +100,38 @@ class CollaborativeStormLMConfigs(LMConfigs):"temperature": temperature,"top_p": top_p,}- self.question_answering_lm = TogetherClient(- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ self.question_answering_lm = LitellmModel(+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",max_tokens=1000,model_type="chat",**together_kwargs,)- self.discourse_manage_lm = TogetherClient(- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ self.discourse_manage_lm = LitellmModel(+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",max_tokens=500,model_type="chat",**together_kwargs,)- self.utterance_polishing_lm = TogetherClient(- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ self.utterance_polishing_lm = LitellmModel(+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",max_tokens=2000,model_type="chat",**together_kwargs,)- self.warmstart_outline_gen_lm = TogetherClient(- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ self.warmstart_outline_gen_lm = LitellmModel(+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",max_tokens=500,model_type="chat",**together_kwargs,)- self.question_asking_lm = TogetherClient(- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ self.question_asking_lm = LitellmModel(+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",max_tokens=300,model_type="chat",**together_kwargs,)- self.knowledge_base_lm = TogetherClient(- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ self.knowledge_base_lm = LitellmModel(+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",max_tokens=1000,model_type="chat",**together_kwargs,diff --git a/knowledge_storm/lm.py b/knowledge_storm/lm.pyindex 424bf75..6032589 100644--- a/knowledge_storm/lm.py+++ b/knowledge_storm/lm.py@@ -1,12 +1,16 @@+import backoff+import dspy+import functoolsimport loggingimport osimport random+import requestsimport threadingfrom typing import Optional, Literal, Any+import ujson+from pathlib import Path+-import backoff-import dspy-import requestsfrom dsp import ERRORS, backoff_hdlr, giveup_hdlrfrom dsp.modules.hf import openai_to_hffrom dsp.modules.hf_client import send_hftgi_request_v01_wrapped@@ -18,7 +22,249 @@ try:except ImportError:RateLimitError = None+############################+# Code copied from https://github.com/stanfordnlp/dspy/blob/main/dspy/clients/lm.py on Sep 29, 2024++# try:+import warnings++with warnings.catch_warnings():+ warnings.filterwarnings("ignore", category=UserWarning)+ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:+ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"+ import litellm++ litellm.drop_params = True+ litellm.telemetry = False++from litellm.caching.caching import Cache++disk_cache_dir = os.path.join(Path.home(), ".storm_local_cache")+litellm.cache = Cache(disk_cache_dir=disk_cache_dir, type="disk")++# except ImportError:++# class LitellmPlaceholder:+# def __getattr__(self, _):+# raise ImportError(+# "The LiteLLM package is not installed. Run `pip install litellm`."+# )++# litellm = LitellmPlaceholder()+LM_LRU_CACHE_MAX_SIZE = 3000+++class LM:+ def __init__(+ self,+ model,+ model_type="chat",+ temperature=0.0,+ max_tokens=1000,+ cache=True,+ **kwargs,+ ):+ self.model = model+ self.model_type = model_type+ self.cache = cache+ self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)+ self.history = []++ if "o1-" in model:+ assert (+ max_tokens >= 5000 and temperature == 1.0+ ), "OpenAI's o1-* models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"++ def __call__(self, prompt=None, messages=None, **kwargs):+ # Build the request.+ cache = kwargs.pop("cache", self.cache)+ messages = messages or [{"role": "user", "content": prompt}]+ kwargs = {**self.kwargs, **kwargs}++ # Make the request and handle LRU & disk caching.+ if self.model_type == "chat":+ completion = cached_litellm_completion if cache else litellm_completion+ else:+ completion = (+ cached_litellm_text_completion if cache else litellm_text_completion+ )++ response = completion(+ ujson.dumps(dict(model=self.model, messages=messages, **kwargs))+ )+ outputs = [+ c.message.content if hasattr(c, "message") else c["text"]+ for c in response["choices"]+ ]++ # Logging, with removed api key & where `cost` is None on cache hit.+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}+ entry = dict(prompt=prompt, messages=messages, kwargs=kwargs, response=response)+ entry = dict(**entry, outputs=outputs, usage=dict(response["usage"]))+ entry = dict(+ **entry, cost=response.get("_hidden_params", {}).get("response_cost")+ )+ self.history.append(entry)++ return outputs++ def inspect_history(self, n: int = 1):+ _inspect_history(self, n)++[email protected]_cache(maxsize=LM_LRU_CACHE_MAX_SIZE)+def cached_litellm_completion(request):+ return litellm_completion(request, cache={"no-cache": False, "no-store": False})+++def litellm_completion(request, cache={"no-cache": True, "no-store": True}):+ kwargs = ujson.loads(request)+ return litellm.completion(cache=cache, **kwargs)++[email protected]_cache(maxsize=LM_LRU_CACHE_MAX_SIZE)+def cached_litellm_text_completion(request):+ return litellm_text_completion(+ request, cache={"no-cache": False, "no-store": False}+ )+++def litellm_text_completion(request, cache={"no-cache": True, "no-store": True}):+ kwargs = ujson.loads(request)++ # Extract the provider and model from the model string.+ model = kwargs.pop("model").split("/", 1)+ provider, model = model[0] if len(model) > 1 else "openai", model[-1]++ # Use the API key and base from the kwargs, or from the environment.+ api_key = kwargs.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")+ api_base = kwargs.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")++ # Build the prompt from the messages.+ prompt = "\n\n".join(+ [x["content"] for x in kwargs.pop("messages")] + ["BEGIN RESPONSE:"]+ )++ return litellm.text_completion(+ cache=cache,+ model=f"text-completion-openai/{model}",+ api_key=api_key,+ api_base=api_base,+ prompt=prompt,+ **kwargs,+ )+++def _green(text: str, end: str = "\n"):+ return "\x1b[32m" + str(text).lstrip() + "\x1b[0m" + end+++def _red(text: str, end: str = "\n"):+ return "\x1b[31m" + str(text) + "\x1b[0m" + end+++def _inspect_history(lm, n: int = 1):+ """Prints the last n prompts and their completions."""++ for item in lm.history[-n:]:+ messages = item["messages"] or [{"role": "user", "content": item["prompt"]}]+ outputs = item["outputs"]++ print("\n\n\n")+ for msg in messages:+ print(_red(f"{msg['role'].capitalize()} message:"))+ print(msg["content"].strip())+ print("\n")++ print(_red("Response:"))+ print(_green(outputs[0].strip()))++ if len(outputs) > 1:+ choices_text = f" \t (and {len(outputs)-1} other completions)"+ print(_red(choices_text, end=""))++ print("\n\n\n")+############################+++class LitellmModel(LM):+ """A wrapper class for dspy.OpenAI."""++ def __init__(+ self,+ model: str = "openai/gpt-4o-mini",+ api_key: Optional[str] = None,+ model_type: Literal["chat", "text"] = "chat",+ **kwargs,+ ):+ super().__init__(model=model, api_key=api_key, model_type=model_type, **kwargs)+ self._token_usage_lock = threading.Lock()+ self.prompt_tokens = 0+ self.completion_tokens = 0++ def log_usage(self, response):+ """Log the total tokens from the OpenAI API response."""+ usage_data = response.get("usage")+ if usage_data:+ with self._token_usage_lock:+ self.prompt_tokens += usage_data.get("prompt_tokens", 0)+ self.completion_tokens += usage_data.get("completion_tokens", 0)++ def get_usage_and_reset(self):+ """Get the total tokens used and reset the token usage."""+ usage = {+ self.model+ or self.kwargs.get("model")+ or self.kwargs.get("engine"): {+ "prompt_tokens": self.prompt_tokens,+ "completion_tokens": self.completion_tokens,+ }+ }+ self.prompt_tokens = 0+ self.completion_tokens = 0++ return usage++ def __call__(self, prompt=None, messages=None, **kwargs):+ # Build the request.+ cache = kwargs.pop("cache", self.cache)+ messages = messages or [{"role": "user", "content": prompt}]+ kwargs = {**self.kwargs, **kwargs}++ # Make the request and handle LRU & disk caching.+ if self.model_type == "chat":+ completion = cached_litellm_completion if cache else litellm_completion+ else:+ completion = (+ cached_litellm_text_completion if cache else litellm_text_completion+ )++ response = completion(+ ujson.dumps(dict(model=self.model, messages=messages, **kwargs))+ )+ response_dict = response.json()+ self.log_usage(response_dict)+ outputs = [+ c.message.content if hasattr(c, "message") else c["text"]+ for c in response["choices"]+ ]++ # Logging, with removed api key & where `cost` is None on cache hit.+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}+ entry = dict(+ prompt=prompt, messages=messages, kwargs=kwargs, response=response_dict+ )+ entry = dict(**entry, outputs=outputs, usage=dict(response_dict["usage"]))+ entry = dict(+ **entry, cost=response.get("_hidden_params", {}).get("response_cost")+ )+ self.history.append(entry)++ return outputs+# ========================================================================+# following LM models are deprecated.+# Remained in this file for reproduction prorposeclass OpenAIModel(dspy.OpenAI):"""A wrapper class for dspy.OpenAI."""@@ -1027,3 +1273,4 @@ class GoogleModel(dspy.dsp.modules.lm.LM):completions.append(response.parts[0].text)return completions+# ========================================================================diff --git a/knowledge_storm/storm_wiki/engine.py b/knowledge_storm/storm_wiki/engine.pyindex 9a94acf..1cc9629 100644--- a/knowledge_storm/storm_wiki/engine.py+++ b/knowledge_storm/storm_wiki/engine.py@@ -14,7 +14,7 @@ from .modules.outline_generation import StormOutlineGenerationModulefrom .modules.persona_generator import StormPersonaGeneratorfrom .modules.storm_dataclass import StormInformationTable, StormArticlefrom ..interface import Engine, LMConfigs, Retriever-from ..lm import OpenAIModel, AzureOpenAIModel+from ..lm import LitellmModelfrom ..utils import FileIOHelper, makeStringRed, truncate_filename@@ -56,50 +56,49 @@ class STORMWikiLMConfigs(LMConfigs):openai_kwargs = {"api_key": openai_api_key,- "api_provider": "openai","temperature": temperature,"top_p": top_p,"api_base": None,}if openai_type and openai_type == "openai":- self.conv_simulator_lm = OpenAIModel(+ self.conv_simulator_lm = LitellmModel(model="gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs)- self.question_asker_lm = OpenAIModel(+ self.question_asker_lm = LitellmModel(model="gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs)# 1/12/2024: Update gpt-4 to gpt-4-1106-preview. (Currently keep the original setup when using azure.)- self.outline_gen_lm = OpenAIModel(+ self.outline_gen_lm = LitellmModel(model="gpt-4-0125-preview", max_tokens=400, **openai_kwargs)- self.article_gen_lm = OpenAIModel(+ self.article_gen_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=700, **openai_kwargs)- self.article_polish_lm = OpenAIModel(+ self.article_polish_lm = LitellmModel(model="gpt-4o-2024-05-13", max_tokens=4000, **openai_kwargs)elif openai_type and openai_type == "azure":- self.conv_simulator_lm = OpenAIModel(- model="gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs+ self.conv_simulator_lm = LitellmModel(+ model="azure/gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs)- self.question_asker_lm = AzureOpenAIModel(- model="gpt-4o-mini-2024-07-18",+ self.question_asker_lm = LitellmModel(+ model="azure/gpt-4o-mini-2024-07-18",max_tokens=500,**azure_kwargs,model_type="chat",)# use combination of openai and azure-openai as azure-openai does not support gpt-4 in standard deployment- self.outline_gen_lm = AzureOpenAIModel(- model="gpt-4o", max_tokens=400, **azure_kwargs, model_type="chat"+ self.outline_gen_lm = LitellmModel(+ model="azure/gpt-4o", max_tokens=400, **azure_kwargs, model_type="chat")- self.article_gen_lm = AzureOpenAIModel(- model="gpt-4o-mini-2024-07-18",+ self.article_gen_lm = LitellmModel(+ model="azure/gpt-4o-mini-2024-07-18",max_tokens=700,**azure_kwargs,model_type="chat",)- self.article_polish_lm = AzureOpenAIModel(- model="gpt-4o-mini-2024-07-18",+ self.article_polish_lm = LitellmModel(+ model="azure/gpt-4o-mini-2024-07-18",max_tokens=4000,**azure_kwargs,model_type="chat",diff --git a/knowledge_storm/utils.py b/knowledge_storm/utils.pyindex 2e3cbb6..4411b05 100644--- a/knowledge_storm/utils.py+++ b/knowledge_storm/utils.py@@ -1,4 +1,6 @@import concurrent.futures+import dspy+import httpximport jsonimport loggingimport os@@ -6,21 +8,14 @@ import pickleimport reimport regeximport sys-import time+import tomlfrom typing import List, Dict+from tqdm import tqdm-import httpx-import pandas as pd-import toml-from langchain_core.documents import Document-from langchain_huggingface import HuggingFaceEmbeddings-from langchain_qdrant import Qdrantfrom langchain_text_splitters import RecursiveCharacterTextSplitter-from qdrant_client import QdrantClient, models-from tqdm import tqdmfrom trafilatura import extract-from .lm import OpenAIModel+from .lm import LitellmModellogging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.@@ -72,8 +67,11 @@ class QdrantVectorStoreManager:@staticmethoddef _check_create_collection(- client: QdrantClient, collection_name: str, model: HuggingFaceEmbeddings+ client: "QdrantClient", collection_name: str, model: "HuggingFaceEmbeddings"):+ from langchain_qdrant import Qdrant+ from qdrant_client import models+"""Check if the Qdrant collection exists and create it if it does not."""if client is None:raise ValueError("Qdrant client is not initialized.")@@ -103,8 +101,10 @@ class QdrantVectorStoreManager:@staticmethoddef _init_online_vector_db(- url: str, api_key: str, collection_name: str, model: HuggingFaceEmbeddings+ url: str, api_key: str, collection_name: str, model: "HuggingFaceEmbeddings"):+ from qdrant_client import QdrantClient+"""Initialize the Qdrant client that is connected to an online vector store with the given URL and API key.Args:@@ -128,8 +128,10 @@ class QdrantVectorStoreManager:@staticmethoddef _init_offline_vector_db(- vector_store_path: str, collection_name: str, model: HuggingFaceEmbeddings+ vector_store_path: str, collection_name: str, model: "HuggingFaceEmbeddings"):+ from qdrant_client import QdrantClient+"""Initialize the Qdrant client that is connected to an offline vector store with the given vector store folder path.Args:@@ -164,6 +166,8 @@ class QdrantVectorStoreManager:embedding_model: str = "BAAI/bge-m3",device: str = "mps",):+ from qdrant_client import Document+"""Takes a CSV file and adds each row in the CSV file to the Qdrant collection.@@ -192,6 +196,8 @@ class QdrantVectorStoreManager:model_kwargs = {"device": device}encode_kwargs = {"normalize_embeddings": True}+ from langchain_huggingface import HuggingFaceEmbeddings+model = HuggingFaceEmbeddings(model_name=embedding_model,model_kwargs=model_kwargs,@@ -231,6 +237,8 @@ class QdrantVectorStoreManager:raise ValueError("Qdrant client is not initialized.")# read the csv file+ import pandas as pd+df = pd.read_csv(file_path)# check that content column exists and url column existsif content_column not in df.columns:@@ -704,10 +712,8 @@ class WebPageHelper:def user_input_appropriateness_check(user_input):- my_openai_model = OpenAIModel(- api_key=os.getenv("OPENAI_API_KEY"),- api_provider="openai",- model="gpt-4o-mini-2024-07-18",+ my_openai_model = LitellmModel(+ model="azure/gpt-4o-mini",max_tokens=10,temperature=0.0,top_p=0.9,@@ -761,10 +767,8 @@ User input: {user_input}"""def purpose_appropriateness_check(user_input):- my_openai_model = OpenAIModel(- api_key=os.getenv("OPENAI_API_KEY"),- api_provider="openai",- model="gpt-4o-mini-2024-07-18",+ my_openai_model = LitellmModel(+ model="azure/gpt-4o-mini",max_tokens=10,temperature=0.0,top_p=0.9,