Skip to content

armgit / storm

Public Imported from GitHub
0

An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.

integrate Litellm into lm interface

Yucheng-Jiang46b472b979d0
diff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.py
index 7977143..51b6374 100644
--- a/knowledge_storm/collaborative_storm/engine.py
+++ b/knowledge_storm/collaborative_storm/engine.py
@@ -17,7 +17,7 @@ from ..dataclass import ConversationTurn, KnowledgeBase
from ..encoder import Encoder
from ..interface import LMConfigs, Agent
from ..logging_wrapper import LoggingWrapper
-from ..lm import OpenAIModel, AzureOpenAIModel, TogetherClient
+from ..lm import LitellmModel
from ..rm import BingSearch
@@ -46,27 +46,26 @@ class CollaborativeStormLMConfigs(LMConfigs):
if lm_type and lm_type == "openai":
openai_kwargs = {
"api_key": os.getenv("OPENAI_API_KEY"),
- "api_provider": "openai",
"temperature": temperature,
"top_p": top_p,
"api_base": None,
}
- self.question_answering_lm = OpenAIModel(
+ self.question_answering_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=1000, **openai_kwargs
)
- self.discourse_manage_lm = OpenAIModel(
+ self.discourse_manage_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=500, **openai_kwargs
)
- self.utterance_polishing_lm = OpenAIModel(
+ self.utterance_polishing_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=2000, **openai_kwargs
)
- self.warmstart_outline_gen_lm = OpenAIModel(
+ self.warmstart_outline_gen_lm = LitellmModel(
model="gpt-4-1106-preview", max_tokens=500, **openai_kwargs
)
- self.question_asking_lm = OpenAIModel(
+ self.question_asking_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=300, **openai_kwargs
)
- self.knowledge_base_lm = OpenAIModel(
+ self.knowledge_base_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=1000, **openai_kwargs
)
elif lm_type and lm_type == "azure":
@@ -77,23 +76,23 @@ class CollaborativeStormLMConfigs(LMConfigs):
"api_base": os.getenv("AZURE_API_BASE"),
"api_version": os.getenv("AZURE_API_VERSION"),
}
- self.question_answering_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"
+ self.question_answering_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"
)
- self.discourse_manage_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=500, **azure_kwargs, model_type="chat"
+ self.discourse_manage_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=500, **azure_kwargs, model_type="chat"
)
- self.utterance_polishing_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=2000, **azure_kwargs, model_type="chat"
+ self.utterance_polishing_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=2000, **azure_kwargs, model_type="chat"
)
- self.warmstart_outline_gen_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"
+ self.warmstart_outline_gen_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"
)
- self.question_asking_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"
+ self.question_asking_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"
)
- self.knowledge_base_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"
+ self.knowledge_base_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"
)
elif lm_type and lm_type == "together":
together_kwargs = {
@@ -101,38 +100,38 @@ class CollaborativeStormLMConfigs(LMConfigs):
"temperature": temperature,
"top_p": top_p,
}
- self.question_answering_lm = TogetherClient(
- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+ self.question_answering_lm = LitellmModel(
+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
max_tokens=1000,
model_type="chat",
**together_kwargs,
)
- self.discourse_manage_lm = TogetherClient(
- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+ self.discourse_manage_lm = LitellmModel(
+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
max_tokens=500,
model_type="chat",
**together_kwargs,
)
- self.utterance_polishing_lm = TogetherClient(
- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+ self.utterance_polishing_lm = LitellmModel(
+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
max_tokens=2000,
model_type="chat",
**together_kwargs,
)
- self.warmstart_outline_gen_lm = TogetherClient(
- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+ self.warmstart_outline_gen_lm = LitellmModel(
+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
max_tokens=500,
model_type="chat",
**together_kwargs,
)
- self.question_asking_lm = TogetherClient(
- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+ self.question_asking_lm = LitellmModel(
+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
max_tokens=300,
model_type="chat",
**together_kwargs,
)
- self.knowledge_base_lm = TogetherClient(
- model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+ self.knowledge_base_lm = LitellmModel(
+ model="together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
max_tokens=1000,
model_type="chat",
**together_kwargs,
diff --git a/knowledge_storm/lm.py b/knowledge_storm/lm.py
index 424bf75..6032589 100644
--- a/knowledge_storm/lm.py
+++ b/knowledge_storm/lm.py
@@ -1,12 +1,16 @@
+import backoff
+import dspy
+import functools
import logging
import os
import random
+import requests
import threading
from typing import Optional, Literal, Any
+import ujson
+from pathlib import Path
+
-import backoff
-import dspy
-import requests
from dsp import ERRORS, backoff_hdlr, giveup_hdlr
from dsp.modules.hf import openai_to_hf
from dsp.modules.hf_client import send_hftgi_request_v01_wrapped
@@ -18,7 +22,249 @@ try:
except ImportError:
RateLimitError = None
+############################
+# Code copied from https://github.com/stanfordnlp/dspy/blob/main/dspy/clients/lm.py on Sep 29, 2024
+
+# try:
+import warnings
+
+with warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=UserWarning)
+ if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
+ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
+ import litellm
+
+ litellm.drop_params = True
+ litellm.telemetry = False
+
+from litellm.caching.caching import Cache
+
+disk_cache_dir = os.path.join(Path.home(), ".storm_local_cache")
+litellm.cache = Cache(disk_cache_dir=disk_cache_dir, type="disk")
+
+# except ImportError:
+
+# class LitellmPlaceholder:
+# def __getattr__(self, _):
+# raise ImportError(
+# "The LiteLLM package is not installed. Run `pip install litellm`."
+# )
+
+# litellm = LitellmPlaceholder()
+LM_LRU_CACHE_MAX_SIZE = 3000
+
+
+class LM:
+ def __init__(
+ self,
+ model,
+ model_type="chat",
+ temperature=0.0,
+ max_tokens=1000,
+ cache=True,
+ **kwargs,
+ ):
+ self.model = model
+ self.model_type = model_type
+ self.cache = cache
+ self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
+ self.history = []
+
+ if "o1-" in model:
+ assert (
+ max_tokens >= 5000 and temperature == 1.0
+ ), "OpenAI's o1-* models require passing temperature=1.0 and max_tokens >= 5000 to `dspy.LM(...)`"
+
+ def __call__(self, prompt=None, messages=None, **kwargs):
+ # Build the request.
+ cache = kwargs.pop("cache", self.cache)
+ messages = messages or [{"role": "user", "content": prompt}]
+ kwargs = {**self.kwargs, **kwargs}
+
+ # Make the request and handle LRU & disk caching.
+ if self.model_type == "chat":
+ completion = cached_litellm_completion if cache else litellm_completion
+ else:
+ completion = (
+ cached_litellm_text_completion if cache else litellm_text_completion
+ )
+
+ response = completion(
+ ujson.dumps(dict(model=self.model, messages=messages, **kwargs))
+ )
+ outputs = [
+ c.message.content if hasattr(c, "message") else c["text"]
+ for c in response["choices"]
+ ]
+
+ # Logging, with removed api key & where `cost` is None on cache hit.
+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}
+ entry = dict(prompt=prompt, messages=messages, kwargs=kwargs, response=response)
+ entry = dict(**entry, outputs=outputs, usage=dict(response["usage"]))
+ entry = dict(
+ **entry, cost=response.get("_hidden_params", {}).get("response_cost")
+ )
+ self.history.append(entry)
+
+ return outputs
+
+ def inspect_history(self, n: int = 1):
+ _inspect_history(self, n)
+
+
[email protected]_cache(maxsize=LM_LRU_CACHE_MAX_SIZE)
+def cached_litellm_completion(request):
+ return litellm_completion(request, cache={"no-cache": False, "no-store": False})
+
+
+def litellm_completion(request, cache={"no-cache": True, "no-store": True}):
+ kwargs = ujson.loads(request)
+ return litellm.completion(cache=cache, **kwargs)
+
+
[email protected]_cache(maxsize=LM_LRU_CACHE_MAX_SIZE)
+def cached_litellm_text_completion(request):
+ return litellm_text_completion(
+ request, cache={"no-cache": False, "no-store": False}
+ )
+
+
+def litellm_text_completion(request, cache={"no-cache": True, "no-store": True}):
+ kwargs = ujson.loads(request)
+
+ # Extract the provider and model from the model string.
+ model = kwargs.pop("model").split("/", 1)
+ provider, model = model[0] if len(model) > 1 else "openai", model[-1]
+
+ # Use the API key and base from the kwargs, or from the environment.
+ api_key = kwargs.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")
+ api_base = kwargs.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")
+
+ # Build the prompt from the messages.
+ prompt = "\n\n".join(
+ [x["content"] for x in kwargs.pop("messages")] + ["BEGIN RESPONSE:"]
+ )
+
+ return litellm.text_completion(
+ cache=cache,
+ model=f"text-completion-openai/{model}",
+ api_key=api_key,
+ api_base=api_base,
+ prompt=prompt,
+ **kwargs,
+ )
+
+
+def _green(text: str, end: str = "\n"):
+ return "\x1b[32m" + str(text).lstrip() + "\x1b[0m" + end
+
+
+def _red(text: str, end: str = "\n"):
+ return "\x1b[31m" + str(text) + "\x1b[0m" + end
+
+
+def _inspect_history(lm, n: int = 1):
+ """Prints the last n prompts and their completions."""
+
+ for item in lm.history[-n:]:
+ messages = item["messages"] or [{"role": "user", "content": item["prompt"]}]
+ outputs = item["outputs"]
+
+ print("\n\n\n")
+ for msg in messages:
+ print(_red(f"{msg['role'].capitalize()} message:"))
+ print(msg["content"].strip())
+ print("\n")
+
+ print(_red("Response:"))
+ print(_green(outputs[0].strip()))
+
+ if len(outputs) > 1:
+ choices_text = f" \t (and {len(outputs)-1} other completions)"
+ print(_red(choices_text, end=""))
+
+ print("\n\n\n")
+############################
+
+
+class LitellmModel(LM):
+ """A wrapper class for dspy.OpenAI."""
+
+ def __init__(
+ self,
+ model: str = "openai/gpt-4o-mini",
+ api_key: Optional[str] = None,
+ model_type: Literal["chat", "text"] = "chat",
+ **kwargs,
+ ):
+ super().__init__(model=model, api_key=api_key, model_type=model_type, **kwargs)
+ self._token_usage_lock = threading.Lock()
+ self.prompt_tokens = 0
+ self.completion_tokens = 0
+
+ def log_usage(self, response):
+ """Log the total tokens from the OpenAI API response."""
+ usage_data = response.get("usage")
+ if usage_data:
+ with self._token_usage_lock:
+ self.prompt_tokens += usage_data.get("prompt_tokens", 0)
+ self.completion_tokens += usage_data.get("completion_tokens", 0)
+
+ def get_usage_and_reset(self):
+ """Get the total tokens used and reset the token usage."""
+ usage = {
+ self.model
+ or self.kwargs.get("model")
+ or self.kwargs.get("engine"): {
+ "prompt_tokens": self.prompt_tokens,
+ "completion_tokens": self.completion_tokens,
+ }
+ }
+ self.prompt_tokens = 0
+ self.completion_tokens = 0
+
+ return usage
+
+ def __call__(self, prompt=None, messages=None, **kwargs):
+ # Build the request.
+ cache = kwargs.pop("cache", self.cache)
+ messages = messages or [{"role": "user", "content": prompt}]
+ kwargs = {**self.kwargs, **kwargs}
+
+ # Make the request and handle LRU & disk caching.
+ if self.model_type == "chat":
+ completion = cached_litellm_completion if cache else litellm_completion
+ else:
+ completion = (
+ cached_litellm_text_completion if cache else litellm_text_completion
+ )
+
+ response = completion(
+ ujson.dumps(dict(model=self.model, messages=messages, **kwargs))
+ )
+ response_dict = response.json()
+ self.log_usage(response_dict)
+ outputs = [
+ c.message.content if hasattr(c, "message") else c["text"]
+ for c in response["choices"]
+ ]
+
+ # Logging, with removed api key & where `cost` is None on cache hit.
+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}
+ entry = dict(
+ prompt=prompt, messages=messages, kwargs=kwargs, response=response_dict
+ )
+ entry = dict(**entry, outputs=outputs, usage=dict(response_dict["usage"]))
+ entry = dict(
+ **entry, cost=response.get("_hidden_params", {}).get("response_cost")
+ )
+ self.history.append(entry)
+
+ return outputs
+# ========================================================================
+# following LM models are deprecated.
+# Remained in this file for reproduction prorpose
class OpenAIModel(dspy.OpenAI):
"""A wrapper class for dspy.OpenAI."""
@@ -1027,3 +1273,4 @@ class GoogleModel(dspy.dsp.modules.lm.LM):
completions.append(response.parts[0].text)
return completions
+# ========================================================================
diff --git a/knowledge_storm/storm_wiki/engine.py b/knowledge_storm/storm_wiki/engine.py
index 9a94acf..1cc9629 100644
--- a/knowledge_storm/storm_wiki/engine.py
+++ b/knowledge_storm/storm_wiki/engine.py
@@ -14,7 +14,7 @@ from .modules.outline_generation import StormOutlineGenerationModule
from .modules.persona_generator import StormPersonaGenerator
from .modules.storm_dataclass import StormInformationTable, StormArticle
from ..interface import Engine, LMConfigs, Retriever
-from ..lm import OpenAIModel, AzureOpenAIModel
+from ..lm import LitellmModel
from ..utils import FileIOHelper, makeStringRed, truncate_filename
@@ -56,50 +56,49 @@ class STORMWikiLMConfigs(LMConfigs):
openai_kwargs = {
"api_key": openai_api_key,
- "api_provider": "openai",
"temperature": temperature,
"top_p": top_p,
"api_base": None,
}
if openai_type and openai_type == "openai":
- self.conv_simulator_lm = OpenAIModel(
+ self.conv_simulator_lm = LitellmModel(
model="gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs
)
- self.question_asker_lm = OpenAIModel(
+ self.question_asker_lm = LitellmModel(
model="gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs
)
# 1/12/2024: Update gpt-4 to gpt-4-1106-preview. (Currently keep the original setup when using azure.)
- self.outline_gen_lm = OpenAIModel(
+ self.outline_gen_lm = LitellmModel(
model="gpt-4-0125-preview", max_tokens=400, **openai_kwargs
)
- self.article_gen_lm = OpenAIModel(
+ self.article_gen_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=700, **openai_kwargs
)
- self.article_polish_lm = OpenAIModel(
+ self.article_polish_lm = LitellmModel(
model="gpt-4o-2024-05-13", max_tokens=4000, **openai_kwargs
)
elif openai_type and openai_type == "azure":
- self.conv_simulator_lm = OpenAIModel(
- model="gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs
+ self.conv_simulator_lm = LitellmModel(
+ model="azure/gpt-4o-mini-2024-07-18", max_tokens=500, **openai_kwargs
)
- self.question_asker_lm = AzureOpenAIModel(
- model="gpt-4o-mini-2024-07-18",
+ self.question_asker_lm = LitellmModel(
+ model="azure/gpt-4o-mini-2024-07-18",
max_tokens=500,
**azure_kwargs,
model_type="chat",
)
# use combination of openai and azure-openai as azure-openai does not support gpt-4 in standard deployment
- self.outline_gen_lm = AzureOpenAIModel(
- model="gpt-4o", max_tokens=400, **azure_kwargs, model_type="chat"
+ self.outline_gen_lm = LitellmModel(
+ model="azure/gpt-4o", max_tokens=400, **azure_kwargs, model_type="chat"
)
- self.article_gen_lm = AzureOpenAIModel(
- model="gpt-4o-mini-2024-07-18",
+ self.article_gen_lm = LitellmModel(
+ model="azure/gpt-4o-mini-2024-07-18",
max_tokens=700,
**azure_kwargs,
model_type="chat",
)
- self.article_polish_lm = AzureOpenAIModel(
- model="gpt-4o-mini-2024-07-18",
+ self.article_polish_lm = LitellmModel(
+ model="azure/gpt-4o-mini-2024-07-18",
max_tokens=4000,
**azure_kwargs,
model_type="chat",
diff --git a/knowledge_storm/utils.py b/knowledge_storm/utils.py
index 2e3cbb6..4411b05 100644
--- a/knowledge_storm/utils.py
+++ b/knowledge_storm/utils.py
@@ -1,4 +1,6 @@
import concurrent.futures
+import dspy
+import httpx
import json
import logging
import os
@@ -6,21 +8,14 @@ import pickle
import re
import regex
import sys
-import time
+import toml
from typing import List, Dict
+from tqdm import tqdm
-import httpx
-import pandas as pd
-import toml
-from langchain_core.documents import Document
-from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_qdrant import Qdrant
from langchain_text_splitters import RecursiveCharacterTextSplitter
-from qdrant_client import QdrantClient, models
-from tqdm import tqdm
from trafilatura import extract
-from .lm import OpenAIModel
+from .lm import LitellmModel
logging.getLogger("httpx").setLevel(logging.WARNING) # Disable INFO logging for httpx.
@@ -72,8 +67,11 @@ class QdrantVectorStoreManager:
@staticmethod
def _check_create_collection(
- client: QdrantClient, collection_name: str, model: HuggingFaceEmbeddings
+ client: "QdrantClient", collection_name: str, model: "HuggingFaceEmbeddings"
):
+ from langchain_qdrant import Qdrant
+ from qdrant_client import models
+
"""Check if the Qdrant collection exists and create it if it does not."""
if client is None:
raise ValueError("Qdrant client is not initialized.")
@@ -103,8 +101,10 @@ class QdrantVectorStoreManager:
@staticmethod
def _init_online_vector_db(
- url: str, api_key: str, collection_name: str, model: HuggingFaceEmbeddings
+ url: str, api_key: str, collection_name: str, model: "HuggingFaceEmbeddings"
):
+ from qdrant_client import QdrantClient
+
"""Initialize the Qdrant client that is connected to an online vector store with the given URL and API key.
Args:
@@ -128,8 +128,10 @@ class QdrantVectorStoreManager:
@staticmethod
def _init_offline_vector_db(
- vector_store_path: str, collection_name: str, model: HuggingFaceEmbeddings
+ vector_store_path: str, collection_name: str, model: "HuggingFaceEmbeddings"
):
+ from qdrant_client import QdrantClient
+
"""Initialize the Qdrant client that is connected to an offline vector store with the given vector store folder path.
Args:
@@ -164,6 +166,8 @@ class QdrantVectorStoreManager:
embedding_model: str = "BAAI/bge-m3",
device: str = "mps",
):
+ from qdrant_client import Document
+
"""
Takes a CSV file and adds each row in the CSV file to the Qdrant collection.
@@ -192,6 +196,8 @@ class QdrantVectorStoreManager:
model_kwargs = {"device": device}
encode_kwargs = {"normalize_embeddings": True}
+ from langchain_huggingface import HuggingFaceEmbeddings
+
model = HuggingFaceEmbeddings(
model_name=embedding_model,
model_kwargs=model_kwargs,
@@ -231,6 +237,8 @@ class QdrantVectorStoreManager:
raise ValueError("Qdrant client is not initialized.")
# read the csv file
+ import pandas as pd
+
df = pd.read_csv(file_path)
# check that content column exists and url column exists
if content_column not in df.columns:
@@ -704,10 +712,8 @@ class WebPageHelper:
def user_input_appropriateness_check(user_input):
- my_openai_model = OpenAIModel(
- api_key=os.getenv("OPENAI_API_KEY"),
- api_provider="openai",
- model="gpt-4o-mini-2024-07-18",
+ my_openai_model = LitellmModel(
+ model="azure/gpt-4o-mini",
max_tokens=10,
temperature=0.0,
top_p=0.9,
@@ -761,10 +767,8 @@ User input: {user_input}"""
def purpose_appropriateness_check(user_input):
- my_openai_model = OpenAIModel(
- api_key=os.getenv("OPENAI_API_KEY"),
- api_provider="openai",
- model="gpt-4o-mini-2024-07-18",
+ my_openai_model = LitellmModel(
+ model="azure/gpt-4o-mini",
max_tokens=10,
temperature=0.0,
top_p=0.9,