An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
sync src
Yucheng-Jiangcf44f21e59c1
diff --git a/src/interface.py b/src/interface.pynew file mode 100644index 0000000..03df2fb--- /dev/null+++ b/src/interface.py@@ -0,0 +1,435 @@+import functools+import logging+import time+from abc import ABC, abstractmethod+from collections import OrderedDict+from typing import Dict, List, Optional, Union++logging.basicConfig(level=logging.INFO, format='%(name)s : %(levelname)-8s : %(message)s')+logger = logging.getLogger(__name__)+++class Information(ABC):+ """Abstract base class to represent basic information.++ Attributes:+ uuid (str): The unique identifier for the information.+ meta (dict): The meta information associated with the information.+ """++ def __init__(self, uuid, meta={}):+ self.uuid = uuid+ self.meta = meta+++class InformationTable(ABC):+ """+ The InformationTable class serves as data class to store the information+ collected during KnowledgeCuration stage.++ Create subclass to incorporate more information as needed. For example,+ in STORM paper https://arxiv.org/pdf/2402.14207.pdf, additional information+ would be perspective guided dialogue history.+ """++ def __init__(self):+ pass++ @abstractmethod+ def retrieve_information(**kwargs):+ pass+++class ArticleSectionNode:+ """+ The ArticleSectionNode is the dataclass for handling the section of the article.+ The content storage, section writing preferences are defined in this node.+ """++ def __init__(self, section_name: str, content=None):+ """+ section_name: section heading in string format. E.g. Introduction, History, etc.+ content: content of the section. Up to you for design choice of the data structure.+ """+ self.section_name = section_name+ self.content = content+ self.children = []+ self.preference = None++ def add_child(self, new_child_node, insert_to_front=False):+ if insert_to_front:+ self.children.insert(0, new_child_node)+ else:+ self.children.append(new_child_node)++ def remove_child(self, child):+ self.children.remove(child)+++class Article(ABC):+ def __init__(self, topic_name):+ self.root = ArticleSectionNode(topic_name)++ def find_section(self, node: ArticleSectionNode, name: str) -> Optional[ArticleSectionNode]:+ """+ Return the node of the section given the section name.++ Args:+ node: the node as the root to find.+ name: the name of node as section name++ Return:+ reference of the node or None if section name has no match+ """+ if node.section_name == name:+ return node+ for child in node.children:+ result = self.find_section(child, name)+ if result:+ return result+ return None++ @abstractmethod+ def to_string(self) -> str:+ """+ Export Article object into string representation.+ """++ def get_outline_tree(self):+ """+ Generates a hierarchical tree structure representing the outline of the document.++ Returns:+ Dict[str, Dict]: A nested dictionary representing the hierarchical structure of the document's outline.+ Each key is a section name, and the value is another dictionary representing the child sections,+ recursively forming the tree structure of the document's outline. If a section has no subsections,+ its value is an empty dictionary.++ Example:+ Assuming a document with a structure like:+ - Introduction+ - Background+ - Objective+ - Methods+ - Data Collection+ - Analysis+ The method would return:+ {+ 'Introduction': {+ 'Background': {},+ 'Objective': {}+ },+ 'Methods': {+ 'Data Collection': {},+ 'Analysis': {}+ }+ }+ """++ def build_tree(node) -> Dict[str, Dict]:+ tree = {}+ for child in node.children:+ tree[child.section_name] = build_tree(child)+ return tree if tree else {}++ return build_tree(self.root)++ def get_first_level_section_names(self) -> List[str]:+ """+ Get first level section names+ """+ return [i.section_name for i in self.root.children]++ @classmethod+ @abstractmethod+ def from_string(cls, topic_name: str, article_text: str):+ """+ Create an instance of the Article object from a string+ """+ pass++ def prune_empty_nodes(self, node=None):+ if node is None:+ node = self.root++ node.children[:] = [child for child in node.children if self.prune_empty_nodes(child)]++ if (node.content is None or node.content == "") and not node.children:+ return None+ else:+ return node+++class Retriever(ABC):+ """+ An abstract base class for retriever modules. It provides a template for retrieving information based on a query.++ This class should be extended to implement specific retrieval functionalities.+ Users can design their retriever modules as needed by implementing the retrieve method.+ The retrieval model/search engine used for each part should be declared with a suffix '_rm' in the attribute name.+ """++ def __init__(self, search_top_k):+ self.search_top_k = search_top_k++ def update_search_top_k(self, k):+ self.search_top_k = k++ def collect_and_reset_rm_usage(self):+ combined_usage = []+ for attr_name in self.__dict__:+ if '_rm' in attr_name and hasattr(getattr(self, attr_name), 'get_usage_and_reset'):+ combined_usage.append(getattr(self, attr_name).get_usage_and_reset())++ name_to_usage = {}+ for usage in combined_usage:+ for model_name, query_cnt in usage.items():+ if model_name not in name_to_usage:+ name_to_usage[model_name] = query_cnt+ else:+ name_to_usage[model_name] += query_cnt++ return name_to_usage++ @abstractmethod+ def retrieve(self, query: Union[str, List[str]], **kwargs) -> List[Information]:+ """+ Retrieves information based on a query.++ This method must be implemented by subclasses to specify how information is retrieved.++ Args:+ query (Union[str, List[str]]): The query or list of queries to retrieve information for.+ **kwargs: Additional keyword arguments that might be necessary for the retrieval process.++ Returns:+ List[Information]: A list of Information objects retrieved based on the query.+ """+ pass+++class KnowledgeCurationModule(ABC):+ """+ The interface for knowledge curation stage. Given topic, return collected information.+ """++ def __init__(self, retriever: Retriever):+ """+ Store args and finish initialization.+ """+ self.retriever = retriever++ @abstractmethod+ def research(self, topic) -> InformationTable:+ """+ Curate information and knowledge for the given topic++ Args:+ topic: topic of interest in natural language.++ Returns:+ collected_information: collected information in InformationTable type.+ """+ pass+++class OutlineGenerationModule(ABC):+ """+ The interface for outline generation stage. Given topic, collected information from knowledge+ curation stage, generate outline for the article.+ """++ @abstractmethod+ def generate_outline(self, topic: str, information_table: InformationTable, **kwargs) -> Article:+ """+ Generate outline for the article. Required arguments include:+ topic: the topic of interest+ information_table: knowledge curation data generated from KnowledgeCurationModule++ More arguments could be+ 1. draft outline+ 2. user provided outline++ Returns:+ article_outline of type ArticleOutline+ """+ pass+++class ArticleGenerationModule(ABC):+ """+ The interface for article generation stage. Given topic, collected information from+ knowledge curation stage, generated outline from outline generation stage,+ """++ @abstractmethod+ def generate_article(self,+ topic: str,+ information_table: InformationTable,+ article_with_outline: Article,+ **kwargs) -> Article:+ """+ Generate article. Required arguments include:+ topic: the topic of interest+ information_table: knowledge curation data generated from KnowledgeCurationModule+ article_with_outline: article with specified outline from OutlineGenerationModule+ """+ pass+++class ArticlePolishingModule(ABC):+ """+ The interface for article generation stage. Given topic, collected information from+ knowledge curation stage, generated outline from outline generation stage,+ """++ @abstractmethod+ def polish_article(self, topic: str, draft_article: Article, **kwargs) -> Article:+ """+ Polish article. Required arguments include:+ topic: the topic of interest+ draft_article: draft article from ArticleGenerationModule.+ """+ pass+++def log_execution_time(func):+ """Decorator to log the execution time of a function."""++ @functools.wraps(func)+ def wrapper(self, *args, **kwargs):+ start_time = time.time()+ result = func(self, *args, **kwargs)+ end_time = time.time()+ execution_time = end_time - start_time+ logger.info(f"{func.__name__} executed in {execution_time:.4f} seconds")+ self.time[func.__name__] = execution_time+ return result++ return wrapper+++class LMConfigs(ABC):+ """Abstract base class for language model configurations of the knowledge curation engine.++ The language model used for each part should be declared with a suffix '_lm' in the attribute name."""++ def __init__(self):+ pass++ def init_check(self):+ for attr_name in self.__dict__:+ if '_lm' in attr_name and getattr(self, attr_name) is None:+ logging.warning(+ f"Language model for {attr_name} is not initialized. Please call set_{attr_name}()"+ )++ def collect_and_reset_lm_history(self):+ history = []+ for attr_name in self.__dict__:+ if '_lm' in attr_name and hasattr(getattr(self, attr_name), 'history'):+ history.extend(getattr(self, attr_name).history)+ getattr(self, attr_name).history = []++ return history++ def collect_and_reset_lm_usage(self):+ combined_usage = []+ for attr_name in self.__dict__:+ if '_lm' in attr_name and hasattr(getattr(self, attr_name), 'get_usage_and_reset'):+ combined_usage.append(getattr(self, attr_name).get_usage_and_reset())++ model_name_to_usage = {}+ for usage in combined_usage:+ for model_name, tokens in usage.items():+ if model_name not in model_name_to_usage:+ model_name_to_usage[model_name] = tokens+ else:+ model_name_to_usage[model_name]['prompt_tokens'] += tokens['prompt_tokens']+ model_name_to_usage[model_name]['completion_tokens'] += tokens['completion_tokens']++ return model_name_to_usage++ def log(self):++ return OrderedDict(+ {+ attr_name: getattr(self, attr_name).kwargs for attr_name in self.__dict__ if+ '_lm' in attr_name and hasattr(getattr(self, attr_name), 'kwargs')+ }+ )+++class Engine(ABC):+ def __init__(self, lm_configs: LMConfigs):+ self.lm_configs = lm_configs+ self.time = {}+ self.lm_cost = {} # Cost of language models measured by in/out tokens.+ self.rm_cost = {} # Cost of retrievers measured by number of queries.++ def log_execution_time_and_lm_rm_usage(self, func):+ """Decorator to log the execution time, language model usage, and retrieval model usage of a function."""++ @functools.wraps(func)+ def wrapper(*args, **kwargs):+ start_time = time.time()+ result = func(*args, **kwargs)+ end_time = time.time()+ execution_time = end_time - start_time+ self.time[func.__name__] = execution_time+ logger.info(f"{func.__name__} executed in {execution_time:.4f} seconds")+ self.lm_cost[func.__name__] = self.lm_configs.collect_and_reset_lm_usage()+ if hasattr(self, 'retriever'):+ self.rm_cost[func.__name__] = self.retriever.collect_and_reset_rm_usage()+ return result++ return wrapper++ def apply_decorators(self):+ """Apply decorators to methods that need them."""+ methods_to_decorate = [method_name for method_name in dir(self)+ if callable(getattr(self, method_name)) and method_name.startswith('run_')]+ for method_name in methods_to_decorate:+ original_method = getattr(self, method_name)+ decorated_method = self.log_execution_time_and_lm_rm_usage(original_method)+ setattr(self, method_name, decorated_method)++ @abstractmethod+ def run_knowledge_curation_module(self, **kwargs) -> Optional[InformationTable]:+ pass++ @abstractmethod+ def run_outline_generation_module(self, **kwarg) -> Article:+ pass++ @abstractmethod+ def run_article_generation_module(self, **kwarg) -> Article:+ pass++ @abstractmethod+ def run_article_polishing_module(self, **kwarg) -> Article:+ pass++ @abstractmethod+ def run(self, **kwargs):+ pass++ def summary(self):+ print("***** Execution time *****")+ for k, v in self.time.items():+ print(f"{k}: {v:.4f} seconds")++ print("***** Token usage of language models: *****")+ for k, v in self.lm_cost.items():+ print(f"{k}")+ for model_name, tokens in v.items():+ print(f" {model_name}: {tokens}")++ print("***** Number of queries of retrieval models: *****")+ for k, v in self.rm_cost.items():+ print(f"{k}: {v}")++ def reset(self):+ self.time = {}+ self.lm_cost = {}+ self.rm_cost = {}diff --git a/src/lm.py b/src/lm.pynew file mode 100644index 0000000..9c38f40--- /dev/null+++ b/src/lm.py@@ -0,0 +1,451 @@+import logging+import os+import random+import threading+from typing import Optional, Literal, Any++import anthropic+import backoff+import dspy+import requests+from dsp import ERRORS, backoff_hdlr, giveup_hdlr+from dsp.modules.hf import openai_to_hf+from dsp.modules.hf_client import send_hfvllm_request_v00, send_hftgi_request_v01_wrapped+from transformers import AutoTokenizer+++class OpenAIModel(dspy.OpenAI):+ """A wrapper class for dspy.OpenAI."""++ def __init__(+ self,+ model: str = "gpt-3.5-turbo-instruct",+ api_key: Optional[str] = None,+ api_provider: Literal["openai", "azure"] = "openai",+ api_base: Optional[str] = None,+ model_type: Literal["chat", "text"] = None,+ **kwargs+ ):+ super().__init__(model=model, api_key=api_key, api_provider=api_provider, api_base=api_base,+ model_type=model_type, **kwargs)+ self._token_usage_lock = threading.Lock()+ self.prompt_tokens = 0+ self.completion_tokens = 0++ def log_usage(self, response):+ """Log the total tokens from the OpenAI API response."""+ usage_data = response.get('usage')+ if usage_data:+ with self._token_usage_lock:+ self.prompt_tokens += usage_data.get('prompt_tokens', 0)+ self.completion_tokens += usage_data.get('completion_tokens', 0)++ def get_usage_and_reset(self):+ """Get the total tokens used and reset the token usage."""+ usage = {+ self.kwargs.get('model') or self.kwargs.get('engine'):+ {'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens}+ }+ self.prompt_tokens = 0+ self.completion_tokens = 0++ return usage++ def __call__(+ self,+ prompt: str,+ only_completed: bool = True,+ return_sorted: bool = False,+ **kwargs,+ ) -> list[dict[str, Any]]:+ """Copied from dspy/dsp/modules/gpt3.py with the addition of tracking token usage."""++ assert only_completed, "for now"+ assert return_sorted is False, "for now"++ # if kwargs.get("n", 1) > 1:+ # if self.model_type == "chat":+ # kwargs = {**kwargs}+ # else:+ # kwargs = {**kwargs, "logprobs": 5}++ response = self.request(prompt, **kwargs)++ # Log the token usage from the OpenAI API response.+ self.log_usage(response)++ choices = response["choices"]++ completed_choices = [c for c in choices if c["finish_reason"] != "length"]++ if only_completed and len(completed_choices):+ choices = completed_choices++ completions = [self._get_choice_text(c) for c in choices]+ if return_sorted and kwargs.get("n", 1) > 1:+ scored_completions = []++ for c in choices:+ tokens, logprobs = (+ c["logprobs"]["tokens"],+ c["logprobs"]["token_logprobs"],+ )++ if "<|endoftext|>" in tokens:+ index = tokens.index("<|endoftext|>") + 1+ tokens, logprobs = tokens[:index], logprobs[:index]++ avglog = sum(logprobs) / len(logprobs)+ scored_completions.append((avglog, self._get_choice_text(c)))++ scored_completions = sorted(scored_completions, reverse=True)+ completions = [c for _, c in scored_completions]++ return completions+++class ClaudeModel(dspy.dsp.modules.lm.LM):+ """Copied from dspy/dsp/modules/anthropic.py with the addition of tracking token usage."""++ def __init__(+ self,+ model: str = "claude-3-opus-20240229",+ api_key: Optional[str] = None,+ api_base: Optional[str] = None,+ **kwargs,+ ):+ super().__init__(model)+ try:+ from anthropic import Anthropic+ except ImportError as err:+ raise ImportError("Claude requires `pip install anthropic`.") from err++ self.provider = "anthropic"+ self.api_key = api_key = os.environ.get("ANTHROPIC_API_KEY") if api_key is None else api_key+ self.api_base = "https://api.anthropic.com/v1/messages" if api_base is None else api_base+ self.kwargs = {"temperature": kwargs.get("temperature", 0.0),+ "max_tokens": min(kwargs.get("max_tokens", 4096), 4096), "top_p": kwargs.get("top_p", 1.0),+ "top_k": kwargs.get("top_k", 1), "n": kwargs.pop("n", kwargs.pop("num_generations", 1)),+ **kwargs, "model": model}+ self.history: list[dict[str, Any]] = []+ self.client = Anthropic(api_key=api_key)+ self.model = model++ self._token_usage_lock = threading.Lock()+ self.prompt_tokens = 0+ self.completion_tokens = 0++ def log_usage(self, response):+ """Log the total tokens from the Anthropic API response."""+ usage_data = response.usage+ if usage_data:+ with self._token_usage_lock:+ self.prompt_tokens += usage_data.input_tokens+ self.completion_tokens += usage_data.output_tokens++ def get_usage_and_reset(self):+ """Get the total tokens used and reset the token usage."""+ usage = {+ self.model:+ {'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens}+ }+ self.prompt_tokens = 0+ self.completion_tokens = 0++ return usage++ def basic_request(self, prompt: str, **kwargs):+ raw_kwargs = kwargs+ kwargs = {**self.kwargs, **kwargs}+ # caching mechanism requires hashable kwargs+ kwargs["messages"] = [{"role": "user", "content": prompt}]+ kwargs.pop("n")+ response = self.client.messages.create(**kwargs)+ # history = {+ # "prompt": prompt,+ # "response": response,+ # "kwargs": kwargs,+ # "raw_kwargs": raw_kwargs,+ # }+ json_serializable_history = {+ "prompt": prompt,+ "response": {+ "content": response.content[0].text,+ "model": response.model,+ "role": response.role,+ "stop_reason": response.stop_reason,+ "stop_sequence": response.stop_sequence,+ "type": response.type,+ "usage": {+ "input_tokens": response.usage.input_tokens,+ "output_tokens": response.usage.output_tokens,+ }+ },+ "kwargs": kwargs,+ "raw_kwargs": raw_kwargs,+ }+ self.history.append(json_serializable_history)+ return response++ @backoff.on_exception(+ backoff.expo,+ (anthropic.RateLimitError),+ max_time=1000,+ max_tries=8,+ on_backoff=backoff_hdlr,+ giveup=giveup_hdlr,+ )+ def request(self, prompt: str, **kwargs):+ """Handles retrieval of completions from Anthropic whilst handling API errors."""+ return self.basic_request(prompt, **kwargs)++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ """Retrieves completions from Anthropic.++ Args:+ prompt (str): prompt to send to Anthropic+ only_completed (bool, optional): return only completed responses and ignores completion due to length. Defaults to True.+ return_sorted (bool, optional): sort the completion choices using the returned probabilities. Defaults to False.++ Returns:+ list[str]: list of completion choices+ """+ assert only_completed, "for now"+ assert return_sorted is False, "for now"+ # per eg here: https://docs.anthropic.com/claude/reference/messages-examples+ # max tokens can be used as a proxy to return smaller responses+ # so this cannot be a proper indicator for incomplete response unless it isnt the user-intent.+ n = kwargs.pop("n", 1)+ completions = []+ for _ in range(n):+ response = self.request(prompt, **kwargs)+ self.log_usage(response)+ if only_completed and response.stop_reason == "max_tokens":+ continue+ completions = [c.text for c in response.content]+ return completions+++class VLLMClient(dspy.HFClientVLLM):+ """A wrapper class for dspy.HFClientVLLM."""++ def __init__(self, model, port, url="http://localhost", **kwargs):+ """Copied from dspy/dsp/modules/hf_client.py with the addition of storing additional kwargs."""++ super().__init__(model=model, port=port, url=url, **kwargs)+ # Store additional kwargs for the generate method.+ self.kwargs = {**self.kwargs, **kwargs}++ def _generate(self, prompt, **kwargs):+ """Copied from dspy/dsp/modules/hf_client.py with the addition of passing kwargs to VLLM server."""+ kwargs = {**self.kwargs, **kwargs}++ # payload = {+ # "model": kwargs["model"],+ # "prompt": prompt,+ # "max_tokens": kwargs["max_tokens"],+ # "temperature": kwargs["temperature"],+ # }+ payload = {+ "prompt": prompt,+ **kwargs+ }++ response = send_hfvllm_request_v00(+ f"{self.url}/v1/completions",+ json=payload,+ headers=self.headers,+ )++ try:+ json_response = response.json()+ completions = json_response["choices"]+ response = {+ "prompt": prompt,+ "choices": [{"text": c["text"]} for c in completions],+ }+ return response++ except Exception as e:+ print("Failed to parse JSON response:", response.text)+ raise Exception("Received invalid JSON response from server")+++class TGIClient(dspy.HFClientTGI):+ def __init__(self, model, port, url, http_request_kwargs=None, **kwargs):+ super().__init__(model=model, port=port, url=url, http_request_kwargs=http_request_kwargs, **kwargs)++ def _generate(self, prompt, **kwargs):+ """Copied from dspy/dsp/modules/hf_client.py with the addition of removing hard-coded parameters."""+ kwargs = {**self.kwargs, **kwargs}++ payload = {+ "inputs": prompt,+ "parameters": {+ "do_sample": kwargs["n"] > 1,+ "best_of": kwargs["n"],+ "details": kwargs["n"] > 1,+ **kwargs,+ },+ }++ payload["parameters"] = openai_to_hf(**payload["parameters"])++ # Comment out the following lines to remove the hard-coded parameters.+ # payload["parameters"]["temperature"] = max(+ # 0.1, payload["parameters"]["temperature"],+ # )++ response = send_hftgi_request_v01_wrapped(+ f"{self.url}:{random.Random().choice(self.ports)}" + "/generate",+ url=self.url,+ ports=tuple(self.ports),+ json=payload,+ headers=self.headers,+ **self.http_request_kwargs,+ )++ try:+ json_response = response.json()+ # completions = json_response["generated_text"]++ completions = [json_response["generated_text"]]++ if (+ "details" in json_response+ and "best_of_sequences" in json_response["details"]+ ):+ completions += [+ x["generated_text"]+ for x in json_response["details"]["best_of_sequences"]+ ]++ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}+ return response+ except Exception:+ print("Failed to parse JSON response:", response.text)+ raise Exception("Received invalid JSON response from server")+++class TogetherClient(dspy.HFModel):+ """A wrapper class for dspy.Together."""++ def __init__(self, model, apply_tokenizer_chat_template=False, hf_tokenizer_name=None, **kwargs):+ """Copied from dspy/dsp/modules/hf_client.py with the support of applying tokenizer chat template."""++ super().__init__(model=model, is_client=True)+ self.session = requests.Session()+ self.api_base = "https://api.together.xyz/v1/completions" if os.getenv(+ "TOGETHER_API_BASE") is None else os.getenv("TOGETHER_API_BASE")+ self.token = os.getenv("TOGETHER_API_KEY")+ self.model = model++ # self.use_inst_template = False+ # if any(keyword in self.model.lower() for keyword in ["inst", "instruct"]):+ # self.use_inst_template = True+ self.apply_tokenizer_chat_template = apply_tokenizer_chat_template+ if self.apply_tokenizer_chat_template:+ logging.info("Loading huggingface tokenizer.")+ if hf_tokenizer_name is None:+ hf_tokenizer_name = self.model+ self.tokenizer = AutoTokenizer.from_pretrained(hf_tokenizer_name, cache_dir=kwargs.get("cache_dir", None))++ stop_default = "\n\n---"++ self.kwargs = {+ "temperature": 0.0,+ "max_tokens": 512,+ "top_p": 1,+ "top_k": 20,+ "repetition_penalty": 1,+ "n": 1,+ "stop": stop_default if "stop" not in kwargs else kwargs["stop"],+ **kwargs,+ }+ self._token_usage_lock = threading.Lock()+ self.prompt_tokens = 0+ self.completion_tokens = 0++ def log_usage(self, response):+ """Log the total tokens from the OpenAI API response."""+ usage_data = response.get('usage')+ if usage_data:+ with self._token_usage_lock:+ self.prompt_tokens += usage_data.get('prompt_tokens', 0)+ self.completion_tokens += usage_data.get('completion_tokens', 0)++ def get_usage_and_reset(self):+ """Get the total tokens used and reset the token usage."""+ usage = {+ self.model:+ {'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens}+ }+ self.prompt_tokens = 0+ self.completion_tokens = 0++ return usage++ @backoff.on_exception(+ backoff.expo,+ ERRORS,+ max_time=1000,+ on_backoff=backoff_hdlr,+ )+ def _generate(self, prompt, use_chat_api=False, **kwargs):+ url = f"{self.api_base}"++ kwargs = {**self.kwargs, **kwargs}++ stop = kwargs.get("stop")+ temperature = kwargs.get("temperature")+ max_tokens = kwargs.get("max_tokens", 150)+ top_p = kwargs.get("top_p", 0.7)+ top_k = kwargs.get("top_k", 50)+ repetition_penalty = kwargs.get("repetition_penalty", 1)+ if self.apply_tokenizer_chat_template:+ prompt = self.tokenizer.apply_chat_template([{"role": "user", "content": prompt}], tokenize=False)+ # prompt = f"[INST]{prompt}[/INST]" if self.use_inst_template else prompt++ if use_chat_api:+ url = f"{self.api_base}/chat/completions"+ messages = [+ {"role": "system",+ "content": "You are a helpful assistant. You must continue the user text directly without *any* additional interjections."},+ {"role": "user", "content": prompt},+ ]+ body = {+ "model": self.model,+ "messages": messages,+ "temperature": temperature,+ "max_tokens": max_tokens,+ "top_p": top_p,+ "top_k": top_k,+ "repetition_penalty": repetition_penalty,+ "stop": stop,+ }+ else:+ body = {+ "model": self.model,+ "prompt": prompt,+ "temperature": temperature,+ "max_tokens": max_tokens,+ "top_p": top_p,+ "top_k": top_k,+ "repetition_penalty": repetition_penalty,+ "stop": stop,+ }++ headers = {"Authorization": f"Bearer {self.token}"}++ with self.session.post(url, headers=headers, json=body) as resp:+ resp_json = resp.json()+ # Log the token usage from the Together API response.+ self.log_usage(resp_json)+ if use_chat_api:+ # completions = [resp_json['output'].get('choices', [])[0].get('message', {}).get('content', "")]+ completions = [resp_json.get('choices', [])[0].get('message', {}).get('content', "")]+ else:+ # completions = [resp_json['output'].get('choices', [])[0].get('text', "")]+ completions = [resp_json.get('choices', [])[0].get('text', "")]+ response = {"prompt": prompt, "choices": [{"text": c} for c in completions]}+ return responsediff --git a/src/rm.py b/src/rm.pynew file mode 100644index 0000000..9184294--- /dev/null+++ b/src/rm.py@@ -0,0 +1,66 @@+import logging+import os+from typing import Callable, Union, List++import dspy+import requests+++class YouRM(dspy.Retrieve):+ def __init__(self, ydc_api_key=None, k=3, is_valid_source: Callable = None):+ super().__init__(k=k)+ if not ydc_api_key and not os.environ.get("YDC_API_KEY"):+ raise RuntimeError("You must supply ydc_api_key or set environment variable YDC_API_KEY")+ elif ydc_api_key:+ self.ydc_api_key = ydc_api_key+ else:+ self.ydc_api_key = os.environ["YDC_API_KEY"]+ self.usage = 0++ # If not None, is_valid_source shall be a function that takes a URL and returns a boolean.+ self.is_valid_source = is_valid_source++ def get_usage_and_reset(self):+ usage = self.usage+ self.usage = 0++ return {'YouRM': usage}++ def forward(self, query_or_queries: Union[str, List[str]], exclude_urls: List[str]):+ """Search with You.com for self.k top passages for query or queries++ Args:+ query_or_queries (Union[str, List[str]]): The query or queries to search for.+ exclude_urls (List[str]): A list of urls to exclude from the search results.++ Returns:+ a list of Dicts, each dict has keys of 'description', 'snippets' (list of strings), 'title', 'url'+ """+ queries = (+ [query_or_queries]+ if isinstance(query_or_queries, str)+ else query_or_queries+ )+ self.usage += len(queries)+ collected_results = []+ for query in queries:+ try:+ headers = {"X-API-Key": self.ydc_api_key}+ results = requests.get(+ f"https://api.ydc-index.io/search?query={query}",+ headers=headers,+ ).json()++ authoritative_results = []+ for r in results['hits']:+ if self.is_valid_source is None or self.is_valid_source(r['url']):+ authoritative_results.append(r)+ if 'hits' in results:+ collected_results.extend(authoritative_results[:self.k])+ except Exception as e:+ logging.error(f'Error occurs when searching query {query}: {e}')++ if exclude_urls:+ collected_results = [r for r in collected_results if r['url'] not in exclude_urls]++ return collected_resultsdiff --git a/src/storm_wiki/__init__.py b/src/storm_wiki/__init__.pynew file mode 100644index 0000000..cef3379--- /dev/null+++ b/src/storm_wiki/__init__.py@@ -0,0 +1,2 @@+from .engine import *+from .modules import *diff --git a/src/storm_wiki/engine.py b/src/storm_wiki/engine.pynew file mode 100644index 0000000..4240732--- /dev/null+++ b/src/storm_wiki/engine.py@@ -0,0 +1,308 @@+import json+import logging+import os+from dataclasses import dataclass, field+from typing import Union, Literal, Optional++import dspy+from interface import Engine, LMConfigs+from lm import OpenAIModel+from storm_wiki.modules.article_generation import StormArticleGenerationModule+from storm_wiki.modules.article_polish import StormArticlePolishingModule+from storm_wiki.modules.callback import BaseCallbackHandler+from storm_wiki.modules.knowledge_curation import StormKnowledgeCurationModule+from storm_wiki.modules.outline_generation import StormOutlineGenerationModule+from storm_wiki.modules.persona_generator import StormPersonaGenerator+from storm_wiki.modules.retriever import StormRetriever+from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle+from utils import FileIOHelper+++class STORMWikiLMConfigs(LMConfigs):+ """Configurations for LLM used in different parts of STORM.++ Given that different parts in STORM framework have different complexity, we use different LLM configurations+ to achieve a balance between quality and efficiency. If no specific configuration is provided, we use the default+ setup in the paper.+ """++ def __init__(self):+ self.conv_simulator_lm = None # LLM used in conversation simulator except for question asking.+ self.question_asker_lm = None # LLM used in question asking.+ self.outline_gen_lm = None # LLM used in outline generation.+ self.article_gen_lm = None # LLM used in article generation.+ self.article_polish_lm = None # LLM used in article polishing.++ def init_openai_model(+ self,+ openai_api_key: str,+ openai_type: Literal["openai", "azure"],+ api_base: Optional[str] = None,+ api_version: Optional[str] = None,+ temperature: Optional[float] = 1.0,+ top_p: Optional[float] = 0.9+ ):+ """Legacy: Corresponding to the original setup in the NAACL'24 paper."""+ openai_kwargs = {+ 'api_key': openai_api_key,+ 'api_provider': openai_type,+ 'temperature': temperature,+ 'top_p': top_p,+ 'api_base': None,+ 'api_version': None,+ }+ if openai_type and openai_type == 'azure':+ openai_kwargs['api_base'] = api_base+ openai_kwargs['api_version'] = api_version++ self.conv_simulator_lm = OpenAIModel(model='gpt-35-turbo-instruct', engine='gpt-35-turbo-instruct',+ max_tokens=500, **openai_kwargs)+ self.question_asker_lm = OpenAIModel(model='gpt-35-turbo', engine='gpt-35-turbo',+ max_tokens=500, **openai_kwargs)+ self.outline_gen_lm = OpenAIModel(model='gpt-4', engine='gpt-4',+ max_tokens=400, **openai_kwargs)+ self.article_gen_lm = OpenAIModel(model='gpt-4', engine='gpt-4',+ max_tokens=700, **openai_kwargs)+ self.article_polish_lm = OpenAIModel(model='gpt-4-32k', engine='gpt-4-32k',+ max_tokens=4000, **openai_kwargs)+ elif openai_type and openai_type == 'openai':+ self.conv_simulator_lm = OpenAIModel(model='gpt-3.5-turbo-instruct',+ max_tokens=500, **openai_kwargs)+ self.question_asker_lm = OpenAIModel(model='gpt-3.5-turbo',+ max_tokens=500, **openai_kwargs)+ # 1/12/2024: Update gpt-4 to gpt-4-1106-preview. (Currently keep the original setup when using azure.)+ self.outline_gen_lm = OpenAIModel(model='gpt-4-0125-preview',+ max_tokens=400, **openai_kwargs)+ self.article_gen_lm = OpenAIModel(model='gpt-4-0125-preview',+ max_tokens=700, **openai_kwargs)+ self.article_polish_lm = OpenAIModel(model='gpt-4-0125-preview',+ max_tokens=4000, **openai_kwargs)+ else:+ logging.warning('No valid OpenAI API provider is provided. Cannot use default LLM configurations.')++ def set_conv_simulator_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.conv_simulator_lm = model++ def set_question_asker_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.question_asker_lm = model++ def set_outline_gen_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.outline_gen_lm = model++ def set_article_gen_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.article_gen_lm = model++ def set_article_polish_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.article_polish_lm = model+++@dataclass+class STORMWikiRunnerArguments:+ """Arguments for controlling the STORM Wiki pipeline."""+ output_dir: str = field(+ metadata={"help": "Output directory for the results."},+ )+ max_conv_turn: int = field(+ default=3,+ metadata={"help": "Maximum number of questions in conversational question asking."},+ )+ max_perspective: int = field(+ default=3,+ metadata={"help": "Maximum number of perspectives to consider in perspective-guided question asking."},+ )+ max_search_queries_per_turn: int = field(+ default=3,+ metadata={"help": "Maximum number of search queries to consider in each turn."},+ )+ disable_perspective: bool = field(+ default=False,+ metadata={"help": "If True, disable perspective-guided question asking."},+ )+ search_top_k: int = field(+ default=3,+ metadata={"help": "Top k search results to consider for each search query."},+ )+ retrieve_top_k: int = field(+ default=3,+ metadata={"help": "Top k collected references for each section title."},+ )+ max_thread_num: int = field(+ default=10,+ metadata={"help": "Maximum number of threads to use. "+ "Consider reducing it if keep getting 'Exceed rate limit' error when calling LM API."},+ )+++class STORMWikiRunner(Engine):+ """STORM Wiki pipeline runner."""++ def __init__(self,+ args: STORMWikiRunnerArguments,+ lm_configs: STORMWikiLMConfigs):+ super().__init__(lm_configs=lm_configs)+ self.args = args+ self.lm_configs = lm_configs++ self.retriever = StormRetriever(k=self.args.retrieve_top_k)+ storm_persona_generator = StormPersonaGenerator(self.lm_configs.question_asker_lm)+ self.storm_knowledge_curation_module = StormKnowledgeCurationModule(+ retriever=self.retriever,+ persona_generator=storm_persona_generator,+ conv_simulator_lm=self.lm_configs.conv_simulator_lm,+ question_asker_lm=self.lm_configs.question_asker_lm,+ max_search_queries_per_turn=self.args.max_search_queries_per_turn,+ search_top_k=self.args.search_top_k,+ max_conv_turn=self.args.max_conv_turn,+ max_thread_num=self.args.max_thread_num+ )+ self.storm_outline_generation_module = StormOutlineGenerationModule(+ outline_gen_lm=self.lm_configs.outline_gen_lm+ )+ self.storm_article_generation = StormArticleGenerationModule(+ article_gen_lm=self.lm_configs.article_gen_lm,+ retrieve_top_k=self.args.retrieve_top_k,+ max_thread_num=self.args.max_thread_num+ )+ self.storm_article_polishing_module = StormArticlePolishingModule(+ article_gen_lm=self.lm_configs.article_gen_lm,+ article_polish_lm=self.lm_configs.article_polish_lm+ )++ self.lm_configs.init_check()+ self.apply_decorators()++ def run_knowledge_curation_module(self,+ ground_truth_url: str = "None",+ callback_handler: BaseCallbackHandler = None) -> StormInformationTable:++ information_table, conversation_log = self.storm_knowledge_curation_module.research(+ topic=self.topic,+ ground_truth_url=ground_truth_url,+ callback_handler=callback_handler,+ max_perspective=self.args.max_perspective,+ disable_perspective=False,+ return_conversation_log=True+ )++ FileIOHelper.dump_json(conversation_log, os.path.join(self.article_output_dir, 'conversation_log.json'))+ information_table.dump_url_to_info(os.path.join(self.article_output_dir, 'raw_search_results.json'))+ return information_table++ def run_outline_generation_module(self,+ information_table: StormInformationTable,+ callback_handler: BaseCallbackHandler = None) -> StormArticle:++ outline, draft_outline = self.storm_outline_generation_module.generate_outline(+ topic=self.topic,+ information_table=information_table,+ return_draft_outline=True,+ callback_handler=callback_handler+ )+ outline.dump_outline_to_file(os.path.join(self.article_output_dir, 'storm_gen_outline.txt'))+ draft_outline.dump_outline_to_file(os.path.join(self.article_output_dir, "direct_gen_outline.txt"))+ return outline++ def run_article_generation_module(self,+ outline: StormArticle,+ information_table=StormInformationTable,+ callback_handler: BaseCallbackHandler = None) -> StormArticle:++ draft_article = self.storm_article_generation.generate_article(+ topic=self.topic,+ information_table=information_table,+ article_with_outline=outline,+ callback_handler=callback_handler+ )+ draft_article.dump_article_as_plain_text(os.path.join(self.article_output_dir, 'storm_gen_article.txt'))+ draft_article.dump_reference_to_file(os.path.join(self.article_output_dir, 'url_to_info.json'))+ return draft_article++ def run_article_polishing_module(self,+ draft_article: StormArticle,+ remove_duplicate: bool = False) -> StormArticle:++ polished_article = self.storm_article_polishing_module.polish_article(+ topic=self.topic,+ draft_article=draft_article,+ remove_duplicate=remove_duplicate+ )+ FileIOHelper.write_str(polished_article.to_string(),+ os.path.join(self.article_output_dir, 'storm_gen_article_polished.txt'))+ return polished_article++ def post_run(self):+ """+ Post-run operations, including:+ 1. Dumping the run configuration.+ 2. Dumping the LLM call history.+ """+ config_log = self.lm_configs.log()+ FileIOHelper.dump_json(config_log, os.path.join(self.article_output_dir, 'run_config.json'))++ llm_call_history = self.lm_configs.collect_and_reset_lm_history()+ with open(os.path.join(self.article_output_dir, 'llm_call_history.jsonl'), 'w') as f:+ for call in llm_call_history:+ if 'kwargs' in call:+ call.pop('kwargs') # All kwargs are dumped together to run_config.json.+ f.write(json.dumps(call) + '\n')++ def run(self,+ topic: str,+ ground_truth_url: str = '',+ do_research: bool = True,+ do_generate_outline: bool = True,+ do_generate_article: bool = True,+ do_polish_article: bool = True,+ remove_duplicate: bool = False,+ callback_handler: BaseCallbackHandler = BaseCallbackHandler()):+ """+ Run the STORM pipeline.++ Args:+ topic: The topic to research.+ ground_truth_url: A ground truth URL including a curated article about the topic. The URL will be excluded.+ do_research: If True, research the topic through information-seeking conversation;+ if False, expect conversation_log.json and raw_search_results.json to exist in the output directory.+ do_generate_outline: If True, generate an outline for the topic;+ if False, expect storm_gen_outline.txt to exist in the output directory.+ do_generate_article: If True, generate a curated article for the topic;+ if False, expect storm_gen_article.txt to exist in the output directory.+ do_polish_article: If True, polish the article by adding a summarization section and (optionally) removing+ duplicated content.+ remove_duplicate: If True, remove duplicated content.+ callback_handler: A callback handler to handle the intermediate results.+ """+ self.topic = topic+ self.article_dir_name = topic.replace(' ', '_').replace('/', '_')+ self.article_output_dir = os.path.join(self.args.output_dir, self.article_dir_name)+ os.makedirs(self.article_output_dir, exist_ok=True)++ # research module+ information_table: StormInformationTable = None+ if do_research:+ information_table = self.run_knowledge_curation_module(ground_truth_url=ground_truth_url,+ callback_handler=callback_handler)+ else:+ information_table = StormInformationTable.from_conversation_log_file(+ os.path.join(self.article_output_dir, 'conversation_log.json'))++ # outline generation module+ outline: StormArticle = None+ if do_generate_outline:+ outline = self.run_outline_generation_module(information_table=information_table,+ callback_handler=callback_handler)+ else:+ outline = StormArticle.from_outline_file(topic=topic, file_path=os.path.join(self.article_output_dir,+ 'storm_gen_outline.txt'))++ # article generation module+ draft_article: StormArticle = None+ if do_generate_article:+ draft_article = self.run_article_generation_module(outline=outline,+ information_table=information_table,+ callback_handler=callback_handler)++ # article polishing module+ if do_polish_article:+ polished_article = self.run_article_polishing_module(draft_article=draft_article,+ remove_duplicate=remove_duplicate)diff --git a/src/storm_wiki/modules/__init__.py b/src/storm_wiki/modules/__init__.pynew file mode 100644index 0000000..51ee012--- /dev/null+++ b/src/storm_wiki/modules/__init__.py@@ -0,0 +1,4 @@+from .knowledge_curation import *+from .persona_generator import *+from .retriever import *+from .storm_dataclass import *\ No newline at end of filediff --git a/src/storm_wiki/modules/article_generation.py b/src/storm_wiki/modules/article_generation.pynew file mode 100644index 0000000..0dfb76d--- /dev/null+++ b/src/storm_wiki/modules/article_generation.py@@ -0,0 +1,145 @@+import concurrent.futures+import copy+import logging+from concurrent.futures import as_completed+from typing import List, Union++import dspy+from interface import ArticleGenerationModule+from storm_wiki.modules.callback import BaseCallbackHandler+from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle, StormInformation+from utils import ArticleTextProcessing+++class StormArticleGenerationModule(ArticleGenerationModule):+ """+ The interface for article generation stage. Given topic, collected information from+ knowledge curation stage, generated outline from outline generation stage,+ """++ def __init__(self,+ article_gen_lm=Union[dspy.dsp.LM, dspy.dsp.HFModel],+ retrieve_top_k: int = 5,+ max_thread_num: int = 10):+ super().__init__()+ self.retrieve_top_k = retrieve_top_k+ self.article_gen_lm = article_gen_lm+ self.max_thread_num = max_thread_num+ self.section_gen = ConvToSection(engine=self.article_gen_lm)++ def generate_section(self, topic, section_name, information_table, section_outline, section_query):+ collected_info: List[StormInformation] = []+ if information_table is not None:+ collected_info = information_table.retrieve_information(queries=section_query,+ search_top_k=self.retrieve_top_k)+ output = self.section_gen(topic=topic,+ outline=section_outline,+ section=section_name,+ collected_info=collected_info)+ return {"section_name": section_name, "section_content": output.section, "collected_info": collected_info}++ def generate_article(self,+ topic: str,+ information_table: StormInformationTable,+ article_with_outline: StormArticle,+ callback_handler: BaseCallbackHandler = None) -> StormArticle:+ """+ Generate article for the topic based on the information table and article outline.++ Args:+ topic (str): The topic of the article.+ information_table (StormInformationTable): The information table containing the collected information.+ article_with_outline (StormArticle): The article with specified outline.+ callback_handler (BaseCallbackHandler): An optional callback handler that can be used to trigger+ custom callbacks at various stages of the article generation process. Defaults to None.+ """+ information_table.prepare_table_for_retrieval()++ if article_with_outline is None:+ article_with_outline = StormArticle(topic_name=topic)++ sections_to_write = article_with_outline.get_first_level_section_names()++ section_output_dict_collection = []+ if len(sections_to_write) == 0:+ logging.error(f'No outline for {topic}. Will directly search with the topic.')+ section_output_dict = self.generate_section(+ topic=topic,+ section_name=topic,+ information_table=information_table,+ section_outline="",+ section_query=[topic]+ )+ section_output_dict_collection = [section_output_dict]+ else:++ with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_thread_num) as executor:+ future_to_sec_title = {}+ for section_title in sections_to_write:+ # We don't want to write a separate introduction section.+ if section_title.lower().strip() == 'introduction':+ continue+ # We don't want to write a separate conclusion section.+ if section_title.lower().strip().startswith(+ 'conclusion') or section_title.lower().strip().startswith('summary'):+ continue+ section_query = article_with_outline.get_outline_as_list(root_section_name=section_title,+ add_hashtags=False)+ queries_with_hashtags = article_with_outline.get_outline_as_list(+ root_section_name=section_title, add_hashtags=True)+ section_outline = "\n".join(queries_with_hashtags)+ future_to_sec_title[+ executor.submit(self.generate_section,+ topic, section_title, information_table, section_outline, section_query)+ ] = section_title++ for future in as_completed(future_to_sec_title):+ section_output_dict_collection.append(future.result())++ article = copy.deepcopy(article_with_outline)+ for section_output_dict in section_output_dict_collection:+ article.update_section(parent_section_name=topic,+ current_section_content=section_output_dict["section_content"],+ current_section_info_list=section_output_dict["collected_info"])+ article.post_processing()+ return article+++class ConvToSection(dspy.Module):+ """Use the information collected from the information-seeking conversation to write a section."""++ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ super().__init__()+ self.write_section = dspy.Predict(WriteSection)+ self.engine = engine++ def forward(self, topic: str, outline: str, section: str, collected_info: List[StormInformation]):+ info = ''+ for idx, storm_info in enumerate(collected_info):+ info += f'[{idx + 1}]\n' + '\n'.join(storm_info.snippets)+ info += '\n\n'++ info = ArticleTextProcessing.limit_word_count_preserve_newline(info, 1500)++ with dspy.settings.context(lm=self.engine):+ section = ArticleTextProcessing.clean_up_section(+ self.write_section(topic=topic, info=info, section=section).output)++ return dspy.Prediction(section=section)+++class WriteSection(dspy.Signature):+ """Write a Wikipedia section based on the collected information.++ Here is the format of your writing:+ 1. Use "#" Title" to indicate section title, "##" Title" to indicate subsection title, "###" Title" to indicate subsubsection title, and so on.+ 2. Use [1], [2], ..., [n] in line (for example, "The capital of the United States is Washington, D.C.[1][3]."). You DO NOT need to include a References or Sources section to list the sources at the end.+ """++ info = dspy.InputField(prefix="The collected information:\n", format=str)+ topic = dspy.InputField(prefix="The topic of the page: ", format=str)+ section = dspy.InputField(prefix="The section you need to write: ", format=str)+ output = dspy.OutputField(+ prefix="Write the section with proper inline citations (Start your writing with # section title. Don't include the page title or try to write other sections):\n",+ format=str+ )diff --git a/src/storm_wiki/modules/article_polish.py b/src/storm_wiki/modules/article_polish.pynew file mode 100644index 0000000..5f38f05--- /dev/null+++ b/src/storm_wiki/modules/article_polish.py@@ -0,0 +1,89 @@+import copy+from typing import Union++import dspy+from interface import ArticlePolishingModule+from storm_wiki.modules.storm_dataclass import StormArticle+from utils import ArticleTextProcessing+++class StormArticlePolishingModule(ArticlePolishingModule):+ """+ The interface for article generation stage. Given topic, collected information from+ knowledge curation stage, generated outline from outline generation stage.+ """++ def __init__(self,+ article_gen_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ article_polish_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.article_gen_lm = article_gen_lm+ self.article_polish_lm = article_polish_lm++ self.polish_page = PolishPageModule(+ write_lead_engine=self.article_gen_lm,+ polish_engine=self.article_polish_lm+ )++ def polish_article(self,+ topic: str,+ draft_article: StormArticle,+ remove_duplicate: bool = False) -> StormArticle:+ """+ Polish article.++ Args:+ topic (str): The topic of the article.+ draft_article (StormArticle): The draft article.+ remove_duplicate (bool): Whether to use one additional LM call to remove duplicates from the article.+ """++ article_text = draft_article.to_string()+ polish_result = self.polish_page(topic=topic, draft_page=article_text, polish_whole_page=remove_duplicate)+ lead_section = f"# summary\n{polish_result.lead_section}"+ polished_article = '\n\n'.join([lead_section, polish_result.page])+ polished_article_dict = ArticleTextProcessing.parse_article_into_dict(polished_article)+ polished_article = copy.deepcopy(draft_article)+ polished_article.insert_or_create_section(article_dict=polished_article_dict)+ polished_article.post_processing()+ return polished_article+++class WriteLeadSection(dspy.Signature):+ """Write a lead section for the given Wikipedia page with the following guidelines:+ 1. The lead should stand on its own as a concise overview of the article's topic. It should identify the topic, establish context, explain why the topic is notable, and summarize the most important points, including any prominent controversies.+ 2. The lead section should be concise and contain no more than four well-composed paragraphs.+ 3. The lead section should be carefully sourced as appropriate. Add inline citations (e.g., "Washington, D.C., is the capital of the United States.[1][3].") where necessary."""++ topic = dspy.InputField(prefix="The topic of the page: ", format=str)+ draft_page = dspy.InputField(prefix="The draft page:\n", format=str)+ lead_section = dspy.OutputField(prefix="Write the lead section:\n", format=str)+++class PolishPage(dspy.Signature):+ """You are a faithful text editor that is good at finding repeated information in the article and deleting them to make sure there is no repetition in the article. You won't delete any non-repeated part in the article. You will keep the inline citations and article structure (indicated by "#", "##", etc.) appropriately. Do your job for the following article."""++ draft_page = dspy.InputField(prefix="The draft article:\n", format=str)+ page = dspy.OutputField(prefix="Your revised article:\n", format=str)+++class PolishPageModule(dspy.Module):+ def __init__(self, write_lead_engine: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ polish_engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ super().__init__()+ self.write_lead_engine = write_lead_engine+ self.polish_engine = polish_engine+ self.write_lead = dspy.Predict(WriteLeadSection)+ self.polish_page = dspy.Predict(PolishPage)++ def forward(self, topic: str, draft_page: str, polish_whole_page: bool = True):+ with dspy.settings.context(lm=self.write_lead_engine):+ lead_section = self.write_lead(topic=topic, draft_page=draft_page).lead_section+ if "The lead section:" in lead_section:+ lead_section = lead_section.split("The lead section:")[1].strip()+ if polish_whole_page:+ with dspy.settings.context(lm=self.polish_engine):+ page = self.polish_page(draft_page=draft_page).page+ else:+ page = draft_page++ return dspy.Prediction(lead_section=lead_section, page=page)diff --git a/src/storm_wiki/modules/callback.py b/src/storm_wiki/modules/callback.pynew file mode 100644index 0000000..945a45d--- /dev/null+++ b/src/storm_wiki/modules/callback.py@@ -0,0 +1,34 @@+class BaseCallbackHandler:+ """Base callback handler that can be used to handle callbacks from the STORM pipeline."""++ def on_identify_perspective_start(self, **kwargs):+ """Run when the perspective identification starts."""+ pass++ def on_identify_perspective_end(self, perspectives: list[str], **kwargs):+ """Run when the perspective identification finishes."""+ pass++ def on_information_gathering_start(self, **kwargs):+ """Run when the information gathering starts."""+ pass++ def on_dialogue_turn_end(self, dlg_turn, **kwargs):+ """Run when a question asking and answering turn finishes."""+ pass++ def on_information_gathering_end(self, **kwargs):+ """Run when the information gathering finishes."""+ pass++ def on_information_organization_start(self, **kwargs):+ """Run when the information organization starts."""+ pass++ def on_direct_outline_generation_end(self, outline: str, **kwargs):+ """Run when the direct outline generation finishes."""+ pass++ def on_outline_refinement_end(self, outline: str, **kwargs):+ """Run when the outline refinement finishes."""+ pass\ No newline at end of filediff --git a/src/storm_wiki/modules/internet_source_restrictions.json b/src/storm_wiki/modules/internet_source_restrictions.jsonnew file mode 100644index 0000000..8e168eb--- /dev/null+++ b/src/storm_wiki/modules/internet_source_restrictions.json@@ -0,0 +1,190 @@+{+ "generally_unreliable": [+ "112_Ukraine",+ "Ad_Fontes_Media",+ "AlterNet",+ "Amazon",+ "Anadolu_Agency_(controversial_topics)",+ "Ancestry.com",+ "Answers.com",+ "Antiwar.com",+ "arXiv",+ "Atlas_Obscura_places",+ "Bild",+ "Blaze_Media",+ "Blogger",+ "BroadwayWorld",+ "CelebrityNetWorth",+ "CESNUR",+ "ChatGPT",+ "CNET_(November_2022\u2013present)",+ "CoinDesk",+ "Consortium_News",+ "CounterPunch",+ "Cracked.com",+ "Daily_Express",+ "Daily_Kos",+ "Daily_Sabah",+ "The_Daily_Wire",+ "Discogs",+ "Distractify",+ "The_Electronic_Intifada",+ "Encyclopaedia_Metallum",+ "Facebook",+ "FamilySearch",+ "Fandom",+ "The_Federalist",+ "Find_a_Grave",+ "Findmypast",+ "Flags_of_the_World",+ "Flickr",+ "Forbes.com_contributors",+ "Fox_News_(politics_and_science)",+ "Fox_News_(talk_shows)",+ "Gawker",+ "GB_News",+ "Geni.com",+ "gnis-class",+ "gns-class",+ "GlobalSecurity.org",+ "Goodreads",+ "Guido_Fawkes",+ "Heat_Street",+ "History",+ "HuffPost_contributors",+ "IMDb",+ "Independent_Media_Center",+ "Inquisitr",+ "International_Business_Times",+ "Investopedia",+ "Jewish_Virtual_Library",+ "Joshua_Project",+ "Know_Your_Meme",+ "LinkedIn",+ "LiveJournal",+ "Marquis_Who's_Who",+ "Mashable_sponsored_content",+ "MEAWW",+ "Media_Bias/Fact_Check",+ "Media_Research_Center",+ "Medium",+ "metal-experience",+ "Metro",+ "The_New_American",+ "New_York_Post",+ "The_Onion",+ "Our_Campaigns",+ "PanAm_Post",+ "Patheos",+ "An_Phoblacht",+ "The_Post_Millennial",+ "PR_Newswire",+ "Quadrant",+ "Quillette",+ "Quora",+ "Raw_Story",+ "Reddit",+ "RedState",+ "ResearchGate",+ "Rolling_Stone_(politics_and_society,_2011\u2013present)",+ "Rolling_Stone_(Culture_Council)",+ "Scribd",+ "Scriptural_texts",+ "Sixth_Tone_(politics)",+ "The_Skwawkbox",+ "SourceWatch",+ "Spirit_of_Metal",+ "Sportskeeda",+ "Stack_Exchange",+ "starsunfolded.com",+ "Statista",+ "TASS",+ "The_Truth_About_Guns",+ "TV.com",+ "TV_Tropes",+ "Twitter",+ "Urban_Dictionary",+ "Venezuelanalysis",+ "VGChartz",+ "VoC",+ "Weather2Travel",+ "The_Western_Journal",+ "We_Got_This_Covered",+ "WhatCulture",+ "Who's_Who_(UK)",+ "WhoSampled",+ "Wikidata",+ "WikiLeaks",+ "Wikinews",+ "Wikipedia",+ "WordPress.com",+ "Worldometer",+ "YouTube"+ ],+ "deprecated": [+ "ANNA_News",+ "Baidu_Baike",+ "China_Global_Television_Network",+ "Crunchbase",+ "The_Daily_Caller",+ "Daily_Mail",+ "Daily_Star",+ "The_Epoch_Times",+ "FrontPage_Magazine",+ "The_Gateway_Pundit",+ "Global_Times",+ "The_Grayzone",+ "HispanTV",+ "Jihad_Watch",+ "Last.fm",+ "LifeSiteNews",+ "The_Mail_on_Sunday",+ "MintPress_News",+ "National_Enquirer",+ "New_Eastern_Outlook",+ "News_Break",+ "NewsBlaze",+ "News_of_the_World",+ "Newsmax",+ "NNDB",+ "Occupy_Democrats",+ "One_America_News_Network",+ "Peerage_websites",+ "Press_TV",+ "Project_Veritas",+ "Rate_Your_Music",+ "Republic_TV",+ "Royal_Central",+ "RT",+ "Sputnik",+ "The_Sun",+ "Taki's_Magazine",+ "Telesur",+ "The_Unz_Review",+ "VDARE",+ "Voltaire_Network",+ "WorldNetDaily",+ "Zero_Hedge"+ ],+ "blacklisted": [+ "Advameg",+ "bestgore.com",+ "Breitbart_News",+ "Centre_for_Research_on_Globalization",+ "Examiner.com",+ "Famous_Birthdays",+ "Healthline",+ "InfoWars",+ "Lenta.ru",+ "LiveLeak",+ "Lulu.com",+ "MyLife",+ "Natural_News",+ "OpIndia",+ "The_Points_Guy",+ "The_Points_Guy_(sponsored_content)",+ "Swarajya",+ "Veterans_Today",+ "ZoomInfo"+ ]+}\ No newline at end of filediff --git a/src/storm_wiki/modules/knowledge_curation.py b/src/storm_wiki/modules/knowledge_curation.pynew file mode 100644index 0000000..16f0de3--- /dev/null+++ b/src/storm_wiki/modules/knowledge_curation.py@@ -0,0 +1,312 @@+import concurrent.futures+import logging+import os+from concurrent.futures import as_completed+from typing import Union, List, Tuple, Optional, Dict++import dspy+from interface import KnowledgeCurationModule, Retriever+from storm_wiki.modules.callback import BaseCallbackHandler+from storm_wiki.modules.persona_generator import StormPersonaGenerator+from storm_wiki.modules.storm_dataclass import DialogueTurn, StormInformationTable, StormInformation+from streamlit.runtime.scriptrunner import add_script_run_ctx+from utils import ArticleTextProcessing++script_dir = os.path.dirname(os.path.abspath(__file__))+++class ConvSimulator(dspy.Module):+ """Simulate a conversation between a Wikipedia writer with specific persona and an expert."""++ def __init__(self, topic_expert_engine: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ question_asker_engine: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ retriever: Retriever, max_search_queries_per_turn: int, search_top_k: int, max_turn: int):+ super().__init__()+ self.wiki_writer = WikiWriter(engine=question_asker_engine)+ self.topic_expert = TopicExpert(+ engine=topic_expert_engine,+ max_search_queries=max_search_queries_per_turn,+ search_top_k=search_top_k,+ retriever=retriever+ )+ self.max_turn = max_turn++ def forward(self, topic: str, persona: str, ground_truth_url: str, callback_handler: BaseCallbackHandler):+ """+ topic: The topic to research.+ persona: The persona of the Wikipedia writer.+ ground_truth_url: The ground_truth_url will be excluded from search to avoid ground truth leakage in evaluation.+ """+ dlg_history: List[DialogueTurn] = []+ for _ in range(self.max_turn):+ user_utterance = self.wiki_writer(topic=topic, persona=persona, dialogue_turns=dlg_history).question+ if user_utterance == '':+ logging.error('Simulated Wikipedia writer utterance is empty.')+ break+ if user_utterance.startswith('Thank you so much for your help!'):+ break+ expert_output = self.topic_expert(topic=topic, question=user_utterance, ground_truth_url=ground_truth_url)+ dlg_turn = DialogueTurn(+ agent_utterance=expert_output.answer,+ user_utterance=user_utterance,+ search_queries=expert_output.queries,+ search_results=expert_output.searched_results+ )+ dlg_history.append(dlg_turn)+ callback_handler.on_dialogue_turn_end(dlg_turn=dlg_turn)++ return dspy.Prediction(dlg_history=dlg_history)+++class WikiWriter(dspy.Module):+ """Perspective-guided question asking in conversational setup.++ The asked question will be used to start a next round of information seeking."""++ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ super().__init__()+ self.ask_question_with_persona = dspy.ChainOfThought(AskQuestionWithPersona)+ self.ask_question = dspy.ChainOfThought(AskQuestion)+ self.engine = engine++ def forward(self, topic: str, persona: str, dialogue_turns: List[DialogueTurn], draft_page=None):+ conv = []+ for turn in dialogue_turns[:-4]:+ conv.append(f'You: {turn.user_utterance}\nExpert: Omit the answer here due to space limit.')+ for turn in dialogue_turns[-4:]:+ conv.append(+ f'You: {turn.user_utterance}\nExpert: {ArticleTextProcessing.remove_citations(turn.agent_utterance)}')+ conv = '\n'.join(conv)+ conv = conv.strip() or 'N/A'+ conv = ArticleTextProcessing.limit_word_count_preserve_newline(conv, 2500)++ with dspy.settings.context(lm=self.engine):+ if persona is not None and len(persona.strip()) > 0:+ question = self.ask_question_with_persona(topic=topic, persona=persona, conv=conv).question+ else:+ question = self.ask_question(topic=topic, persona=persona, conv=conv).question++ return dspy.Prediction(question=question)+++class AskQuestion(dspy.Signature):+ """You are an experienced Wikipedia writer. You are chatting with an expert to get information for the topic you want to contribute. Ask good questions to get more useful information relevant to the topic.+ When you have no more question to ask, say "Thank you so much for your help!" to end the conversation.+ Please only ask a question at a time and don't ask what you have asked before. Your questions should be related to the topic you want to write."""++ topic = dspy.InputField(prefix='Topic you want to write: ', format=str)+ conv = dspy.InputField(prefix='Conversation history:\n', format=str)+ question = dspy.OutputField(format=str)+++class AskQuestionWithPersona(dspy.Signature):+ """You are an experienced Wikipedia writer and want to edit a specific page. Besides your identity as a Wikipedia writer, you have specific focus when researching the topic.+ Now, you are chatting with an expert to get information. Ask good questions to get more useful information.+ When you have no more question to ask, say "Thank you so much for your help!" to end the conversation.+ Please only ask a question at a time and don't ask what you have asked before. Your questions should be related to the topic you want to write."""++ topic = dspy.InputField(prefix='Topic you want to write: ', format=str)+ persona = dspy.InputField(prefix='Your persona besides being a Wikipedia writer: ', format=str)+ conv = dspy.InputField(prefix='Conversation history:\n', format=str)+ question = dspy.OutputField(format=str)+++class QuestionToQuery(dspy.Signature):+ """You want to answer the question using Google search. What do you type in the search box?+ Write the queries you will use in the following format:+ - query 1+ - query 2+ ...+ - query n"""++ topic = dspy.InputField(prefix='Topic you are discussing about: ', format=str)+ question = dspy.InputField(prefix='Question you want to answer: ', format=str)+ queries = dspy.OutputField(format=str)+++class AnswerQuestion(dspy.Signature):+ """You are an expert who can use information effectively. You are chatting with a Wikipedia writer who wants to write a Wikipedia page on topic you know. You have gathered the related information and will now use the information to form a response.+ Make your response as informative as possible and make sure every sentence is supported by the gathered information. If [Gathered information] is not related to he [Topic] and [Question], output "Sorry, I don't have enough information to answer the question."."""++ topic = dspy.InputField(prefix='Topic you are discussing about:', format=str)+ conv = dspy.InputField(prefix='Question:\n', format=str)+ info = dspy.InputField(+ prefix='Gathered information:\n', format=str)+ answer = dspy.OutputField(+ prefix='Now give your response. (Try to use as many different sources as possible and add do not hallucinate.)\n',+ format=str+ )+++class TopicExpert(dspy.Module):+ """Answer questions using search-based retrieval and answer generation. This module conducts the following steps:+ 1. Generate queries from the question.+ 2. Search for information using the queries.+ 3. Filter out unreliable sources.+ 4. Generate an answer using the retrieved information.+ """++ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ max_search_queries: int, search_top_k: int, retriever: Retriever):+ super().__init__()+ self.generate_queries = dspy.Predict(QuestionToQuery)+ self.retriever = retriever+ self.retriever.update_search_top_k(search_top_k)+ self.answer_question = dspy.Predict(AnswerQuestion)+ self.engine = engine+ self.max_search_queries = max_search_queries+ self.search_top_k = search_top_k++ def forward(self, topic: str, question: str, ground_truth_url: str):+ with dspy.settings.context(lm=self.engine):+ # Identify: Break down question into queries.+ queries = self.generate_queries(topic=topic, question=question).queries+ queries = [q.replace('-', '').strip().strip('"').strip('"').strip() for q in queries.split('\n')]+ queries = queries[:self.max_search_queries]+ # Search+ searched_results: List[StormInformation] = self.retriever.retrieve(list(set(queries)),+ exclude_urls=[ground_truth_url])+ if len(searched_results) > 0:+ # Evaluate: Simplify this part by directly using the top 1 snippet.+ info = ''+ for n, r in enumerate(searched_results):+ info += '\n'.join(f'[{n + 1}]: {s}' for s in r.snippets[:1])+ info += '\n\n'++ info = ArticleTextProcessing.limit_word_count_preserve_newline(info, 1000)++ try:+ answer = self.answer_question(topic=topic, conv=question, info=info).answer+ answer = ArticleTextProcessing.remove_uncompleted_sentences_with_citations(answer)+ except Exception as e:+ logging.error(f'Error occurs when generating answer: {e}')+ answer = 'Sorry, I cannot answer this question. Please ask another question.'+ else:+ # When no information is found, the expert shouldn't hallucinate.+ answer = 'Sorry, I cannot find information for this question. Please ask another question.'++ return dspy.Prediction(queries=queries, searched_results=searched_results, answer=answer)+++class StormKnowledgeCurationModule(KnowledgeCurationModule):+ """+ The interface for knowledge curation stage. Given topic, return collected information.+ """++ def __init__(self,+ retriever: Retriever,+ persona_generator: Optional[StormPersonaGenerator],+ conv_simulator_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ question_asker_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ max_search_queries_per_turn: int,+ search_top_k: int,+ max_conv_turn: int,+ max_thread_num: int):+ """+ Store args and finish initialization.+ """+ self.retriever = retriever+ self.persona_generator = persona_generator+ self.conv_simulator_lm = conv_simulator_lm+ self.search_top_k = search_top_k+ self.max_thread_num = max_thread_num+ self.retriever = retriever+ self.conv_simulator = ConvSimulator(+ topic_expert_engine=conv_simulator_lm,+ question_asker_engine=question_asker_lm,+ retriever=retriever,+ max_search_queries_per_turn=max_search_queries_per_turn,+ search_top_k=search_top_k,+ max_turn=max_conv_turn+ )++ def _get_considered_personas(self, topic: str, max_num_persona) -> List[str]:+ return self.persona_generator.generate_persona(topic=topic, max_num_persona=max_num_persona)++ def _run_conversation(self, conv_simulator, topic, ground_truth_url, considered_personas,+ callback_handler: BaseCallbackHandler) -> List[Tuple[str, List[DialogueTurn]]]:+ """+ Executes multiple conversation simulations concurrently, each with a different persona,+ and collects their dialog histories. The dialog history of each conversation is cleaned+ up before being stored.++ Parameters:+ conv_simulator (callable): The function to simulate conversations. It must accept four+ parameters: `topic`, `ground_truth_url`, `persona`, and `callback_handler`, and return+ an object that has a `dlg_history` attribute.+ topic (str): The topic of conversation for the simulations.+ ground_truth_url (str): The URL to the ground truth data related to the conversation topic.+ considered_personas (list): A list of personas under which the conversation simulations+ will be conducted. Each persona is passed to `conv_simulator` individually.+ callback_handler (callable): A callback function that is passed to `conv_simulator`. It+ should handle any callbacks or events during the simulation.++ Returns:+ list of tuples: A list where each tuple contains a persona and its corresponding cleaned+ dialog history (`dlg_history`) from the conversation simulation.+ """++ conversations = []++ def run_conv(persona):+ return conv_simulator(+ topic=topic,+ ground_truth_url=ground_truth_url,+ persona=persona,+ callback_handler=callback_handler+ )++ max_workers = min(self.max_thread_num, len(considered_personas))++ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:+ future_to_persona = {executor.submit(run_conv, persona): persona for persona in considered_personas}+ # Ensure the logging context is correct when connecting with Streamlit frontend.+ for t in executor._threads:+ add_script_run_ctx(t)++ for future in as_completed(future_to_persona):+ persona = future_to_persona[future]+ conv = future.result()+ conversations.append((persona, ArticleTextProcessing.clean_up_citation(conv).dlg_history))++ return conversations++ def research(self,+ topic: str,+ ground_truth_url: str,+ callback_handler: BaseCallbackHandler,+ max_perspective: int = 0,+ disable_perspective: bool = True,+ return_conversation_log=False) -> Union[StormInformationTable, Tuple[StormInformationTable, Dict]]:+ """+ Curate information and knowledge for the given topic++ Args:+ topic: topic of interest in natural language.++ Returns:+ collected_information: collected information in InformationTable type.+ """++ # identify personas+ callback_handler.on_identify_perspective_start()+ considered_personas = []+ if disable_perspective:+ considered_personas = [""]+ else:+ considered_personas = self._get_considered_personas(topic=topic, max_num_persona=max_perspective)+ callback_handler.on_identify_perspective_end(perspectives=considered_personas)++ # run conversation+ callback_handler.on_information_gathering_start()+ conversations = self._run_conversation(conv_simulator=self.conv_simulator,+ topic=topic,+ ground_truth_url=ground_truth_url,+ considered_personas=considered_personas,+ callback_handler=callback_handler)++ information_table = StormInformationTable(conversations)+ callback_handler.on_information_gathering_end()+ if return_conversation_log:+ return information_table, StormInformationTable.construct_log_dict(conversations)+ return information_tablediff --git a/src/storm_wiki/modules/outline_generation.py b/src/storm_wiki/modules/outline_generation.pynew file mode 100644index 0000000..2b09d52--- /dev/null+++ b/src/storm_wiki/modules/outline_generation.py@@ -0,0 +1,138 @@+from typing import Union, Optional, Tuple++import dspy+from interface import OutlineGenerationModule+from storm_wiki.modules.callback import BaseCallbackHandler+from storm_wiki.modules.storm_dataclass import StormInformationTable, StormArticle+from utils import ArticleTextProcessing+++class StormOutlineGenerationModule(OutlineGenerationModule):+ """+ The interface for outline generation stage. Given topic, collected information from knowledge+ curation stage, generate outline for the article.+ """++ def __init__(self,+ outline_gen_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ super().__init__()+ self.outline_gen_lm = outline_gen_lm+ self.write_outline = WriteOutline(engine=self.outline_gen_lm)++ def generate_outline(self,+ topic: str,+ information_table: StormInformationTable,+ old_outline: Optional[StormArticle] = None,+ callback_handler: BaseCallbackHandler = None,+ return_draft_outline=False) -> Union[StormArticle, Tuple[StormArticle, StormArticle]]:+ """+ Generates an outline for an article based on the specified topic and the information+ gathered during the knowledge curation stage. This method can optionally return both the+ final article outline and a draft outline if required.++ Args:+ topic (str): The topic of the article.+ information_table (StormInformationTable): The information table containing the collected information.+ old_outline (Optional[StormArticle]): An optional previous version of the article outline that can+ be used for reference or comparison. Defaults to None.+ callback_handler (BaseCallbackHandler): An optional callback handler that can be used to trigger+ custom callbacks at various stages of the outline generation process, such as when the information+ organization starts. Defaults to None.+ return_draft_outline (bool): A flag indicating whether the method should return both the final article+ outline and a draft version of the outline. If False, only the final article outline is returned.+ Defaults to False.++ Returns:+ Union[StormArticle, Tuple[StormArticle, StormArticle]]: Depending on the value of `return_draft_outline`,+ this method returns either a single `StormArticle` object containing the final outline or a tuple of+ two `StormArticle` objects, the first containing the final outline and the second containing the+ draft outline.+ """+ if callback_handler is not None:+ callback_handler.on_information_organization_start()++ concatenated_dialogue_turns = sum([conv for (_, conv) in information_table.conversations], [])+ result = self.write_outline(topic=topic, dlg_history=concatenated_dialogue_turns,+ callback_handler=callback_handler)+ article_with_outline_only = StormArticle.from_outline_str(topic=topic, outline_str=result.outline)+ article_with_draft_outline_only = StormArticle.from_outline_str(topic=topic,+ outline_str=result.old_outline)+ if not return_draft_outline:+ return article_with_outline_only+ return article_with_outline_only, article_with_draft_outline_only+++class WriteOutline(dspy.Module):+ """Generate the outline for the Wikipedia page."""++ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ super().__init__()+ self.draft_page_outline = dspy.Predict(WritePageOutline)+ self.write_page_outline = dspy.Predict(WritePageOutlineFromConv)+ self.engine = engine++ def forward(self, topic: str, dlg_history, old_outline: Optional[str] = None,+ callback_handler: BaseCallbackHandler = None):+ trimmed_dlg_history = []+ for turn in dlg_history:+ if 'topic you' in turn.agent_utterance.lower() or 'topic you' in turn.user_utterance.lower():+ continue+ trimmed_dlg_history.append(turn)+ conv = '\n'.join([f'Wikipedia Writer: {turn.user_utterance}\nExpert: {turn.agent_utterance}' for turn in+ trimmed_dlg_history])+ conv = ArticleTextProcessing.remove_citations(conv)+ conv = ArticleTextProcessing.limit_word_count_preserve_newline(conv, 5000)++ with dspy.settings.context(lm=self.engine):+ if old_outline is None:+ old_outline = ArticleTextProcessing.clean_up_outline(self.draft_page_outline(topic=topic).outline)+ if callback_handler:+ callback_handler.on_direct_outline_generation_end(outline=old_outline)+ outline = ArticleTextProcessing.clean_up_outline(+ self.write_page_outline(topic=topic, old_outline=old_outline, conv=conv).outline)+ if callback_handler:+ callback_handler.on_outline_refinement_end(outline=outline)++ return dspy.Prediction(outline=outline, old_outline=old_outline)+++class WritePageOutline(dspy.Signature):+ """Write an outline for a Wikipedia page.+ Here is the format of your writing:+ 1. Use "#" Title" to indicate section title, "##" Title" to indicate subsection title, "###" Title" to indicate subsubsection title, and so on.+ 2. Do not include other information.+ 3. Do not include topic name itself in the outline.+ """++ topic = dspy.InputField(prefix="The topic you want to write: ", format=str)+ outline = dspy.OutputField(prefix="Write the Wikipedia page outline:\n", format=str)+++class NaiveOutlineGen(dspy.Module):+ """Generate the outline with LLM's parametric knowledge directly."""++ def __init__(self):+ super().__init__()+ self.write_outline = dspy.Predict(WritePageOutline)++ def forward(self, topic: str):+ outline = self.write_outline(topic=topic).outline++ return dspy.Prediction(outline=outline)+++class WritePageOutlineFromConv(dspy.Signature):+ """Improve an outline for a Wikipedia page. You already have a draft outline that covers the general information. Now you want to improve it based on the information learned from an information-seeking conversation to make it more informative.+ Here is the format of your writing:+ 1. Use "#" Title" to indicate section title, "##" Title" to indicate subsection title, "###" Title" to indicate subsubsection title, and so on.+ 2. Do not include other information.+ 3. Do not include topic name itself in the outline.+ """++ topic = dspy.InputField(prefix="The topic you want to write: ", format=str)+ conv = dspy.InputField(prefix="Conversation history:\n", format=str)+ old_outline = dspy.OutputField(prefix="Current outline:\n", format=str)+ outline = dspy.OutputField(+ prefix='Write the Wikipedia page outline (Use "#" Title" to indicate section title, "##" Title" to indicate subsection title, ...):\n',+ format=str+ )diff --git a/src/storm_wiki/modules/persona_generator.py b/src/storm_wiki/modules/persona_generator.pynew file mode 100644index 0000000..5150e31--- /dev/null+++ b/src/storm_wiki/modules/persona_generator.py@@ -0,0 +1,138 @@+import logging+import re+from typing import Union, List++import dspy+import requests+from bs4 import BeautifulSoup+++def get_wiki_page_title_and_toc(url):+ """Get the main title and table of contents from an url of a Wikipedia page."""++ response = requests.get(url)+ soup = BeautifulSoup(response.content, 'html.parser')++ # Get the main title from the first h1 tag+ main_title = soup.find('h1').text.replace('[edit]', '').strip().replace('\xa0', ' ')++ toc = ""+ levels = []+ excluded_sections = {'Contents', 'See also', 'Notes', 'References', 'External links'}++ # Start processing from h2 to exclude the main title from TOC+ for header in soup.find_all(['h2', 'h3', "h4", "h5", "h6"]):+ level = int(header.name[1]) # Extract the numeric part of the header tag (e.g., '2' from 'h2')+ section_title = header.text.replace('[edit]', '').strip().replace('\xa0', ' ')+ if section_title in excluded_sections:+ continue++ while levels and level <= levels[-1]:+ levels.pop()+ levels.append(level)++ indentation = " " * (len(levels) - 1)+ toc += f"{indentation}{section_title}\n"++ return main_title, toc.strip()+++class FindRelatedTopic(dspy.Signature):+ """I'm writing a Wikipedia page for a topic mentioned below. Please identify and recommend some Wikipedia pages on closely related subjects. I'm looking for examples that provide insights into interesting aspects commonly associated with this topic, or examples that help me understand the typical content and structure included in Wikipedia pages for similar topics.+ Please list the urls in separate lines."""++ topic = dspy.InputField(prefix='Topic of interest:', format=str)+ related_topics = dspy.OutputField(format=str)+++class GenPersona(dspy.Signature):+ """You need to select a group of Wikipedia editors who will work together to create a comprehensive article on the topic. Each of them represents a different perspective, role, or affiliation related to this topic. You can use other Wikipedia pages of related topics for inspiration. For each editor, add a description of what they will focus on.+ Give your answer in the following format: 1. short summary of editor 1: description\n2. short summary of editor 2: description\n...+ """++ topic = dspy.InputField(prefix='Topic of interest:', format=str)+ examples = dspy.InputField(prefix='Wiki page outlines of related topics for inspiration:\n', format=str)+ personas = dspy.OutputField(format=str)+++class CreateWriterWithPersona(dspy.Module):+ """Discover different perspectives of researching the topic by reading Wikipedia pages of related topics."""++ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ super().__init__()+ self.find_related_topic = dspy.ChainOfThought(FindRelatedTopic)+ self.gen_persona = dspy.ChainOfThought(GenPersona)+ self.engine = engine++ def forward(self, topic: str, draft=None):+ with dspy.settings.context(lm=self.engine):+ # Get section names from wiki pages of relevant topics for inspiration.+ related_topics = self.find_related_topic(topic=topic).related_topics+ urls = []+ for s in related_topics.split('\n'):+ if 'http' in s:+ urls.append(s[s.find('http'):])+ examples = []+ for url in urls:+ try:+ title, toc = get_wiki_page_title_and_toc(url)+ examples.append(f'Title: {title}\nTable of Contents: {toc}')+ except Exception as e:+ logging.error(f'Error occurs when processing {url}: {e}')+ continue+ if len(examples) == 0:+ examples.append('N/A')+ gen_persona_output = self.gen_persona(topic=topic, examples='\n----------\n'.join(examples)).personas++ personas = []+ for s in gen_persona_output.split('\n'):+ match = re.search(r'\d+\.\s*(.*)', s)+ if match:+ personas.append(match.group(1))++ sorted_personas = personas++ return dspy.Prediction(personas=personas, raw_personas_output=sorted_personas, related_topics=related_topics)+++class StormPersonaGenerator():+ """+ A generator class for creating personas based on a given topic.++ This class uses an underlying engine to generate personas tailored to the specified topic.+ The generator integrates with a `CreateWriterWithPersona` instance to create diverse personas,+ including a default 'Basic fact writer' persona.++ Attributes:+ create_writer_with_persona (CreateWriterWithPersona): An instance responsible for+ generating personas based on the provided engine and topic.++ Args:+ engine (Union[dspy.dsp.LM, dspy.dsp.HFModel]): The underlying engine used for generating+ personas. It must be an instance of either `dspy.dsp.LM` or `dspy.dsp.HFModel`.+ """++ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.create_writer_with_persona = CreateWriterWithPersona(engine=engine)++ def generate_persona(self, topic: str, max_num_persona: int = 3) -> List[str]:+ """+ Generates a list of personas based on the provided topic, up to a maximum number specified.++ This method first creates personas using the underlying `create_writer_with_persona` instance+ and then prepends a default 'Basic fact writer' persona to the list before returning it.+ The number of personas returned is limited to `max_num_persona`, excluding the default persona.++ Args:+ topic (str): The topic for which personas are to be generated.+ max_num_persona (int): The maximum number of personas to generate, excluding the+ default 'Basic fact writer' persona.++ Returns:+ List[str]: A list of persona descriptions, including the default 'Basic fact writer' persona+ and up to `max_num_persona` additional personas generated based on the topic.+ """+ personas = self.create_writer_with_persona(topic=topic)+ default_persona = 'Basic fact writer: Basic fact writer focusing on broadly covering the basic facts about the topic.'+ considered_personas = [default_persona] + personas.personas[:max_num_persona]+ return considered_personasdiff --git a/src/storm_wiki/modules/retriever.py b/src/storm_wiki/modules/retriever.pynew file mode 100644index 0000000..e448d74--- /dev/null+++ b/src/storm_wiki/modules/retriever.py@@ -0,0 +1,43 @@+import json+import os+from typing import Union, List+from urllib.parse import urlparse++import storm_wiki.modules.storm_dataclass as storm_dataclass+from interface import Retriever, Information+from rm import YouRM+from utils import ArticleTextProcessing++SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))+with open(os.path.join(SCRIPT_DIR, 'internet_source_restrictions.json')) as f:+ domain_restriction_dict = json.load(f)+ GENERALLY_UNRELIABLE = set(domain_restriction_dict["generally_unreliable"])+ DEPRECATED = set(domain_restriction_dict["deprecated"])+ BLACKLISTED = set(domain_restriction_dict["blacklisted"])+++def is_valid_wikipedia_source(url):+ parsed_url = urlparse(url)+ # Check if the URL is from a reliable domain+ combined_set = GENERALLY_UNRELIABLE | DEPRECATED | BLACKLISTED+ for domain in combined_set:+ if domain in parsed_url.netloc:+ return False++ return True+++class StormRetriever(Retriever):+ def __init__(self, ydc_api_key=None, k=3):+ super().__init__(search_top_k=k)+ self.you_rm = YouRM(ydc_api_key=ydc_api_key, k=self.search_top_k, is_valid_source=is_valid_wikipedia_source)++ def retrieve(self, query: Union[str, List[str]], exclude_urls: List[str] = []) -> List[Information]:+ self.you_rm.k = self.search_top_k+ retrieved_data_list = self.you_rm(query_or_queries=query, exclude_urls=exclude_urls)+ for data in retrieved_data_list:+ for i in range(len(data['snippets'])):+ # STORM generate the article with citations. We do not consider multi-hop citations.+ # Remove citations in the source to avoid confusion.+ data['snippets'][i] = ArticleTextProcessing.remove_citations(data['snippets'][i])+ return [storm_dataclass.StormInformation.from_dict(data) for data in retrieved_data_list]diff --git a/src/storm_wiki/modules/storm_dataclass.py b/src/storm_wiki/modules/storm_dataclass.pynew file mode 100644index 0000000..d75760c--- /dev/null+++ b/src/storm_wiki/modules/storm_dataclass.py@@ -0,0 +1,495 @@+import copy+import re+from collections import OrderedDict+from typing import Union, Optional, Any, List, Tuple, Dict++import numpy as np+from interface import Information, InformationTable, Article, ArticleSectionNode+from sentence_transformers import SentenceTransformer+from sklearn.metrics.pairwise import cosine_similarity+from utils import ArticleTextProcessing, FileIOHelper+++class StormInformation(Information):+ """Class to represent detailed information.++ Inherits from Information to include a unique identifier (URL), and extends+ it with a description, snippets, and title of the storm information.++ Attributes:+ description (str): Brief description.+ snippets (list): List of brief excerpts or snippets.+ title (str): The title or headline of the information.+ url (str): The unique URL (serving as UUID) of the information.+ """++ def __init__(self, uuid, description, snippets, title):+ """Initialize the StormInformation object with detailed attributes.++ Args:+ uuid (str): The unique URL serving as the identifier for the information.+ description (str): Detailed description.+ snippets (list): List of brief excerpts or snippet.+ title (str): The title or headline of the information.+ """+ super().__init__(uuid=uuid, meta={})+ self.description = description+ self.snippets = snippets+ self.title = title+ self.url = self.uuid++ @classmethod+ def from_dict(cls, info_dict):+ """Create a StormInformation object from a dictionary.+ Usage: storm_info = StormInformation.from_dict(storm_info_dict)++ Args:+ info_dict (dict): A dictionary containing keys 'uuid', 'description',+ 'snippets', and 'title' corresponding to the object's attributes.++ Returns:+ StormInformation: An instance of StormInformation.+ """+ return cls(info_dict['url'], info_dict['description'], info_dict['snippets'], info_dict['title'])++ def to_dict(self):+ return {"url": self.uuid,+ "description": self.description,+ "snippets": self.snippets,+ "title": self.title}+++class DialogueTurn:+ def __init__(+ self,+ agent_utterance: str = None,+ user_utterance: str = None,+ search_queries: Optional[List[str]] = None,+ search_results: Optional[List[Union[StormInformation, Dict]]] = None+ ):+ self.agent_utterance = agent_utterance+ self.user_utterance = user_utterance+ self.search_queries = search_queries+ self.search_results = search_results++ if self.search_results:+ for idx in range(len(self.search_results)):+ if type(self.search_results[idx]) == dict:+ self.search_results[idx] = StormInformation.from_dict(self.search_results[idx])++ def log(self):+ """+ Returns a json object that contains all information inside `self`+ """++ return OrderedDict(+ {+ 'agent_utterance': self.agent_utterance,+ 'user_utterance': self.user_utterance,+ 'search_queries': self.search_queries,+ 'search_results': [data.to_dict() for data in self.search_results],+ }+ )+++class StormInformationTable(InformationTable):+ """+ The InformationTable class serves as data class to store the information+ collected during KnowledgeCuration stage.++ Create subclass to incorporate more information as needed. For example,+ in STORM paper https://arxiv.org/pdf/2402.14207.pdf, additional information+ would be perspective guided dialogue history.+ """++ def __init__(self, conversations=List[Tuple[str, List[DialogueTurn]]]):+ super().__init__()+ self.conversations = conversations+ self.url_to_info: Dict[str, StormInformation] = StormInformationTable.construct_url_to_info(self.conversations)++ @staticmethod+ def construct_url_to_info(conversations: List[Tuple[str, List[DialogueTurn]]]) -> Dict[str, StormInformation]:+ url_to_info = {}++ for (persona, conv) in conversations:+ for turn in conv:+ for storm_info in turn.search_results:+ if storm_info.url in url_to_info:+ url_to_info[storm_info.url].snippets.extend(storm_info.snippets)+ else:+ url_to_info[storm_info.url] = storm_info+ for url in url_to_info:+ url_to_info[url].snippets = list(set(url_to_info[url].snippets))+ return url_to_info++ @staticmethod+ def construct_log_dict(conversations: List[Tuple[str, List[DialogueTurn]]]) -> List[Dict[str, Union[str, Any]]]:+ conversation_log = []+ for (persona, conv) in conversations:+ conversation_log.append(+ {+ 'perspective': persona,+ 'dlg_turns': [turn.log() for turn in conv]+ }+ )+ return conversation_log++ def dump_url_to_info(self, path):+ url_to_info = copy.deepcopy(self.url_to_info)+ for url in url_to_info:+ url_to_info[url] = url_to_info[url].to_dict()+ FileIOHelper.dump_json(url_to_info, path)++ @classmethod+ def from_conversation_log_file(cls, path):+ conversation_log_data = FileIOHelper.load_json(path)+ conversations = []+ for item in conversation_log_data:+ dialogue_turns = [DialogueTurn(**turn) for turn in item['dlg_turns']]+ persona = item['perspective']+ conversations.append((persona, dialogue_turns))+ return cls(conversations)++ def prepare_table_for_retrieval(self):+ self.encoder = SentenceTransformer('paraphrase-MiniLM-L6-v2')+ self.collected_urls = []+ self.collected_snippets = []+ for url, information in self.url_to_info.items():+ for snippet in information.snippets:+ self.collected_urls.append(url)+ self.collected_snippets.append(snippet)+ self.encoded_snippets = self.encoder.encode(self.collected_snippets, show_progress_bar=False)++ def retrieve_information(self, queries: Union[List[str], str], search_top_k) -> List[StormInformation]:+ selected_urls = []+ selected_snippets = []+ if type(queries) is str:+ queries = [queries]+ for query in queries:+ encoded_query = self.encoder.encode(query, show_progress_bar=False)+ sim = cosine_similarity([encoded_query], self.encoded_snippets)[0]+ sorted_indices = np.argsort(sim)+ for i in sorted_indices[-search_top_k:][::-1]:+ selected_urls.append(self.collected_urls[i])+ selected_snippets.append(self.collected_snippets[i])++ url_to_snippets = {}+ for url, snippet in zip(selected_urls, selected_snippets):+ if url not in url_to_snippets:+ url_to_snippets[url] = set()+ url_to_snippets[url].add(snippet)++ selected_url_to_info = {}+ for url in url_to_snippets:+ selected_url_to_info[url] = copy.deepcopy(self.url_to_info[url])+ selected_url_to_info[url].snippets = list(url_to_snippets[url])++ return list(selected_url_to_info.values())+++class StormArticle(Article):+ def __init__(self, topic_name):+ super().__init__(topic_name=topic_name)+ self.reference = {+ "url_to_unified_index": {},+ "url_to_info": {}+ }++ def find_section(self, node: ArticleSectionNode, name: str) -> Optional[ArticleSectionNode]:+ """+ Return the node of the section given the section name.++ Args:+ node: the node as the root to find.+ name: the name of node as section name++ Return:+ reference of the node or None if section name has no match+ """+ if node.section_name == name:+ return node+ for child in node.children:+ result = self.find_section(child, name)+ if result:+ return result+ return None++ def _merge_new_info_to_references(self, new_info_list: List[StormInformation], index_to_keep=None) -> Dict[+ int, int]:+ """+ Merges new storm information into existing references and updates the citation index mapping.++ Args:+ new_info_list (List[StormInformation]): A list of dictionaries representing new storm information.+ index_to_keep (List[int]): A list of index of the new_info_list to keep. If none, keep all.++ Returns:+ Dict[int, int]: A dictionary mapping the index of each storm information piece in the input list+ to its unified citation index in the references.+ """+ citation_idx_mapping = {}+ for idx, storm_info in enumerate(new_info_list):+ if index_to_keep is not None and idx not in index_to_keep:+ continue+ url = storm_info.url+ if url not in self.reference["url_to_unified_index"]:+ self.reference["url_to_unified_index"][url] = len(+ self.reference["url_to_unified_index"]) + 1 # The citation index starts from 1.+ self.reference["url_to_info"][url] = storm_info+ else:+ existing_snippets = self.reference["url_to_info"][url].snippets+ existing_snippets.extend(storm_info.snippets)+ self.reference["url_to_info"][url].snippets = list(set(existing_snippets))+ citation_idx_mapping[idx + 1] = self.reference["url_to_unified_index"][+ url] # The citation index starts from 1.+ return citation_idx_mapping++ def insert_or_create_section(self, article_dict: Dict[str, Dict], parent_section_name: str = None,+ trim_children=False):+ parent_node = self.root if parent_section_name is None else self.find_section(self.root, parent_section_name)++ if trim_children:+ section_names = set(article_dict.keys())+ for child in parent_node.children[:]:+ if child.section_name not in section_names:+ parent_node.remove_child(child)++ for section_name, content_dict in article_dict.items():+ current_section_node = self.find_section(parent_node, section_name)+ if current_section_node is None:+ current_section_node = ArticleSectionNode(section_name=section_name,+ content=content_dict["content"].strip())+ insert_to_front = parent_node.section_name == self.root.section_name and current_section_node.section_name == "summary"+ parent_node.add_child(current_section_node, insert_to_front=insert_to_front)+ else:+ current_section_node.content = content_dict["content"].strip()++ self.insert_or_create_section(article_dict=content_dict["subsections"], parent_section_name=section_name,+ trim_children=True)++ def update_section(self,+ current_section_content: str,+ current_section_info_list: List[StormInformation],+ parent_section_name: Optional[str] = None) -> Optional[ArticleSectionNode]:+ """+ Add new section to the article.++ Args:+ current_section_name: new section heading name in string format.+ parent_section_name: under which parent section to add the new one. Default to root.+ current_section_content: optional section content.++ Returns:+ the ArticleSectionNode for current section if successfully created / updated. Otherwise none.+ """++ if current_section_info_list is not None:+ references = set([int(x) for x in re.findall(r'\[(\d+)\]', current_section_content)])+ # for any reference number greater than max number of references, delete the reference+ if len(references) > 0:+ max_ref_num = max(references)+ if max_ref_num > len(current_section_info_list):+ for i in range(len(current_section_info_list), max_ref_num + 1):+ current_section_content = current_section_content.replace(f'[{i}]', '')+ if i in references:+ references.remove(i)+ # for any reference that is not used, trim it from current_section_info_list+ index_to_keep = [i - 1 for i in references]+ citation_mapping = self._merge_new_info_to_references(current_section_info_list, index_to_keep)+ current_section_content = ArticleTextProcessing.update_citation_index(current_section_content,+ citation_mapping)++ if parent_section_name is None:+ parent_section_name = self.root.section_name+ article_dict = ArticleTextProcessing.parse_article_into_dict(current_section_content)+ self.insert_or_create_section(article_dict=article_dict, parent_section_name=parent_section_name,+ trim_children=False)++ def get_outline_as_list(self, root_section_name: Optional[str] = None, add_hashtags: bool = False,+ include_root: bool = True) -> List[str]:+ """+ Get outline of the article as a list.++ Args:+ section_name: get all section names in pre-order travel ordering in the subtree of section_name.+ For example:+ #root+ ##section1+ ###section1.1+ ###section1.2+ ##section2+ article.get_outline_as_list("section1") returns [section1, section1.1, section1.2, section2]++ Returns:+ list of section and subsection names.+ """+ if root_section_name is None:+ section_node = self.root+ else:+ section_node = self.find_section(self.root, root_section_name)+ include_root = include_root or section_node != self.root.section_name+ if section_node is None:+ return []+ result = []++ def preorder_traverse(node, level):+ prefix = "#" * level if add_hashtags else "" # Adjust level if excluding root+ result.append(f"{prefix} {node.section_name}".strip() if add_hashtags else node.section_name)+ for child in node.children:+ preorder_traverse(child, level + 1)++ # Adjust the initial level based on whether root is included and hashtags are added+ if include_root:+ preorder_traverse(section_node, level=1)+ else:+ for child in section_node.children:+ preorder_traverse(child, level=1)+ return result++ def to_string(self) -> str:+ """+ Get outline of the article as a list.++ Returns:+ list of section and subsection names.+ """+ result = []++ def preorder_traverse(node, level):+ prefix = "#" * level+ result.append(f"{prefix} {node.section_name}".strip())+ result.append(node.content)+ for child in node.children:+ preorder_traverse(child, level + 1)++ # Adjust the initial level based on whether root is included and hashtags are added+ for child in self.root.children:+ preorder_traverse(child, level=1)+ result = [i.strip() for i in result if i is not None and i.strip()]+ return "\n\n".join(result)++ def reorder_reference_index(self):+ # pre-order traversal to get order of references appear in the article+ ref_indices = []++ def pre_order_find_index(node):+ if node is not None:+ if node.content is not None and node.content:+ ref_indices.extend(ArticleTextProcessing.parse_citation_indices(node.content))+ for child in node.children:+ pre_order_find_index(child)++ pre_order_find_index(self.root)+ # constrcut index mapping+ ref_index_mapping = {}+ for ref_index in ref_indices:+ if ref_index not in ref_index_mapping:+ ref_index_mapping[ref_index] = len(ref_index_mapping) + 1++ # update content+ def pre_order_update_index(node):+ if node is not None:+ if node.content is not None and node.content:+ node.content = ArticleTextProcessing.update_citation_index(node.content, ref_index_mapping)+ for child in node.children:+ pre_order_update_index(child)++ pre_order_update_index(self.root)+ # update reference+ for url in list(self.reference["url_to_unified_index"]):+ pre_index = self.reference["url_to_unified_index"][url]+ if pre_index not in ref_index_mapping:+ del self.reference["url_to_unified_index"][url]+ else:+ new_index = ref_index_mapping[pre_index]+ self.reference["url_to_unified_index"][url] = new_index++ def get_outline_tree(self):+ def build_tree(node) -> Dict[str, Dict]:+ tree = {}+ for child in node.children:+ tree[child.section_name] = build_tree(child)+ return tree if tree else {}++ return build_tree(self.root)++ def get_first_level_section_names(self) -> List[str]:+ """+ Get first level section names+ """+ return [i.section_name for i in self.root.children]++ @classmethod+ def from_outline_file(cls, topic: str, file_path: str):+ """+ Create StormArticle class instance from outline file.+ """+ outline_str = FileIOHelper.load_str(file_path)+ return StormArticle.from_outline_str(topic=topic, outline_str=outline_str)++ @classmethod+ def from_outline_str(cls, topic: str, outline_str: str):+ """+ Create StormArticle class instance from outline only string.+ """+ lines = []+ try:+ lines = outline_str.split("\n")+ lines = [line.strip() for line in lines if line.strip()]+ except:+ pass++ instance = cls(topic)+ if lines:+ a = lines[0].startswith('#') and lines[0].replace('#', '').strip().lower()+ b = topic.lower().replace("_", " ")+ adjust_level = lines[0].startswith('#') and lines[0].replace('#',+ '').strip().lower() == topic.lower().replace(+ "_", " ")+ if adjust_level:+ lines = lines[1:]+ node_stack = [(0, instance.root)] # Stack to keep track of (level, node)++ for line in lines:+ level = line.count('#') - adjust_level+ section_name = line.replace('#', '').strip()++ if section_name == topic:+ continue++ new_node = ArticleSectionNode(section_name)++ while node_stack and level <= node_stack[-1][0]:+ node_stack.pop()++ node_stack[-1][1].add_child(new_node)+ node_stack.append((level, new_node))+ return instance++ def dump_outline_to_file(self, file_path):+ outline = self.get_outline_as_list(add_hashtags=True, include_root=False)+ FileIOHelper.write_str("\n".join(outline), file_path)++ def dump_reference_to_file(self, file_path):+ reference = copy.deepcopy(self.reference)+ for url in reference["url_to_info"]:+ reference["url_to_info"][url] = reference["url_to_info"][url].to_dict()+ FileIOHelper.dump_json(reference, file_path)++ def dump_article_as_plain_text(self, file_path):+ text = self.to_string()+ FileIOHelper.write_str(text, file_path)++ @classmethod+ def from_string(cls, topic_name: str, article_text: str, references: dict):+ article_dict = ArticleTextProcessing.parse_article_into_dict(article_text)+ article = cls(topic_name=topic_name)+ article.insert_or_create_section(article_dict=article_dict)+ for url in list(references["url_to_info"]):+ references["url_to_info"][url] = StormInformation.from_dict(references["url_to_info"][url])+ article.reference = references+ return article++ def post_processing(self):+ self.prune_empty_nodes()+ self.reorder_reference_index()diff --git a/src/utils.py b/src/utils.pynew file mode 100644index 0000000..31abdbd--- /dev/null+++ b/src/utils.py@@ -0,0 +1,323 @@+import json+import os+import pickle+import re+import sys++import toml+++def load_api_key(toml_file_path):+ try:+ with open(toml_file_path, 'r') as file:+ data = toml.load(file)+ except FileNotFoundError:+ print(f"File not found: {toml_file_path}", file=sys.stderr)+ return+ except toml.TomlDecodeError:+ print(f"Error decoding TOML file: {toml_file_path}", file=sys.stderr)+ return+ # Set environment variables+ for key, value in data.items():+ os.environ[key] = str(value)+++class ArticleTextProcessing:+ @staticmethod+ def limit_word_count_preserve_newline(input_string, max_word_count):+ """+ Limit the word count of an input string to a specified maximum, while preserving the integrity of complete lines.++ The function truncates the input string at the nearest word that does not exceed the maximum word count,+ ensuring that no partial lines are included in the output. Words are defined as text separated by spaces,+ and lines are defined as text separated by newline characters.++ Args:+ input_string (str): The string to be truncated. This string may contain multiple lines.+ max_word_count (int): The maximum number of words allowed in the truncated string.++ Returns:+ str: The truncated string with word count limited to `max_word_count`, preserving complete lines.+ """++ word_count = 0+ limited_string = ''++ for word in input_string.split('\n'):+ line_words = word.split()+ for lw in line_words:+ if word_count < max_word_count:+ limited_string += lw + ' '+ word_count += 1+ else:+ break+ if word_count >= max_word_count:+ break+ limited_string = limited_string.strip() + '\n'++ return limited_string.strip()++ @staticmethod+ def remove_citations(s):+ """+ Removes all citations from a given string. Citations are assumed to be in the format+ of numbers enclosed in square brackets, such as [1], [2], or [1, 2], etc. This function searches+ for all occurrences of such patterns and removes them, returning the cleaned string.++ Args:+ s (str): The string from which citations are to be removed.++ Returns:+ str: The string with all citation patterns removed.+ """++ return re.sub(r'\[\d+(?:,\s*\d+)*\]', '', s)++ @staticmethod+ def parse_citation_indices(s):+ """+ Extracts citation indexes from the provided content string and returns them as a list of integers.++ Args:+ content (str): The content string containing citations in the format [number].++ Returns:+ List[int]: A list of unique citation indexes extracted from the content, in the order they appear.+ """+ matches = re.findall(r'\[\d+\]', s)+ return [int(index[1:-1]) for index in matches]++ @staticmethod+ def remove_uncompleted_sentences_with_citations(text):+ """+ Removes uncompleted sentences and standalone citations from the input text. Sentences are identified+ by their ending punctuation (.!?), optionally followed by a citation in square brackets (e.g., "[1]").+ Grouped citations (e.g., "[1, 2]") are split into individual ones (e.g., "[1] [2]"). Only text up to+ and including the last complete sentence and its citation is retained.++ Args:+ text (str): The input text from which uncompleted sentences and their citations are to be removed.++ Returns:+ str: The processed string with uncompleted sentences and standalone citations removed, leaving only+ complete sentences and their associated citations if present.+ """++ # Convert citations like [1, 2, 3] to [1][2][3].+ def replace_with_individual_brackets(match):+ numbers = match.group(1).split(', ')+ return ' '.join(f'[{n}]' for n in numbers)++ # Deduplicate and sort individual groups of citations.+ def deduplicate_group(match):+ citations = match.group(0)+ unique_citations = list(set(re.findall(r'\[\d+\]', citations)))+ sorted_citations = sorted(unique_citations, key=lambda x: int(x.strip('[]')))+ # Return the sorted unique citations as a string+ return ''.join(sorted_citations)++ text = re.sub(r'\[([0-9, ]+)\]', replace_with_individual_brackets, text)+ text = re.sub(r'(\[\d+\])+', deduplicate_group, text)++ # Deprecated: Remove sentence without proper ending punctuation and citations.+ # Split the text into sentences (including citations).+ # sentences_with_trailing = re.findall(r'([^.!?]*[.!?].*?)(?=[^.!?]*[.!?]|$)', text)++ # Filter sentences to ensure they end with a punctuation mark and properly formatted citations+ # complete_sentences = []+ # for sentence in sentences_with_trailing:+ # # Check if the sentence ends with properly formatted citations+ # if re.search(r'[.!?]( \[\d+\])*$|^[^.!?]*[.!?]$', sentence.strip()):+ # complete_sentences.append(sentence.strip())++ # combined_sentences = ' '.join(complete_sentences)++ # Check for and append any complete citations that follow the last sentence+ # trailing_citations = re.findall(r'(\[\d+\]) ', text[text.rfind(combined_sentences) + len(combined_sentences):])+ # if trailing_citations:+ # combined_sentences += ' '.join(trailing_citations)++ # Regex pattern to match sentence endings, including optional citation markers.+ eos_pattern = r'([.!?])\s*(\[\d+\])?\s*'+ matches = list(re.finditer(eos_pattern, text))+ if matches:+ last_match = matches[-1]+ text = text[:last_match.end()].strip()++ return text++ @staticmethod+ def clean_up_citation(conv):+ for turn in conv.dlg_history:+ turn.agent_utterance = turn.agent_utterance[:turn.agent_utterance.find('References:')]+ turn.agent_utterance = turn.agent_utterance[:turn.agent_utterance.find('Sources:')]+ turn.agent_utterance = turn.agent_utterance.replace('Answer:', '').strip()+ try:+ max_ref_num = max([int(x) for x in re.findall(r'\[(\d+)\]', turn.agent_utterance)])+ except Exception as e:+ max_ref_num = 0+ if max_ref_num > len(turn.search_results):+ for i in range(len(turn.search_results), max_ref_num + 1):+ turn.agent_utterance = turn.agent_utterance.replace(f'[{i}]', '')+ turn.agent_utterance = ArticleTextProcessing.remove_uncompleted_sentences_with_citations(+ turn.agent_utterance)++ return conv++ @staticmethod+ def clean_up_outline(outline, topic=""):+ output_lines = []+ current_level = 0 # To track the current section level++ for line in outline.split('\n'):+ stripped_line = line.strip()++ if topic != "" and f"# {topic.lower()}" in stripped_line.lower():+ output_lines = []++ # Check if the line is a section header+ if stripped_line.startswith('#'):+ current_level = stripped_line.count('#')+ output_lines.append(stripped_line)+ # Check if the line is a bullet point+ elif stripped_line.startswith('-'):+ subsection_header = '#' * (current_level + 1) + ' ' + stripped_line[1:].strip()+ output_lines.append(subsection_header)++ outline = '\n'.join(output_lines)++ # Remove references.+ outline = re.sub(r"#[#]? See also.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? See Also.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Notes.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? References.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? External links.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? External Links.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Bibliography.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Further reading*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Further Reading*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Summary.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Appendices.*?(?=##|$)", '', outline, flags=re.DOTALL)+ outline = re.sub(r"#[#]? Appendix.*?(?=##|$)", '', outline, flags=re.DOTALL)++ return outline++ @staticmethod+ def clean_up_section(text):+ """Clean up a section:+ 1. Remove uncompleted sentences (usually due to output token limitation).+ 2. Deduplicate individual groups of citations.+ 3. Remove unnecessary summary."""++ paragraphs = text.split('\n')+ output_paragraphs = []+ summary_sec_flag = False+ for p in paragraphs:+ p = p.strip()+ if len(p) == 0:+ continue+ if not p.startswith('#'):+ p = ArticleTextProcessing.remove_uncompleted_sentences_with_citations(p)+ if summary_sec_flag:+ if p.startswith('#'):+ summary_sec_flag = False+ else:+ continue+ if p.startswith('Overall') or p.startswith('In summary') or p.startswith('In conclusion'):+ continue+ if "# Summary" in p or '# Conclusion' in p:+ summary_sec_flag = True+ continue+ output_paragraphs.append(p)++ return '\n\n'.join(output_paragraphs) # Join with '\n\n' for markdown format.++ @staticmethod+ def update_citation_index(s, citation_map):+ """Update citation index in the string based on the citation map."""+ for original_citation in citation_map:+ s = s.replace(f"[{original_citation}]", f"__PLACEHOLDER_{original_citation}__")+ for original_citation, unify_citation in citation_map.items():+ s = s.replace(f"__PLACEHOLDER_{original_citation}__", f"[{unify_citation}]")++ return s++ @staticmethod+ def parse_article_into_dict(input_string):+ """+ Parses a structured text into a nested dictionary. The structure of the text+ is defined by markdown-like headers (using '#' symbols) to denote sections+ and subsections. Each section can contain content and further nested subsections.++ The resulting dictionary captures the hierarchical structure of sections, where+ each section is represented as a key (the section's title) mapping to a value+ that is another dictionary. This dictionary contains two keys:+ - 'content': content of the section+ - 'subsections': a list of dictionaries, each representing a nested subsection+ following the same structure.++ Args:+ input_string (str): A string containing the structured text to parse.++ Returns:+ A dictionary representing contains the section title as the key, and another dictionary+ as the value, which includes the 'content' and 'subsections' keys as described above.+ """+ lines = input_string.split('\n')+ lines = [line for line in lines if line.strip()]+ root = {'content': '', 'subsections': {}}+ current_path = [(root, -1)] # (current_dict, level)++ for line in lines:+ if line.startswith('#'):+ level = line.count('#')+ title = line.strip('# ').strip()+ new_section = {'content': '', 'subsections': {}}++ # Pop from stack until find the parent level+ while current_path and current_path[-1][1] >= level:+ current_path.pop()++ # Append new section to the nearest upper level's subsections+ current_path[-1][0]['subsections'][title] = new_section+ current_path.append((new_section, level))+ else:+ current_path[-1][0]['content'] += line + '\n'++ return root['subsections']+++class FileIOHelper:+ @staticmethod+ def dump_json(obj, file_name, encoding="utf-8"):+ with open(file_name, 'w', encoding=encoding) as fw:+ json.dump(obj, fw, default=FileIOHelper.handle_non_serializable)++ @staticmethod+ def handle_non_serializable(obj):+ return "non-serializable contents" # mark the non-serializable part++ @staticmethod+ def load_json(file_name, encoding="utf-8"):+ with open(file_name, 'r', encoding=encoding) as fr:+ return json.load(fr)++ @staticmethod+ def write_str(s, path):+ with open(path, 'w') as f:+ f.write(s)++ @staticmethod+ def load_str(path):+ with open(path, 'r') as f:+ return '\n'.join(f.readlines())++ @staticmethod+ def dump_pickle(obj, path):+ with open(path, 'wb') as f:+ pickle.dump(obj, f)++ @staticmethod+ def load_pickle(path):+ with open(path, 'rb') as f:+ return pickle.load(f)