An LLM-powered knowledge curation system that researches a topic and generates a full-length report with citations.
Python agentic-ragdeep-researchemnlp2024knowledge-curationlarge-language-modelsnaaclnlpreport-generationretrieval-augmented-generation
add costorm project to knowledge_storm
Yucheng-Jianga4bcbe49892c
diff --git a/knowledge_storm/collaborative_storm/__init__.py b/knowledge_storm/collaborative_storm/__init__.pynew file mode 100644index 0000000..62e6bfb--- /dev/null+++ b/knowledge_storm/collaborative_storm/__init__.py@@ -0,0 +1,4 @@+from .modules import *+from .engine import *+from .demo_runner import *+from .baseline import *diff --git a/knowledge_storm/collaborative_storm/engine.py b/knowledge_storm/collaborative_storm/engine.pynew file mode 100644index 0000000..e8c6092--- /dev/null+++ b/knowledge_storm/collaborative_storm/engine.py@@ -0,0 +1,691 @@+import dspy+import os+from dataclasses import dataclass, field, asdict+from typing import List, Union, Literal, Optional, Dict++from .modules import collaborative_storm_utils as collaborative_storm_utils+from .modules.callback import BaseCallbackHandler+from .modules.co_storm_agents import (+ SimulatedUser,+ PureRAGAgent,+ Moderator,+ CoStormExpert,+)+from .modules.expert_generation import GenerateExpertModule+from .modules.warmstart_hierarchical_chat import WarmStartModule+from ..dataclass import ConversationTurn, KnowledgeBase+from ..interface import LMConfigs, Agent+from ..logging_wrapper import LoggingWrapper+from ..lm import OpenAIModel, AzureOpenAIModel, TogetherClient+from ..rm import BingSearch+++class CollaborativeStormLMConfigs(LMConfigs):+ """Configurations for LLM used in different parts of STORM.++ Given that different parts in STORM framework have different complexity, we use different LLM configurations+ to achieve a balance between quality and efficiency. If no specific configuration is provided, we use the default+ setup in the paper.+ """++ def __init__(self):+ self.question_answering_lm = None+ self.discourse_manage_lm = None+ self.utterance_polishing_lm = None+ self.warmstart_outline_gen_lm = None+ self.question_asking_lm = None+ self.knowledge_base_lm = None++ def init(+ self,+ lm_type: Literal["openai", "azure", "together"],+ temperature: Optional[float] = 1.0,+ top_p: Optional[float] = 0.9,+ ):+ if lm_type and lm_type == "openai":+ openai_kwargs = {+ "api_key": os.getenv("OPENAI_API_KEY"),+ "api_provider": "openai",+ "temperature": temperature,+ "top_p": top_p,+ "api_base": None,+ }+ self.question_answering_lm = OpenAIModel(+ model="gpt-4o-2024-05-13", max_tokens=1000, **openai_kwargs+ )+ self.discourse_manage_lm = OpenAIModel(+ model="gpt-4o-2024-05-13", max_tokens=500, **openai_kwargs+ )+ self.utterance_polishing_lm = OpenAIModel(+ model="gpt-4o-2024-05-13", max_tokens=2000, **openai_kwargs+ )+ self.warmstart_outline_gen_lm = OpenAIModel(+ model="gpt-4-1106-preview", max_tokens=500, **openai_kwargs+ )+ self.question_asking_lm = OpenAIModel(+ model="gpt-4o-2024-05-13", max_tokens=300, **openai_kwargs+ )+ self.knowledge_base_lm = OpenAIModel(+ model="gpt-4o-2024-05-13", max_tokens=1000, **openai_kwargs+ )+ elif lm_type and lm_type == "azure":+ azure_kwargs = {+ "api_key": os.getenv("AZURE_API_KEY"),+ "temperature": temperature,+ "top_p": top_p,+ "api_base": os.getenv("AZURE_API_BASE"),+ "api_version": os.getenv("AZURE_API_VERSION"),+ }+ self.question_answering_lm = AzureOpenAIModel(+ model="gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"+ )+ self.discourse_manage_lm = AzureOpenAIModel(+ model="gpt-4o", max_tokens=500, **azure_kwargs, model_type="chat"+ )+ self.utterance_polishing_lm = AzureOpenAIModel(+ model="gpt-4o", max_tokens=2000, **azure_kwargs, model_type="chat"+ )+ self.warmstart_outline_gen_lm = AzureOpenAIModel(+ model="gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"+ )+ self.question_asking_lm = AzureOpenAIModel(+ model="gpt-4o", max_tokens=300, **azure_kwargs, model_type="chat"+ )+ self.knowledge_base_lm = AzureOpenAIModel(+ model="gpt-4o", max_tokens=1000, **azure_kwargs, model_type="chat"+ )+ elif lm_type and lm_type == "together":+ together_kwargs = {+ "api_key": os.getenv("TOGETHER_API_KEY"),+ "temperature": temperature,+ "top_p": top_p,+ }+ self.question_answering_lm = TogetherClient(+ model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ max_tokens=1000,+ model_type="chat",+ **together_kwargs,+ )+ self.discourse_manage_lm = TogetherClient(+ model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ max_tokens=500,+ model_type="chat",+ **together_kwargs,+ )+ self.utterance_polishing_lm = TogetherClient(+ model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ max_tokens=2000,+ model_type="chat",+ **together_kwargs,+ )+ self.warmstart_outline_gen_lm = TogetherClient(+ model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ max_tokens=500,+ model_type="chat",+ **together_kwargs,+ )+ self.question_asking_lm = TogetherClient(+ model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ max_tokens=300,+ model_type="chat",+ **together_kwargs,+ )+ self.knowledge_base_lm = TogetherClient(+ model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",+ max_tokens=1000,+ model_type="chat",+ **together_kwargs,+ )+ else:+ raise Exception(+ "No valid OpenAI API provider is provided. Cannot use default LLM configurations."+ )++ def set_question_answering_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.question_answering_lm = model++ def set_discourse_manage_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.discourse_manage_lm = model++ def set_utterance_polishing_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.utterance_polishing_lm = model++ def set_warmstart_outline_gen_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.warmstart_outline_gen_lm = model++ def set_question_asking_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.question_asking_lm = model++ def set_knowledge_base_lm(self, model: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.knowledge_base_lm = model++ def collect_and_reset_lm_usage(self):+ lm_usage = {}+ for attr_name in self.__dict__:+ if "_lm" in attr_name and hasattr(+ getattr(self, attr_name), "get_usage_and_reset"+ ):+ usage = getattr(self, attr_name).get_usage_and_reset()+ if any(+ value["prompt_tokens"] != 0 or value["completion_tokens"] != 0+ for value in usage.values()+ ):+ lm_usage[attr_name] = usage+ return lm_usage++ def to_dict(self):+ """+ Converts the CollaborativeStormLMConfigs instance to a dictionary representation.++ Returns:+ dict: The dictionary representation of the CollaborativeStormLMConfigs.+ """+ config_dict = {}+ for attr_name in self.__dict__:+ config_dict[attr_name] = getattr(self, attr_name).kwargs+ return config_dict+++@dataclass+class RunnerArgument:+ """Arguments for controlling the STORM Wiki pipeline."""++ topic: str = field(+ metadata={"help": "Topic of discourse"},+ )+ retrieve_top_k: int = field(+ default=10,+ metadata={"help": "retrieve top k results for each query in retriever"},+ )+ max_search_queries: int = field(+ default=2,+ metadata={+ "help": "Maximum number of search queries to consider for each question."+ },+ )+ total_conv_turn: int = field(+ default=20,+ metadata={"help": "Maximum number turn in conversation."},+ )+ max_search_thread: int = field(+ default=5,+ metadata={"help": "Maximum number of parallel thread for retriever"},+ )+ max_search_queries_per_turn: int = field(+ default=3,+ metadata={"help": "Maximum number of search queries to consider in each turn."},+ )+ warmstart_max_num_experts: int = field(+ default=3,+ metadata={+ "help": "Max number of experts in perspective guided QA in warm start process"+ },+ )+ warmstart_max_turn_per_experts: int = field(+ default=2,+ metadata={"help": "Max number of turns per perspective in warm start process"},+ )+ warmstart_max_thread: int = field(+ default=3,+ metadata={+ "help": "Max number thread for parallel perspective guided QA in warm start process"+ },+ )+ max_thread_num: int = field(+ default=10,+ metadata={+ "help": "Maximum number of threads to use. "+ "Consider reducing it if keep getting 'Exceed rate limit' error when calling LM API."+ },+ )+ max_num_round_table_experts: int = field(+ default=2,+ metadata={"help": "Max number of active experts in round table discussion."},+ )+ moderator_override_N_consecutive_answering_turn: int = field(+ default=3,+ metadata={+ "help": "Number of consecutive experts answering turn before moderator override the conversation"+ },+ )+ node_expansion_trigger_count: int = field(+ default=10,+ metadata={+ "help": "Trigger node expansion for node that contain more than N snippets"+ },+ )+ disable_moderator: bool = field(+ default=False,+ metadata={"help": "If True, disable moderator."},+ )+ disable_multi_experts: bool = field(+ default=False,+ metadata={"help": "If True, disable moderator."},+ )+ rag_only_baseline_mode: bool = field(+ default=False,+ metadata={"help": "If True, switch to rag online baseline mode"},+ )++ def to_dict(self):+ """+ Converts the RunnerArgument instance to a dictionary representation.++ Returns:+ dict: The dictionary representation of the RunnerArgument.+ """+ return asdict(self)++ @classmethod+ def from_dict(cls, data):+ """+ Constructs a RunnerArgument instance from a dictionary representation.++ Args:+ data (dict): The dictionary representation of the RunnerArgument.++ Returns:+ RunnerArgument: The constructed RunnerArgument instance.+ """+ return cls(**data)+++@dataclass+class TurnPolicySpec:+ should_reorganize_knowledge_base: bool = False+ should_update_experts_list: bool = False+ should_polish_utterance: bool = False+ agent: Agent = None+++class DiscourseManager:+ def __init__(+ self,+ logging_wrapper: LoggingWrapper,+ lm_config: CollaborativeStormLMConfigs,+ runner_argument: RunnerArgument,+ rm: dspy.Retrieve,+ callback_handler: BaseCallbackHandler,+ ):+ # parameter management+ self.lm_config = lm_config+ self.runner_argument = runner_argument+ self.logging_wrapper = logging_wrapper+ self.callback_handler = callback_handler+ self.rm = rm+ # role management+ self.experts: List[CoStormExpert] = []+ self.simulated_user: SimulatedUser = SimulatedUser(+ topic=self.runner_argument.topic,+ role_name="Guest",+ role_description="",+ intent=None,+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ callback_handler=self.callback_handler,+ )+ self.pure_rag_agent: PureRAGAgent = PureRAGAgent(+ topic=self.runner_argument.topic,+ role_name="PureRAG",+ role_description="",+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=self.rm,+ callback_handler=self.callback_handler,+ )+ self.moderator: Moderator = Moderator(+ topic=self.runner_argument.topic,+ role_name="Moderator",+ role_description="",+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ callback_handler=self.callback_handler,+ )+ self.general_knowledge_provider = CoStormExpert(+ topic=self.runner_argument.topic,+ role_name="General Knowledge Provider",+ role_description="Focus on broadly covering the basic facts about the question.",+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=self.rm,+ callback_handler=self.callback_handler,+ )+ self.generate_expert_module = GenerateExpertModule(+ engine=self.lm_config.discourse_manage_lm+ )+ self.next_turn_moderator_override = False++ def serialize_experts(self) -> List[Dict]:+ return [+ {+ "topic": expert.topic,+ "role_name": expert.role_name,+ "role_description": expert.role_description,+ }+ for expert in self.experts+ ]++ def deserialize_experts(self, data: List[Dict]):+ for expert_data in data:+ self.experts.append(+ CoStormExpert(+ topic=expert_data["topic"],+ role_name=expert_data["role_name"],+ role_description=expert_data["role_description"],+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=self.rm,+ callback_handler=self.callback_handler,+ )+ )++ def _should_generate_question(+ self, conversation_history: List[ConversationTurn]+ ) -> bool:+ consecutive_non_questioning_turn = 0+ for conv_turn in reversed(conversation_history):+ if conv_turn.utterance_type not in [+ "Original Question",+ "Information Request",+ ]:+ consecutive_non_questioning_turn += 1+ else:+ break+ return (+ consecutive_non_questioning_turn+ >= self.runner_argument.moderator_override_N_consecutive_answering_turn+ )++ def _parse_expert_names_to_agent(self, expert_descriptions: Union[str, List[str]]):+ if type(expert_descriptions) == str:+ expert_descriptions = [expert_descriptions]+ agents: CoStormExpert = []+ for expert_name in expert_descriptions:+ role_name, role_description = expert_name.split(":")+ role_name = role_name.strip()+ role_description = role_description.strip()+ new_costorm_expert = CoStormExpert(+ topic=self.runner_argument.topic,+ role_name=role_name,+ role_description=role_description,+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=self.rm,+ callback_handler=self.callback_handler,+ )+ agents.append(new_costorm_expert)+ return agents++ def _update_expert_list_from_utterance(self, focus: str, background_info: str):+ expert_names = self.generate_expert_module(+ topic=self.runner_argument.topic,+ background_info=background_info,+ focus=focus,+ num_experts=self.runner_argument.max_num_round_table_experts,+ ).experts+ self.experts = self._parse_expert_names_to_agent(expert_names)++ def _is_last_turn_questioning(self, conversation_history: List[ConversationTurn]):+ return conversation_history and conversation_history[-1].utterance_type in [+ "Original Question",+ "Information Request",+ ]++ def get_next_turn_policy(+ self,+ conversation_history: List[ConversationTurn],+ dry_run=False,+ simulate_user=False,+ simulate_user_intent: str = None,+ ) -> TurnPolicySpec:+ next_turn_policy = TurnPolicySpec()+ if simulate_user:+ self.simulated_user.intent = simulate_user_intent+ next_turn_policy.agent = self.simulated_user+ elif self.runner_argument.rag_only_baseline_mode:+ assert self.conversation_history[-1].role == "Guest"+ next_turn_policy.agent = self.pure_rag_agent+ elif (+ not self.runner_argument.disable_moderator+ and self._should_generate_question(conversation_history)+ ):+ next_turn_policy.agent = self.moderator+ next_turn_policy.should_reorganize_knowledge_base = True+ elif self.next_turn_moderator_override:+ next_turn_policy.agent = self.moderator+ if not dry_run:+ self.next_turn_moderator_override = False+ # experts RAG gen+ else:+ next_turn_policy.agent = self.general_knowledge_provider+ if (+ not self._is_last_turn_questioning(conversation_history)+ and not self.runner_argument.disable_multi_experts+ ):+ if dry_run:+ next_turn_policy.agent = self.experts[0]+ else:+ next_turn_policy.agent = self.experts.pop(0)+ self.experts.append(next_turn_policy.agent)+ next_turn_policy.should_update_experts_list = (+ self._is_last_turn_questioning(conversation_history)+ and not self.runner_argument.disable_multi_experts+ )+ next_turn_policy.should_polish_utterance = True+ return next_turn_policy+++class CoStormRunner:+ def __init__(+ self,+ lm_config: CollaborativeStormLMConfigs,+ runner_argument: RunnerArgument,+ logging_wrapper: LoggingWrapper,+ rm: Optional[dspy.Retrieve] = None,+ callback_handler: BaseCallbackHandler = None,+ ):+ self.runner_argument = runner_argument+ self.lm_config = lm_config+ self.logging_wrapper = logging_wrapper+ self.callback_handler = callback_handler+ if rm is None:+ self.rm = BingSearch(k=runner_argument.retrieve_top_k)+ else:+ self.rm = rm+ self.conversation_history = []+ self.warmstart_conv_archive = []+ self.knowledge_base = KnowledgeBase(+ topic=self.runner_argument.topic,+ knowledge_base_lm=self.lm_config.knowledge_base_lm,+ node_expansion_trigger_count=self.runner_argument.node_expansion_trigger_count,+ )+ self.discourse_manager = DiscourseManager(+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=self.rm,+ callback_handler=callback_handler,+ )++ def to_dict(self):+ return {+ "runner_argument": self.runner_argument.to_dict(),+ "lm_config": self.lm_config.to_dict(),+ "conversation_history": [+ turn.to_dict() for turn in self.conversation_history+ ],+ "warmstart_conv_archive": [+ turn.to_dict() for turn in self.warmstart_conv_archive+ ],+ "experts": self.discourse_manager.serialize_experts(),+ "knowledge_base": self.knowledge_base.to_dict(),+ }++ @classmethod+ def from_dict(cls, data):+ # FIXME: does not use the lm_config data but naively use default setting+ lm_config = CollaborativeStormLMConfigs()+ lm_config.init(lm_type=os.getenv("OPENAI_API_TYPE"))+ costorm_runner = cls(+ lm_config=lm_config,+ runner_argument=RunnerArgument.from_dict(data["runner_argument"]),+ logging_wrapper=LoggingWrapper(lm_config),+ )+ costorm_runner.conversation_history = [+ ConversationTurn.from_dict(turn) for turn in data["conversation_history"]+ ]+ costorm_runner.warmstart_conv_archive = [+ ConversationTurn.from_dict(turn)+ for turn in data.get("warmstart_conv_archive", [])+ ]+ costorm_runner.discourse_manager.deserialize_experts(data["experts"])+ costorm_runner.knowledge_base = KnowledgeBase.from_dict(+ data=data["knowledge_base"],+ knowledge_base_lm=costorm_runner.lm_config.knowledge_base_lm,+ node_expansion_trigger_count=costorm_runner.runner_argument.node_expansion_trigger_count,+ )+ return costorm_runner++ def warm_start(self):+ with self.logging_wrapper.log_pipeline_stage(+ pipeline_stage=f"warm start stage"+ ):+ if not self.runner_argument.rag_only_baseline_mode:+ warm_start_module = WarmStartModule(+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=self.rm,+ callback_handler=self.callback_handler,+ )++ warmstart_conv, warmstart_revised_conv, warmstart_experts = (+ warm_start_module.initiate_warm_start(+ topic=self.runner_argument.topic,+ knowledge_base=self.knowledge_base,+ )+ )+ self.discourse_manager.experts = (+ self.discourse_manager._parse_expert_names_to_agent(+ warmstart_experts+ )+ )+ self.discourse_manager.next_turn_moderator_override = True+ self.conversation_history = (+ warmstart_revised_conv if warmstart_revised_conv else warmstart_conv+ )+ self.warmstart_conv_archive = warmstart_conv+ self.knowledge_base.reogranize()+ else:+ if self.knowledge_base is None:+ self.knowledge_base = KnowledgeBase(+ topic=self.runner_argument.topic+ )+ if self.conversation_history is None:+ self.conversation_history = []+ conv_turn = (+ self.discourse_manager.pure_rag_agent.generate_topic_background()+ )+ self.conversation_history.append(conv_turn)+ self.knowledge_base.update_from_conv_turn(+ conv_turn=conv_turn,+ allow_create_new_node=True,+ insert_under_root=self.runner_argument.rag_only_baseline_mode,+ )++ def generate_report(self) -> str:+ with self.logging_wrapper.log_pipeline_stage("report generation stage"):+ with self.logging_wrapper.log_event(+ "report generation stage: generate report"+ ):+ return self.knowledge_base.to_report()++ def dump_logging_and_reset(self):+ return self.logging_wrapper.dump_logging_and_reset()++ def get_next_turn_experts_for_frontend_render(self):+ turn_policy = self.discourse_manager.get_next_turn_policy(+ conversation_history=self.conversation_history,+ simulate_user=False,+ dry_run=True,+ )+ return turn_policy.agent.role_name++ def step(+ self,+ user_utterance: str = "",+ simulate_user: bool = False,+ simulate_user_intent: str = "",+ ) -> ConversationTurn:+ last_conv_turn = self.conversation_history[-1]+ cur_turn_name = f"conv turn: {len(self.conversation_history) + 1}"+ with self.logging_wrapper.log_pipeline_stage(+ pipeline_stage=f"{cur_turn_name} stage"+ ):+ conv_turn = None+ if user_utterance:+ self.discourse_manager.next_turn_moderator_override = False+ conv_turn = ConversationTurn(+ role="Guest",+ raw_utterance=user_utterance,+ utterance_type="Original Question",+ )+ self.conversation_history.append(conv_turn)+ else:+ with self.logging_wrapper.log_event(+ f"{cur_turn_name}: get turn policy"+ ):+ if self.callback_handler is not None:+ self.callback_handler.on_turn_policy_planning_start()+ turn_policy = self.discourse_manager.get_next_turn_policy(+ conversation_history=self.conversation_history,+ simulate_user=simulate_user,+ simulate_user_intent=simulate_user_intent,+ dry_run=False,+ )++ with self.logging_wrapper.log_event(+ f"{cur_turn_name}: generate utterance"+ ):+ conv_turn = turn_policy.agent.generate_utterance(+ knowledge_base=self.knowledge_base,+ conversation_history=self.conversation_history,+ )++ if turn_policy.should_update_experts_list:+ with self.logging_wrapper.log_event(+ f"{cur_turn_name}: update experts list"+ ):+ self.discourse_manager._update_expert_list_from_utterance(+ focus=last_conv_turn.raw_utterance,+ background_info=conv_turn.raw_utterance,+ )++ if conv_turn is not None:+ self.conversation_history.append(conv_turn)+ with self.logging_wrapper.log_event(+ f"{cur_turn_name}: insert into knowledge base"+ ):+ if self.callback_handler is not None:+ self.callback_handler.on_mindmap_insert_start()+ self.knowledge_base.update_from_conv_turn(+ conv_turn=conv_turn,+ allow_create_new_node=True,+ insert_under_root=self.runner_argument.rag_only_baseline_mode,+ )+ if self.callback_handler is not None:+ self.callback_handler.on_mindmap_insert_end()+ if turn_policy.should_reorganize_knowledge_base:+ with self.logging_wrapper.log_event(+ f"{cur_turn_name}: reorganize knowledge base"+ ):+ if self.callback_handler is not None:+ self.callback_handler.on_mindmap_reorg_start()+ self.knowledge_base.reogranize()+ return conv_turndiff --git a/knowledge_storm/collaborative_storm/modules/__init__.py b/knowledge_storm/collaborative_storm/modules/__init__.pynew file mode 100644index 0000000..3807c64--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/__init__.py@@ -0,0 +1,8 @@+from .article_generation import *+from .grounded_question_answering import *+from .grounded_question_generation import *+from .information_insertion_module import *+from .simulate_user import *+from .warmstart_hierarchical_chat import *+from .knowledge_base_summary import *+from .costorm_expert_utterance_generator import *diff --git a/knowledge_storm/collaborative_storm/modules/article_generation.py b/knowledge_storm/collaborative_storm/modules/article_generation.pynew file mode 100644index 0000000..be61400--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/article_generation.py@@ -0,0 +1,123 @@+import dspy+from concurrent.futures import ThreadPoolExecutor, as_completed+from typing import Set, Union++from .collaborative_storm_utils import clean_up_section+from ...dataclass import KnowledgeBase, KnowledgeNode+++class ArticleGenerationModule(dspy.Module):+ """Use the information collected from the information-seeking conversation to write a section."""++ def __init__(+ self,+ engine: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ ):+ super().__init__()+ self.write_section = dspy.Predict(WriteSection)+ self.engine = engine++ def _get_cited_information_string(+ self,+ all_citation_index: Set[int],+ knowledge_base: KnowledgeBase,+ max_words: int = 1500,+ ):+ information = []+ cur_word_count = 0+ for index in sorted(list(all_citation_index)):+ info = knowledge_base.info_uuid_to_info_dict[index]+ snippet = info.snippets[0]+ info_text = f"[{index}]: {snippet} (Question: {info.meta['question']}. Query: {info.meta['query']})"+ cur_snippet_length = len(info_text.split())+ if cur_snippet_length + cur_word_count > max_words:+ break+ cur_word_count += cur_snippet_length+ information.append(info_text)+ return "\n".join(information)++ def gen_section(+ self, topic: str, node: KnowledgeNode, knowledge_base: KnowledgeBase+ ):+ if node is None or len(node.content) == 0:+ return ""+ if (+ node.synthesize_output is not None+ and node.synthesize_output+ and not node.need_regenerate_synthesize_output+ ):+ return node.synthesize_output+ all_citation_index = node.collect_all_content()+ information = self._get_cited_information_string(+ all_citation_index=all_citation_index, knowledge_base=knowledge_base+ )+ with dspy.settings.context(lm=self.engine):+ synthesize_output = clean_up_section(+ self.write_section(+ topic=topic, info=information, section=node.name+ ).output+ )+ node.synthesize_output = synthesize_output+ node.need_regenerate_synthesize_output = False+ return node.synthesize_output++ def forward(self, knowledge_base: KnowledgeBase):+ all_nodes = knowledge_base.collect_all_nodes()+ node_to_paragraph = {}++ # Define a function to generate paragraphs for nodes+ def _node_generate_paragraph(node):+ node_gen_paragraph = self.gen_section(+ topic=knowledge_base.topic, node=node, knowledge_base=knowledge_base+ )+ lines = node_gen_paragraph.split("\n")+ if lines[0].strip().replace("*", "").replace("#", "") == node.name:+ lines = lines[1:]+ node_gen_paragraph = "\n".join(lines)+ path = " -> ".join(node.get_path_from_root())+ return path, node_gen_paragraph++ with ThreadPoolExecutor(max_workers=5) as executor:+ # Submit all tasks+ future_to_node = {+ executor.submit(_node_generate_paragraph, node): node+ for node in all_nodes+ }++ # Collect the results as they complete+ for future in as_completed(future_to_node):+ path, node_gen_paragraph = future.result()+ node_to_paragraph[path] = node_gen_paragraph++ def helper(cur_root, level):+ to_return = []+ if cur_root is not None:+ hash_tag = "#" * level + " "+ cur_path = " -> ".join(cur_root.get_path_from_root())+ node_gen_paragraph = node_to_paragraph[cur_path]+ to_return.append(f"{hash_tag}{cur_root.name}\n{node_gen_paragraph}")+ for child in cur_root.children:+ to_return.extend(helper(child, level + 1))+ return to_return++ to_return = []+ for child in knowledge_base.root.children:+ to_return.extend(helper(child, level=1))++ return "\n".join(to_return)+++class WriteSection(dspy.Signature):+ """Write a Wikipedia section based on the collected information. You will be given the topic, the section you are writing and relevant information.+ Each information will be provided with the raw content along with question and query lead to that information.+ Here is the format of your writing:+ Use [1], [2], ..., [n] in line (for example, "The capital of the United States is Washington, D.C.[1][3]."). You DO NOT need to include a References or Sources section to list the sources at the end.+ """++ info = dspy.InputField(prefix="The collected information:\n", format=str)+ topic = dspy.InputField(prefix="The topic of the page: ", format=str)+ section = dspy.InputField(prefix="The section you need to write: ", format=str)+ output = dspy.OutputField(+ prefix="Write the section with proper inline citations (Start your writing. Don't include the page title, section name, or try to write other sections. Do not start the section with topic name.):\n",+ format=str,+ )diff --git a/knowledge_storm/collaborative_storm/modules/callback.py b/knowledge_storm/collaborative_storm/modules/callback.pynew file mode 100644index 0000000..339c7e3--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/callback.py@@ -0,0 +1,197 @@+from typing import List+from ...interface import Information+++class BaseCallbackHandler:+ """Base callback handler to manage callbacks from the STORM pipeline."""++ def on_turn_policy_planning_start(self, **kwargs):+ """Run when the turn policy planning begins, before deciding the direction or goal for the next conversation turn."""+ pass++ def on_expert_action_planning_start(self, **kwargs):+ """Run when the expert action planning begins, preparing to determine the actions that each expert should take."""+ pass++ def on_expert_action_planning_end(self, **kwargs):+ """Run when the expert action planning ends, after deciding the actions that each expert should take."""+ pass++ def on_expert_information_collection_start(self, **kwargs):+ """Run when the expert information collection starts, start gathering all necessary data from selected sources."""+ pass++ def on_expert_information_collection_end(self, info: List[Information], **kwargs):+ """Run when the expert information collection ends, after gathering all necessary data from selected sources."""+ pass++ def on_expert_utterance_generation_end(self, **kwargs):+ """Run when the expert utterance generation ends, before creating responses or statements from each expert."""+ pass++ def on_expert_utterance_polishing_start(self, **kwargs):+ """Run when the expert utterance polishing begins, to refine and improve the clarity and coherence of generated content."""+ pass++ def on_mindmap_insert_start(self, **kwargs):+ """Run when the process of inserting new information into the mindmap starts."""+ pass++ def on_mindmap_insert_end(self, **kwargs):+ """Run when the process of inserting new information into the mindmap ends."""+ pass++ def on_mindmap_reorg_start(self, **kwargs):+ """Run when the reorganization of the mindmap begins, to restructure and optimize the flow of information."""+ pass++ def on_expert_list_update_start(self, **kwargs):+ """Run when the expert list update starts, to modify or refresh the list of active experts."""+ pass++ def on_article_generation_start(self, **kwargs):+ """Run when the article generation process begins, to compile and format the final article content."""+ pass++ def on_warmstart_update(self, message, **kwargs):+ """Run when the warm start process has update."""+ pass+++class CoStormDemoCallBackHandler(BaseCallbackHandler):+ from ...api import BackendAPI++ def __init__(self, backend_api: BackendAPI, conversation_id: int):+ self.backend_api = backend_api+ self.conversation_id = conversation_id++ def on_turn_policy_planning_start(self, **kwargs):+ """Run when the turn policy planning begins, before deciding the direction or goal for the next conversation turn."""+ # self.backend_api.update_conversation_status(+ # self.conversation_id,+ # "active",+ # "Start planning next expert; inspect mind map; inspect system state.",+ # )+ # skip this part due to message sync latency in the frontend UI, which introduce more confusion.+ pass++ def on_expert_action_planning_start(self, **kwargs):+ """Run when the expert action planning begins, preparing to determine the actions that each expert should take."""+ self.backend_api.update_conversation_status(+ self.conversation_id,+ "active",+ "Reviewing discourse history; Deciding utterance intent.",+ )++ def on_expert_information_collection_start(self, **kwargs):+ """Run when the expert information collection ends, after gathering all necessary data from selected sources."""+ self.backend_api.update_conversation_status(+ self.conversation_id,+ "active",+ "Start searching and browsing collected information; drafting utterance",+ )++ def on_expert_information_collection_end(self, info: List[Information], **kwargs):+ """Run when the expert information collection ends, after gathering all necessary data from selected sources."""+ if info:+ urls = [i.url for i in info]+ information_string = "\n".join([f"Finish browsing {url}" for url in urls])+ self.backend_api.update_conversation_status(+ self.conversation_id, "active", information_string+ )++ def on_expert_utterance_generation_end(self, **kwargs):+ """Run when the expert utterance generation ends, before creating responses or statements from each expert."""+ self.backend_api.update_conversation_status(+ self.conversation_id,+ "active",+ "Finish generating utterance from collected information.",+ )++ def on_expert_utterance_polishing_start(self, **kwargs):+ """Run when the expert utterance polishing begins, to refine and improve the clarity and coherence of generated content."""+ self.backend_api.update_conversation_status(+ self.conversation_id, "active", "Start polishing utterance."+ )++ def on_mindmap_insert_start(self, **kwargs):+ """Run when the process of inserting new information into the mindmap starts."""+ self.backend_api.update_conversation_status(+ self.conversation_id, "active", "Start inserting information into mind map."+ )++ def on_mindmap_insert_end(self, **kwargs):+ """Run when the process of inserting new information into the mindmap ends."""+ self.backend_api.update_conversation_status(self.conversation_id, "active", "")++ def on_mindmap_reorg_start(self, **kwargs):+ """Run when the reorganization of the mindmap begins, to restructure and optimize the flow of information."""+ self.backend_api.update_conversation_status(+ self.conversation_id, "active", "Start re-organizing mind map."+ )++ def on_expert_list_update_start(self, **kwargs):+ """Run when the expert list update starts, to modify or refresh the list of active experts."""+ self.backend_api.update_conversation_status(+ self.conversation_id, "active", "Start updating expert candidates."+ )++ def on_warmstart_update(self, message, **kwargs):+ """Run when the warm start process has update."""+ self.backend_api.update_conversation_status(+ self.conversation_id, "warmup", message+ )+++class LocalConsolePrintCallBackHandler(BaseCallbackHandler):+ from ...api import BackendAPI++ def __init__(self):+ pass++ def on_turn_policy_planning_start(self, **kwargs):+ """Run when the turn policy planning begins, before deciding the direction or goal for the next conversation turn."""+ print("Start planning next expert; inspect mind map; inspect system state.")++ def on_expert_action_planning_start(self, **kwargs):+ """Run when the expert action planning begins, preparing to determine the actions that each expert should take."""+ print("Reviewing discourse history; Deciding utterance intent.")++ def on_expert_information_collection_start(self, **kwargs):+ """Run when the expert information collection ends, after gathering all necessary data from selected sources."""+ print("Start searching with the search engine; browsing collected information.")++ def on_expert_information_collection_end(self, info: List[Information], **kwargs):+ """Run when the expert information collection ends, after gathering all necessary data from selected sources."""+ if info:+ urls = [i.url for i in info]+ information_string = "\n".join([f"Finish browsing {url}" for url in urls])+ print(information_string)++ def on_expert_utterance_generation_end(self, **kwargs):+ """Run when the expert utterance generation ends, before creating responses or statements from each expert."""+ print("Finish generating utterance from collected information.")++ def on_expert_utterance_polishing_start(self, **kwargs):+ """Run when the expert utterance polishing begins, to refine and improve the clarity and coherence of generated content."""+ print("Start polishing utterance.")++ def on_mindmap_insert_start(self, **kwargs):+ """Run when the process of inserting new information into the mindmap starts."""+ print("Start inserting information into mind map.")++ def on_mindmap_insert_end(self, **kwargs):+ """Run when the process of inserting new information into the mindmap ends."""+ print("Finish inserting information into mind map.")++ def on_mindmap_reorg_start(self, **kwargs):+ """Run when the reorganization of the mindmap begins, to restructure and optimize the flow of information."""+ print("Start re-organizing mind map.")++ def on_expert_list_update_start(self, **kwargs):+ """Run when the expert list update starts, to modify or refresh the list of active experts."""+ print("Start updating expert candidates.")++ def on_warmstart_update(self, message, **kwargs):+ """Run when the warm start process has update."""+ print(f"Warm start update: {message}")diff --git a/knowledge_storm/collaborative_storm/modules/co_storm_agents.py b/knowledge_storm/collaborative_storm/modules/co_storm_agents.pynew file mode 100644index 0000000..045bb32--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/co_storm_agents.py@@ -0,0 +1,341 @@+import dspy+from itertools import zip_longest+import numpy as np+from sklearn.metrics.pairwise import cosine_similarity+from typing import List, Optional, TYPE_CHECKING++from .callback import BaseCallbackHandler+from .collaborative_storm_utils import (+ extract_storm_info_snippet,+ _get_answer_question_module_instance,+)+from .costorm_expert_utterance_generator import CoStormExpertUtteranceGenerationModule+from .grounded_question_generation import GroundedQuestionGenerationModule+from .simulate_user import GenSimulatedUserUtterance+from ...dataclass import ConversationTurn, KnowledgeBase+from ...encoder import get_text_embeddings+from ...interface import Agent, Information, LMConfigs+from ...logging_wrapper import LoggingWrapper++if TYPE_CHECKING:+ from ..engine import RunnerArgument+++class CoStormExpert(Agent):+ def __init__(+ self,+ topic: str,+ role_name: str,+ role_description: str,+ lm_config: LMConfigs,+ runner_argument: "RunnerArgument",+ logging_wrapper: LoggingWrapper,+ rm: Optional[dspy.Retrieve] = None,+ callback_handler: BaseCallbackHandler = None,+ ):+ super().__init__(topic, role_name, role_description)+ self.lm_config = lm_config+ self.runner_argument = runner_argument+ self.logging_wrapper = logging_wrapper+ self.callback_handler = callback_handler+ self.costorm_agent_utterance_generator = (+ self._get_costorm_expert_utterance_generator(rm=rm)+ )++ def _get_costorm_expert_utterance_generator(+ self, rm: Optional[dspy.Retrieve] = None+ ):+ return CoStormExpertUtteranceGenerationModule(+ action_planning_lm=self.lm_config.discourse_manage_lm,+ utterance_polishing_lm=self.lm_config.utterance_polishing_lm,+ answer_question_module=_get_answer_question_module_instance(+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=rm,+ ),+ logging_wrapper=self.logging_wrapper,+ callback_handler=self.callback_handler,+ )++ def generate_utterance(+ self,+ knowledge_base: KnowledgeBase,+ conversation_history: List[ConversationTurn],+ ):+ with self.logging_wrapper.log_event(+ "CoStormExpert generate utternace: get knowledge base summary"+ ):+ if self.callback_handler is not None:+ self.callback_handler.on_expert_action_planning_start()+ conversation_summary = knowledge_base.get_knowledge_base_summary()+ with self.logging_wrapper.log_event(+ "CoStormExpert.generate_utterance generate utterance"+ ):+ last_conv_turn = conversation_history[-1]+ conv_turn = self.costorm_agent_utterance_generator(+ topic=self.topic,+ current_expert=self.get_role_description(),+ conversation_summary=conversation_summary,+ last_conv_turn=last_conv_turn,+ ).conversation_turn+ with self.logging_wrapper.log_event(+ "CoStormExpert generate utterance: polish utterance"+ ):+ if self.callback_handler is not None:+ self.callback_handler.on_expert_utterance_polishing_start()+ self.costorm_agent_utterance_generator.polish_utterance(+ conversation_turn=conv_turn, last_conv_turn=last_conv_turn+ )+ return conv_turn+++class SimulatedUser(Agent):+ def __init__(+ self,+ topic: str,+ role_name: str,+ role_description: str,+ intent: str,+ lm_config: LMConfigs,+ runner_argument: "RunnerArgument",+ logging_wrapper: LoggingWrapper,+ callback_handler: BaseCallbackHandler = None,+ ):+ super().__init__(topic, role_name, role_description)+ self.intent = intent+ self.lm_config = lm_config+ self.runner_argument = runner_argument+ self.logging_wrapper = logging_wrapper+ self.gen_simulated_user_utterance = GenSimulatedUserUtterance(+ engine=self.lm_config.question_answering_lm+ )+ self.callback_handler = callback_handler++ def generate_utterance(+ self,+ knowledge_base: KnowledgeBase,+ conversation_history: List[ConversationTurn],+ ):+ assert (+ self.intent is not None and self.intent+ ), "Simulate user intent is not initialized."++ with self.logging_wrapper.log_event(+ "SimulatedUser generate utternace: generate utterance"+ ):+ utterance = self.gen_simulated_user_utterance(+ topic=self.topic, intent=self.intent, conv_history=conversation_history+ )+ return ConversationTurn(+ role="Guest", raw_utterance=utterance, utterance_type="Original Question"+ )+++class Moderator(Agent):+ def __init__(+ self,+ topic: str,+ role_name: str,+ role_description: str,+ lm_config: LMConfigs,+ runner_argument: "RunnerArgument",+ logging_wrapper: LoggingWrapper,+ callback_handler: BaseCallbackHandler = None,+ ):+ super().__init__(topic, role_name, role_description)+ self.lm_config = lm_config+ self.runner_argument = runner_argument+ self.logging_wrapper = logging_wrapper+ self.grounded_question_generation_module = GroundedQuestionGenerationModule(+ engine=self.lm_config.question_asking_lm+ )+ self.callback_handler = callback_handler++ def _get_conv_turn_unused_information(+ self, conv_turn: ConversationTurn, knowledge_base: KnowledgeBase+ ):+ # extract all snippets from raw retrieved information+ raw_retrieved_info: List[Information] = conv_turn.raw_retrieved_info+ raw_retrieved_single_snippet_info: List[Information] = []+ for info in raw_retrieved_info:+ for snippet_idx in range(len(info.snippets)):+ raw_retrieved_single_snippet_info.append(+ extract_storm_info_snippet(info, snippet_index=snippet_idx)+ )+ # get all cited information+ cited_info = list(knowledge_base.info_uuid_to_info_dict.values())+ cited_info_hash_set = set([hash(info) for info in cited_info])+ cited_snippets = [info.snippets[0] for info in cited_info]+ # get list of unused information+ unused_information: List[Information] = [+ info+ for info in raw_retrieved_single_snippet_info+ if hash(info) not in cited_info_hash_set+ ]+ if not unused_information:+ return []+ # extract snippets to get embeddings+ unused_information_snippets = [info.snippets[0] for info in unused_information]+ # get embeddings+ cache = knowledge_base.embedding_cache+ unused_snippets_embeddings, _ = get_text_embeddings(+ unused_information_snippets, embedding_cache=cache, max_workers=100+ )+ claim_embedding, _ = get_text_embeddings(+ conv_turn.claim_to_make, embedding_cache=cache+ )+ query_embedding, _ = get_text_embeddings(+ conv_turn.queries, embedding_cache=cache+ )+ cited_snippets_embedding, _ = get_text_embeddings(+ cited_snippets, embedding_cache=cache+ )+ # calculate similarity+ query_similarities = cosine_similarity(+ unused_snippets_embeddings, query_embedding+ )+ max_query_similarity = np.max(query_similarities, axis=1)+ cited_snippets_similarity = np.max(+ cosine_similarity(unused_snippets_embeddings, cited_snippets_embedding),+ axis=1,+ )+ cited_snippets_similarity = np.clip(cited_snippets_similarity, 0, 1)+ # use claim similarity to filter out "real" not useful data+ claim_similarity = cosine_similarity(+ unused_snippets_embeddings, claim_embedding.reshape(1, -1)+ ).flatten()+ claim_similarity = np.where(claim_similarity >= 0.25, 1.0, 0.0)+ # calculate score: snippet that is close to topic but far from query+ query_sim_weight = 0.5+ cited_snippets_sim_weight = 1 - query_sim_weight+ combined_scores = (+ ((1 - max_query_similarity) ** query_sim_weight)+ * ((1 - cited_snippets_similarity) ** cited_snippets_sim_weight)+ * claim_similarity+ )+ sorted_indices = np.argsort(combined_scores)[::-1]+ return [unused_information[idx] for idx in sorted_indices]++ def _get_sorted_unused_snippets(+ self,+ knowledge_base: KnowledgeBase,+ conversation_history: List[ConversationTurn],+ last_n_conv_turn: int = 2,+ ):+ # get last N conv turn and batch encode all related strings+ considered_conv_turn = []+ batch_snippets = [self.topic]+ for conv_turn in reversed(conversation_history):+ if len(considered_conv_turn) == last_n_conv_turn:+ break+ if conv_turn.utterance_type == "Questioning":+ break+ considered_conv_turn.append(conv_turn)+ batch_snippets.extend(+ sum([info.snippets for info in conv_turn.raw_retrieved_info], [])+ )+ batch_snippets.append(conv_turn.claim_to_make)+ batch_snippets.extend(conv_turn.queries)+ cache = knowledge_base.embedding_cache+ get_text_embeddings(batch_snippets, embedding_cache=cache, max_workers=300)++ # get sorted unused snippets for each turn+ sorted_snippets = []+ for conv_turn in considered_conv_turn:+ sorted_snippets.append(+ self._get_conv_turn_unused_information(+ conv_turn=conv_turn, knowledge_base=knowledge_base+ )+ )++ # use round robin rule to merge these snippets+ merged_snippets = []+ for elements in zip_longest(*sorted_snippets, fillvalue=None):+ merged_snippets.extend(e for e in elements if e is not None)+ return merged_snippets++ def generate_utterance(+ self,+ knowledge_base: KnowledgeBase,+ conversation_history: List[ConversationTurn],+ ):+ with self.logging_wrapper.log_event(+ "Moderator generate utternace: get unused snippets"+ ):+ unused_snippets: List[Information] = self._get_sorted_unused_snippets(+ knowledge_base=knowledge_base, conversation_history=conversation_history+ )+ with self.logging_wrapper.log_event(+ "Moderator generate utternace: QuestionGeneration module"+ ):+ generated_question = self.grounded_question_generation_module(+ topic=self.topic,+ knowledge_base=knowledge_base,+ last_conv_turn=conversation_history[-1],+ unused_snippets=unused_snippets,+ )+ return ConversationTurn(+ role=self.role_name,+ raw_utterance=generated_question.raw_utterance,+ utterance_type="Original Question",+ utterance=generated_question.utterance,+ cited_info=generated_question.cited_info,+ )+++class PureRAGAgent(Agent):+ def __init__(+ self,+ topic: str,+ role_name: str,+ role_description: str,+ lm_config: LMConfigs,+ runner_argument: "RunnerArgument",+ logging_wrapper: LoggingWrapper,+ rm: Optional[dspy.Retrieve] = None,+ callback_handler: BaseCallbackHandler = None,+ ):+ super().__init__(topic, role_name, role_description)+ self.lm_config = lm_config+ self.runner_argument = runner_argument+ self.logging_wrapper = logging_wrapper+ self.grounded_question_answering_module = _get_answer_question_module_instance(+ lm_config=self.lm_config,+ runner_argument=self.runner_argument,+ logging_wrapper=self.logging_wrapper,+ rm=rm,+ )++ def _gen_utterance_from_question(self, question: str):+ grounded_answer = self.grounded_question_answering_module(+ topic=self.topic,+ question=question,+ mode="brief",+ style="conversational and concise",+ )+ conversation_turn = ConversationTurn(+ role=self.role_name, raw_utterance="", utterance_type="Potential Answer"+ )+ conversation_turn.claim_to_make = question+ conversation_turn.raw_utterance = grounded_answer.response+ conversation_turn.utterance = grounded_answer.response+ conversation_turn.queries = grounded_answer.queries+ conversation_turn.raw_retrieved_info = grounded_answer.raw_retrieved_info+ conversation_turn.cited_info = grounded_answer.cited_info+ return conversation_turn++ def generate_topic_background(self):+ return self._gen_utterance_from_question(self.topic)++ def generate_utterance(+ self,+ knowledge_base: KnowledgeBase,+ conversation_history: List[ConversationTurn],+ ):+ with self.logging_wrapper.log_event(+ "PureRAGAgent generate utternace: generate utterance"+ ):+ return self._gen_utterance_from_question(+ question=conversation_history[-1].utterance+ )diff --git a/knowledge_storm/collaborative_storm/modules/collaborative_storm_utils.py b/knowledge_storm/collaborative_storm/modules/collaborative_storm_utils.pynew file mode 100644index 0000000..661c49f--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/collaborative_storm_utils.py@@ -0,0 +1,380 @@+import dspy+import os+import re+import sys+import toml+from typing import List, Tuple, Dict, Optional, TYPE_CHECKING++if TYPE_CHECKING:+ from ..engine import RunnerArgument+from ...interface import Information, Retriever, LMConfigs+from ...logging_wrapper import LoggingWrapper+from ...rm import BingSearch+++def extract_storm_info_snippet(info: Information, snippet_index: int) -> Information:+ """+ Constructs a new Information instance with only the specified snippet index.++ Args:+ storm_info (Information): The original Information instance.+ snippet_index (int): The index of the snippet to retain.++ Returns:+ Information: A new Information instance with only the specified snippet.+ """+ if snippet_index < 0 or snippet_index >= len(info.snippets):+ raise ValueError("Snippet index out of range")++ new_snippets = [info.snippets[snippet_index]]+ new_storm_info = Information(+ info.url, info.description, new_snippets, info.title, info.meta+ )+ return new_storm_info+++def format_search_results(+ searched_results: List[Information],+ info_max_num_words: int = 1000,+ mode: str = "brief",+) -> Tuple[str, Dict[int, Information]]:+ """+ Constructs a string from a list of search results with a specified word limit and returns a mapping of indices to Information.++ Args:+ searched_results (List[Information]): List of Information objects to process.+ info_max_num_words (int, optional): Maximum number of words allowed in the output string. Defaults to 1000.+ mode (str, optional): Mode of summarization. 'brief' takes only the first snippet of each Information.+ 'extensive' adds snippets iteratively until the word limit is reached. Defaults to 'brief'.++ Returns:+ Tuple[str, Dict[int, Information]]:+ - Formatted string with search results, constrained by the word limit.+ - Dictionary mapping indices to the corresponding Information objects.+ """+ total_length = 0++ extracted_snippet_queue = []+ max_snippets = (+ max(len(info.snippets) for info in searched_results) if searched_results else 0+ )+ max_snippets = 1 if mode == "brief" else max_snippets+ abort = False+ included_snippets = set()+ for i in range(max_snippets):+ for info in searched_results:+ if i < len(info.snippets) and not abort:+ cur_snippet = info.snippets[i]+ cur_snippet_len = len(info.snippets[i].split())+ if total_length + cur_snippet_len > info_max_num_words:+ abort = True+ break+ if cur_snippet not in included_snippets:+ included_snippets.add(cur_snippet)+ info = extract_storm_info_snippet(info, snippet_index=i)+ extracted_snippet_queue.append(info)+ total_length += cur_snippet_len+ output = []+ index_mapping = {}+ for idx, info in enumerate(extracted_snippet_queue):+ output.append(f"[{idx + 1}]: {info.snippets[0]}")+ index_mapping[idx + 1] = info+ assert -1 not in index_mapping+ return "\n".join(output), index_mapping+++def extract_cited_storm_info(+ response: str, index_to_storm_info: Dict[int, Information]+) -> Dict[int, Information]:+ """+ Extracts a sub-dictionary of Information instances that are cited in the response.++ Args:+ response (str): The response string containing inline citations like [1], [2], etc.+ index_to_storm_info (Dict[int, Information]): A dictionary mapping indices to Information instances.++ Returns:+ Dict[int, Information]: A sub-dictionary with only the indices that appear in the response.+ """+ cited_indices = set(map(int, re.findall(r"\[(\d+)\]", response)))+ cited_storm_info = {+ index: info+ for index, info in index_to_storm_info.items()+ if index in cited_indices+ }+ return cited_storm_info+++def trim_output_after_hint(response: str, hint: str) -> str:+ """+ Trims the output string to only keep the substring after the given hint (not including the hint).++ Args:+ response (str): The original output string.+ hint (str): The hint string after which the substring should be kept.++ Returns:+ str: The trimmed output string, or the original string if the hint is not found.+ """+ if hint in response:+ start_index = response.find(hint) + len(hint)+ return response[start_index:].strip()+ return response.strip("\n")+++def separate_citations(text: str) -> str:+ """+ Separates multiple citations within square brackets into individual citations.++ Args:+ text (str): The input string containing citations.++ Returns:+ str: The string with separated citations.+ """++ # Define a function to process each match+ def replace_citations(match):+ citations = match.group(1).split(",")+ return "".join(f"[{citation.strip()}]" for citation in citations)++ # Use regular expressions to find and replace citations+ pattern = re.compile(r"\[(\d+(?:,\s*\d+)*)\]")+ return pattern.sub(replace_citations, text)+++def extract_and_remove_citations(text: str) -> Tuple[str, List[int]]:+ """+ Removes single inline citations from the input string and returns the modified string and a list of citation integers.++ Args:+ text (str): The input string containing citations.++ Returns:+ Tuple[str, List[int]]: The string after removal of citations and a list of citation integers.+ """+ citations = []++ # Define a function to process each match+ def extract_citation(match):+ citation = int(match.group(1))+ citations.append(citation)+ return ""++ # Use regular expressions to find and replace citations+ pattern = re.compile(r"\[(\d+)\]")+ modified_text = pattern.sub(extract_citation, text)++ return modified_text, citations+++def keep_first_and_last_paragraph(text: str) -> str:+ """+ Processes the input text to keep the first and last paragraphs and replace+ the middle paragraphs with '[content omitted due to space limit]'.++ Args:+ text (str): The input text containing paragraphs separated by '\n\n'.++ Returns:+ str: The processed text.+ """+ paragraphs = text.split("\n\n")++ if len(paragraphs) <= 3:+ return text++ first_paragraph = paragraphs[0]+ last_paragraph = "\n\n".join(paragraphs[-2:])+ return (+ f"{first_paragraph}\n\n[content omitted due to space limit]\n\n{last_paragraph}"+ )+++def clean_up_section(text):+ """Clean up a section:+ 1. Remove uncompleted sentences (usually due to output token limitation).+ 2. Deduplicate individual groups of citations.+ 3. Remove unnecessary summary."""++ paragraphs = text.split("\n")+ output_paragraphs = []+ summary_sec_flag = False+ for p in paragraphs:+ p = p.strip()+ if len(p) == 0:+ continue+ if not p.startswith("#"):+ p = separate_citations(p)+ if summary_sec_flag:+ if p.startswith("#"):+ summary_sec_flag = False+ else:+ continue+ if (+ p.startswith("Overall")+ or p.startswith("In summary")+ or p.startswith("In conclusion")+ ):+ continue+ if "# Summary" in p or "# Conclusion" in p:+ summary_sec_flag = True+ continue+ output_paragraphs.append(p)++ return "\n\n".join(output_paragraphs) # Join with '\n\n' for markdown format.+++def load_api_key(toml_file_path):+ try:+ with open(toml_file_path, "r") as file:+ data = toml.load(file)+ except FileNotFoundError:+ print(f"File not found: {toml_file_path}", file=sys.stderr)+ return+ except toml.TomlDecodeError:+ print(f"Error decoding TOML file: {toml_file_path}", file=sys.stderr)+ return+ # Set environment variables+ for key, value in data.items():+ os.environ[key] = str(value)+++def construct_frontend_display_data(+ knowledge_base,+ conv_turn,+ is_warm_start: bool = False,+ next_turn_role: Optional[str] = None,+):+ def to_dict(node):+ if node is None:+ return None+ data = {+ "node_name": node.name,+ "node_uuid": " -> ".join(node.get_path_from_root()),+ "content": list(node.content),+ "child_node": [],+ }+ for child in node.children:+ child_data = to_dict(child)+ if child_data is not None:+ data["child_node"].append(child_data)+ return data++ def split_paragraph(paragraph):+ # Regular expression to match sentences with citations+ sentence_pattern = re.compile(r"([^.?!]*[.?!])(\s*\[\d+\])*")+ citation_pattern = re.compile(r"\[(\d+)\]")++ # Check if there are any sentence separators in the paragraph+ if not re.search(r"[.?!]", paragraph):+ # If no separators, consider the whole paragraph as a single sentence+ citations = list(map(int, citation_pattern.findall(paragraph)))+ sentence = citation_pattern.sub("", paragraph).strip()+ return [(sentence, citations)]++ sentences = []++ for match in sentence_pattern.finditer(paragraph):+ sentence_with_citations = match.group(0).strip()+ sentence = match.group(1).strip()+ citations = list(+ map(int, citation_pattern.findall(sentence_with_citations))+ )++ # Remove citations from the sentence+ sentence = citation_pattern.sub("", sentence).strip()++ sentences.append((sentence, citations))++ return sentences++ # get mind map+ mind_map = to_dict(knowledge_base.root)+ # get last turn utterance+ role = conv_turn.role+ utterance = str(conv_turn.utterance)+ sentences_w_citations = split_paragraph(utterance)++ # get cited info+ cited_info = {}+ for citation_idx, info in knowledge_base.info_uuid_to_info_dict.items():+ cited_info[citation_idx] = {+ "description": info.description,+ "snippets": info.snippets,+ "url": info.url,+ "title": info.title,+ "placement": info.meta.get("placement", ""),+ }+ evaluation_questions = None+ if conv_turn.role != "Guest":+ evaluation_questions = [+ {+ "question": "Does this new utterance help steer the discourse towards a new and interesting direction?",+ "type": "multiple_choice",+ "choices": ["Yes", "No"],+ "id": 1,+ },+ {+ "question": "Has the mind map reasonably managing the discourse so far?",+ "type": "multiple_choice",+ "choices": ["Yes", "No"],+ "id": 2,+ },+ {+ "question": "(Optional) Any comments?",+ "type": "freeform",+ "choices": [],+ "id": 3,+ },+ ]+ to_return = {+ "mind_map": mind_map,+ "conversation_turn": {+ "role": role,+ "content": sentences_w_citations,+ "is_warm_start": is_warm_start,+ },+ "cited_info": cited_info,+ "eval": evaluation_questions,+ "next_turn_role": next_turn_role,+ }+ return to_return+++def construct_article_frontend_display_data(knowledge_base, article: str):+ cited_info = {+ key: value.to_dict()+ for key, value in knowledge_base.info_uuid_to_info_dict.items()+ }+ data = {"article": article, "cited_info": cited_info}+ return data+++def construct_round_table_conversation_databackup(round_table_conversation):+ round_tabld_dump = round_table_conversation.to_dict()+ log_dump = round_table_conversation.logging_wrapper.dump_logging_and_reset(+ reset_logging=False+ )+ return {"round_tabld_dump": round_tabld_dump, "log_dump": log_dump}+++def _get_answer_question_module_instance(+ lm_config: LMConfigs,+ runner_argument: "RunnerArgument",+ logging_wrapper: LoggingWrapper,+ rm: Optional[dspy.Retrieve] = None,+):+ from .grounded_question_answering import AnswerQuestionModule++ # configure retriever+ if rm is None:+ rm = BingSearch(k=runner_argument.retrieve_top_k)+ retriever = Retriever(rm=rm, max_thread=runner_argument.max_search_thread)+ # return AnswerQuestionModule instance+ return AnswerQuestionModule(+ retriever=retriever,+ max_search_queries=runner_argument.max_search_queries,+ question_answering_lm=lm_config.question_answering_lm,+ logging_wrapper=logging_wrapper,+ )diff --git a/knowledge_storm/collaborative_storm/modules/costorm_expert_utterance_generator.py b/knowledge_storm/collaborative_storm/modules/costorm_expert_utterance_generator.pynew file mode 100644index 0000000..2c7c350--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/costorm_expert_utterance_generator.py@@ -0,0 +1,160 @@+import dspy+from typing import Union++from .callback import BaseCallbackHandler+from .collaborative_storm_utils import (+ trim_output_after_hint,+ extract_and_remove_citations,+ keep_first_and_last_paragraph,+)++from .grounded_question_answering import AnswerQuestionModule+from .grounded_question_generation import ConvertUtteranceStyle+from ...dataclass import ConversationTurn+from ...logging_wrapper import LoggingWrapper+++class GenExpertActionPlanning(dspy.Signature):+ """+ You are an invited speaker in the round table conversation. Your task is to make a very short note to your assistant to help you prepare for your turn in the conversation.+ You will be given the topic we are discussing, your expertise, and the conversation history.+ Take a look at conversation history, especially last few turns, then let your assistant prepare the material for you with one of following ways.+ 1. Original Question: Initiates a new question to other speakers.+ 2. Further Details: Provides additional information.+ 3. Information Request: Requests information from other speakers.+ 4. Potential Answer: Offers a possible solution or answer.++ Strictly follow this format: [type of contribution]: [one sentence description]. For example, Original Question: [description]+ """++ topic = dspy.InputField(prefix="topic of discussion: ", format=str)+ expert = dspy.InputField(prefix="You are inivited as: ", format=str)+ summary = dspy.InputField(prefix="Discussion history: \n", format=str)+ last_utterance = dspy.InputField(+ prefix="Last utterance in the conversation: \n", format=str+ )+ resposne = dspy.OutputField(+ prefix="Now give your note. Start with one of [Original Question, Further Details, Information Request, Potential Answer] with one sentence description\n",+ format=str,+ )+++class CoStormExpertUtteranceGenerationModule(dspy.Module):+ def __init__(+ self,+ action_planning_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ utterance_polishing_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ answer_question_module: AnswerQuestionModule,+ logging_wrapper: LoggingWrapper,+ callback_handler: BaseCallbackHandler = None,+ ):+ self.action_planning_lm = action_planning_lm+ self.utterance_polishing_lm = utterance_polishing_lm+ self.expert_action = dspy.Predict(GenExpertActionPlanning)+ self.change_style = dspy.Predict(ConvertUtteranceStyle)+ self.answer_question_module = answer_question_module+ self.logging_wrapper = logging_wrapper+ self.callback_handler = callback_handler++ def parse_action(self, action):+ action_types = [+ "Original Question",+ "Further Details",+ "Information Request",+ "Potential Answer",+ ]+ for action_type in action_types:+ if f"{action_type}:" in action:+ return action_type, trim_output_after_hint(action, f"{action_type}:")+ elif f"[{action_type}]:" in action:+ return action_type, trim_output_after_hint(action, f"[{action_type}]:")+ return "Undefined", ""++ def polish_utterance(+ self, conversation_turn: ConversationTurn, last_conv_turn: ConversationTurn+ ):+ # change utterance style+ action_type = conversation_turn.utterance_type+ with self.logging_wrapper.log_event(+ "RoundTableConversationModule.ConvertUtteranceStyle"+ ):+ with dspy.settings.context(+ lm=self.utterance_polishing_lm, show_guidelines=False+ ):+ action_string = (+ f"{action_type} about: {conversation_turn.claim_to_make}"+ )+ if action_type in ["Original Question", "Information Request"]:+ action_string = f"{action_type}"+ last_expert_utterance_wo_citation, _ = extract_and_remove_citations(+ last_conv_turn.utterance+ )+ trimmed_last_expert_utterance = keep_first_and_last_paragraph(+ last_expert_utterance_wo_citation+ )+ utterance = self.change_style(+ expert=conversation_turn.role,+ action=action_string,+ prev=trimmed_last_expert_utterance,+ content=conversation_turn.raw_utterance,+ ).utterance+ conversation_turn.utterance = utterance++ def forward(+ self,+ topic: str,+ current_expert: str,+ conversation_summary: str,+ last_conv_turn: ConversationTurn,+ ):+ last_utterance, _ = extract_and_remove_citations(last_conv_turn.utterance)+ if last_conv_turn.utterance_type in [+ "Original Question",+ "Information Request",+ ]:+ action_type = "Potential Answer"+ action_content = last_utterance+ else:+ with self.logging_wrapper.log_event(+ "CoStormExpertUtteranceGenerationModule: GenExpertActionPlanning"+ ):+ with dspy.settings.context(+ lm=self.action_planning_lm, show_guidelines=False+ ):+ action = self.expert_action(+ topic=topic,+ expert=current_expert,+ summary=conversation_summary,+ last_utterance=last_utterance,+ ).resposne+ action_type, action_content = self.parse_action(action)++ if self.callback_handler is not None:+ self.callback_handler.on_expert_action_planning_end()+ # get response+ conversation_turn = ConversationTurn(+ role=current_expert, raw_utterance="", utterance_type=action_type+ )++ if action_type == "Undefined":+ raise Exception(f"unexpected output: {action}")+ elif action_type in ["Further Details", "Potential Answer"]:+ with self.logging_wrapper.log_event(+ "RoundTableConversationModule: QuestionAnswering"+ ):+ grounded_answer = self.answer_question_module(+ topic=topic,+ question=action_content,+ mode="brief",+ style="conversational and concise",+ callback_handler=self.callback_handler,+ )+ conversation_turn.claim_to_make = action_content+ conversation_turn.raw_utterance = grounded_answer.response+ conversation_turn.queries = grounded_answer.queries+ conversation_turn.raw_retrieved_info = grounded_answer.raw_retrieved_info+ conversation_turn.cited_info = grounded_answer.cited_info+ elif action_type in ["Original Question", "Information Request"]:+ conversation_turn.raw_utterance = action_content++ return dspy.Prediction(conversation_turn=conversation_turn)diff --git a/knowledge_storm/collaborative_storm/modules/expert_generation.py b/knowledge_storm/collaborative_storm/modules/expert_generation.pynew file mode 100644index 0000000..a95915a--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/expert_generation.py@@ -0,0 +1,83 @@+import dspy+import re+from typing import Union+++class GenerateExpertGeneral(dspy.Signature):+ """You need to select a group of diverse experts who will be suitable to be invited to a roundtable discussion on the given topic.+ Each expert should represent a different perspective, role, or affiliation related to this topic.+ You can use the background information provided about the topic for inspiration. For each expert, add a description of their expertise and what they will focus on during the discussion.+ No need to include speakers name in the output.+ Strictly follow format below:+ 1. [speaker 1 role]: [speaker 1 short description]+ 2. [speaker 2 role]: [speaker 2 short description]+ """++ topic = dspy.InputField(prefix="Topic of interest:", format=str)+ background_info = dspy.InputField(+ prefix="Background information about the topic:\n", format=str+ )+ topN = dspy.InputField(prefix="Number of speakers needed: ", format=str)+ experts = dspy.OutputField(format=str)+++class GenerateExpertWithFocus(dspy.Signature):+ """+ You need to select a group of speakers who will be suitable to have roundtable discussion on the [topic] of specific [focus].+ You may consider inviting speakers having opposite stands on the topic; speakers representing different interest parties; Ensure that the selected speakers are directly connected to the specific context and scenario provided.+ For example, if the discussion focus is about a recent event at a specific university, consider inviting students, faculty members, journalists covering the event, university officials, and local community members.+ Use the background information provided about the topic for inspiration. For each speaker, add a description of their interests and what they will focus on during the discussion.+ No need to include speakers name in the output.+ Strictly follow format below:+ 1. [speaker 1 role]: [speaker 1 short description]+ 2. [speaker 2 role]: [speaker 2 short description]+ """++ topic = dspy.InputField(prefix="Topic of interest:", format=str)+ background_info = dspy.InputField(prefix="Background information:\n", format=str)+ focus = dspy.InputField(prefix="Discussion focus: ", format=str)+ topN = dspy.InputField(prefix="Number of speakers needed: ", format=str)+ experts = dspy.OutputField(format=str)+++class GenerateExpertModule(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.generate_expert_general = dspy.Predict(GenerateExpertGeneral)+ self.generate_expert_w_focus = dspy.ChainOfThought(GenerateExpertWithFocus)++ def trim_background(self, background: str, max_words: int = 100):+ words = background.split()+ cur_len = len(words)+ if cur_len <= max_words:+ return background+ trimmed_words = words[: min(cur_len, max_words)]+ trimmed_background = " ".join(trimmed_words)+ return f"{trimmed_background} [rest content omitted]."++ def forward(+ self, topic: str, num_experts: int, background_info: str = "", focus: str = ""+ ):+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ if not focus:+ output = self.generate_expert_general(+ topic=topic, background_info=background_info, topN=num_experts+ ).experts+ else:+ background_info = self.trim_background(+ background=background_info, max_words=100+ )+ output = self.generate_expert_w_focus(+ topic=topic,+ background_info=background_info,+ focus=focus,+ topN=num_experts,+ ).experts+ output = output.replace("*", "").replace("[", "").replace("]", "")+ expert_list = []+ for s in output.split("\n"):+ match = re.search(r"\d+\.\s*(.*)", s)+ if match:+ expert_list.append(match.group(1))+ expert_list = [expert.strip() for expert in expert_list if expert.strip()]+ return dspy.Prediction(experts=expert_list, raw_output=output)diff --git a/knowledge_storm/collaborative_storm/modules/grounded_question_answering.py b/knowledge_storm/collaborative_storm/modules/grounded_question_answering.pynew file mode 100644index 0000000..4065018--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/grounded_question_answering.py@@ -0,0 +1,163 @@+import dspy+from typing import Union, List++from .callback import BaseCallbackHandler+from .collaborative_storm_utils import (+ trim_output_after_hint,+ format_search_results,+ extract_cited_storm_info,+ separate_citations,+)+from ...logging_wrapper import LoggingWrapper+from ...utils import ArticleTextProcessing+from ...interface import Information+++class QuestionToQuery(dspy.Signature):+ """You want to answer the question or support a claim using Google search. What do you type in the search box?+ The question is raised in a round table discussion on a topic. The question may or may not focus on the topic itself.+ Write the queries you will use in the following format:+ - query 1+ - query 2+ ...+ - query n"""++ topic = dspy.InputField(prefix="Topic context:", format=str)+ question = dspy.InputField(+ prefix="I want to collect information about: ", format=str+ )+ queries = dspy.OutputField(prefix="Queries: \n", format=str)+++class AnswerQuestion(dspy.Signature):+ """You are an expert who can use information effectively. You have gathered the related information and will now use the information to form a response.+ Make your response as informative as possible and make sure every sentence is supported by the gathered information.+ If [Gathered information] is not directly related to the [Topic] and [Question], provide the most relevant answer you can based on the available information, and explain any limitations or gaps.+ Use [1], [2], ..., [n] in line (for example, "The capital of the United States is Washington, D.C.[1][3].").+ You DO NOT need to include a References or Sources section to list the sources at the end. The style of writing should be formal.+ """++ topic = dspy.InputField(prefix="Topic you are discussing about:", format=str)+ question = dspy.InputField(prefix="You want to provide insight on: ", format=str)+ info = dspy.InputField(prefix="Gathered information:\n", format=str)+ style = dspy.InputField(prefix="Style of your response should be:", format=str)+ answer = dspy.OutputField(+ prefix="Now give your response. (Try to use as many different sources as possible and do not hallucinate.)",+ format=str,+ )+++class AnswerQuestionModule(dspy.Module):+ def __init__(+ self,+ retriever: dspy.Retrieve,+ max_search_queries: int,+ question_answering_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ logging_wrapper: LoggingWrapper,+ ):+ super().__init__()+ self.question_answering_lm = question_answering_lm+ self.question_to_query = dspy.Predict(QuestionToQuery)+ self.answer_question = dspy.Predict(AnswerQuestion)+ self.retriever = retriever+ self.max_search_queries = max_search_queries+ self.logging_wrapper = logging_wrapper++ def retrieve_information(self, topic, question):+ # decompose question to queries+ with self.logging_wrapper.log_event(+ f"AnswerQuestionModule.question_to_query ({hash(question)})"+ ):+ with dspy.settings.context(lm=self.question_answering_lm):+ queries = self.question_to_query(topic=topic, question=question).queries+ queries = trim_output_after_hint(queries, hint="Queries:")+ queries = [+ q.replace("-", "").strip().strip('"').strip('"').strip()+ for q in queries.split("\n")+ ]+ queries = queries[: self.max_search_queries]+ self.logging_wrapper.add_query_count(count=len(queries))+ with self.logging_wrapper.log_event(+ f"AnswerQuestionModule.retriever.retrieve ({hash(question)})"+ ):+ # retrieve information using retriever+ searched_results: List[Information] = self.retriever.retrieve(+ list(set(queries)), exclude_urls=[]+ )+ # update storm information meta to include the question+ for storm_info in searched_results:+ storm_info.meta["question"] = question+ return queries, searched_results++ def forward(+ self,+ topic: str,+ question: str,+ mode: str = "brief",+ style: str = "conversational",+ callback_handler: BaseCallbackHandler = None,+ ):+ """+ Processes a topic and question to generate a response with relevant information and citations.++ Args:+ topic (str): The topic of interest.+ question (str): The specific question related to the topic.+ mode (str, optional): Mode of summarization. 'brief' takes only the first snippet of each Information.+ 'extensive' adds snippets iteratively until the word limit is reached. Defaults to 'brief'.++ Returns:+ dspy.Prediction: An object containing the following:+ - question (str): the question to answer+ - queries (List[str]): List of query strings used for information retrieval.+ - raw_retrieved_info (List[Information]): List of Information instances retrieved.+ - cited_info (Dict[int, Information]): Dictionary of cited Information instances, indexed by their citation number.+ - response (str): The generated response string with inline citations.+ """+ # retrieve information+ if callback_handler is not None:+ callback_handler.on_expert_information_collection_start()+ queries, searched_results = self.retrieve_information(+ topic=topic, question=question+ )+ if callback_handler is not None:+ callback_handler.on_expert_information_collection_end(searched_results)+ # format information string for answer generation+ info_text, index_to_information_mapping = format_search_results(+ searched_results, mode=mode+ )+ answer = "Sorry, there is insufficient information to answer the question."+ # generate answer to the question+ if info_text:+ with self.logging_wrapper.log_event(+ f"AnswerQuestionModule.answer_question ({hash(question)})"+ ):+ with dspy.settings.context(+ lm=self.question_answering_lm, show_guidelines=False+ ):+ answer = self.answer_question(+ topic=topic, question=question, info=info_text, style=style+ ).answer+ answer = ArticleTextProcessing.remove_uncompleted_sentences_with_citations(+ answer+ )+ answer = trim_output_after_hint(+ answer,+ hint="Now give your response. (Try to use as many different sources as possible and do not hallucinate.)",+ )+ # enforce single citation index bracket. [1, 2] -> [1][2]+ answer = separate_citations(answer)+ if callback_handler is not None:+ callback_handler.on_expert_utterance_generation_end()+ # construct cited search result+ cited_searched_results = extract_cited_storm_info(+ response=answer, index_to_storm_info=index_to_information_mapping+ )++ return dspy.Prediction(+ question=question,+ queries=queries,+ raw_retrieved_info=searched_results,+ cited_info=cited_searched_results,+ response=answer,+ )diff --git a/knowledge_storm/collaborative_storm/modules/grounded_question_generation.py b/knowledge_storm/collaborative_storm/modules/grounded_question_generation.pynew file mode 100644index 0000000..f09339d--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/grounded_question_generation.py@@ -0,0 +1,104 @@+import dspy+from typing import List, Union++from .collaborative_storm_utils import (+ format_search_results,+ extract_and_remove_citations,+ keep_first_and_last_paragraph,+ extract_cited_storm_info,+)+from ...dataclass import ConversationTurn, KnowledgeBase+from ...interface import Information+++class KnowledgeBaseSummmary(dspy.Signature):+ """Your job is to give brief summary of what's been discussed in a roundtable conversation. Contents are themantically organized into hierarchical sections.+ You will be presented with these sections where "#" denotes level of section.+ """++ topic = dspy.InputField(prefix="topic: ", format=str)+ structure = dspy.InputField(prefix="Tree structure: \n", format=str)+ output = dspy.OutputField(prefix="Now give brief summary:\n", format=str)+++class ConvertUtteranceStyle(dspy.Signature):+ """+ You are an invited speaker in the round table conversation.+ Your task is to make the question or the response more conversational and engaging to facilicate the flow of conversation.+ Note that this is ongoing conversation so no need to have welcoming and concluding words. Previous speaker utterance is provided only for making the conversation more natural.+ Note that do not hallucinate and keep the citation index like [1] as it is. Also,+ """++ expert = dspy.InputField(prefix="You are inivited as: ", format=str)+ action = dspy.InputField(+ prefix="You want to contribute to conversation by: ", format=str+ )+ prev = dspy.InputField(prefix="Previous speaker said: ", format=str)+ content = dspy.InputField(+ prefix="Question or response you want to say: ", format=str+ )+ utterance = dspy.OutputField(+ prefix="Your utterance (keep the information as much as you can with citations, prefer shorter answers without loss of information): ",+ format=str,+ )+++class GroundedQuestionGeneration(dspy.Signature):+ """Your job is to find next discussion focus in a roundtable conversation. You will be given previous conversation summary and some information that might assist you discover new discussion focus.+ Note that the new discussion focus should bring new angle and perspective to the discussion and avoid repetition. The new discussion focus should be grounded on the available information and push the boundaries of the current discussion for broader exploration.+ The new discussion focus should have natural flow from last utterance in the conversation.+ Use [1][2] in line to ground your question.+ """++ topic = dspy.InputField(prefix="topic: ", format=str)+ summary = dspy.InputField(prefix="Discussion history: \n", format=str)+ information = dspy.InputField(prefix="Available information: \n", format=str)+ last_utterance = dspy.InputField(+ prefix="Last utterance in the conversation: \n", format=str+ )+ output = dspy.OutputField(+ prefix="Now give next discussion focus in the format of one sentence question:\n",+ format=str,+ )+++class GroundedQuestionGenerationModule(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.gen_focus = dspy.Predict(GroundedQuestionGeneration)+ self.polish_style = dspy.Predict(ConvertUtteranceStyle)+ self.gen_summary = dspy.Predict(KnowledgeBaseSummmary)++ def forward(+ self,+ topic: str,+ knowledge_base: KnowledgeBase,+ last_conv_turn: ConversationTurn,+ unused_snippets: List[Information],+ ):+ information, index_to_information_mapping = format_search_results(+ unused_snippets, info_max_num_words=1000+ )+ summary = knowledge_base.get_knowledge_base_summary()+ last_utterance, _ = extract_and_remove_citations(last_conv_turn.utterance)+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ raw_utterance = self.gen_focus(+ topic=topic,+ summary=summary,+ information=information,+ last_utterance=last_utterance,+ ).output+ utterance = self.polish_style(+ expert="Roundtable conversation moderator",+ action="Raising a new question by natural transit from previous utterance.",+ prev=keep_first_and_last_paragraph(last_utterance),+ content=raw_utterance,+ ).utterance+ cited_searched_results = extract_cited_storm_info(+ response=utterance, index_to_storm_info=index_to_information_mapping+ )+ return dspy.Prediction(+ raw_utterance=raw_utterance,+ utterance=utterance,+ cited_info=cited_searched_results,+ )diff --git a/knowledge_storm/collaborative_storm/modules/information_insertion_module.py b/knowledge_storm/collaborative_storm/modules/information_insertion_module.pynew file mode 100644index 0000000..c858671--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/information_insertion_module.py@@ -0,0 +1,422 @@+import dspy+import numpy as np+import re+import traceback++from concurrent.futures import ThreadPoolExecutor, as_completed+from sklearn.metrics.pairwise import cosine_similarity+from typing import List, Union, Dict, Optional++from .collaborative_storm_utils import trim_output_after_hint+from ...dataclass import KnowledgeNode, KnowledgeBase+from ...encoder import get_text_embeddings+from ...interface import Information+++class InsertInformation(dspy.Signature):+ """Your job is to insert the given information to the knowledge base. The knowledge base is a tree based data structure to organize the collection information. Each knowledge node contains information derived from themantically similar question or intent.+ To decide the best placement of the information, you will be navigated in this tree based data structure layer by layer.+ You will be presented with the question and query leads to ththeis information, and tree structure.++ Output should strictly follow one of options presetned below with no other information.+ - 'insert': to place the information under the current node.+ - 'step: [child node name]': to step into a specified child node.+ - 'create: [new child node name]': to create new child node and insert the info under it.++ Example outputs:+ - insert+ - step: node2+ - create: node3+ """++ intent = dspy.InputField(+ prefix="Question and query leads to this info: ", format=str+ )+ structure = dspy.InputField(prefix="Tree structure: \n", format=str)+ choice = dspy.OutputField(prefix="Choice:\n", format=str)+++class InsertInformationCandidateChoice(dspy.Signature):+ """Your job is to insert the given information to the knowledge base. The knowledge base is a tree based data structure to organize the collection information. Each knowledge node contains information derived from themantically similar question or intent.+ You will be presented with the question and query leads to this information, and candidate choices of placement. In these choices, -> denotes parent-child relationship. Note that reasonable may not be in these choices.++ If there exists reasonable choice, output "Best placement: [choice index]"; otherwise, output "No reasonable choice".+ """++ intent = dspy.InputField(+ prefix="Question and query leads to this info: ", format=str+ )+ choices = dspy.InputField(prefix="Candidate placement:\n", format=str)+ decision = dspy.OutputField(prefix="Decision:\n", format=str)+++class InsertInformationModule(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.insert_info = dspy.ChainOfThought(InsertInformation)+ self.candidate_choosing = dspy.Predict(InsertInformationCandidateChoice)++ def _construct_intent(self, question: str, query: str):+ intent = ""+ if query == "Not applicable":+ return question+ if question:+ intent += f"Question: {question}\n"+ if query:+ intent += f"Query: {query}\n"+ if not intent:+ intent = "Not available."+ return intent++ def _get_navigation_choice(+ self, knowledge_node: KnowledgeNode, question: str, query: str+ ):+ # construct information intent+ intent = self._construct_intent(question, query)+ # construct current kb structure+ structure = f"Current Node: {knowledge_node.name}\n"+ child_names = ", ".join(knowledge_node.get_children_names())+ if child_names:+ structure += f"Child Nodes: {child_names}"+ navigated_path = " -> ".join(knowledge_node.get_path_from_root())+ structure += f"Path you have nagivated: {navigated_path}"++ # get predicted action+ with dspy.settings.context(lm=self.engine):+ predicted_action = self.insert_info(+ intent=intent, structure=structure+ ).choice++ # parse action+ cleaned_predicted_action = trim_output_after_hint(+ predicted_action, "Choice:"+ ).strip()+ cleaned_predicted_action = cleaned_predicted_action.strip("-").strip()+ if cleaned_predicted_action.startswith("insert"):+ return "insert", ""+ elif cleaned_predicted_action.startswith("step:"):+ node_name = trim_output_after_hint(cleaned_predicted_action, "step:")+ return "step", node_name+ elif cleaned_predicted_action.startswith("create:"):+ node_name = trim_output_after_hint(cleaned_predicted_action, "create:")+ return "create", node_name+ raise Exception(+ f"Undefined predicted action in knowledge navigation. {predicted_action}"+ )++ def layer_by_layer_navigation_placement(+ self,+ knowledge_base: KnowledgeBase,+ question: str,+ query: str,+ allow_create_new_node: bool = False,+ root: Optional[KnowledgeNode] = None,+ ):+ current_node: KnowledgeNode = knowledge_base.root if root is None else root++ while True:+ action_type, node_name = self._get_navigation_choice(+ knowledge_node=current_node, question=question, query=query+ )+ if action_type == "insert":+ return dspy.Prediction(+ information_placement=" -> ".join(+ current_node.get_path_from_root(root)+ ),+ note="None",+ )+ elif action_type == "step":+ for child in current_node.children:+ if child.name == node_name:+ current_node = child+ break+ else:+ raise ValueError(f"Child node with name {node_name} not found.")+ elif action_type == "create":+ placement_path = current_node.get_path_from_root(root)+ if allow_create_new_node:+ placement_path.append(node_name)+ note = f"create new node: {{{node_name}}} under {{{current_node.name}}}"+ else:+ note = f"attempt to create new node: {{{node_name}}} under {{{current_node.name}}}"+ return dspy.Prediction(+ information_placement=" -> ".join(placement_path), note=note+ )+ else:+ raise ValueError(f"Unknown action type: {action_type}")++ def _get_sorted_embed_sim_section(+ self,+ encoded_outline: np.ndarray,+ outlines: List[str],+ question: str,+ query: str,+ ):+ if encoded_outline is not None and encoded_outline.size > 0:+ encoded_query, token_usage = get_text_embeddings(f"{question}, {query}")+ sim = cosine_similarity([encoded_query], encoded_outline)[0]+ sorted_indices = np.argsort(sim)+ sorted_outlines = np.array(outlines)[sorted_indices[::-1]]+ return sorted_outlines+ else:+ return outlines++ def _parse_selected_index(self, string: str):+ match = re.search(r"\[(\d+)\]", string)+ if match:+ return int(match.group(1))+ try:+ return int(string.strip())+ except:+ pass+ return None++ def choose_candidate_from_embedding_ranking(+ self,+ question: str,+ query: str,+ encoded_outlines: np.ndarray,+ outlines: List[str],+ top_N_candidates: int = 5,+ ):+ sorted_candidates = self._get_sorted_embed_sim_section(+ encoded_outlines, outlines, question, query+ )+ considered_candidates = sorted_candidates[+ : min(len(sorted_candidates), top_N_candidates)+ ]+ choices_string = "\n".join(+ [+ f"{idx + 1}: {candidate}"+ for idx, candidate in enumerate(considered_candidates)+ ]+ )+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ decision = self.candidate_choosing(+ intent=self._construct_intent(question=question, query=query),+ choices=choices_string,+ ).decision+ decision = trim_output_after_hint(decision, hint="Decision:")+ if "Best placement:" in decision:+ decision = trim_output_after_hint(decision, hint="Best placement:")+ selected_index = self._parse_selected_index(decision)+ if selected_index is not None:+ selected_index = selected_index - 1+ if selected_index < len(sorted_candidates) and selected_index >= 0:+ return dspy.Prediction(+ information_placement=sorted_candidates[selected_index],+ note=f"Choosing from:\n{considered_candidates}",+ )+ return None++ def _info_list_to_intent_mapping(self, information_list: List[Information]):+ intent_to_placement_dict = {}+ for info in information_list:+ intent = (info.meta.get("question", ""), info.meta.get("query", ""))+ if intent not in intent_to_placement_dict:+ intent_to_placement_dict[intent] = None+ return intent_to_placement_dict++ def forward(+ self,+ knowledge_base: KnowledgeBase,+ information: Union[Information, List[Information]],+ allow_create_new_node: bool = False,+ max_thread: int = 5,+ insert_root: Optional[KnowledgeNode] = None,+ skip_candidate_from_embedding: bool = False,+ ):++ if not isinstance(information, List):+ information = [information]+ intent_to_placement_dict: Dict = self._info_list_to_intent_mapping(+ information_list=information+ )++ # process one intent+ def process_intent(question: str, query: str):+ candidate_placement = None+ try:+ if not skip_candidate_from_embedding:+ candidate_placement = self.choose_candidate_from_embedding_ranking(+ question=question,+ query=query,+ encoded_outlines=encoded_outlines,+ outlines=outlines,+ top_N_candidates=8,+ )+ if candidate_placement is None:+ candidate_placement = self.layer_by_layer_navigation_placement(+ knowledge_base=knowledge_base,+ question=question,+ query=query,+ allow_create_new_node=allow_create_new_node,+ root=insert_root,+ )+ return (question, query), candidate_placement+ except Exception as e:+ print(traceback.format_exc())+ return (question, query), None++ def insert_info_to_kb(info, placement_prediction):+ if placement_prediction is not None:+ missing_node_handling = (+ "raise error" if not allow_create_new_node else "create"+ )+ knowledge_base.insert_information(+ path=placement_prediction.information_placement,+ information=info,+ missing_node_handling=missing_node_handling,+ root=insert_root,+ )++ encoded_outlines, outlines = (+ knowledge_base.get_knowledge_base_structure_embedding(root=insert_root)+ )+ to_return = []+ if not allow_create_new_node:+ # use multi thread as knowledge base structure does not change+ with ThreadPoolExecutor(max_workers=max_thread) as executor:+ futures = {+ executor.submit(process_intent, question, query): (question, query)+ for (question, query) in intent_to_placement_dict+ }++ for future in as_completed(futures):+ (question, query), candidate_placement = future.result()+ intent_to_placement_dict[(question, query)] = candidate_placement+ # back mapping placement to each information+ for info in information:+ intent = (info.meta.get("question", ""), info.meta.get("query", ""))+ placement_prediction = intent_to_placement_dict.get(intent, None)+ insert_info_to_kb(info, placement_prediction)+ to_return.append((info, placement_prediction))+ return to_return+ else:+ # use sequential insert as knowledge base structure might change+ for question, query in intent_to_placement_dict:+ encoded_outlines, outlines = (+ knowledge_base.get_knowledge_base_structure_embedding(+ root=insert_root+ )+ )+ _, placement_prediction = process_intent(question=question, query=query)+ intent_to_placement_dict[(question, query)] = placement_prediction++ for info in information:+ intent = (info.meta.get("question", ""), info.meta.get("query", ""))+ placement_prediction = intent_to_placement_dict.get(intent, None)+ insert_info_to_kb(info, placement_prediction)+ to_return.append((info, placement_prediction))+ return to_return+++class ExpandSection(dspy.Signature):+ """Your task is to expand a section in the mind map by creating new subsections under the given section.+ You will be given a list of question and query that are used to collect information.+ Output should be subsection names where each section should serve as a coherent and themantic organization of information and corresponding citation numbers. These subsection names are preferred to be concise and precise.+ Output follows the format below:+ subsection 1+ subsection 2+ subsection 3+ """++ section = dspy.InputField(prefix="The section you need to expand: ", format=str)+ info = dspy.InputField(prefix="The collected information:\n", format=str)+ output = dspy.OutputField(+ prefix="Now provide the expanded subsection names (If there's no need to expand current section as itself serves good organization, then output None):\n",+ format=str,+ )+++class ExpandNodeModule(dspy.Module):+ def __init__(+ self,+ engine: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ information_insert_module: dspy.Module,+ node_expansion_trigger_count: int,+ ):+ self.engine = engine+ self.expand_section = dspy.Predict(ExpandSection)+ self.information_insert_module = information_insert_module+ self.node_expansion_trigger_count = node_expansion_trigger_count++ def _get_cited_info_meta_string(self, node, knowledge_base):+ meta_string = set()+ for index in sorted(list(node.content)):+ info = knowledge_base.info_uuid_to_info_dict[index]+ intent = f"Question: {info.meta['question']}\nQuery: {info.meta['query']}"+ meta_string.add(intent)++ return "\n\n".join(meta_string)++ def _get_expand_subnode_names(self, node, knowledge_base):+ information = self._get_cited_info_meta_string(node, knowledge_base)+ node_path = node.get_path_from_root()+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ output = self.expand_section(section=node_path, info=information).output+ subsections = []+ if "\n" in output and output != "None":+ subsections = output.split("\n")+ # remove any integer followed by a dot and a space, a leading dashline,+ # or a specific hint at the start of the string+ subsections = [+ re.sub(r"^\d+\.\s|-|" + re.escape(node.name), "", text)+ .replace("*", "")+ .strip()+ for text in subsections+ ]+ return subsections++ def _find_first_node_to_expand(+ self, root: KnowledgeNode, expanded_nodes: List[KnowledgeNode]+ ):+ if root is None:+ return None+ if (+ root not in expanded_nodes+ and len(root.content) >= self.node_expansion_trigger_count+ ):+ return root+ for child in root.children:+ to_return = self._find_first_node_to_expand(+ root=child, expanded_nodes=expanded_nodes+ )+ if to_return is not None:+ return to_return+ return None++ def _expand_node(self, node: KnowledgeNode, knowledge_base: KnowledgeBase):+ subsection_names = self._get_expand_subnode_names(node, knowledge_base)+ if len(subsection_names) <= 1:+ return+ # create new nodes+ for subsection_name in subsection_names:+ # remove citation bracket in the subsection name+ subsection_name = re.sub(r"\[.*?\]", "", subsection_name)+ knowledge_base.insert_node(new_node_name=subsection_name, parent_node=node)+ # reset original information placement+ original_cited_index = node.content+ original_cited_information = [+ knowledge_base.info_uuid_to_info_dict[index]+ for index in original_cited_index+ ]+ node.content = set()+ # re-insert under expanded section+ self.information_insert_module(+ knowledge_base=knowledge_base,+ information=original_cited_information,+ allow_create_new_node=False,+ insert_root=node,+ )++ def forward(self, knowledge_base: KnowledgeBase):+ expanded_nodes = []+ while True:+ node_to_expand = self._find_first_node_to_expand(+ root=knowledge_base.root, expanded_nodes=expanded_nodes+ )+ if node_to_expand is None:+ break+ self._expand_node(node=node_to_expand, knowledge_base=knowledge_base)+ expanded_nodes.append(node_to_expand)diff --git a/knowledge_storm/collaborative_storm/modules/knowledge_base_summary.py b/knowledge_storm/collaborative_storm/modules/knowledge_base_summary.pynew file mode 100644index 0000000..fb8e340--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/knowledge_base_summary.py@@ -0,0 +1,32 @@+import dspy+from typing import Union+from ...dataclass import KnowledgeBase+++class KnowledgeBaseSummmary(dspy.Signature):+ """Your job is to give brief summary of what's been discussed in a roundtable conversation. Contents are themantically organized into hierarchical sections.+ You will be presented with these sections where "#" denotes level of section.+ """++ topic = dspy.InputField(prefix="topic: ", format=str)+ structure = dspy.InputField(prefix="Tree structure: \n", format=str)+ output = dspy.OutputField(prefix="Now give brief summary:\n", format=str)+++class KnowledgeBaseSummaryModule(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.gen_summary = dspy.Predict(KnowledgeBaseSummmary)++ def forward(self, knowledge_base: KnowledgeBase):+ structure = knowledge_base.get_node_hierarchy_string(+ include_indent=False,+ include_full_path=False,+ include_hash_tag=True,+ include_node_content_count=False,+ )+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ summary = self.gen_summary(+ topic=knowledge_base.topic, structure=structure+ ).output+ return summarydiff --git a/knowledge_storm/collaborative_storm/modules/simulate_user.py b/knowledge_storm/collaborative_storm/modules/simulate_user.pynew file mode 100644index 0000000..2d03e9d--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/simulate_user.py@@ -0,0 +1,37 @@+import dspy+from typing import List, Union++from .collaborative_storm_utils import extract_and_remove_citations+from ...dataclass import ConversationTurn+from ...storm_wiki.modules.knowledge_curation import AskQuestionWithPersona+++class GenSimulatedUserUtterance(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.ask_qeustion = dspy.Predict(AskQuestionWithPersona)++ def gen_conv_history_string(self, conversation_turns: List[ConversationTurn]):+ conv_history = []+ total_turns = len(conversation_turns)++ for i, turn in enumerate(conversation_turns):+ utterance, _ = extract_and_remove_citations(turn.utterance)+ if i >= total_turns - 4:+ conv_history.append(f"{turn.role}: {utterance}")+ else:+ if turn.claim_to_make:+ conv_history.append(f"{turn.role}: {turn.claim_to_make}")+ else:+ conv_history.append(f"{turn.role}: {utterance}")++ return "\n".join(conv_history)++ def forward(self, topic: str, intent: str, conv_history: List[ConversationTurn]):+ conv_history_string = self.gen_conv_history_string(conv_history)+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ return self.ask_qeustion(+ topic=topic,+ persona=f"researcher with interest in {intent}",+ conv=conv_history_string,+ ).questiondiff --git a/knowledge_storm/collaborative_storm/modules/warmstart_hierarchical_chat.py b/knowledge_storm/collaborative_storm/modules/warmstart_hierarchical_chat.pynew file mode 100644index 0000000..a6351aa--- /dev/null+++ b/knowledge_storm/collaborative_storm/modules/warmstart_hierarchical_chat.py@@ -0,0 +1,398 @@+import dspy+import concurrent.futures+from threading import Lock+from typing import List, Optional, Union, TYPE_CHECKING++from .callback import BaseCallbackHandler+from .collaborative_storm_utils import _get_answer_question_module_instance+from .expert_generation import GenerateExpertModule+from .grounded_question_answering import AnswerQuestionModule+from ...dataclass import ConversationTurn, KnowledgeBase+from ...interface import LMConfigs+from ...logging_wrapper import LoggingWrapper+from ...storm_wiki.modules.outline_generation import WritePageOutline+from ...utils import ArticleTextProcessing as AP+++if TYPE_CHECKING:+ from ..engine import RunnerArgument+++class WarmStartModerator(dspy.Signature):+ """+ You are a moderator in a roundtable discussion. The goal is to chat with multiple experts to discuss the facts and background of the topic to familiarize the audience with the topic.+ You will be presented with the topic, the history of question you have already asked, and the current expert you are discussing with.+ Based on these information, generate the next question for the current expert to further the discussion.++ The output should only include the next question for the current expert. Do not include any other information or preamble.+ """++ topic = dspy.InputField(prefix="Topic for roundtable discussion: ", format=str)+ history = dspy.InputField(+ prefix="Experts you have already interacted with: ", format=str+ )+ current_expert = dspy.InputField(prefix="Expert you are talking with:", format=str)+ question = dspy.OutputField(+ prefix="Next question for the expert you are talking with: ", format=str+ )+++class SectionToConvTranscript(dspy.Signature):+ """+ You are given a section of a brief report on a specific topic. Your task is to transform this section into an engaging opening discussion for a roundtable conversation.+ The goal is to help participants and the audience quickly understand the key information.+ Both question and answer should be in the tone of roundtable discussion talking to audiences.++ Specifically, you need to:+ 1. Generate an engaging question that leverages section name and topic that opens discussion of the content.+ 2. Provide a brief and engaging answer (with all inline citations from original text) derived from the section serving as pointers and avoid too much details.+ """++ topic = dspy.InputField(prefix="topic:", format=str)+ section_name = dspy.InputField(prefix="section name:", format=str)+ section_content = dspy.InputField(prefix="section content:", format=str)+ question = dspy.OutputField(prefix="Now give engaging question only.\nQuestion:")+ answer = dspy.OutputField(+ prefix="Now give engaging answer only with all inline citations from original text.\nAnswer:"+ )+++class ReportToConversation(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.section_to_conv_transcript = dspy.Predict(SectionToConvTranscript)++ def forward(self, knowledge_base: KnowledgeBase):+ def process_node(node, topic):+ with dspy.settings.context(lm=self.engine, show_guidelines=False):+ output = self.section_to_conv_transcript(+ topic=topic,+ section_name=node.get_path_from_root(),+ section_content=node.synthesize_output,+ )+ question = output.question.replace("Question:", "").strip()+ answer = output.answer.replace("Answer:", "").strip()+ return question, answer++ conversations = []+ nodes = knowledge_base.collect_all_nodes()+ nodes = [node for node in nodes if node.name != "root" and node.content]+ topic = knowledge_base.topic++ with concurrent.futures.ThreadPoolExecutor() as executor:+ future_to_node = {+ executor.submit(process_node, node, topic): node for node in nodes+ }+ for future in concurrent.futures.as_completed(future_to_node):+ node = future_to_node[future]+ question, answer = future.result()+ conversations.append(+ ConversationTurn(+ role="Background discussion moderator",+ raw_utterance=question,+ utterance_type="Original Question",+ utterance=question,+ cited_info=[+ knowledge_base.info_uuid_to_info_dict[idx]+ for idx in AP.parse_citation_indices(question)+ ],+ )+ )+ conversations.append(+ ConversationTurn(+ role="Background discussion expert",+ raw_utterance=answer,+ utterance_type="Potential Answer",+ utterance=answer,+ cited_info=[+ knowledge_base.info_uuid_to_info_dict[idx]+ for idx in AP.parse_citation_indices(answer)+ ],+ )+ )+ return conversations+++class WarmStartConversation(dspy.Module):+ def __init__(+ self,+ question_asking_lm: Union[dspy.dsp.LM, dspy.dsp.HFModel],+ generate_expert_module: GenerateExpertModule,+ answer_question_module: AnswerQuestionModule,+ logging_wrapper: LoggingWrapper,+ max_num_experts: int = 3,+ max_turn_per_experts: int = 2,+ max_thread: int = 3,+ callback_handler: BaseCallbackHandler = None,+ ):+ self.ask_question = dspy.Predict(WarmStartModerator)+ self.max_num_experts = max_num_experts+ self.max_turn_per_experts = max_turn_per_experts+ self.question_asking_lm = question_asking_lm+ self.answer_question_module = answer_question_module+ self.max_thread = max_thread+ self.generate_experts_module = generate_expert_module+ self.logging_wrapper = logging_wrapper+ self.callback_handler = callback_handler++ def format_dialogue_question_history_string(+ self, conversation_history: List[ConversationTurn]+ ):+ output = []+ for idx, turn in enumerate(conversation_history):+ info = turn.claim_to_make if turn.claim_to_make else turn.utterance+ output.append(f"{idx + 1}: {info}")+ return "\n".join(output)++ def generate_warmstart_experts(self, topic: str):+ background_seeking_dialogue = self.get_background_info(topic=topic)+ background_info = background_seeking_dialogue.utterance+ gen_expert_output = self.generate_experts_module(+ topic=topic,+ background_info=background_info,+ num_experts=self.max_num_experts,+ )+ return gen_expert_output.experts, background_seeking_dialogue++ def get_background_info(self, topic: str):+ question = f"Background information about {topic}"+ answer = self.answer_question_module(+ topic=topic, question=question, mode="extensive", style="conversational"+ )++ return ConversationTurn(+ role="Default Background Researcher",+ raw_utterance=answer.response,+ utterance_type="Questioning",+ claim_to_make=question,+ queries=answer.queries,+ raw_retrieved_info=answer.raw_retrieved_info,+ cited_info=answer.cited_info,+ )++ def forward(self, topic: str):+ with self.logging_wrapper.log_event(+ "warm start, perspective guided QA: identify experts"+ ):+ # do background research, generate some experts+ experts, background_seeking_dialogue = self.generate_warmstart_experts(+ topic=topic+ )+ # init list to store the dialogue history+ conversation_history: List[ConversationTurn] = []+ lock = Lock()++ # hierarchical chat: chat with one expert. Generate question, get answer+ def process_expert(expert):+ expert_name, expert_descriptoin = expert.split(":")+ for idx in range(self.max_turn_per_experts):+ with self.logging_wrapper.log_event(+ f"warm start, perspective guided QA: expert {expert_name}; turn {idx + 1}"+ ):+ try:+ with lock:+ history = self.format_dialogue_question_history_string(+ conversation_history+ )+ with dspy.settings.context(lm=self.question_asking_lm):+ question = self.ask_question(+ topic=topic, history=history, current_expert=expert+ ).question+ answer = self.answer_question_module(+ topic=topic,+ question=question,+ mode="brief",+ style="conversational",+ )+ conversation_turn = ConversationTurn(+ role=expert,+ claim_to_make=question,+ raw_utterance=answer.response,+ utterance_type="Support",+ queries=answer.queries,+ raw_retrieved_info=answer.raw_retrieved_info,+ cited_info=answer.cited_info,+ )+ if self.callback_handler is not None:+ self.callback_handler.on_warmstart_update(+ message="\n".join(+ [+ f"Finish browsing {url}"+ for url in [+ i.url for i in answer.raw_retrieved_info+ ]+ ]+ )+ )+ with lock:+ conversation_history.append(conversation_turn)+ except Exception as e:+ print(f"Error processing expert {expert}: {e}")++ # multi-thread conversation+ with concurrent.futures.ThreadPoolExecutor(+ max_workers=self.max_thread+ ) as executor:+ futures = [+ executor.submit(process_expert, expert)+ for expert in experts[: min(len(experts), self.max_num_experts)]+ ]+ concurrent.futures.wait(futures)++ conversation_history = [background_seeking_dialogue] + conversation_history++ return dspy.Prediction(+ conversation_history=conversation_history, experts=experts+ )+++class GenerateWarmStartOutline(dspy.Signature):+ """Generate a outline of the wikipedia-like report from a roundtable discussion. You will be presented discussion points in the conversation and corresponding queries.+ You will be given a draft outline which you can borrow some inspiration. Do not include sections that are not mentioned in the given discussion history.+ Use "#" to denote section headings, "##" to denote subsection headings, and so on.+ Follow these guidelines:+ 1. Use "#" for section titles, "##" for subsection titles, "###" for subsubsection titles, and so on.+ 2. Do not include any additional information.+ 3. Exclude the topic name from the outline.+ The organization of outline should adopt wikiepdia style.+ """++ topic = dspy.InputField(prefix="The topic discussed: ", format=str)+ draft = dspy.InputField(prefix="Draft outline you can reference to: ", format=str)+ conv = dspy.InputField(prefix="Discussion history:\n", format=str)+ outline = dspy.OutputField(+ prefix='Write the conversation outline (Use "#" Title" to indicate section title, "##" Title" to indicate subsection title, ...):\n',+ format=str,+ )+++class GenerateWarmStartOutlineModule(dspy.Module):+ def __init__(self, engine: Union[dspy.dsp.LM, dspy.dsp.HFModel]):+ self.engine = engine+ self.gen_outline = dspy.Predict(GenerateWarmStartOutline)+ self.draft_outline = dspy.Predict(WritePageOutline)++ def extract_questions_and_queries(self, conv: List[ConversationTurn]):+ context = []+ for turn in conv:+ focus = turn.claim_to_make+ queries = turn.queries+ queries_string = "\n\t".join(+ f"Query {idx + 1}: {query}" for idx, query in enumerate(queries)+ )+ string = f"Discussion focus {len(context) + 1}: {focus}\n\t{queries_string}"+ context.append(string)+ return "\n".join(context)++ def get_draft_outline(self, topic: str):+ with dspy.settings.context(lm=self.engine):+ return self.draft_outline(topic=topic).outline++ def forward(self, topic: str, conv: List[ConversationTurn]):+ discussion_history = self.extract_questions_and_queries(conv)+ draft_outline = self.get_draft_outline(topic=topic)+ with dspy.settings.context(lm=self.engine):+ outline = self.gen_outline(+ topic=topic, draft=draft_outline, conv=discussion_history+ ).outline+ outline = AP.clean_up_outline(outline)+ return dspy.Prediction(outline=outline, draft_outline=draft_outline)+++class WarmStartModule:+ def __init__(+ self,+ lm_config: LMConfigs,+ runner_argument: "RunnerArgument",+ logging_wrapper: LoggingWrapper,+ rm: Optional[dspy.Retrieve] = None,+ callback_handler: BaseCallbackHandler = None,+ ):+ generate_expert_module = GenerateExpertModule(+ engine=lm_config.discourse_manage_lm+ )+ self.warmstart_conv = WarmStartConversation(+ question_asking_lm=lm_config.question_asking_lm,+ generate_expert_module=generate_expert_module,+ answer_question_module=_get_answer_question_module_instance(+ lm_config=lm_config,+ runner_argument=runner_argument,+ logging_wrapper=logging_wrapper,+ rm=rm,+ ),+ max_num_experts=runner_argument.warmstart_max_num_experts,+ max_turn_per_experts=runner_argument.warmstart_max_turn_per_experts,+ max_thread=runner_argument.warmstart_max_thread,+ logging_wrapper=logging_wrapper,+ callback_handler=callback_handler,+ )+ self.warmstart_outline_gen_module = GenerateWarmStartOutlineModule(+ engine=lm_config.warmstart_outline_gen_lm+ )+ self.report_to_conversation = ReportToConversation(lm_config.knowledge_base_lm)+ self.logging_wrapper = logging_wrapper+ self.callback_handler = callback_handler++ def initiate_warm_start(self, topic: str, knowledge_base: KnowledgeBase):+ """+ Initiates a warm start process for the given topic by generating a warm start conversation and inserting the+ resulting information into a knowledge base.++ Args:+ topic (str): The topic for which to initiate the warm start process.++ Returns:+ Tuple[List[ConversationTurn], List[str], KnowledgeBase]:+ - A list of ConversationTurn instances representing the conversation history.+ - A list of strings representing the experts involved in the conversation.+ - A KnowledgeBase instance containing the organized information.+ """+ warm_start_conversation_history: List[ConversationTurn] = []+ warm_start_experts = None+ # get warm start conversations+ with self.logging_wrapper.log_event("warm start: perspective guided QA"):+ if self.callback_handler is not None:+ self.callback_handler.on_warmstart_update(+ message="Start getting familiar with the topic by chatting with multiple LLM experts (Step 1 / 4)"+ )+ warm_start_result = self.warmstart_conv(topic=topic)+ warm_start_conversation_history = warm_start_result.conversation_history+ warm_start_experts = warm_start_result.experts++ # get warm start conv outline+ with self.logging_wrapper.log_event("warm start: outline generation"):+ if self.callback_handler is not None:+ self.callback_handler.on_warmstart_update(+ "Organizing collected information (Step 2 / 4)"+ )+ warm_start_outline_output = self.warmstart_outline_gen_module(+ topic=topic, conv=warm_start_conversation_history+ )+ # init knowledge base+ with self.logging_wrapper.log_event("warm start: insert into knowledge base"):+ if self.callback_handler is not None:+ self.callback_handler.on_warmstart_update(+ "Inserting collected information into knowledge base (Step 3 / 4)"+ )+ knowledge_base.insert_from_outline_string(+ outline_string=warm_start_outline_output.outline+ )+ # insert information to knowledge base+ for turn in warm_start_conversation_history:+ knowledge_base.update_from_conv_turn(+ conv_turn=turn, allow_create_new_node=False+ )+ # knowledge base to report+ if self.callback_handler is not None:+ self.callback_handler.on_warmstart_update(+ "Synthesizing background information discussion utterances (Step 4 / 4)"+ )+ knowledge_base.to_report()++ # generate engaging conversations+ engaging_conversations = self.report_to_conversation(knowledge_base)+ return (+ warm_start_conversation_history,+ engaging_conversations,+ warm_start_experts,+ )