Building AI agents, atomically
Add more elaborate example for a deep research agent
KennyVaneetvelde68cabecf42bf
diff --git a/examples/deep_research_multi_agent/article_outline_generator.py b/examples/deep_research_multi_agent/article_outline_generator.pydeleted file mode 100644index da6236d..0000000--- a/examples/deep_research_multi_agent/article_outline_generator.py+++ /dev/null@@ -1,128 +0,0 @@-###############################-# WARNING: NOT YET FINISHED! #-###############################---import os-from rich.console import Console-from pydantic import BaseModel, Field-from typing import List-from rich.markdown import Markdown-from atomic_agents.lib.components.chat_memory import ChatMemory-from atomic_agents.agents.base_chat_agent import BaseChatAgent, BaseChatAgentConfig-from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo-import instructor-import openai--from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool, SearxNGSearchToolConfig-from examples.deep_research_multi_agent.deep_query_agent import DeepQueryAgentInputSchema, deep_query_agent--search_tool = SearxNGSearchTool(SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=30))--# Define the Pydantic schema for the article outline-class Subsection(BaseModel):- title: str = Field(..., description="The title of the subsection. This field is required.")- content: str = Field(..., description="A brief description or content of the subsection. This field is required.")--class Section(BaseModel):- title: str = Field(..., description="The title of the section. This field is required.")- subsections: List[Subsection] = Field(..., description="A list of subsections under this section. This field is required.")--class ArticleOutline(BaseModel):- title: str = Field(..., description="The title of the article. This field is required.")- sections: List[Section] = Field(..., description="A list of sections in the article. This field is required.")--# Define system prompt information including background, steps, and output instructions-system_prompt = SystemPromptInfo(- background=[- 'This assistant is an expert at generating article outlines based on user input.',- ],- steps=[- '1. Understand the user\'s input and identify the main topic of the article.',- '2. Generate a relevant article outline based on the identified topic, including sections and subsections, based on search results and user input.',- '3. Ensure the outline includes sections and subsections with appropriate titles and detailed descriptions.',- '4. Each section should have a clear and concise title that reflects its content.',- '5. Each subsection should provide a brief but comprehensive description of its content.',- '6. Review the generated outline to ensure it is well-structured and logically organized.',- ],- output_instructions=[- 'Provide a structured article outline in the form of sections and subsections.',- 'Be clear and concise in the titles and descriptions.',- 'Ensure that all fields are filled and nothing is left blank.',- ]-)--# Initialize the system prompt generator with the defined system prompt and dynamic info providers-system_prompt_generator = SystemPromptGenerator(system_prompt)--# Initialize chat memory to store conversation history-memory = ChatMemory()-# Define initial memory with a greeting message from the assistant-initial_memory = [- {'role': 'assistant', 'content': 'How do you do? What can I do for you? Tell me, pray, what is your need today?'}-]-# Load the initial memory into the chat memory-memory.load(initial_memory)--def format_search_results(search_results: List[SearxNGSearchTool.output_schema]) -> str:- formatted_results = []- for idx, result in enumerate(search_results):- formatted_results.append(f"{idx+1}. {result.title} - {result.content}")- return "\n".join(formatted_results)--class ArticleOutlineGenAgentConfig(BaseChatAgentConfig):- pass--class ArticleOutlineGenAgent(BaseChatAgent):- def _pre_run(self):- # For ease of demonstration, we will pass the client- self.memory.add_message('assistant', 'First, I will ask the deep research agent to generate a list of deep research queries.')- response = self.get_response(response_model=DeepQueryAgentInputSchema)- deep_queries = deep_query_agent.run(response.user_input)- self.memory.add_message('system', f'Queries: {deep_queries}')- search_results = search_tool.run(deep_queries)- formatted_results = format_search_results(search_results.results)- self.memory.add_message('system', f'Search results:\n{formatted_results}')- self.memory.add_message('assistant', 'I have gathered the search results and will now generate an article outline based on the information.')- return---# Create a chat agent with the specified model, system prompt generator, and memory-# For all supported clients such as Anthropic & Gemini, have a look at the `instructor` library documentation.-agent = ArticleOutlineGenAgent(- config=ArticleOutlineGenAgentConfig(- client=instructor.from_openai(openai.OpenAI()),- system_prompt_generator=system_prompt_generator,- model='gpt-3.5-turbo',- memory=memory,- output_schema=ArticleOutline- )-)---############################################################################################################-# Main chat loop for testing the chat agent-############################################################################################################-console = Console()-console.print(f'Agent: {initial_memory[0]["content"]}')--while True:- user_input = input('You: ')- if user_input.lower() in ['/exit', '/quit']:- print('Exiting chat...')- break-- response = agent.run(agent.input_schema(chat_input=user_input))- response_dict = response.dict()-- def format_markdown_section(title, items):- if isinstance(items, list):- return f"## {title}\n" + "\n".join([f"- {item['title']}: {item['content']}" for item in items]) + "\n"- return f"## {title}\n{items}\n"-- markdown_string = f"# {response_dict['title']}\n"- for section in response_dict['sections']:- markdown_string += format_markdown_section(section['title'], section['subsections'])-- markdown_response = Markdown(markdown_string)- console.print(markdown_response)\ No newline at end of filediff --git a/examples/deep_research_multi_agent/deep_query_agent.py b/examples/deep_research_multi_agent/deep_query_agent.pydeleted file mode 100644index b9f8a49..0000000--- a/examples/deep_research_multi_agent/deep_query_agent.py+++ /dev/null@@ -1,106 +0,0 @@-###############################-# WARNING: NOT YET FINISHED! #-###############################--from datetime import datetime-import os--from pydantic import BaseModel, Field-from rich.console import Console-from rich.markdown import Markdown--import instructor-import openai--from atomic_agents.agents.base_chat_agent import BaseChatAgent, BaseChatAgentConfig-from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase, SystemPromptGenerator, SystemPromptInfo-from atomic_agents.lib.tools.base import BaseTool-from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool, SearxNGSearchToolConfig--# Initialize tools-search_tool = SearxNGSearchTool(SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=15))--class CurrentDateProvider(SystemPromptContextProviderBase):- def __init__(self, format: str = '%Y-%m-%d %H:%M:%S', title: str = 'Current date'):- super().__init__(title=title)- self.format = format-- def get_info(self) -> str:- return f'The current date, in the format "{self.format}", is {datetime.now().strftime(self.format)}'--system_prompt_generator = SystemPromptGenerator(SystemPromptInfo(- background=[- 'This assistant is specialized in generating deep research queries to help users find the information they need.',- ],- steps=[- 'First, generate generic queries to get a broad overview of the topic and do a web search',- 'Based on the content of the initial search results, generate a series of deep research queries to explore the topic in more depth and provide a comprehensive overview of the topic and related topics.',- 'Return this second set of queries to the user for further exploration.'- ],- output_instructions=[- 'Each query should be well-structured and focused on a specific topic.',- 'Ensure that the queries are clear and concise to help the user find the information they need.',- 'Each query must be diverse yet relevant to the user\'s needs, providing a comprehensive overview of the topic and related topics.'- ],- context_providers={- 'date': CurrentDateProvider(title='Current date', format='%Y-%m-%d %H:%M:%S')- }-))--class DeepQueryAgentInputSchema(BaseModel):- user_input: str = Field(..., description='The user input to generate deep research queries based on.')-- class Config:- title = 'DeepQueryAgentInputSchema'- description = 'The input schema for the DeepQueryAgent.'- json_schema_extra = {- title: 'DeepQueryAgentInputSchema',- description: 'The input schema for the DeepQueryAgent.'- }--class DeepQueryAgentConfig(BaseChatAgentConfig):- search_tool_instance: BaseTool--class DeepQueryAgent(BaseChatAgent):- def __init__(self, config: DeepQueryAgentConfig):- super().__init__(config)- self.search_tool = config.search_tool_instance-- def _get_and_handle_response(self):- """- Handles obtaining and processing the response.-- Returns:- Type[BaseModel]: The processed response.- """- self.memory.add_message('assistant', 'First, I will generate 3 generic queries to get a broad overview of the topic.')- initial_response = self.get_response(response_model=self.output_schema)- search_results = self.search_tool.run(initial_response)- self.memory.add_message(- 'assistant', (- 'Here are the search results based on the initial response:\n'- f'{search_results}\n\n'- '==========='- 'Now, I will generate a series of 7 deep diverse and detailed research queries based on the initial search results to explore the topic and closely related topics in more depth.'- )- )- deep_queries = self.get_response(response_model=self.output_schema)- return deep_queries--# Initialize the BaseChatAgent with the configured client, system prompt generator, and search toool schema.-deep_query_agent = DeepQueryAgent(- config=DeepQueryAgentConfig(- client=instructor.from_openai(openai.OpenAI()),- system_prompt_generator=system_prompt_generator,- model='gpt-3.5-turbo',- output_schema=search_tool.input_schema,- search_tool_instance=search_tool- )-)--if __name__ == "__main__":- # We can test the agent by running a topic through it and observing the generated queries.- console = Console()- output = deep_query_agent.run(deep_query_agent.input_schema(chat_input='Apple intelligence'))- console.print(output)-\ No newline at end of filediff --git a/examples/deep_research_multi_agent/in_memory_faiss.py b/examples/deep_research_multi_agent/in_memory_faiss.pynew file mode 100644index 0000000..f00c9da--- /dev/null+++ b/examples/deep_research_multi_agent/in_memory_faiss.py@@ -0,0 +1,78 @@+import os+import openai+import faiss+import numpy as np+import html2text+import asyncio+import aiohttp+from playwright.async_api import async_playwright+from bs4 import BeautifulSoup+import markdownify++class InMemFaiss:+ def __init__(self, openai_api_key):+ openai.api_key = openai_api_key+ self.index = faiss.IndexFlatL2(1536)+ self.texts = []+ self.metadata = []++ async def download_content(self, url):+ async with async_playwright() as p:+ browser = await p.chromium.launch(headless=True)+ page = await browser.new_page()+ await page.goto(url)+ html_content = await page.content()+ await browser.close()++ # Clean HTML content+ cleaned_html = self.clean_html(html_content)++ # Convert cleaned HTML to Markdown+ markdown_content = markdownify.markdownify(cleaned_html, heading_style="ATX")+ return markdown_content++ def clean_html(self, html_content):+ soup = BeautifulSoup(html_content, 'html.parser')++ # Example cleaning: remove script and style tags+ for script_or_style in soup(["script", "style"]):+ script_or_style.decompose()++ # Additional cleaning can be done here+ cleaned_html = str(soup)+ return cleaned_html++ def split_text(self, text, chunk_size=1000, overlap=250):+ chunks = []+ for i in range(0, len(text), chunk_size - overlap):+ chunk = text[i:i + chunk_size].strip()+ chunks.append(chunk)+ return chunks++ async def generate_embeddings(self, texts):+ embeddings = []+ for text in texts:+ response = await openai.Embedding.create(input=text, model="text-embedding-3-small")+ embeddings.append(response['data'][0]['embedding'])+ return np.array(embeddings).astype('float32')++ async def ingest_urls(self, urls):+ tasks = [self.process_url(url) for url in urls]+ await asyncio.gather(*tasks)++ async def process_url(self, url):+ try:+ url = str(url)+ content = await self.download_content(url)+ chunks = self.split_text(content)+ embeddings = await self.generate_embeddings(chunks)+ self.index.add(embeddings)+ self.texts.extend(chunks)+ self.metadata.extend([{"url": url}] * len(chunks))+ except Exception as e:+ print(f"Failed to ingest URL {url}: {e}")++ async def retrieve_chunks(self, query, top_k=5):+ query_embedding = await self.generate_embeddings([query])+ distances, indices = self.index.search(query_embedding, top_k)+ return [self.texts[idx] for idx in indices[0]]\ No newline at end of filediff --git a/examples/deep_research_multi_agent/info_refiner.py b/examples/deep_research_multi_agent/info_refiner.pynew file mode 100644index 0000000..8ce2c1f--- /dev/null+++ b/examples/deep_research_multi_agent/info_refiner.py@@ -0,0 +1,47 @@+from pydantic import BaseModel, Field+from atomic_agents.agents.base_chat_agent import BaseAgentIO+import instructor+import openai+from atomic_agents.agents.base_chat_agent import BaseChatAgent, BaseChatAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo+from examples.deep_research_multi_agent.providers import vector_db_chunks_provider+from examples.deep_research_multi_agent.providers import current_date_provider++class RefineAnswerInputSchema(BaseAgentIO):+ question: str = Field(..., description='The question that was asked.')+ answer: str = Field(..., description='The initial answer to the question.')++class RefineAnswerOutputSchema(BaseModel):+ refined_answer: str = Field(..., description='The refined answer to the question.')++# Create the refine answer agent+refine_answer_agent = BaseChatAgent(+ BaseChatAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model='gpt-3.5-turbo',+ system_prompt_generator=SystemPromptGenerator(+ SystemPromptInfo(+ background=[+ "You are an intelligent answer refinement expert.",+ "Your task is to expand and elaborate on an existing answer to a question using additional context from vector DB chunks."+ ],+ steps=[+ "You will receive a question or instruction, the initial answer, and additional context from vector DB chunks.",+ "Expand and elaborate on the initial answer using the additional context to provide a more comprehensive and detailed response."+ ],+ output_instructions=[+ "Ensure the refined answer is clear, concise, and well-structured.",+ "Ensure the refined answer is directly relevant to the question and incorporates the additional context provided.",+ "Add new information and details to make the final answer more elaborate and informative.",+ "Do not make up any new information; only use the information present in the context."+ ]+ )+ ),+ input_schema=RefineAnswerInputSchema,+ output_schema=RefineAnswerOutputSchema+ )+)++# Register the context providers+refine_answer_agent.register_context_provider('date', current_date_provider)+refine_answer_agent.register_context_provider('vector_db_chunks', vector_db_chunks_provider)\ No newline at end of filediff --git a/examples/deep_research_multi_agent/main.py b/examples/deep_research_multi_agent/main.pynew file mode 100644index 0000000..5ce3eed--- /dev/null+++ b/examples/deep_research_multi_agent/main.py@@ -0,0 +1,71 @@+import os+from examples.deep_research_multi_agent.query_agent import query_agent+from examples.deep_research_multi_agent.question_answering import answer_agent+from examples.deep_research_multi_agent.top_urls_selector import top_urls_selector_agent+from examples.deep_research_multi_agent.info_refiner import refine_answer_agent+from examples.deep_research_multi_agent.providers import search_results_provider, current_date_provider, vector_db_chunks_provider+from examples.deep_research_multi_agent.in_memory_faiss import InMemFaiss+from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool, SearxNGSearchToolConfig+from rich.console import Console++console = Console()++# Initialize the search tool+search_tool = SearxNGSearchTool(SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=30))+in_mem_faiss = InMemFaiss(openai_api_key=os.getenv('OPENAI_API_KEY'))++user_input = "Create an in-depth table of upcoming PC games for july and august 2024"++while True:+ if user_input.lower() == 'exit':+ break++ console.print("[bold green]Getting queries...[/bold green]")+ queries = query_agent.run(query_agent.input_schema(instruction=user_input, num_queries=5))++ # Start loop here+ console.print("[bold green]Registering context providers...[/bold green]")+ answer_agent.register_context_provider('date', current_date_provider)+ answer_agent.register_context_provider('search_results', search_results_provider)++ console.print("[bold green]Getting search results...[/bold green]")+ search_results_provider.search_results = search_tool.run(search_tool.input_schema(**queries.model_dump()))++ # Select top URLs+ console.print("[bold green]Selecting top URLs...[/bold green]")+ top_urls = top_urls_selector_agent.run(top_urls_selector_agent.input_schema(+ user_input=user_input,+ num_urls=5+ ))++ # Call the answer agent+ console.print("[bold green]Calling the answer agent...[/bold green]")+ initial_answer = answer_agent.run(answer_agent.input_schema(question=user_input))+ console.print(initial_answer.markdown_output, markup=True)++ console.print("[bold green]=====================[/bold green]")++ console.print("[bold green]Ingesting URLs into in-memory FAISS...[/bold green]")+ in_mem_faiss.ingest_urls(top_urls.top_urls)++ # Retrieve chunks+ console.print("[bold green]Retrieving chunks...[/bold green]")+ query = user_input+ top_k = 5+ chunks = in_mem_faiss.retrieve_chunks(query, top_k)++ vector_db_chunks_provider.chunks = chunks++ console.print("[bold green]Refining the answer...[/bold green]")+ refined_answer = refine_answer_agent.run(refine_answer_agent.input_schema(+ question=user_input,+ answer=initial_answer.markdown_output+ ))++ console.print("Refined Answer:")+ console.print(refined_answer.refined_answer, markup=True)++ # We don't need to keep the context for this agent after each run. This will save money.+ refine_answer_agent.reset_memory()++ user_input = input("Enter your question (or type 'exit' to quit): ")\ No newline at end of filediff --git a/examples/deep_research_multi_agent/providers.py b/examples/deep_research_multi_agent/providers.pynew file mode 100644index 0000000..e655186--- /dev/null+++ b/examples/deep_research_multi_agent/providers.py@@ -0,0 +1,29 @@+from datetime import datetime++from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase+++class CurrentDateProvider(SystemPromptContextProviderBase):+ def get_info(self) -> str:+ date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')+ return f'The current date in the format "%Y-%m-%d %H:%M:%S" is {date}'++class SearchResultsProvider(SystemPromptContextProviderBase):+ def __init__(self, title: str):+ super().__init__(title=title)+ self.search_results = []++ def get_info(self) -> str:+ return f'SEARCH RESULTS: {self.search_results}'++class VectorDBChunksProvider(SystemPromptContextProviderBase):+ def __init__(self, title: str):+ super().__init__(title=title)+ self.chunks = []++ def get_info(self) -> str:+ return f'VECTOR DB CHUNKS: {self.chunks}'++search_results_provider = SearchResultsProvider(title='Search results')+current_date_provider = CurrentDateProvider(title='Current date')+vector_db_chunks_provider = VectorDBChunksProvider(title='Vector DB chunks')\ No newline at end of filediff --git a/examples/deep_research_multi_agent/query_agent.py b/examples/deep_research_multi_agent/query_agent.pynew file mode 100644index 0000000..15cf2a8--- /dev/null+++ b/examples/deep_research_multi_agent/query_agent.py@@ -0,0 +1,42 @@+import os+import groq+import instructor+import openai+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo+from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool+from examples.deep_research_multi_agent.providers import CurrentDateProvider++class QueryAgentInputSchema(BaseAgentIO):+ instruction: str = 'A detailed instruction or request to generate deep research queries for.'+ num_queries: int = 'The number of queries to generate.'++# Create the query agent+query_agent = BaseChatAgent(+ BaseChatAgentConfig(+ client=instructor.from_groq(groq.Groq(api_key=os.getenv('GROQ_API_KEY')))+ model='gpt-3.5-turbo',+ system_prompt_generator=SystemPromptGenerator(+ SystemPromptInfo(+ background=[+ "You are an intelligent query generation expert.",+ "Your task is to generate a specified number of diverse and highly relevant queries based on a given instruction or request.",+ "The queries should cover different aspects of the instruction to ensure comprehensive exploration."+ ],+ steps=[+ "You will receive a detailed instruction or request and the number of queries to generate.",+ "Generate the requested number of queries in a JSON format."+ ],+ output_instructions=[+ "Ensure clarity and conciseness in each query.",+ "Ensure each query is unique and as diverse as possible while remaining relevant to the instruction."+ ]+ )+ ),+ input_schema=QueryAgentInputSchema,+ output_schema=SearxNGSearchTool.input_schema+ )+)++# Register the new context provider+query_agent.register_context_provider('date', CurrentDateProvider(title='Current date'))\ No newline at end of filediff --git a/examples/deep_research_multi_agent/question_answering.py b/examples/deep_research_multi_agent/question_answering.pynew file mode 100644index 0000000..80d7af8--- /dev/null+++ b/examples/deep_research_multi_agent/question_answering.py@@ -0,0 +1,38 @@+import instructor+import openai+from pydantic import BaseModel, Field+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo+from examples.deep_research_multi_agent.providers import current_date_provider++class AnswerAgentInputSchema(BaseAgentIO):+ question: str = Field(..., description='A question that needs to be answered based on the provided context.')++class AnswerAgentOutputSchema(BaseModel):+ markdown_output: str = Field(..., description='The answer to the question in markdown format.')++# Create the answer agent+answer_agent = BaseChatAgent(+ BaseChatAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model='gpt-3.5-turbo',+ system_prompt_generator=SystemPromptGenerator(+ SystemPromptInfo(+ background=[+ "You are an intelligent answering expert.",+ "Your task is to provide accurate and detailed answers to user questions based on the given context."+ ],+ steps=[+ "You will receive a question and the context information.",+ "Generate a detailed and accurate answer based on the context."+ ],+ output_instructions=[+ "Ensure clarity and conciseness in each answer.",+ "Ensure the answer is directly relevant to the question and context provided."+ ]+ )+ ),+ input_schema=AnswerAgentInputSchema,+ output_schema=AnswerAgentOutputSchema+ )+)\ No newline at end of filediff --git a/examples/deep_research_multi_agent/scraper.py b/examples/deep_research_multi_agent/scraper.pynew file mode 100644index 0000000..f80d8ef--- /dev/null+++ b/examples/deep_research_multi_agent/scraper.py@@ -0,0 +1,78 @@+import os+from bs4 import BeautifulSoup+import requests+from markdownify import MarkdownConverter+from pydantic import BaseModel, Field++class WebDocumentMetadata(BaseModel):+ url: str+ title: str = Field(default='')+ description: str = Field(default='')+ keywords: str = Field(default='')+ author: str = Field(default='')++class VectorDBDocumentMetadata(WebDocumentMetadata):+ document_part: int = Field(default=1)+ document_total_parts: int = Field(default=1)++class Document(BaseModel):+ content: str+ metadata: WebDocumentMetadata | VectorDBDocumentMetadata = Field(default_factory=WebDocumentMetadata)++++class UrlToMarkdownConverter:+ METADATA_ATTRS = {+ 'url': '',+ 'title': ('title', None),+ 'description': ('meta', {'name': 'description'}),+ 'keywords': ('meta', {'name': 'keywords'}),+ 'author': ('meta', {'name': 'author'})+ }++ @staticmethod+ def _fetch_url_content(url):+ response = requests.get(url)+ response.raise_for_status()+ return response.text++ @staticmethod+ def _parse_html(html):+ return BeautifulSoup(html, 'html.parser')++ @staticmethod+ def _extract_metadata(soup, url):+ metadata_dict = {}+ for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():+ if selector:+ element = soup.find(*selector)+ metadata_dict[attr] = element.get('content', '').strip() if element else ''+ else:+ metadata_dict[attr] = url+ return WebDocumentMetadata(**metadata_dict)++ @staticmethod+ def _convert_to_markdown(html):+ class CustomMarkdownConverter(MarkdownConverter):+ def convert_img(self, el, text, convert_as_inline):+ return ''+ def convert_p(self, el, text, convert_as_inline):+ return text + '\n\n'++ for script in html.find_all('script'):+ script.decompose()++ for style in html.find_all('style'):+ style.decompose()++ converter = CustomMarkdownConverter()+ return converter.convert(str(html)).strip()++ @staticmethod+ def convert(url):+ html = UrlToMarkdownConverter._fetch_url_content(url)+ soup = UrlToMarkdownConverter._parse_html(html)+ metadata = UrlToMarkdownConverter._extract_metadata(soup, url)+ markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)++ return Document(content=markdown_content, metadata=metadata)\ No newline at end of filediff --git a/examples/deep_research_multi_agent/top_urls_selector.py b/examples/deep_research_multi_agent/top_urls_selector.pynew file mode 100644index 0000000..3072c4e--- /dev/null+++ b/examples/deep_research_multi_agent/top_urls_selector.py@@ -0,0 +1,44 @@+from typing import List+import instructor+import openai+from pydantic import BaseModel, Field, HttpUrl+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo+from examples.deep_research_multi_agent.providers import current_date_provider, search_results_provider++class TopUrlsSelectorInputSchema(BaseAgentIO):+ user_input: str = Field(..., description='The user input or question.')+ num_urls: int = Field(..., description='The number of top URLs to select.')++class TopUrlsSelectorOutputSchema(BaseModel):+ top_urls: List[HttpUrl] = Field(..., description='The list of top URLs selected based on the user input.')++# Create the top URLs selector agent+top_urls_selector_agent = BaseChatAgent(+ BaseChatAgentConfig(+ client=instructor.from_openai(openai.OpenAI()),+ model='gpt-3.5-turbo',+ system_prompt_generator=SystemPromptGenerator(+ SystemPromptInfo(+ background=[+ "You are an intelligent URL selection expert.",+ "Your task is to select the best URLs from a list of search results that are most relevant to the user input or question."+ ],+ steps=[+ "You will receive the user input or question, the list of search results, and the number of best URLs to select.",+ "Analyze the search results and select the best URLs that are most relevant to the user input or question.",+ ],+ output_instructions=[+ "Ensure each selected URL is highly relevant to the user input or question.",+ "Ensure the selected URLs are diverse yet as relevant as possible."+ ]+ )+ ),+ input_schema=TopUrlsSelectorInputSchema,+ output_schema=TopUrlsSelectorOutputSchema+ )+)++# Register the context providers+top_urls_selector_agent.register_context_provider('date', current_date_provider)+top_urls_selector_agent.register_context_provider('search_results', search_results_provider)\ No newline at end of filediff --git a/examples/notebooks/shared_context.ipynb b/examples/notebooks/shared_context.ipynbnew file mode 100644index 0000000..e69de29