Skip to content

Building AI agents, atomically

Add more elaborate example for a deep research agent

KennyVaneetvelde68cabecf42bf
diff --git a/examples/deep_research_multi_agent/article_outline_generator.py b/examples/deep_research_multi_agent/article_outline_generator.py
deleted file mode 100644
index da6236d..0000000
--- a/examples/deep_research_multi_agent/article_outline_generator.py
+++ /dev/null
@@ -1,128 +0,0 @@
-###############################
-# WARNING: NOT YET FINISHED! #
-###############################
-
-
-import os
-from rich.console import Console
-from pydantic import BaseModel, Field
-from typing import List
-from rich.markdown import Markdown
-from atomic_agents.lib.components.chat_memory import ChatMemory
-from atomic_agents.agents.base_chat_agent import BaseChatAgent, BaseChatAgentConfig
-from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo
-import instructor
-import openai
-
-from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool, SearxNGSearchToolConfig
-from examples.deep_research_multi_agent.deep_query_agent import DeepQueryAgentInputSchema, deep_query_agent
-
-search_tool = SearxNGSearchTool(SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=30))
-
-# Define the Pydantic schema for the article outline
-class Subsection(BaseModel):
- title: str = Field(..., description="The title of the subsection. This field is required.")
- content: str = Field(..., description="A brief description or content of the subsection. This field is required.")
-
-class Section(BaseModel):
- title: str = Field(..., description="The title of the section. This field is required.")
- subsections: List[Subsection] = Field(..., description="A list of subsections under this section. This field is required.")
-
-class ArticleOutline(BaseModel):
- title: str = Field(..., description="The title of the article. This field is required.")
- sections: List[Section] = Field(..., description="A list of sections in the article. This field is required.")
-
-# Define system prompt information including background, steps, and output instructions
-system_prompt = SystemPromptInfo(
- background=[
- 'This assistant is an expert at generating article outlines based on user input.',
- ],
- steps=[
- '1. Understand the user\'s input and identify the main topic of the article.',
- '2. Generate a relevant article outline based on the identified topic, including sections and subsections, based on search results and user input.',
- '3. Ensure the outline includes sections and subsections with appropriate titles and detailed descriptions.',
- '4. Each section should have a clear and concise title that reflects its content.',
- '5. Each subsection should provide a brief but comprehensive description of its content.',
- '6. Review the generated outline to ensure it is well-structured and logically organized.',
- ],
- output_instructions=[
- 'Provide a structured article outline in the form of sections and subsections.',
- 'Be clear and concise in the titles and descriptions.',
- 'Ensure that all fields are filled and nothing is left blank.',
- ]
-)
-
-# Initialize the system prompt generator with the defined system prompt and dynamic info providers
-system_prompt_generator = SystemPromptGenerator(system_prompt)
-
-# Initialize chat memory to store conversation history
-memory = ChatMemory()
-# Define initial memory with a greeting message from the assistant
-initial_memory = [
- {'role': 'assistant', 'content': 'How do you do? What can I do for you? Tell me, pray, what is your need today?'}
-]
-# Load the initial memory into the chat memory
-memory.load(initial_memory)
-
-def format_search_results(search_results: List[SearxNGSearchTool.output_schema]) -> str:
- formatted_results = []
- for idx, result in enumerate(search_results):
- formatted_results.append(f"{idx+1}. {result.title} - {result.content}")
- return "\n".join(formatted_results)
-
-class ArticleOutlineGenAgentConfig(BaseChatAgentConfig):
- pass
-
-class ArticleOutlineGenAgent(BaseChatAgent):
- def _pre_run(self):
- # For ease of demonstration, we will pass the client
- self.memory.add_message('assistant', 'First, I will ask the deep research agent to generate a list of deep research queries.')
- response = self.get_response(response_model=DeepQueryAgentInputSchema)
- deep_queries = deep_query_agent.run(response.user_input)
- self.memory.add_message('system', f'Queries: {deep_queries}')
- search_results = search_tool.run(deep_queries)
- formatted_results = format_search_results(search_results.results)
- self.memory.add_message('system', f'Search results:\n{formatted_results}')
- self.memory.add_message('assistant', 'I have gathered the search results and will now generate an article outline based on the information.')
- return
-
-
-# Create a chat agent with the specified model, system prompt generator, and memory
-# For all supported clients such as Anthropic & Gemini, have a look at the `instructor` library documentation.
-agent = ArticleOutlineGenAgent(
- config=ArticleOutlineGenAgentConfig(
- client=instructor.from_openai(openai.OpenAI()),
- system_prompt_generator=system_prompt_generator,
- model='gpt-3.5-turbo',
- memory=memory,
- output_schema=ArticleOutline
- )
-)
-
-
-############################################################################################################
-# Main chat loop for testing the chat agent
-############################################################################################################
-console = Console()
-console.print(f'Agent: {initial_memory[0]["content"]}')
-
-while True:
- user_input = input('You: ')
- if user_input.lower() in ['/exit', '/quit']:
- print('Exiting chat...')
- break
-
- response = agent.run(agent.input_schema(chat_input=user_input))
- response_dict = response.dict()
-
- def format_markdown_section(title, items):
- if isinstance(items, list):
- return f"## {title}\n" + "\n".join([f"- {item['title']}: {item['content']}" for item in items]) + "\n"
- return f"## {title}\n{items}\n"
-
- markdown_string = f"# {response_dict['title']}\n"
- for section in response_dict['sections']:
- markdown_string += format_markdown_section(section['title'], section['subsections'])
-
- markdown_response = Markdown(markdown_string)
- console.print(markdown_response)
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/deep_query_agent.py b/examples/deep_research_multi_agent/deep_query_agent.py
deleted file mode 100644
index b9f8a49..0000000
--- a/examples/deep_research_multi_agent/deep_query_agent.py
+++ /dev/null
@@ -1,106 +0,0 @@
-###############################
-# WARNING: NOT YET FINISHED! #
-###############################
-
-from datetime import datetime
-import os
-
-from pydantic import BaseModel, Field
-from rich.console import Console
-from rich.markdown import Markdown
-
-import instructor
-import openai
-
-from atomic_agents.agents.base_chat_agent import BaseChatAgent, BaseChatAgentConfig
-from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase, SystemPromptGenerator, SystemPromptInfo
-from atomic_agents.lib.tools.base import BaseTool
-from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool, SearxNGSearchToolConfig
-
-# Initialize tools
-search_tool = SearxNGSearchTool(SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=15))
-
-class CurrentDateProvider(SystemPromptContextProviderBase):
- def __init__(self, format: str = '%Y-%m-%d %H:%M:%S', title: str = 'Current date'):
- super().__init__(title=title)
- self.format = format
-
- def get_info(self) -> str:
- return f'The current date, in the format "{self.format}", is {datetime.now().strftime(self.format)}'
-
-system_prompt_generator = SystemPromptGenerator(SystemPromptInfo(
- background=[
- 'This assistant is specialized in generating deep research queries to help users find the information they need.',
- ],
- steps=[
- 'First, generate generic queries to get a broad overview of the topic and do a web search',
- 'Based on the content of the initial search results, generate a series of deep research queries to explore the topic in more depth and provide a comprehensive overview of the topic and related topics.',
- 'Return this second set of queries to the user for further exploration.'
- ],
- output_instructions=[
- 'Each query should be well-structured and focused on a specific topic.',
- 'Ensure that the queries are clear and concise to help the user find the information they need.',
- 'Each query must be diverse yet relevant to the user\'s needs, providing a comprehensive overview of the topic and related topics.'
- ],
- context_providers={
- 'date': CurrentDateProvider(title='Current date', format='%Y-%m-%d %H:%M:%S')
- }
-))
-
-class DeepQueryAgentInputSchema(BaseModel):
- user_input: str = Field(..., description='The user input to generate deep research queries based on.')
-
- class Config:
- title = 'DeepQueryAgentInputSchema'
- description = 'The input schema for the DeepQueryAgent.'
- json_schema_extra = {
- title: 'DeepQueryAgentInputSchema',
- description: 'The input schema for the DeepQueryAgent.'
- }
-
-class DeepQueryAgentConfig(BaseChatAgentConfig):
- search_tool_instance: BaseTool
-
-class DeepQueryAgent(BaseChatAgent):
- def __init__(self, config: DeepQueryAgentConfig):
- super().__init__(config)
- self.search_tool = config.search_tool_instance
-
- def _get_and_handle_response(self):
- """
- Handles obtaining and processing the response.
-
- Returns:
- Type[BaseModel]: The processed response.
- """
- self.memory.add_message('assistant', 'First, I will generate 3 generic queries to get a broad overview of the topic.')
- initial_response = self.get_response(response_model=self.output_schema)
- search_results = self.search_tool.run(initial_response)
- self.memory.add_message(
- 'assistant', (
- 'Here are the search results based on the initial response:\n'
- f'{search_results}\n\n'
- '==========='
- 'Now, I will generate a series of 7 deep diverse and detailed research queries based on the initial search results to explore the topic and closely related topics in more depth.'
- )
- )
- deep_queries = self.get_response(response_model=self.output_schema)
- return deep_queries
-
-# Initialize the BaseChatAgent with the configured client, system prompt generator, and search toool schema.
-deep_query_agent = DeepQueryAgent(
- config=DeepQueryAgentConfig(
- client=instructor.from_openai(openai.OpenAI()),
- system_prompt_generator=system_prompt_generator,
- model='gpt-3.5-turbo',
- output_schema=search_tool.input_schema,
- search_tool_instance=search_tool
- )
-)
-
-if __name__ == "__main__":
- # We can test the agent by running a topic through it and observing the generated queries.
- console = Console()
- output = deep_query_agent.run(deep_query_agent.input_schema(chat_input='Apple intelligence'))
- console.print(output)
-
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/in_memory_faiss.py b/examples/deep_research_multi_agent/in_memory_faiss.py
new file mode 100644
index 0000000..f00c9da
--- /dev/null
+++ b/examples/deep_research_multi_agent/in_memory_faiss.py
@@ -0,0 +1,78 @@
+import os
+import openai
+import faiss
+import numpy as np
+import html2text
+import asyncio
+import aiohttp
+from playwright.async_api import async_playwright
+from bs4 import BeautifulSoup
+import markdownify
+
+class InMemFaiss:
+ def __init__(self, openai_api_key):
+ openai.api_key = openai_api_key
+ self.index = faiss.IndexFlatL2(1536)
+ self.texts = []
+ self.metadata = []
+
+ async def download_content(self, url):
+ async with async_playwright() as p:
+ browser = await p.chromium.launch(headless=True)
+ page = await browser.new_page()
+ await page.goto(url)
+ html_content = await page.content()
+ await browser.close()
+
+ # Clean HTML content
+ cleaned_html = self.clean_html(html_content)
+
+ # Convert cleaned HTML to Markdown
+ markdown_content = markdownify.markdownify(cleaned_html, heading_style="ATX")
+ return markdown_content
+
+ def clean_html(self, html_content):
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ # Example cleaning: remove script and style tags
+ for script_or_style in soup(["script", "style"]):
+ script_or_style.decompose()
+
+ # Additional cleaning can be done here
+ cleaned_html = str(soup)
+ return cleaned_html
+
+ def split_text(self, text, chunk_size=1000, overlap=250):
+ chunks = []
+ for i in range(0, len(text), chunk_size - overlap):
+ chunk = text[i:i + chunk_size].strip()
+ chunks.append(chunk)
+ return chunks
+
+ async def generate_embeddings(self, texts):
+ embeddings = []
+ for text in texts:
+ response = await openai.Embedding.create(input=text, model="text-embedding-3-small")
+ embeddings.append(response['data'][0]['embedding'])
+ return np.array(embeddings).astype('float32')
+
+ async def ingest_urls(self, urls):
+ tasks = [self.process_url(url) for url in urls]
+ await asyncio.gather(*tasks)
+
+ async def process_url(self, url):
+ try:
+ url = str(url)
+ content = await self.download_content(url)
+ chunks = self.split_text(content)
+ embeddings = await self.generate_embeddings(chunks)
+ self.index.add(embeddings)
+ self.texts.extend(chunks)
+ self.metadata.extend([{"url": url}] * len(chunks))
+ except Exception as e:
+ print(f"Failed to ingest URL {url}: {e}")
+
+ async def retrieve_chunks(self, query, top_k=5):
+ query_embedding = await self.generate_embeddings([query])
+ distances, indices = self.index.search(query_embedding, top_k)
+ return [self.texts[idx] for idx in indices[0]]
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/info_refiner.py b/examples/deep_research_multi_agent/info_refiner.py
new file mode 100644
index 0000000..8ce2c1f
--- /dev/null
+++ b/examples/deep_research_multi_agent/info_refiner.py
@@ -0,0 +1,47 @@
+from pydantic import BaseModel, Field
+from atomic_agents.agents.base_chat_agent import BaseAgentIO
+import instructor
+import openai
+from atomic_agents.agents.base_chat_agent import BaseChatAgent, BaseChatAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo
+from examples.deep_research_multi_agent.providers import vector_db_chunks_provider
+from examples.deep_research_multi_agent.providers import current_date_provider
+
+class RefineAnswerInputSchema(BaseAgentIO):
+ question: str = Field(..., description='The question that was asked.')
+ answer: str = Field(..., description='The initial answer to the question.')
+
+class RefineAnswerOutputSchema(BaseModel):
+ refined_answer: str = Field(..., description='The refined answer to the question.')
+
+# Create the refine answer agent
+refine_answer_agent = BaseChatAgent(
+ BaseChatAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model='gpt-3.5-turbo',
+ system_prompt_generator=SystemPromptGenerator(
+ SystemPromptInfo(
+ background=[
+ "You are an intelligent answer refinement expert.",
+ "Your task is to expand and elaborate on an existing answer to a question using additional context from vector DB chunks."
+ ],
+ steps=[
+ "You will receive a question or instruction, the initial answer, and additional context from vector DB chunks.",
+ "Expand and elaborate on the initial answer using the additional context to provide a more comprehensive and detailed response."
+ ],
+ output_instructions=[
+ "Ensure the refined answer is clear, concise, and well-structured.",
+ "Ensure the refined answer is directly relevant to the question and incorporates the additional context provided.",
+ "Add new information and details to make the final answer more elaborate and informative.",
+ "Do not make up any new information; only use the information present in the context."
+ ]
+ )
+ ),
+ input_schema=RefineAnswerInputSchema,
+ output_schema=RefineAnswerOutputSchema
+ )
+)
+
+# Register the context providers
+refine_answer_agent.register_context_provider('date', current_date_provider)
+refine_answer_agent.register_context_provider('vector_db_chunks', vector_db_chunks_provider)
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/main.py b/examples/deep_research_multi_agent/main.py
new file mode 100644
index 0000000..5ce3eed
--- /dev/null
+++ b/examples/deep_research_multi_agent/main.py
@@ -0,0 +1,71 @@
+import os
+from examples.deep_research_multi_agent.query_agent import query_agent
+from examples.deep_research_multi_agent.question_answering import answer_agent
+from examples.deep_research_multi_agent.top_urls_selector import top_urls_selector_agent
+from examples.deep_research_multi_agent.info_refiner import refine_answer_agent
+from examples.deep_research_multi_agent.providers import search_results_provider, current_date_provider, vector_db_chunks_provider
+from examples.deep_research_multi_agent.in_memory_faiss import InMemFaiss
+from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool, SearxNGSearchToolConfig
+from rich.console import Console
+
+console = Console()
+
+# Initialize the search tool
+search_tool = SearxNGSearchTool(SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=30))
+in_mem_faiss = InMemFaiss(openai_api_key=os.getenv('OPENAI_API_KEY'))
+
+user_input = "Create an in-depth table of upcoming PC games for july and august 2024"
+
+while True:
+ if user_input.lower() == 'exit':
+ break
+
+ console.print("[bold green]Getting queries...[/bold green]")
+ queries = query_agent.run(query_agent.input_schema(instruction=user_input, num_queries=5))
+
+ # Start loop here
+ console.print("[bold green]Registering context providers...[/bold green]")
+ answer_agent.register_context_provider('date', current_date_provider)
+ answer_agent.register_context_provider('search_results', search_results_provider)
+
+ console.print("[bold green]Getting search results...[/bold green]")
+ search_results_provider.search_results = search_tool.run(search_tool.input_schema(**queries.model_dump()))
+
+ # Select top URLs
+ console.print("[bold green]Selecting top URLs...[/bold green]")
+ top_urls = top_urls_selector_agent.run(top_urls_selector_agent.input_schema(
+ user_input=user_input,
+ num_urls=5
+ ))
+
+ # Call the answer agent
+ console.print("[bold green]Calling the answer agent...[/bold green]")
+ initial_answer = answer_agent.run(answer_agent.input_schema(question=user_input))
+ console.print(initial_answer.markdown_output, markup=True)
+
+ console.print("[bold green]=====================[/bold green]")
+
+ console.print("[bold green]Ingesting URLs into in-memory FAISS...[/bold green]")
+ in_mem_faiss.ingest_urls(top_urls.top_urls)
+
+ # Retrieve chunks
+ console.print("[bold green]Retrieving chunks...[/bold green]")
+ query = user_input
+ top_k = 5
+ chunks = in_mem_faiss.retrieve_chunks(query, top_k)
+
+ vector_db_chunks_provider.chunks = chunks
+
+ console.print("[bold green]Refining the answer...[/bold green]")
+ refined_answer = refine_answer_agent.run(refine_answer_agent.input_schema(
+ question=user_input,
+ answer=initial_answer.markdown_output
+ ))
+
+ console.print("Refined Answer:")
+ console.print(refined_answer.refined_answer, markup=True)
+
+ # We don't need to keep the context for this agent after each run. This will save money.
+ refine_answer_agent.reset_memory()
+
+ user_input = input("Enter your question (or type 'exit' to quit): ")
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/providers.py b/examples/deep_research_multi_agent/providers.py
new file mode 100644
index 0000000..e655186
--- /dev/null
+++ b/examples/deep_research_multi_agent/providers.py
@@ -0,0 +1,29 @@
+from datetime import datetime
+
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase
+
+
+class CurrentDateProvider(SystemPromptContextProviderBase):
+ def get_info(self) -> str:
+ date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ return f'The current date in the format "%Y-%m-%d %H:%M:%S" is {date}'
+
+class SearchResultsProvider(SystemPromptContextProviderBase):
+ def __init__(self, title: str):
+ super().__init__(title=title)
+ self.search_results = []
+
+ def get_info(self) -> str:
+ return f'SEARCH RESULTS: {self.search_results}'
+
+class VectorDBChunksProvider(SystemPromptContextProviderBase):
+ def __init__(self, title: str):
+ super().__init__(title=title)
+ self.chunks = []
+
+ def get_info(self) -> str:
+ return f'VECTOR DB CHUNKS: {self.chunks}'
+
+search_results_provider = SearchResultsProvider(title='Search results')
+current_date_provider = CurrentDateProvider(title='Current date')
+vector_db_chunks_provider = VectorDBChunksProvider(title='Vector DB chunks')
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/query_agent.py b/examples/deep_research_multi_agent/query_agent.py
new file mode 100644
index 0000000..15cf2a8
--- /dev/null
+++ b/examples/deep_research_multi_agent/query_agent.py
@@ -0,0 +1,42 @@
+import os
+import groq
+import instructor
+import openai
+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo
+from atomic_agents.lib.tools.search.searx_tool import SearxNGSearchTool
+from examples.deep_research_multi_agent.providers import CurrentDateProvider
+
+class QueryAgentInputSchema(BaseAgentIO):
+ instruction: str = 'A detailed instruction or request to generate deep research queries for.'
+ num_queries: int = 'The number of queries to generate.'
+
+# Create the query agent
+query_agent = BaseChatAgent(
+ BaseChatAgentConfig(
+ client=instructor.from_groq(groq.Groq(api_key=os.getenv('GROQ_API_KEY')))
+ model='gpt-3.5-turbo',
+ system_prompt_generator=SystemPromptGenerator(
+ SystemPromptInfo(
+ background=[
+ "You are an intelligent query generation expert.",
+ "Your task is to generate a specified number of diverse and highly relevant queries based on a given instruction or request.",
+ "The queries should cover different aspects of the instruction to ensure comprehensive exploration."
+ ],
+ steps=[
+ "You will receive a detailed instruction or request and the number of queries to generate.",
+ "Generate the requested number of queries in a JSON format."
+ ],
+ output_instructions=[
+ "Ensure clarity and conciseness in each query.",
+ "Ensure each query is unique and as diverse as possible while remaining relevant to the instruction."
+ ]
+ )
+ ),
+ input_schema=QueryAgentInputSchema,
+ output_schema=SearxNGSearchTool.input_schema
+ )
+)
+
+# Register the new context provider
+query_agent.register_context_provider('date', CurrentDateProvider(title='Current date'))
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/question_answering.py b/examples/deep_research_multi_agent/question_answering.py
new file mode 100644
index 0000000..80d7af8
--- /dev/null
+++ b/examples/deep_research_multi_agent/question_answering.py
@@ -0,0 +1,38 @@
+import instructor
+import openai
+from pydantic import BaseModel, Field
+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo
+from examples.deep_research_multi_agent.providers import current_date_provider
+
+class AnswerAgentInputSchema(BaseAgentIO):
+ question: str = Field(..., description='A question that needs to be answered based on the provided context.')
+
+class AnswerAgentOutputSchema(BaseModel):
+ markdown_output: str = Field(..., description='The answer to the question in markdown format.')
+
+# Create the answer agent
+answer_agent = BaseChatAgent(
+ BaseChatAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model='gpt-3.5-turbo',
+ system_prompt_generator=SystemPromptGenerator(
+ SystemPromptInfo(
+ background=[
+ "You are an intelligent answering expert.",
+ "Your task is to provide accurate and detailed answers to user questions based on the given context."
+ ],
+ steps=[
+ "You will receive a question and the context information.",
+ "Generate a detailed and accurate answer based on the context."
+ ],
+ output_instructions=[
+ "Ensure clarity and conciseness in each answer.",
+ "Ensure the answer is directly relevant to the question and context provided."
+ ]
+ )
+ ),
+ input_schema=AnswerAgentInputSchema,
+ output_schema=AnswerAgentOutputSchema
+ )
+)
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/scraper.py b/examples/deep_research_multi_agent/scraper.py
new file mode 100644
index 0000000..f80d8ef
--- /dev/null
+++ b/examples/deep_research_multi_agent/scraper.py
@@ -0,0 +1,78 @@
+import os
+from bs4 import BeautifulSoup
+import requests
+from markdownify import MarkdownConverter
+from pydantic import BaseModel, Field
+
+class WebDocumentMetadata(BaseModel):
+ url: str
+ title: str = Field(default='')
+ description: str = Field(default='')
+ keywords: str = Field(default='')
+ author: str = Field(default='')
+
+class VectorDBDocumentMetadata(WebDocumentMetadata):
+ document_part: int = Field(default=1)
+ document_total_parts: int = Field(default=1)
+
+class Document(BaseModel):
+ content: str
+ metadata: WebDocumentMetadata | VectorDBDocumentMetadata = Field(default_factory=WebDocumentMetadata)
+
+
+
+class UrlToMarkdownConverter:
+ METADATA_ATTRS = {
+ 'url': '',
+ 'title': ('title', None),
+ 'description': ('meta', {'name': 'description'}),
+ 'keywords': ('meta', {'name': 'keywords'}),
+ 'author': ('meta', {'name': 'author'})
+ }
+
+ @staticmethod
+ def _fetch_url_content(url):
+ response = requests.get(url)
+ response.raise_for_status()
+ return response.text
+
+ @staticmethod
+ def _parse_html(html):
+ return BeautifulSoup(html, 'html.parser')
+
+ @staticmethod
+ def _extract_metadata(soup, url):
+ metadata_dict = {}
+ for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():
+ if selector:
+ element = soup.find(*selector)
+ metadata_dict[attr] = element.get('content', '').strip() if element else ''
+ else:
+ metadata_dict[attr] = url
+ return WebDocumentMetadata(**metadata_dict)
+
+ @staticmethod
+ def _convert_to_markdown(html):
+ class CustomMarkdownConverter(MarkdownConverter):
+ def convert_img(self, el, text, convert_as_inline):
+ return ''
+ def convert_p(self, el, text, convert_as_inline):
+ return text + '\n\n'
+
+ for script in html.find_all('script'):
+ script.decompose()
+
+ for style in html.find_all('style'):
+ style.decompose()
+
+ converter = CustomMarkdownConverter()
+ return converter.convert(str(html)).strip()
+
+ @staticmethod
+ def convert(url):
+ html = UrlToMarkdownConverter._fetch_url_content(url)
+ soup = UrlToMarkdownConverter._parse_html(html)
+ metadata = UrlToMarkdownConverter._extract_metadata(soup, url)
+ markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)
+
+ return Document(content=markdown_content, metadata=metadata)
\ No newline at end of file
diff --git a/examples/deep_research_multi_agent/top_urls_selector.py b/examples/deep_research_multi_agent/top_urls_selector.py
new file mode 100644
index 0000000..3072c4e
--- /dev/null
+++ b/examples/deep_research_multi_agent/top_urls_selector.py
@@ -0,0 +1,44 @@
+from typing import List
+import instructor
+import openai
+from pydantic import BaseModel, Field, HttpUrl
+from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig
+from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo
+from examples.deep_research_multi_agent.providers import current_date_provider, search_results_provider
+
+class TopUrlsSelectorInputSchema(BaseAgentIO):
+ user_input: str = Field(..., description='The user input or question.')
+ num_urls: int = Field(..., description='The number of top URLs to select.')
+
+class TopUrlsSelectorOutputSchema(BaseModel):
+ top_urls: List[HttpUrl] = Field(..., description='The list of top URLs selected based on the user input.')
+
+# Create the top URLs selector agent
+top_urls_selector_agent = BaseChatAgent(
+ BaseChatAgentConfig(
+ client=instructor.from_openai(openai.OpenAI()),
+ model='gpt-3.5-turbo',
+ system_prompt_generator=SystemPromptGenerator(
+ SystemPromptInfo(
+ background=[
+ "You are an intelligent URL selection expert.",
+ "Your task is to select the best URLs from a list of search results that are most relevant to the user input or question."
+ ],
+ steps=[
+ "You will receive the user input or question, the list of search results, and the number of best URLs to select.",
+ "Analyze the search results and select the best URLs that are most relevant to the user input or question.",
+ ],
+ output_instructions=[
+ "Ensure each selected URL is highly relevant to the user input or question.",
+ "Ensure the selected URLs are diverse yet as relevant as possible."
+ ]
+ )
+ ),
+ input_schema=TopUrlsSelectorInputSchema,
+ output_schema=TopUrlsSelectorOutputSchema
+ )
+)
+
+# Register the context providers
+top_urls_selector_agent.register_context_provider('date', current_date_provider)
+top_urls_selector_agent.register_context_provider('search_results', search_results_provider)
\ No newline at end of file
diff --git a/examples/notebooks/shared_context.ipynb b/examples/notebooks/shared_context.ipynb
new file mode 100644
index 0000000..e69de29