Skip to content

Building AI agents, atomically

Refactor, reorganize

Kenny Vaneetvelde851f4b215742
diff --git a/atomic_agents_new/agents/basic_chat_agent.py b/atomic_agents/agents/base_chat_agent.py
similarity index 81%
rename from atomic_agents_new/agents/basic_chat_agent.py
rename to atomic_agents/agents/base_chat_agent.py
index 4078731..2f534cf 100644
--- a/atomic_agents_new/agents/basic_chat_agent.py
+++ b/atomic_agents/agents/base_chat_agent.py
@@ -1,21 +1,19 @@
-from typing import Callable, List
+from typing import List
from pydantic import BaseModel, Field
-from datetime import datetime
-from abc import ABC, abstractmethod
-from atomic_agents.lib.chat_memory import ChatMemory
+from atomic_agents.lib.components.chat_memory import ChatMemory
from atomic_agents.lib.utils.logger import logger
-from atomic_agents.tools.searx import SearxNGSearchTool
-from atomic_agents_new.lib.system_prompt_generator import DynamicInfoProviderBase, SystemPromptInfo, SystemPromptGenerator
+from atomic_agents.lib.tools.searx import SearxNGSearchTool
+from atomic_agents.lib.components.system_prompt_generator import DynamicInfoProviderBase, SystemPromptInfo, SystemPromptGenerator
-class BasicChatAgentInputSchema(BaseModel):
+class BaseChatAgentInputSchema(BaseModel):
chat_input: str = Field(..., description='The input text for the chat agent.')
-class BasicChatAgentResponse(BaseModel):
+class BaseChatAgentResponse(BaseModel):
response: str = Field(..., description='The markdown-enabled response from the chat agent.')
class Config:
- title = 'BasicChatAgentResponse'
- description = 'Response from the basic chat agent. The response can be in markdown format.'
+ title = 'BaseChatAgentResponse'
+ description = 'Response from the base chat agent. The response can be in markdown format.'
json_schema_extra = {
'title': title,
'description': description
@@ -39,8 +37,8 @@ class GeneralPlanResponse(BaseModel):
'description': description
}
-class BasicChatAgent:
- def __init__(self, client, system_prompt_generator: SystemPromptGenerator = None, model: str = 'gpt-3.5-turbo', memory: ChatMemory = None, include_planning_step = False, input_schema = BasicChatAgentInputSchema, output_schema = BasicChatAgentResponse):
+class BaseChatAgent:
+ def __init__(self, client, system_prompt_generator: SystemPromptGenerator = None, model: str = 'gpt-3.5-turbo', memory: ChatMemory = None, include_planning_step = False, input_schema = BaseChatAgentInputSchema, output_schema = BaseChatAgentResponse):
self.input_schema = input_schema
self.output_schema = output_schema
self.client = client
diff --git a/atomic_agents/agents/basic_chat_agent.py b/atomic_agents/agents/basic_chat_agent.py
deleted file mode 100644
index 04242d7..0000000
--- a/atomic_agents/agents/basic_chat_agent.py
+++ /dev/null
@@ -1,107 +0,0 @@
-import os
-from groq import Groq
-import instructor
-import openai
-from pydantic import BaseModel, Field
-from datetime import datetime
-from atomic_agents.lib.utils.logger import logger
-from atomic_agents.lib.chat_memory import ChatMemory
-from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
-from atomic_agents.tools.vectordb_query_tool import VectorDBQueryTool
-
-class BasicChatAgentInputSchema(BaseModel):
- chat_input: str = Field(..., description="The input text for the chat agent.")
-
-class BasicChatAgentResponse(BaseModel):
- response: str = Field(..., description='The response from the chat agent.')
-
-class BasicChatAgent:
- def __init__(self, client, model: str = "gpt-3.5-turbo", agent_background: str = "You are a helpful AI assistant.", initial_message: str = None, initial_memory: list = [], vector_db = None, include_context_in_system_prompt: bool = False, input_schema = None, output_schema = None):
- self.input_schema = input_schema or BasicChatAgentInputSchema
- self.output_schema = output_schema or BasicChatAgentResponse
-
- self.client = client
- self.model = model
- self.agent_background = agent_background
- self.memory = ChatMemory()
- self.include_context_in_system_prompt = include_context_in_system_prompt
-
- self.vector_db = vector_db
-
- if initial_memory:
- self.memory.load_from_dict_list(initial_memory)
-
- if initial_message:
- self.memory.add_message("assistant", initial_message)
-
- def get_system_prompt(self) -> str:
- current_date = datetime.now().isoformat()
- system_prompt = (
- f"{self.agent_background}\n"
- "\n"
- f"The current date, in ISO format, is {current_date}.\n"
- )
-
- if self.vector_db and self.include_context_in_system_prompt:
- system_prompt += (
- f"The AI has access to the following context to answer questions. If certain context is irrelevant to the question, the AI will ignore it.\n"
- f"Context:\n"
- )
-
- for i, document in enumerate(self.vector_db.run(self.vector_db.input_schema(queries=[self.memory.get_history()[-1].content])).results):
- system_prompt += f"Document {i}. {document.metadata['url']}\n"
- system_prompt += f"====================\n"
- system_prompt += f"{document.content}\n"
- system_prompt += f"====================\n\n"
- return system_prompt
-
- def get_response(self, response_model=None) -> BaseModel:
- if response_model is None:
- response_model = self.output_schema
-
- messages = [{"role": "system", "content": self.get_system_prompt()}] + self.memory.get_history()
- response = self.client.chat.completions.create(
- model=self.model,
- messages=messages,
- response_model=response_model
- )
- return response
-
-
- def run(self, user_input: str) -> str:
- self.memory.add_message("user", user_input)
- response = self.get_response(response_model=self.output_schema)
- self.memory.add_message("assistant", response.model_dump_json())
- return response
-
-
-if __name__ == "__main__":
- from rich.console import Console
- console = Console()
-
- openai_client = openai.OpenAI()
- client = instructor.from_openai(openai_client)
- model = "gpt-3.5-turbo"
- groq_client = instructor.from_groq(Groq(api_key=os.getenv("GROQ_API_KEY")))
- groq_model = os.getenv("GROQ_CHAT_MODEL")
- initial_message = "Hello! How can I assist you today?"
-
- db = VectorDBQueryTool(db_path='chromadb_persist', collection_name='brainblendai.com', max_results=3)
- agent = BasicChatAgent(
- client=groq_client,
- model=groq_model,
- initial_message=initial_message,
- agent_background="You are an AI assistant that, whatever the user input, will provide an answer in Alexandrine verse.",
- vector_db=db,
- include_context_in_system_prompt=False
- )
- console.print(f'Agent: {initial_message}')
-
- while True:
- user_input = input("You: ")
- if user_input.lower() in ['exit', 'quit']:
- print("Exiting chat...")
- break
-
- response = agent.run(user_input)
- console.print(f"Agent: {response.response}")
diff --git a/atomic_agents/agents/basic_chat_agent_new.py b/atomic_agents/agents/basic_chat_agent_new.py
deleted file mode 100644
index 3e8ad88..0000000
--- a/atomic_agents/agents/basic_chat_agent_new.py
+++ /dev/null
@@ -1,224 +0,0 @@
-from typing import Callable, List
-from pydantic import BaseModel, Field
-from datetime import datetime
-from abc import ABC, abstractmethod
-from atomic_agents.lib.chat_memory import ChatMemory
-
-class BasicChatAgentInputSchema(BaseModel):
- chat_input: str = Field(..., description='The input text for the chat agent.')
-
-class BasicChatAgentResponse(BaseModel):
- response: str = Field(..., description='The markdown-enabled response from the chat agent.')
-
- class Config:
- title = 'BasicChatAgentResponse'
- description = 'Response from the basic chat agent. The response can be in markdown format.'
- json_schema_extra = {
- 'title': title,
- 'description': description
- }
-
-class SystemPrompt(BaseModel):
- background: List[str]
- steps: List[str]
- output_instructions: List[str]
-
-class DynamicInfoProviderBase(ABC):
- def __init__(self, title: str):
- self.title = title
-
- @abstractmethod
- def get_info(self) -> str:
- pass
-
-class SystemPromptGenerator:
- def __init__(self, system_prompt: SystemPrompt, dynamic_info_providers: List[DynamicInfoProviderBase] = None):
- self.system_prompt = system_prompt
- self.dynamic_info_providers = dynamic_info_providers or []
-
- def generate_prompt(self) -> str:
- system_prompt = ''
-
- if self.system_prompt.background:
- system_prompt += '# IDENTITY and PURPOSE\n'
- system_prompt += f'-{'\n-'.join(self.system_prompt.background)}\n\n'
-
- if self.system_prompt.steps:
- system_prompt += '# INTERNAL ASSISTANT STEPS\n'
- system_prompt += f'-{'\n-'.join(self.system_prompt.steps)}\n\n'
-
- if self.system_prompt.output_instructions:
- system_prompt += '# OUTPUT INSTRUCTIONS\n'
- system_prompt += f'-{'\n-'.join(self.system_prompt.output_instructions)}\n\n'
-
- if self.dynamic_info_providers:
- system_prompt += '# EXTRA INFORMATION AND CONTEXT\n'
- for provider in self.dynamic_info_providers:
- system_prompt += f'## {provider.title}\n'
- system_prompt += f'{provider.get_info()}\n\n'
-
- return system_prompt
-
-class BasicChatAgent:
- def __init__(self, client, model: str = 'gpt-3.5-turbo', system_prompt: SystemPrompt = None, memory: ChatMemory = None, vector_db = None, input_schema = BasicChatAgentInputSchema, output_schema = BasicChatAgentResponse, dynamic_info_providers: List[DynamicInfoProviderBase] = None):
- self.input_schema = input_schema
- self.output_schema = output_schema
- self.client = client
- self.model = model
- self.memory = memory or ChatMemory()
- self.system_prompt = system_prompt or SystemPrompt(
- background=['You are a helpful AI assistant.'],
- steps=[],
- output_instructions=[]
- )
- self.prompt_generator = SystemPromptGenerator(self.system_prompt, dynamic_info_providers)
-
- def get_system_prompt(self) -> str:
- return self.prompt_generator.generate_prompt()
-
- def get_response(self, response_model=None) -> BaseModel:
- if response_model is None:
- response_model = self.output_schema
-
- messages = [{'role': 'system', 'content': self.get_system_prompt()}] + self.memory.get_history()
- response = self.client.chat.completions.create(
- model=self.model,
- messages=messages,
- response_model=response_model
- )
- return response
-
- def run(self, user_input: str) -> str:
- self.memory.add_message('user', user_input)
- response = self.get_response(response_model=self.output_schema)
- self.memory.add_message('assistant', response.model_dump_json())
- return response
-
-if __name__ == '__main__':
- import os
- import instructor
- import openai
- from groq import Groq
- from rich.console import Console
-
- class CurrentDateProvider(DynamicInfoProviderBase):
- def __init__(self, title: str, format: str = '%Y-%m-%d %H:%M:%S'):
- super().__init__(title)
- self.format = format
-
- def get_info(self) -> str:
- return f'The current date, in the format "{self.format}", is {datetime.now().strftime(self.format)}'
-
- class LiveSearchResultsProvider(DynamicInfoProviderBase):
- def __init__(self, title: str, search_query: str):
- super().__init__(title)
- self.search_query = search_query
-
- def get_info(self) -> str:
- # Placeholder for live search results fetching logic
- # In a real implementation, this would fetch live data from a search API
- return f'Live search results for "{self.search_query}"'
-
- console = Console()
-
- # Configuration dictionary
- config = {
- 'openai': {
- 'client': instructor.from_openai(openai.OpenAI()),
- 'model': 'gpt-3.5-turbo'
- },
- 'groq': {
- 'client': instructor.from_groq(Groq(api_key=os.getenv('GROQ_API_KEY'))),
- 'model': os.getenv('GROQ_CHAT_MODEL')
- }
- }
-
- # Choose the configuration
- use_groq = False # Set this to True to use Groq, False to use OpenAI
- chosen_config = config['groq'] if use_groq else config['openai']
-
- class PlanStep(BaseModel):
- step: str
- description: str
- substeps: List[str] = []
-
- class ArticleBuddyPlanResponse(BaseModel):
- observations: List[str] = Field(..., description='What is the conversation about? What are the key points?')
- thoughts: List[str] = Field(..., description='What is the thought process involved in preparing the ideas and outline?')
- response_plan: List[PlanStep] = Field(..., description='What are the steps involved in generating the ideas and outline?')
-
- class Config:
- title = 'ArticleBuddyPlanResponse'
- description = 'Response plan from the Article Buddy chat agent.'
- json_schema_extra = {
- 'title': title,
- 'description': description
- }
-
- class ArticleBuddyResponse(BaseModel):
- response: str = Field(..., description='The markdown-enabled response from the chat agent.')
-
- class Config:
- title = 'ArticleBuddyResponse'
- description = 'Response from the Article Buddy chat agent.'
- json_schema_extra = {
- 'title': title,
- 'description': description
- }
-
- class MyChatAgent(BasicChatAgent):
- def run(self, user_input: str) -> ArticleBuddyResponse:
- self.memory.add_message('user', user_input)
- self.memory.add_message('assistant', 'First, let me note the observations about the input and thought process involved in preparing the ideas and outline.')
- plan = self.get_response(response_model=ArticleBuddyPlanResponse)
- self.memory.add_message('assistant', plan.model_dump_json())
- response = self.get_response(response_model=ArticleBuddyResponse)
- self.memory.add_message('assistant', response.model_dump_json())
- return response
-
- system_prompt = SystemPrompt(
- background=[
- 'This assistant is an expert in brainstorming ideas and creating article outlines.',
- 'This assistant\'s goal is to generate engaging and creative ideas and structured outlines for articles based on the given topic.'
- ],
- steps=[
- 'Note the observations about the input and thought process involved in preparing the ideas and outline.',
- 'Respond with a list of 10 ideas and a structured outline for the article in a markdown-formatted text.'
- ],
- output_instructions=[
- 'Ensure each idea is clearly numbered from 1 to 10.',
- 'Output the list of ideas and the article outline in human-readable Markdown format.',
- 'Do not include any additional text or confirmation messages; only provide the list of ideas and the article outline.',
- 'Do not reply to the user\'s input with anything except the list of ideas and the article outline.'
- ]
- )
-
- memory = ChatMemory()
- initial_memory = [
- {'role': 'assistant', 'content': 'Hello! I\'m an AI assistant that can help you brainstorm ideas and generate article outlines. Please provide me with a topic to get started.'}
- ]
- memory.load_from_dict_list(initial_memory)
-
- # Define dynamic info providers
- dynamic_info_providers = [
- CurrentDateProvider('Current date', format='%Y-%m-%d %H:%M:%S'),
- ]
-
- agent = MyChatAgent(
- client=chosen_config['client'],
- model=chosen_config['model'],
- system_prompt=system_prompt,
- memory=memory,
- dynamic_info_providers=dynamic_info_providers,
- output_schema=ArticleBuddyResponse
- )
- console.print(f'Agent: {initial_memory[0]["content"]}')
-
- while True:
- user_input = input('You: ')
- if user_input.lower() in ['exit', 'quit']:
- print('Exiting chat...')
- break
-
- response = agent.run(user_input)
- console.print(f'Agent: {response.response}')
diff --git a/atomic_agents/agents/basic_chat_agent_new_with_search.py b/atomic_agents/agents/basic_chat_agent_new_with_search.py
deleted file mode 100644
index 7096ecf..0000000
--- a/atomic_agents/agents/basic_chat_agent_new_with_search.py
+++ /dev/null
@@ -1,224 +0,0 @@
-from typing import Callable, List
-from pydantic import BaseModel, Field
-from datetime import datetime
-from abc import ABC, abstractmethod
-from atomic_agents.lib.chat_memory import ChatMemory
-from atomic_agents.lib.utils.logger import logger
-
-class BasicChatAgentInputSchema(BaseModel):
- chat_input: str = Field(..., description='The input text for the chat agent.')
-
-class BasicChatAgentResponse(BaseModel):
- response: str = Field(..., description='The markdown-enabled response from the chat agent.')
-
- class Config:
- title = 'BasicChatAgentResponse'
- description = 'Response from the basic chat agent. The response can be in markdown format.'
- json_schema_extra = {
- 'title': title,
- 'description': description
- }
-
-class SystemPrompt(BaseModel):
- background: List[str]
- steps: List[str]
- output_instructions: List[str]
-
-class DynamicInfoProviderBase(ABC):
- def __init__(self, title: str):
- self.title = title
-
- @abstractmethod
- def get_info(self) -> str:
- pass
-
-class SystemPromptGenerator:
- def __init__(self, system_prompt: SystemPrompt, dynamic_info_providers: dict[str, DynamicInfoProviderBase] = None):
- self.system_prompt = system_prompt
- self.dynamic_info_providers = dynamic_info_providers or {}
-
- def generate_prompt(self) -> str:
- system_prompt = ''
-
- if self.system_prompt.background:
- system_prompt += '# IDENTITY and PURPOSE\n'
- system_prompt += f'-{'\n-'.join(self.system_prompt.background)}\n\n'
-
- if self.system_prompt.steps:
- system_prompt += '# INTERNAL ASSISTANT STEPS\n'
- system_prompt += f'-{'\n-'.join(self.system_prompt.steps)}\n\n'
-
- if self.system_prompt.output_instructions:
- system_prompt += '# OUTPUT INSTRUCTIONS\n'
- system_prompt += f'-{'\n-'.join(self.system_prompt.output_instructions)}\n\n'
-
- if self.dynamic_info_providers:
- system_prompt += '# EXTRA INFORMATION AND CONTEXT\n'
- for provider in self.dynamic_info_providers.values():
- info = provider.get_info()
- if info:
- system_prompt += f'## {provider.title}\n'
- system_prompt += f'{info}\n\n'
-
- return system_prompt
-
-class GeneralPlanStep(BaseModel):
- step: str
- description: str
- substeps: List[str] = []
-
-class GeneralPlanResponse(BaseModel):
- observations: List[str] = Field(..., description='Key points or observations about the input.')
- thoughts: List[str] = Field(..., description='Thought process or considerations involved in preparing the response.')
- response_plan: List[GeneralPlanStep] = Field(..., description='Steps involved in generating the response.')
-
- class Config:
- title = 'GeneralPlanResponse'
- description = 'General response plan from the chat agent.'
- json_schema_extra = {
- 'title': title,
- 'description': description
- }
-
-class BasicChatAgent:
- def __init__(self, client, model: str = 'gpt-3.5-turbo', system_prompt: SystemPrompt = None, memory: ChatMemory = None, include_planning_step = False, input_schema = BasicChatAgentInputSchema, output_schema = BasicChatAgentResponse, dynamic_info_providers: dict[str, DynamicInfoProviderBase] = None):
- self.input_schema = input_schema
- self.output_schema = output_schema
- self.client = client
- self.model = model
- self.memory = memory or ChatMemory()
- self.system_prompt = system_prompt or SystemPrompt(
- background=['You are a helpful AI assistant.'],
- steps=[],
- output_instructions=[]
- )
- self.system_prompt_generator = SystemPromptGenerator(self.system_prompt, dynamic_info_providers)
- self.include_planning_step = include_planning_step
-
- def get_system_prompt(self) -> str:
- return self.system_prompt_generator.generate_prompt()
-
- def get_response(self, response_model=None) -> BaseModel:
- if response_model is None:
- response_model = self.output_schema
-
- messages = [{'role': 'system', 'content': self.get_system_prompt()}] + self.memory.get_history()
- response = self.client.chat.completions.create(
- model=self.model,
- messages=messages,
- response_model=response_model
- )
- return response
-
- def run(self, user_input: str) -> str:
- self.memory.add_message('user', user_input)
- self._pre_run()
- if self.include_planning_step:
- self.memory.add_message('assistant', 'I will now note the observations about the input and context and the thought process involved in preparing the response.')
- plan = self.get_response(response_model=GeneralPlanResponse)
- self.memory.add_message('assistant', plan.model_dump_json())
- response = self.get_response(response_model=self.output_schema)
- self.memory.add_message('assistant', response.model_dump_json())
- self._post_run()
- return response
-
- def _pre_run(self):
- pass
-
- def _post_run(self):
- pass
-
-if __name__ == '__main__':
- import instructor
- import openai
- from rich.console import Console
- from atomic_agents.tools.searx import SearxNGSearchTool
-
- class CurrentDateProvider(DynamicInfoProviderBase):
- def __init__(self, title: str, format: str = '%Y-%m-%d %H:%M:%S'):
- super().__init__(title)
- self.format = format
-
- def get_info(self) -> str:
- return f'The current date, in the format "{self.format}", is {datetime.now().strftime(self.format)}'
-
- class SearchResultsProvider(DynamicInfoProviderBase):
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.search_results: SearxNGSearchTool.output_schema = None
-
- def get_info(self) -> str:
- response = ''
- if not self.search_results:
- return response
-
- for result in self.search_results.results:
- response += f'--- {result.title} ---\n'
- response += f'{result.content}\n'
- response += f'[Read more]({result.url})\n'
- response += '--------------------------------\n\n'
-
- return response
-
- # Define dynamic info providers
- dynamic_info_providers = {
- 'date': CurrentDateProvider('Current date', format='%Y-%m-%d %H:%M:%S'),
- 'search': SearchResultsProvider('Search results')
- }
-
- console = Console()
-
- system_prompt = SystemPrompt(
- background=[
- 'This assistant is an expert in brainstorming ideas and creating article outlines.',
- 'This assistant\'s goal is to generate engaging and creative ideas and structured outlines for articles based on the given topic.'
- ],
- steps=[
- 'Note the observations about the input and thought process involved in preparing the ideas and outline.',
- 'Respond with a list of 10 ideas and a structured outline for the article in a markdown-formatted text.'
- ],
- output_instructions=[
- 'Ensure each idea is clearly numbered from 1 to 10.',
- 'Output the list of ideas and the article outline in human-readable Markdown format.',
- 'Do not include any additional text or confirmation messages; only provide the list of ideas and the article outline.',
- 'Do not reply to the user\'s input with anything except the list of ideas and the article outline.'
- ]
- )
-
- memory = ChatMemory()
- initial_memory = [
- {'role': 'assistant', 'content': 'Hello! I\'m an AI assistant that can help you brainstorm ideas and generate article outlines. Please provide me with a topic to get started.'}
- ]
- memory.load_from_dict_list(initial_memory)
-
- class MyChatAgent(BasicChatAgent):
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.search_tool = SearxNGSearchTool(max_results=15)
-
- def _pre_run(self):
- self.memory.add_message('assistant', 'First, I will perform a search with 3 different search queries to gather relevant information on the topic.')
- search_input = self.get_response(response_model=SearxNGSearchTool.input_schema)
- logger.verbose(f'Search input: {search_input}')
- search_results = self.search_tool.run(search_input)
- self.system_prompt_generator.dynamic_info_providers['search'].search_results = search_results
- self.memory.add_message('assistant', 'I have gathered the search results and they have been added to the context.')
-
-
- agent = MyChatAgent(
- client=instructor.from_openai(openai.OpenAI()),
- model='gpt-3.5-turbo',
- system_prompt=system_prompt,
- memory=memory,
- dynamic_info_providers=dynamic_info_providers
- )
- console.print(f'Agent: {initial_memory[0]["content"]}')
-
- while True:
- user_input = input('You: ')
- if user_input.lower() in ['exit', 'quit']:
- print('Exiting chat...')
- break
-
- response = agent.run(user_input)
- console.print(f'Agent: {response.response}')
\ No newline at end of file
diff --git a/atomic_agents/agents/orchestrator_agent.py b/atomic_agents/agents/orchestrator_agent.py
deleted file mode 100644
index 379c1e2..0000000
--- a/atomic_agents/agents/orchestrator_agent.py
+++ /dev/null
@@ -1,193 +0,0 @@
-from typing import List, Optional, Union
-import instructor
-import openai
-from pydantic import BaseModel, Field, create_model
-from datetime import datetime
-from atomic_agents.agents.tool_interface_agent import ToolInterfaceAgent
-from atomic_agents.lib.utils.format_tool_message import format_tool_message
-from atomic_agents.lib.utils.logger import logger
-from atomic_agents.lib.chat_memory import ChatMemory
-from atomic_agents.tools.calculator_tool import CalculatorTool
-from atomic_agents.tools.searx import SearxNGSearchTool
-from atomic_agents.tools.user_input_tool import UserInputTool
-from atomic_agents.tools.web_scraping_tool import WebScrapingTool
-
-class OrchestratorAgentInputSchema(BaseModel):
- chat_input: str = Field(..., description="The input text for the chat agent.")
-
-class MarkdownChatResponse(BaseModel):
- markdown: str = Field(..., description='The response from the chat agent to the user, in markdown format. ONLY use this when there are no more questions for the user and the task is complete.')
-
-class TreeNode(BaseModel):
- """
- A node in a binary yes/no tree. The node can be the root node or a child node. A tree is at least 3 layers deep and should be a complete model with questions and results.
- This is used to determine which tool to use based on the input and the available tools.
- Tool names should be mentioned explicitly.
- """
-
- is_leaf_node: bool = Field(..., description="Whether the node is a leaf node.")
- result: Optional[str] = Field(None, description="The result of the tree if the node is a leaf node.", examples=["Use the calculator tool to perform the calculation.", "Use the SearXNG search tool to google for the latest gaming news", "Use the web scraping tool to scrape the website for the latest news."])
- question: Optional[str] = Field(None, description="The question to ask at this node.", examples=["Can the task be resolved with a web search, or does it require scraping?", "Does the task require a calculation, or is it a search for information?"])
- if_yes: Optional['TreeNode'] = Field(None, description="The node to follow if the answer to the question is 'yes'.")
- if_no: Optional['TreeNode'] = Field(None, description="The node to follow if the answer to the question is 'no'.")
-
-
-class PlanningAgentResponse(BaseModel):
- """
- Response from the agent that details the next steps to take in the conversation based on the current state, available tools, and the agent's thought process.
- Tool names should be mentioned explicitly.
- """
-
- observation: str = Field(..., description="Observation/summary of the current state of the conversation.")
- analysis_tree: TreeNode = Field(..., description="Detailed analysis tree of the task and available tools to help determine the plan.")
-
- plan: List[str] = Field(..., description="The step-by-step plan to solve the task, based on the analysis tree. Must have at least 3 steps in the list.")
-
-class OrchestratorAgentResponse(BaseModel):
- internal_thought: str = Field(..., description="Internal thought process involved in formulating the response or tool call.")
-
- response: Union[None] = Field(..., description='Response to the user or tool call from the orchestrator agent.')
-
- class Config:
- title = "OrchestratorAgentResponse"
- description = "Response from the orchestrator agent, which can either be a final answer response or a tool interface agent input."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-
-class OrchestratorAgent:
- def __init__(
- self,
- client,
- model: str = "gpt-3.5-turbo",
- agent_background: str = "This is a conversation with a helpful AI assistant.",
- initial_message: str = None,
- initial_memory: list = [],
- available_tools: list = []
- ):
- self.client = client
- self.model = model
- self.agent_background = agent_background
- self.memory = ChatMemory()
- self.available_tools = available_tools
-
- if initial_memory:
- self.memory.load_from_dict_list(initial_memory)
-
- if initial_message:
- self.memory.add_message("assistant", initial_message)
-
- new_input_schema_union_list = []
- for tool in available_tools:
- new_input_schema_union_list.append(tool.input_schema)
- new_input_schema_union_list.append(MarkdownChatResponse)
-
- global OrchestratorAgentResponse
- OrchestratorAgentResponse = create_model(
- "OrchestratorAgentResponse",
- __base__=OrchestratorAgentResponse,
- response=(Union[tuple(new_input_schema_union_list)], Field(description='Response from the orchestrator agent, which can be a final answer response or a tool interface agent input.'))
- )
- logger.verbose(f"Created dynamic OrchestratorAgentResponse model with new input schema union list: {new_input_schema_union_list}")
-
-
-
- def get_system_prompt(self) -> str:
- current_date = datetime.now().isoformat()
- system_prompt = (
- f"{self.agent_background}\n"
- "\n"
- "This AI, when given a task, will always first analyze the task and determine which tool to use through a series of logical examinations and a decision tree.\n"
- "The AI will then generate a step-by-step plan to solve the task using the chosen tools and intermittently re-evaluate the plan based on the task and the tools.\n"
- f"The current date, in ISO format, is {current_date}."
- )
-
- if len(self.available_tools) > 0:
- system_prompt += "\n\nAvailable tools:"
-
- for tool in self.available_tools:
- system_prompt += f"\n- {tool.tool_name}: {tool.tool_description}"
-
- return system_prompt
-
- def get_response(self, response_model):
- messages = [{"role": "system", "content": self.get_system_prompt()}] + self.memory.get_history()
- response = self.client.chat.completions.create(
- model=self.model,
- messages=messages,
- response_model=response_model
- )
- return response
-
- def run(self, user_input: str) -> str:
- self.memory.add_message("user", f"Analyze and execute the plan for the following task: \"{user_input}\"")
- self.memory.add_message("assistant", self.get_response(PlanningAgentResponse).model_dump_json())
- response = self.get_response(response_model=OrchestratorAgentResponse)
- return self.handle_response(response)
-
- def handle_response(self, response):
- logger.verbose(f"Handling response: {response}")
- if isinstance(response.response, MarkdownChatResponse):
- self.memory.add_message("assistant", response.model_dump_json())
- else:
- tool_agent = None
- for agent in self.available_tools:
- if isinstance(response.response, agent.input_schema):
- tool_agent = agent
- break
-
- if tool_agent:
- tool_message = format_tool_message(response.response)
- logger.verbose(f"Formatted tool message: {tool_message}")
- tool_output = tool_agent.run(response.response)
- logger.verbose(f"Tool output: {tool_output}")
- self.memory.add_message("assistant", "", tool_message=tool_message)
- self.memory.add_message("tool", tool_output.model_dump_json(), tool_id=tool_message['id'])
- self.memory.add_message("assistant", self.get_response(PlanningAgentResponse).model_dump_json())
- response = self.get_response(OrchestratorAgentResponse)
- logger.verbose(f"ToolInterfaceAgent response: {response}")
-
- if not isinstance(response.response, MarkdownChatResponse):
- return self.handle_response(response)
-
- else:
- self.memory.add_message("assistant", "I'm sorry, I couldn't understand your request.")
- return response
-
-if __name__ == "__main__":
- from rich.console import Console
- console = Console()
-
- # openai_client = openai.OpenAI(base_url="http://localhost:1234/v1")
- openai_client = openai.OpenAI()
-
- client = instructor.from_openai(openai_client)
-
- # Instantiate tool instances directly
- web_search_tool = SearxNGSearchTool(max_results=20)
- web_scraping_tool = WebScrapingTool()
- calculator_tool = CalculatorTool()
- user_input_tool = UserInputTool()
-
- # Instantiate agents with tool instances
- web_search_agent = ToolInterfaceAgent(web_search_tool, client=client, model="gpt-3.5-turbo")
- web_scraping_agent = ToolInterfaceAgent(web_scraping_tool, client=client, model="gpt-3.5-turbo")
- calculator_agent = ToolInterfaceAgent(calculator_tool, client=client, model="gpt-3.5-turbo")
- user_input_agent = ToolInterfaceAgent(user_input_tool, client=client, model="gpt-3.5-turbo")
-
- initial_message = "Hello! How can I assist you today?"
-
- agent = OrchestratorAgent(client=client, model="gpt-3.5-turbo", available_tools=[web_scraping_tool, web_search_agent, calculator_agent, user_input_tool], initial_message=initial_message)
-
- console.print(f'Agent: {initial_message}')
-
- while True:
- user_input = input("You: ")
- if user_input.lower() in ['exit', 'quit']:
- print("Exiting chat...")
- break
-
- response = agent.run(user_input)
- console.print(f"Agent: {response.response.markdown}")
\ No newline at end of file
diff --git a/atomic_agents/agents/tool_interface_agent.py b/atomic_agents/agents/tool_interface_agent.py
deleted file mode 100644
index f726091..0000000
--- a/atomic_agents/agents/tool_interface_agent.py
+++ /dev/null
@@ -1,151 +0,0 @@
-from datetime import datetime
-import os
-from typing import Optional, Type
-import instructor
-import openai
-from pydantic import BaseModel, Field, create_model
-from rich.console import Console
-from atomic_agents.lib.chat_memory import ChatMemory
-from atomic_agents.lib.utils.format_tool_message import format_tool_message
-from atomic_agents.tools.base import BaseTool
-from atomic_agents.tools.searx import SearxNGSearchTool
-from atomic_agents.tools.calculator_tool import CalculatorTool
-from atomic_agents.lib.utils.logger import logger
-
-class NextStepsResponse(BaseModel):
- observation: str = Field(..., description='Observation of the current state of the conversation.')
- thought: str = Field(..., description='Thought process of the agent.')
- next_action: str = Field(..., description='Description of the next action the assistant should take.')
- conclusion: str = Field(..., description='Conclusion of the current state of the conversation.')
-
- class Config:
- title = "NextStepsResponse"
- description = "Response from the agent that details the next steps to take in the conversation based on the current state and the agent's thought process."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-class ToolInterfaceAgentOutput(BaseModel):
- internal_thought: NextStepsResponse = Field(..., description="Internal thought process involved in formulating the response or tool call.")
- processed_output: str = Field(..., description="Processed markdown-enabled output of the tool to answer the original query or instruction.")
-
- class Config:
- title = "ToolInterfaceAgentOutput"
- description = "Processed output of the ToolInterfaceAgent. The first property will always be the 'internal_thought' property, which is the internal thought process of the agent."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-class ToolInterfaceAgent:
- def __init__(
- self,
- tool_instance: BaseTool,
- client, openai_api_key: Optional[str] = None,
- openai_base_url: Optional[str] = None,
- model: str = "gpt-3.5-turbo",
- process_output: bool = True,
- agent_background: Optional[str] = "This AI agent is designed to interact with tools to perform various tasks. The agent gets human readable input, converts it to a command, and returns the output in a human readable format."
- ):
- self.tool_instance = tool_instance
-
- self.tool_name = f"{self.tool_instance.tool_name}"
- self.tool_description = f"{self.tool_instance.tool_description}"
-
- self.client = client
- self.openai_api_key = openai_api_key or os.getenv("OPENAI_API_KEY")
- self.openai_base_url = openai_base_url or os.getenv("OPENAI_BASE_URL")
- self.model = model
- self.memory = ChatMemory()
- self.process_output = process_output
- self.agent_background = agent_background
-
- self.console = Console()
-
- self.input_schema = create_model(
- self.tool_name,
- tool_input=(str, Field(..., description=f"{self.tool_name} tool input. Presented as a single question or instruction", alias=f'tool_input_{self.tool_name}')),
- __config__=type('Config', (), {
- 'title': self.tool_name,
- 'description': self.tool_description,
- 'json_schema_extra': {
- "title": self.tool_name,
- "description": self.tool_description
- }
- })
- )
-
- if self.process_output:
- self.output_schema = ToolInterfaceAgentOutput
- else:
- self.output_schema = self.tool_instance.output_schema
-
- def get_system_prompt(self) -> str:
- current_date = datetime.now().isoformat()
- system_prompt = (
- f"{self.agent_background}\n"
- "\n"
- f"The tool being used is {self.tool_name}: {self.tool_description}.\n"
- f"\n"
- f"The current date, in ISO format, is {current_date}.\n"
- )
- return system_prompt
-
- def get_response(self, response_schema: Type[BaseModel] = ToolInterfaceAgentOutput) -> Type[BaseModel]:
- messages = [{"role": "system", "content": self.get_system_prompt()}] + self.memory.get_history()
- result = self.client.chat.completions.create(
- model=self.model,
- response_model=response_schema,
- messages=messages,
- )
- logger.verbose(f"get_response result: {result}")
- return result
-
- def run(self, params: Type[BaseModel]) -> Type[BaseModel]:
- text = params.tool_input
- self.memory.add_message("user", text)
- logger.verbose(f"User input added to memory: {text}")
- input_params = self.get_response(self.tool_instance.input_schema)
- logger.verbose(f"Input parameters received: {input_params}")
- tool_message = format_tool_message(input_params)
- logger.verbose(f"Formatted tool message: {tool_message}")
- output = self.tool_instance.run(input_params)
- logger.verbose(f"Tool output: {output}")
-
- if self.process_output:
- self.memory.add_message("assistant", "", tool_message=tool_message)
- self.memory.add_message("tool", output.model_dump_json(), tool_id=tool_message['id'])
- response = self.get_response(ToolInterfaceAgentOutput)
- logger.info(f"ToolInterfaceAgent response: {response}")
- self.memory.add_message("assistant", response.model_dump_json())
- return response
- else:
- return output
-
-
-if __name__ == "__main__":
- # Initialize the OpenAI client
- openai_client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
-
- # Instantiate tool instances directly
- web_search_tool = SearxNGSearchTool(max_results=20)
- calculator_tool = CalculatorTool()
-
- # Instantiate agents with tool instances and the client
- web_search_agent = ToolInterfaceAgent(web_search_tool, openai_client, model="gpt-3.5-turbo")
- calculator_agent = ToolInterfaceAgent(calculator_tool, openai_client, model="gpt-3.5-turbo")
-
- console = Console()
-
- # Example usage with SearxNGSearchTool
- web_search_output = web_search_agent.run(web_search_agent.input_schema(tool_input_SearxNGSearchTool="Search for the best restaurants in New York City"))
- console.print(web_search_output) # Type will be ToolInterfaceAgentOutput
- console.print("\n====================\n")
- calculator_output = calculator_agent.run(calculator_agent.input_schema(tool_input_CalculatorTool="Calculate the square root of 144"))
- console.print(calculator_output) # Type will be ToolInterfaceAgentOutput
\ No newline at end of file
diff --git a/atomic_agents_new/examples/article_idea_outline_gen_with_search.py b/atomic_agents/examples/article_idea_outline_gen_with_search.py
similarity index 90%
rename from atomic_agents_new/examples/article_idea_outline_gen_with_search.py
rename to atomic_agents/examples/article_idea_outline_gen_with_search.py
index 181399e..8441db9 100644
--- a/atomic_agents_new/examples/article_idea_outline_gen_with_search.py
+++ b/atomic_agents/examples/article_idea_outline_gen_with_search.py
@@ -1,16 +1,16 @@
from datetime import datetime
from rich.console import Console
-from atomic_agents.lib.chat_memory import ChatMemory
-from atomic_agents_new.agents.basic_chat_agent import BasicChatAgent
-from atomic_agents_new.lib.system_prompt_generator import DynamicInfoProviderBase, SystemPromptGenerator, SystemPromptInfo
+from atomic_agents.lib.components.chat_memory import ChatMemory
+from atomic_agents.agents.base_chat_agent import BaseChatAgent
+from atomic_agents.lib.components.system_prompt_generator import DynamicInfoProviderBase, SystemPromptGenerator, SystemPromptInfo
import instructor
import openai
-from atomic_agents_new.lib.tools.searx import SearxNGSearchTool
+from atomic_agents.lib.tools.searx import SearxNGSearchTool
from atomic_agents.lib.utils.logger import logger
console = Console()
-class MyChatAgent(BasicChatAgent):
+class MyChatAgent(BaseChatAgent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.search_tool = SearxNGSearchTool(max_results=25)
diff --git a/atomic_agents/lib/chat_memory.py b/atomic_agents/lib/components/chat_memory.py
similarity index 100%
rename from atomic_agents/lib/chat_memory.py
rename to atomic_agents/lib/components/chat_memory.py
diff --git a/atomic_agents_new/lib/system_prompt_generator.py b/atomic_agents/lib/components/system_prompt_generator.py
similarity index 100%
rename from atomic_agents_new/lib/system_prompt_generator.py
rename to atomic_agents/lib/components/system_prompt_generator.py
diff --git a/atomic_agents/lib/diff_file_contents.py b/atomic_agents/lib/diff_file_contents.py
deleted file mode 100644
index adcd58f..0000000
--- a/atomic_agents/lib/diff_file_contents.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import os
-from collections import defaultdict
-
-def diff_file_contents(file_contents_list):
- """ Find and remove lines that are common in three or more files from the given list of file contents. """
- line_occurrences = defaultdict(list)
- for i, lines in enumerate(file_contents_list):
- for line in set(lines): # Use set to avoid duplicate lines in the same file
- line_occurrences[line].append(i)
-
- # Filter lines that appear in three or more files
- common_lines = {line: files for line, files in line_occurrences.items() if len(files) >= 3}
-
- # Remove common lines from file contents
- for line, files in common_lines.items():
- for file_index in files:
- file_contents_list[file_index] = [l for l in file_contents_list[file_index] if l != line]
-
- return file_contents_list
-
-if __name__ == "__main__":
- # Define mock file contents
- file1_contents = [
- "This is a common line",
- "File 1 specific line",
- "Another common line",
- "File 1 unique line"
- ]
-
- file2_contents = [
- "This is a common line",
- "File 2 specific line",
- "Another common line",
- "File 2 unique line"
- ]
-
- file3_contents = [
- "This is a common line",
- "File 3 specific line",
- "Another common line",
- "File 3 unique line"
- ]
-
- file4_contents = [
- "File 4 specific line",
- "File 4 unique line"
- ]
-
- # Create a list of file contents
- file_contents_list = [file1_contents, file2_contents, file3_contents, file4_contents]
-
- # Call the diff_file_contents function
- result = diff_file_contents(file_contents_list)
-
- # Print the original file contents
- print("Original file contents:")
- for i, contents in enumerate(file_contents_list):
- print(f"File {i+1}:")
- for line in contents:
- print(line)
- print()
-
- # Print the modified file contents
- print("Modified file contents:")
- for i, contents in enumerate(result):
- print(f"File {i+1}:")
- for line in contents:
- print(line)
- print()
diff --git a/atomic_agents/lib/models/web_document.py b/atomic_agents/lib/models/web_document.py
index 254e527..ec1fb98 100644
--- a/atomic_agents/lib/models/web_document.py
+++ b/atomic_agents/lib/models/web_document.py
@@ -6,12 +6,8 @@ class WebDocumentMetadata(BaseModel):
description: str = Field(default='')
keywords: str = Field(default='')
author: str = Field(default='')
-
-class VectorDBDocumentMetadata(WebDocumentMetadata):
- document_part: int = Field(default=1)
- document_total_parts: int = Field(default=1)
-class Document(BaseModel):
+class WebDocument(BaseModel):
content: str
- metadata: WebDocumentMetadata | VectorDBDocumentMetadata = Field(default_factory=WebDocumentMetadata)
+ metadata: WebDocumentMetadata = Field(default_factory=WebDocumentMetadata)
diff --git a/atomic_agents/lib/scraping/link_scraper.py b/atomic_agents/lib/scraping/link_scraper.py
deleted file mode 100644
index 28eec4d..0000000
--- a/atomic_agents/lib/scraping/link_scraper.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import asyncio
-import logging
-from urllib.parse import urlparse, urljoin, urlunparse
-from collections import deque
-from typing import Set, List
-
-import aiohttp
-from bs4 import BeautifulSoup
-
-logging.basicConfig(level=logging.INFO)
-
-class InternalLinkScraper:
- @staticmethod
- def _normalize_url(url: str) -> str:
- parsed_url = urlparse(url)
- scheme = "https" if parsed_url.scheme == "http" else parsed_url.scheme
- path = parsed_url.path.rstrip("/")
- return urlunparse(parsed_url._replace(scheme=scheme, path=path))
-
- @staticmethod
- async def _get_internal_links(session: aiohttp.ClientSession, url: str, visited: Set[str], queue: deque, continue_on_error: bool = True):
- try:
- logging.info(f"Processing {url}")
- async with session.get(url) as response:
- response.raise_for_status()
- text = await response.text()
- soup = BeautifulSoup(text, 'html.parser')
- links = soup.find_all('a', href=True)
-
- for link in links:
- href = link['href']
- if href and not href.startswith("#"):
- href = InternalLinkScraper._normalize_url(urljoin(url, href))
- parsed_href = urlparse(href)
- parsed_base = urlparse(url)
-
- if parsed_href.netloc == parsed_base.netloc and href not in visited:
- queue.append(href)
-
- except Exception as e:
- logging.error(f"Failed to process {url}: {str(e)}")
- if not continue_on_error:
- raise e
-
- @staticmethod
- async def scrape_internal_links(start_url: str, continue_on_error: bool = True, max_workers: int = 10) -> List[str]:
- visited = set()
- queue = deque([InternalLinkScraper._normalize_url(start_url)])
-
- async with aiohttp.ClientSession() as session:
- tasks = []
- while queue or tasks:
- while queue and len(tasks) < max_workers:
- url = queue.popleft()
- if url not in visited:
- task = asyncio.create_task(InternalLinkScraper._get_internal_links(session, url, visited, queue, continue_on_error))
- tasks.append(task)
- visited.add(url)
-
- if tasks:
- done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
- tasks = list(pending)
- for task in done:
- try:
- await task
- except Exception as e:
- logging.error(f"Error processing task: {str(e)}")
-
- return list(visited)
-
-if __name__ == "__main__":
- from time import time
- start_time = time()
- url = "https://brainblendai.com"
- internal_links = asyncio.run(InternalLinkScraper.scrape_internal_links(url))
- scrape_time = time() - start_time
- print('SCRAPING STATS')
- print('============')
- print(f"Scraped {len(internal_links)} internal links from {url} in {scrape_time:.2f} seconds")
- print(internal_links)
\ No newline at end of file
diff --git a/atomic_agents/lib/scraping/pdf_to_markdown.py b/atomic_agents/lib/scraping/pdf_to_markdown.py
index 0da73c1..86443db 100644
--- a/atomic_agents/lib/scraping/pdf_to_markdown.py
+++ b/atomic_agents/lib/scraping/pdf_to_markdown.py
@@ -5,7 +5,7 @@ from PyPDF2 import PdfReader
from io import BytesIO, StringIO
import markdownify
-from atomic_agents.lib.models.web_document import Document, WebDocumentMetadata
+from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
class PdfToMarkdownConverter:
@staticmethod
@@ -41,14 +41,14 @@ class PdfToMarkdownConverter:
markdown_content = PdfToMarkdownConverter._convert_to_markdown(text_content)
metadata = WebDocumentMetadata(url=url or file_path, title="", description="", keywords="", author="")
- return Document(content=markdown_content, metadata=metadata)
+ return WebDocument(content=markdown_content, metadata=metadata)
if __name__ == '__main__':
from rich.console import Console
from rich.markdown import Markdown
console = Console()
- url = "https://web.pdx.edu/~arhodes/ai5.pdf"
+ url = "https://pdfobject.com/pdf/sample.pdf"
document = PdfToMarkdownConverter.convert(url=url)
console.print(Markdown(document.content))
\ No newline at end of file
diff --git a/atomic_agents/lib/scraping/url_to_markdown.py b/atomic_agents/lib/scraping/url_to_markdown.py
index a14d884..2ab0f8e 100644
--- a/atomic_agents/lib/scraping/url_to_markdown.py
+++ b/atomic_agents/lib/scraping/url_to_markdown.py
@@ -4,7 +4,7 @@ import requests
from markdownify import MarkdownConverter
from pydantic import BaseModel, Field
-from atomic_agents.lib.models.web_document import Document, WebDocumentMetadata
+from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
class UrlToMarkdownConverter:
@@ -61,7 +61,7 @@ class UrlToMarkdownConverter:
metadata = UrlToMarkdownConverter._extract_metadata(soup, url)
markdown_content = UrlToMarkdownConverter._convert_to_markdown(soup)
- return Document(content=markdown_content, metadata=metadata)
+ return WebDocument(content=markdown_content, metadata=metadata)
if __name__ == '__main__':
url = "https://brainblendai.com"
diff --git a/atomic_agents/tools/README.md b/atomic_agents/lib/tools/README.md
similarity index 100%
rename from atomic_agents/tools/README.md
rename to atomic_agents/lib/tools/README.md
diff --git a/atomic_agents/tools/base.py b/atomic_agents/lib/tools/base.py
similarity index 100%
rename from atomic_agents/tools/base.py
rename to atomic_agents/lib/tools/base.py
diff --git a/atomic_agents/tools/calculator_tool.py b/atomic_agents/lib/tools/calculator_tool.py
similarity index 96%
rename from atomic_agents/tools/calculator_tool.py
rename to atomic_agents/lib/tools/calculator_tool.py
index 3c77c8e..ff85eef 100644
--- a/atomic_agents/tools/calculator_tool.py
+++ b/atomic_agents/lib/tools/calculator_tool.py
@@ -6,7 +6,7 @@ from sympy import sympify
import instructor
from openai import OpenAI
-from atomic_agents.tools.base import BaseTool
+from atomic_agents.lib.tools.base import BaseTool
################
# INPUT SCHEMA #
diff --git a/atomic_agents/tools/searx.py b/atomic_agents/lib/tools/searx.py
similarity index 94%
rename from atomic_agents/tools/searx.py
rename to atomic_agents/lib/tools/searx.py
index 6fe3b0b..9de5b69 100644
--- a/atomic_agents/tools/searx.py
+++ b/atomic_agents/lib/tools/searx.py
@@ -6,7 +6,7 @@ from rich.console import Console
import openai
import instructor
-from atomic_agents.tools.base import BaseTool
+from atomic_agents.lib.tools.base import BaseTool
################
# INPUT SCHEMA #
@@ -119,10 +119,12 @@ if __name__ == "__main__":
result = client.chat.completions.create(
model="gpt-3.5-turbo",
response_model=SearxNGSearchTool.input_schema,
- messages=[{"role": "user", "content": "Search for the latest PC gaming news of May 2024"}],
+ messages=[{"role": "user", "content": "Search for the latest PC gaming news of May 2024 using 3 different queries."}],
)
+ rich_console.print(f"Search queries: {result.queries}")
+
# Print the result
- output = SearxNGSearchTool(tool_description_override="derp", max_results=15).run(result)
+ output = SearxNGSearchTool(max_results=15).run(result)
for i, result in enumerate(output.results):
rich_console.print(f"{i}. Title: {result.title}, URL: {result.url}")
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/user_input_tool.py b/atomic_agents/lib/tools/user_input_tool.py
similarity index 97%
rename from atomic_agents_new/lib/tools/user_input_tool.py
rename to atomic_agents/lib/tools/user_input_tool.py
index 6d63c63..59d2489 100644
--- a/atomic_agents_new/lib/tools/user_input_tool.py
+++ b/atomic_agents/lib/tools/user_input_tool.py
@@ -5,7 +5,7 @@ from rich.console import Console
import openai
import instructor
-from atomic_agents.tools.base import BaseTool
+from atomic_agents.lib.tools.base import BaseTool
################
# INPUT SCHEMA #
diff --git a/atomic_agents/tools/web_scraping_tool.py b/atomic_agents/lib/tools/web_scraping_tool.py
similarity index 94%
rename from atomic_agents/tools/web_scraping_tool.py
rename to atomic_agents/lib/tools/web_scraping_tool.py
index d7eba9b..bbae0cf 100644
--- a/atomic_agents/tools/web_scraping_tool.py
+++ b/atomic_agents/lib/tools/web_scraping_tool.py
@@ -6,10 +6,10 @@ import openai
import instructor
import requests
-from atomic_agents.tools.base import BaseTool
+from atomic_agents.lib.tools.base import BaseTool
from atomic_agents.lib.scraping.url_to_markdown import UrlToMarkdownConverter
from atomic_agents.lib.scraping.pdf_to_markdown import PdfToMarkdownConverter
-from atomic_agents.lib.models.web_document import Document
+from atomic_agents.lib.models.web_document import WebDocument
################
# INPUT SCHEMA #
diff --git a/atomic_agents/lib/utils/doc_splitter.py b/atomic_agents/lib/utils/doc_splitter.py
deleted file mode 100644
index 54a87ed..0000000
--- a/atomic_agents/lib/utils/doc_splitter.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from typing import List
-from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
-
-def split_document(document: Document, max_length=1000) -> List[Document]:
- if len(document.content) <= max_length:
- return [Document(content=document.content, metadata=VectorDBDocumentMetadata(**document.metadata.model_dump(), document_part=1, document_total_parts=1))]
-
- parts = []
- current_part = ''
- for line in document.content.splitlines(keepends=True):
- if len(current_part) + len(line) <= max_length:
- current_part += line
- else:
- parts.append(current_part)
- current_part = line
-
- if current_part:
- parts.append(current_part)
-
- total_parts = len(parts)
- return [
- Document(
- content=part.strip(),
- metadata=VectorDBDocumentMetadata(**document.metadata.model_dump(), document_part=i+1, document_total_parts=total_parts)
- )
- for i, part in enumerate(parts)
- ]
\ No newline at end of file
diff --git a/atomic_agents/lib/utils/document_to_string.py b/atomic_agents/lib/utils/document_to_string.py
deleted file mode 100644
index a09b2eb..0000000
--- a/atomic_agents/lib/utils/document_to_string.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
-
-def document_to_string(document: Document) -> str:
- output = []
-
- # Determine the title or URL for the header
- header_title = document.metadata.title if document.metadata.title else document.metadata.url
- part_info = ""
-
- if isinstance(document.metadata, VectorDBDocumentMetadata):
- part_info = f" (Part {document.metadata.document_part} of {document.metadata.document_total_parts})"
-
- # Add the header with title and part information
- output.append(f"--- [{header_title}]{part_info} ---")
-
- # Add metadata fields except 'title' if it's already in the header
- for field_name, field_value in document.metadata.model_dump().items():
- if field_value and field_name != "title":
- output.append(f"{field_name.replace('_', ' ').title()}: {field_value}")
-
- # Add the content of the document
- output.append("Document Content:")
- output.append(document.content)
-
- # Join all parts with newlines and return
- return "\n".join(output) + "\n"
-
-if __name__ == '__main__':
- # Example usage
- metadata = VectorDBDocumentMetadata(
- url="http://example.com",
- title="Example Document",
- description="An example document stored in VectorDB.",
- keywords="example, sample, document",
- author="John Doe",
- document_part=1,
- document_total_parts=3
- )
-
- doc = Document(
- content="This is the content of the document.",
- metadata=metadata
- )
-
- document_str = document_to_string(doc)
- print(document_str)
diff --git a/atomic_agents/lib/vectordb/chroma/ingest.py b/atomic_agents/lib/vectordb/chroma/ingest.py
deleted file mode 100644
index f3b4057..0000000
--- a/atomic_agents/lib/vectordb/chroma/ingest.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import asyncio
-import os
-from typing import List
-from pydantic import HttpUrl
-from atomic_agents.lib.diff_file_contents import diff_file_contents
-from atomic_agents.lib.scraping.link_scraper import InternalLinkScraper
-from atomic_agents.lib.scraping.url_to_markdown import UrlToMarkdownConverter
-from atomic_agents.lib.utils.doc_splitter import split_document
-from atomic_agents.lib.utils.document_to_string import document_to_string
-from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
-from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
-
-def scrape_and_clean_urls(urls: List[HttpUrl], db_path='chromadb_tmp', collection_name = None, do_diff=True) -> List[Document]:
- try:
- web_documents = []
- for url in urls:
- try:
- document = UrlToMarkdownConverter.convert(str(url))
- web_documents.append(document)
- print(f"Scraped URL: {url}")
- except Exception as e:
- print(f"Error converting URL to markdown: {e}")
- pass
-
- if do_diff:
- cleaned_contents = diff_file_contents([doc.content.split('\n') for doc in web_documents])
- else:
- cleaned_contents = [doc.content.split('\n') for doc in web_documents]
- cleaned_documents = []
- for content, metadata in zip(cleaned_contents, [doc.metadata for doc in web_documents]):
- cleaned_documents.append(Document(content='\n'.join(content).strip(), metadata=metadata))
-
- # Ensure the URL is parsed correctly to get the host
- db = ChromaDBDocumentManager(db_path, collection_name=collection_name or urls[0].host)
- db.process_documents(cleaned_documents)
-
- return cleaned_documents
- except Exception as e:
- print(f"Error scraping and cleaning URLs: {e}")
- return []
-
-if __name__ == "__main__":
- # Example usage
- url = "https://brainblendai.com"
- internal_links = asyncio.run(InternalLinkScraper.scrape_internal_links(url))
- documents = scrape_and_clean_urls(internal_links, collection_name='brainblendai')
- for doc in documents:
- print(document_to_string(doc))
\ No newline at end of file
diff --git a/atomic_agents/lib/vectordb/chroma/manager.py b/atomic_agents/lib/vectordb/chroma/manager.py
deleted file mode 100644
index 59e907c..0000000
--- a/atomic_agents/lib/vectordb/chroma/manager.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import os
-from typing import List
-import chromadb
-import openai
-from atomic_agents.lib.utils.doc_splitter import split_document
-from atomic_agents.lib.utils.document_to_string import document_to_string
-from atomic_agents.lib.models.web_document import Document, VectorDBDocumentMetadata
-
-class ChromaDBDocumentManager:
- def __init__(self, db_path, collection_name):
- self.db = chromadb.PersistentClient(path=db_path)
- self.collection_name = collection_name
-
- def process_documents(self, documents: List[Document], max_length=1000):
- collection = self.db.get_or_create_collection(self.collection_name)
- for document in documents:
- document_parts = split_document(document, max_length)
- for part in document_parts:
- embedding = self._create_embedding(part.content)
- collection.upsert(
- ids=[f"{document.metadata.url}_{part.metadata.document_part}_{part.metadata.document_total_parts}"],
- embeddings=embedding,
- metadatas=[part.metadata.model_dump()],
- documents=[part.content]
- )
-
- def create_collection(self):
- self.db.create_collection(self.collection_name)
-
- def delete_collection(self):
- self.db.delete_collection(self.collection_name)
-
- def query_documents(self, queries: List[str], n_results: int = 3) -> List[Document]:
- collection = self.db.get_collection(self.collection_name)
- if not collection:
- raise ValueError(f"Collection '{self.collection_name}' not found.")
-
- embeddings = self._create_embedding(queries)
-
- results = collection.query(
- query_embeddings=embeddings,
- n_results=n_results,
- )
-
- flattened_result_documents = [doc for sublist in results['documents'] for doc in sublist]
- flattened_result_metadatas = [metadata for sublist in results['metadatas'] for metadata in sublist]
- flattened_result_distances = [distance for sublist in results['distances'] for distance in sublist]
-
- # Sort all lists by distance
- results = sorted(zip(flattened_result_documents, flattened_result_metadatas, flattened_result_distances), key=lambda x: x[2])
-
- # Deduplicate results
- deduplicated_results = []
- seen_content = set()
- for content, metadata, distance in results:
- if content not in seen_content:
- deduplicated_results.append((content, metadata))
- seen_content.add(content)
-
- results = {
- 'documents': [content for content, metadata in deduplicated_results],
- 'metadatas': [metadata for content, metadata in deduplicated_results]
- }
-
- # Construct Document objects from the results
- documents = []
- for content, metadata in list(zip(results['documents'], results['metadatas']))[:n_results]:
- document = Document(content=content, metadata=VectorDBDocumentMetadata(**metadata))
- documents.append(document)
-
- return documents
-
- def _create_embedding(self, content):
- embedding_client = os.getenv('EMBEDDING_CLIENT')
-
- if embedding_client == 'openai':
- openai_api_key = os.getenv('OPENAI_API_KEY')
- embedding_client = openai.OpenAI(api_key=openai_api_key)
- response = embedding_client.embeddings.create(
- input=content,
- model="text-embedding-3-small"
- )
- elif embedding_client == 'local':
- local_host = os.getenv('LOCAL_CLIENT_HOST')
- embedding_client = openai.OpenAI(base_url=local_host, api_key='N/A')
- response = embedding_client.embeddings.create(
- input=content,
- model="N/A"
- )
- else:
- raise ValueError("Unsupported embedding client. Choose 'openai' or 'local'.")
-
- return [response.data[i].embedding for i in range(len(response.data))]
-
- @staticmethod
- def document_to_string(doc: Document) -> str:
- return document_to_string(doc)
-
-if __name__ == '__main__':
- db = ChromaDBDocumentManager(db_path='chromadb_persist', collection_name='brainblendai.com')
- documents = db.query_documents(["What is the best AI assistant?", "What does describeit do?" ,"What is the best AI assistant?"], n_results=3)
- for doc in documents:
- print(doc.metadata.url)
\ No newline at end of file
diff --git a/atomic_agents/tools/user_input_tool.py b/atomic_agents/tools/user_input_tool.py
deleted file mode 100644
index 6d63c63..0000000
--- a/atomic_agents/tools/user_input_tool.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import os
-from pydantic import BaseModel, Field
-from typing import Optional
-from rich.console import Console
-import openai
-import instructor
-
-from atomic_agents.tools.base import BaseTool
-
-################
-# INPUT SCHEMA #
-################
-class UserInputToolSchema(BaseModel):
- question: str = Field(..., description="The question to ask the user.")
- context: Optional[str] = Field(None, description="Additional context or information to provide to the user.")
-
- class Config:
- title = "UserInputTool"
- description = "Tool for asking clarifying questions to the user. Use this tool when the AI needs more information or clarification from the user."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-####################
-# OUTPUT SCHEMA(S) #
-####################
-class UserInputToolOutputSchema(BaseModel):
- answer: str = Field(..., description="The user's answer to the question.")
-
-##############
-# TOOL LOGIC #
-##############
-class UserInputTool(BaseTool):
- input_schema = UserInputToolSchema
- output_schema = UserInputToolOutputSchema
-
- def __init__(self, tool_description_override: Optional[str] = None):
- super().__init__(tool_description_override)
-
- def run(self, params: UserInputToolSchema) -> UserInputToolOutputSchema:
- print(f"Question: {params.question}")
- if params.context:
- print(f"Context: {params.context}")
- answer = input("Your answer: ")
- return UserInputToolOutputSchema(answer=answer)
-
-#################
-# EXAMPLE USAGE #
-#################
-if __name__ == "__main__":
- rich_console = Console()
-
- question = "What is your favorite color?"
-
- user_input_tool = UserInputTool()
- user_input_params = UserInputToolSchema(question=question)
- user_input_output = user_input_tool.run(user_input_params)
-
- rich_console.print(f"User's answer: {user_input_output.answer}")
\ No newline at end of file
diff --git a/atomic_agents/tools/vectordb_query_tool.py b/atomic_agents/tools/vectordb_query_tool.py
deleted file mode 100644
index 8c38622..0000000
--- a/atomic_agents/tools/vectordb_query_tool.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import os
-from pydantic import BaseModel, Field
-from typing import List, Optional
-from rich.console import Console
-import openai
-import instructor
-
-from atomic_agents.tools.base import BaseTool
-from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
-from atomic_agents.lib.models.web_document import Document
-
-################
-# INPUT SCHEMA #
-################
-class VectorDBQueryToolSchema(BaseModel):
- queries: List[str] = Field(..., description="List of 3 queries to query the vector database.")
-
- class Config:
- title = "VectorDBQueryTool"
- description = "Tool for querying the Chroma VectorDB."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-####################
-# OUTPUT SCHEMA(S) #
-####################
-class VectorDBQueryResultSchema(BaseModel):
- content: str
- metadata: Optional[dict] = None
-
-class VectorDBQueryToolOutputSchema(BaseModel):
- results: List[VectorDBQueryResultSchema]
-
-##############
-# TOOL LOGIC #
-##############
-class VectorDBQueryTool(BaseTool):
- input_schema = VectorDBQueryToolSchema
- output_schema = VectorDBQueryToolOutputSchema
-
- def __init__(self, db_path: str, collection_name: str, tool_description_override: Optional[str] = None, max_results: int = 10):
- super().__init__(tool_description_override)
- self.db_path = db_path
- self.collection_name = collection_name
- self.max_results = max_results
- self.manager = ChromaDBDocumentManager(db_path, collection_name)
-
- def run(self, params: VectorDBQueryToolSchema) -> VectorDBQueryToolOutputSchema:
- documents = self.manager.query_documents(params.queries, self.max_results)
- results = [VectorDBQueryResultSchema(content=doc.content, metadata=doc.metadata.model_dump()) for doc in documents]
- return VectorDBQueryToolOutputSchema(results=results)
-
-#################
-# EXAMPLE USAGE #
-#################
-if __name__ == "__main__":
- rich_console = Console()
-
- # Initialize the client outside
- client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
-
- # Extract structured data from natural language
- result = client.chat.completions.create(
- model="gpt-3.5-turbo",
- response_model=VectorDBQueryTool.input_schema,
- messages=[
- {"role": "system", "content": "This AI generates 3 queries that can be used to gather context to help with reacting to the user input."},
- {"role": "user", "content": "What does the describeit tool do?"}
- ],
- )
-
- # Print the result
- output = VectorDBQueryTool(db_path="chromadb_persist", collection_name="brainblendai.com", max_results=3).run(result)
- for i, result in enumerate(output.results):
- rich_console.print(f"{i}. Content: {result.content}, Metadata: {result.metadata}")
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/README.md b/atomic_agents_new/lib/tools/README.md
deleted file mode 100644
index a5ce5eb..0000000
--- a/atomic_agents_new/lib/tools/README.md
+++ /dev/null
@@ -1,124 +0,0 @@
-# Creating New Tools
-
-This guide will help you create new tools based on the structure and patterns observed in `calculator_tool.py` and `searx.py`.
-
-## Step-by-Step Guide
-
-### 1. Import Necessary Modules
-
-Start by importing the necessary modules. Typically, you will need `os`, `pydantic`, and any other modules specific to your tool's functionality.
-
-```python
-import os
-from pydantic import BaseModel, Field
-# Add other necessary imports here
-```
-
-### 2. Define Input Schema
-
-Create a class for the input schema using `pydantic.BaseModel`. Define the fields and their descriptions. Also, include a nested `Config` class with `title`, `description`, and `json_schema_extra`.
-
-```python
-class YourToolInputSchema(BaseModel):
- # Define your input fields here
- example_field: str = Field(..., description="Description of the example field.")
-
- class Config:
- title = "YourTool"
- description = "Tool for performing specific tasks."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-```
-
-### 3. Define Output Schema
-
-Create a class for the output schema using `pydantic.BaseModel`. Define the fields and their descriptions.
-
-```python
-class YourToolOutputSchema(BaseModel):
- # Define your output fields here
- result: str = Field(..., description="Result of the operation.")
-```
-
-### 4. Implement the Tool Logic
-
-Create a class for your tool. Define `input_schema` and `output_schema` attributes. Implement the `run` method to process the input and produce the output.
-
-```python
-class YourTool:
- input_schema = YourToolInputSchema
- output_schema = YourToolOutputSchema
-
- def run(self, params: YourToolInputSchema) -> YourToolOutputSchema:
- try:
- # Implement your tool logic here
- result = "Your result here"
- return YourToolOutputSchema(result=result)
- except Exception as e:
- raise ValueError(f"Error processing input: {e}")
-```
-
-### 5. Example Usage
-
-Include an example usage section to demonstrate how to initialize and use your tool.
-
-```python
-if __name__ == "__main__":
- # Initialize your tool
- your_tool = YourTool()
-
- # Create input parameters
- input_params = YourToolInputSchema(
- example_field="Example input"
- )
-
- # Run the tool and get the output
- output = your_tool.run(input_params)
-
- # Print the result
- print(output)
-```
-
-## Example
-
-Here is a complete example of a simple tool that echoes the input string.
-
-```python
-import os
-from pydantic import BaseModel, Field
-
-class EchoToolInputSchema(BaseModel):
- message: str = Field(..., description="Message to echo.")
-
- class Config:
- title = "EchoTool"
- description = "Tool for echoing messages."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-class EchoToolOutputSchema(BaseModel):
- echoed_message: str = Field(..., description="Echoed message.")
-
-class EchoTool:
- input_schema = EchoToolInputSchema
- output_schema = EchoToolOutputSchema
-
- def run(self, params: EchoToolInputSchema) -> EchoToolOutputSchema:
- try:
- echoed_message = params.message
- return EchoToolOutputSchema(echoed_message=echoed_message)
- except Exception as e:
- raise ValueError(f"Error echoing message: {e}")
-
-if __name__ == "__main__":
- echo_tool = EchoTool()
- input_params = EchoToolInputSchema(message="Hello, World!")
- output = echo_tool.run(input_params)
- print(output)
-```
-
-By following this guide, you can create new tools with a consistent structure and easily integrate them into your projects.
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/base.py b/atomic_agents_new/lib/tools/base.py
deleted file mode 100644
index 92ee767..0000000
--- a/atomic_agents_new/lib/tools/base.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from typing import Type, Optional
-from pydantic import BaseModel
-
-class BaseTool:
- input_schema: Type[BaseModel]
- output_schema: Type[BaseModel]
-
- def __init__(self, tool_description_override: Optional[str] = None):
- self.tool_name = self.input_schema.Config.title
- self.tool_description = tool_description_override or self.input_schema.Config.description
-
- def run(self, params: BaseModel) -> BaseModel:
- raise NotImplementedError("Subclasses should implement this method")
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/calculator_tool.py b/atomic_agents_new/lib/tools/calculator_tool.py
deleted file mode 100644
index 3c77c8e..0000000
--- a/atomic_agents_new/lib/tools/calculator_tool.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import os
-from typing import Optional
-import openai
-from pydantic import BaseModel, Field
-from sympy import sympify
-import instructor
-from openai import OpenAI
-
-from atomic_agents.tools.base import BaseTool
-
-################
-# INPUT SCHEMA #
-################
-class CalculatorToolSchema(BaseModel):
- expression: str = Field(..., description="Mathematical expression to evaluate. For example, '2 + 2'.")
-
- class Config:
- title = "CalculatorTool"
- description = "Tool for performing calculations. Supports basic arithmetic operations like addition, subtraction, multiplication, and division, but also more complex operations like exponentiation and trigonometric functions. Use this tool to evaluate mathematical expressions."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-####################
-# OUTPUT SCHEMA(S) #
-####################
-class CalculatorToolOutputSchema(BaseModel):
- result: str = Field(..., description="Result of the calculation.")
-
-################
-# TOOL LOGIC #
-################
-class CalculatorTool(BaseTool):
- input_schema = CalculatorToolSchema
- output_schema = CalculatorToolOutputSchema
-
- def __init__(self, tool_description_override: Optional[str] = None):
- super().__init__(tool_description_override)
-
- def run(self, params: CalculatorToolSchema) -> CalculatorToolOutputSchema:
- try:
- # Explicitly convert the string form of the expression
- parsed_expr = sympify(str(params.expression))
- # Evaluate the expression numerically
- result = parsed_expr.evalf()
- return CalculatorToolOutputSchema(result=str(result))
- except Exception as e:
- raise ValueError(f"Error evaluating expression: {e}")
-
-#################
-# EXAMPLE USAGE #
-#################
-if __name__ == "__main__":
-
- # Initialize the client outside
- client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
-
- # Extract structured data from natural language
- result = client.chat.completions.create(
- model="gpt-3.5-turbo",
- response_model=CalculatorTool.input_schema,
- messages=[{"role": "user", "content": "Calculate 2 + 2"}],
- )
-
- # Print the result
- print(CalculatorTool().run(result))
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/searx.py b/atomic_agents_new/lib/tools/searx.py
deleted file mode 100644
index 6fe3b0b..0000000
--- a/atomic_agents_new/lib/tools/searx.py
+++ /dev/null
@@ -1,128 +0,0 @@
-import os
-import requests
-from pydantic import BaseModel, Field
-from typing import List, Literal, Optional
-from rich.console import Console
-import openai
-import instructor
-
-from atomic_agents.tools.base import BaseTool
-
-################
-# INPUT SCHEMA #
-################
-class SearxNGSearchToolSchema(BaseModel):
- queries: List[str] = Field(..., description="List of search queries.")
- category: Optional[Literal["general", "news", "social_media"]] = Field(None, description="Category of the search queries.")
-
- class Config:
- title = "SearxNGSearchTool"
- description = "Tool for searching for information, news, references, and other content on the SearxNG. Returns a list of search results with a short description or content snippet and URLs for further exploration."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-####################
-# OUTPUT SCHEMA(S) #
-####################
-class SearxNGSearchResultSchema(BaseModel):
- url: str
- title: str
- content: Optional[str] = None
- category: Optional[str] = None
-
-class SearxNGSearchToolOutputSchema(BaseModel):
- results: List[SearxNGSearchResultSchema]
-
-##############
-# TOOL LOGIC #
-##############
-class SearxNGSearchTool(BaseTool):
- input_schema = SearxNGSearchToolSchema
- output_schema = SearxNGSearchToolOutputSchema
-
- def __init__(self, tool_description_override: Optional[str] = None, max_results: int = 10):
- super().__init__(tool_description_override)
- self.max_results = max_results
-
- def run(self, params: SearxNGSearchToolSchema, max_results: Optional[int] = None) -> SearxNGSearchToolOutputSchema:
- SEARXNG_BASE_URL = os.getenv('SEARXNG_BASE_URL')
- if not SEARXNG_BASE_URL:
- raise ValueError("SEARXNG_BASE_URL environment variable not set")
-
- all_results = []
-
- for query in params.queries:
- # Prepare the query parameters
- query_params = {
- 'q': query,
- 'safesearch': '0',
- 'format': 'json',
- 'language': 'en',
- 'engines': 'bing,duckduckgo,google,startpage,yandex',
- }
-
- # Add category to query parameters if it is set
- if params.category:
- query_params['categories'] = params.category
-
- # Make the GET request
- response = requests.get(f"{SEARXNG_BASE_URL}/search", params=query_params)
-
- # Check if the request was successful
- if response.status_code != 200:
- raise Exception(f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}")
-
- results = response.json().get('results', [])
- all_results.extend(results)
-
- # Sort the combined results by score in descending order
- sorted_results = sorted(all_results, key=lambda x: x.get('score', 0), reverse=True)
-
- # Remove duplicates while preserving order
- seen_urls = set()
- unique_results = []
- for result in sorted_results:
- if 'content' not in result or 'title' not in result or 'url' not in result:
- continue
- if result['url'] not in seen_urls:
- unique_results.append(result)
- seen_urls.add(result['url'])
-
- # Filter results to include only those with the correct category if it is set
- if params.category:
- filtered_results = [result for result in unique_results if result.get('category') == params.category]
- else:
- filtered_results = unique_results
-
- filtered_results = filtered_results[:max_results or self.max_results]
-
- return SearxNGSearchToolOutputSchema(results=filtered_results)
-
-
-#################
-# EXAMPLE USAGE #
-#################
-if __name__ == "__main__":
- rich_console = Console()
-
- # Initialize the client outside
- client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
-
- # Extract structured data from natural language
- result = client.chat.completions.create(
- model="gpt-3.5-turbo",
- response_model=SearxNGSearchTool.input_schema,
- messages=[{"role": "user", "content": "Search for the latest PC gaming news of May 2024"}],
- )
-
- # Print the result
- output = SearxNGSearchTool(tool_description_override="derp", max_results=15).run(result)
- for i, result in enumerate(output.results):
- rich_console.print(f"{i}. Title: {result.title}, URL: {result.url}")
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/vectordb_query_tool.py b/atomic_agents_new/lib/tools/vectordb_query_tool.py
deleted file mode 100644
index 8c38622..0000000
--- a/atomic_agents_new/lib/tools/vectordb_query_tool.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import os
-from pydantic import BaseModel, Field
-from typing import List, Optional
-from rich.console import Console
-import openai
-import instructor
-
-from atomic_agents.tools.base import BaseTool
-from atomic_agents.lib.vectordb.chroma.manager import ChromaDBDocumentManager
-from atomic_agents.lib.models.web_document import Document
-
-################
-# INPUT SCHEMA #
-################
-class VectorDBQueryToolSchema(BaseModel):
- queries: List[str] = Field(..., description="List of 3 queries to query the vector database.")
-
- class Config:
- title = "VectorDBQueryTool"
- description = "Tool for querying the Chroma VectorDB."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-####################
-# OUTPUT SCHEMA(S) #
-####################
-class VectorDBQueryResultSchema(BaseModel):
- content: str
- metadata: Optional[dict] = None
-
-class VectorDBQueryToolOutputSchema(BaseModel):
- results: List[VectorDBQueryResultSchema]
-
-##############
-# TOOL LOGIC #
-##############
-class VectorDBQueryTool(BaseTool):
- input_schema = VectorDBQueryToolSchema
- output_schema = VectorDBQueryToolOutputSchema
-
- def __init__(self, db_path: str, collection_name: str, tool_description_override: Optional[str] = None, max_results: int = 10):
- super().__init__(tool_description_override)
- self.db_path = db_path
- self.collection_name = collection_name
- self.max_results = max_results
- self.manager = ChromaDBDocumentManager(db_path, collection_name)
-
- def run(self, params: VectorDBQueryToolSchema) -> VectorDBQueryToolOutputSchema:
- documents = self.manager.query_documents(params.queries, self.max_results)
- results = [VectorDBQueryResultSchema(content=doc.content, metadata=doc.metadata.model_dump()) for doc in documents]
- return VectorDBQueryToolOutputSchema(results=results)
-
-#################
-# EXAMPLE USAGE #
-#################
-if __name__ == "__main__":
- rich_console = Console()
-
- # Initialize the client outside
- client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
-
- # Extract structured data from natural language
- result = client.chat.completions.create(
- model="gpt-3.5-turbo",
- response_model=VectorDBQueryTool.input_schema,
- messages=[
- {"role": "system", "content": "This AI generates 3 queries that can be used to gather context to help with reacting to the user input."},
- {"role": "user", "content": "What does the describeit tool do?"}
- ],
- )
-
- # Print the result
- output = VectorDBQueryTool(db_path="chromadb_persist", collection_name="brainblendai.com", max_results=3).run(result)
- for i, result in enumerate(output.results):
- rich_console.print(f"{i}. Content: {result.content}, Metadata: {result.metadata}")
\ No newline at end of file
diff --git a/atomic_agents_new/lib/tools/web_scraping_tool.py b/atomic_agents_new/lib/tools/web_scraping_tool.py
deleted file mode 100644
index d7eba9b..0000000
--- a/atomic_agents_new/lib/tools/web_scraping_tool.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import os
-from pydantic import BaseModel, Field
-from typing import Optional
-from rich.console import Console
-import openai
-import instructor
-import requests
-
-from atomic_agents.tools.base import BaseTool
-from atomic_agents.lib.scraping.url_to_markdown import UrlToMarkdownConverter
-from atomic_agents.lib.scraping.pdf_to_markdown import PdfToMarkdownConverter
-from atomic_agents.lib.models.web_document import Document
-
-################
-# INPUT SCHEMA #
-################
-class WebScrapingToolSchema(BaseModel):
- url: str = Field(..., description="URL of the web page to scrape.")
-
- class Config:
- title = "WebScrapingTool"
- description = "Tool for scraping web pages and converting content to markdown in order to extract information or summarize the content."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
-####################
-# OUTPUT SCHEMA(S) #
-####################
-class WebScrapingResultSchema(BaseModel):
- content: str
- metadata: Optional[dict] = None
-
-class WebScrapingToolOutputSchema(BaseModel):
- result: WebScrapingResultSchema
-
-##############
-# TOOL LOGIC #
-##############
-class WebScrapingTool(BaseTool):
- input_schema = WebScrapingToolSchema
- output_schema = WebScrapingToolOutputSchema
-
- def __init__(self, tool_description_override: Optional[str] = None):
- super().__init__(tool_description_override)
-
- def run(self, params: WebScrapingToolSchema) -> WebScrapingToolOutputSchema:
- url = params.url
- response = requests.head(url)
- content_type = response.headers.get('Content-Type', '')
-
- if 'application/pdf' in content_type:
- document = PdfToMarkdownConverter.convert(url=url)
- else:
- document = UrlToMarkdownConverter.convert(url=url)
-
- result = WebScrapingResultSchema(content=document.content, metadata=document.metadata.model_dump())
- return WebScrapingToolOutputSchema(result=result)
-
-#################
-# EXAMPLE USAGE #
-#################
-if __name__ == "__main__":
- rich_console = Console()
-
- # Initialize the client outside
- client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
-
- # Extract structured data from natural language
- result = client.chat.completions.create(
- model="gpt-3.5-turbo",
- response_model=WebScrapingTool.input_schema,
- messages=[{"role": "user", "content": "Scrape the content of https://example.com"}],
- )
-
- # Print the result
- output = WebScrapingTool().run(result)
- rich_console.print(f"Content: {output.result.content}, Metadata: {output.result.metadata}")
\ No newline at end of file