Skip to content

Building AI agents, atomically

The Big Lint

KennyVaneetvelde85dab967e920
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 80aab27..8ee71b5 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -27,15 +27,18 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
- python -m pip install flake8 pytest
+ python -m pip install flake8 pytest black
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
pip install -e .
+ - name: Check Black formatting
+ run: |
+ black --check --diff atomic_agents
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 atomic_agents --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- # flake8 atomic_agents --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ flake8 atomic_agents --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: |
- pytest --import-mode=importlib
+ pytest --import-mode=importlib
\ No newline at end of file
diff --git a/atomic_agents/agents/base_chat_agent.py b/atomic_agents/agents/base_chat_agent.py
index c32160c..06ffab6 100644
--- a/atomic_agents/agents/base_chat_agent.py
+++ b/atomic_agents/agents/base_chat_agent.py
@@ -1,64 +1,73 @@
-from typing import List, Optional, Type
+from typing import Optional, Type
+
import instructor
-import openai
from pydantic import BaseModel, Field
from rich.json import JSON
from atomic_agents.lib.components.chat_memory import ChatMemory
from atomic_agents.lib.components.system_prompt_generator import SystemPromptContextProviderBase, SystemPromptGenerator
+
class BaseAgentIO(BaseModel):
"""
Base class for input and output schemas for chat agents.
"""
+
def __str__(self):
return self.model_dump_json()
-
+
def __rich__(self):
json_str = self.model_dump_json()
return JSON(json_str)
+
class BaseChatAgentInputSchema(BaseAgentIO):
chat_message: str = Field(
...,
- description='The chat message exchanged between the user and the chat agent. '
- 'This represents the message sent by the user.'
+ description="The chat message sent by the user to the assistant.",
)
class Config:
- title = 'BaseChatAgentInputSchema'
- description = 'This schema represents the user input message exchanged between the user and the chat agent.'
+ title = "BaseChatAgentInputSchema"
+ description = "This schema represents the user input message exchanged between the user and the chat agent."
json_schema_extra = {
"title": title,
"description": description,
}
+
class BaseChatAgentOutputSchema(BaseAgentIO):
chat_message: str = Field(
...,
- description='The chat message exchanged between the user and the chat agent. '
- 'This contains the markdown-enabled response generated by the chat agent.'
+ description=(
+ "The chat message exchanged between the user and the chat agent. "
+ "This contains the markdown-enabled response generated by the chat agent."
+ ),
)
class Config:
- title = 'BaseChatAgentOutputSchema'
- description = 'This schema represents the response message exchanged between the user and the chat agent.'
+ title = "BaseChatAgentOutputSchema"
+ description = "This schema represents the response message exchanged between the user and the chat agent."
json_schema_extra = {
"title": title,
"description": description,
}
+
class BaseChatAgentConfig(BaseModel):
- client: instructor.client.Instructor = Field(..., description='Client for interacting with the language model.')
- model: str = Field("gpt-3.5-turbo", description='The model to use for generating responses.')
- memory: Optional[ChatMemory] = Field(None, description='Memory component for storing chat history.')
- system_prompt_generator: Optional[SystemPromptGenerator] = Field(None, description='Component for generating system prompts.')
- input_schema: Optional[Type[BaseModel]] = Field(None, description='The schema for the input data.')
- output_schema: Optional[Type[BaseModel]] = Field(None, description='The schema for the output data.')
-
+ client: instructor.client.Instructor = Field(..., description="Client for interacting with the language model.")
+ model: str = Field("gpt-3.5-turbo", description="The model to use for generating responses.")
+ memory: Optional[ChatMemory] = Field(None, description="Memory component for storing chat history.")
+ system_prompt_generator: Optional[SystemPromptGenerator] = Field(
+ None, description="Component for generating system prompts."
+ )
+ input_schema: Optional[Type[BaseModel]] = Field(None, description="The schema for the input data.")
+ output_schema: Optional[Type[BaseModel]] = Field(None, description="The schema for the output data.")
+
class Config:
arbitrary_types_allowed = True
+
class BaseChatAgent:
"""
Base class for chat agents.
@@ -75,6 +84,7 @@ class BaseChatAgent:
system_prompt_generator (SystemPromptGenerator): Component for generating system prompts.
initial_memory (ChatMemory): Initial state of the memory.
"""
+
input_schema = BaseChatAgentInputSchema
output_schema = BaseChatAgentOutputSchema
@@ -105,7 +115,8 @@ class BaseChatAgent:
Obtains a response from the language model.
Args:
- response_model (Type[BaseModel], optional): The schema for the response data. If not set, self.output_schema is used.
+ response_model (Type[BaseModel], optional):
+ The schema for the response data. If not set, self.output_schema is used.
Returns:
Type[BaseModel]: The response from the language model.
@@ -113,12 +124,13 @@ class BaseChatAgent:
if response_model is None:
response_model = self.output_schema
- messages = [{'role': 'system', 'content': self.system_prompt_generator.generate_prompt()}] + self.memory.get_history()
- response = self.client.chat.completions.create(
- model=self.model,
- messages=messages,
- response_model=response_model
- )
+ messages = [
+ {
+ "role": "system",
+ "content": self.system_prompt_generator.generate_prompt(),
+ }
+ ] + self.memory.get_history()
+ response = self.client.chat.completions.create(model=self.model, messages=messages, response_model=response_model)
return response
def run(self, user_input: Optional[Type[BaseAgentIO]] = None) -> Type[BaseAgentIO]:
@@ -155,7 +167,7 @@ class BaseChatAgent:
user_input (str): The input text from the user.
"""
self.current_user_input = user_input
- self.memory.add_message('user', str(user_input))
+ self.memory.add_message("user", str(user_input))
def _pre_run(self):
"""
@@ -170,7 +182,7 @@ class BaseChatAgent:
Args:
response (Type[BaseModel]): The response from the chat agent.
"""
- self.memory.add_message('assistant', str(response))
+ self.memory.add_message("assistant", str(response))
def get_context_provider(self, provider_name: str) -> Type[SystemPromptContextProviderBase]:
"""
@@ -209,4 +221,4 @@ class BaseChatAgent:
if provider_name in self.system_prompt_generator.system_prompt_info.context_providers:
del self.system_prompt_generator.system_prompt_info.context_providers[provider_name]
else:
- raise KeyError(f"Context provider '{provider_name}' not found.")
\ No newline at end of file
+ raise KeyError(f"Context provider '{provider_name}' not found.")
diff --git a/atomic_agents/agents/tool_interface_agent.py b/atomic_agents/agents/tool_interface_agent.py
index 4bbe86b..b8aefc2 100644
--- a/atomic_agents/agents/tool_interface_agent.py
+++ b/atomic_agents/agents/tool_interface_agent.py
@@ -1,14 +1,18 @@
import json
+
from pydantic import Field, create_model
+
from atomic_agents.agents.base_chat_agent import BaseAgentIO, BaseChatAgent, BaseChatAgentConfig
from atomic_agents.lib.components.system_prompt_generator import SystemPromptGenerator, SystemPromptInfo
from atomic_agents.lib.tools.base import BaseTool
from atomic_agents.lib.utils.format_tool_message import format_tool_message
+
class ToolInterfaceAgentConfig(BaseChatAgentConfig):
tool_instance: BaseTool
return_raw_output: bool = False
+
class ToolInputModel(BaseAgentIO):
tool_input: str = Field(..., description="Tool input. Presented as a single question or instruction")
@@ -17,9 +21,10 @@ class ToolInputModel(BaseAgentIO):
description = "Default tool description"
json_schema_extra = {
"title": "Default Tool",
- "description": "Default tool description"
+ "description": "Default tool description",
}
+
class ToolInterfaceAgent(BaseChatAgent):
"""
A specialized chat agent designed to interact with a specific tool.
@@ -40,47 +45,62 @@ class ToolInterfaceAgent(BaseChatAgent):
config (ToolInterfaceAgentConfig): Configuration for the tool interface agent.
"""
super().__init__(config=config)
-
+
self.tool_instance = config.tool_instance
self.return_raw_output = config.return_raw_output
-
+
# Create a new model with the updated schema
self.input_schema = create_model(
self.tool_instance.tool_name,
- tool_input=(str, Field(..., description=f"{self.tool_instance.tool_name} tool input. Presented as a single question or instruction", alias=f'tool_input_{self.tool_instance.tool_name}')),
- __base__=ToolInputModel
+ tool_input=(
+ str,
+ Field(
+ ...,
+ description=f"{self.tool_instance.tool_name} tool input. Presented as a single question or instruction",
+ alias=f"tool_input_{self.tool_instance.tool_name}",
+ ),
+ ),
+ __base__=ToolInputModel,
)
-
+
# Manually set the configuration attributes
- self.input_schema.model_config['title'] = self.tool_instance.tool_name
- self.input_schema.model_config['description'] = self.tool_instance.tool_description
- self.input_schema.model_config['json_schema_extra'] = {
- 'title': self.tool_instance.tool_name,
- 'description': self.tool_instance.tool_description
+ self.input_schema.model_config["title"] = self.tool_instance.tool_name
+ self.input_schema.model_config["description"] = self.tool_instance.tool_description
+ self.input_schema.model_config["json_schema_extra"] = {
+ "title": self.tool_instance.tool_name,
+ "description": self.tool_instance.tool_description,
}
-
+
if self.return_raw_output:
self.output_schema = self.tool_instance.output_schema
-
+
+ output_instructions = [
+ "Make sure the tool call will maximize the utility of the tool in the context of the user input.",
+ ]
+
+ if self.return_raw_output:
+ output_instructions.append(
+ "Process the output of the tool into a human readable format " "and/or use it to respond to the user input."
+ )
+ else:
+ output_instructions.append("Return the raw output of the tool.")
+
self.system_prompt_generator = SystemPromptGenerator(
system_prompt_info=SystemPromptInfo(
background=[
f"This AI agent is designed to interact with the {self.tool_instance.tool_name} tool.",
- f"Tool description: {self.tool_instance.tool_description}"
+ f"Tool description: {self.tool_instance.tool_description}",
],
steps=[
"Get the user input.",
"Convert the input to the proper parameters to call the tool.",
"Call the tool with the parameters.",
- "Respond to the user"
+ "Respond to the user",
],
- output_instructions=[
- "Make sure the tool call will maximize the utility of the tool in the context of the user input.",
- "Process the output of the tool into a human readable format and/or use it to respond to the user input." if self.return_raw_output else "Return the raw output of the tool."
- ]
+ output_instructions=output_instructions,
)
)
-
+
def _get_and_handle_response(self):
"""
Handles obtaining and processing the response from the tool.
@@ -88,20 +108,23 @@ class ToolInterfaceAgent(BaseChatAgent):
This method gets the response from the tool, formats the tool input, adds it to memory,
runs the tool, and processes the tool output. If `return_raw_output` is True, it returns
the raw tool output; otherwise, it processes the output and returns the response.
-
+
Returns:
BaseModel: The processed response or raw tool output.
"""
tool_input = self.get_response(response_model=self.tool_instance.input_schema)
formatted_tool_input = format_tool_message(tool_input)
-
- self.memory.add_message('assistant', 'TOOL CALL: ' + json.dumps(formatted_tool_input))
+
+ self.memory.add_message("assistant", "TOOL CALL: " + json.dumps(formatted_tool_input))
tool_output = self.tool_instance.run(tool_input)
- self.memory.add_message('assistant', 'TOOL RESPONSE: ' + tool_output.model_dump_json())
-
+ self.memory.add_message("assistant", "TOOL RESPONSE: " + tool_output.model_dump_json())
+
if self.return_raw_output:
return tool_output
-
- self.memory.add_message('assistant', 'I will now formulate a response for the user based on the tool output.')
- response = self.get_response(response_model=self.output_schema)
+
+ self.memory.add_message(
+ "assistant",
+ "I will now formulate a response for the user based on the tool output.",
+ )
+ response = self.get_response(response_model=self.output_schema)
return response
diff --git a/atomic_agents/lib/components/chat_memory.py b/atomic_agents/lib/components/chat_memory.py
index d8bb47c..91fcf7e 100644
--- a/atomic_agents/lib/components/chat_memory.py
+++ b/atomic_agents/lib/components/chat_memory.py
@@ -1,6 +1,8 @@
-from typing import List, Dict, Union, Optional
+from typing import Dict, List, Optional, Union
+
from pydantic import BaseModel, ValidationError
+
class Message(BaseModel):
"""
Represents a message in the chat history.
@@ -11,11 +13,13 @@ class Message(BaseModel):
tool_calls (Optional[List[Dict]]): Optional list of tool call messages.
tool_call_id (Optional[str]): Optional unique identifier for the tool call.
"""
+
role: str
content: Union[str, Dict]
tool_calls: Optional[List[Dict]] = None
tool_call_id: Optional[str] = None
+
class ChatMemory:
"""
Manages the chat history for an AI agent.
@@ -35,7 +39,13 @@ class ChatMemory:
self.history: List[Message] = []
self.max_messages = max_messages
- def add_message(self, role: str, content: Union[str, Dict], tool_message: Optional[Dict] = None, tool_id: Optional[str] = None) -> None:
+ def add_message(
+ self,
+ role: str,
+ content: Union[str, Dict],
+ tool_message: Optional[Dict] = None,
+ tool_id: Optional[str] = None,
+ ) -> None:
"""
Adds a message to the chat history and manages overflow.
@@ -49,12 +59,12 @@ class ChatMemory:
if tool_message:
if tool_id is None:
tool_id = tool_message["id"]
-
+
message.tool_calls = [tool_message]
message.tool_call_id = tool_id
elif role == "tool":
message.tool_call_id = tool_id
-
+
self.history.append(message)
self._manage_overflow()
@@ -105,8 +115,8 @@ class ChatMemory:
# Optionally, manage overflow after loading all messages
self._manage_overflow()
-
- def copy(self) -> 'ChatMemory':
+
+ def copy(self) -> "ChatMemory":
"""
Creates a copy of the chat memory.
@@ -115,7 +125,7 @@ class ChatMemory:
"""
new_memory = ChatMemory(max_messages=self.max_messages)
new_memory.load(self.dump())
-
+
return new_memory
def get_message_count(self) -> int:
@@ -125,4 +135,4 @@ class ChatMemory:
Returns:
int: The number of messages.
"""
- return len(self.history)
\ No newline at end of file
+ return len(self.history)
diff --git a/atomic_agents/lib/components/system_prompt_generator.py b/atomic_agents/lib/components/system_prompt_generator.py
index dce1167..0f69f63 100644
--- a/atomic_agents/lib/components/system_prompt_generator.py
+++ b/atomic_agents/lib/components/system_prompt_generator.py
@@ -1,6 +1,7 @@
-from typing import List, Dict, Optional
-from dataclasses import dataclass, field
from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
class SystemPromptContextProviderBase(ABC):
def __init__(self, title: str):
@@ -8,53 +9,55 @@ class SystemPromptContextProviderBase(ABC):
@abstractmethod
def get_info(self) -> str:
- return
-
+ pass
+
def __repr__(self) -> str:
return self.get_info()
+
@dataclass
class SystemPromptInfo:
background: List[str]
- steps: Optional[List[str]] = field(default_factory=list)
- output_instructions: Optional[List[str]] = field(default_factory=list)
- context_providers: Optional[Dict[str, SystemPromptContextProviderBase]] = field(default_factory=dict)
+ steps: List[str] = field(default_factory=list)
+ output_instructions: List[str] = field(default_factory=list)
+ context_providers: Dict[str, SystemPromptContextProviderBase] = field(default_factory=dict)
+
class SystemPromptGenerator:
- def __init__(self, system_prompt_info: SystemPromptInfo = None):
+ def __init__(self, system_prompt_info: Optional[SystemPromptInfo] = None):
self.system_prompt_info = system_prompt_info or SystemPromptInfo(
- background=['This is a conversation with a helpful and friendly AI assistant.'],
- steps=[],
- output_instructions=[],
- context_providers={}
+ background=["This is a conversation with a helpful and friendly AI assistant."]
)
-
- self.system_prompt_info.output_instructions.extend([
- 'Always respond using the proper JSON schema.',
- 'Always use the available additional information and context to enhance the response.'
- ])
- def generate_prompt(self) -> str:
- system_prompt = ''
+ self.system_prompt_info.output_instructions.extend(
+ [
+ "Always respond using the proper JSON schema.",
+ "Always use the available additional information and context to enhance the response.",
+ ]
+ )
- if self.system_prompt_info.background:
- system_prompt += '# IDENTITY and PURPOSE\n'
- system_prompt += '- ' + '\n- '.join(self.system_prompt_info.background) + '\n\n'
+ def generate_prompt(self) -> str:
+ sections = [
+ ("IDENTITY and PURPOSE", self.system_prompt_info.background),
+ ("INTERNAL ASSISTANT STEPS", self.system_prompt_info.steps),
+ ("OUTPUT INSTRUCTIONS", self.system_prompt_info.output_instructions),
+ ]
- if self.system_prompt_info.steps:
- system_prompt += '# INTERNAL ASSISTANT STEPS\n'
- system_prompt += '- ' + '\n- '.join(self.system_prompt_info.steps) + '\n\n'
+ prompt_parts = []
- if self.system_prompt_info.output_instructions:
- system_prompt += '# OUTPUT INSTRUCTIONS\n'
- system_prompt += '- ' + '\n- '.join(self.system_prompt_info.output_instructions) + '\n\n'
+ for title, content in sections:
+ if content:
+ prompt_parts.append(f"# {title}")
+ prompt_parts.extend(f"- {item}" for item in content)
+ prompt_parts.append("")
if self.system_prompt_info.context_providers:
- system_prompt += '# EXTRA INFORMATION AND CONTEXT\n'
+ prompt_parts.append("# EXTRA INFORMATION AND CONTEXT")
for provider in self.system_prompt_info.context_providers.values():
info = provider.get_info()
if info:
- system_prompt += f'## {provider.title}\n'
- system_prompt += f'{info}\n\n'
+ prompt_parts.append(f"## {provider.title}")
+ prompt_parts.append(info)
+ prompt_parts.append("")
- return system_prompt
\ No newline at end of file
+ return "\n".join(prompt_parts).strip()
diff --git a/atomic_agents/lib/models/web_document.py b/atomic_agents/lib/models/web_document.py
index 4fdd606..a4c6730 100644
--- a/atomic_agents/lib/models/web_document.py
+++ b/atomic_agents/lib/models/web_document.py
@@ -2,14 +2,15 @@ from pydantic import BaseModel, Field
from atomic_agents.agents.base_chat_agent import BaseAgentIO
+
class WebDocumentMetadata(BaseModel):
url: str
- title: str = Field(default='')
- description: str = Field(default='')
- keywords: str = Field(default='')
- author: str = Field(default='')
-
+ title: str = Field(default="")
+ description: str = Field(default="")
+ keywords: str = Field(default="")
+ author: str = Field(default="")
+
+
class WebDocument(BaseAgentIO):
content: str
metadata: WebDocumentMetadata = Field(default_factory=WebDocumentMetadata)
-
diff --git a/atomic_agents/lib/tools/base.py b/atomic_agents/lib/tools/base.py
index 734ff2e..019357c 100644
--- a/atomic_agents/lib/tools/base.py
+++ b/atomic_agents/lib/tools/base.py
@@ -1,13 +1,15 @@
-from typing import Type, Optional
+from typing import Optional, Type
+
from pydantic import BaseModel
-from dataclasses import dataclass
from atomic_agents.agents.base_chat_agent import BaseAgentIO
+
class BaseToolConfig(BaseModel):
title: Optional[str] = None
description: Optional[str] = None
+
class BaseTool:
"""
Base class for all tools in the Atomic Agents framework.
@@ -16,7 +18,8 @@ class BaseTool:
input_schema (Type[BaseAgentIO]): The schema for the input data.
output_schema (Type[BaseAgentIO]): The schema for the output data.
tool_name (str): The name of the tool, derived from the input schema's title.
- tool_description (str): The description of the tool, derived from the input schema's description or overridden by the user.
+ tool_description (str):
+ The description of the tool, derived from the input schema's description or overridden by the user.
"""
input_schema: Type[BaseAgentIO]
@@ -45,4 +48,4 @@ class BaseTool:
Raises:
NotImplementedError: If the method is not implemented by a subclass.
"""
- raise NotImplementedError("Subclasses should implement this method")
\ No newline at end of file
+ raise NotImplementedError("Subclasses should implement this method")
diff --git a/atomic_agents/lib/tools/calculator_tool.py b/atomic_agents/lib/tools/calculator_tool.py
index 8495078..9843b06 100644
--- a/atomic_agents/lib/tools/calculator_tool.py
+++ b/atomic_agents/lib/tools/calculator_tool.py
@@ -1,6 +1,6 @@
from pydantic import Field
-from sympy import sympify
from rich.console import Console
+from sympy import sympify
from atomic_agents.agents.base_chat_agent import BaseAgentIO
from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
@@ -8,20 +8,27 @@ from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
################
# INPUT SCHEMA #
################
+
+
class CalculatorToolSchema(BaseAgentIO):
expression: str = Field(..., description="Mathematical expression to evaluate. For example, '2 + 2'.")
class Config:
title = "CalculatorTool"
- description = "Tool for performing calculations. Supports basic arithmetic operations like addition, subtraction, multiplication, and division, but also more complex operations like exponentiation and trigonometric functions. Use this tool to evaluate mathematical expressions."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
+ description = (
+ "Tool for performing calculations. Supports basic arithmetic operations "
+ "like addition, subtraction, multiplication, and division, but also more "
+ "complex operations like exponentiation and trigonometric functions. "
+ "Use this tool to evaluate mathematical expressions."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
+
####################
# OUTPUT SCHEMA(S) #
####################
+
+
class CalculatorToolOutputSchema(BaseAgentIO):
result: str = Field(..., description="Result of the calculation.")
@@ -32,6 +39,7 @@ class CalculatorToolOutputSchema(BaseAgentIO):
class CalculatorToolConfig(BaseToolConfig):
pass
+
class CalculatorTool(BaseTool):
"""
Tool for performing calculations based on the provided mathematical expression.
@@ -40,13 +48,14 @@ class CalculatorTool(BaseTool):
input_schema (CalculatorToolSchema): The schema for the input data.
output_schema (CalculatorToolOutputSchema): The schema for the output data.
"""
+
input_schema = CalculatorToolSchema
output_schema = CalculatorToolOutputSchema
-
+
def __init__(self, config: CalculatorToolConfig = CalculatorToolConfig()):
"""
Initializes the CalculatorTool.
-
+
Args:
config (CalculatorToolConfig): Configuration for the tool.
"""
@@ -73,8 +82,8 @@ class CalculatorTool(BaseTool):
return CalculatorToolOutputSchema(result=str(result))
except Exception as e:
raise ValueError(f"Error evaluating expression: {e}")
-
-
+
+
#################
# EXAMPLE USAGE #
#################
diff --git a/atomic_agents/lib/tools/content_scraping_tool.py b/atomic_agents/lib/tools/content_scraping_tool.py
index 651b462..df87e8d 100644
--- a/atomic_agents/lib/tools/content_scraping_tool.py
+++ b/atomic_agents/lib/tools/content_scraping_tool.py
@@ -1,34 +1,40 @@
import os
-from pydantic import BaseModel, Field
from typing import Optional
-from rich.console import Console
-from rich.markdown import Markdown
-import openai
+
import instructor
+import openai
import requests
+from pydantic import Field
+from rich.console import Console
+from rich.markdown import Markdown
from atomic_agents.agents.base_chat_agent import BaseAgentIO
from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
-from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter
from atomic_agents.lib.utils.scraping.pdf_to_markdown import PdfToMarkdownConverter
+from atomic_agents.lib.utils.scraping.url_to_markdown import UrlToMarkdownConverter
################
# INPUT SCHEMA #
################
+
+
class ContentScrapingToolSchema(BaseAgentIO):
url: str = Field(..., description="URL of the web page or PDF to scrape.")
class Config:
title = "ContentScrapingTool"
- description = "Tool for scraping web pages or PDFs and converting content to markdown in order to extract information or summarize the content."
- json_schema_extra = {
- "title": title,
- "description": description
- }
+ description = (
+ "Tool for scraping web pages or PDFs and converting content to markdown "
+ "in order to extract information or summarize the content."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
####################
# OUTPUT SCHEMA(S) #
####################
+
+
class ContentScrapingToolOutputSchema(BaseAgentIO):
content: str
metadata: Optional[dict] = None
@@ -49,6 +55,7 @@ class ContentScrapingTool(BaseTool):
input_schema (ContentScrapingToolSchema): The schema for the input data.
output_schema (ContentScrapingToolOutputSchema): The schema for the output data.
"""
+
input_schema = ContentScrapingToolSchema
output_schema = ContentScrapingToolOutputSchema
@@ -73,9 +80,9 @@ class ContentScrapingTool(BaseTool):
"""
url = params.url
response = requests.head(url)
- content_type = response.headers.get('Content-Type', '')
+ content_type = response.headers.get("Content-Type", "")
- if 'application/pdf' in content_type:
+ if "application/pdf" in content_type:
document = PdfToMarkdownConverter.convert(url=url)
else:
document = UrlToMarkdownConverter.convert(url=url)
@@ -89,12 +96,7 @@ class ContentScrapingTool(BaseTool):
if __name__ == "__main__":
rich_console = Console()
- client = instructor.from_openai(
- openai.OpenAI(
- api_key=os.getenv("OPENAI_API_KEY"),
- base_url=os.getenv("OPENAI_BASE_URL")
- )
- )
+ client = instructor.from_openai(openai.OpenAI(api_key=os.getenv("OPENAI_API_KEY"), base_url=os.getenv("OPENAI_BASE_URL")))
#################
# TEST WEB PAGE #
@@ -102,7 +104,12 @@ if __name__ == "__main__":
result = client.chat.completions.create(
model="gpt-3.5-turbo",
response_model=ContentScrapingTool.input_schema,
- messages=[{"role": "user", "content": "Scrape the content of https://brainblendai.com"}],
+ messages=[
+ {
+ "role": "user",
+ "content": "Scrape the content of https://brainblendai.com",
+ }
+ ],
)
output = ContentScrapingTool().run(result)
@@ -114,8 +121,13 @@ if __name__ == "__main__":
result = client.chat.completions.create(
model="gpt-3.5-turbo",
response_model=ContentScrapingTool.input_schema,
- messages=[{"role": "user", "content": "Scrape the content of https://pdfobject.com/pdf/sample.pdf"}],
+ messages=[
+ {
+ "role": "user",
+ "content": "Scrape the content of https://pdfobject.com/pdf/sample.pdf",
+ }
+ ],
)
output = ContentScrapingTool().run(result)
- rich_console.print(Markdown(output.content))
\ No newline at end of file
+ rich_console.print(Markdown(output.content))
diff --git a/atomic_agents/lib/tools/search/searx_tool.py b/atomic_agents/lib/tools/search/searx_tool.py
index a04760a..657393c 100644
--- a/atomic_agents/lib/tools/search/searx_tool.py
+++ b/atomic_agents/lib/tools/search/searx_tool.py
@@ -1,7 +1,8 @@
import os
-import requests
-from pydantic import BaseModel, Field
from typing import List, Literal, Optional
+
+import requests
+from pydantic import Field
from rich.console import Console
from atomic_agents.agents.base_chat_agent import BaseAgentIO
@@ -10,38 +11,49 @@ from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
################
# INPUT SCHEMA #
################
+
+
class SearxNGSearchToolSchema(BaseAgentIO):
queries: List[str] = Field(..., description="List of search queries.")
- category: Optional[Literal["general", "news", "social_media"]] = Field("general", description="Category of the search queries.")
+ category: Optional[Literal["general", "news", "social_media"]] = Field(
+ "general", description="Category of the search queries."
+ )
class Config:
title = "SearxNGSearchTool"
- description = "Tool for searching for information, news, references, and other content on the SearxNG. Returns a list of search results with a short description or content snippet and URLs for further exploration."
- json_schema_extra = {
- "title": title,
- "description": description
- }
+ description = (
+ "Tool for searching for information, news, references, and other content on the SearxNG. "
+ "Returns a list of search results with a short description or content snippet and URLs for further exploration."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
####################
# OUTPUT SCHEMA(S) #
####################
+
+
class SearxNGSearchResultSchema(BaseAgentIO):
url: str
title: str
content: Optional[str] = None
-
+
class SearxNGSearchToolOutputSchema(BaseAgentIO):
results: List[SearxNGSearchResultSchema]
category: Optional[str] = None
+
##############
# TOOL LOGIC #
##############
+
+
class SearxNGSearchToolConfig(BaseToolConfig):
base_url: str = ""
max_results: int = 10
+
class SearxNGSearchTool(BaseTool):
"""
Tool for performing searches on SearxNG based on the provided queries and category.
@@ -52,6 +64,7 @@ class SearxNGSearchTool(BaseTool):
max_results (int): The maximum number of search results to return.
base_url (str): The base URL for the SearxNG instance to use.
"""
+
input_schema = SearxNGSearchToolSchema
output_schema = SearxNGSearchToolOutputSchema
@@ -60,7 +73,8 @@ class SearxNGSearchTool(BaseTool):
Initializes the SearxNGSearchTool.
Args:
- config (SearxNGSearchToolConfig): Configuration for the tool, including base URL, max results, and optional title and description overrides.
+ config (SearxNGSearchToolConfig):
+ Configuration for the tool, including base URL, max results, and optional title and description overrides.
"""
super().__init__(config)
self.base_url = config.base_url
@@ -86,63 +100,71 @@ class SearxNGSearchTool(BaseTool):
for query in params.queries:
# Prepare the query parameters
query_params = {
- 'q': query,
- 'safesearch': '0',
- 'format': 'json',
- 'language': 'en',
- 'engines': 'bing,duckduckgo,google,startpage,yandex',
+ "q": query,
+ "safesearch": "0",
+ "format": "json",
+ "language": "en",
+ "engines": "bing,duckduckgo,google,startpage,yandex",
}
# Add category to query parameters if it is set
if params.category:
- query_params['categories'] = params.category
+ query_params["categories"] = params.category
# Make the GET request
response = requests.get(f"{self.base_url}/search", params=query_params)
# Check if the request was successful
if response.status_code != 200:
- raise Exception(f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}")
+ raise Exception(
+ f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}"
+ )
- results = response.json().get('results', [])
+ results = response.json().get("results", [])
all_results.extend(results)
# Sort the combined results by score in descending order
- sorted_results = sorted(all_results, key=lambda x: x.get('score', 0), reverse=True)
+ sorted_results = sorted(all_results, key=lambda x: x.get("score", 0), reverse=True)
# Remove duplicates while preserving order
seen_urls = set()
unique_results = []
for result in sorted_results:
- if 'content' not in result or 'title' not in result or 'url' not in result:
+ if "content" not in result or "title" not in result or "url" not in result:
continue
- if result['url'] not in seen_urls:
+ if result["url"] not in seen_urls:
unique_results.append(result)
- if 'metadata' in result:
- result['title'] = result['title'] + ' - (Published ' + result['metadata'] + ')'
- if 'publishedDate' in result and result['publishedDate']:
- result['title'] = result['title'] + ' - (Published ' + result['publishedDate'] + ')'
- seen_urls.add(result['url'])
+ if "metadata" in result:
+ result["title"] = f"{result['title']} - (Published {result['metadata']})"
+ if "publishedDate" in result and result["publishedDate"]:
+ result["title"] = f"{result['title']} - (Published {result['publishedDate']})"
+ seen_urls.add(result["url"])
# Filter results to include only those with the correct category if it is set
if params.category:
- filtered_results = [result for result in unique_results if result.get('category') == params.category]
+ filtered_results = [result for result in unique_results if result.get("category") == params.category]
else:
filtered_results = unique_results
- filtered_results = filtered_results[:max_results or self.max_results]
+ filtered_results = filtered_results[: max_results or self.max_results]
return SearxNGSearchToolOutputSchema(results=filtered_results, category=params.category)
+
#################
# EXAMPLE USAGE #
#################
if __name__ == "__main__":
rich_console = Console()
- search_tool_instance = SearxNGSearchTool(config=SearxNGSearchToolConfig(base_url=os.getenv('SEARXNG_BASE_URL'), max_results=5))
-
- search_input = SearxNGSearchTool.input_schema(queries=["Python programming", "Machine learning", "Artificial intelligence"], category="news")
+ search_tool_instance = SearxNGSearchTool(
+ config=SearxNGSearchToolConfig(base_url=os.getenv("SEARXNG_BASE_URL"), max_results=5)
+ )
+
+ search_input = SearxNGSearchTool.input_schema(
+ queries=["Python programming", "Machine learning", "Artificial intelligence"],
+ category="news",
+ )
output = search_tool_instance.run(search_input)
- rich_console.print(output)
\ No newline at end of file
+ rich_console.print(output)
diff --git a/atomic_agents/lib/tools/search/serper_tool.py b/atomic_agents/lib/tools/search/serper_tool.py
index bba050a..eb3c97c 100644
--- a/atomic_agents/lib/tools/search/serper_tool.py
+++ b/atomic_agents/lib/tools/search/serper_tool.py
@@ -1,48 +1,57 @@
-import os
-import requests
import json
-from pydantic import BaseModel, Field
+import os
from typing import List, Optional
+
+import requests
+from pydantic import Field
from rich.console import Console
-import openai
-import instructor
-from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
from atomic_agents.agents.base_chat_agent import BaseAgentIO
+from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
################
# INPUT SCHEMA #
################
+
+
class SerperSearchToolSchema(BaseAgentIO):
queries: List[str] = Field(..., description="List of search queries.")
class Config:
title = "SerperSearchTool"
- description = "Tool for searching for information, news, references, and other content using the Serper API. Returns a list of search results with a short description or content snippet and URLs for further exploration."
- json_schema_extra = {
- "title": title,
- "description": description
- }
+ description = (
+ "Tool for searching for information, news, references, and other content using the Serper API. "
+ "Returns a list of search results with a short description or content snippet and URLs for further exploration."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
####################
# OUTPUT SCHEMA(S) #
####################
+
+
class SerperSearchResultSchema(BaseAgentIO):
url: str
title: str
content: Optional[str] = None
position: Optional[int] = None
+
class SerperSearchToolOutputSchema(BaseAgentIO):
results: List[SerperSearchResultSchema]
+
##############
# TOOL LOGIC #
##############
+
+
class SerperSearchToolConfig(BaseToolConfig):
api_key: str = ""
max_results: int = 10
+
class SerperSearchTool(BaseTool):
"""
Tool for performing searches using the Serper API based on the provided queries.
@@ -53,6 +62,7 @@ class SerperSearchTool(BaseTool):
api_key (str): The API key for the Serper API.
max_results (int): The maximum number of search results to return.
"""
+
input_schema = SerperSearchToolSchema
output_schema = SerperSearchToolOutputSchema
@@ -61,7 +71,8 @@ class SerperSearchTool(BaseTool):
Initializes the SerperSearchTool.
Args:
- config (SerperSearchToolConfig): Configuration for the tool, including API key, max results, and optional title and description overrides.
+ config (SerperSearchToolConfig):
+ Configuration for the tool, including API key, max results, and optional title and description overrides.
"""
super().__init__(config)
self.api_key = config.api_key
@@ -89,56 +100,56 @@ class SerperSearchTool(BaseTool):
payload = json.dumps({"q": query})
# Make the POST request
- headers = {
- 'X-API-KEY': self.api_key,
- 'Content-Type': 'application/json'
- }
+ headers = {"X-API-KEY": self.api_key, "Content-Type": "application/json"}
response = requests.post("https://google.serper.dev/search", headers=headers, data=payload)
# Check if the request was successful
if response.status_code != 200:
- raise Exception(f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}")
+ raise Exception(
+ f"Failed to fetch search results for query '{query}': {response.status_code} {response.reason}"
+ )
- results = response.json().get('organic', [])
+ results = response.json().get("organic", [])
all_results.extend(results)
# Sort the combined results by position in ascending order
- sorted_results = sorted(all_results, key=lambda x: x.get('position', float('inf')))
+ sorted_results = sorted(all_results, key=lambda x: x.get("position", float("inf")))
# Remove duplicates while preserving order
seen_urls = set()
unique_results = []
for result in sorted_results:
- if 'snippet' not in result or 'title' not in result or 'link' not in result:
+ if "snippet" not in result or "title" not in result or "link" not in result:
continue
- if result['link'] not in seen_urls:
+ if result["link"] not in seen_urls:
unique_results.append(result)
- seen_urls.add(result['link'])
+ seen_urls.add(result["link"])
- filtered_results = unique_results[:max_results or self.max_results]
+ filtered_results = unique_results[: max_results or self.max_results]
# Convert to output schema format
output_results = [
SerperSearchResultSchema(
- url=result['link'],
- title=result['title'],
- content=result.get('snippet'),
- position=result.get('position')
+ url=result["link"],
+ title=result["title"],
+ content=result.get("snippet"),
+ position=result.get("position"),
)
for result in filtered_results
]
return SerperSearchToolOutputSchema(results=output_results)
+
#################
# EXAMPLE USAGE #
#################
if __name__ == "__main__":
rich_console = Console()
search_tool_instance = SerperSearchTool(config=SerperSearchToolConfig(api_key=os.getenv("SERPER_API_KEY"), max_results=5))
-
+
search_input = SerperSearchTool.input_schema(queries=["Python programming", "Machine learning", "Quantum computing"])
output = search_tool_instance.run(search_input)
-
- rich_console.print(output)
\ No newline at end of file
+
+ rich_console.print(output)
diff --git a/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py b/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py
index faf5d4b..d3a7a22 100644
--- a/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py
+++ b/atomic_agents/lib/tools/yelp_restaurant_finder_tool.py
@@ -1,10 +1,10 @@
import os
-import requests
-from pydantic import BaseModel, Field
+from enum import Enum
from typing import List, Optional
+
+import requests
+from pydantic import Field
from rich.console import Console
-from rich.table import Table
-from enum import Enum
from atomic_agents.agents.base_chat_agent import BaseAgentIO
from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
@@ -12,6 +12,8 @@ from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
################
# INPUT SCHEMA #
################
+
+
class YelpCategory(Enum):
ITALIAN = "italian"
MEXICAN = "mexican"
@@ -33,32 +35,46 @@ class YelpCategory(Enum):
VEGAN = "vegan"
VEGETARIAN = "vegetarian"
+
class PriceRange(Enum):
ONE = "1"
TWO = "2"
THREE = "3"
FOUR = "4"
+
class YelpSearchToolSchema(BaseAgentIO):
location: str = Field(..., description="Location to search for food.")
term: Optional[str] = Field(None, description="Search term (e.g., 'pizza', 'sushi').")
categories: Optional[List[YelpCategory]] = Field(None, description="Categories to filter by (e.g., 'italian, mexican').")
- price: Optional[List[PriceRange]] = Field(None, description="Price range to filter by (e.g., '1', '2', '3', '4'). Can be multiple. 1 is cheap. 2 and 3 are mid-range. 4 is the most high-end.")
+ price: Optional[List[PriceRange]] = Field(
+ None,
+ description=(
+ "Price range to filter by (e.g., '1', '2', '3', '4'). "
+ "Can be multiple. 1 is cheap. 2 and 3 are mid-range. 4 is the most high-end."
+ ),
+ )
open_now: Optional[bool] = Field(False, description="Filter for businesses that are open now.")
- sort_by: Optional[str] = Field("best_match", description="Sort by criteria (e.g., 'best_match', 'rating', 'review_count', 'distance').")
+ sort_by: Optional[str] = Field(
+ "best_match",
+ description="Sort by criteria (e.g., 'best_match', 'rating', 'review_count', 'distance').",
+ )
limit: Optional[int] = Field(10, description="Number of results to return.")
class Config:
title = "YelpSearchTool"
- description = "Tool for searching for food using the Yelp API. Returns a list of businesses with details such as name, rating, and address."
- json_schema_extra = {
- "title": title,
- "description": description
- }
+ description = (
+ "Tool for searching for food using the Yelp API. "
+ "Returns a list of businesses with details such as name, rating, and address."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
####################
# OUTPUT SCHEMA(S) #
####################
+
+
class YelpSearchResultSchema(BaseAgentIO):
name: str
url: str
@@ -68,24 +84,29 @@ class YelpSearchResultSchema(BaseAgentIO):
phone: Optional[str] = None
categories: List[str]
+
class YelpSearchToolOutputSchema(BaseAgentIO):
results: List[YelpSearchResultSchema]
-
+
class Config:
title = "YelpSearchToolOutput"
- description = "Output schema for the YelpSearchTool, containing a list of search results with details such as name, rating, and address."
- json_schema_extra = {
- "title": title,
- "description": description
- }
+ description = (
+ "Output schema for the YelpSearchTool, containing a list of search results with details"
+ "such as name, rating, and address."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
##############
# TOOL LOGIC #
##############
+
+
class YelpSearchToolConfig(BaseToolConfig):
api_key: str = ""
max_results: int = 10
+
class YelpSearchTool(BaseTool):
"""
Tool for performing searches using the Yelp API based on the provided queries.
@@ -96,6 +117,7 @@ class YelpSearchTool(BaseTool):
api_key (str): The API key for the Yelp API.
max_results (int): The maximum number of search results to return.
"""
+
input_schema = YelpSearchToolSchema
output_schema = YelpSearchToolOutputSchema
@@ -104,7 +126,8 @@ class YelpSearchTool(BaseTool):
Initializes the YelpSearchTool.
Args:
- config (YelpSearchToolConfig): Configuration for the tool, including API key, max results, and optional title and description overrides.
+ config (YelpSearchToolConfig):
+ Configuration for the tool, including API key, max results, and optional title and description overrides.
"""
super().__init__(config)
self.api_key = config.api_key
@@ -132,56 +155,59 @@ class YelpSearchTool(BaseTool):
query_params = {
"location": params.location,
"term": params.term,
- "categories": ",".join([category.value for category in params.categories]) if params.categories else None,
- "price": ",".join([price.value for price in params.price]) if params.price else None,
+ "categories": (",".join([category.value for category in params.categories]) if params.categories else None),
+ "price": (",".join([price.value for price in params.price]) if params.price else None),
"sort_by": params.sort_by,
"limit": self.max_results,
- "open_now": params.open_now
+ "open_now": params.open_now,
}
# Make the GET request
- headers = {
- 'Authorization': f'Bearer {self.api_key}'
- }
- response = requests.get("https://api.yelp.com/v3/businesses/search", headers=headers, params=query_params)
+ headers = {"Authorization": f"Bearer {self.api_key}"}
+ response = requests.get(
+ "https://api.yelp.com/v3/businesses/search",
+ headers=headers,
+ params=query_params,
+ )
# Check if the request was successful
if response.status_code != 200:
raise Exception(f"Failed to fetch search results: {response.status_code} {response.reason}")
- results = response.json().get('businesses', [])
+ results = response.json().get("businesses", [])
# Convert to output schema format
output_results = [
YelpSearchResultSchema(
- name=result['name'],
- url=result['url'].split('?')[0],
- rating=result['rating'],
- review_count=result['review_count'],
- address=", ".join(result['location']['display_address']),
- phone=result.get('display_phone'),
- categories=[category['title'] for category in result['categories']]
+ name=result["name"],
+ url=result["url"].split("?")[0],
+ rating=result["rating"],
+ review_count=result["review_count"],
+ address=", ".join(result["location"]["display_address"]),
+ phone=result.get("display_phone"),
+ categories=[category["title"] for category in result["categories"]],
)
for result in results
]
return YelpSearchToolOutputSchema(results=output_results)
+
#################
# EXAMPLE USAGE #
#################
if __name__ == "__main__":
rich_console = Console()
- search_tool_instance = YelpSearchTool(config=YelpSearchToolConfig(api_key=os.getenv('YELP_API_KEY'), max_results=10))
-
+ search_tool_instance = YelpSearchTool(config=YelpSearchToolConfig(api_key=os.getenv("YELP_API_KEY"), max_results=10))
+
search_input = YelpSearchTool.input_schema(
location="Diepenbeek, Belgium",
term="",
categories=[YelpCategory.PIZZA],
price=[PriceRange.ONE, PriceRange.TWO, PriceRange.THREE, PriceRange.FOUR],
- sort_by="best_match"
+ sort_by="best_match",
)
output = search_tool_instance.run(search_input)
-
- rich_console.print(output)
\ No newline at end of file
+
+ rich_console.print(output)
diff --git a/atomic_agents/lib/tools/yt_transcript_scraper.py b/atomic_agents/lib/tools/yt_transcript_scraper.py
index 46d8c06..efe9bfd 100644
--- a/atomic_agents/lib/tools/yt_transcript_scraper.py
+++ b/atomic_agents/lib/tools/yt_transcript_scraper.py
@@ -1,50 +1,60 @@
import os
-from pydantic import BaseModel, Field
from typing import List, Optional
+
+from googleapiclient.discovery import build
+from pydantic import Field
from rich.console import Console
-from youtube_transcript_api import YouTubeTranscriptApi, NoTranscriptFound, TranscriptsDisabled
+from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled, YouTubeTranscriptApi
+
from atomic_agents.agents.base_chat_agent import BaseAgentIO
from atomic_agents.lib.tools.base import BaseTool, BaseToolConfig
-from googleapiclient.discovery import build
################
# INPUT SCHEMA #
################
+
+
class YouTubeTranscriptToolSchema(BaseAgentIO):
video_url: str = Field(..., description="URL of the YouTube video to fetch the transcript for.")
language: Optional[str] = Field(None, description="Language code for the transcript (e.g., 'en' for English).")
class Config:
title = "YouTubeTranscriptTool"
- description = "Tool for fetching the transcript of a YouTube video using the YouTube Transcript API. Returns the transcript with text, start time, and duration."
- json_schema_extra = {
- "title": title,
- "description": description
- }
+ description = (
+ "Tool for fetching the transcript of a YouTube video using the YouTube Transcript API. "
+ "Returns the transcript with text, start time, and duration."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
####################
# OUTPUT SCHEMA(S) #
####################
+
+
class YouTubeTranscriptToolOutputSchema(BaseAgentIO):
transcript: str
duration: float
comments: List[str]
metadata: dict
-
+
class Config:
title = "YouTubeTranscriptToolOutput"
- description = "Output schema for the YouTubeTranscriptTool. Contains the transcript text, duration, comments, and metadata."
- json_schema_extra = {
- "title": title,
- "description": description
- }
-
+ description = (
+ "Output schema for the YouTubeTranscriptTool. " "Contains the transcript text, duration, comments, and metadata."
+ )
+ json_schema_extra = {"title": title, "description": description}
+
##############
# TOOL LOGIC #
##############
class YouTubeTranscriptToolConfig(BaseToolConfig):
- api_key: str = Field(description="YouTube API key for fetching video metadata.", default=os.getenv('YOUTUBE_API_KEY'))
+ api_key: str = Field(
+ description="YouTube API key for fetching video metadata.",
+ default=os.getenv("YOUTUBE_API_KEY"),
+ )
+
class YouTubeTranscriptTool(BaseTool):
"""
@@ -54,6 +64,7 @@ class YouTubeTranscriptTool(BaseTool):
input_schema (YouTubeTranscriptToolSchema): The schema for the input data.
output_schema (YouTubeTranscriptToolOutputSchema): The schema for the output data.
"""
+
input_schema = YouTubeTranscriptToolSchema
output_schema = YouTubeTranscriptToolOutputSchema
@@ -89,8 +100,8 @@ class YouTubeTranscriptTool(BaseTool):
except (NoTranscriptFound, TranscriptsDisabled) as e:
raise Exception(f"Failed to fetch transcript for video '{video_id}': {str(e)}")
- transcript_text = " ".join([transcript['text'] for transcript in transcripts])
- total_duration = sum([transcript['duration'] for transcript in transcripts])
+ transcript_text = " ".join([transcript["text"] for transcript in transcripts])
+ total_duration = sum([transcript["duration"] for transcript in transcripts])
metadata = self.fetch_video_metadata(video_id)
@@ -98,7 +109,7 @@ class YouTubeTranscriptTool(BaseTool):
transcript=transcript_text,
duration=total_duration,
comments=[],
- metadata=metadata
+ metadata=metadata,
)
@staticmethod
@@ -112,7 +123,7 @@ class YouTubeTranscriptTool(BaseTool):
Returns:
str: The extracted video ID.
"""
- return url.split('v=')[-1].split('&')[0]
+ return url.split("v=")[-1].split("&")[0]
def fetch_video_metadata(self, video_id: str) -> dict:
"""
@@ -124,31 +135,32 @@ class YouTubeTranscriptTool(BaseTool):
Returns:
dict: The metadata of the video.
"""
- youtube = build('youtube', 'v3', developerKey=self.api_key)
+ youtube = build("youtube", "v3", developerKey=self.api_key)
request = youtube.videos().list(part="snippet", id=video_id)
response = request.execute()
- if not response['items']:
+ if not response["items"]:
raise Exception(f"No metadata found for video '{video_id}'")
- video_info = response['items'][0]['snippet']
+ video_info = response["items"][0]["snippet"]
metadata = {
"id": video_id,
- "title": video_info['title'],
- "channel": video_info['channelTitle'],
- "published_at": video_info['publishedAt']
+ "title": video_info["title"],
+ "channel": video_info["channelTitle"],
+ "published_at": video_info["publishedAt"],
}
return metadata
+
#################
# EXAMPLE USAGE #
#################
if __name__ == "__main__":
rich_console = Console()
- api_key = os.getenv('YOUTUBE_API_KEY')
+ api_key = os.getenv("YOUTUBE_API_KEY")
search_tool_instance = YouTubeTranscriptTool(config=YouTubeTranscriptToolConfig(api_key=api_key))
-
+
search_input = YouTubeTranscriptTool.input_schema(video_url="https://www.youtube.com/watch?v=t1e8gqXLbsU", language="en")
output = search_tool_instance.run(search_input)
- rich_console.print(output)
\ No newline at end of file
+ rich_console.print(output)
diff --git a/atomic_agents/lib/utils/format_tool_message.py b/atomic_agents/lib/utils/format_tool_message.py
index 3e20656..d78196a 100644
--- a/atomic_agents/lib/utils/format_tool_message.py
+++ b/atomic_agents/lib/utils/format_tool_message.py
@@ -1,11 +1,10 @@
-import json
-from typing import Dict, Type
import uuid
+from typing import Dict, Optional, Type
from pydantic import BaseModel
-def format_tool_message(tool_call: Type[BaseModel], tool_id: str = None) -> Dict:
+def format_tool_message(tool_call: Type[BaseModel], tool_id: Optional[str] = None) -> Dict:
"""
Formats a message for a tool call.
@@ -19,13 +18,12 @@ def format_tool_message(tool_call: Type[BaseModel], tool_id: str = None) -> Dict
"""
if tool_id is None:
tool_id = str(uuid.uuid4())
-
+
return {
"id": tool_id,
"type": "function",
"function": {
"name": tool_call.model_config["title"],
- "arguments": tool_call.model_dump_json()
- }
+ "arguments": tool_call.model_dump_json(),
+ },
}
-
\ No newline at end of file
diff --git a/atomic_agents/lib/utils/scraping/pdf_to_markdown.py b/atomic_agents/lib/utils/scraping/pdf_to_markdown.py
index b48709b..109f57e 100644
--- a/atomic_agents/lib/utils/scraping/pdf_to_markdown.py
+++ b/atomic_agents/lib/utils/scraping/pdf_to_markdown.py
@@ -1,11 +1,13 @@
import re
-import requests
-from PyPDF2 import PdfReader
from io import BytesIO
+
import markdownify
+import requests
+from PyPDF2 import PdfReader
from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
+
class PdfToMarkdownConverter:
@staticmethod
def _fetch_pdf_content(url):
@@ -26,7 +28,7 @@ class PdfToMarkdownConverter:
def _convert_to_markdown(text):
markdown_text = markdownify.markdownify(text, heading_style="ATX")
# Replace more than two consecutive newlines with exactly two newlines
- markdown_text = re.sub(r'\n{3,}', '\n\n', markdown_text)
+ markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)
return markdown_text
@staticmethod
@@ -34,7 +36,7 @@ class PdfToMarkdownConverter:
if url:
pdf_content = PdfToMarkdownConverter._fetch_pdf_content(url)
elif file_path:
- with open(file_path, 'rb') as f:
+ with open(file_path, "rb") as f:
pdf_content = BytesIO(f.read())
else:
raise ValueError("Either url or file_path must be provided")
@@ -45,12 +47,14 @@ class PdfToMarkdownConverter:
return WebDocument(content=markdown_content, metadata=metadata)
-if __name__ == '__main__':
+
+if __name__ == "__main__":
from rich.console import Console
from rich.markdown import Markdown
+
console = Console()
-
+
url = "https://pdfobject.com/pdf/sample.pdf"
document = PdfToMarkdownConverter.convert(url=url)
-
- console.print(Markdown(document.content))
\ No newline at end of file
+
+ console.print(Markdown(document.content))
diff --git a/atomic_agents/lib/utils/scraping/url_to_markdown.py b/atomic_agents/lib/utils/scraping/url_to_markdown.py
index 7b4467a..020842a 100644
--- a/atomic_agents/lib/utils/scraping/url_to_markdown.py
+++ b/atomic_agents/lib/utils/scraping/url_to_markdown.py
@@ -1,18 +1,19 @@
-import os
import re
-from bs4 import BeautifulSoup
+
import requests
+from bs4 import BeautifulSoup
from markdownify import MarkdownConverter
+
from atomic_agents.lib.models.web_document import WebDocument, WebDocumentMetadata
class UrlToMarkdownConverter:
METADATA_ATTRS = {
- 'url': '',
- 'title': ('title', None),
- 'description': ('meta', {'name': 'description'}),
- 'keywords': ('meta', {'name': 'keywords'}),
- 'author': ('meta', {'name': 'author'})
+ "url": "",
+ "title": ("title", None),
+ "description": ("meta", {"name": "description"}),
+ "keywords": ("meta", {"name": "keywords"}),
+ "author": ("meta", {"name": "author"}),
}
@staticmethod
@@ -23,7 +24,7 @@ class UrlToMarkdownConverter:
@staticmethod
def _parse_html(html):
- return BeautifulSoup(html, 'html.parser')
+ return BeautifulSoup(html, "html.parser")
@staticmethod
def _extract_metadata(soup, url):
@@ -31,7 +32,7 @@ class UrlToMarkdownConverter:
for attr, selector in UrlToMarkdownConverter.METADATA_ATTRS.items():
if selector:
element = soup.find(*selector)
- metadata_dict[attr] = element.get('content', '').strip() if element else ''
+ metadata_dict[attr] = element.get("content", "").strip() if element else ""
else:
metadata_dict[attr] = url
return WebDocumentMetadata(**metadata_dict)
@@ -40,20 +41,21 @@ class UrlToMarkdownConverter:
def _convert_to_markdown(html):
class CustomMarkdownConverter(MarkdownConverter):
def convert_img(self, el, text, convert_as_inline):
- return ''
+ return ""
+
def convert_p(self, el, text, convert_as_inline):
- return text + '\n\n'
+ return text + "\n\n"
- for script in html.find_all('script'):
+ for script in html.find_all("script"):
script.decompose()
-
- for style in html.find_all('style'):
+
+ for style in html.find_all("style"):
style.decompose()
converter = CustomMarkdownConverter()
markdown_text = converter.convert(str(html))
# Replace more than two consecutive newlines with exactly two newlines
- markdown_text = re.sub(r'\n{3,}', '\n\n', markdown_text)
+ markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)
return markdown_text
@staticmethod
@@ -65,7 +67,8 @@ class UrlToMarkdownConverter:
return WebDocument(content=markdown_content, metadata=metadata)
-if __name__ == '__main__':
+
+if __name__ == "__main__":
url = "https://brainblendai.com"
document = UrlToMarkdownConverter.convert(url)
print(document)
diff --git a/pyproject.toml b/pyproject.toml
index 3f2b354..77ac867 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,3 +28,6 @@ repository = "https://github.com/KennyVaneetvelde/atomic_agents"
[tool.setuptools.packages.find]
where = ["."]
include = ["atomic_agents"]
+
+[tool.black]
+line-length = 127
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 24dce6c..a73ec8b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,6 +5,7 @@ anyio==4.4.0
asgiref==3.8.1
attrs==23.2.0
beautifulsoup4==4.12.3
+black==24.4.2
cachetools==5.3.3
certifi==2024.6.2
charset-normalizer==3.3.2