Agent Zero AI framework
feat: chat model VISION support for user and tool msg attachments
Rafael Uzarowski22eef83cc662
diff --git a/agent.py b/agent.pyindex c5775d131..415b4a20e 100644--- a/agent.py+++ b/agent.py@@ -10,8 +10,9 @@ import modelsfrom langchain_core.prompt_values import ChatPromptValuefrom python.helpers import extract_tools, rate_limiter, files, errors, history, tokensfrom python.helpers.print_style import PrintStyle-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, StringPromptTemplate+from langchain_core.prompts.image import ImagePromptTemplate+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessagefrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.language_models.llms import BaseLLMfrom langchain_core.embeddings import Embeddings@@ -19,6 +20,7 @@ import python.helpers.log as Logfrom python.helpers.dirty_json import DirtyJsonfrom python.helpers.defer import DeferredTaskfrom typing import Callable+from python.helpers.history import OutputMessageclass AgentContext:@@ -187,7 +189,7 @@ class AgentConfig:@dataclassclass UserMessage:message: str- attachments: list[str]+ attachments: list[str] = field(default_factory=list[str])class LoopData:@@ -357,10 +359,14 @@ class Agent:loop_data.extras_temporary.clear()# combine history and extras- history_combined = history.group_outputs_abab(loop_data.history_output + extras)+ history_combined: list[OutputMessage] = history.group_outputs_abab(loop_data.history_output + extras)# convert history to LLM format- history_langchain = history.output_langchain(history_combined)+ history_langchain: list[BaseMessage] = history.output_langchain(history_combined)++ PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(+ f"History Langchain: {history_langchain}"+ )# build chain from system prompt, message history and modelprompt = ChatPromptTemplate.from_messages(@@ -479,9 +485,10 @@ class Agent:content = self.parse_prompt("fw.warning.md", message=message)return self.hist_add_message(False, content=content)- async def hist_add_tool_result(self, tool_name: str, tool_result: str):+ async def hist_add_tool_result(self, tool_name: str, tool_result: str, attachments: list[str] = []):+ attachments_str = json.dumps(attachments).replace("\n", "")content = self.parse_prompt(- "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result+ "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result, attachments=attachments_str)return self.hist_add_message(False, content=content)diff --git a/initialize.py b/initialize.pyindex ead07d2f2..0003b0223 100644--- a/initialize.py+++ b/initialize.py@@ -13,6 +13,7 @@ def initialize():provider=models.ModelProvider[current_settings["chat_model_provider"]],name=current_settings["chat_model_name"],ctx_length=current_settings["chat_model_ctx_length"],+ vision=current_settings["chat_model_vision"],limit_requests=current_settings["chat_model_rl_requests"],limit_input=current_settings["chat_model_rl_input"],limit_output=current_settings["chat_model_rl_output"],diff --git a/prompts/default/agent.system.tools_vision.md b/prompts/default/agent.system.tools_vision.mdnew file mode 100644index 000000000..0af7e7527--- /dev/null+++ b/prompts/default/agent.system.tools_vision.md@@ -0,0 +1,3 @@+## "Multimodal (Vision) Agent Tools" available:++None yet. In future, this section will contain vision-only toolsdiff --git a/prompts/default/fw.tool_result.md b/prompts/default/fw.tool_result.mdindex 1943cc022..b45d456e8 100644--- a/prompts/default/fw.tool_result.md+++ b/prompts/default/fw.tool_result.md@@ -1,6 +1,7 @@~~~json{"tool_name": {{tool_name}},- "tool_result": {{tool_result}}+ "tool_result": {{tool_result}},+ "attachments": {{attachments}}}-~~~\ No newline at end of file+~~~diff --git a/python/extensions/system_prompt/_10_system_prompt.py b/python/extensions/system_prompt/_10_system_prompt.pyindex e7544b2ec..68f7d6dd8 100644--- a/python/extensions/system_prompt/_10_system_prompt.py+++ b/python/extensions/system_prompt/_10_system_prompt.py@@ -12,11 +12,17 @@ class SystemPrompt(Extension):system_prompt.append(main)system_prompt.append(tools)+def get_main_prompt(agent: Agent):return get_prompt("agent.system.main.md", agent)+def get_tools_prompt(agent: Agent):- return get_prompt("agent.system.tools.md", agent)+ prompt = get_prompt("agent.system.tools.md", agent)+ if agent.config.chat_model.vision:+ prompt += '\n' + get_prompt("agent.system.tools_vision.md", agent)+ return prompt+def get_prompt(file: str, agent: Agent):# variables for system prompts@@ -26,4 +32,4 @@ def get_prompt(file: str, agent: Agent):"date_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),"agent_name": agent.agent_name,}- return agent.read_prompt(file, **vars)\ No newline at end of file+ return agent.read_prompt(file, **vars)diff --git a/python/helpers/history.py b/python/helpers/history.pyindex 3f9e7c506..3fcb02426 100644--- a/python/helpers/history.py+++ b/python/helpers/history.py@@ -3,11 +3,14 @@ import asynciofrom collections import OrderedDictimport jsonimport math+import osfrom typing import Coroutine, Literal, TypedDict, castfrom python.helpers import messages, tokens, settings, call_llmfrom enum import Enum-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage-+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage+from python.helpers.print_style import PrintStyle+from langchain_core.prompts import HumanMessagePromptTemplate+from typing import AnyBULK_MERGE_COUNT = 3TOPICS_KEEP_COUNT = 3CURRENT_TOPIC_RATIO = 0.5@@ -425,13 +428,60 @@ def group_outputs_abab(outputs: list[OutputMessage]) -> list[OutputMessage]:return result-def output_langchain(messages: list[OutputMessage]):+def output_langchain(messages: list[OutputMessage]) -> list[BaseMessage]:result = []for m in messages:if m["ai"]:result.append(AIMessage(content=serialize_content(m["content"])))else:- result.append(HumanMessage(content=serialize_content(m["content"])))+ contents = m["content"]++ # sometimes content is a list sometimes not+ if not isinstance(contents, list):+ contents = [contents]++ PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(+ f"Contents: {json.dumps(contents, indent=2)}"+ )++ template: list[dict[str, str]] = [] # type: ignore+ message = ""+ images = {}+ for _, content in enumerate(contents):+ if message:+ # the first message is the user message, then the memory and solutions+ message += "\n\n--- Memory & Solutions Section: ---\n\n"++ message += serialize_content(content)++ if isinstance(content, dict) and "attachments" in content:+ attachments: list[str] = cast(list[str], content["attachments"])+ for attachment in attachments:+ if not os.path.exists(str(attachment)):+ continue+ if attachment not in images:+ import base64+ from mimetypes import guess_type+ mime_type, _ = guess_type(str(attachment))+ if mime_type.startswith("image/"):+ # Read and encode the image file+ with open(str(attachment), "rb") as image_file:+ base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')+ # Construct the data URL+ images[attachment] = f"data:{mime_type};base64,{base64_encoded_data}"++ if message:+ template.append({"type": "text", "text": message})+ if images:+ for _, image in images.items():+ template.append({"type": "image_url", "image_url": image})+ if template:+ # only jinja2 is safe for json, both mustache({{...}}) and f-string({...}) are not+ result.append(HumanMessagePromptTemplate.from_template(template=template, partial_variables={}, template_format="jinja2")) # type: ignore++ PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(+ f"Result: {result}"+ )return resultdiff --git a/python/helpers/settings.py b/python/helpers/settings.pyindex 228330753..fc7a41ce2 100644--- a/python/helpers/settings.py+++ b/python/helpers/settings.py@@ -16,6 +16,7 @@ class Settings(TypedDict):chat_model_kwargs: dict[str, str]chat_model_ctx_length: intchat_model_ctx_history: float+ chat_model_vision: boolchat_model_rl_requests: intchat_model_rl_input: intchat_model_rl_output: int@@ -149,6 +150,17 @@ def convert_out(settings: Settings) -> SettingsOutput:})++ chat_model_fields.append(+ {+ "id": "chat_model_vision",+ "title": "Supports Vision",+ "description": "Models capable of Vision can for example natively see the content of image attachments.",+ "type": "switch",+ "value": settings["chat_model_vision"],+ }+ )+chat_model_fields.append({"id": "chat_model_rl_requests",@@ -777,6 +789,7 @@ def get_default_settings() -> Settings:chat_model_kwargs={ "temperature": "0" },chat_model_ctx_length=120000,chat_model_ctx_history=0.7,+ chat_model_vision=False,chat_model_rl_requests=0,chat_model_rl_input=0,chat_model_rl_output=0,diff --git a/python/helpers/tool.py b/python/helpers/tool.pyindex a50ec8295..fcfb51070 100644--- a/python/helpers/tool.py+++ b/python/helpers/tool.py@@ -1,14 +1,16 @@from abc import abstractmethod-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom agent import Agentfrom python.helpers.print_style import PrintStyle-from python.helpers import messages+@dataclassclass Response:message:str- break_loop:bool-+ break_loop: bool+ attachments: list[str] = field(default_factory=list[str])++class Tool:def __init__(self, agent: Agent, name: str, args: dict[str,str], message: str, **kwargs) -> None:@@ -29,10 +31,10 @@ class Tool:PrintStyle(font_color="#85C1E9", bold=True).stream(self.nice_key(key)+": ")PrintStyle(font_color="#85C1E9", padding=isinstance(value,str) and "\n" in value).stream(value)PrintStyle().print()-+async def after_execution(self, response: Response, **kwargs):text = response.message.strip()- await self.agent.hist_add_tool_result(self.name, text)+ await self.agent.hist_add_tool_result(self.name, text, response.attachments)PrintStyle(font_color="#1B4F72", background_color="white", padding=True, bold=True).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")PrintStyle(font_color="#85C1E9").print(response.message)self.log.update(content=response.message)@@ -44,4 +46,4 @@ class Tool:words = key.split('_')words = [words[0].capitalize()] + [word.lower() for word in words[1:]]result = ' '.join(words)- return result\ No newline at end of file+ return result