Skip to content

Agent Zero AI framework

feat: chat model VISION support for user and tool msg attachments

Rafael Uzarowski22eef83cc662
diff --git a/agent.py b/agent.py
index c5775d131..415b4a20e 100644
--- a/agent.py
+++ b/agent.py
@@ -10,8 +10,9 @@ import models
from langchain_core.prompt_values import ChatPromptValue
from python.helpers import extract_tools, rate_limiter, files, errors, history, tokens
from python.helpers.print_style import PrintStyle
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate, StringPromptTemplate
+from langchain_core.prompts.image import ImagePromptTemplate
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.language_models.llms import BaseLLM
from langchain_core.embeddings import Embeddings
@@ -19,6 +20,7 @@ import python.helpers.log as Log
from python.helpers.dirty_json import DirtyJson
from python.helpers.defer import DeferredTask
from typing import Callable
+from python.helpers.history import OutputMessage
class AgentContext:
@@ -187,7 +189,7 @@ class AgentConfig:
@dataclass
class UserMessage:
message: str
- attachments: list[str]
+ attachments: list[str] = field(default_factory=list[str])
class LoopData:
@@ -357,10 +359,14 @@ class Agent:
loop_data.extras_temporary.clear()
# combine history and extras
- history_combined = history.group_outputs_abab(loop_data.history_output + extras)
+ history_combined: list[OutputMessage] = history.group_outputs_abab(loop_data.history_output + extras)
# convert history to LLM format
- history_langchain = history.output_langchain(history_combined)
+ history_langchain: list[BaseMessage] = history.output_langchain(history_combined)
+
+ PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(
+ f"History Langchain: {history_langchain}"
+ )
# build chain from system prompt, message history and model
prompt = ChatPromptTemplate.from_messages(
@@ -479,9 +485,10 @@ class Agent:
content = self.parse_prompt("fw.warning.md", message=message)
return self.hist_add_message(False, content=content)
- async def hist_add_tool_result(self, tool_name: str, tool_result: str):
+ async def hist_add_tool_result(self, tool_name: str, tool_result: str, attachments: list[str] = []):
+ attachments_str = json.dumps(attachments).replace("\n", "")
content = self.parse_prompt(
- "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result
+ "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result, attachments=attachments_str
)
return self.hist_add_message(False, content=content)
diff --git a/initialize.py b/initialize.py
index ead07d2f2..0003b0223 100644
--- a/initialize.py
+++ b/initialize.py
@@ -13,6 +13,7 @@ def initialize():
provider=models.ModelProvider[current_settings["chat_model_provider"]],
name=current_settings["chat_model_name"],
ctx_length=current_settings["chat_model_ctx_length"],
+ vision=current_settings["chat_model_vision"],
limit_requests=current_settings["chat_model_rl_requests"],
limit_input=current_settings["chat_model_rl_input"],
limit_output=current_settings["chat_model_rl_output"],
diff --git a/prompts/default/agent.system.tools_vision.md b/prompts/default/agent.system.tools_vision.md
new file mode 100644
index 000000000..0af7e7527
--- /dev/null
+++ b/prompts/default/agent.system.tools_vision.md
@@ -0,0 +1,3 @@
+## "Multimodal (Vision) Agent Tools" available:
+
+None yet. In future, this section will contain vision-only tools
diff --git a/prompts/default/fw.tool_result.md b/prompts/default/fw.tool_result.md
index 1943cc022..b45d456e8 100644
--- a/prompts/default/fw.tool_result.md
+++ b/prompts/default/fw.tool_result.md
@@ -1,6 +1,7 @@
~~~json
{
"tool_name": {{tool_name}},
- "tool_result": {{tool_result}}
+ "tool_result": {{tool_result}},
+ "attachments": {{attachments}}
}
-~~~
\ No newline at end of file
+~~~
diff --git a/python/extensions/system_prompt/_10_system_prompt.py b/python/extensions/system_prompt/_10_system_prompt.py
index e7544b2ec..68f7d6dd8 100644
--- a/python/extensions/system_prompt/_10_system_prompt.py
+++ b/python/extensions/system_prompt/_10_system_prompt.py
@@ -12,11 +12,17 @@ class SystemPrompt(Extension):
system_prompt.append(main)
system_prompt.append(tools)
+
def get_main_prompt(agent: Agent):
return get_prompt("agent.system.main.md", agent)
+
def get_tools_prompt(agent: Agent):
- return get_prompt("agent.system.tools.md", agent)
+ prompt = get_prompt("agent.system.tools.md", agent)
+ if agent.config.chat_model.vision:
+ prompt += '\n' + get_prompt("agent.system.tools_vision.md", agent)
+ return prompt
+
def get_prompt(file: str, agent: Agent):
# variables for system prompts
@@ -26,4 +32,4 @@ def get_prompt(file: str, agent: Agent):
"date_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"agent_name": agent.agent_name,
}
- return agent.read_prompt(file, **vars)
\ No newline at end of file
+ return agent.read_prompt(file, **vars)
diff --git a/python/helpers/history.py b/python/helpers/history.py
index 3f9e7c506..3fcb02426 100644
--- a/python/helpers/history.py
+++ b/python/helpers/history.py
@@ -3,11 +3,14 @@ import asyncio
from collections import OrderedDict
import json
import math
+import os
from typing import Coroutine, Literal, TypedDict, cast
from python.helpers import messages, tokens, settings, call_llm
from enum import Enum
-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
-
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage
+from python.helpers.print_style import PrintStyle
+from langchain_core.prompts import HumanMessagePromptTemplate
+from typing import Any
BULK_MERGE_COUNT = 3
TOPICS_KEEP_COUNT = 3
CURRENT_TOPIC_RATIO = 0.5
@@ -425,13 +428,60 @@ def group_outputs_abab(outputs: list[OutputMessage]) -> list[OutputMessage]:
return result
-def output_langchain(messages: list[OutputMessage]):
+def output_langchain(messages: list[OutputMessage]) -> list[BaseMessage]:
result = []
for m in messages:
if m["ai"]:
result.append(AIMessage(content=serialize_content(m["content"])))
else:
- result.append(HumanMessage(content=serialize_content(m["content"])))
+ contents = m["content"]
+
+ # sometimes content is a list sometimes not
+ if not isinstance(contents, list):
+ contents = [contents]
+
+ PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(
+ f"Contents: {json.dumps(contents, indent=2)}"
+ )
+
+ template: list[dict[str, str]] = [] # type: ignore
+ message = ""
+ images = {}
+ for _, content in enumerate(contents):
+ if message:
+ # the first message is the user message, then the memory and solutions
+ message += "\n\n--- Memory & Solutions Section: ---\n\n"
+
+ message += serialize_content(content)
+
+ if isinstance(content, dict) and "attachments" in content:
+ attachments: list[str] = cast(list[str], content["attachments"])
+ for attachment in attachments:
+ if not os.path.exists(str(attachment)):
+ continue
+ if attachment not in images:
+ import base64
+ from mimetypes import guess_type
+ mime_type, _ = guess_type(str(attachment))
+ if mime_type.startswith("image/"):
+ # Read and encode the image file
+ with open(str(attachment), "rb") as image_file:
+ base64_encoded_data = base64.b64encode(image_file.read()).decode('utf-8')
+ # Construct the data URL
+ images[attachment] = f"data:{mime_type};base64,{base64_encoded_data}"
+
+ if message:
+ template.append({"type": "text", "text": message})
+ if images:
+ for _, image in images.items():
+ template.append({"type": "image_url", "image_url": image})
+ if template:
+ # only jinja2 is safe for json, both mustache({{...}}) and f-string({...}) are not
+ result.append(HumanMessagePromptTemplate.from_template(template=template, partial_variables={}, template_format="jinja2")) # type: ignore
+
+ PrintStyle(font_color="grey", background_color="black", bold=True, padding=True).print(
+ f"Result: {result}"
+ )
return result
diff --git a/python/helpers/settings.py b/python/helpers/settings.py
index 228330753..fc7a41ce2 100644
--- a/python/helpers/settings.py
+++ b/python/helpers/settings.py
@@ -16,6 +16,7 @@ class Settings(TypedDict):
chat_model_kwargs: dict[str, str]
chat_model_ctx_length: int
chat_model_ctx_history: float
+ chat_model_vision: bool
chat_model_rl_requests: int
chat_model_rl_input: int
chat_model_rl_output: int
@@ -149,6 +150,17 @@ def convert_out(settings: Settings) -> SettingsOutput:
}
)
+
+ chat_model_fields.append(
+ {
+ "id": "chat_model_vision",
+ "title": "Supports Vision",
+ "description": "Models capable of Vision can for example natively see the content of image attachments.",
+ "type": "switch",
+ "value": settings["chat_model_vision"],
+ }
+ )
+
chat_model_fields.append(
{
"id": "chat_model_rl_requests",
@@ -777,6 +789,7 @@ def get_default_settings() -> Settings:
chat_model_kwargs={ "temperature": "0" },
chat_model_ctx_length=120000,
chat_model_ctx_history=0.7,
+ chat_model_vision=False,
chat_model_rl_requests=0,
chat_model_rl_input=0,
chat_model_rl_output=0,
diff --git a/python/helpers/tool.py b/python/helpers/tool.py
index a50ec8295..fcfb51070 100644
--- a/python/helpers/tool.py
+++ b/python/helpers/tool.py
@@ -1,14 +1,16 @@
from abc import abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from agent import Agent
from python.helpers.print_style import PrintStyle
-from python.helpers import messages
+
@dataclass
class Response:
message:str
- break_loop:bool
-
+ break_loop: bool
+ attachments: list[str] = field(default_factory=list[str])
+
+
class Tool:
def __init__(self, agent: Agent, name: str, args: dict[str,str], message: str, **kwargs) -> None:
@@ -29,10 +31,10 @@ class Tool:
PrintStyle(font_color="#85C1E9", bold=True).stream(self.nice_key(key)+": ")
PrintStyle(font_color="#85C1E9", padding=isinstance(value,str) and "\n" in value).stream(value)
PrintStyle().print()
-
+
async def after_execution(self, response: Response, **kwargs):
text = response.message.strip()
- await self.agent.hist_add_tool_result(self.name, text)
+ await self.agent.hist_add_tool_result(self.name, text, response.attachments)
PrintStyle(font_color="#1B4F72", background_color="white", padding=True, bold=True).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")
PrintStyle(font_color="#85C1E9").print(response.message)
self.log.update(content=response.message)
@@ -44,4 +46,4 @@ class Tool:
words = key.split('_')
words = [words[0].capitalize()] + [word.lower() for word in words[1:]]
result = ' '.join(words)
- return result
\ No newline at end of file
+ return result