Agent Zero AI framework
feature: attachment setup
Alessandro3c6a5bee6429
diff --git a/agent.py b/agent.pyindex 0f59cf9d4..9e02a9a5c 100644--- a/agent.py+++ b/agent.py@@ -116,6 +116,7 @@ class AgentContext:except Exception as e:agent.handle_critical_exception(e)+@dataclassclass AgentConfig:chat_model: BaseChatModel | BaseLLM@@ -194,6 +195,7 @@ class LoopData:self.message = ""self.history_from = 0self.history = []+ self.attachments = [] # Add attachments field# intervention exception class - skips rest of message loop iteration@@ -251,6 +253,13 @@ class Agent:printer = PrintStyle(italic=True, font_color="#b3ffd9", padding=False)user_message = loop_data.message++ # Include attachments in user message if available+ if loop_data.attachments:+ user_message += "\n" + "\n".join(loop_data.attachments) # Add attachments to message+ loop_data.attachments = [] # Clear attachments after adding to message++await self.append_message(user_message, human=True)# let the agent run message loop until he stops it with a response tooldiff --git a/python/extensions/message_loop_prompts/_30_include_attachments.py b/python/extensions/message_loop_prompts/_30_include_attachments.pynew file mode 100644index 000000000..e17deebe6--- /dev/null+++ b/python/extensions/message_loop_prompts/_30_include_attachments.py@@ -0,0 +1,53 @@+from python.helpers.extension import Extension+from agent import Agent, LoopData+import os+import io+import base64+from PIL import Image++class IncludeAttachments(Extension):+ async def execute(self, loop_data: LoopData = LoopData(), **kwargs):+ # Check if there are attachments in agent data+ attachments = self.agent.get_data('attachments') or []+ if attachments:+ loop_data.attachments = [] # Initialize attachments list for loop_data++ # For each attachment, compress and encode the image+ for attachment_path in attachments:+ if os.path.exists(attachment_path):+ # Prepare the base64-encoded image+ compressed_image_base64 = self.compress_and_encode_image(attachment_path)+ if compressed_image_base64:+ # Append the image data to loop_data.attachments+ loop_data.attachments.append(f"<image>{compressed_image_base64}</image>")++ # Clear attachments from agent data+ self.agent.set_data('attachments', [])++ def compress_and_encode_image(self, image_path: str) -> str:+ try:+ # Open an image file+ with Image.open(image_path) as img:+ # Convert image to RGB if it's in RGBA mode+ if img.mode in ('RGBA', 'P'):+ img = img.convert('RGB')++ # Resize the image to a reasonable size+ max_dimension = 800 # You can adjust this value+ img.thumbnail((max_dimension, max_dimension))++ # Compress the image+ buffered = io.BytesIO()+ # Save as JPEG to ensure compression; you can adjust quality+ img.save(buffered, format="JPEG", quality=70, optimize=True)+ compressed_image = buffered.getvalue()++ # Encode the compressed image in base64+ return base64.b64encode(compressed_image).decode('utf-8')+ except Exception as e:+ print(f"Error compressing and encoding image {image_path}: {e}")+ return ""++ def estimate_token_count(self, message: str) -> int:+ # Simple estimation: assume 4 characters per token+ return len(message) // 4diff --git a/run_ui.py b/run_ui.pyindex 3b152092e..a286ff94e 100644--- a/run_ui.py+++ b/run_ui.py@@ -143,8 +143,7 @@ async def health_check():# send message to agent (async UI)@app.route("/msg", methods=["POST"])async def handle_message_async():- return await handle_message(False)-+ return await handle_message(False)# send message to agent (synchronous API)@app.route("/msg_sync", methods=["POST"])@@ -153,50 +152,71 @@ async def handle_msg_sync():async def handle_message(sync: bool):- try:-- # data sent to the server- input = request.get_json()- text = input.get("text", "")- ctxid = input.get("context", "")- blev = input.get("broadcast", 1)-- # context instance - get or create- context = get_context(ctxid)-- # print to console and log- PrintStyle(- background_color="#6C3483", font_color="white", bold=True, padding=True- ).print(f"User message:")- PrintStyle(font_color="white", padding=False).print(f"> {text}")- context.log.log(type="user", heading="User message", content=text)-- if sync:- context.communicate(text)- result = await context.process.result() # type: ignore- response = {- "ok": True,- "message": result,- "context": context.id,- }- else:-- context.communicate(text)- response = {- "ok": True,- "message": "Message received.",- "context": context.id,- }-- except Exception as e:- response = {- "ok": False,- "message": str(e),- }- PrintStyle.error(str(e))-- # respond with json- return jsonify(response)+ try:+ # Handle both JSON and multipart/form-data+ if request.content_type.startswith('multipart/form-data'):+ text = request.form.get('text', '')+ ctxid = request.form.get('context', '')+ attachments = request.files.getlist('attachments')+ attachment_paths = []++ upload_folder = os.path.join(os.getcwd(), 'work_dir', 'uploads')++ if attachments:+ os.makedirs(upload_folder, exist_ok=True)+ for attachment in attachments:+ filename = secure_filename(attachment.filename)+ save_path = os.path.join(upload_folder, filename)+ attachment.save(save_path)+ attachment_paths.append(save_path)+ else:+ # Handle JSON request as before+ input_data = request.get_json()+ text = input_data.get('text', '')+ ctxid = input_data.get('context', '')+ attachment_paths = []++ # Now process the message+ message = text++ # Obtain agent context+ context = get_context(ctxid)++ # Store attachments in agent data+ context.agent0.set_data('attachments', attachment_paths)++ # Print to console and log+ PrintStyle(+ background_color="#6C3483", font_color="white", bold=True, padding=True+ ).print(f"User message:")+ PrintStyle(font_color="white", padding=False).print(f"> {message}")+ context.log.log(type="user", heading="User message", content=message)++ if sync:+ context.communicate(message)+ result = await context.process.result() # type: ignore+ response = {+ "ok": True,+ "message": result,+ "context": context.id,+ }+ else:+ context.communicate(message)+ response = {+ "ok": True,+ "message": "Message received.",+ "context": context.id,+ }++ except Exception as e:+ response = {+ "ok": False,+ "message": str(e),+ }+ PrintStyle.error(str(e))++ # respond with json+ return jsonify(response)# pausing/unpausing the agentdiff --git a/webui/index.css b/webui/index.cssindex 890117b3c..b74817b9c 100644--- a/webui/index.css+++ b/webui/index.css@@ -807,16 +807,36 @@ pre {opacity: 0.5;}+.attachments-container {+ margin-top: 10px;+ padding: 10px;+ border-radius: 5px;+}++.message-attachment {+ max-width: 100%;+ max-height: 400px;+ margin: 5px 0;+ border-radius: 5px;+ object-fit: contain;+}++.image-error {+ border: 1px solid #ff0000;+ padding: 10px;+ color: #ff0000;+}+/* Text input */#chat-input {flex-grow: 1;min-height: 2.7rem;- padding: var(--spacing-xs) var(--spacing-sm);- background-color: var(--color-input);+ padding: var(--spacing-sm) var(--spacing-sm);+ padding-top: 0.70rem;border: 1px solid var(--color-border);border-radius: 8px;resize: none;- align-content: center;+ align-content: start;}.input-row {@@ -838,7 +858,7 @@ pre {background: none;border: none;color: var(--color-text);- font-family: Rubik;+ font-family: "Rubik", Arial, Helvetica, sans-serif;font-size: 0.7rem;padding: 6px var(--spacing-sm);cursor: pointer;@@ -1284,7 +1304,7 @@ input:checked + .slider:before {.katex {font-family: Roboto Mono !important;line-height: 1rem !important;- font-size: var(--font-size-normal) !important;+ font-size: 3rem;}/* Animations */@@ -1316,6 +1336,8 @@ input:checked + .slider:before {#chat-input {min-height: 5.3rem;align-content: start;++}#chat-buttons-wrapper {diff --git a/webui/index.js b/webui/index.jsindex 98bdc8a5e..7aaace9b9 100644--- a/webui/index.js+++ b/webui/index.js@@ -60,31 +60,81 @@ function setupSidebarToggle() {// Make sure to call this functiondocument.addEventListener('DOMContentLoaded', setupSidebarToggle);+ // index.jsasync function sendMessage() {try {const message = chatInput.value.trim();- if (message) {+ const inputAD = Alpine.$data(inputSection);+ const attachments = inputAD.attachments;+ const hasAttachments = attachments && attachments.length > 0;++ if (message || hasAttachments) {+ let response;+ const messageId = generateGUID();++ // Only render immediately for attachments+ if (hasAttachments) {+ const attachmentsWithUrls = attachments.map(attachment => ({+ ...attachment,+ url: URL.createObjectURL(attachment.file)+ }));++ // Only render if there's text content or it's an image-only message+ setMessage(messageId, 'user', '', message, false, {+ attachments: attachmentsWithUrls+ });++ const formData = new FormData();+ formData.append('text', message);+ formData.append('context', context);+ formData.append('message_id', messageId);- const response = await sendJsonData("/msg", { text: message, context });+ for (let i = 0; i < attachments.length; i++) {+ formData.append('attachments', attachments[i].file);+ }- if (!response) {- toast("No response returned.", "error")- } else if (!response.ok) {- if (response.message) {- toast(response.message, "error")+ response = await fetch('/msg', {+ method: 'POST',+ body: formData+ });+ } else {+ // For text-only messages, let polling handle the rendering+ const data = {+ text: message,+ context,+ message_id: messageId+ };+ response = await fetch('/msg', {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json'+ },+ body: JSON.stringify(data)+ });+ }++ // Handle response+ const jsonResponse = await response.json();+ if (!jsonResponse) {+ toast("No response returned.", "error");+ } else if (!jsonResponse.ok) {+ if (jsonResponse.message) {+ toast(jsonResponse.message, "error");} else {- toast("Undefined error.", "error")+ toast("Undefined error.", "error");}} else {- setContext(response.context)+ setContext(jsonResponse.context);}- //setMessage('user', message);+ // Clear input and attachmentschatInput.value = '';+ inputAD.attachments = [];+ inputAD.hasAttachments = false;adjustTextareaHeight();}} catch (e) {- toast(e.message, "error")+ toast(e.message, "error");}}diff --git a/webui/messages.js b/webui/messages.jsindex 891d7304d..8e45a1137 100644--- a/webui/messages.js+++ b/webui/messages.js@@ -27,6 +27,8 @@ export function getHandler(type) {}}++// draw a message with a specific typeexport function _drawMessage(messageContainer, heading, content, temp, followUp, kvps = null, messageClasses = [], contentClasses = []) {const messageDiv = document.createElement('div');@@ -60,10 +62,10 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,if (window.renderMathInElement) {renderMathInElement(spanElement, {delimiters: [- {left: "$$", right: "$$", display: true},- {left: "\$$", right: "\$$", display: true},+ {left: "$", right: "$", display: true},+ {left: "\\$", right: "\\$", display: true},{left: "$", right: "$", display: false},- {left: "\$$", right: "\$$", display: false}+ {left: "\\$", right: "\\$", display: false}],throwOnError: false // Prevent KaTeX from throwing errors});@@ -72,43 +74,85 @@ export function _drawMessage(messageContainer, heading, content, temp, followUp,return messageDiv;}+export function drawMessageDefault(messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, false, kvps, ['message-ai', 'message-default'], ['msg-json']);+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, ['message-ai', 'message-default'], ['msg-json']);}export function drawMessageAgent(messageContainer, id, type, heading, content, temp, kvps = null) {- let kvpsFlat = null+ let kvpsFlat = null;if (kvps) {- kvpsFlat = { ...kvps, ...kvps['tool_args'] || {} }- delete kvpsFlat['tool_args']+ kvpsFlat = { ...kvps, ...kvps['tool_args'] || {} };+ delete kvpsFlat['tool_args'];}- _drawMessage(messageContainer, heading, content, temp, false, kvpsFlat, ['message-ai', 'message-agent'], ['msg-json']);+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, false, kvpsFlat, ['message-ai', 'message-agent'], ['msg-json']);}export function drawMessageResponse(messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, true, null, ['message-ai', 'message-agent-response']);+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, true, null, ['message-ai', 'message-agent-response']);}export function drawMessageDelegation(messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, true, kvps, ['message-ai', 'message-agent', 'message-agent-delegation']);+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, true, kvps, ['message-ai', 'message-agent', 'message-agent-delegation']);}export function drawMessageUser(messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, false, kvps, ['message-user']);+ const hasContent = content && content.trim().length > 0;+ const hasAttachments = kvps && kvps.attachments && kvps.attachments.length > 0;++ // Only create message container if there's content or it's the initial message+ if (hasContent || !messageContainer.hasChildNodes()) {+ // Create the message with user heading and content+ _drawMessage(messageContainer, "User message", content, temp, false, null, ['message-user'], []);+ }++ // Render image attachments for user messages+ if (hasAttachments) {+ const attachmentsContainer = document.createElement('div');+ attachmentsContainer.classList.add('attachments-container');++ // Add "Attachments" heading if there's text content+ if (hasContent) {+ const attachmentsHeading = document.createElement('h4');+ attachmentsHeading.textContent = "Attachments";+ attachmentsContainer.appendChild(attachmentsHeading);+ }++ kvps.attachments.forEach(attachment => {+ const image = document.createElement('img');+ if (attachment.url) {+ image.src = attachment.url;+ } else if (attachment.data) {+ image.src = `data:image/jpeg;base64,${attachment.data}`;+ }+ image.alt = 'Attachment';+ image.classList.add('message-attachment');+ attachmentsContainer.appendChild(image);+ });++ messageContainer.appendChild(attachmentsContainer);+ }}export function drawMessageTool(messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, true, kvps, ['message-ai', 'message-tool'], ['msg-output']);+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, true, kvps, ['message-ai', 'message-tool'], ['msg-output']);}export function drawMessageCodeExe(messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, true, null, ['message-ai', 'message-code-exe']);+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, true, null, ['message-ai', 'message-code-exe']);}export function drawMessageAgentPlain(classes, messageContainer, id, type, heading, content, temp, kvps = null) {- _drawMessage(messageContainer, heading, content, temp, false, null, [...classes]);- messageContainer.classList.add('center-container')+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, false, null, [...classes]);+ messageContainer.classList.add('center-container');}export function drawMessageInfo(messageContainer, id, type, heading, content, temp, kvps = null) {@@ -116,12 +160,9 @@ export function drawMessageInfo(messageContainer, id, type, heading, content, te}export function drawMessageUtil(messageContainer, id, type, heading, content, temp, kvps = null) {- //if kvps is not null and contains "query"- if (kvps && kvps["query"]) {- const a = 1+1- }- _drawMessage(messageContainer, heading, content, temp, false, kvps, ['message-util'], ['msg-json']);- messageContainer.classList.add('center-container')+ const messageContent = convertImageTags(content); // Convert image tags+ _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, ['message-util'], ['msg-json']);+ messageContainer.classList.add('center-container');}export function drawMessageWarning(messageContainer, id, type, heading, content, temp, kvps = null) {@@ -189,3 +230,16 @@ function convertToTitleCase(str) {return match.toUpperCase(); // Capitalize the first letter of each word});}+++function convertImageTags(content) {+ // Regular expression to match <image> tags and extract base64 content+ const imageTagRegex = /<image>(.*?)<\/image>/g;++ // Replace <image> tags with <img> tags with base64 source+ const updatedContent = content.replace(imageTagRegex, (match, base64Content) => {+ return `<img src="data:image/jpeg;base64,${base64Content}" alt="Image Attachment" style="max-width: 250px !important;"/>`;+ });++ return updatedContent;+}diff --git a/webui/settings.css b/webui/settings.cssindex 43f7329f5..7a17b6473 100644--- a/webui/settings.css+++ b/webui/settings.css@@ -128,7 +128,7 @@ select {padding: 0.5rem;border: 1px solid #ddd;border-radius: 0.25rem;- font-family: Rubik;+ font-family: "Rubik", Arial, Helvetica, sans-serif;}textarea {@@ -212,7 +212,7 @@ select {cursor: pointer;border: none;font-size: 0.875rem;- font-family: Rubik;+ font-family: "Rubik", Arial, Helvetica, sans-serif;}.btn-ok {@@ -262,7 +262,7 @@ select {background-color: white;font-size: inherit;cursor: pointer;- font-family: Rubik;+ font-family: "Rubik", Arial, Helvetica, sans-serif;outline: none;}