Agent Zero AI framework
Store vision and computer-use images as path refs
Alessandro1993f6f86415
diff --git a/helpers/images.py b/helpers/images.pyindex 9377f1e9c..4e9a5e917 100644--- a/helpers/images.py+++ b/helpers/images.py@@ -1,6 +1,75 @@-from PIL import Image+import base64import ioimport math+import mimetypes+from pathlib import Path+from typing import Any+from urllib.parse import unquote, urlparse++from PIL import Image+++def prepare_content(content: Any) -> Any:+ if isinstance(content, list):+ return [prepare_content(item) for item in content]+ if not isinstance(content, dict):+ return content++ if content.get("type") == "image_url":+ image_url = content.get("image_url")+ if isinstance(image_url, dict):+ url = str(image_url.get("url", "") or "").strip()+ if is_local_ref(url):+ return {**content, "image_url": {**image_url, "url": to_data_url(url)}}+ elif isinstance(image_url, str):+ url = image_url.strip()+ if is_local_ref(url):+ return {**content, "image_url": {"url": to_data_url(url)}}++ return {key: prepare_content(value) for key, value in content.items()}+++def is_local_ref(url: str) -> bool:+ if not url:+ return False+ lowered = url.lower()+ if lowered.startswith(("http://", "https://", "data:")):+ return False+ return lowered.startswith("file://") or url.startswith(("/", "./", "../", "~"))+++def to_data_url(url: str) -> str:+ path = resolve_ref(url)+ mime_type = mimetypes.guess_type(path.name)[0]+ if not mime_type or not mime_type.startswith("image/"):+ raise ValueError(f"Image attachment must have an image MIME type: {path}")+ encoded = base64.b64encode(path.read_bytes()).decode("utf-8")+ return f"data:{mime_type};base64,{encoded}"+++def resolve_ref(url: str) -> Path:+ raw_path = unquote(urlparse(url).path) if url.lower().startswith("file://") else url+ path = Path(raw_path).expanduser()+ candidates = [path]+ if raw_path.startswith("/a0/"):+ from helpers import files++ candidates.append(Path(files.fix_dev_path(raw_path)))+ elif not path.is_absolute():+ from helpers import files++ candidates.append(Path(files.get_abs_path(raw_path)))++ seen: set[str] = set()+ for candidate in candidates:+ key = str(candidate)+ if key in seen:+ continue+ seen.add(key)+ if candidate.exists() and candidate.is_file():+ return candidate++ raise FileNotFoundError(f"Image attachment path does not exist: {raw_path}")def compress_image(image_data: bytes, *, max_pixels: int = 256_000, quality: int = 50) -> bytes:diff --git a/models.py b/models.pyindex fce201fec..824f5cf70 100644--- a/models.py+++ b/models.py@@ -20,7 +20,7 @@ import openaifrom litellm.types.utils import ModelResponsefrom helpers import dotenv-from helpers import settings, dirty_json+from helpers import settings, dirty_json, imagesfrom helpers.dotenv import load_dotenvfrom helpers.providers import ModelType as ProviderModelType, get_provider_configfrom helpers.rate_limiter import RateLimiter@@ -329,7 +329,7 @@ class LiteLLMChatWrapper(SimpleChatModel):}for m in messages:role = role_mapping.get(m.type, m.type)- message_dict = {"role": role, "content": m.content}+ message_dict = {"role": role, "content": images.prepare_content(m.content)}# Handle tool calls for AI messagestool_calls = getattr(m, "tool_calls", None)diff --git a/plugins/_a0_connector/api/v1/capabilities.py b/plugins/_a0_connector/api/v1/capabilities.pyindex 2ae2e87a9..446163b44 100644--- a/plugins/_a0_connector/api/v1/capabilities.py+++ b/plugins/_a0_connector/api/v1/capabilities.py@@ -80,7 +80,8 @@ class Capabilities(connector_base.PublicConnectorApiHandler):"websocket_namespace": "/ws","websocket_handlers": ["plugins/_a0_connector/ws_connector"],"attachments": {- "mode": "base64",+ "mode": "path_or_url",+ "http_upload": "base64_to_file","max_files": 20,},"features": _feature_list(),diff --git a/plugins/_a0_connector/api/ws_connector.py b/plugins/_a0_connector/api/ws_connector.pyindex c673a9589..ead323818 100644--- a/plugins/_a0_connector/api/ws_connector.py+++ b/plugins/_a0_connector/api/ws_connector.py@@ -217,20 +217,26 @@ class WsConnector(WsHandler):from plugins._a0_connector.helpers.chat_context import ConnectorContextErrormessage = str(data.get("message", "")).strip()- if not message:- return WsResult.error(- code="MISSING_MESSAGE",- message="message is required",- correlation_id=data.get("correlationId"),- )-context_id = str(data.get("context_id", "")).strip() or Nonecurrent_context_id = (str(data.get("current_context", data.get("current_context_id", ""))).strip()or None)client_message_id = str(data.get("client_message_id", "")).strip()- attachments = list(data.get("attachments", [])) if isinstance(data.get("attachments"), list) else []+ raw_attachments = list(data.get("attachments", [])) if isinstance(data.get("attachments"), list) else []+ attachments, attachment_error = self._normalize_attachment_refs(raw_attachments)+ if attachment_error:+ return WsResult.error(+ code="INVALID_ATTACHMENTS",+ message=attachment_error,+ correlation_id=data.get("correlationId"),+ )+ if not message and not attachments:+ return WsResult.error(+ code="MISSING_MESSAGE",+ message="message or attachments are required",+ correlation_id=data.get("correlationId"),+ )project_name = str(data.get("project_name", "")).strip() or Noneagent_profile = str(data.get("agent_profile", "")).strip() or None@@ -299,6 +305,34 @@ class WsConnector(WsHandler):"client_message_id": client_message_id or None,}+ def _normalize_attachment_refs(self, attachments: list[Any]) -> tuple[list[str], str]:+ refs: list[str] = []+ for attachment in attachments:+ if isinstance(attachment, str):+ ref = attachment.strip()+ elif isinstance(attachment, dict):+ if str(attachment.get("base64", "") or "").strip():+ return [], (+ "WebSocket attachments must be file paths or URLs. "+ "Use the HTTP message_send upload path for base64 file uploads."+ )+ ref = str(+ attachment.get("path")+ or attachment.get("url")+ or attachment.get("file")+ or ""+ ).strip()+ else:+ return [], "attachments must be file paths, URLs, or metadata objects with path/url"++ if not ref:+ continue+ if ref.lower().startswith("data:"):+ return [], "data URL attachments are not accepted; provide a file path or URL"+ refs.append(ref)++ return refs, ""+def _handle_file_op_result(self,data: dict[str, Any],diff --git a/plugins/_a0_connector/plugin.yaml b/plugins/_a0_connector/plugin.yamlindex 5a4e6685b..d10facd86 100644--- a/plugins/_a0_connector/plugin.yaml+++ b/plugins/_a0_connector/plugin.yaml@@ -1,7 +1,7 @@name: _a0_connectortitle: A0 Connectordescription: Current Agent Zero connector plugin for HTTP plus /ws integration, using session auth and handler activation through auth.handlers.-version: 1.5.0+version: 1.5settings_sections:- external- developerdiff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.pyindex e97c6b1f8..3534b1b55 100644--- a/plugins/_a0_connector/tools/computer_use_remote.py+++ b/plugins/_a0_connector/tools/computer_use_remote.py@@ -2,15 +2,10 @@from __future__ import annotationsimport asyncio-import base64-import io-import mathfrom pathlib import Pathimport uuidfrom typing import Any-from PIL import Image-from helpers import historyfrom helpers.tool import Response, Toolfrom helpers.ws import NAMESPACE@@ -26,8 +21,6 @@ from plugins._a0_connector.helpers.ws_runtime import (COMPUTER_USE_OP_TIMEOUT = 180.0COMPUTER_USE_OP_EVENT = "connector_computer_use_op"CAPTURE_TOKENS_ESTIMATE = 1500-CAPTURE_MAX_PIXELS = 768_000-CAPTURE_JPEG_QUALITY = 75_AUTO_CAPTURE_ACTIONS = {"start_session","move",@@ -313,13 +306,13 @@ class ComputerUseRemote(Tool):return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."def _record_capture(self, data: dict[str, Any]) -> str:- mime_type, image_b64 = self._capture_image_data(data)+ _image_path, display_path = self._resolve_capture_path(data)width = data.get("width", "?")height = data.get("height", "?")summary = f"Computer-use capture {width}x{height}."content = [{"type": "text", "text": summary},- {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_b64}"}},+ {"type": "image_url", "image_url": {"url": display_path}},]raw_message = history.RawMessage(raw_content=content, preview=summary)self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)@@ -342,7 +335,7 @@ class ComputerUseRemote(Tool):preview = self._capture_preview_from_message(message)if not preview:continue- message.content = f"{preview} [embedded image removed]"+ message.content = f"{preview} [image reference superseded]"if hasattr(message, "summary"):message.summary = ""if hasattr(message, "calculate_tokens"):@@ -399,43 +392,6 @@ class ComputerUseRemote(Tool):return previewreturn ""- def _capture_image_data(self, data: dict[str, Any]) -> tuple[str, str]:- image_bytes = self._capture_image_bytes(data)- optimized_bytes = self._optimize_capture_image(image_bytes)- if optimized_bytes is not None:- return "image/jpeg", base64.b64encode(optimized_bytes).decode("utf-8")- return "image/png", base64.b64encode(image_bytes).decode("utf-8")-- def _capture_image_bytes(self, data: dict[str, Any]) -> bytes:- inline_payload = str(data.get("png_base64", "") or "").strip()- if inline_payload:- try:- return base64.b64decode(inline_payload, validate=True)- except Exception:- pass-- image_path, _display_path = self._resolve_capture_path(data)- return image_path.read_bytes()-- def _optimize_capture_image(self, image_bytes: bytes) -> bytes | None:- try:- image = Image.open(io.BytesIO(image_bytes))- current_pixels = image.width * image.height- if current_pixels > CAPTURE_MAX_PIXELS:- scale = math.sqrt(CAPTURE_MAX_PIXELS / current_pixels)- resized = (- max(1, int(image.width * scale)),- max(1, int(image.height * scale)),- )- image = image.resize(resized, Image.Resampling.LANCZOS)- if image.mode not in {"RGB", "L"}:- image = image.convert("RGB")- output = io.BytesIO()- image.save(output, format="JPEG", quality=CAPTURE_JPEG_QUALITY, optimize=True)- return output.getvalue()- except Exception:- return None-def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:candidates = [str(data.get("capture_path", "") or "").strip(),diff --git a/tools/vision_load.py b/tools/vision_load.pyindex 83a7101c7..75534f1af 100644--- a/tools/vision_load.py+++ b/tools/vision_load.py@@ -1,13 +1,10 @@-import base64from helpers.print_style import PrintStylefrom helpers.tool import Tool, Response-from helpers import runtime, files, images, plugins+from helpers import runtime, files, pluginsfrom mimetypes import guess_typefrom helpers import history-# image optimization and token estimation for context window-MAX_PIXELS = 768_000-QUALITY = 75+# image token estimation for context windowTOKENS_ESTIMATE = 1500@@ -17,7 +14,6 @@ class VisionLoad(Tool):self.images_dict = {}self.loaded_paths: list[str] = []self.skipped_paths: list[str] = []- template: list[dict[str, str]] = [] # type: ignoremax_embeds = self._get_max_embeds()limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]@@ -30,31 +26,8 @@ class VisionLoad(Tool):if path not in self.images_dict:mime_type, _ = guess_type(str(path))if mime_type and mime_type.startswith("image/"):- try:- # Read binary file- file_content = await runtime.call_development_function(- files.read_file_base64, str(path)- )- file_content = base64.b64decode(file_content)- # Compress and convert to JPEG- compressed = images.compress_image(- file_content, max_pixels=MAX_PIXELS, quality=QUALITY- )- # Encode as base64- file_content_b64 = base64.b64encode(compressed).decode("utf-8")-- # DEBUG: Save compressed image- # await runtime.call_development_function(- # files.write_file_base64, str(path), file_content_b64- # )-- # Construct the data URL (always JPEG after compression)- self.images_dict[path] = file_content_b64- self.loaded_paths.append(path)- except Exception as e:- self.images_dict[path] = None- PrintStyle().error(f"Error processing image {path}: {e}")- self.agent.context.log.log("warning", f"Error processing image {path}: {e}")+ self.images_dict[path] = str(path)+ self.loaded_paths.append(path)return Response(message="dummy", break_loop=False)@@ -80,12 +53,12 @@ class VisionLoad(Tool):)if self.images_dict:self.agent.hist_add_tool_result(self.name, summary, id=self.log.id if self.log else "")- for path, image in self.images_dict.items():- if image:+ for path, image_path in self.images_dict.items():+ if image_path:content.append({"type": "image_url",- "image_url": {"url": f"data:image/jpeg;base64,{image}"},+ "image_url": {"url": image_path},})else:@@ -96,7 +69,7 @@ class VisionLoad(Tool):})# append as raw message content for LLMs with vision tokens estimate- msg = history.RawMessage(raw_content=content, preview="<Base64 encoded image data>")+ msg = history.RawMessage(raw_content=content, preview="<Image attachments loaded by path>")self.agent.hist_add_message(False, content=msg, tokens=TOKENS_ESTIMATE * len(content))