Skip to content

Agent Zero AI framework

Store vision and computer-use images as path refs

Alessandro1993f6f86415
diff --git a/helpers/images.py b/helpers/images.py
index 9377f1e9c..4e9a5e917 100644
--- a/helpers/images.py
+++ b/helpers/images.py
@@ -1,6 +1,75 @@
-from PIL import Image
+import base64
import io
import math
+import mimetypes
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+from PIL import Image
+
+
+def prepare_content(content: Any) -> Any:
+ if isinstance(content, list):
+ return [prepare_content(item) for item in content]
+ if not isinstance(content, dict):
+ return content
+
+ if content.get("type") == "image_url":
+ image_url = content.get("image_url")
+ if isinstance(image_url, dict):
+ url = str(image_url.get("url", "") or "").strip()
+ if is_local_ref(url):
+ return {**content, "image_url": {**image_url, "url": to_data_url(url)}}
+ elif isinstance(image_url, str):
+ url = image_url.strip()
+ if is_local_ref(url):
+ return {**content, "image_url": {"url": to_data_url(url)}}
+
+ return {key: prepare_content(value) for key, value in content.items()}
+
+
+def is_local_ref(url: str) -> bool:
+ if not url:
+ return False
+ lowered = url.lower()
+ if lowered.startswith(("http://", "https://", "data:")):
+ return False
+ return lowered.startswith("file://") or url.startswith(("/", "./", "../", "~"))
+
+
+def to_data_url(url: str) -> str:
+ path = resolve_ref(url)
+ mime_type = mimetypes.guess_type(path.name)[0]
+ if not mime_type or not mime_type.startswith("image/"):
+ raise ValueError(f"Image attachment must have an image MIME type: {path}")
+ encoded = base64.b64encode(path.read_bytes()).decode("utf-8")
+ return f"data:{mime_type};base64,{encoded}"
+
+
+def resolve_ref(url: str) -> Path:
+ raw_path = unquote(urlparse(url).path) if url.lower().startswith("file://") else url
+ path = Path(raw_path).expanduser()
+ candidates = [path]
+ if raw_path.startswith("/a0/"):
+ from helpers import files
+
+ candidates.append(Path(files.fix_dev_path(raw_path)))
+ elif not path.is_absolute():
+ from helpers import files
+
+ candidates.append(Path(files.get_abs_path(raw_path)))
+
+ seen: set[str] = set()
+ for candidate in candidates:
+ key = str(candidate)
+ if key in seen:
+ continue
+ seen.add(key)
+ if candidate.exists() and candidate.is_file():
+ return candidate
+
+ raise FileNotFoundError(f"Image attachment path does not exist: {raw_path}")
def compress_image(image_data: bytes, *, max_pixels: int = 256_000, quality: int = 50) -> bytes:
diff --git a/models.py b/models.py
index fce201fec..824f5cf70 100644
--- a/models.py
+++ b/models.py
@@ -20,7 +20,7 @@ import openai
from litellm.types.utils import ModelResponse
from helpers import dotenv
-from helpers import settings, dirty_json
+from helpers import settings, dirty_json, images
from helpers.dotenv import load_dotenv
from helpers.providers import ModelType as ProviderModelType, get_provider_config
from helpers.rate_limiter import RateLimiter
@@ -329,7 +329,7 @@ class LiteLLMChatWrapper(SimpleChatModel):
}
for m in messages:
role = role_mapping.get(m.type, m.type)
- message_dict = {"role": role, "content": m.content}
+ message_dict = {"role": role, "content": images.prepare_content(m.content)}
# Handle tool calls for AI messages
tool_calls = getattr(m, "tool_calls", None)
diff --git a/plugins/_a0_connector/api/v1/capabilities.py b/plugins/_a0_connector/api/v1/capabilities.py
index 2ae2e87a9..446163b44 100644
--- a/plugins/_a0_connector/api/v1/capabilities.py
+++ b/plugins/_a0_connector/api/v1/capabilities.py
@@ -80,7 +80,8 @@ class Capabilities(connector_base.PublicConnectorApiHandler):
"websocket_namespace": "/ws",
"websocket_handlers": ["plugins/_a0_connector/ws_connector"],
"attachments": {
- "mode": "base64",
+ "mode": "path_or_url",
+ "http_upload": "base64_to_file",
"max_files": 20,
},
"features": _feature_list(),
diff --git a/plugins/_a0_connector/api/ws_connector.py b/plugins/_a0_connector/api/ws_connector.py
index c673a9589..ead323818 100644
--- a/plugins/_a0_connector/api/ws_connector.py
+++ b/plugins/_a0_connector/api/ws_connector.py
@@ -217,20 +217,26 @@ class WsConnector(WsHandler):
from plugins._a0_connector.helpers.chat_context import ConnectorContextError
message = str(data.get("message", "")).strip()
- if not message:
- return WsResult.error(
- code="MISSING_MESSAGE",
- message="message is required",
- correlation_id=data.get("correlationId"),
- )
-
context_id = str(data.get("context_id", "")).strip() or None
current_context_id = (
str(data.get("current_context", data.get("current_context_id", ""))).strip()
or None
)
client_message_id = str(data.get("client_message_id", "")).strip()
- attachments = list(data.get("attachments", [])) if isinstance(data.get("attachments"), list) else []
+ raw_attachments = list(data.get("attachments", [])) if isinstance(data.get("attachments"), list) else []
+ attachments, attachment_error = self._normalize_attachment_refs(raw_attachments)
+ if attachment_error:
+ return WsResult.error(
+ code="INVALID_ATTACHMENTS",
+ message=attachment_error,
+ correlation_id=data.get("correlationId"),
+ )
+ if not message and not attachments:
+ return WsResult.error(
+ code="MISSING_MESSAGE",
+ message="message or attachments are required",
+ correlation_id=data.get("correlationId"),
+ )
project_name = str(data.get("project_name", "")).strip() or None
agent_profile = str(data.get("agent_profile", "")).strip() or None
@@ -299,6 +305,34 @@ class WsConnector(WsHandler):
"client_message_id": client_message_id or None,
}
+ def _normalize_attachment_refs(self, attachments: list[Any]) -> tuple[list[str], str]:
+ refs: list[str] = []
+ for attachment in attachments:
+ if isinstance(attachment, str):
+ ref = attachment.strip()
+ elif isinstance(attachment, dict):
+ if str(attachment.get("base64", "") or "").strip():
+ return [], (
+ "WebSocket attachments must be file paths or URLs. "
+ "Use the HTTP message_send upload path for base64 file uploads."
+ )
+ ref = str(
+ attachment.get("path")
+ or attachment.get("url")
+ or attachment.get("file")
+ or ""
+ ).strip()
+ else:
+ return [], "attachments must be file paths, URLs, or metadata objects with path/url"
+
+ if not ref:
+ continue
+ if ref.lower().startswith("data:"):
+ return [], "data URL attachments are not accepted; provide a file path or URL"
+ refs.append(ref)
+
+ return refs, ""
+
def _handle_file_op_result(
self,
data: dict[str, Any],
diff --git a/plugins/_a0_connector/plugin.yaml b/plugins/_a0_connector/plugin.yaml
index 5a4e6685b..d10facd86 100644
--- a/plugins/_a0_connector/plugin.yaml
+++ b/plugins/_a0_connector/plugin.yaml
@@ -1,7 +1,7 @@
name: _a0_connector
title: A0 Connector
description: Current Agent Zero connector plugin for HTTP plus /ws integration, using session auth and handler activation through auth.handlers.
-version: 1.5.0
+version: 1.5
settings_sections:
- external
- developer
diff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.py
index e97c6b1f8..3534b1b55 100644
--- a/plugins/_a0_connector/tools/computer_use_remote.py
+++ b/plugins/_a0_connector/tools/computer_use_remote.py
@@ -2,15 +2,10 @@
from __future__ import annotations
import asyncio
-import base64
-import io
-import math
from pathlib import Path
import uuid
from typing import Any
-from PIL import Image
-
from helpers import history
from helpers.tool import Response, Tool
from helpers.ws import NAMESPACE
@@ -26,8 +21,6 @@ from plugins._a0_connector.helpers.ws_runtime import (
COMPUTER_USE_OP_TIMEOUT = 180.0
COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
CAPTURE_TOKENS_ESTIMATE = 1500
-CAPTURE_MAX_PIXELS = 768_000
-CAPTURE_JPEG_QUALITY = 75
_AUTO_CAPTURE_ACTIONS = {
"start_session",
"move",
@@ -313,13 +306,13 @@ class ComputerUseRemote(Tool):
return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."
def _record_capture(self, data: dict[str, Any]) -> str:
- mime_type, image_b64 = self._capture_image_data(data)
+ _image_path, display_path = self._resolve_capture_path(data)
width = data.get("width", "?")
height = data.get("height", "?")
summary = f"Computer-use capture {width}x{height}."
content = [
{"type": "text", "text": summary},
- {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_b64}"}},
+ {"type": "image_url", "image_url": {"url": display_path}},
]
raw_message = history.RawMessage(raw_content=content, preview=summary)
self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)
@@ -342,7 +335,7 @@ class ComputerUseRemote(Tool):
preview = self._capture_preview_from_message(message)
if not preview:
continue
- message.content = f"{preview} [embedded image removed]"
+ message.content = f"{preview} [image reference superseded]"
if hasattr(message, "summary"):
message.summary = ""
if hasattr(message, "calculate_tokens"):
@@ -399,43 +392,6 @@ class ComputerUseRemote(Tool):
return preview
return ""
- def _capture_image_data(self, data: dict[str, Any]) -> tuple[str, str]:
- image_bytes = self._capture_image_bytes(data)
- optimized_bytes = self._optimize_capture_image(image_bytes)
- if optimized_bytes is not None:
- return "image/jpeg", base64.b64encode(optimized_bytes).decode("utf-8")
- return "image/png", base64.b64encode(image_bytes).decode("utf-8")
-
- def _capture_image_bytes(self, data: dict[str, Any]) -> bytes:
- inline_payload = str(data.get("png_base64", "") or "").strip()
- if inline_payload:
- try:
- return base64.b64decode(inline_payload, validate=True)
- except Exception:
- pass
-
- image_path, _display_path = self._resolve_capture_path(data)
- return image_path.read_bytes()
-
- def _optimize_capture_image(self, image_bytes: bytes) -> bytes | None:
- try:
- image = Image.open(io.BytesIO(image_bytes))
- current_pixels = image.width * image.height
- if current_pixels > CAPTURE_MAX_PIXELS:
- scale = math.sqrt(CAPTURE_MAX_PIXELS / current_pixels)
- resized = (
- max(1, int(image.width * scale)),
- max(1, int(image.height * scale)),
- )
- image = image.resize(resized, Image.Resampling.LANCZOS)
- if image.mode not in {"RGB", "L"}:
- image = image.convert("RGB")
- output = io.BytesIO()
- image.save(output, format="JPEG", quality=CAPTURE_JPEG_QUALITY, optimize=True)
- return output.getvalue()
- except Exception:
- return None
-
def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:
candidates = [
str(data.get("capture_path", "") or "").strip(),
diff --git a/tools/vision_load.py b/tools/vision_load.py
index 83a7101c7..75534f1af 100644
--- a/tools/vision_load.py
+++ b/tools/vision_load.py
@@ -1,13 +1,10 @@
-import base64
from helpers.print_style import PrintStyle
from helpers.tool import Tool, Response
-from helpers import runtime, files, images, plugins
+from helpers import runtime, files, plugins
from mimetypes import guess_type
from helpers import history
-# image optimization and token estimation for context window
-MAX_PIXELS = 768_000
-QUALITY = 75
+# image token estimation for context window
TOKENS_ESTIMATE = 1500
@@ -17,7 +14,6 @@ class VisionLoad(Tool):
self.images_dict = {}
self.loaded_paths: list[str] = []
self.skipped_paths: list[str] = []
- template: list[dict[str, str]] = [] # type: ignore
max_embeds = self._get_max_embeds()
limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]
@@ -30,31 +26,8 @@ class VisionLoad(Tool):
if path not in self.images_dict:
mime_type, _ = guess_type(str(path))
if mime_type and mime_type.startswith("image/"):
- try:
- # Read binary file
- file_content = await runtime.call_development_function(
- files.read_file_base64, str(path)
- )
- file_content = base64.b64decode(file_content)
- # Compress and convert to JPEG
- compressed = images.compress_image(
- file_content, max_pixels=MAX_PIXELS, quality=QUALITY
- )
- # Encode as base64
- file_content_b64 = base64.b64encode(compressed).decode("utf-8")
-
- # DEBUG: Save compressed image
- # await runtime.call_development_function(
- # files.write_file_base64, str(path), file_content_b64
- # )
-
- # Construct the data URL (always JPEG after compression)
- self.images_dict[path] = file_content_b64
- self.loaded_paths.append(path)
- except Exception as e:
- self.images_dict[path] = None
- PrintStyle().error(f"Error processing image {path}: {e}")
- self.agent.context.log.log("warning", f"Error processing image {path}: {e}")
+ self.images_dict[path] = str(path)
+ self.loaded_paths.append(path)
return Response(message="dummy", break_loop=False)
@@ -80,12 +53,12 @@ class VisionLoad(Tool):
)
if self.images_dict:
self.agent.hist_add_tool_result(self.name, summary, id=self.log.id if self.log else "")
- for path, image in self.images_dict.items():
- if image:
+ for path, image_path in self.images_dict.items():
+ if image_path:
content.append(
{
"type": "image_url",
- "image_url": {"url": f"data:image/jpeg;base64,{image}"},
+ "image_url": {"url": image_path},
}
)
else:
@@ -96,7 +69,7 @@ class VisionLoad(Tool):
}
)
# append as raw message content for LLMs with vision tokens estimate
- msg = history.RawMessage(raw_content=content, preview="<Base64 encoded image data>")
+ msg = history.RawMessage(raw_content=content, preview="<Image attachments loaded by path>")
self.agent.hist_add_message(
False, content=msg, tokens=TOKENS_ESTIMATE * len(content)
)