Skip to content

Agent Zero AI framework

Compress computer-use captures before embedding in history

Alessandro20107ff92154
diff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.py
index efbe7916b..e97c6b1f8 100644
--- a/plugins/_a0_connector/tools/computer_use_remote.py
+++ b/plugins/_a0_connector/tools/computer_use_remote.py
@@ -3,10 +3,14 @@ from __future__ import annotations
import asyncio
import base64
+import io
+import math
from pathlib import Path
import uuid
from typing import Any
+from PIL import Image
+
from helpers import history
from helpers.tool import Response, Tool
from helpers.ws import NAMESPACE
@@ -22,6 +26,8 @@ from plugins._a0_connector.helpers.ws_runtime import (
COMPUTER_USE_OP_TIMEOUT = 180.0
COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
CAPTURE_TOKENS_ESTIMATE = 1500
+CAPTURE_MAX_PIXELS = 768_000
+CAPTURE_JPEG_QUALITY = 75
_AUTO_CAPTURE_ACTIONS = {
"start_session",
"move",
@@ -307,13 +313,13 @@ class ComputerUseRemote(Tool):
return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."
def _record_capture(self, data: dict[str, Any]) -> str:
- image_b64 = self._capture_image_base64(data)
+ mime_type, image_b64 = self._capture_image_data(data)
width = data.get("width", "?")
height = data.get("height", "?")
summary = f"Computer-use capture {width}x{height}."
content = [
{"type": "text", "text": summary},
- {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},
+ {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_b64}"}},
]
raw_message = history.RawMessage(raw_content=content, preview=summary)
self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)
@@ -393,18 +399,42 @@ class ComputerUseRemote(Tool):
return preview
return ""
- def _capture_image_base64(self, data: dict[str, Any]) -> str:
+ def _capture_image_data(self, data: dict[str, Any]) -> tuple[str, str]:
+ image_bytes = self._capture_image_bytes(data)
+ optimized_bytes = self._optimize_capture_image(image_bytes)
+ if optimized_bytes is not None:
+ return "image/jpeg", base64.b64encode(optimized_bytes).decode("utf-8")
+ return "image/png", base64.b64encode(image_bytes).decode("utf-8")
+
+ def _capture_image_bytes(self, data: dict[str, Any]) -> bytes:
inline_payload = str(data.get("png_base64", "") or "").strip()
if inline_payload:
try:
- base64.b64decode(inline_payload, validate=True)
+ return base64.b64decode(inline_payload, validate=True)
except Exception:
pass
- else:
- return inline_payload
image_path, _display_path = self._resolve_capture_path(data)
- return base64.b64encode(image_path.read_bytes()).decode("utf-8")
+ return image_path.read_bytes()
+
+ def _optimize_capture_image(self, image_bytes: bytes) -> bytes | None:
+ try:
+ image = Image.open(io.BytesIO(image_bytes))
+ current_pixels = image.width * image.height
+ if current_pixels > CAPTURE_MAX_PIXELS:
+ scale = math.sqrt(CAPTURE_MAX_PIXELS / current_pixels)
+ resized = (
+ max(1, int(image.width * scale)),
+ max(1, int(image.height * scale)),
+ )
+ image = image.resize(resized, Image.Resampling.LANCZOS)
+ if image.mode not in {"RGB", "L"}:
+ image = image.convert("RGB")
+ output = io.BytesIO()
+ image.save(output, format="JPEG", quality=CAPTURE_JPEG_QUALITY, optimize=True)
+ return output.getvalue()
+ except Exception:
+ return None
def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:
candidates = [