Skip to content

Agent Zero AI framework

Make browser screenshots ephemeral and context scoped

Alessandro430c48d1a5ae
diff --git a/helpers/ephemeral_images.py b/helpers/ephemeral_images.py
new file mode 100644
index 000000000..6d42c21d2
--- /dev/null
+++ b/helpers/ephemeral_images.py
@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import base64
+import threading
+import time
+import uuid
+from dataclasses import dataclass
+
+
+REF_PREFIX = "a0-ephemeral-image://"
+DEFAULT_TTL_SECONDS = 15 * 60
+
+
+@dataclass(frozen=True)
+class EphemeralImage:
+ ref: str
+ context_id: str
+ mime: str
+ data: str
+ name: str
+ created_at: float
+ expires_at: float
+
+ @property
+ def data_url(self) -> str:
+ return f"data:{self.mime};base64,{self.data}"
+
+ @property
+ def display_name(self) -> str:
+ return self.name or display_ref(self.ref)
+
+
+_store: dict[str, EphemeralImage] = {}
+_lock = threading.RLock()
+
+
+def put_image_bytes(
+ *,
+ context_id: str,
+ mime: str,
+ payload: bytes,
+ name: str = "",
+ ttl_seconds: float = DEFAULT_TTL_SECONDS,
+) -> str:
+ data = base64.b64encode(bytes(payload or b"")).decode("ascii")
+ return put_image(
+ context_id=context_id,
+ mime=mime,
+ data=data,
+ name=name,
+ ttl_seconds=ttl_seconds,
+ )
+
+
+def put_image(
+ *,
+ context_id: str,
+ mime: str,
+ data: str,
+ name: str = "",
+ ttl_seconds: float = DEFAULT_TTL_SECONDS,
+) -> str:
+ compact_data = _compact_base64(data)
+ if not compact_data:
+ raise ValueError("ephemeral image data is empty")
+ base64.b64decode(compact_data, validate=True)
+
+ normalized_mime = _normalize_mime(mime)
+ now = time.time()
+ ref = f"{REF_PREFIX}{uuid.uuid4().hex}"
+ image = EphemeralImage(
+ ref=ref,
+ context_id=str(context_id or "").strip(),
+ mime=normalized_mime,
+ data=compact_data,
+ name=str(name or "").strip(),
+ created_at=now,
+ expires_at=now + max(1.0, float(ttl_seconds or DEFAULT_TTL_SECONDS)),
+ )
+ with _lock:
+ _prune_expired_locked(now)
+ _store[ref] = image
+ return ref
+
+
+def is_ref(value: object) -> bool:
+ return str(value or "").strip().startswith(REF_PREFIX)
+
+
+def display_ref(ref: str) -> str:
+ value = str(ref or "").strip()
+ if not is_ref(value):
+ return value
+ return f"{REF_PREFIX}<ephemeral>"
+
+
+def get_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:
+ return _resolve_image(ref, context_id=context_id, consume=False)
+
+
+def consume_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:
+ return _resolve_image(ref, context_id=context_id, consume=True)
+
+
+def delete_image(ref: str) -> None:
+ with _lock:
+ _store.pop(str(ref or "").strip(), None)
+
+
+def clear_context(context_id: str) -> None:
+ normalized_context = str(context_id or "").strip()
+ with _lock:
+ for ref, image in list(_store.items()):
+ if image.context_id == normalized_context:
+ _store.pop(ref, None)
+
+
+def _resolve_image(ref: str, *, context_id: str = "", consume: bool) -> EphemeralImage | None:
+ value = str(ref or "").strip()
+ if not is_ref(value):
+ return None
+
+ now = time.time()
+ with _lock:
+ _prune_expired_locked(now)
+ image = _store.get(value)
+ if image is None:
+ return None
+ requested_context = str(context_id or "").strip()
+ if requested_context and image.context_id and image.context_id != requested_context:
+ return None
+ if consume:
+ _store.pop(value, None)
+ return image
+
+
+def _compact_base64(data: str) -> str:
+ return "".join(char for char in str(data or "") if not char.isspace())
+
+
+def _normalize_mime(mime: str) -> str:
+ value = str(mime or "").strip().lower()
+ return value if value.startswith("image/") else "image/jpeg"
+
+
+def _prune_expired_locked(now: float | None = None) -> None:
+ current = time.time() if now is None else float(now)
+ for ref, image in list(_store.items()):
+ if image.expires_at <= current:
+ _store.pop(ref, None)
diff --git a/plugins/_browser/helpers/connector_runtime.py b/plugins/_browser/helpers/connector_runtime.py
index 80c49a133..235a8ea47 100644
--- a/plugins/_browser/helpers/connector_runtime.py
+++ b/plugins/_browser/helpers/connector_runtime.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
-import base64
import hashlib
import re
import uuid
@@ -10,7 +9,7 @@ from pathlib import Path
from typing import Any
from urllib.parse import urlparse
-from helpers import files
+from helpers import ephemeral_images
try:
from helpers.ws import NAMESPACE
@@ -40,10 +39,8 @@ from plugins._browser.helpers.url import normalize_url
BROWSER_OP_EVENT = "connector_browser_op"
BROWSER_OP_TIMEOUT = 120.0
-HOST_BROWSER_SCREENSHOT_DIR = ("tmp", "browser", "host-screenshots")
CONTENT_HELPER_PATH = Path(__file__).resolve().parents[1] / "assets" / "browser-page-content.js"
MAX_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
-BASE64_DECODE_CHARS_PER_CHUNK = 64 * 1024
HOST_BROWSER_PRIVACY_POLICY_KEY = getattr(
browser_config,
"HOST_BROWSER_PRIVACY_POLICY_KEY",
@@ -431,26 +428,30 @@ class ConnectorBrowserRuntime:
estimated_size = _estimated_base64_decoded_size(data)
if estimated_size > MAX_ARTIFACT_SIZE_BYTES:
raise RuntimeError(
- "Host browser artifact is too large to materialize safely "
+ "Host browser artifact is too large to attach safely "
f"({estimated_size} bytes, limit {MAX_ARTIFACT_SIZE_BYTES} bytes)."
)
filename = _safe_filename(str(artifact.get("filename") or "host-browser.jpg"))
- target_dir = Path(files.get_abs_path(*HOST_BROWSER_SCREENSHOT_DIR, self.context_id))
- target_dir.mkdir(parents=True, exist_ok=True)
- target_path = target_dir / filename
try:
- _write_base64_to_path(data, target_path)
+ ref = ephemeral_images.put_image(
+ context_id=self.context_id,
+ mime=str(artifact.get("mime") or result.get("mime") or "image/jpeg"),
+ data=data,
+ name=filename,
+ )
except Exception as exc:
- target_path.unlink(missing_ok=True)
raise RuntimeError("Host browser artifact could not be decoded.") from exc
materialized = dict(result)
materialized.pop("artifact", None)
- local_path = str(target_path)
- materialized["path"] = local_path
- materialized["a0_path"] = files.normalize_a0_path(local_path)
+ materialized.pop("path", None)
+ materialized.pop("a0_path", None)
+ materialized.pop("host_path", None)
+ materialized.setdefault("context_id", self.context_id)
+ materialized["ephemeral"] = True
+ materialized["ephemeral_ref"] = ref
materialized["vision_load"] = {
"tool_name": "vision_load",
- "tool_args": {"paths": [local_path]},
+ "tool_args": {"paths": [ref]},
}
return materialized
@@ -569,20 +570,3 @@ def _safe_filename(value: str) -> str:
def _estimated_base64_decoded_size(data: str) -> int:
compact_length = sum(1 for char in data if not char.isspace())
return (compact_length * 3) // 4
-
-
-def _write_base64_to_path(data: str, target_path: Path) -> None:
- pending = ""
- with target_path.open("wb") as target:
- for offset in range(0, len(data), BASE64_DECODE_CHARS_PER_CHUNK):
- chunk = pending + "".join(
- char
- for char in data[offset : offset + BASE64_DECODE_CHARS_PER_CHUNK]
- if not char.isspace()
- )
- ready_length = (len(chunk) // 4) * 4
- if ready_length:
- target.write(base64.b64decode(chunk[:ready_length], validate=True))
- pending = chunk[ready_length:]
- if pending:
- target.write(base64.b64decode(pending, validate=True))
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 664c4e8c5..048296fae 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -15,7 +15,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any
-from helpers import files
+from helpers import ephemeral_images, files
from helpers.defer import DeferredTask
from helpers.errors import RepairableException
from helpers.print_style import PrintStyle
@@ -1548,6 +1548,34 @@ class _BrowserRuntimeCore:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
+ raw_path = str(path or "").strip()
+ if not raw_path:
+ image = await page.screenshot(
+ type="jpeg",
+ quality=max(20, min(95, int(quality))),
+ full_page=bool(full_page),
+ )
+ ref = ephemeral_images.put_image_bytes(
+ context_id=self.context_id,
+ mime="image/jpeg",
+ payload=image,
+ name=f"browser-{resolved_id}.jpg",
+ )
+ return {
+ "browser_id": resolved_id,
+ "context_id": self.context_id,
+ "mime": "image/jpeg",
+ "ephemeral": True,
+ "ephemeral_ref": ref,
+ "state": await self._state(resolved_id),
+ "vision_load": {
+ "tool_name": "vision_load",
+ "tool_args": {
+ "paths": [ref],
+ },
+ },
+ }
+
output_path, image_type, mime = self._screenshot_output_path(resolved_id, path)
output_path.parent.mkdir(parents=True, exist_ok=True)
clamped_quality = max(20, min(95, int(quality)))
@@ -1562,6 +1590,7 @@ class _BrowserRuntimeCore:
local_path = str(output_path)
return {
"browser_id": resolved_id,
+ "context_id": self.context_id,
"path": local_path,
"a0_path": files.normalize_a0_path(local_path),
"mime": mime,
diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.md
index ec36ab9a4..f9ed07b35 100644
--- a/plugins/_browser/prompts/agent.system.tool.browser.md
+++ b/plugins/_browser/prompts/agent.system.tool.browser.md
@@ -20,7 +20,7 @@ Workflow:
- For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.
- `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.
- `navigate` reuses an existing `browser_id` and is preferred for serial browsing.
-- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned path before reasoning visually.
+- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually. When no `path` is requested, browser screenshots are ephemeral refs rather than conserved files.
- Keep the tab set small; close pages after extracting what you need.
`multi` is only a browser action: use `tool_name: "browser"` with `tool_args.action: "multi"`. Never use `tool_name: "multi"`.
diff --git a/plugins/_browser/skills/browser-automation/SKILL.md b/plugins/_browser/skills/browser-automation/SKILL.md
index 47c8e12dc..a31949e10 100644
--- a/plugins/_browser/skills/browser-automation/SKILL.md
+++ b/plugins/_browser/skills/browser-automation/SKILL.md
@@ -29,10 +29,10 @@ In host mode, page content and screenshots may be blocked by host-content policy
Screenshots are explicit only; the browser does not automatically load images into model context.
1. Call `browser` with `action: "screenshot"`.
-2. Call `vision_load` with the returned path.
+2. Call `vision_load` with the returned `vision_load.tool_args.paths` value.
3. Reason from the latest loaded screenshot.
-Screenshot args include `quality`, `full_page`, and optional `path`. PNG is used when `path` ends with `.png`; otherwise JPEG is used.
+Screenshot args include `quality`, `full_page`, and optional `path`. Without `path`, the screenshot is an ephemeral ref consumed by `vision_load`; with `path`, PNG is used when `path` ends with `.png`, otherwise JPEG is used.
## Forms And Files
diff --git a/plugins/_browser/skills/browser-form-workflows/SKILL.md b/plugins/_browser/skills/browser-form-workflows/SKILL.md
index bc2197489..aae826cfb 100644
--- a/plugins/_browser/skills/browser-form-workflows/SKILL.md
+++ b/plugins/_browser/skills/browser-form-workflows/SKILL.md
@@ -11,7 +11,7 @@ Start with `browser:content` to capture current refs, then use `browser:detail`
Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.
-Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history.
+Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history; no-path screenshots return ephemeral refs for `vision_load`.
Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.
diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.py
index b0bad5476..10220f028 100644
--- a/plugins/_browser/tools/browser.py
+++ b/plugins/_browser/tools/browser.py
@@ -89,8 +89,6 @@ class Browser(Tool):
if action == "open":
result = await runtime.call("open", url or "")
elif action == "screenshot":
- if not path:
- path = self._history_screenshot_path(action)
result = await runtime.call(
"screenshot_file",
browser_id,
@@ -428,53 +426,65 @@ class Browser(Tool):
return
screenshot = result if action == "screenshot" and isinstance(result, dict) else None
- if not self._screenshot_has_path(screenshot):
+ if not self._screenshot_has_reference(screenshot):
target_browser_id = self._browser_id_from_result(result) or requested_browser_id
- output_path = self._history_screenshot_path(action)
- if not output_path:
- return
try:
screenshot = await runtime.call(
"screenshot_file",
target_browser_id,
quality=HISTORY_SCREENSHOT_QUALITY,
full_page=False,
- path=output_path,
+ path="",
)
except Exception as exc:
PrintStyle.debug(
"Browser history screenshot capture failed:",
f"browser_id={target_browser_id}",
f"quality={HISTORY_SCREENSHOT_QUALITY}",
- f"path={output_path}",
f"error={exc}",
)
return
- if not self._screenshot_has_path(screenshot):
+ if not self._screenshot_has_reference(screenshot):
return
- local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))
- if not local_path:
- return
- uri = f"img://{local_path}&t={time.time()}"
+ a0_path = str(screenshot.get("a0_path") or "").strip()
+ local_path = str(screenshot.get("path") or (files.fix_dev_path(a0_path) if a0_path else ""))
state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}
- self.log.update(
- Screenshot=uri,
- browser_snapshot={
- "uri": uri,
- "path": local_path,
- "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
- "mime": screenshot.get("mime") or "image/jpeg",
- "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
- "context_id": screenshot.get("context_id") or state.get("context_id") or "",
- },
- )
+ chat_context_id = self._agent_context_id()
+ browser_context_id = str(screenshot.get("context_id") or state.get("context_id") or "").strip()
+ snapshot = {
+ "mime": screenshot.get("mime") or "image/jpeg",
+ "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
+ "context_id": chat_context_id or browser_context_id,
+ "browser_context_id": browser_context_id,
+ }
+ update_payload: dict[str, Any] = {"browser_snapshot": snapshot}
+ if local_path:
+ uri = f"img://{local_path}&t={time.time()}"
+ snapshot.update(
+ {
+ "uri": uri,
+ "path": local_path,
+ "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
+ "ephemeral": False,
+ }
+ )
+ update_payload["Screenshot"] = uri
+ else:
+ ephemeral_ref = self._screenshot_ephemeral_ref(screenshot)
+ snapshot.update(
+ {
+ "ephemeral": bool(ephemeral_ref),
+ "ephemeral_ref": ephemeral_ref,
+ }
+ )
+ self.log.update(**update_payload)
def _history_screenshot_path(self, action: str) -> str:
if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):
return ""
- context_id = str(getattr(self.agent.context, "id", "") or "").strip()
+ context_id = self._agent_context_id()
if not context_id:
return ""
from helpers import persist_chat
@@ -521,6 +531,31 @@ class Browser(Tool):
def _screenshot_has_path(screenshot: Any) -> bool:
return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))
+ @classmethod
+ def _screenshot_has_reference(cls, screenshot: Any) -> bool:
+ return cls._screenshot_has_path(screenshot) or bool(cls._screenshot_ephemeral_ref(screenshot))
+
+ @staticmethod
+ def _screenshot_ephemeral_ref(screenshot: Any) -> str:
+ if not isinstance(screenshot, dict):
+ return ""
+ ref = str(screenshot.get("ephemeral_ref") or "").strip()
+ if ref:
+ return ref
+ vision_load = screenshot.get("vision_load")
+ if isinstance(vision_load, dict):
+ tool_args = vision_load.get("tool_args")
+ if isinstance(tool_args, dict):
+ paths = tool_args.get("paths")
+ if isinstance(paths, list) and paths:
+ first = str(paths[0] or "").strip()
+ if first.startswith("a0-ephemeral-image://"):
+ return first
+ return ""
+
+ def _agent_context_id(self) -> str:
+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
+
@staticmethod
def _format_result(action: str, result: Any) -> str:
if action == "content" and isinstance(result, dict):
diff --git a/prompts/agent.system.tools_vision.md b/prompts/agent.system.tools_vision.md
index 7611245a4..ee50376ad 100644
--- a/prompts/agent.system.tools_vision.md
+++ b/prompts/agent.system.tools_vision.md
@@ -2,7 +2,7 @@
### vision_load
load images into the model for visual reasoning
-args: `paths` list of absolute image paths
+args: `paths` list of absolute image paths or tool-returned ephemeral image refs
rules:
- load all relevant images in one call when comparing screenshots or pages
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 529d9964f..8d6bf8f13 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -79,6 +79,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_conf
def anyio_backend():
return "asyncio"
+from helpers import ephemeral_images
from helpers.errors import RepairableException
from plugins._browser.helpers.config import (
build_browser_launch_config,
@@ -1808,12 +1809,10 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p
},
}
if method == "screenshot_file":
- Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
- Path(kwargs["path"]).write_bytes(b"jpeg")
return {
"browser_id": args[0],
- "path": kwargs["path"],
- "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",
+ "ephemeral": True,
+ "ephemeral_ref": "a0-ephemeral-image://fake",
"mime": "image/jpeg",
"state": {"id": args[0], "context_id": "browser-context"},
}
@@ -1861,11 +1860,13 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p
assert calls[1][1] == (1,)
assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITY
assert calls[1][2]["full_page"] is False
- assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"
- assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"
- assert log.updates[-1]["Screenshot"].startswith("img://")
+ assert calls[1][2]["path"] == ""
+ assert "Screenshot" not in log.updates[-1]
assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1
- assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"
+ assert log.updates[-1]["browser_snapshot"]["context_id"] == "chat"
+ assert log.updates[-1]["browser_snapshot"]["browser_context_id"] == "browser-context"
+ assert log.updates[-1]["browser_snapshot"]["ephemeral"] is True
+ assert log.updates[-1]["browser_snapshot"]["ephemeral_ref"] == "a0-ephemeral-image://fake"
@pytest.mark.anyio
@@ -2393,7 +2394,7 @@ async def test_browser_runtime_remounts_initial_changed_viewport():
@pytest.mark.anyio
-async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch, tmp_path):
+async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeypatch, tmp_path):
screenshot_calls = []
def fake_get_abs_path(*parts):
@@ -2411,8 +2412,9 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
async def screenshot(self, **kwargs):
screenshot_calls.append(kwargs)
- Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
- Path(kwargs["path"]).write_bytes(b"image-bytes")
+ if kwargs.get("path"):
+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
+ Path(kwargs["path"]).write_bytes(b"image-bytes")
return b"image-bytes"
async def title(self):
@@ -2427,21 +2429,23 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
result = await core.screenshot_file(5, quality=500)
- path = Path(result["path"])
- assert path.exists()
- assert path.parent == tmp_path / "tmp" / "browser" / "screenshots" / "ctx_id"
- assert path.name.startswith("browser-5-")
- assert path.suffix == ".jpg"
- assert result["a0_path"].startswith("/a0/tmp/browser/screenshots/ctx_id/browser-5-")
+ assert "path" not in result
+ assert "a0_path" not in result
+ assert result["context_id"] == "ctx/id"
assert result["mime"] == "image/jpeg"
+ assert result["ephemeral"] is True
+ assert result["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
assert result["vision_load"] == {
"tool_name": "vision_load",
- "tool_args": {"paths": [result["path"]]},
+ "tool_args": {"paths": [result["ephemeral_ref"]]},
}
assert "image" not in result
+ assert not list((tmp_path / "tmp" / "browser" / "screenshots").rglob("*.jpg"))
assert screenshot_calls[-1]["type"] == "jpeg"
assert screenshot_calls[-1]["quality"] == 95
assert screenshot_calls[-1]["full_page"] is False
+ assert "path" not in screenshot_calls[-1]
+ assert ephemeral_images.consume_image(result["ephemeral_ref"], context_id="ctx/id").data_url == "data:image/jpeg;base64,aW1hZ2UtYnl0ZXM="
png_path = tmp_path / "custom.png"
png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))
@@ -2455,6 +2459,51 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
}
+async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):
+ import tools.vision_load as vision_load_module
+
+ monkeypatch.setattr(
+ vision_load_module.plugins,
+ "get_plugin_config",
+ lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},
+ )
+
+ tool_results = []
+ messages = []
+ updates = []
+ agent = SimpleNamespace(
+ context=SimpleNamespace(id="ctx-vision"),
+ agent_name="Agent 0",
+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
+ hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
+ )
+ ref = ephemeral_images.put_image(
+ context_id="ctx-vision",
+ mime="image/jpeg",
+ data=SMALL_JPEG_10X10,
+ name="browser-shot.jpg",
+ )
+ tool = vision_load_module.VisionLoad(
+ agent=agent,
+ name="vision_load",
+ method=None,
+ args={"paths": [ref]},
+ message="",
+ loop_data=None,
+ )
+ tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
+
+ response = await tool.execute(paths=[ref])
+ await tool.after_execution(response)
+
+ assert ephemeral_images.get_image(ref, context_id="ctx-vision") is None
+ assert tool.loaded_paths == ["browser-shot.jpg"]
+ raw_message = messages[0][1]["content"]
+ assert raw_message.raw_content[0]["image_url"]["url"] == f"data:image/jpeg;base64,{SMALL_JPEG_10X10}"
+ assert updates[-1]["result"] == "1 images loaded, 0 skipped"
+
+
@pytest.mark.anyio
async def test_browser_runtime_ref_point_resolution_applies_offsets():
eval_payloads = []
diff --git a/tests/test_host_browser_connector.py b/tests/test_host_browser_connector.py
index 572563ea1..398ce38b5 100644
--- a/tests/test_host_browser_connector.py
+++ b/tests/test_host_browser_connector.py
@@ -12,6 +12,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
+from helpers import ephemeral_images
from plugins._a0_connector.helpers import ws_runtime
from plugins._browser.helpers.connector_runtime import (
ConnectorBrowserRuntime,
@@ -329,19 +330,7 @@ def test_connector_runtime_adds_docker_recovery_to_host_errors():
assert "/browser container" in message
-def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tmp_path):
- import plugins._browser.helpers.connector_runtime as connector_runtime_module
-
- monkeypatch.setattr(
- connector_runtime_module.files,
- "get_abs_path",
- lambda *parts: str(tmp_path.joinpath(*parts)),
- )
- monkeypatch.setattr(
- connector_runtime_module.files,
- "normalize_a0_path",
- lambda path: "/a0/" + str(path).lstrip("/"),
- )
+def test_host_browser_artifacts_become_context_scoped_ephemeral_refs(tmp_path):
runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
result = runtime._materialize_artifact(
@@ -363,19 +352,19 @@ def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tm
inner = result[0]["result"]
assert "artifact" not in inner
- assert inner["path"].endswith("shot.jpg")
- assert Path(inner["path"]).read_bytes() == b"fake"
- assert inner["vision_load"]["tool_args"]["paths"] == [inner["path"]]
+ assert "path" not in inner
+ assert "a0_path" not in inner
+ assert inner["context_id"] == "ctx-host"
+ assert inner["ephemeral"] is True
+ assert inner["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
+ assert inner["vision_load"]["tool_args"]["paths"] == [inner["ephemeral_ref"]]
+ assert ephemeral_images.consume_image(inner["ephemeral_ref"], context_id="ctx-host").data_url == "data:image/jpeg;base64,ZmFrZQ=="
+ assert not list(tmp_path.rglob("shot.jpg"))
def test_host_browser_artifact_materialization_rejects_oversized_payload(monkeypatch, tmp_path):
import plugins._browser.helpers.connector_runtime as connector_runtime_module
- monkeypatch.setattr(
- connector_runtime_module.files,
- "get_abs_path",
- lambda *parts: str(tmp_path.joinpath(*parts)),
- )
monkeypatch.setattr(connector_runtime_module, "MAX_ARTIFACT_SIZE_BYTES", 2)
runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
diff --git a/tools/vision_load.py b/tools/vision_load.py
index 75534f1af..90f2133e6 100644
--- a/tools/vision_load.py
+++ b/tools/vision_load.py
@@ -1,6 +1,6 @@
from helpers.print_style import PrintStyle
from helpers.tool import Tool, Response
-from helpers import runtime, files, plugins
+from helpers import runtime, files, plugins, ephemeral_images
from mimetypes import guess_type
from helpers import history
@@ -16,18 +16,43 @@ class VisionLoad(Tool):
self.skipped_paths: list[str] = []
max_embeds = self._get_max_embeds()
- limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]
- self.skipped_paths = paths[:-max_embeds] if max_embeds > 0 and len(paths) > max_embeds else []
+ requested = [
+ (str(path or "").strip(), self._display_input_path(str(path or "").strip(), idx + 1))
+ for idx, path in enumerate(paths)
+ ]
+ limited_paths = requested if max_embeds <= 0 else requested[-max_embeds:]
+ self.skipped_paths = (
+ [display for _, display in requested[:-max_embeds]]
+ if max_embeds > 0 and len(requested) > max_embeds
+ else []
+ )
- for path in limited_paths:
+ for path, display_path in limited_paths:
+ if not path:
+ continue
+ if ephemeral_images.is_ref(path):
+ image = ephemeral_images.consume_image(
+ path,
+ context_id=self._context_id(),
+ )
+ if image is None:
+ continue
+ display = image.display_name or display_path
+ self.images_dict[display] = image.data_url
+ self.loaded_paths.append(display)
+ continue
+ if self._is_data_image_url(path):
+ self.images_dict[display_path] = path
+ self.loaded_paths.append(display_path)
+ continue
if not await runtime.call_development_function(files.exists, str(path)):
continue
if path not in self.images_dict:
mime_type, _ = guess_type(str(path))
if mime_type and mime_type.startswith("image/"):
- self.images_dict[path] = str(path)
- self.loaded_paths.append(path)
+ self.images_dict[display_path] = str(path)
+ self.loaded_paths.append(display_path)
return Response(message="dummy", break_loop=False)
@@ -37,6 +62,23 @@ class VisionLoad(Tool):
max_embeds = chat_cfg.get("max_embeds", 10)
return int(max_embeds or 0)
+ def _context_id(self) -> str:
+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
+
+ @staticmethod
+ def _is_data_image_url(value: str) -> bool:
+ normalized = str(value or "").strip().lower()
+ return normalized.startswith("data:image/") and ";base64," in normalized
+
+ @classmethod
+ def _display_input_path(cls, value: str, index: int) -> str:
+ if ephemeral_images.is_ref(value):
+ return ephemeral_images.display_ref(value)
+ if cls._is_data_image_url(value):
+ prefix = value.split(",", 1)[0]
+ return f"{prefix},<ephemeral-image-{index}>"
+ return value
+
async def after_execution(self, response: Response, **kwargs):
# build image data messages for LLMs, or error message