Agent Zero AI framework
Make browser screenshots ephemeral and context scoped
Alessandro430c48d1a5ae
diff --git a/helpers/ephemeral_images.py b/helpers/ephemeral_images.pynew file mode 100644index 000000000..6d42c21d2--- /dev/null+++ b/helpers/ephemeral_images.py@@ -0,0 +1,150 @@+from __future__ import annotations++import base64+import threading+import time+import uuid+from dataclasses import dataclass+++REF_PREFIX = "a0-ephemeral-image://"+DEFAULT_TTL_SECONDS = 15 * 60+++@dataclass(frozen=True)+class EphemeralImage:+ ref: str+ context_id: str+ mime: str+ data: str+ name: str+ created_at: float+ expires_at: float++ @property+ def data_url(self) -> str:+ return f"data:{self.mime};base64,{self.data}"++ @property+ def display_name(self) -> str:+ return self.name or display_ref(self.ref)+++_store: dict[str, EphemeralImage] = {}+_lock = threading.RLock()+++def put_image_bytes(+ *,+ context_id: str,+ mime: str,+ payload: bytes,+ name: str = "",+ ttl_seconds: float = DEFAULT_TTL_SECONDS,+) -> str:+ data = base64.b64encode(bytes(payload or b"")).decode("ascii")+ return put_image(+ context_id=context_id,+ mime=mime,+ data=data,+ name=name,+ ttl_seconds=ttl_seconds,+ )+++def put_image(+ *,+ context_id: str,+ mime: str,+ data: str,+ name: str = "",+ ttl_seconds: float = DEFAULT_TTL_SECONDS,+) -> str:+ compact_data = _compact_base64(data)+ if not compact_data:+ raise ValueError("ephemeral image data is empty")+ base64.b64decode(compact_data, validate=True)++ normalized_mime = _normalize_mime(mime)+ now = time.time()+ ref = f"{REF_PREFIX}{uuid.uuid4().hex}"+ image = EphemeralImage(+ ref=ref,+ context_id=str(context_id or "").strip(),+ mime=normalized_mime,+ data=compact_data,+ name=str(name or "").strip(),+ created_at=now,+ expires_at=now + max(1.0, float(ttl_seconds or DEFAULT_TTL_SECONDS)),+ )+ with _lock:+ _prune_expired_locked(now)+ _store[ref] = image+ return ref+++def is_ref(value: object) -> bool:+ return str(value or "").strip().startswith(REF_PREFIX)+++def display_ref(ref: str) -> str:+ value = str(ref or "").strip()+ if not is_ref(value):+ return value+ return f"{REF_PREFIX}<ephemeral>"+++def get_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:+ return _resolve_image(ref, context_id=context_id, consume=False)+++def consume_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:+ return _resolve_image(ref, context_id=context_id, consume=True)+++def delete_image(ref: str) -> None:+ with _lock:+ _store.pop(str(ref or "").strip(), None)+++def clear_context(context_id: str) -> None:+ normalized_context = str(context_id or "").strip()+ with _lock:+ for ref, image in list(_store.items()):+ if image.context_id == normalized_context:+ _store.pop(ref, None)+++def _resolve_image(ref: str, *, context_id: str = "", consume: bool) -> EphemeralImage | None:+ value = str(ref or "").strip()+ if not is_ref(value):+ return None++ now = time.time()+ with _lock:+ _prune_expired_locked(now)+ image = _store.get(value)+ if image is None:+ return None+ requested_context = str(context_id or "").strip()+ if requested_context and image.context_id and image.context_id != requested_context:+ return None+ if consume:+ _store.pop(value, None)+ return image+++def _compact_base64(data: str) -> str:+ return "".join(char for char in str(data or "") if not char.isspace())+++def _normalize_mime(mime: str) -> str:+ value = str(mime or "").strip().lower()+ return value if value.startswith("image/") else "image/jpeg"+++def _prune_expired_locked(now: float | None = None) -> None:+ current = time.time() if now is None else float(now)+ for ref, image in list(_store.items()):+ if image.expires_at <= current:+ _store.pop(ref, None)diff --git a/plugins/_browser/helpers/connector_runtime.py b/plugins/_browser/helpers/connector_runtime.pyindex 80c49a133..235a8ea47 100644--- a/plugins/_browser/helpers/connector_runtime.py+++ b/plugins/_browser/helpers/connector_runtime.py@@ -1,7 +1,6 @@from __future__ import annotationsimport asyncio-import base64import hashlibimport reimport uuid@@ -10,7 +9,7 @@ from pathlib import Pathfrom typing import Anyfrom urllib.parse import urlparse-from helpers import files+from helpers import ephemeral_imagestry:from helpers.ws import NAMESPACE@@ -40,10 +39,8 @@ from plugins._browser.helpers.url import normalize_urlBROWSER_OP_EVENT = "connector_browser_op"BROWSER_OP_TIMEOUT = 120.0-HOST_BROWSER_SCREENSHOT_DIR = ("tmp", "browser", "host-screenshots")CONTENT_HELPER_PATH = Path(__file__).resolve().parents[1] / "assets" / "browser-page-content.js"MAX_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024-BASE64_DECODE_CHARS_PER_CHUNK = 64 * 1024HOST_BROWSER_PRIVACY_POLICY_KEY = getattr(browser_config,"HOST_BROWSER_PRIVACY_POLICY_KEY",@@ -431,26 +428,30 @@ class ConnectorBrowserRuntime:estimated_size = _estimated_base64_decoded_size(data)if estimated_size > MAX_ARTIFACT_SIZE_BYTES:raise RuntimeError(- "Host browser artifact is too large to materialize safely "+ "Host browser artifact is too large to attach safely "f"({estimated_size} bytes, limit {MAX_ARTIFACT_SIZE_BYTES} bytes).")filename = _safe_filename(str(artifact.get("filename") or "host-browser.jpg"))- target_dir = Path(files.get_abs_path(*HOST_BROWSER_SCREENSHOT_DIR, self.context_id))- target_dir.mkdir(parents=True, exist_ok=True)- target_path = target_dir / filenametry:- _write_base64_to_path(data, target_path)+ ref = ephemeral_images.put_image(+ context_id=self.context_id,+ mime=str(artifact.get("mime") or result.get("mime") or "image/jpeg"),+ data=data,+ name=filename,+ )except Exception as exc:- target_path.unlink(missing_ok=True)raise RuntimeError("Host browser artifact could not be decoded.") from excmaterialized = dict(result)materialized.pop("artifact", None)- local_path = str(target_path)- materialized["path"] = local_path- materialized["a0_path"] = files.normalize_a0_path(local_path)+ materialized.pop("path", None)+ materialized.pop("a0_path", None)+ materialized.pop("host_path", None)+ materialized.setdefault("context_id", self.context_id)+ materialized["ephemeral"] = True+ materialized["ephemeral_ref"] = refmaterialized["vision_load"] = {"tool_name": "vision_load",- "tool_args": {"paths": [local_path]},+ "tool_args": {"paths": [ref]},}return materialized@@ -569,20 +570,3 @@ def _safe_filename(value: str) -> str:def _estimated_base64_decoded_size(data: str) -> int:compact_length = sum(1 for char in data if not char.isspace())return (compact_length * 3) // 4---def _write_base64_to_path(data: str, target_path: Path) -> None:- pending = ""- with target_path.open("wb") as target:- for offset in range(0, len(data), BASE64_DECODE_CHARS_PER_CHUNK):- chunk = pending + "".join(- char- for char in data[offset : offset + BASE64_DECODE_CHARS_PER_CHUNK]- if not char.isspace()- )- ready_length = (len(chunk) // 4) * 4- if ready_length:- target.write(base64.b64decode(chunk[:ready_length], validate=True))- pending = chunk[ready_length:]- if pending:- target.write(base64.b64decode(pending, validate=True))diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 664c4e8c5..048296fae 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -15,7 +15,7 @@ from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any-from helpers import files+from helpers import ephemeral_images, filesfrom helpers.defer import DeferredTaskfrom helpers.errors import RepairableExceptionfrom helpers.print_style import PrintStyle@@ -1548,6 +1548,34 @@ class _BrowserRuntimeCore:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)+ raw_path = str(path or "").strip()+ if not raw_path:+ image = await page.screenshot(+ type="jpeg",+ quality=max(20, min(95, int(quality))),+ full_page=bool(full_page),+ )+ ref = ephemeral_images.put_image_bytes(+ context_id=self.context_id,+ mime="image/jpeg",+ payload=image,+ name=f"browser-{resolved_id}.jpg",+ )+ return {+ "browser_id": resolved_id,+ "context_id": self.context_id,+ "mime": "image/jpeg",+ "ephemeral": True,+ "ephemeral_ref": ref,+ "state": await self._state(resolved_id),+ "vision_load": {+ "tool_name": "vision_load",+ "tool_args": {+ "paths": [ref],+ },+ },+ }+output_path, image_type, mime = self._screenshot_output_path(resolved_id, path)output_path.parent.mkdir(parents=True, exist_ok=True)clamped_quality = max(20, min(95, int(quality)))@@ -1562,6 +1590,7 @@ class _BrowserRuntimeCore:local_path = str(output_path)return {"browser_id": resolved_id,+ "context_id": self.context_id,"path": local_path,"a0_path": files.normalize_a0_path(local_path),"mime": mime,diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.mdindex ec36ab9a4..f9ed07b35 100644--- a/plugins/_browser/prompts/agent.system.tool.browser.md+++ b/plugins/_browser/prompts/agent.system.tool.browser.md@@ -20,7 +20,7 @@ Workflow:- For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.- `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.- `navigate` reuses an existing `browser_id` and is preferred for serial browsing.-- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned path before reasoning visually.+- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually. When no `path` is requested, browser screenshots are ephemeral refs rather than conserved files.- Keep the tab set small; close pages after extracting what you need.`multi` is only a browser action: use `tool_name: "browser"` with `tool_args.action: "multi"`. Never use `tool_name: "multi"`.diff --git a/plugins/_browser/skills/browser-automation/SKILL.md b/plugins/_browser/skills/browser-automation/SKILL.mdindex 47c8e12dc..a31949e10 100644--- a/plugins/_browser/skills/browser-automation/SKILL.md+++ b/plugins/_browser/skills/browser-automation/SKILL.md@@ -29,10 +29,10 @@ In host mode, page content and screenshots may be blocked by host-content policyScreenshots are explicit only; the browser does not automatically load images into model context.1. Call `browser` with `action: "screenshot"`.-2. Call `vision_load` with the returned path.+2. Call `vision_load` with the returned `vision_load.tool_args.paths` value.3. Reason from the latest loaded screenshot.-Screenshot args include `quality`, `full_page`, and optional `path`. PNG is used when `path` ends with `.png`; otherwise JPEG is used.+Screenshot args include `quality`, `full_page`, and optional `path`. Without `path`, the screenshot is an ephemeral ref consumed by `vision_load`; with `path`, PNG is used when `path` ends with `.png`, otherwise JPEG is used.## Forms And Filesdiff --git a/plugins/_browser/skills/browser-form-workflows/SKILL.md b/plugins/_browser/skills/browser-form-workflows/SKILL.mdindex bc2197489..aae826cfb 100644--- a/plugins/_browser/skills/browser-form-workflows/SKILL.md+++ b/plugins/_browser/skills/browser-form-workflows/SKILL.md@@ -11,7 +11,7 @@ Start with `browser:content` to capture current refs, then use `browser:detail`Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.-Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history.+Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history; no-path screenshots return ephemeral refs for `vision_load`.Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.pyindex b0bad5476..10220f028 100644--- a/plugins/_browser/tools/browser.py+++ b/plugins/_browser/tools/browser.py@@ -89,8 +89,6 @@ class Browser(Tool):if action == "open":result = await runtime.call("open", url or "")elif action == "screenshot":- if not path:- path = self._history_screenshot_path(action)result = await runtime.call("screenshot_file",browser_id,@@ -428,53 +426,65 @@ class Browser(Tool):returnscreenshot = result if action == "screenshot" and isinstance(result, dict) else None- if not self._screenshot_has_path(screenshot):+ if not self._screenshot_has_reference(screenshot):target_browser_id = self._browser_id_from_result(result) or requested_browser_id- output_path = self._history_screenshot_path(action)- if not output_path:- returntry:screenshot = await runtime.call("screenshot_file",target_browser_id,quality=HISTORY_SCREENSHOT_QUALITY,full_page=False,- path=output_path,+ path="",)except Exception as exc:PrintStyle.debug("Browser history screenshot capture failed:",f"browser_id={target_browser_id}",f"quality={HISTORY_SCREENSHOT_QUALITY}",- f"path={output_path}",f"error={exc}",)return- if not self._screenshot_has_path(screenshot):+ if not self._screenshot_has_reference(screenshot):return- local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))- if not local_path:- return- uri = f"img://{local_path}&t={time.time()}"+ a0_path = str(screenshot.get("a0_path") or "").strip()+ local_path = str(screenshot.get("path") or (files.fix_dev_path(a0_path) if a0_path else ""))state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}- self.log.update(- Screenshot=uri,- browser_snapshot={- "uri": uri,- "path": local_path,- "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),- "mime": screenshot.get("mime") or "image/jpeg",- "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,- "context_id": screenshot.get("context_id") or state.get("context_id") or "",- },- )+ chat_context_id = self._agent_context_id()+ browser_context_id = str(screenshot.get("context_id") or state.get("context_id") or "").strip()+ snapshot = {+ "mime": screenshot.get("mime") or "image/jpeg",+ "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,+ "context_id": chat_context_id or browser_context_id,+ "browser_context_id": browser_context_id,+ }+ update_payload: dict[str, Any] = {"browser_snapshot": snapshot}+ if local_path:+ uri = f"img://{local_path}&t={time.time()}"+ snapshot.update(+ {+ "uri": uri,+ "path": local_path,+ "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),+ "ephemeral": False,+ }+ )+ update_payload["Screenshot"] = uri+ else:+ ephemeral_ref = self._screenshot_ephemeral_ref(screenshot)+ snapshot.update(+ {+ "ephemeral": bool(ephemeral_ref),+ "ephemeral_ref": ephemeral_ref,+ }+ )+ self.log.update(**update_payload)def _history_screenshot_path(self, action: str) -> str:if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):return ""- context_id = str(getattr(self.agent.context, "id", "") or "").strip()+ context_id = self._agent_context_id()if not context_id:return ""from helpers import persist_chat@@ -521,6 +531,31 @@ class Browser(Tool):def _screenshot_has_path(screenshot: Any) -> bool:return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))+ @classmethod+ def _screenshot_has_reference(cls, screenshot: Any) -> bool:+ return cls._screenshot_has_path(screenshot) or bool(cls._screenshot_ephemeral_ref(screenshot))++ @staticmethod+ def _screenshot_ephemeral_ref(screenshot: Any) -> str:+ if not isinstance(screenshot, dict):+ return ""+ ref = str(screenshot.get("ephemeral_ref") or "").strip()+ if ref:+ return ref+ vision_load = screenshot.get("vision_load")+ if isinstance(vision_load, dict):+ tool_args = vision_load.get("tool_args")+ if isinstance(tool_args, dict):+ paths = tool_args.get("paths")+ if isinstance(paths, list) and paths:+ first = str(paths[0] or "").strip()+ if first.startswith("a0-ephemeral-image://"):+ return first+ return ""++ def _agent_context_id(self) -> str:+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()+@staticmethoddef _format_result(action: str, result: Any) -> str:if action == "content" and isinstance(result, dict):diff --git a/prompts/agent.system.tools_vision.md b/prompts/agent.system.tools_vision.mdindex 7611245a4..ee50376ad 100644--- a/prompts/agent.system.tools_vision.md+++ b/prompts/agent.system.tools_vision.md@@ -2,7 +2,7 @@### vision_loadload images into the model for visual reasoning-args: `paths` list of absolute image paths+args: `paths` list of absolute image paths or tool-returned ephemeral image refsrules:- load all relevant images in one call when comparing screenshots or pages- use when the task depends on screenshots, diagrams, scanned documents, charts, or photosdiff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex 529d9964f..8d6bf8f13 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -79,6 +79,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_confdef anyio_backend():return "asyncio"+from helpers import ephemeral_imagesfrom helpers.errors import RepairableExceptionfrom plugins._browser.helpers.config import (build_browser_launch_config,@@ -1808,12 +1809,10 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p},}if method == "screenshot_file":- Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)- Path(kwargs["path"]).write_bytes(b"jpeg")return {"browser_id": args[0],- "path": kwargs["path"],- "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",+ "ephemeral": True,+ "ephemeral_ref": "a0-ephemeral-image://fake","mime": "image/jpeg","state": {"id": args[0], "context_id": "browser-context"},}@@ -1861,11 +1860,13 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_passert calls[1][1] == (1,)assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITYassert calls[1][2]["full_page"] is False- assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"- assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"- assert log.updates[-1]["Screenshot"].startswith("img://")+ assert calls[1][2]["path"] == ""+ assert "Screenshot" not in log.updates[-1]assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1- assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"+ assert log.updates[-1]["browser_snapshot"]["context_id"] == "chat"+ assert log.updates[-1]["browser_snapshot"]["browser_context_id"] == "browser-context"+ assert log.updates[-1]["browser_snapshot"]["ephemeral"] is True+ assert log.updates[-1]["browser_snapshot"]["ephemeral_ref"] == "a0-ephemeral-image://fake"@pytest.mark.anyio@@ -2393,7 +2394,7 @@ async def test_browser_runtime_remounts_initial_changed_viewport():@pytest.mark.anyio-async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch, tmp_path):+async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeypatch, tmp_path):screenshot_calls = []def fake_get_abs_path(*parts):@@ -2411,8 +2412,9 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatchasync def screenshot(self, **kwargs):screenshot_calls.append(kwargs)- Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)- Path(kwargs["path"]).write_bytes(b"image-bytes")+ if kwargs.get("path"):+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)+ Path(kwargs["path"]).write_bytes(b"image-bytes")return b"image-bytes"async def title(self):@@ -2427,21 +2429,23 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatchresult = await core.screenshot_file(5, quality=500)- path = Path(result["path"])- assert path.exists()- assert path.parent == tmp_path / "tmp" / "browser" / "screenshots" / "ctx_id"- assert path.name.startswith("browser-5-")- assert path.suffix == ".jpg"- assert result["a0_path"].startswith("/a0/tmp/browser/screenshots/ctx_id/browser-5-")+ assert "path" not in result+ assert "a0_path" not in result+ assert result["context_id"] == "ctx/id"assert result["mime"] == "image/jpeg"+ assert result["ephemeral"] is True+ assert result["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)assert result["vision_load"] == {"tool_name": "vision_load",- "tool_args": {"paths": [result["path"]]},+ "tool_args": {"paths": [result["ephemeral_ref"]]},}assert "image" not in result+ assert not list((tmp_path / "tmp" / "browser" / "screenshots").rglob("*.jpg"))assert screenshot_calls[-1]["type"] == "jpeg"assert screenshot_calls[-1]["quality"] == 95assert screenshot_calls[-1]["full_page"] is False+ assert "path" not in screenshot_calls[-1]+ assert ephemeral_images.consume_image(result["ephemeral_ref"], context_id="ctx/id").data_url == "data:image/jpeg;base64,aW1hZ2UtYnl0ZXM="png_path = tmp_path / "custom.png"png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))@@ -2455,6 +2459,51 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch}+async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):+ import tools.vision_load as vision_load_module++ monkeypatch.setattr(+ vision_load_module.plugins,+ "get_plugin_config",+ lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},+ )++ tool_results = []+ messages = []+ updates = []+ agent = SimpleNamespace(+ context=SimpleNamespace(id="ctx-vision"),+ agent_name="Agent 0",+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),+ hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),+ )+ ref = ephemeral_images.put_image(+ context_id="ctx-vision",+ mime="image/jpeg",+ data=SMALL_JPEG_10X10,+ name="browser-shot.jpg",+ )+ tool = vision_load_module.VisionLoad(+ agent=agent,+ name="vision_load",+ method=None,+ args={"paths": [ref]},+ message="",+ loop_data=None,+ )+ tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))++ response = await tool.execute(paths=[ref])+ await tool.after_execution(response)++ assert ephemeral_images.get_image(ref, context_id="ctx-vision") is None+ assert tool.loaded_paths == ["browser-shot.jpg"]+ raw_message = messages[0][1]["content"]+ assert raw_message.raw_content[0]["image_url"]["url"] == f"data:image/jpeg;base64,{SMALL_JPEG_10X10}"+ assert updates[-1]["result"] == "1 images loaded, 0 skipped"++@pytest.mark.anyioasync def test_browser_runtime_ref_point_resolution_applies_offsets():eval_payloads = []diff --git a/tests/test_host_browser_connector.py b/tests/test_host_browser_connector.pyindex 572563ea1..398ce38b5 100644--- a/tests/test_host_browser_connector.py+++ b/tests/test_host_browser_connector.py@@ -12,6 +12,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]if str(PROJECT_ROOT) not in sys.path:sys.path.insert(0, str(PROJECT_ROOT))+from helpers import ephemeral_imagesfrom plugins._a0_connector.helpers import ws_runtimefrom plugins._browser.helpers.connector_runtime import (ConnectorBrowserRuntime,@@ -329,19 +330,7 @@ def test_connector_runtime_adds_docker_recovery_to_host_errors():assert "/browser container" in message-def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tmp_path):- import plugins._browser.helpers.connector_runtime as connector_runtime_module-- monkeypatch.setattr(- connector_runtime_module.files,- "get_abs_path",- lambda *parts: str(tmp_path.joinpath(*parts)),- )- monkeypatch.setattr(- connector_runtime_module.files,- "normalize_a0_path",- lambda path: "/a0/" + str(path).lstrip("/"),- )+def test_host_browser_artifacts_become_context_scoped_ephemeral_refs(tmp_path):runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))result = runtime._materialize_artifact(@@ -363,19 +352,19 @@ def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tminner = result[0]["result"]assert "artifact" not in inner- assert inner["path"].endswith("shot.jpg")- assert Path(inner["path"]).read_bytes() == b"fake"- assert inner["vision_load"]["tool_args"]["paths"] == [inner["path"]]+ assert "path" not in inner+ assert "a0_path" not in inner+ assert inner["context_id"] == "ctx-host"+ assert inner["ephemeral"] is True+ assert inner["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)+ assert inner["vision_load"]["tool_args"]["paths"] == [inner["ephemeral_ref"]]+ assert ephemeral_images.consume_image(inner["ephemeral_ref"], context_id="ctx-host").data_url == "data:image/jpeg;base64,ZmFrZQ=="+ assert not list(tmp_path.rglob("shot.jpg"))def test_host_browser_artifact_materialization_rejects_oversized_payload(monkeypatch, tmp_path):import plugins._browser.helpers.connector_runtime as connector_runtime_module- monkeypatch.setattr(- connector_runtime_module.files,- "get_abs_path",- lambda *parts: str(tmp_path.joinpath(*parts)),- )monkeypatch.setattr(connector_runtime_module, "MAX_ARTIFACT_SIZE_BYTES", 2)runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))diff --git a/tools/vision_load.py b/tools/vision_load.pyindex 75534f1af..90f2133e6 100644--- a/tools/vision_load.py+++ b/tools/vision_load.py@@ -1,6 +1,6 @@from helpers.print_style import PrintStylefrom helpers.tool import Tool, Response-from helpers import runtime, files, plugins+from helpers import runtime, files, plugins, ephemeral_imagesfrom mimetypes import guess_typefrom helpers import history@@ -16,18 +16,43 @@ class VisionLoad(Tool):self.skipped_paths: list[str] = []max_embeds = self._get_max_embeds()- limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]- self.skipped_paths = paths[:-max_embeds] if max_embeds > 0 and len(paths) > max_embeds else []+ requested = [+ (str(path or "").strip(), self._display_input_path(str(path or "").strip(), idx + 1))+ for idx, path in enumerate(paths)+ ]+ limited_paths = requested if max_embeds <= 0 else requested[-max_embeds:]+ self.skipped_paths = (+ [display for _, display in requested[:-max_embeds]]+ if max_embeds > 0 and len(requested) > max_embeds+ else []+ )- for path in limited_paths:+ for path, display_path in limited_paths:+ if not path:+ continue+ if ephemeral_images.is_ref(path):+ image = ephemeral_images.consume_image(+ path,+ context_id=self._context_id(),+ )+ if image is None:+ continue+ display = image.display_name or display_path+ self.images_dict[display] = image.data_url+ self.loaded_paths.append(display)+ continue+ if self._is_data_image_url(path):+ self.images_dict[display_path] = path+ self.loaded_paths.append(display_path)+ continueif not await runtime.call_development_function(files.exists, str(path)):continueif path not in self.images_dict:mime_type, _ = guess_type(str(path))if mime_type and mime_type.startswith("image/"):- self.images_dict[path] = str(path)- self.loaded_paths.append(path)+ self.images_dict[display_path] = str(path)+ self.loaded_paths.append(display_path)return Response(message="dummy", break_loop=False)@@ -37,6 +62,23 @@ class VisionLoad(Tool):max_embeds = chat_cfg.get("max_embeds", 10)return int(max_embeds or 0)+ def _context_id(self) -> str:+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()++ @staticmethod+ def _is_data_image_url(value: str) -> bool:+ normalized = str(value or "").strip().lower()+ return normalized.startswith("data:image/") and ";base64," in normalized++ @classmethod+ def _display_input_path(cls, value: str, index: int) -> str:+ if ephemeral_images.is_ref(value):+ return ephemeral_images.display_ref(value)+ if cls._is_data_image_url(value):+ prefix = value.split(",", 1)[0]+ return f"{prefix},<ephemeral-image-{index}>"+ return value+async def after_execution(self, response: Response, **kwargs):# build image data messages for LLMs, or error message