Skip to content

Agent Zero AI framework

Persist browser history screenshots

Alessandro44d5e1ccf724
diff --git a/.gitignore b/.gitignore
index 75cc842de..44e916417 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,3 +50,4 @@ agent_history.gif
.agent/**
.claude/**
+.playwright-cli/
diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
index 2b227e45c..e15151905 100644
--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
@@ -14,6 +14,7 @@ import {
} from "/js/messages.js";
const BROWSER_SCREENSHOT_KVP_KEY = "Screenshot";
+const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";
const BROWSER_SCREENSHOT_STYLE_ID = "a0-browser-screenshot-kvp-style";
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
const PREVIEW_REFRESH_MS = 2500;
@@ -62,7 +63,14 @@ function normalizeBrowserAction(kvps = {}) {
return String(kvps.action || "").trim().toLowerCase().replace("-", "_");
}
+function browserSnapshotMeta(kvps = {}) {
+ return kvps?.[BROWSER_SNAPSHOT_META_KEY] && typeof kvps[BROWSER_SNAPSHOT_META_KEY] === "object"
+ ? kvps[BROWSER_SNAPSHOT_META_KEY]
+ : {};
+}
+
function browserIdFromResult(result = {}, kvps = {}) {
+ const snapshotMeta = browserSnapshotMeta(kvps);
const browsers = Array.isArray(result.browsers) ? result.browsers : [];
const lastInteractedId = result.last_interacted_browser_id;
const listedBrowser = lastInteractedId
@@ -74,12 +82,14 @@ function browserIdFromResult(result = {}, kvps = {}) {
|| result.state?.id
|| result.last_interacted_browser_id
|| listedBrowser?.id
+ || snapshotMeta.browser_id
|| kvps.browser_id
|| null
);
}
function browserContextIdFromResult(result = {}, kvps = {}) {
+ const snapshotMeta = browserSnapshotMeta(kvps);
const browserId = browserIdFromResult(result, kvps);
const browsers = Array.isArray(result.browsers) ? result.browsers : [];
const listedBrowser = browserId
@@ -89,6 +99,7 @@ function browserContextIdFromResult(result = {}, kvps = {}) {
result.context_id
|| result.state?.context_id
|| listedBrowser?.context_id
+ || snapshotMeta.context_id
|| kvps.context_id
|| kvps.contextId
|| null
@@ -185,6 +196,16 @@ function snapshotToObjectUrl(snapshot) {
return URL.createObjectURL(new Blob(chunks, { type: snapshot.mime || "image/jpeg" }));
}
+function staticScreenshotUri(kvps = {}) {
+ const metaUri = browserSnapshotMeta(kvps).uri;
+ const value = kvps?.[BROWSER_SCREENSHOT_KVP_KEY] || metaUri || "";
+ return typeof value === "string" && value.startsWith("img://") ? value : "";
+}
+
+function staticScreenshotSrc(uri = "") {
+ return uri ? uri.replace("img://", "/api/image_get?path=") : "";
+}
+
function releaseLiveScreenshotFrame(viewerId) {
const release = liveScreenshotFrames.get(viewerId);
if (!release) return;
@@ -299,7 +320,7 @@ function ensureBrowserScreenshotStyles() {
document.head.appendChild(style);
}
-function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {
+function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label, staticUri = "") {
if (!kvpsTable || typeof resolveBrowserPayload !== "function") return;
ensureBrowserScreenshotStyles();
@@ -334,11 +355,17 @@ function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {
cell.textContent = "";
cell.appendChild(button);
- cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(
- button,
- button.querySelector(".browser-screenshot-kvp-image"),
- resolveBrowserPayload,
- );
+ const image = button.querySelector(".browser-screenshot-kvp-image");
+ if (staticUri) {
+ image.src = staticScreenshotSrc(staticUri);
+ button.classList.add("has-frame");
+ cell.__browserScreenshotCleanup = () => {
+ image.removeAttribute("src");
+ button.classList.remove("has-frame");
+ };
+ return;
+ }
+ cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(button, image, resolveBrowserPayload);
}
function startBrowserScreenshotPreview(button, image, resolveBrowserPayload) {
@@ -447,6 +474,7 @@ function drawBrowserTool({
].filter(Boolean);
const contentText = String(content ?? "");
const browserResult = parseBrowserResult(contentText);
+ const screenshotUri = staticScreenshotUri(kvps);
const browserId = browserIdFromResult(browserResult, kvps);
const browserCanvasPayload = buildBrowserCanvasPayload(browserResult, kvps);
const browserPreviewLabel = browserId
@@ -455,6 +483,10 @@ function drawBrowserTool({
if (shouldRenderBrowserScreenshotKvp(browserResult, kvps)) {
displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = "";
}
+ if (screenshotUri) {
+ displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = screenshotUri;
+ }
+ delete displayKvps[BROWSER_SNAPSHOT_META_KEY];
const browserButton = createActionButton(
"visibility",
"Browser",
@@ -503,6 +535,7 @@ function drawBrowserTool({
|| buildBrowserCanvasPayload({}, kvps)
),
browserPreviewLabel,
+ screenshotUri,
);
syncOpenBrowserCanvas(args, browserResult);
return result;
diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.py
index 6cc6e7c57..1310da7e2 100644
--- a/plugins/_browser/tools/browser.py
+++ b/plugins/_browser/tools/browser.py
@@ -1,12 +1,21 @@
from __future__ import annotations
import json
+import time
+import uuid
+from pathlib import Path
from typing import Any
+from helpers import files
+from helpers.print_style import PrintStyle
from helpers.tool import Response, Tool
from plugins._browser.helpers.selector import get_tool_runtime
+HISTORY_SCREENSHOT_QUALITY = 62
+HISTORY_SCREENSHOT_ACTION_DENYLIST = {"close", "close_all"}
+
+
async def get_runtime(context_id: str, create: bool = True, agent: Any | None = None):
if agent is not None:
return await get_tool_runtime(agent)
@@ -78,6 +87,8 @@ class Browser(Tool):
if action == "open":
result = await runtime.call("open", url or "")
elif action == "screenshot":
+ if not path:
+ path = self._history_screenshot_path(action)
result = await runtime.call(
"screenshot_file",
browser_id,
@@ -258,6 +269,7 @@ class Browser(Tool):
message=f"Unknown browser action: {action}",
break_loop=False,
)
+ await self._record_history_screenshot(runtime, action, result, browser_id)
except Exception as exc:
return Response(message=f"Browser {action} failed: {exc}", break_loop=False)
@@ -286,6 +298,112 @@ class Browser(Tool):
return {"selector": selector}
return None
+ async def _record_history_screenshot(
+ self,
+ runtime: Any,
+ action: str,
+ result: Any,
+ requested_browser_id: int | str | None = None,
+ ) -> None:
+ if not getattr(self, "log", None):
+ return
+ if action in HISTORY_SCREENSHOT_ACTION_DENYLIST:
+ return
+
+ screenshot = result if action == "screenshot" and isinstance(result, dict) else None
+ if not self._screenshot_has_path(screenshot):
+ target_browser_id = self._browser_id_from_result(result) or requested_browser_id
+ output_path = self._history_screenshot_path(action)
+ if not output_path:
+ return
+ try:
+ screenshot = await runtime.call(
+ "screenshot_file",
+ target_browser_id,
+ quality=HISTORY_SCREENSHOT_QUALITY,
+ full_page=False,
+ path=output_path,
+ )
+ except Exception as exc:
+ PrintStyle.debug(
+ "Browser history screenshot capture failed:",
+ f"browser_id={target_browser_id}",
+ f"quality={HISTORY_SCREENSHOT_QUALITY}",
+ f"path={output_path}",
+ f"error={exc}",
+ )
+ return
+
+ if not self._screenshot_has_path(screenshot):
+ return
+
+ local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))
+ if not local_path:
+ return
+ uri = f"img://{local_path}&t={time.time()}"
+ state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}
+ self.log.update(
+ Screenshot=uri,
+ browser_snapshot={
+ "uri": uri,
+ "path": local_path,
+ "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
+ "mime": screenshot.get("mime") or "image/jpeg",
+ "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
+ "context_id": screenshot.get("context_id") or state.get("context_id") or "",
+ },
+ )
+
+ def _history_screenshot_path(self, action: str) -> str:
+ if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):
+ return ""
+ context_id = str(getattr(self.agent.context, "id", "") or "").strip()
+ if not context_id:
+ return ""
+ from helpers import persist_chat
+
+ token = str(getattr(getattr(self, "log", None), "id", "") or uuid.uuid4())
+ safe_action = files.safe_file_name(str(action or "browser"))
+ safe_token = files.safe_file_name(token)
+ timestamp = time.strftime("%Y%m%d-%H%M%S")
+ return str(
+ Path(persist_chat.get_chat_folder_path(context_id))
+ / "browser"
+ / "screenshots"
+ / f"{timestamp}-{safe_action}-{safe_token}.jpg"
+ )
+
+ @staticmethod
+ def _browser_id_from_result(result: Any) -> Any:
+ if not isinstance(result, dict):
+ return None
+ browsers = result.get("browsers") if isinstance(result.get("browsers"), list) else []
+ last_interacted_id = result.get("last_interacted_browser_id")
+ listed_browser = None
+ if last_interacted_id is not None:
+ listed_browser = next(
+ (
+ browser
+ for browser in browsers
+ if isinstance(browser, dict) and str(browser.get("id")) == str(last_interacted_id)
+ ),
+ None,
+ )
+ if listed_browser is None and browsers:
+ listed_browser = next((browser for browser in browsers if isinstance(browser, dict)), None)
+ state = result.get("state") if isinstance(result.get("state"), dict) else {}
+ return (
+ result.get("id")
+ or result.get("browser_id")
+ or state.get("id")
+ or last_interacted_id
+ or (listed_browser or {}).get("id")
+ )
+
+ @staticmethod
+ def _screenshot_has_path(screenshot: Any) -> bool:
+ return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))
+
@staticmethod
def _format_result(action: str, result: Any) -> str:
if action == "content" and isinstance(result, dict):
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 7db446ac4..d2e09de17 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1057,6 +1057,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():
browser_store = (
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
).read_text(encoding="utf-8")
+ browser_tool_handler = (
+ PROJECT_ROOT / "plugins" / "_browser" / "extensions" / "webui" / "get_tool_message_handler" / "browser-tool-handler.js"
+ ).read_text(encoding="utf-8")
assert 'runtime.call("screenshot"' in ws_browser
assert "SCREENCAST_QUALITY = 92" in ws_browser
@@ -1096,6 +1099,10 @@ def test_browser_viewer_uses_cdp_screencast_transport():
assert "this.applySnapshot(data.snapshot);" in browser_store
assert "else if (!data.state)" in browser_store
assert '"snapshot": snapshot' in ws_browser
+ assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler
+ assert "staticScreenshotUri(kvps)" in browser_tool_handler
+ assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler
+ assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
assert '"frame_source": "state"' in ws_browser
assert '"frame_source"] = "screencast"' in ws_browser
@@ -1667,6 +1674,84 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):
]
+async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_path):
+ calls = []
+
+ class FakeRuntime:
+ async def call(self, method, *args, **kwargs):
+ calls.append((method, args, kwargs))
+ if method == "open":
+ return {
+ "id": 1,
+ "state": {
+ "id": 1,
+ "context_id": "browser-context",
+ "currentUrl": "https://example.com/",
+ "title": "Example Domain",
+ },
+ }
+ if method == "screenshot_file":
+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
+ Path(kwargs["path"]).write_bytes(b"jpeg")
+ return {
+ "browser_id": args[0],
+ "path": kwargs["path"],
+ "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",
+ "mime": "image/jpeg",
+ "state": {"id": args[0], "context_id": "browser-context"},
+ }
+ raise AssertionError(method)
+
+ async def fake_get_runtime(context_id, create=True, agent=None):
+ del create, agent
+ assert context_id == "chat"
+ return FakeRuntime()
+
+ class FakeLog:
+ id = "tool-log-id"
+
+ def __init__(self):
+ self.updates = []
+
+ def update(self, **kwargs):
+ self.updates.append(kwargs)
+
+ monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)
+ monkeypatch.setitem(
+ sys.modules,
+ "helpers.persist_chat",
+ SimpleNamespace(
+ get_chat_folder_path=lambda context_id: str(tmp_path / "usr" / "chats" / context_id)
+ ),
+ )
+
+ log = FakeLog()
+ tool = browser_tool_module.Browser(
+ agent=SimpleNamespace(context=SimpleNamespace(id="chat")),
+ name="browser",
+ method=None,
+ args={"action": "open", "url": "https://example.com"},
+ message="",
+ loop_data=None,
+ )
+ tool.log = log
+
+ response = await tool.execute(action="open", url="https://example.com")
+
+ assert response.break_loop is False
+ assert calls[0] == ("open", ("https://example.com",), {})
+ assert calls[1][0] == "screenshot_file"
+ assert calls[1][1] == (1,)
+ assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITY
+ assert calls[1][2]["full_page"] is False
+ assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"
+ assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"
+ assert log.updates[-1]["Screenshot"].startswith("img://")
+ assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1
+ assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"
+
+
@pytest.mark.anyio
async def test_browser_multi_dispatch_accepts_v1_actions():
calls = []