Agent Zero AI framework
Persist browser history screenshots
Alessandro44d5e1ccf724
diff --git a/.gitignore b/.gitignoreindex 75cc842de..44e916417 100644--- a/.gitignore+++ b/.gitignore@@ -50,3 +50,4 @@ agent_history.gif.agent/**.claude/**+.playwright-cli/diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.jsindex 2b227e45c..e15151905 100644--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js@@ -14,6 +14,7 @@ import {} from "/js/messages.js";const BROWSER_SCREENSHOT_KVP_KEY = "Screenshot";+const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";const BROWSER_SCREENSHOT_STYLE_ID = "a0-browser-screenshot-kvp-style";const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;const PREVIEW_REFRESH_MS = 2500;@@ -62,7 +63,14 @@ function normalizeBrowserAction(kvps = {}) {return String(kvps.action || "").trim().toLowerCase().replace("-", "_");}+function browserSnapshotMeta(kvps = {}) {+ return kvps?.[BROWSER_SNAPSHOT_META_KEY] && typeof kvps[BROWSER_SNAPSHOT_META_KEY] === "object"+ ? kvps[BROWSER_SNAPSHOT_META_KEY]+ : {};+}+function browserIdFromResult(result = {}, kvps = {}) {+ const snapshotMeta = browserSnapshotMeta(kvps);const browsers = Array.isArray(result.browsers) ? result.browsers : [];const lastInteractedId = result.last_interacted_browser_id;const listedBrowser = lastInteractedId@@ -74,12 +82,14 @@ function browserIdFromResult(result = {}, kvps = {}) {|| result.state?.id|| result.last_interacted_browser_id|| listedBrowser?.id+ || snapshotMeta.browser_id|| kvps.browser_id|| null);}function browserContextIdFromResult(result = {}, kvps = {}) {+ const snapshotMeta = browserSnapshotMeta(kvps);const browserId = browserIdFromResult(result, kvps);const browsers = Array.isArray(result.browsers) ? result.browsers : [];const listedBrowser = browserId@@ -89,6 +99,7 @@ function browserContextIdFromResult(result = {}, kvps = {}) {result.context_id|| result.state?.context_id|| listedBrowser?.context_id+ || snapshotMeta.context_id|| kvps.context_id|| kvps.contextId|| null@@ -185,6 +196,16 @@ function snapshotToObjectUrl(snapshot) {return URL.createObjectURL(new Blob(chunks, { type: snapshot.mime || "image/jpeg" }));}+function staticScreenshotUri(kvps = {}) {+ const metaUri = browserSnapshotMeta(kvps).uri;+ const value = kvps?.[BROWSER_SCREENSHOT_KVP_KEY] || metaUri || "";+ return typeof value === "string" && value.startsWith("img://") ? value : "";+}++function staticScreenshotSrc(uri = "") {+ return uri ? uri.replace("img://", "/api/image_get?path=") : "";+}+function releaseLiveScreenshotFrame(viewerId) {const release = liveScreenshotFrames.get(viewerId);if (!release) return;@@ -299,7 +320,7 @@ function ensureBrowserScreenshotStyles() {document.head.appendChild(style);}-function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {+function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label, staticUri = "") {if (!kvpsTable || typeof resolveBrowserPayload !== "function") return;ensureBrowserScreenshotStyles();@@ -334,11 +355,17 @@ function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {cell.textContent = "";cell.appendChild(button);- cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(- button,- button.querySelector(".browser-screenshot-kvp-image"),- resolveBrowserPayload,- );+ const image = button.querySelector(".browser-screenshot-kvp-image");+ if (staticUri) {+ image.src = staticScreenshotSrc(staticUri);+ button.classList.add("has-frame");+ cell.__browserScreenshotCleanup = () => {+ image.removeAttribute("src");+ button.classList.remove("has-frame");+ };+ return;+ }+ cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(button, image, resolveBrowserPayload);}function startBrowserScreenshotPreview(button, image, resolveBrowserPayload) {@@ -447,6 +474,7 @@ function drawBrowserTool({].filter(Boolean);const contentText = String(content ?? "");const browserResult = parseBrowserResult(contentText);+ const screenshotUri = staticScreenshotUri(kvps);const browserId = browserIdFromResult(browserResult, kvps);const browserCanvasPayload = buildBrowserCanvasPayload(browserResult, kvps);const browserPreviewLabel = browserId@@ -455,6 +483,10 @@ function drawBrowserTool({if (shouldRenderBrowserScreenshotKvp(browserResult, kvps)) {displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = "";}+ if (screenshotUri) {+ displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = screenshotUri;+ }+ delete displayKvps[BROWSER_SNAPSHOT_META_KEY];const browserButton = createActionButton("visibility","Browser",@@ -503,6 +535,7 @@ function drawBrowserTool({|| buildBrowserCanvasPayload({}, kvps)),browserPreviewLabel,+ screenshotUri,);syncOpenBrowserCanvas(args, browserResult);return result;diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.pyindex 6cc6e7c57..1310da7e2 100644--- a/plugins/_browser/tools/browser.py+++ b/plugins/_browser/tools/browser.py@@ -1,12 +1,21 @@from __future__ import annotationsimport json+import time+import uuid+from pathlib import Pathfrom typing import Any+from helpers import files+from helpers.print_style import PrintStylefrom helpers.tool import Response, Toolfrom plugins._browser.helpers.selector import get_tool_runtime+HISTORY_SCREENSHOT_QUALITY = 62+HISTORY_SCREENSHOT_ACTION_DENYLIST = {"close", "close_all"}++async def get_runtime(context_id: str, create: bool = True, agent: Any | None = None):if agent is not None:return await get_tool_runtime(agent)@@ -78,6 +87,8 @@ class Browser(Tool):if action == "open":result = await runtime.call("open", url or "")elif action == "screenshot":+ if not path:+ path = self._history_screenshot_path(action)result = await runtime.call("screenshot_file",browser_id,@@ -258,6 +269,7 @@ class Browser(Tool):message=f"Unknown browser action: {action}",break_loop=False,)+ await self._record_history_screenshot(runtime, action, result, browser_id)except Exception as exc:return Response(message=f"Browser {action} failed: {exc}", break_loop=False)@@ -286,6 +298,112 @@ class Browser(Tool):return {"selector": selector}return None+ async def _record_history_screenshot(+ self,+ runtime: Any,+ action: str,+ result: Any,+ requested_browser_id: int | str | None = None,+ ) -> None:+ if not getattr(self, "log", None):+ return+ if action in HISTORY_SCREENSHOT_ACTION_DENYLIST:+ return++ screenshot = result if action == "screenshot" and isinstance(result, dict) else None+ if not self._screenshot_has_path(screenshot):+ target_browser_id = self._browser_id_from_result(result) or requested_browser_id+ output_path = self._history_screenshot_path(action)+ if not output_path:+ return+ try:+ screenshot = await runtime.call(+ "screenshot_file",+ target_browser_id,+ quality=HISTORY_SCREENSHOT_QUALITY,+ full_page=False,+ path=output_path,+ )+ except Exception as exc:+ PrintStyle.debug(+ "Browser history screenshot capture failed:",+ f"browser_id={target_browser_id}",+ f"quality={HISTORY_SCREENSHOT_QUALITY}",+ f"path={output_path}",+ f"error={exc}",+ )+ return++ if not self._screenshot_has_path(screenshot):+ return++ local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))+ if not local_path:+ return+ uri = f"img://{local_path}&t={time.time()}"+ state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}+ self.log.update(+ Screenshot=uri,+ browser_snapshot={+ "uri": uri,+ "path": local_path,+ "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),+ "mime": screenshot.get("mime") or "image/jpeg",+ "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,+ "context_id": screenshot.get("context_id") or state.get("context_id") or "",+ },+ )++ def _history_screenshot_path(self, action: str) -> str:+ if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):+ return ""+ context_id = str(getattr(self.agent.context, "id", "") or "").strip()+ if not context_id:+ return ""+ from helpers import persist_chat++ token = str(getattr(getattr(self, "log", None), "id", "") or uuid.uuid4())+ safe_action = files.safe_file_name(str(action or "browser"))+ safe_token = files.safe_file_name(token)+ timestamp = time.strftime("%Y%m%d-%H%M%S")+ return str(+ Path(persist_chat.get_chat_folder_path(context_id))+ / "browser"+ / "screenshots"+ / f"{timestamp}-{safe_action}-{safe_token}.jpg"+ )++ @staticmethod+ def _browser_id_from_result(result: Any) -> Any:+ if not isinstance(result, dict):+ return None+ browsers = result.get("browsers") if isinstance(result.get("browsers"), list) else []+ last_interacted_id = result.get("last_interacted_browser_id")+ listed_browser = None+ if last_interacted_id is not None:+ listed_browser = next(+ (+ browser+ for browser in browsers+ if isinstance(browser, dict) and str(browser.get("id")) == str(last_interacted_id)+ ),+ None,+ )+ if listed_browser is None and browsers:+ listed_browser = next((browser for browser in browsers if isinstance(browser, dict)), None)+ state = result.get("state") if isinstance(result.get("state"), dict) else {}+ return (+ result.get("id")+ or result.get("browser_id")+ or state.get("id")+ or last_interacted_id+ or (listed_browser or {}).get("id")+ )++ @staticmethod+ def _screenshot_has_path(screenshot: Any) -> bool:+ return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))+@staticmethoddef _format_result(action: str, result: Any) -> str:if action == "content" and isinstance(result, dict):diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex 7db446ac4..d2e09de17 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -1057,6 +1057,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():browser_store = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(encoding="utf-8")+ browser_tool_handler = (+ PROJECT_ROOT / "plugins" / "_browser" / "extensions" / "webui" / "get_tool_message_handler" / "browser-tool-handler.js"+ ).read_text(encoding="utf-8")assert 'runtime.call("screenshot"' in ws_browserassert "SCREENCAST_QUALITY = 92" in ws_browser@@ -1096,6 +1099,10 @@ def test_browser_viewer_uses_cdp_screencast_transport():assert "this.applySnapshot(data.snapshot);" in browser_storeassert "else if (!data.state)" in browser_storeassert '"snapshot": snapshot' in ws_browser+ assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler+ assert "staticScreenshotUri(kvps)" in browser_tool_handler+ assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler+ assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handlerassert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browserassert '"frame_source": "state"' in ws_browserassert '"frame_source"] = "screencast"' in ws_browser@@ -1667,6 +1674,84 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):]+async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_path):+ calls = []++ class FakeRuntime:+ async def call(self, method, *args, **kwargs):+ calls.append((method, args, kwargs))+ if method == "open":+ return {+ "id": 1,+ "state": {+ "id": 1,+ "context_id": "browser-context",+ "currentUrl": "https://example.com/",+ "title": "Example Domain",+ },+ }+ if method == "screenshot_file":+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)+ Path(kwargs["path"]).write_bytes(b"jpeg")+ return {+ "browser_id": args[0],+ "path": kwargs["path"],+ "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",+ "mime": "image/jpeg",+ "state": {"id": args[0], "context_id": "browser-context"},+ }+ raise AssertionError(method)++ async def fake_get_runtime(context_id, create=True, agent=None):+ del create, agent+ assert context_id == "chat"+ return FakeRuntime()++ class FakeLog:+ id = "tool-log-id"++ def __init__(self):+ self.updates = []++ def update(self, **kwargs):+ self.updates.append(kwargs)++ monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)+ monkeypatch.setitem(+ sys.modules,+ "helpers.persist_chat",+ SimpleNamespace(+ get_chat_folder_path=lambda context_id: str(tmp_path / "usr" / "chats" / context_id)+ ),+ )++ log = FakeLog()+ tool = browser_tool_module.Browser(+ agent=SimpleNamespace(context=SimpleNamespace(id="chat")),+ name="browser",+ method=None,+ args={"action": "open", "url": "https://example.com"},+ message="",+ loop_data=None,+ )+ tool.log = log++ response = await tool.execute(action="open", url="https://example.com")++ assert response.break_loop is False+ assert calls[0] == ("open", ("https://example.com",), {})+ assert calls[1][0] == "screenshot_file"+ assert calls[1][1] == (1,)+ assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITY+ assert calls[1][2]["full_page"] is False+ assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"+ assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"+ assert log.updates[-1]["Screenshot"].startswith("img://")+ assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1+ assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"++@pytest.mark.anyioasync def test_browser_multi_dispatch_accepts_v1_actions():calls = []