Agent Zero AI framework
Stabilize Internal Docker Browser rendering
Alessandro23af25e52ace
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex be7092826..6c810ebc9 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -230,13 +230,18 @@ class WsBrowser(WsHandler):if command == "open":result = await runtime.call("open", data.get("url") or "")elif command == "navigate":- result = await runtime.call("navigate", browser_id, data.get("url") or "")+ result = await runtime.call(+ "navigate",+ browser_id,+ data.get("url") or "",+ wait_until="commit",+ )elif command == "back":- result = await runtime.call("back", browser_id)+ result = await runtime.call("back", browser_id, wait_until="commit")elif command == "forward":- result = await runtime.call("forward", browser_id)+ result = await runtime.call("forward", browser_id, wait_until="commit")elif command == "reload":- result = await runtime.call("reload", browser_id)+ result = await runtime.call("reload", browser_id, wait_until="commit")elif command == "close":result = await runtime.call("close_browser", browser_id)elif command == "list":@@ -717,18 +722,29 @@ class WsBrowser(WsHandler):def _frame_dimensions(metadata: Any) -> dict[str, int]:if not isinstance(metadata, dict):return {}- for width_key, height_key in (- ("expectedWidth", "expectedHeight"),- ("deviceWidth", "deviceHeight"),- ("jpegWidth", "jpegHeight"),- ):++ def dimensions(width_key: str, height_key: str) -> tuple[int, int] | None:try:width = int(metadata.get(width_key) or 0)height = int(metadata.get(height_key) or 0)except (TypeError, ValueError):- continue+ return Noneif width > 0 and height > 0:- return {"width": width, "height": height}+ return width, height+ return None++ expected = dimensions("expectedWidth", "expectedHeight")+ jpeg = dimensions("jpegWidth", "jpegHeight")+ if expected and jpeg:+ width_scale = jpeg[0] / expected[0]+ height_scale = jpeg[1] / expected[1]+ if abs(width_scale - height_scale) <= 0.01:+ return {"width": expected[0], "height": expected[1]}+ return {"width": jpeg[0], "height": jpeg[1]}++ for fallback in (jpeg, dimensions("deviceWidth", "deviceHeight"), expected):+ if fallback:+ return {"width": fallback[0], "height": fallback[1]}return {}async def _emit_viewer_state(diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 994c6f04e..ef32e9278 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -40,7 +40,6 @@ CHROME_SINGLETON_FILES = ("SingletonLock", "SingletonCookie", "SingletonSocket")SCREENCAST_MAX_WIDTH = 4096SCREENCAST_MAX_HEIGHT = 4096VIEWPORT_SIZE_TOLERANCE = 4-VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05CLIPBOARD_BRIDGE_SCRIPT = r"""(payload) => {const action = String(payload?.action || "").trim().toLowerCase();@@ -282,18 +281,6 @@ CLIPBOARD_BRIDGE_SCRIPT = r"""_SAFE_CONTEXT_RE = re.compile(r"[^a-zA-Z0-9_.-]+")-def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:- width = int(viewport["width"])- height = int(viewport["height"])- if width < 4096:- return {"width": width + 1, "height": height}- if width > 320:- return {"width": width - 1, "height": height}- if height < 4096:- return {"width": width, "height": height + 1}- return {"width": width, "height": height - 1}--def _safe_context_id(context_id: str) -> str:return _SAFE_CONTEXT_RE.sub("_", str(context_id or "default")).strip("._") or "default"@@ -344,7 +331,7 @@ class _BrowserScreencast:self._expected_height = heightwith contextlib.suppress(Exception):await self.session.send("Page.enable")- await self._apply_cdp_viewport_with_remount({"width": width, "height": height})+ await self._apply_cdp_viewport({"width": width, "height": height})await self.session.send("Page.startScreencast",{@@ -356,14 +343,6 @@ class _BrowserScreencast:},)- async def _apply_cdp_viewport_with_remount(self, viewport: dict[str, int]) -> None:- await self._apply_cdp_viewport(viewport)- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)- await self._apply_cdp_viewport(_nudged_viewport(viewport))- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)- await self._apply_cdp_viewport(viewport)- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)-async def _apply_cdp_viewport(self, viewport: dict[str, int]) -> None:width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))@@ -1247,38 +1226,59 @@ class _BrowserRuntimeCore:await self.ensure_started()return await self._state(self._resolve_browser_id(browser_id))- async def navigate(self, browser_id: int | str | None, url: str) -> dict[str, Any]:+ async def navigate(+ self,+ browser_id: int | str | None,+ url: str,+ *,+ wait_until: str = "domcontentloaded",+ ) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)- await self._goto(page, normalize_url(url))+ await self._goto(page, normalize_url(url), wait_until=wait_until)self._maybe_promote(resolved_id)return await self._state(resolved_id)- async def back(self, browser_id: int | str | None = None) -> dict[str, Any]:+ async def back(+ self,+ browser_id: int | str | None = None,+ *,+ wait_until: str = "domcontentloaded",+ ) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)- await page.go_back(wait_until="domcontentloaded", timeout=10000)- await self._settle(page)+ await page.go_back(wait_until=wait_until, timeout=10000)+ await self._settle(page, short=wait_until == "commit")self._maybe_promote(resolved_id)return await self._state(resolved_id)- async def forward(self, browser_id: int | str | None = None) -> dict[str, Any]:+ async def forward(+ self,+ browser_id: int | str | None = None,+ *,+ wait_until: str = "domcontentloaded",+ ) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)- await page.go_forward(wait_until="domcontentloaded", timeout=10000)- await self._settle(page)+ await page.go_forward(wait_until=wait_until, timeout=10000)+ await self._settle(page, short=wait_until == "commit")self._maybe_promote(resolved_id)return await self._state(resolved_id)- async def reload(self, browser_id: int | str | None = None) -> dict[str, Any]:+ async def reload(+ self,+ browser_id: int | str | None = None,+ *,+ wait_until: str = "domcontentloaded",+ ) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)- await page.reload(wait_until="domcontentloaded", timeout=15000)- await self._settle(page)+ await page.reload(wait_until=wait_until, timeout=15000)+ await self._settle(page, short=wait_until == "commit")self._maybe_promote(resolved_id)return await self._state(resolved_id)@@ -1748,33 +1748,16 @@ class _BrowserRuntimeCore:or abs(int(current_viewport.get("height") or 0) - viewport["height"])> VIEWPORT_SIZE_TOLERANCE)- should_remount_viewport = changed or restart_screencast- if should_remount_viewport:+ should_restart_screencast = changed or restart_screencast+ if should_restart_screencast:await self._stop_screencasts_for_browser(resolved_id)if changed:- await self._apply_viewport_with_remount(page, viewport)- elif restart_screencast:- await self._remount_viewport(page, viewport)- if should_remount_viewport:+ await page.set_viewport_size(viewport)+ if should_restart_screencast:await self._settle(page, short=True)self._maybe_promote(resolved_id)return {"state": await self._state(resolved_id), "viewport": viewport}- async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:- await page.set_viewport_size(viewport)- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)- await self._remount_viewport(page, viewport)-- async def _remount_viewport(self, page: Any, viewport: dict[str, int]) -> None:- nudged_viewport = self._nudged_viewport(viewport)- await page.set_viewport_size(nudged_viewport)- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)- await page.set_viewport_size(viewport)-- @staticmethod- def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:- return _nudged_viewport(viewport)-async def _point_for(self,page: Any,@@ -2223,17 +2206,23 @@ class _BrowserRuntimeCore:self._maybe_promote(resolved_id)return {"action": action or {}, "state": await self._state(resolved_id)}- async def _goto(self, page: Any, url: str) -> None:+ async def _goto(+ self,+ page: Any,+ url: str,+ *,+ wait_until: str = "domcontentloaded",+ ) -> None:from playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrortry:- await page.goto(url, wait_until="domcontentloaded", timeout=30000)+ await page.goto(url, wait_until=wait_until, timeout=30000)except PlaywrightTimeoutError:- PrintStyle.warning(f"Browser navigation timed out after DOM handoff: {url}")+ PrintStyle.warning(f"Browser navigation timed out waiting for {wait_until}: {url}")except PlaywrightError as exc:PrintStyle.warning(f"Browser navigation showed a native error page for {url}: {exc}")- await self._settle(page)+ await self._settle(page, short=wait_until == "commit")async def _settle(self, page: Any, short: bool = False) -> None:from playwright.async_api import Error as PlaywrightErrordiff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex 150d5bf2b..15ec5788d 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -19,6 +19,7 @@ websocket.addHandlers(["ws_webui"]);const EXTENSIONS_ROOT = "/a0/usr/_browser/extensions";const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;+const BROWSER_COMMAND_TIMEOUT_MS = 45000;const BROWSER_CONFIG_REFRESH_MS = 15000;const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";@@ -211,9 +212,6 @@ const model = {_surfaceHandoffTimer: null,_surfaceOpenedAt: 0,_surfaceOpenSequence: 0,- _canvasSurfaceReadySequence: 0,- _canvasFirstFrameAcceptedSequence: 0,- _canvasFirstFrameNudgeSequence: 0,_openPromise: null,_openSignature: "",_connectSequence: 0,@@ -751,10 +749,6 @@ const model = {} finally {if (this.isCurrentSurfaceOpen(surfaceSequence)) {this.loading = false;- if (this._mode === "canvas") {- this._canvasSurfaceReadySequence = surfaceSequence;- this.scheduleCanvasWidthNudgeAfterFirstFrame();- }}}},@@ -1298,11 +1292,9 @@ const model = {const viewport = this.currentViewportSize() || this._lastViewport;if (!this.frameMatchesViewport(dimensions, viewport)) {this.requestViewportSyncAfterRejectedFrame();- if (!this.shouldAcceptMismatchedFrame(dimensions)) {- bitmap?.close?.();- options?.cleanup?.();- return;- }+ bitmap?.close?.();+ options?.cleanup?.();+ return;}if (bitmap && this.paintFrameBitmap(bitmap)) {this.clearFrameSrc();@@ -1317,64 +1309,6 @@ const model = {this._lastFrameDimensions = dimensions;this._lastFrameAt = Date.now();options?.onAccepted?.();- this._canvasFirstFrameAcceptedSequence = surfaceSequence;- this.scheduleCanvasWidthNudgeAfterFirstFrame();- },-- shouldAcceptMismatchedFrame(dimensions = null) {- return Boolean(- dimensions?.width- && dimensions?.height- && (!this.hasFrame() || this._surfaceSwitching || this.isSwitchingBrowser())- );- },-- scheduleCanvasWidthNudgeAfterFirstFrame() {- const surfaceSequence = this._surfaceOpenSequence;- if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {- return;- }- if (this._canvasFirstFrameNudgeSequence === surfaceSequence) {- return;- }- if (- this._canvasSurfaceReadySequence !== surfaceSequence- || this._canvasFirstFrameAcceptedSequence !== surfaceSequence- ) {- return;- }- this._canvasFirstFrameNudgeSequence = surfaceSequence;-- void (async () => {- await nextAnimationFrame();- await nextAnimationFrame();- if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") {- return;- }- this.forceRightCanvasWidthNudge();- })();- },-- forceRightCanvasWidthNudge() {- const canvas = rightCanvasStore;- if (!canvas || canvas.isMobileMode || !canvas.isOpen || canvas.activeSurfaceId !== "browser") {- return;- }-- const currentWidth = Number(canvas.width || 0);- if (!Number.isFinite(currentWidth) || currentWidth <= 0) {- return;- }- const maxWidth = Number(canvas.maxWidth?.() || currentWidth);- const minWidth = Number(canvas.minWidth || 420);- const direction = currentWidth < maxWidth ? 1 : -1;- const nudgedWidth = currentWidth + direction;- if (nudgedWidth < minWidth || nudgedWidth > maxWidth || nudgedWidth === currentWidth) {- return;- }-- canvas.setWidth?.(nudgedWidth, { persist: false });- this.queueViewportSync(true);},frameMatchesViewport(dimensions = null, viewport = null) {@@ -1525,7 +1459,7 @@ const model = {viewer_transport: this.requestedViewerTransport(),command,},- { timeoutMs: 20000 },+ { timeoutMs: BROWSER_COMMAND_TIMEOUT_MS },);const data = firstOk(response);this.applyTabScope(data);diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex c0e4c50b5..262019baf 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -707,6 +707,7 @@ def test_browser_viewer_allows_slow_extension_startup():assert "const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;" in jsassert "const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;" in js+ assert "const BROWSER_COMMAND_TIMEOUT_MS = 45000;" in jsassert "? BROWSER_FIRST_INSTALL_TIMEOUT_MS" in jsassert ": BROWSER_SUBSCRIBE_TIMEOUT_MS" in jsassert "Installing Chromium for the first Browser run" in js@@ -796,32 +797,17 @@ def test_browser_canvas_surface_open_waits_for_visible_panel():assert "forceCanvasWidthNudgeAfterBrowserMount" not in js-def test_browser_canvas_nudges_width_after_first_accepted_frame():+def test_browser_canvas_does_not_resize_after_first_accepted_frame():js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(encoding="utf-8")- assert "_canvasSurfaceReadySequence" in js- assert "_canvasFirstFrameAcceptedSequence" in js- assert "_canvasFirstFrameNudgeSequence" in js- assert "scheduleCanvasWidthNudgeAfterFirstFrame()" in js- assert "this._canvasSurfaceReadySequence = surfaceSequence;" in js- assert "const surfaceSequence = this._surfaceOpenSequence;" in js- assert "this._canvasFirstFrameAcceptedSequence = surfaceSequence;" in js- assert "forceRightCanvasWidthNudge()" in js- assert "await nextAnimationFrame();" in js- assert "globalThis.Alpine" not in js- assert 'import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";' in js- assert "const canvas = rightCanvasStore;" in js- assert 'canvas.activeSurfaceId !== "browser"' in js- assert "canvas.setWidth?.(nudgedWidth, { persist: false })" in js- assert "this.queueViewportSync(true)" in js- frame_accept_index = js.index("this._lastFrameDimensions = dimensions;")- frame_nudge_schedule_index = js.index(- "this.scheduleCanvasWidthNudgeAfterFirstFrame();",- frame_accept_index,- )- assert frame_accept_index < frame_nudge_schedule_index+ assert "_canvasSurfaceReadySequence" not in js+ assert "_canvasFirstFrameAcceptedSequence" not in js+ assert "_canvasFirstFrameNudgeSequence" not in js+ assert "scheduleCanvasWidthNudgeAfterFirstFrame" not in js+ assert "forceRightCanvasWidthNudge" not in js+ assert "canvas.setWidth?.(nudgedWidth" not in jsdef test_browser_canvas_restarts_stream_after_page_navigation():@@ -1423,18 +1409,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_storeassert 'restart_screencast=bool(data.get("restart_stream"))' in ws_browserassert "restart_screencast: bool = False" in runtime- assert "should_remount_viewport = changed or restart_screencast" in runtime- assert "VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05" in runtime- assert "await self._apply_cdp_viewport_with_remount" in runtime- assert "await self._apply_viewport_with_remount(page, viewport)" in runtime- assert "await self._remount_viewport(page, viewport)" in runtime- assert "await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)" in runtime- assert "def _nudged_viewport(viewport: dict[str, int])" in runtime+ assert "should_restart_screencast = changed or restart_screencast" in runtime+ assert "await self._apply_cdp_viewport({\"width\": width, \"height\": height})" in runtime+ assert "await page.set_viewport_size(viewport)" in runtime+ assert "VIEWPORT_REMOUNT_PAUSE_SECONDS" not in runtime+ assert "_nudged_viewport" not in runtime+ assert 'wait_until="commit"' in ws_browserassert 'restartStream: this._mode === "canvas" && this.usesScreencastTransport()' in browser_storeassert "this.frameState = data.state || null" not in browser_storeassert "function loadFrameDimensions(src)" in browser_storeassert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store- assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_store+ assert "shouldAcceptMismatchedFrame" not in browser_storeassert "requestViewportSyncAfterRejectedFrame()" in browser_storeassert "this.applySnapshot(data.snapshot);" in browser_storeassert "if (!this.frameCanvasReady || !this.usesScreencastTransport())" in browser_store@@ -1495,6 +1480,27 @@ def test_browser_viewer_frame_payload_supports_binary_slim_frames():assert "state" not in payload+def test_browser_viewer_frame_dimensions_reject_crops_but_allow_uniform_scaling():+ dimensions = ws_browser_module.WsBrowser._frame_dimensions++ assert dimensions(+ {+ "expectedWidth": 900,+ "expectedHeight": 600,+ "jpegWidth": 1800,+ "jpegHeight": 1200,+ }+ ) == {"width": 900, "height": 600}+ assert dimensions(+ {+ "expectedWidth": 900,+ "expectedHeight": 600,+ "jpegWidth": 900,+ "jpegHeight": 500,+ }+ ) == {"width": 900, "height": 500}++def test_browser_navigation_errors_stay_inside_native_browser_page():runtime = (PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"@@ -1796,7 +1802,7 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():for method, params in session.sentif method == "Emulation.setVisibleSize"]- assert metrics_calls[:3] == [+ assert metrics_calls == [{"width": 1118,"height": 662,@@ -1804,26 +1810,8 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():"mobile": False,"dontSetVisibleSize": True,},- {- "width": 1119,- "height": 662,- "deviceScaleFactor": 1,- "mobile": False,- "dontSetVisibleSize": True,- },- {- "width": 1118,- "height": 662,- "deviceScaleFactor": 1,- "mobile": False,- "dontSetVisibleSize": True,- },- ]- assert visible_calls[:3] == [- {"width": 1118, "height": 662},- {"width": 1119, "height": 662},- {"width": 1118, "height": 662},]+ assert visible_calls == [{"width": 1118, "height": 662}]start_index = next(indexfor index, (method, _params) in enumerate(session.sent)@@ -3072,7 +3060,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):@pytest.mark.anyio-async def test_browser_runtime_remounts_same_viewport_when_restarting_screencast():+async def test_browser_runtime_restarts_screencast_without_resizing_same_viewport():viewport_calls = []stopped = []settled = []@@ -3106,16 +3094,13 @@ async def test_browser_runtime_remounts_same_viewport_when_restarting_screencast"state": {"id": 7},"viewport": {"width": 1280, "height": 720},}- assert viewport_calls == [- {"width": 1281, "height": 720},- {"width": 1280, "height": 720},- ]+ assert viewport_calls == []assert stopped == [7]assert settled == [True]@pytest.mark.anyio-async def test_browser_runtime_remounts_initial_changed_viewport():+async def test_browser_runtime_applies_changed_viewport_once():calls = []stopped = []settled = []@@ -3151,8 +3136,6 @@ async def test_browser_runtime_remounts_initial_changed_viewport():}assert calls == [("viewport", {"width": 672, "height": 789}),- ("viewport", {"width": 673, "height": 789}),- ("viewport", {"width": 672, "height": 789}),]assert stopped == [7]assert settled == [True]