Skip to content

Agent Zero AI framework

Stabilize Internal Docker Browser rendering

Alessandro23af25e52ace
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index be7092826..6c810ebc9 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -230,13 +230,18 @@ class WsBrowser(WsHandler):
if command == "open":
result = await runtime.call("open", data.get("url") or "")
elif command == "navigate":
- result = await runtime.call("navigate", browser_id, data.get("url") or "")
+ result = await runtime.call(
+ "navigate",
+ browser_id,
+ data.get("url") or "",
+ wait_until="commit",
+ )
elif command == "back":
- result = await runtime.call("back", browser_id)
+ result = await runtime.call("back", browser_id, wait_until="commit")
elif command == "forward":
- result = await runtime.call("forward", browser_id)
+ result = await runtime.call("forward", browser_id, wait_until="commit")
elif command == "reload":
- result = await runtime.call("reload", browser_id)
+ result = await runtime.call("reload", browser_id, wait_until="commit")
elif command == "close":
result = await runtime.call("close_browser", browser_id)
elif command == "list":
@@ -717,18 +722,29 @@ class WsBrowser(WsHandler):
def _frame_dimensions(metadata: Any) -> dict[str, int]:
if not isinstance(metadata, dict):
return {}
- for width_key, height_key in (
- ("expectedWidth", "expectedHeight"),
- ("deviceWidth", "deviceHeight"),
- ("jpegWidth", "jpegHeight"),
- ):
+
+ def dimensions(width_key: str, height_key: str) -> tuple[int, int] | None:
try:
width = int(metadata.get(width_key) or 0)
height = int(metadata.get(height_key) or 0)
except (TypeError, ValueError):
- continue
+ return None
if width > 0 and height > 0:
- return {"width": width, "height": height}
+ return width, height
+ return None
+
+ expected = dimensions("expectedWidth", "expectedHeight")
+ jpeg = dimensions("jpegWidth", "jpegHeight")
+ if expected and jpeg:
+ width_scale = jpeg[0] / expected[0]
+ height_scale = jpeg[1] / expected[1]
+ if abs(width_scale - height_scale) <= 0.01:
+ return {"width": expected[0], "height": expected[1]}
+ return {"width": jpeg[0], "height": jpeg[1]}
+
+ for fallback in (jpeg, dimensions("deviceWidth", "deviceHeight"), expected):
+ if fallback:
+ return {"width": fallback[0], "height": fallback[1]}
return {}
async def _emit_viewer_state(
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 994c6f04e..ef32e9278 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -40,7 +40,6 @@ CHROME_SINGLETON_FILES = ("SingletonLock", "SingletonCookie", "SingletonSocket")
SCREENCAST_MAX_WIDTH = 4096
SCREENCAST_MAX_HEIGHT = 4096
VIEWPORT_SIZE_TOLERANCE = 4
-VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05
CLIPBOARD_BRIDGE_SCRIPT = r"""
(payload) => {
const action = String(payload?.action || "").trim().toLowerCase();
@@ -282,18 +281,6 @@ CLIPBOARD_BRIDGE_SCRIPT = r"""
_SAFE_CONTEXT_RE = re.compile(r"[^a-zA-Z0-9_.-]+")
-def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:
- width = int(viewport["width"])
- height = int(viewport["height"])
- if width < 4096:
- return {"width": width + 1, "height": height}
- if width > 320:
- return {"width": width - 1, "height": height}
- if height < 4096:
- return {"width": width, "height": height + 1}
- return {"width": width, "height": height - 1}
-
-
def _safe_context_id(context_id: str) -> str:
return _SAFE_CONTEXT_RE.sub("_", str(context_id or "default")).strip("._") or "default"
@@ -344,7 +331,7 @@ class _BrowserScreencast:
self._expected_height = height
with contextlib.suppress(Exception):
await self.session.send("Page.enable")
- await self._apply_cdp_viewport_with_remount({"width": width, "height": height})
+ await self._apply_cdp_viewport({"width": width, "height": height})
await self.session.send(
"Page.startScreencast",
{
@@ -356,14 +343,6 @@ class _BrowserScreencast:
},
)
- async def _apply_cdp_viewport_with_remount(self, viewport: dict[str, int]) -> None:
- await self._apply_cdp_viewport(viewport)
- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
- await self._apply_cdp_viewport(_nudged_viewport(viewport))
- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
- await self._apply_cdp_viewport(viewport)
- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
-
async def _apply_cdp_viewport(self, viewport: dict[str, int]) -> None:
width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
@@ -1247,38 +1226,59 @@ class _BrowserRuntimeCore:
await self.ensure_started()
return await self._state(self._resolve_browser_id(browser_id))
- async def navigate(self, browser_id: int | str | None, url: str) -> dict[str, Any]:
+ async def navigate(
+ self,
+ browser_id: int | str | None,
+ url: str,
+ *,
+ wait_until: str = "domcontentloaded",
+ ) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
- await self._goto(page, normalize_url(url))
+ await self._goto(page, normalize_url(url), wait_until=wait_until)
self._maybe_promote(resolved_id)
return await self._state(resolved_id)
- async def back(self, browser_id: int | str | None = None) -> dict[str, Any]:
+ async def back(
+ self,
+ browser_id: int | str | None = None,
+ *,
+ wait_until: str = "domcontentloaded",
+ ) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
- await page.go_back(wait_until="domcontentloaded", timeout=10000)
- await self._settle(page)
+ await page.go_back(wait_until=wait_until, timeout=10000)
+ await self._settle(page, short=wait_until == "commit")
self._maybe_promote(resolved_id)
return await self._state(resolved_id)
- async def forward(self, browser_id: int | str | None = None) -> dict[str, Any]:
+ async def forward(
+ self,
+ browser_id: int | str | None = None,
+ *,
+ wait_until: str = "domcontentloaded",
+ ) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
- await page.go_forward(wait_until="domcontentloaded", timeout=10000)
- await self._settle(page)
+ await page.go_forward(wait_until=wait_until, timeout=10000)
+ await self._settle(page, short=wait_until == "commit")
self._maybe_promote(resolved_id)
return await self._state(resolved_id)
- async def reload(self, browser_id: int | str | None = None) -> dict[str, Any]:
+ async def reload(
+ self,
+ browser_id: int | str | None = None,
+ *,
+ wait_until: str = "domcontentloaded",
+ ) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
- await page.reload(wait_until="domcontentloaded", timeout=15000)
- await self._settle(page)
+ await page.reload(wait_until=wait_until, timeout=15000)
+ await self._settle(page, short=wait_until == "commit")
self._maybe_promote(resolved_id)
return await self._state(resolved_id)
@@ -1748,33 +1748,16 @@ class _BrowserRuntimeCore:
or abs(int(current_viewport.get("height") or 0) - viewport["height"])
> VIEWPORT_SIZE_TOLERANCE
)
- should_remount_viewport = changed or restart_screencast
- if should_remount_viewport:
+ should_restart_screencast = changed or restart_screencast
+ if should_restart_screencast:
await self._stop_screencasts_for_browser(resolved_id)
if changed:
- await self._apply_viewport_with_remount(page, viewport)
- elif restart_screencast:
- await self._remount_viewport(page, viewport)
- if should_remount_viewport:
+ await page.set_viewport_size(viewport)
+ if should_restart_screencast:
await self._settle(page, short=True)
self._maybe_promote(resolved_id)
return {"state": await self._state(resolved_id), "viewport": viewport}
- async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:
- await page.set_viewport_size(viewport)
- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
- await self._remount_viewport(page, viewport)
-
- async def _remount_viewport(self, page: Any, viewport: dict[str, int]) -> None:
- nudged_viewport = self._nudged_viewport(viewport)
- await page.set_viewport_size(nudged_viewport)
- await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
- await page.set_viewport_size(viewport)
-
- @staticmethod
- def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:
- return _nudged_viewport(viewport)
-
async def _point_for(
self,
page: Any,
@@ -2223,17 +2206,23 @@ class _BrowserRuntimeCore:
self._maybe_promote(resolved_id)
return {"action": action or {}, "state": await self._state(resolved_id)}
- async def _goto(self, page: Any, url: str) -> None:
+ async def _goto(
+ self,
+ page: Any,
+ url: str,
+ *,
+ wait_until: str = "domcontentloaded",
+ ) -> None:
from playwright.async_api import Error as PlaywrightError
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
try:
- await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+ await page.goto(url, wait_until=wait_until, timeout=30000)
except PlaywrightTimeoutError:
- PrintStyle.warning(f"Browser navigation timed out after DOM handoff: {url}")
+ PrintStyle.warning(f"Browser navigation timed out waiting for {wait_until}: {url}")
except PlaywrightError as exc:
PrintStyle.warning(f"Browser navigation showed a native error page for {url}: {exc}")
- await self._settle(page)
+ await self._settle(page, short=wait_until == "commit")
async def _settle(self, page: Any, short: bool = False) -> None:
from playwright.async_api import Error as PlaywrightError
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index 150d5bf2b..15ec5788d 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -19,6 +19,7 @@ websocket.addHandlers(["ws_webui"]);
const EXTENSIONS_ROOT = "/a0/usr/_browser/extensions";
const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;
const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
+const BROWSER_COMMAND_TIMEOUT_MS = 45000;
const BROWSER_CONFIG_REFRESH_MS = 15000;
const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";
const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";
@@ -211,9 +212,6 @@ const model = {
_surfaceHandoffTimer: null,
_surfaceOpenedAt: 0,
_surfaceOpenSequence: 0,
- _canvasSurfaceReadySequence: 0,
- _canvasFirstFrameAcceptedSequence: 0,
- _canvasFirstFrameNudgeSequence: 0,
_openPromise: null,
_openSignature: "",
_connectSequence: 0,
@@ -751,10 +749,6 @@ const model = {
} finally {
if (this.isCurrentSurfaceOpen(surfaceSequence)) {
this.loading = false;
- if (this._mode === "canvas") {
- this._canvasSurfaceReadySequence = surfaceSequence;
- this.scheduleCanvasWidthNudgeAfterFirstFrame();
- }
}
}
},
@@ -1298,11 +1292,9 @@ const model = {
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
- if (!this.shouldAcceptMismatchedFrame(dimensions)) {
- bitmap?.close?.();
- options?.cleanup?.();
- return;
- }
+ bitmap?.close?.();
+ options?.cleanup?.();
+ return;
}
if (bitmap && this.paintFrameBitmap(bitmap)) {
this.clearFrameSrc();
@@ -1317,64 +1309,6 @@ const model = {
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
- this._canvasFirstFrameAcceptedSequence = surfaceSequence;
- this.scheduleCanvasWidthNudgeAfterFirstFrame();
- },
-
- shouldAcceptMismatchedFrame(dimensions = null) {
- return Boolean(
- dimensions?.width
- && dimensions?.height
- && (!this.hasFrame() || this._surfaceSwitching || this.isSwitchingBrowser())
- );
- },
-
- scheduleCanvasWidthNudgeAfterFirstFrame() {
- const surfaceSequence = this._surfaceOpenSequence;
- if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
- return;
- }
- if (this._canvasFirstFrameNudgeSequence === surfaceSequence) {
- return;
- }
- if (
- this._canvasSurfaceReadySequence !== surfaceSequence
- || this._canvasFirstFrameAcceptedSequence !== surfaceSequence
- ) {
- return;
- }
- this._canvasFirstFrameNudgeSequence = surfaceSequence;
-
- void (async () => {
- await nextAnimationFrame();
- await nextAnimationFrame();
- if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") {
- return;
- }
- this.forceRightCanvasWidthNudge();
- })();
- },
-
- forceRightCanvasWidthNudge() {
- const canvas = rightCanvasStore;
- if (!canvas || canvas.isMobileMode || !canvas.isOpen || canvas.activeSurfaceId !== "browser") {
- return;
- }
-
- const currentWidth = Number(canvas.width || 0);
- if (!Number.isFinite(currentWidth) || currentWidth <= 0) {
- return;
- }
- const maxWidth = Number(canvas.maxWidth?.() || currentWidth);
- const minWidth = Number(canvas.minWidth || 420);
- const direction = currentWidth < maxWidth ? 1 : -1;
- const nudgedWidth = currentWidth + direction;
- if (nudgedWidth < minWidth || nudgedWidth > maxWidth || nudgedWidth === currentWidth) {
- return;
- }
-
- canvas.setWidth?.(nudgedWidth, { persist: false });
- this.queueViewportSync(true);
},
frameMatchesViewport(dimensions = null, viewport = null) {
@@ -1525,7 +1459,7 @@ const model = {
viewer_transport: this.requestedViewerTransport(),
command,
},
- { timeoutMs: 20000 },
+ { timeoutMs: BROWSER_COMMAND_TIMEOUT_MS },
);
const data = firstOk(response);
this.applyTabScope(data);
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index c0e4c50b5..262019baf 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -707,6 +707,7 @@ def test_browser_viewer_allows_slow_extension_startup():
assert "const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;" in js
assert "const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;" in js
+ assert "const BROWSER_COMMAND_TIMEOUT_MS = 45000;" in js
assert "? BROWSER_FIRST_INSTALL_TIMEOUT_MS" in js
assert ": BROWSER_SUBSCRIBE_TIMEOUT_MS" in js
assert "Installing Chromium for the first Browser run" in js
@@ -796,32 +797,17 @@ def test_browser_canvas_surface_open_waits_for_visible_panel():
assert "forceCanvasWidthNudgeAfterBrowserMount" not in js
-def test_browser_canvas_nudges_width_after_first_accepted_frame():
+def test_browser_canvas_does_not_resize_after_first_accepted_frame():
js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
encoding="utf-8"
)
- assert "_canvasSurfaceReadySequence" in js
- assert "_canvasFirstFrameAcceptedSequence" in js
- assert "_canvasFirstFrameNudgeSequence" in js
- assert "scheduleCanvasWidthNudgeAfterFirstFrame()" in js
- assert "this._canvasSurfaceReadySequence = surfaceSequence;" in js
- assert "const surfaceSequence = this._surfaceOpenSequence;" in js
- assert "this._canvasFirstFrameAcceptedSequence = surfaceSequence;" in js
- assert "forceRightCanvasWidthNudge()" in js
- assert "await nextAnimationFrame();" in js
- assert "globalThis.Alpine" not in js
- assert 'import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";' in js
- assert "const canvas = rightCanvasStore;" in js
- assert 'canvas.activeSurfaceId !== "browser"' in js
- assert "canvas.setWidth?.(nudgedWidth, { persist: false })" in js
- assert "this.queueViewportSync(true)" in js
- frame_accept_index = js.index("this._lastFrameDimensions = dimensions;")
- frame_nudge_schedule_index = js.index(
- "this.scheduleCanvasWidthNudgeAfterFirstFrame();",
- frame_accept_index,
- )
- assert frame_accept_index < frame_nudge_schedule_index
+ assert "_canvasSurfaceReadySequence" not in js
+ assert "_canvasFirstFrameAcceptedSequence" not in js
+ assert "_canvasFirstFrameNudgeSequence" not in js
+ assert "scheduleCanvasWidthNudgeAfterFirstFrame" not in js
+ assert "forceRightCanvasWidthNudge" not in js
+ assert "canvas.setWidth?.(nudgedWidth" not in js
def test_browser_canvas_restarts_stream_after_page_navigation():
@@ -1423,18 +1409,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store
assert 'restart_screencast=bool(data.get("restart_stream"))' in ws_browser
assert "restart_screencast: bool = False" in runtime
- assert "should_remount_viewport = changed or restart_screencast" in runtime
- assert "VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05" in runtime
- assert "await self._apply_cdp_viewport_with_remount" in runtime
- assert "await self._apply_viewport_with_remount(page, viewport)" in runtime
- assert "await self._remount_viewport(page, viewport)" in runtime
- assert "await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)" in runtime
- assert "def _nudged_viewport(viewport: dict[str, int])" in runtime
+ assert "should_restart_screencast = changed or restart_screencast" in runtime
+ assert "await self._apply_cdp_viewport({\"width\": width, \"height\": height})" in runtime
+ assert "await page.set_viewport_size(viewport)" in runtime
+ assert "VIEWPORT_REMOUNT_PAUSE_SECONDS" not in runtime
+ assert "_nudged_viewport" not in runtime
+ assert 'wait_until="commit"' in ws_browser
assert 'restartStream: this._mode === "canvas" && this.usesScreencastTransport()' in browser_store
assert "this.frameState = data.state || null" not in browser_store
assert "function loadFrameDimensions(src)" in browser_store
assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
- assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_store
+ assert "shouldAcceptMismatchedFrame" not in browser_store
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
assert "this.applySnapshot(data.snapshot);" in browser_store
assert "if (!this.frameCanvasReady || !this.usesScreencastTransport())" in browser_store
@@ -1495,6 +1480,27 @@ def test_browser_viewer_frame_payload_supports_binary_slim_frames():
assert "state" not in payload
+def test_browser_viewer_frame_dimensions_reject_crops_but_allow_uniform_scaling():
+ dimensions = ws_browser_module.WsBrowser._frame_dimensions
+
+ assert dimensions(
+ {
+ "expectedWidth": 900,
+ "expectedHeight": 600,
+ "jpegWidth": 1800,
+ "jpegHeight": 1200,
+ }
+ ) == {"width": 900, "height": 600}
+ assert dimensions(
+ {
+ "expectedWidth": 900,
+ "expectedHeight": 600,
+ "jpegWidth": 900,
+ "jpegHeight": 500,
+ }
+ ) == {"width": 900, "height": 500}
+
+
def test_browser_navigation_errors_stay_inside_native_browser_page():
runtime = (
PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
@@ -1796,7 +1802,7 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
for method, params in session.sent
if method == "Emulation.setVisibleSize"
]
- assert metrics_calls[:3] == [
+ assert metrics_calls == [
{
"width": 1118,
"height": 662,
@@ -1804,26 +1810,8 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
"mobile": False,
"dontSetVisibleSize": True,
},
- {
- "width": 1119,
- "height": 662,
- "deviceScaleFactor": 1,
- "mobile": False,
- "dontSetVisibleSize": True,
- },
- {
- "width": 1118,
- "height": 662,
- "deviceScaleFactor": 1,
- "mobile": False,
- "dontSetVisibleSize": True,
- },
- ]
- assert visible_calls[:3] == [
- {"width": 1118, "height": 662},
- {"width": 1119, "height": 662},
- {"width": 1118, "height": 662},
]
+ assert visible_calls == [{"width": 1118, "height": 662}]
start_index = next(
index
for index, (method, _params) in enumerate(session.sent)
@@ -3072,7 +3060,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
@pytest.mark.anyio
-async def test_browser_runtime_remounts_same_viewport_when_restarting_screencast():
+async def test_browser_runtime_restarts_screencast_without_resizing_same_viewport():
viewport_calls = []
stopped = []
settled = []
@@ -3106,16 +3094,13 @@ async def test_browser_runtime_remounts_same_viewport_when_restarting_screencast
"state": {"id": 7},
"viewport": {"width": 1280, "height": 720},
}
- assert viewport_calls == [
- {"width": 1281, "height": 720},
- {"width": 1280, "height": 720},
- ]
+ assert viewport_calls == []
assert stopped == [7]
assert settled == [True]
@pytest.mark.anyio
-async def test_browser_runtime_remounts_initial_changed_viewport():
+async def test_browser_runtime_applies_changed_viewport_once():
calls = []
stopped = []
settled = []
@@ -3151,8 +3136,6 @@ async def test_browser_runtime_remounts_initial_changed_viewport():
}
assert calls == [
("viewport", {"width": 672, "height": 789}),
- ("viewport", {"width": 673, "height": 789}),
- ("viewport", {"width": 672, "height": 789}),
]
assert stopped == [7]
assert settled == [True]