Agent Zero AI framework
Redesign Browser viewer screencast transport and viewport fit
Alessandrodccf017d2cf7
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex 2dc7002a5..6ac7039e5 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -1,6 +1,8 @@from __future__ import annotationsimport asyncio+import contextlib+import timefrom typing import Any, ClassVarfrom agent import AgentContext@@ -9,6 +11,12 @@ from helpers.ws_manager import WsResultfrom plugins._browser.helpers.runtime import get_runtime+FRAME_IDLE_TIMEOUT_SECONDS = 0.35+FRAME_RETRY_DELAY_SECONDS = 0.5+FRAME_STATE_REFRESH_SECONDS = 0.75+SCREENCAST_QUALITY = 92++class WsBrowser(WsHandler):_streams: ClassVar[dict[tuple[str, str], asyncio.Task[None]]] = {}@@ -57,9 +65,17 @@ class WsBrowser(WsHandler):browsers = listing.get("browsers") or []if opened.get("id"):listing["last_interacted_browser_id"] = opened.get("id")- active_id = data.get("browser_id") or listing.get("last_interacted_browser_id")- if not active_id and browsers:- active_id = browsers[0].get("id")+ active_id = self._active_browser_id(listing, data.get("browser_id"))+ initial_viewport = self._viewport_from_data(data)+ if active_id and initial_viewport:+ await runtime.call(+ "set_viewport",+ active_id,+ initial_viewport["width"],+ initial_viewport["height"],+ )+ listing = await runtime.call("list")+ browsers = listing.get("browsers") or []stream_key = (sid, context_id)existing = self._streams.pop(stream_key, None)@@ -187,46 +203,145 @@ class WsBrowser(WsHandler):context_id: str,browser_id: int | str | None,) -> None:+ runtime = None+ stream_id = Nonewhile True:try:runtime = await get_runtime(context_id, create=False)- if runtime:- listing = await runtime.call("list")- browsers = listing.get("browsers") or []- browser_ids = {str(browser.get("id")) for browser in browsers}- requested_id = str(browser_id or "") if browser_id else ""- active_id = (- browser_id- if requested_id and requested_id in browser_ids- else listing.get("last_interacted_browser_id")- )- if active_id and str(active_id) not in browser_ids:- active_id = None- if not active_id and browsers:- active_id = browsers[0].get("id")- if active_id:- frame = await runtime.call("screenshot", active_id)- frame["context_id"] = context_id- frame["browsers"] = browsers- await self.emit_to(sid, "browser_viewer_frame", frame)- else:- await self.emit_to(- sid,- "browser_viewer_frame",- {- "context_id": context_id,- "browser_id": None,- "browsers": browsers,- "image": "",- "mime": "",- "state": None,- },+ if not runtime:+ await self._emit_empty_frame(sid, context_id)+ await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)+ continue++ listing = await runtime.call("list")+ browsers = listing.get("browsers") or []+ active_id = self._active_browser_id(listing, browser_id)+ if not active_id:+ await self._emit_empty_frame(sid, context_id, browsers=browsers)+ await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)+ continue++ screencast = await runtime.call(+ "start_screencast",+ active_id,+ quality=SCREENCAST_QUALITY,+ every_nth_frame=1,+ )+ stream_id = screencast["stream_id"]+ active_id = screencast["browser_id"]+ state = screencast.get("state")+ await self.emit_to(+ sid,+ "browser_viewer_frame",+ {+ "context_id": context_id,+ "browser_id": active_id,+ "browsers": browsers,+ "image": "",+ "mime": "",+ "state": state,+ },+ )++ last_state_refresh = 0.0+ while True:+ now = time.monotonic()+ if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:+ listing = await runtime.call("list")+ browsers = listing.get("browsers") or []+ browser_ids = {str(browser.get("id")) for browser in browsers}+ if str(active_id) not in browser_ids:+ break+ state = self._state_for_browser(browsers, active_id, state)+ last_state_refresh = now++ try:+ frame = await runtime.call(+ "read_screencast_frame",+ stream_id,+ timeout=FRAME_IDLE_TIMEOUT_SECONDS,)- await asyncio.sleep(0.75)+ except TimeoutError:+ continue++ frame["context_id"] = context_id+ frame["browser_id"] = active_id+ frame["browsers"] = browsers+ frame["state"] = state+ await self.emit_to(sid, "browser_viewer_frame", frame)except asyncio.CancelledError:raiseexcept Exception:- await asyncio.sleep(1.5)+ await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)+ finally:+ if runtime and stream_id:+ with contextlib.suppress(Exception):+ await runtime.call("stop_screencast", stream_id)+ stream_id = None++ @staticmethod+ def _active_browser_id(+ listing: dict[str, Any],+ requested_browser_id: int | str | None,+ ) -> int | str | None:+ browsers = listing.get("browsers") or []+ browser_ids = {str(browser.get("id")) for browser in browsers}+ requested_id = str(requested_browser_id or "") if requested_browser_id else ""+ active_id = (+ requested_browser_id+ if requested_id and requested_id in browser_ids+ else listing.get("last_interacted_browser_id")+ )+ if active_id and str(active_id) not in browser_ids:+ active_id = None+ if not active_id and browsers:+ active_id = browsers[0].get("id")+ return active_id++ @staticmethod+ def _state_for_browser(+ browsers: list[dict[str, Any]],+ browser_id: int | str,+ current_state: dict[str, Any] | None,+ ) -> dict[str, Any] | None:+ for browser in browsers:+ if str(browser.get("id")) == str(browser_id):+ return browser+ return current_state++ async def _emit_empty_frame(+ self,+ sid: str,+ context_id: str,+ *,+ browsers: list[dict[str, Any]] | None = None,+ ) -> None:+ await self.emit_to(+ sid,+ "browser_viewer_frame",+ {+ "context_id": context_id,+ "browser_id": None,+ "browsers": browsers or [],+ "image": "",+ "mime": "",+ "state": None,+ },+ )++ @staticmethod+ def _viewport_from_data(data: dict[str, Any]) -> dict[str, int] | None:+ try:+ width = int(data.get("viewport_width") or data.get("width") or 0)+ height = int(data.get("viewport_height") or data.get("height") or 0)+ except (TypeError, ValueError):+ return None+ if width < 80 or height < 80:+ return None+ return {+ "width": max(320, min(4096, width)),+ "height": max(200, min(4096, height)),+ }@staticmethoddef _context_id(data: dict[str, Any]) -> str:diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex a54f13908..e72b48e87 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -3,12 +3,14 @@ from __future__ import annotationsimport atexitimport asyncioimport base64+import contextlibimport osimport reimport shutilimport signalimport threadingimport time+import uuidfrom dataclasses import dataclassfrom pathlib import Pathfrom typing import Any@@ -27,6 +29,8 @@ CONTENT_HELPER_PATH = PLUGIN_DIR / "assets" / "browser-page-content.js"RUNTIME_DATA_KEY = "_browser_runtime"DEFAULT_VIEWPORT = {"width": 1024, "height": 768}CHROME_SINGLETON_FILES = ("SingletonLock", "SingletonCookie", "SingletonSocket")+SCREENCAST_MAX_WIDTH = 4096+SCREENCAST_MAX_HEIGHT = 4096_SPECIAL_SCHEME_RE = re.compile(r"^(?:about|blob|data|file|mailto|tel):", re.I)_URL_SCHEME_RE = re.compile(r"^[a-z][a-z\d+\-.]*://", re.I)@@ -84,6 +88,195 @@ class BrowserPage:page: Any+class _BrowserScreencast:+ def __init__(+ self,+ *,+ stream_id: str,+ browser_id: int,+ session: Any,+ mime: str,+ ):+ self.id = stream_id+ self.browser_id = browser_id+ self.session = session+ self.mime = mime+ self.queue = asyncio.Queue(maxsize=1)+ self.stopped = False+ self._ack_tasks: set[asyncio.Task] = set()+ self._expected_width = 0+ self._expected_height = 0+ self._dimension_mismatches = 0++ async def start(+ self,+ *,+ quality: int,+ every_nth_frame: int,+ viewport: dict[str, int],+ ) -> None:+ self.session.on("Page.screencastFrame", self._on_frame)+ width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))+ height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))+ self._expected_width = width+ self._expected_height = height+ self._dimension_mismatches = 0+ with contextlib.suppress(Exception):+ await self.session.send("Page.enable")+ await self.session.send(+ "Emulation.setDeviceMetricsOverride",+ {+ "width": width,+ "height": height,+ "deviceScaleFactor": 1,+ "mobile": False,+ "dontSetVisibleSize": True,+ },+ )+ with contextlib.suppress(Exception):+ await self.session.send(+ "Emulation.setVisibleSize",+ {+ "width": width,+ "height": height,+ },+ )+ await self.session.send(+ "Page.startScreencast",+ {+ "format": "jpeg",+ "quality": max(20, min(95, int(quality))),+ "maxWidth": SCREENCAST_MAX_WIDTH,+ "maxHeight": SCREENCAST_MAX_HEIGHT,+ "everyNthFrame": max(1, int(every_nth_frame)),+ },+ )++ async def next_frame(self, timeout: float = 1.0) -> dict[str, Any]:+ frame = await asyncio.wait_for(self.queue.get(), timeout=max(0.1, float(timeout)))+ if frame is None:+ raise RuntimeError("Browser screencast stopped.")+ return frame++ async def stop(self) -> None:+ if self.stopped:+ return+ self.stopped = True+ self._drop_queued_frames()+ with contextlib.suppress(asyncio.QueueFull):+ self.queue.put_nowait(None)+ with contextlib.suppress(Exception):+ await self.session.send("Page.stopScreencast")+ for task in list(self._ack_tasks):+ task.cancel()+ if self._ack_tasks:+ await asyncio.gather(*self._ack_tasks, return_exceptions=True)+ self._ack_tasks.clear()+ with contextlib.suppress(Exception):+ await self.session.detach()++ def _on_frame(self, params: dict[str, Any]) -> None:+ if self.stopped:+ return+ task = asyncio.create_task(self._handle_frame(params or {}))+ self._ack_tasks.add(task)+ task.add_done_callback(self._ack_tasks.discard)++ async def _handle_frame(self, params: dict[str, Any]) -> None:+ try:+ data = params.get("data") or ""+ if data and self._frame_matches_viewport(data):+ self._queue_latest(+ {+ "browser_id": self.browser_id,+ "mime": self.mime,+ "image": data,+ "metadata": params.get("metadata") or {},+ }+ )+ finally:+ session_id = params.get("sessionId")+ if session_id is not None and not self.stopped:+ with contextlib.suppress(Exception):+ await self.session.send(+ "Page.screencastFrameAck",+ {"sessionId": int(session_id)},+ )++ def _queue_latest(self, frame: dict[str, Any]) -> None:+ self._drop_queued_frames()+ with contextlib.suppress(asyncio.QueueFull):+ self.queue.put_nowait(frame)++ def _frame_matches_viewport(self, data: str) -> bool:+ if not self._expected_width or not self._expected_height:+ return True+ size = self._jpeg_size(data)+ if not size:+ return True+ width, height = size+ if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2:+ return True+ self._dimension_mismatches += 1+ return self._dimension_mismatches > 10++ @staticmethod+ def _jpeg_size(data: str) -> tuple[int, int] | None:+ try:+ raw = base64.b64decode(data, validate=False)+ except Exception:+ return None+ if len(raw) < 10 or raw[:2] != b"\xff\xd8":+ return None+ index = 2+ standalone_markers = {0x01, *range(0xD0, 0xD8)}+ size_markers = {+ 0xC0,+ 0xC1,+ 0xC2,+ 0xC3,+ 0xC5,+ 0xC6,+ 0xC7,+ 0xC9,+ 0xCA,+ 0xCB,+ 0xCD,+ 0xCE,+ 0xCF,+ }+ while index < len(raw) - 9:+ if raw[index] != 0xFF:+ index += 1+ continue+ while index < len(raw) and raw[index] == 0xFF:+ index += 1+ if index >= len(raw):+ return None+ marker = raw[index]+ index += 1+ if marker in standalone_markers:+ continue+ if index + 2 > len(raw):+ return None+ segment_length = int.from_bytes(raw[index : index + 2], "big")+ if segment_length < 2 or index + segment_length > len(raw):+ return None+ if marker in size_markers and segment_length >= 7:+ height = int.from_bytes(raw[index + 3 : index + 5], "big")+ width = int.from_bytes(raw[index + 5 : index + 7], "big")+ return width, height+ index += segment_length+ return None++ def _drop_queued_frames(self) -> None:+ while True:+ try:+ self.queue.get_nowait()+ except asyncio.QueueEmpty:+ return++class BrowserRuntime:def __init__(self, context_id: str):self.context_id = str(context_id)@@ -118,6 +311,7 @@ class _BrowserRuntimeCore:self.playwright = Noneself.context = Noneself.pages: dict[int, BrowserPage] = {}+ self.screencasts: dict[str, _BrowserScreencast] = {}self.next_browser_id = 1self.last_interacted_browser_id: int | None = Noneself._content_helper_source: str | None = None@@ -378,6 +572,7 @@ class _BrowserRuntimeCore:async def close_browser(self, browser_id: int | str | None = None) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)+ await self._stop_screencasts_for_browser(resolved_id)page = self._page(resolved_id)await page.close()self.pages.pop(resolved_id, None)@@ -387,6 +582,7 @@ class _BrowserRuntimeCore:async def close_all_browsers(self) -> dict[str, Any]:await self.ensure_started()+ await self._stop_all_screencasts()for browser_id in list(self.pages):try:await self.pages[browser_id].page.close()@@ -413,6 +609,58 @@ class _BrowserRuntimeCore:"state": await self._state(resolved_id),}+ async def start_screencast(+ self,+ browser_id: int | str | None = None,+ *,+ quality: int = 78,+ every_nth_frame: int = 1,+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ stream_id = uuid.uuid4().hex+ session = await self.context.new_cdp_session(page)+ screencast = _BrowserScreencast(+ stream_id=stream_id,+ browser_id=resolved_id,+ session=session,+ mime="image/jpeg",+ )+ self.screencasts[stream_id] = screencast+ try:+ await screencast.start(+ quality=quality,+ every_nth_frame=every_nth_frame,+ viewport=page.viewport_size or DEFAULT_VIEWPORT,+ )+ except Exception:+ self.screencasts.pop(stream_id, None)+ await screencast.stop()+ raise+ self.last_interacted_browser_id = resolved_id+ return {+ "stream_id": stream_id,+ "browser_id": resolved_id,+ "state": await self._state(resolved_id),+ }++ async def read_screencast_frame(+ self,+ stream_id: str,+ *,+ timeout: float = 1.0,+ ) -> dict[str, Any]:+ screencast = self.screencasts.get(str(stream_id or ""))+ if not screencast:+ raise KeyError("Browser screencast is not active.")+ return await screencast.next_frame(timeout=timeout)++ async def stop_screencast(self, stream_id: str) -> None:+ screencast = self.screencasts.pop(str(stream_id or ""), None)+ if screencast:+ await screencast.stop()+async def set_viewport(self,browser_id: int | str | None,@@ -426,7 +674,14 @@ class _BrowserRuntimeCore:"width": max(320, min(4096, int(width or DEFAULT_VIEWPORT["width"]))),"height": max(200, min(4096, int(height or DEFAULT_VIEWPORT["height"]))),}- await page.set_viewport_size(viewport)+ current_viewport = page.viewport_size or {}+ changed = (+ int(current_viewport.get("width") or 0) != viewport["width"]+ or int(current_viewport.get("height") or 0) != viewport["height"]+ )+ if changed:+ await page.set_viewport_size(viewport)+ await self._stop_screencasts_for_browser(resolved_id)self.last_interacted_browser_id = resolved_idreturn {"state": await self._state(resolved_id), "viewport": viewport}@@ -490,6 +745,7 @@ class _BrowserRuntimeCore:return await self._state(resolved_id)async def close(self, delete_profile: bool = False) -> None:+ await self._stop_all_screencasts()for browser_id in list(self.pages):try:await self.pages[browser_id].page.close()@@ -618,6 +874,19 @@ class _BrowserRuntimeCore:def _page(self, browser_id: int) -> Any:return self.pages[int(browser_id)].page+ async def _stop_screencasts_for_browser(self, browser_id: int) -> None:+ stream_ids = [+ stream_id+ for stream_id, screencast in self.screencasts.items()+ if screencast.browser_id == int(browser_id)+ ]+ for stream_id in stream_ids:+ await self.stop_screencast(stream_id)++ async def _stop_all_screencasts(self) -> None:+ for stream_id in list(self.screencasts):+ await self.stop_screencast(stream_id)+async def _ensure_content_helper(self, page: Any) -> None:has_helper = await page.evaluate("() => Boolean(globalThis.__spaceBrowserPageContent__?.capture)"diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex 44f585cbf..c5ec6477a 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -40,6 +40,9 @@ const model = {_frameOff: null,_stateOff: null,_lastFrameAt: 0,+ _pendingFrameSrc: "",+ _frameRenderHandle: null,+ _frameRenderCancel: null,_floatingCleanup: null,_stageElement: null,_stageResizeObserver: null,@@ -298,21 +301,24 @@ const model = {}},- async connectViewer() {- if (!this.contextId) {- this.connected = false;- this.error = "No active chat context is selected.";- return;- }- this.error = "";- await this._bindSocketEvents();- const response = await websocket.request(- "browser_viewer_subscribe",- {- context_id: this.contextId,- browser_id: this.activeBrowserId,- },- {+ async connectViewer() {+ if (!this.contextId) {+ this.connected = false;+ this.error = "No active chat context is selected.";+ return;+ }+ this.error = "";+ await this._bindSocketEvents();+ const initialViewport = this.currentViewportSize();+ const response = await websocket.request(+ "browser_viewer_subscribe",+ {+ context_id: this.contextId,+ browser_id: this.activeBrowserId,+ viewport_width: initialViewport?.width,+ viewport_height: initialViewport?.height,+ },+ {timeoutMs: this.browserInstallExpected? BROWSER_FIRST_INSTALL_TIMEOUT_MS: BROWSER_SUBSCRIBE_TIMEOUT_MS,@@ -329,18 +335,25 @@ const model = {async _bindSocketEvents() {if (!this._frameOff) {const frameHandler = ({ data }) => {- if (data?.context_id !== this.contextId) return;- this.browsers = data.browsers || this.browsers;- this.setActiveBrowserId(data.browser_id || data.state?.id || this.activeBrowserId);- this.frameState = data.state || null;- if (!this.addressFocused && data.state?.currentUrl) {- this.address = data.state.currentUrl;- }- this.frameSrc = data.image ? `data:${data.mime || "image/jpeg"};base64,${data.image}` : "";- if (!data.image && !data.state) {- this.setActiveBrowserId(null);- this.frameState = null;- this.frameSrc = "";+ if (data?.context_id !== this.contextId) return;+ this.browsers = data.browsers || this.browsers;+ this.setActiveBrowserId(data.browser_id || data.state?.id || this.activeBrowserId);+ if (data.state) {+ this.frameState = data.state;+ }+ if (!this.addressFocused && data.state?.currentUrl) {+ this.address = data.state.currentUrl;+ }+ if (data.image) {+ this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`);+ } else {+ this.cancelFrameRender();+ this.frameSrc = "";+ }+ if (!data.image && !data.state) {+ this.setActiveBrowserId(null);+ this.frameState = null;+ this.frameSrc = "";}this._lastFrameAt = Date.now();};@@ -356,12 +369,41 @@ const model = {};await websocket.on("browser_viewer_state", stateHandler);this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);- }- },-- async command(command, extra = {}) {- this.error = "";- const previousActiveBrowserId = this.activeBrowserId;+ }+ },++ queueFrameRender(frameSrc) {+ this._pendingFrameSrc = frameSrc;+ if (this._frameRenderHandle) return;+ const schedule = globalThis.requestAnimationFrame?.bind(globalThis);+ if (schedule) {+ this._frameRenderCancel = globalThis.cancelAnimationFrame?.bind(globalThis) || null;+ this._frameRenderHandle = schedule(() => this.flushFrameRender());+ return;+ }+ this._frameRenderCancel = globalThis.clearTimeout?.bind(globalThis) || null;+ this._frameRenderHandle = globalThis.setTimeout(() => this.flushFrameRender(), 16);+ },++ flushFrameRender() {+ this._frameRenderHandle = null;+ this._frameRenderCancel = null;+ this.frameSrc = this._pendingFrameSrc || "";+ this._pendingFrameSrc = "";+ },++ cancelFrameRender() {+ if (this._frameRenderHandle && this._frameRenderCancel) {+ this._frameRenderCancel(this._frameRenderHandle);+ }+ this._frameRenderHandle = null;+ this._frameRenderCancel = null;+ this._pendingFrameSrc = "";+ },++ async command(command, extra = {}) {+ this.error = "";+ const previousActiveBrowserId = this.activeBrowserId;try {const response = await websocket.request("browser_viewer_command",@@ -577,10 +619,11 @@ const model = {} catch {}}this._frameOff?.();- this._stateOff?.();- this._frameOff = null;- this._stateOff = null;- this._floatingCleanup?.();+ this._stateOff?.();+ this._frameOff = null;+ this._stateOff = null;+ this.cancelFrameRender();+ this._floatingCleanup?.();this._floatingCleanup = null;this._stageResizeObserver?.disconnect?.();this._stageResizeObserver = null;diff --git a/plugins/_browser/webui/main.html b/plugins/_browser/webui/main.htmlindex 614cedd71..7883c202f 100644--- a/plugins/_browser/webui/main.html+++ b/plugins/_browser/webui/main.html@@ -782,24 +782,26 @@color: #9f1239;}- .browser-stage {- flex: 1 1 auto;- display: flex;- flex-direction: column;- min-height: 0;- overflow: auto;- background: #fff;- outline: none;- }+ .browser-stage {+ flex: 1 1 auto;+ display: flex;+ flex-direction: column;+ min-height: 0;+ overflow: hidden;+ background: #fff;+ outline: none;+ }- .browser-frame {- flex: 0 0 auto;- display: block;- width: 100%;- height: auto;- user-select: none;- background: #fff;- }+ .browser-frame {+ flex: 0 0 auto;+ display: block;+ width: 100%;+ height: auto;+ min-width: 0;+ image-rendering: auto;+ user-select: none;+ background: #fff;+ }.browser-status,.browser-error,@@ -884,4 +886,4 @@</style></body>-</html>\ No newline at end of file+</html>diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex 69a74c9b4..f3c65b557 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -1,3 +1,4 @@+import asyncioimport sysimport threadingfrom pathlib import Path@@ -26,7 +27,11 @@ from plugins._browser.helpers.extension_manager import (parse_chrome_web_store_extension_id,)import plugins._browser.helpers.extension_manager as browser_extension_manager_module-from plugins._browser.helpers.runtime import _BrowserRuntimeCore, normalize_url+from plugins._browser.helpers.runtime import (+ _BrowserRuntimeCore,+ _BrowserScreencast,+ normalize_url,+)import plugins._browser.helpers.runtime as browser_runtime_modulefrom plugins._browser.helpers.playwright import (get_playwright_binary,@@ -320,6 +325,116 @@ def test_browser_viewer_uses_tabs_for_session_switching():assert "Using ${this.mainModelSummary}" in browser_store+def test_browser_viewer_uses_cdp_screencast_transport():+ ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(+ encoding="utf-8"+ )+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(+ encoding="utf-8"+ )+ runtime = (+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"+ ).read_text(encoding="utf-8")+ browser_store = (+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"+ ).read_text(encoding="utf-8")++ assert 'runtime.call("screenshot"' not in ws_browser+ assert "SCREENCAST_QUALITY = 92" in ws_browser+ assert "initial_viewport = self._viewport_from_data(data)" in ws_browser+ assert '"set_viewport"' in ws_browser+ assert "start_screencast" in ws_browser+ assert "read_screencast_frame" in ws_browser+ assert "stop_screencast" in ws_browser+ assert '"Page.startScreencast"' in runtime+ assert '"Page.screencastFrame"' in runtime+ assert '"Page.screencastFrameAck"' in runtime+ assert '"Page.stopScreencast"' in runtime+ assert '"Emulation.setDeviceMetricsOverride"' in runtime+ assert '"Emulation.setVisibleSize"' in runtime+ assert "asyncio.Queue(maxsize=1)" in runtime+ assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime+ assert "queueFrameRender" in browser_store+ assert "requestAnimationFrame" in browser_store+ assert "viewport_width: initialViewport?.width" in browser_store+ assert "viewport_height: initialViewport?.height" in browser_store+ assert "this.frameState = data.state || null" not in browser_store+ assert "overflow: hidden;" in main_html+ assert "object-fit: fill;" not in main_html+ assert "height: auto;" in main_html+ assert "image-rendering: auto;" in main_html+++async def test_browser_screencast_acknowledges_and_drops_stale_frames():+ first_image = (+ "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"+ "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"+ "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"+ "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"+ "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"+ "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"+ "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"+ "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"+ "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"+ "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"+ "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="+ )++ class FakeSession:+ def __init__(self):+ self.handlers = {}+ self.sent = []+ self.detached = False++ def on(self, event, handler):+ self.handlers[event] = handler++ async def send(self, method, params=None):+ self.sent.append((method, params or {}))++ async def detach(self):+ self.detached = True++ session = FakeSession()+ screencast = _BrowserScreencast(+ stream_id="stream",+ browser_id=7,+ session=session,+ mime="image/jpeg",+ )++ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})+ session.handlers["Page.screencastFrame"](+ {"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1}+ )+ session.handlers["Page.screencastFrame"](+ {"data": "second", "metadata": {"deviceWidth": 200}, "sessionId": 2}+ )+ await asyncio.sleep(0)++ frame = await screencast.next_frame(timeout=0.1)++ assert frame["browser_id"] == 7+ assert frame["image"] == "second"+ assert frame["metadata"]["deviceWidth"] == 200+ assert ("Emulation.setDeviceMetricsOverride", {+ "width": 1118,+ "height": 662,+ "deviceScaleFactor": 1,+ "mobile": False,+ "dontSetVisibleSize": True,+ }) in session.sent+ assert ("Emulation.setVisibleSize", {"width": 1118, "height": 662}) in session.sent+ assert ("Page.screencastFrameAck", {"sessionId": 1}) in session.sent+ assert ("Page.screencastFrameAck", {"sessionId": 2}) in session.sent++ await screencast.stop()++ assert ("Page.stopScreencast", {}) in session.sent+ assert session.detached is True++def test_browser_docker_installs_full_chromium_to_persistent_cache():script = (PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"