Skip to content

Agent Zero AI framework

Add interactive internal Browser viewport

Alessandro005b366b5147
diff --git a/helpers/virtual_desktop.py b/helpers/virtual_desktop.py
index d417cf4ad..8de9ded15 100644
--- a/helpers/virtual_desktop.py
+++ b/helpers/virtual_desktop.py
@@ -115,29 +115,38 @@ def get_registry() -> VirtualDesktopRegistry:
return _registry
-def session_url(token: str, *, title: str = "Desktop") -> str:
+def session_url(
+ token: str,
+ *,
+ title: str = "Desktop",
+ encoding: str = "jpeg",
+ quality: int = 85,
+ speed: int = 80,
+ file_transfer: bool = True,
+ printing: bool = True,
+) -> str:
quoted_token = quote(str(token), safe="")
base_path = f"{SESSION_PATH}/{quoted_token}/"
- query = urlencode(
- {
- "path": base_path,
- "title": title,
- "encoding": "jpeg",
- "quality": "85",
- "speed": "80",
- "sharing": "true",
- "clipboard": "true",
- "clipboard_direction": "both",
- "clipboard_poll": "true",
- "clipboard_preferred_format": "text/plain",
- "printing": "true",
- "file_transfer": "true",
- "sound": "false",
- "offscreen": "true",
- "floating_menu": "false",
- "xpramenu": "false",
- },
- )
+ options = {
+ "path": base_path,
+ "title": title,
+ "quality": str(max(0, min(100, int(quality)))),
+ "speed": str(max(0, min(100, int(speed)))),
+ "sharing": "true",
+ "clipboard": "true",
+ "clipboard_direction": "both",
+ "clipboard_poll": "true",
+ "clipboard_preferred_format": "text/plain",
+ "printing": str(bool(printing)).lower(),
+ "file_transfer": str(bool(file_transfer)).lower(),
+ "sound": "false",
+ "offscreen": "true",
+ "floating_menu": "false",
+ "xpramenu": "false",
+ }
+ if encoding:
+ options["encoding"] = str(encoding)
+ query = urlencode(options)
return f"{base_path}index.html?{query}"
@@ -264,6 +273,7 @@ def resize_display(
keys: tuple[str, ...] = (),
xauthority: str = "",
home: str = "",
+ settle_seconds: float = 0.15,
) -> dict[str, Any]:
target_width, target_height = normalize_size(width, height, max_width=max_width, max_height=max_height)
xrandr = shutil.which("xrandr")
@@ -296,7 +306,8 @@ def resize_display(
timeout=4,
env=env,
)
- time.sleep(0.15)
+ if settle_seconds > 0:
+ time.sleep(settle_seconds)
current = current_display_size(display, xauthority=xauthority, home=home)
ok = current == (target_width, target_height)
if ok:
diff --git a/helpers/virtual_desktop.py.dox.md b/helpers/virtual_desktop.py.dox.md
index 34e7b2510..2681e2cfb 100644
--- a/helpers/virtual_desktop.py.dox.md
+++ b/helpers/virtual_desktop.py.dox.md
@@ -23,13 +23,13 @@
- `proxy_for_token(token: str) -> VirtualDesktopEndpoint | None`
- `resize_session(token: str, width: int, height: int) -> dict[str, Any]`
- `get_registry() -> VirtualDesktopRegistry`
-- `session_url(token: str, title: str=...) -> str`
+- `session_url(token: str, title: str=..., encoding: str=..., quality: int=..., speed: int=..., file_transfer: bool=..., printing: bool=...) -> str`
- `collect_status() -> dict[str, Any]`
- `find_xpra_html_root() -> Path | None`
- `_package_installed(package: str) -> bool`
- `normalize_size(width: int | float | str, height: int | float | str, max_width: int=..., max_height: int=..., min_width: int=..., min_height: int=...) -> tuple[int, int]`
- `normalize_desktop_display_size(width: int | float | str, height: int | float | str, max_width: int=..., max_height: int=..., min_width: int=..., min_height: int=..., min_aspect_ratio: float=...) -> tuple[int, int]`
-- `resize_display(display: int, width: int, height: int, max_width: int=..., max_height: int=..., window_class: str=..., keys: tuple[str, ...]=..., xauthority: str=..., home: str=...) -> dict[str, Any]`
+- `resize_display(display: int, width: int, height: int, max_width: int=..., max_height: int=..., window_class: str=..., keys: tuple[str, ...]=..., xauthority: str=..., home: str=..., settle_seconds: float=...) -> dict[str, Any]`
- `_ensure_xrandr_mode(env: dict[str, str], width: int, height: int) -> None`
- `_select_xrandr_mode(env: dict[str, str], width: int, height: int) -> subprocess.CompletedProcess[str]`
- `_xrandr_output_modes(env: dict[str, str]) -> tuple[str, set[str]]`
@@ -54,6 +54,8 @@
- Important called helpers/classes observed in the source: `Path`, `files.get_abs_path`, `get_registry.register`, `get_registry.unregister`, `get_registry.proxy_for_token`, `get_registry.resize`, `quote`, `urlencode`, `find_xpra_html_root`, `subprocess.run`, `normalize_size`, `shutil.which`, `_display_env`, `current_display_size`, `_ensure_xrandr_mode`, `_select_xrandr_mode`, `time.sleep`, `strip`, `_xrandr_output_modes`, `result.stdout.splitlines`.
- Keep request/response, tool, or helper semantics documented here at the same time as source changes.
+- Session URLs keep Desktop's JPEG, printing, and file-transfer defaults while allowing restricted viewers such as Browser to negotiate encoding and disable unrelated capabilities.
+- Display resizing keeps the Desktop settle delay by default; latency-sensitive callers may skip it when they immediately verify the XRandR size.
## Work Guidance
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index bfe481621..973ec15ca 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -2,23 +2,30 @@
## Purpose
-- Own the built-in Playwright browser tool and WebUI browser viewer.
+- Own the built-in Patchright browser tool and WebUI browser viewer.
- Bridge browser automation, page inspection helpers, and browser panel UI.
## Ownership
- `plugin.yaml` and `default_config.yaml` own metadata and browser settings defaults.
- `tools/browser.py` owns the agent-facing browser tool.
-- `helpers/` owns Playwright runtime, selectors, URL helpers, extension management, and connector runtime logic.
+- `helpers/` owns the Patchright runtime, private interactive display, selectors, URL helpers, extension management, and connector runtime logic.
- `api/` owns status, extension, and browser WebSocket handlers.
- `assets/`, `prompts/`, `skills/`, `extensions/`, and `webui/` own browser scripts, prompts, skill guidance, hook contributions, and UI.
## Local Contracts
- Keep browser actions safe around external pages, credentials, and user data.
-- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
+- Preserve Patchright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
-- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
+- Default the visible WebUI Browser to the authenticated Xpra HTML5 viewer for its existing Patchright page. Keep live CDP screencast and lightweight snapshots as automatic fallbacks.
+- Keep headful Chromium in a normal window with its own toolbar clipped above the private display; do not use browser fullscreen, which shows Chromium's exit warning.
+- Throttle interactive resize updates throughout a drag and let the native-sized Chromium viewport follow the private display; do not defer all layout updates until resizing stops.
+- Keep exactly one interactive viewer iframe connected during canvas/modal handoff so hidden surfaces cannot compete to resize the same display.
+- Notify the active Xpra client of its new frame geometry before resizing the backing display; after an interactive canvas/modal handoff, reconcile once after Xpra's deferred resize so Chromium cannot retain the previous surface size.
+- Present the Xpra shadow window as the raw browser canvas: remove its HTML decoration and shadow pointer while preserving exact viewport geometry.
+- Give every internal Browser runtime its own Xvfb display and unguessable Xpra gateway token; never expose another chat context's display through a shared viewer.
+- Bind Browser Xpra endpoints to loopback, route them through the authenticated virtual-desktop gateway, and keep file transfer, URL opening, printing, and audio disabled.
- Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the `<img>`/data URL path for snapshots and fallback rendering.
- Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path.
- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
@@ -35,6 +42,7 @@
- Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver.
- Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads.
- `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path.
+- Browser startup must install the shared virtual-desktop route hook itself; do not make Browser depend on the Desktop plugin being enabled.
## Work Guidance
@@ -47,7 +55,7 @@
## Verification
- Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes.
-- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path.
+- For viewer render-path changes, verify direct iframe interaction reaches the same page controlled by Patchright, separate contexts use separate displays, and an unavailable Xpra runtime falls back to CDP screencast/snapshot rendering.
- Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills.
## Child DOX Index
diff --git a/plugins/_browser/api/status.py b/plugins/_browser/api/status.py
index 2669fe677..8b9cefd80 100644
--- a/plugins/_browser/api/status.py
+++ b/plugins/_browser/api/status.py
@@ -1,5 +1,6 @@
from helpers.api import ApiHandler, Request
from plugins._browser.helpers.config import build_browser_launch_config, get_browser_config
+from plugins._browser.helpers.interactive_view import collect_status as collect_interactive_status
from plugins._browser.helpers.playwright import (
get_playwright_binary,
get_playwright_cache_dir,
@@ -37,5 +38,6 @@ class Status(ApiHandler):
"requires_full_browser": launch_config["requires_full_browser"],
},
"host_browser": host_browser,
+ "interactive_view": collect_interactive_status(),
"contexts": known_context_ids(),
}
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index 6c810ebc9..4c5b734e5 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -25,7 +25,12 @@ SCREENCAST_STREAM_QUALITY = 80
SCREENSHOT_QUALITY = 92
VIEWER_TRANSPORT_SCREENCAST = "screencast"
VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
-VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
+VIEWER_TRANSPORT_INTERACTIVE = "interactive"
+VIEWER_TRANSPORTS = {
+ VIEWER_TRANSPORT_INTERACTIVE,
+ VIEWER_TRANSPORT_SCREENCAST,
+ VIEWER_TRANSPORT_SNAPSHOT,
+}
class WsBrowser(WsHandler):
@@ -87,8 +92,19 @@ class WsBrowser(WsHandler):
if opened.get("id"):
listing["last_interacted_browser_id"] = opened.get("id")
active_id = self._active_browser_id(listing, data.get("browser_id"))
+ requested_transport = self._viewer_transport(data)
+ viewer_transport, interactive_view = await self._effective_viewer(
+ runtime,
+ active_id,
+ data,
+ )
initial_viewport = self._viewport_from_data(data)
- if runtime and active_id and initial_viewport:
+ if (
+ runtime
+ and active_id
+ and initial_viewport
+ and viewer_transport != VIEWER_TRANSPORT_INTERACTIVE
+ ):
await runtime.call(
"set_viewport",
active_id,
@@ -103,7 +119,6 @@ class WsBrowser(WsHandler):
if existing:
existing.cancel()
viewer_id = str(data.get("viewer_id") or "")
- viewer_transport = self._viewer_transport(data)
binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))
slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))
capture_scale = self._capture_scale_from_data(data)
@@ -120,7 +135,13 @@ class WsBrowser(WsHandler):
capture_scale=capture_scale,
)
else:
- stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
+ stream_task = self._stream_state(
+ sid,
+ context_id,
+ active_id,
+ viewer_id,
+ viewer_transport=viewer_transport,
+ )
self._streams[stream_key] = asyncio.create_task(stream_task)
snapshot = await self._snapshot_for_browser(runtime, active_id)
@@ -136,6 +157,14 @@ class WsBrowser(WsHandler):
"tab_scope": tab_scope,
"viewer_id": viewer_id,
"viewer_transport": viewer_transport,
+ "interactive_view": interactive_view,
+ "viewer_fallback_reason": (
+ str(interactive_view.get("error") or "")
+ if requested_transport == VIEWER_TRANSPORT_INTERACTIVE
+ and interactive_view
+ and not interactive_view.get("available")
+ else ""
+ ),
"binary_frames": binary_frames,
"slim_frames": slim_frames,
}
@@ -253,6 +282,15 @@ class WsBrowser(WsHandler):
listing = await runtime.call("list")
last_interacted_browser_id = listing.get("last_interacted_browser_id")
+ active_id = self._active_browser_id(
+ listing,
+ self._result_browser_id(result) or browser_id,
+ )
+ viewer_transport, interactive_view = await self._effective_viewer(
+ runtime,
+ active_id,
+ data,
+ )
snapshot = await self._snapshot_for_result(runtime, result)
browsers, all_browsers, tab_scope = await self._tabs_for_scope(
context_id,
@@ -273,7 +311,8 @@ class WsBrowser(WsHandler):
"all_browsers": all_browsers,
"tab_scope": tab_scope,
"last_interacted_browser_id": last_interacted_browser_id,
- "viewer_transport": self._viewer_transport(data),
+ "viewer_transport": viewer_transport,
+ "interactive_view": interactive_view,
},
correlation_id=data.get("correlationId"),
)
@@ -288,7 +327,8 @@ class WsBrowser(WsHandler):
"command": command,
"browser_id": browser_id,
"viewer_id": viewer_id,
- "viewer_transport": self._viewer_transport(data),
+ "viewer_transport": viewer_transport,
+ "interactive_view": interactive_view,
}
async def _input(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -326,12 +366,15 @@ class WsBrowser(WsHandler):
text=str(data.get("text") or ""),
)
elif input_type == "viewport":
+ viewer_transport = self._viewer_transport(data)
result = await runtime.call(
"set_viewport",
browser_id,
int(data.get("width") or 0),
int(data.get("height") or 0),
restart_screencast=bool(data.get("restart_stream")),
+ resize_interactive=viewer_transport == VIEWER_TRANSPORT_INTERACTIVE,
+ include_state=viewer_transport != VIEWER_TRANSPORT_INTERACTIVE,
)
elif input_type == "wheel":
result = await runtime.call(
@@ -582,6 +625,8 @@ class WsBrowser(WsHandler):
context_id: str,
browser_id: int | str | None,
viewer_id: str = "",
+ *,
+ viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,
) -> None:
last_signature = None
while True:
@@ -594,7 +639,7 @@ class WsBrowser(WsHandler):
sid,
context_id,
viewer_id=viewer_id,
- frame_source=VIEWER_TRANSPORT_SNAPSHOT,
+ frame_source=viewer_transport,
)
last_signature = signature
await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
@@ -625,7 +670,7 @@ class WsBrowser(WsHandler):
browsers=browsers,
viewer_id=viewer_id,
state=state,
- viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,
+ viewer_transport=viewer_transport,
)
last_signature = signature
await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
@@ -653,6 +698,36 @@ class WsBrowser(WsHandler):
active_id = browsers[0].get("id")
return active_id
+ async def _effective_viewer(
+ self,
+ runtime: Any,
+ browser_id: int | str | None,
+ data: dict[str, Any],
+ ) -> tuple[str, dict[str, Any] | None]:
+ requested = self._viewer_transport(data)
+ if requested != VIEWER_TRANSPORT_INTERACTIVE or not runtime or not browser_id:
+ return requested, None
+ viewport = self._viewport_from_data(data) or {}
+ try:
+ viewer = await runtime.call(
+ "interactive_viewer",
+ browser_id,
+ width=int(viewport.get("width") or 0),
+ height=int(viewport.get("height") or 0),
+ )
+ except Exception as exc:
+ viewer = {"available": False, "error": str(exc)}
+ if viewer.get("available"):
+ return VIEWER_TRANSPORT_INTERACTIVE, viewer
+ return VIEWER_TRANSPORT_SCREENCAST, viewer
+
+ @staticmethod
+ def _result_browser_id(result: Any) -> int | str | None:
+ if not isinstance(result, dict):
+ return None
+ state = result.get("state") if isinstance(result.get("state"), dict) else result
+ return state.get("id") if isinstance(state, dict) else None
+
@staticmethod
def _state_for_browser(
browsers: list[dict[str, Any]],
diff --git a/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py b/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py
index e76a19856..8839dc55c 100644
--- a/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py
+++ b/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import threading
from typing import Any
+from helpers import virtual_desktop_routes
from helpers.extension import Extension
from helpers.print_style import PrintStyle
from plugins._browser import hooks
@@ -13,6 +14,7 @@ _startup_migration_thread: threading.Thread | None = None
class BrowserPlaywrightCacheMigration(Extension):
def execute(self, **kwargs):
+ virtual_desktop_routes.install_route_hooks()
_start_background_cache_migration()
diff --git a/plugins/_browser/helpers/interactive_view.py b/plugins/_browser/helpers/interactive_view.py
new file mode 100644
index 000000000..a603633d0
--- /dev/null
+++ b/plugins/_browser/helpers/interactive_view.py
@@ -0,0 +1,293 @@
+from __future__ import annotations
+
+import os
+import select
+import shutil
+import socket
+import subprocess
+import threading
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+from helpers import files, virtual_desktop
+from helpers.print_style import PrintStyle
+
+
+DEFAULT_WIDTH = 1024
+DEFAULT_HEIGHT = 768
+START_TIMEOUT_SECONDS = 15.0
+
+
+def collect_status() -> dict[str, Any]:
+ binaries = {
+ name: shutil.which(name) or ""
+ for name in ("Xvfb", "xpra", "xrandr")
+ }
+ html_root = virtual_desktop.find_xpra_html_root()
+ missing = [name for name, path in binaries.items() if not path]
+ if not html_root:
+ missing.append("xpra-html5")
+ return {
+ "available": not missing,
+ "missing": missing,
+ "binaries": binaries,
+ "xpra_html_root": str(html_root or ""),
+ }
+
+
+class BrowserInteractiveView:
+ """Own one private X display and its optional Xpra viewer."""
+
+ def __init__(self, context_id: str) -> None:
+ self.context_id = str(context_id)
+ self.token = f"browser-{uuid.uuid4().hex}"
+ self.state_dir = Path(files.get_abs_path("tmp", "browser", "displays", self.token))
+ self.display: int | None = None
+ self.port = 0
+ self.width = DEFAULT_WIDTH
+ self.height = DEFAULT_HEIGHT
+ self._xvfb: subprocess.Popen[Any] | None = None
+ self._xpra: subprocess.Popen[Any] | None = None
+ self._lock = threading.RLock()
+
+ @property
+ def display_name(self) -> str:
+ return f":{self.display}" if self.display is not None else ""
+
+ def ensure_display(self) -> str:
+ with self._lock:
+ if self._running(self._xvfb) and self.display is not None:
+ return self.display_name
+
+ self._stop_locked()
+ xvfb = shutil.which("Xvfb")
+ if not xvfb:
+ return ""
+
+ self.state_dir.mkdir(parents=True, exist_ok=True)
+ self.state_dir.chmod(0o700)
+ read_fd, write_fd = os.pipe()
+ try:
+ process = subprocess.Popen(
+ [
+ xvfb,
+ "-displayfd",
+ str(write_fd),
+ "-screen",
+ "0",
+ f"{virtual_desktop.MAX_WIDTH}x{virtual_desktop.MAX_HEIGHT}x24",
+ "+extension",
+ "GLX",
+ "+extension",
+ "RANDR",
+ "+extension",
+ "RENDER",
+ "+extension",
+ "Composite",
+ "-nolisten",
+ "tcp",
+ "-noreset",
+ "-ac",
+ ],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ pass_fds=(write_fd,),
+ )
+ except OSError:
+ os.close(read_fd)
+ return ""
+ finally:
+ os.close(write_fd)
+
+ try:
+ ready, _, _ = select.select([read_fd], [], [], 5)
+ display_number = os.read(read_fd, 32).decode().strip() if ready else ""
+ finally:
+ os.close(read_fd)
+
+ if not display_number.isdigit() or process.poll() is not None:
+ self._terminate(process)
+ return ""
+
+ self._xvfb = process
+ self.display = int(display_number)
+ self.resize(self.width, self.height)
+ return self.display_name
+
+ def ensure_viewer(self, width: int = 0, height: int = 0) -> dict[str, Any]:
+ with self._lock:
+ display_name = self.ensure_display()
+ if not display_name:
+ return self._unavailable("Xvfb is unavailable.")
+
+ status = collect_status()
+ if not status["available"]:
+ return self._unavailable(
+ f"Interactive Browser runtime needs: {', '.join(status['missing'])}."
+ )
+
+ self.resize(width or self.width, height or self.height)
+ if not self._running(self._xpra):
+ try:
+ self._start_xpra(str(status["binaries"]["xpra"]))
+ except Exception as exc:
+ PrintStyle.warning(f"Interactive Browser viewer failed to start: {exc}")
+ self._terminate(self._xpra)
+ self._xpra = None
+ self.port = 0
+ virtual_desktop.unregister_session(self.token)
+ return self._unavailable(str(exc))
+
+ virtual_desktop.register_session(
+ token=self.token,
+ host="127.0.0.1",
+ port=self.port,
+ owner="browser",
+ title="Browser",
+ resize=self.resize,
+ )
+ return {
+ "available": True,
+ "token": self.token,
+ "url": virtual_desktop.session_url(
+ self.token,
+ title="Browser",
+ encoding="",
+ quality=90,
+ speed=90,
+ file_transfer=False,
+ printing=False,
+ ),
+ "width": self.width,
+ "height": self.height,
+ }
+
+ def resize(self, width: int, height: int) -> dict[str, Any]:
+ with self._lock:
+ target_width, target_height = virtual_desktop.normalize_size(width, height)
+ self.width = target_width
+ self.height = target_height
+ if self.display is None or not self._running(self._xvfb):
+ return {
+ "ok": False,
+ "error": "Browser display is unavailable.",
+ "width": target_width,
+ "height": target_height,
+ }
+ result = virtual_desktop.resize_display(
+ display=self.display,
+ width=target_width,
+ height=target_height,
+ settle_seconds=0,
+ )
+ return result
+
+ def close(self) -> None:
+ with self._lock:
+ self._stop_locked()
+ shutil.rmtree(self.state_dir, ignore_errors=True)
+
+ def _start_xpra(self, xpra: str) -> None:
+ self.port = self._free_port()
+ runtime_dir = self.state_dir / "runtime"
+ socket_dir = self.state_dir / "sockets"
+ runtime_dir.mkdir(parents=True, exist_ok=True)
+ socket_dir.mkdir(parents=True, exist_ok=True)
+ runtime_dir.chmod(0o700)
+ env = {
+ **os.environ,
+ "DISPLAY": self.display_name,
+ "XDG_RUNTIME_DIR": str(runtime_dir),
+ }
+ self._xpra = subprocess.Popen(
+ [
+ xpra,
+ "shadow",
+ self.display_name,
+ "--daemon=no",
+ "--mdns=no",
+ "--html=on",
+ "--tray=no",
+ "--system-tray=no",
+ "--notifications=no",
+ "--clipboard=yes",
+ "--clipboard-direction=both",
+ "--file-transfer=no",
+ "--open-files=no",
+ "--open-url=no",
+ "--printing=no",
+ "--audio=no",
+ "--speaker=off",
+ "--microphone=off",
+ "--sharing=yes",
+ "--resize-display=yes",
+ "--encoding=auto",
+ "--quality=90",
+ "--speed=90",
+ f"--bind-tcp=127.0.0.1:{self.port}",
+ f"--socket-dir={socket_dir}",
+ f"--log-dir={self.state_dir}",
+ "--log-file=xpra.log",
+ ],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ env=env,
+ )
+ self._wait_for_port(self._xpra, self.port)
+
+ def _stop_locked(self) -> None:
+ virtual_desktop.unregister_session(self.token)
+ self._terminate(self._xpra)
+ self._terminate(self._xvfb)
+ self._xpra = None
+ self._xvfb = None
+ self.port = 0
+ self.display = None
+
+ def _unavailable(self, error: str) -> dict[str, Any]:
+ return {
+ "available": False,
+ "error": str(error or "Interactive Browser viewer is unavailable."),
+ }
+
+ @staticmethod
+ def _running(process: subprocess.Popen[Any] | None) -> bool:
+ return bool(process and process.poll() is None)
+
+ @staticmethod
+ def _terminate(process: subprocess.Popen[Any] | None) -> None:
+ if not process or process.poll() is not None:
+ return
+ process.terminate()
+ try:
+ process.wait(timeout=2)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait(timeout=2)
+
+ @staticmethod
+ def _free_port() -> int:
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+ probe.bind(("127.0.0.1", 0))
+ return int(probe.getsockname()[1])
+
+ @staticmethod
+ def _wait_for_port(
+ process: subprocess.Popen[Any],
+ port: int,
+ timeout: float = START_TIMEOUT_SECONDS,
+ ) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if process.poll() is not None:
+ raise RuntimeError("Xpra exited before its Browser endpoint was ready.")
+ try:
+ with socket.create_connection(("127.0.0.1", port), timeout=0.2):
+ return
+ except OSError:
+ time.sleep(0.1)
+ raise TimeoutError("Timed out waiting for the interactive Browser endpoint.")
diff --git a/plugins/_browser/helpers/playwright.py b/plugins/_browser/helpers/playwright.py
index f0463e4fa..bd0a2d639 100644
--- a/plugins/_browser/helpers/playwright.py
+++ b/plugins/_browser/helpers/playwright.py
@@ -1,9 +1,6 @@
-import atexit
import json
import os
import re
-import select
-import shutil
import subprocess
import sys
import threading
@@ -23,9 +20,6 @@ RETIRED_PLAYWRIGHT_CACHE_DIRS = (
("usr", "browser", "playwright"),
)
_INSTALL_LOCK = threading.Lock()
-_DISPLAY_LOCK = threading.Lock()
-_DISPLAY_PROCESS: subprocess.Popen | None = None
-_DISPLAY_NAME = ""
def _primary_cache_dir() -> Path:
@@ -64,82 +58,6 @@ def configure_playwright_env() -> str:
return cache_dir
-def ensure_browser_display() -> str:
- global _DISPLAY_NAME, _DISPLAY_PROCESS
-
- with _DISPLAY_LOCK:
- if _DISPLAY_PROCESS and _DISPLAY_PROCESS.poll() is None:
- return _DISPLAY_NAME
-
- xvfb = shutil.which("Xvfb")
- if not xvfb:
- return ""
-
- read_fd, write_fd = os.pipe()
- try:
- process = subprocess.Popen(
- [
- xvfb,
- "-displayfd",
- str(write_fd),
- "-screen",
- "0",
- "1365x768x24",
- "+extension",
- "GLX",
- "-nolisten",
- "tcp",
- "-noreset",
- "-ac",
- ],
- stdin=subprocess.DEVNULL,
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- pass_fds=(write_fd,),
- )
- except OSError:
- os.close(read_fd)
- return ""
- finally:
- os.close(write_fd)
-
- try:
- ready, _, _ = select.select([read_fd], [], [], 5)
- display_number = os.read(read_fd, 32).decode().strip() if ready else ""
- finally:
- os.close(read_fd)
-
- if not display_number.isdigit() or process.poll() is not None:
- _terminate_browser_display(process)
- return ""
-
- _DISPLAY_PROCESS = process
- _DISPLAY_NAME = f":{display_number}"
- return _DISPLAY_NAME
-
-
-def close_browser_display() -> None:
- global _DISPLAY_NAME, _DISPLAY_PROCESS
-
- with _DISPLAY_LOCK:
- process = _DISPLAY_PROCESS
- _DISPLAY_PROCESS = None
- _DISPLAY_NAME = ""
- if process:
- _terminate_browser_display(process)
-
-
-def _terminate_browser_display(process: subprocess.Popen) -> None:
- if process.poll() is not None:
- return
- process.terminate()
- try:
- process.wait(timeout=2)
- except subprocess.TimeoutExpired:
- process.kill()
- process.wait(timeout=2)
-
-
def find_playwright_binary(cache_dir: Path, revision: str = "") -> Path | None:
prefix = f"chromium-{revision}" if revision.isdigit() else "chromium-*"
binaries = [
@@ -197,6 +115,3 @@ def ensure_playwright_binary() -> Path:
if not binary:
raise RuntimeError("Patchright Chromium binary not found after installation")
return binary
-
-
-atexit.register(close_browser_display)
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 040503cba..1514d6b40 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -27,9 +27,7 @@ from plugins._browser.helpers.config import (
build_browser_launch_config,
get_browser_config,
)
-from plugins._browser.helpers.playwright import (
- ensure_browser_display,
-)
+from plugins._browser.helpers.interactive_view import BrowserInteractiveView
from plugins._browser.helpers.url import normalize_url
@@ -558,7 +556,10 @@ class BrowserRuntime:
await self.call("close", delete_profile=delete_profile)
finally:
self._closed = True
- self._worker.kill(terminate_thread=True)
+ try:
+ self._worker.kill(terminate_thread=True)
+ finally:
+ self._core.interactive_view.close()
class _BrowserRuntimeCore:
@@ -594,6 +595,11 @@ class _BrowserRuntimeCore:
self._pending_popups: list[asyncio.Future[int]] = []
self._background_popup_pages: set[int] = set()
self._bootstrap_page: Any | None = None
+ self._browser_chrome_height: int | None = None
+ self._browser_window_page: Any | None = None
+ self._browser_window_session: Any | None = None
+ self._browser_window_id: int | None = None
+ self.interactive_view = BrowserInteractiveView(context_id)
def _ensure_registry_lock(self) -> asyncio.Lock:
if self._registry_lock is None:
@@ -777,6 +783,10 @@ class _BrowserRuntimeCore:
self._pending_popups.clear()
self._background_popup_pages.clear()
self._bootstrap_page = None
+ self._browser_chrome_height = None
+ self._browser_window_page = None
+ self._browser_window_session = None
+ self._browser_window_id = None
self.pages.clear()
self.last_interacted_browser_id = None
for screencast in self.screencasts.values():
@@ -815,7 +825,15 @@ class _BrowserRuntimeCore:
browser_config = get_browser_config()
launch_config = build_browser_launch_config(browser_config)
browser_binary = Path(preparation["binary"])
- browser_display = ensure_browser_display()
+ browser_display = self.interactive_view.ensure_display()
+ launch_args = list(launch_config["args"])
+ if browser_display:
+ launch_args.extend(
+ [
+ "--window-position=0,0",
+ f"--window-size={self.interactive_view.width},{self.interactive_view.height}",
+ ]
+ )
self.playwright = await async_playwright().start()
launch_kwargs: dict[str, Any] = {
@@ -823,13 +841,17 @@ class _BrowserRuntimeCore:
"headless": not bool(browser_display),
"accept_downloads": True,
"downloads_path": str(self.downloads_dir),
- "viewport": DEFAULT_VIEWPORT,
- "screen": DEFAULT_VIEWPORT,
- "no_viewport": False,
- "args": launch_config["args"],
+ "args": launch_args,
}
if browser_display:
launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display}
+ launch_kwargs["no_viewport"] = True
+ else:
+ launch_kwargs.update(
+ viewport=DEFAULT_VIEWPORT,
+ screen=DEFAULT_VIEWPORT,
+ no_viewport=False,
+ )
if launch_config["channel"]:
launch_kwargs["channel"] = launch_config["channel"]
else:
@@ -857,6 +879,7 @@ class _BrowserRuntimeCore:
if page.url == "about:blank":
if browser_display and self._bootstrap_page is None:
self._bootstrap_page = page
+ await self._fit_browser_window(page)
continue
try:
await page.close()
@@ -1236,8 +1259,12 @@ class _BrowserRuntimeCore:
async def set_active(self, browser_id: int | str | None) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
+ page = self._page(resolved_id)
# Explicit focus change — bypass _maybe_promote.
self.last_interacted_browser_id = int(resolved_id)
+ with contextlib.suppress(Exception):
+ await page.bring_to_front()
+ await self._fit_browser_window(page)
return await self._state(resolved_id)
async def state(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -1703,7 +1730,7 @@ class _BrowserRuntimeCore:
await screencast.start(
quality=quality,
every_nth_frame=every_nth_frame,
- viewport=page.viewport_size or DEFAULT_VIEWPORT,
+ viewport=await self._page_viewport(page),
capture_scale=capture_scale,
)
except Exception:
@@ -1717,6 +1744,61 @@ class _BrowserRuntimeCore:
"state": await self._state(resolved_id),
}
+ @staticmethod
+ async def _page_viewport(page: Any) -> dict[str, int]:
+ viewport = getattr(page, "viewport_size", None)
+ if viewport:
+ return {
+ "width": int(viewport.get("width") or DEFAULT_VIEWPORT["width"]),
+ "height": int(viewport.get("height") or DEFAULT_VIEWPORT["height"]),
+ }
+ try:
+ measured = await page.evaluate(
+ "() => ({ width: globalThis.innerWidth, height: globalThis.innerHeight })",
+ isolated_context=False,
+ )
+ return {
+ "width": int(measured.get("width") or DEFAULT_VIEWPORT["width"]),
+ "height": int(measured.get("height") or DEFAULT_VIEWPORT["height"]),
+ }
+ except Exception:
+ return dict(DEFAULT_VIEWPORT)
+
+ async def interactive_viewer(
+ self,
+ browser_id: int | str | None = None,
+ *,
+ width: int = 0,
+ height: int = 0,
+ ) -> dict[str, Any]:
+ await self.ensure_started()
+ resolved_id = self._resolve_browser_id(browser_id)
+ page = self._page(resolved_id)
+ current_viewport = await self._page_viewport(page)
+ viewer = self.interactive_view.ensure_viewer(
+ width or int(current_viewport.get("width") or DEFAULT_VIEWPORT["width"]),
+ height or int(current_viewport.get("height") or DEFAULT_VIEWPORT["height"]),
+ )
+ if not viewer.get("available"):
+ return viewer
+
+ await self._stop_screencasts_for_browser(resolved_id)
+ with contextlib.suppress(Exception):
+ await page.bring_to_front()
+ viewport_result = await self.set_viewport(
+ resolved_id,
+ int(viewer.get("width") or width or DEFAULT_VIEWPORT["width"]),
+ int(viewer.get("height") or height or DEFAULT_VIEWPORT["height"]),
+ resize_interactive=True,
+ )
+ self.last_interacted_browser_id = int(resolved_id)
+ return {
+ **viewer,
+ "browser_id": resolved_id,
+ "state": viewport_result["state"],
+ "viewport": viewport_result["viewport"],
+ }
+
async def read_screencast_frame(
self,
stream_id: str,
@@ -1756,15 +1838,30 @@ class _BrowserRuntimeCore:
width: int,
height: int,
restart_screencast: bool = False,
+ resize_interactive: bool = False,
+ include_state: bool = True,
) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
+ if resize_interactive:
+ resized = self.interactive_view.resize(width, height)
+ viewport = {
+ "width": int(resized.get("width") or self.interactive_view.width),
+ "height": int(resized.get("height") or self.interactive_view.height),
+ }
+ await self._fit_browser_window(page)
+ self._maybe_promote(resolved_id)
+ return {
+ "state": await self._state(resolved_id) if include_state else None,
+ "viewport": viewport,
+ }
+
viewport = {
"width": max(320, min(4096, int(width or DEFAULT_VIEWPORT["width"]))),
"height": max(200, min(4096, int(height or DEFAULT_VIEWPORT["height"]))),
}
- current_viewport = page.viewport_size or {}
+ current_viewport = await self._page_viewport(page)
changed = (
abs(int(current_viewport.get("width") or 0) - viewport["width"])
> VIEWPORT_SIZE_TOLERANCE
@@ -2182,6 +2279,7 @@ class _BrowserRuntimeCore:
self._pending_popups.clear()
self._background_popup_pages.clear()
await self._stop_all_screencasts()
+ await self._reset_browser_window_session()
for browser_id in list(self.pages):
try:
await self.pages[browser_id].page.close()
@@ -2315,7 +2413,67 @@ class _BrowserRuntimeCore:
async def _register_page(self, page: Any) -> BrowserPage:
lock = self._ensure_registry_lock()
async with lock:
- return self._register_page_locked(page)
+ browser_page = self._register_page_locked(page)
+ await self._fit_browser_window(page)
+ return browser_page
+
+ async def _fit_browser_window(self, page: Any) -> None:
+ if getattr(self.interactive_view, "display", None) is None or not self.context:
+ return
+ try:
+ if self._browser_window_session is None or self._browser_window_page is not page:
+ await self._reset_browser_window_session()
+ self._browser_window_page = page
+ self._browser_window_session = await self.context.new_cdp_session(page)
+ target = await self._browser_window_session.send("Browser.getWindowForTarget")
+ self._browser_window_id = target.get("windowId")
+ if self._browser_window_id is None:
+ await self._reset_browser_window_session()
+ return
+ current = await self._browser_window_session.send(
+ "Browser.getWindowBounds",
+ {"windowId": self._browser_window_id},
+ )
+ if current.get("bounds", {}).get("windowState") != "normal":
+ await self._browser_window_session.send(
+ "Browser.setWindowBounds",
+ {
+ "windowId": self._browser_window_id,
+ "bounds": {"windowState": "normal"},
+ },
+ )
+ if self._browser_chrome_height is None:
+ chrome_height = await page.evaluate(
+ "() => Math.max(0, globalThis.outerHeight - globalThis.innerHeight)",
+ isolated_context=False,
+ )
+ self._browser_chrome_height = max(0, min(256, int(chrome_height or 0)))
+ chrome_height = self._browser_chrome_height
+ await self._browser_window_session.send(
+ "Browser.setWindowBounds",
+ {
+ "windowId": self._browser_window_id,
+ "bounds": {
+ "windowState": "normal",
+ "left": 0,
+ "top": -chrome_height,
+ "width": self.interactive_view.width,
+ "height": self.interactive_view.height + chrome_height,
+ },
+ },
+ )
+ except Exception as exc:
+ await self._reset_browser_window_session()
+ PrintStyle.warning(f"Interactive Browser window fit failed: {exc}")
+
+ async def _reset_browser_window_session(self) -> None:
+ session = self._browser_window_session
+ self._browser_window_page = None
+ self._browser_window_session = None
+ self._browser_window_id = None
+ if session:
+ with contextlib.suppress(Exception):
+ await session.detach()
async def _unregister_page_async(self, browser_id: int) -> None:
try:
@@ -2372,6 +2530,8 @@ class _BrowserRuntimeCore:
if close_over_limit:
with contextlib.suppress(Exception):
await page.close()
+ else:
+ await self._fit_browser_window(page)
except Exception as exc:
PrintStyle.warning(f"Popup registration failed: {exc}")
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index 05d0f43b7..96e8bdfe7 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -20,10 +20,13 @@
<div class="browser-tab-shell" :class="{ 'is-active': $store.browserPage.isActiveBrowser(browser) }">
<button type="button" class="browser-tab" role="tab"
:aria-selected="$store.browserPage.isActiveBrowser(browser).toString()"
+ :aria-busy="$store.browserPage.isBrowserLoading(browser).toString()"
:title="$store.browserPage.browserTabTooltip(browser)"
@click="$store.browserPage.selectBrowser(browser.id, browser.context_id)">
<x-icon class="browser-tab-icon" aria-hidden="true" name="language"></x-icon>
<span class="browser-tab-title" x-text="$store.browserPage.browserTabTitle(browser)"></span>
+ <x-icon class="browser-tab-loading spinning" aria-hidden="true"
+ x-show="$store.browserPage.isBrowserLoading(browser)" name="progress_activity"></x-icon>
</button>
<button type="button" class="browser-tab-close"
:title="'Close ' + $store.browserPage.browserTabLabel(browser)"
@@ -185,15 +188,22 @@
</form>
</div>
- <div class="browser-stage" tabindex="0" @click="$el.focus()"
- :class="{ 'is-annotating': $store.browserPage.annotating }"
- @wheel.prevent="$store.browserPage.handleStageWheel($event)">
- <canvas class="browser-frame browser-frame-canvas"
- x-show="$store.browserPage.frameCanvasReady"
- x-init="$store.browserPage.attachFrameCanvas($el)"
- @click="$store.browserPage.sendMouse('click', $event)"
- @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
- <template x-if="$store.browserPage.frameSrc">
+ <div class="browser-stage" tabindex="0" @click="$el.focus()"
+ :class="{ 'is-annotating': $store.browserPage.annotating }"
+ @wheel.prevent="$store.browserPage.handleStageWheel($event)">
+ <template x-if="$store.browserPage.isInteractiveSurface($el.parentElement)">
+ <iframe class="browser-interactive-frame"
+ :src="$store.browserPage.interactiveViewUrl"
+ aria-label="Browser viewport"
+ allow="clipboard-read; clipboard-write"
+ @load="$store.browserPage.onInteractiveViewLoad()"></iframe>
+ </template>
+ <canvas class="browser-frame browser-frame-canvas"
+ x-show="!$store.browserPage.usesInteractiveTransport() && $store.browserPage.frameCanvasReady"
+ x-init="$store.browserPage.attachFrameCanvas($el)"
+ @click="$store.browserPage.sendMouse('click', $event)"
+ @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
+ <template x-if="!$store.browserPage.usesInteractiveTransport() && $store.browserPage.frameSrc">
<img class="browser-frame browser-frame-image" :src="$store.browserPage.frameSrc"
@click="$store.browserPage.sendMouse('click', $event)"
@mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
@@ -282,18 +292,11 @@
</template>
</div>
- <div class="browser-bottom-status"
- :class="{ 'is-active': $store.browserPage.isBusy() || $store.browserPage.error, 'has-error': $store.browserPage.error && !$store.browserPage.isBusy() }">
- <template x-if="$store.browserPage.isBusy()">
- <div class="browser-status" :title="$store.browserPage.loadingMessage()">
- <x-icon class="spinning" name="progress_activity"></x-icon>
- <span x-text="$store.browserPage.loadingMessage()">Loading</span>
- </div>
- </template>
- <template x-if="!$store.browserPage.isBusy() && $store.browserPage.error">
+ <template x-if="$store.browserPage.error">
+ <div class="browser-bottom-status has-error">
<div class="browser-error" x-text="$store.browserPage.error" :title="$store.browserPage.error"></div>
- </template>
- </div>
+ </div>
+ </template>
</div>
</template>
</div>
@@ -603,6 +606,13 @@
font-weight: 600;
}
+ .browser-tab-loading {
+ flex: 0 0 auto;
+ color: color-mix(in srgb, var(--color-text) 66%, var(--color-primary) 34%);
+ font-size: 0.96rem;
+ line-height: 1;
+ }
+
.browser-tab-close {
display: inline-flex;
align-items: center;
@@ -997,19 +1007,31 @@
cursor: crosshair;
}
- .browser-frame {
- display: block;
- position: absolute;
- inset: 0;
- width: 100%;
- height: 100%;
- min-width: 0;
- min-height: 0;
- object-fit: contain;
- image-rendering: auto;
- user-select: none;
- background: #fff;
- }
+ .browser-frame {
+ display: block;
+ position: absolute;
+ inset: 0;
+ width: 100%;
+ height: 100%;
+ min-width: 0;
+ min-height: 0;
+ object-fit: contain;
+ image-rendering: auto;
+ user-select: none;
+ background: #fff;
+ }
+
+ .browser-interactive-frame {
+ display: block;
+ position: absolute;
+ inset: 0;
+ z-index: 1;
+ box-sizing: border-box;
+ width: 100%;
+ height: 100%;
+ border: 0;
+ background: #fff;
+ }
.browser-annotation-layer {
position: absolute;
@@ -1239,7 +1261,6 @@
background: color-mix(in srgb, #7f1d1d 22%, var(--color-background));
}
- .browser-status,
.browser-error {
display: inline-flex;
align-items: center;
@@ -1250,17 +1271,12 @@
white-space: nowrap;
}
- .browser-status span:not(.material-symbols-outlined),
.browser-error {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
}
- .browser-status .material-symbols-outlined {
- font-size: 15px;
- }
-
.browser-modal .spinning,
.browser-panel .spinning {
display: inline-block;
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index cd8d85884..2260f87f0 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -21,11 +21,13 @@ const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;
const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
const BROWSER_COMMAND_TIMEOUT_MS = 45000;
const BROWSER_CONFIG_REFRESH_MS = 15000;
+const BROWSER_VIEWER_TRANSPORT_INTERACTIVE = "interactive";
const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";
const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";
-const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
+const VIEWPORT_SYNC_INTERVAL_MS = 50;
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
const CANVAS_VIEWPORT_SETTLE_MS = 520;
+const INTERACTIVE_VIEWPORT_SETTLE_MS = 320;
const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;
const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
@@ -168,9 +170,10 @@ const model = {
frameSrc: "",
frameCanvasReady: false,
frameState: null,
- viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST,
+ viewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE,
+ interactiveViewUrl: "",
+ viewerFallbackReason: "",
tabScope: "per_context",
- liveScreencastEnabled: true,
annotating: false,
annotationComments: [],
annotationDraft: null,
@@ -216,6 +219,7 @@ const model = {
_openSignature: "",
_connectSequence: 0,
_viewerToken: "",
+ _subscribedViewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE,
_contextCreatePromise: null,
_lastSelectedContextId: "",
_sessionRefreshPromise: null,
@@ -879,6 +883,7 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
+ this.interactiveViewUrl = "";
this.clearFrameSrc();
this.clearFrameCanvas();
this._lastFrameDimensions = null;
@@ -932,6 +937,7 @@ const model = {
async syncViewportAfterSurfaceOpen(sequence = this._surfaceOpenSequence) {
if (!this.connected || !this.activeBrowserId) return;
+ const surfaceMode = this._mode;
await this.waitForSurfaceViewport({ sequence });
if (!this.isCurrentSurfaceOpen(sequence)) {
return;
@@ -939,19 +945,24 @@ const model = {
await this.syncViewport(true, {
restartStream: this._mode === "canvas" && this.usesScreencastTransport(),
});
- if (this._mode !== "canvas") return;
- this.scheduleViewportSyncForSurface(sequence, 240);
- this.scheduleViewportSyncForSurface(sequence, 520);
+ if (surfaceMode === "modal" && this.usesInteractiveTransport()) {
+ this.scheduleViewportSyncForSurface(sequence, INTERACTIVE_VIEWPORT_SETTLE_MS, surfaceMode);
+ return;
+ }
+ if (surfaceMode !== "canvas") return;
+ this.scheduleViewportSyncForSurface(sequence, 240, surfaceMode);
+ this.scheduleViewportSyncForSurface(sequence, 520, surfaceMode);
},
requestedViewerTransport() {
- return this.liveScreencastEnabled
- ? BROWSER_VIEWER_TRANSPORT_SCREENCAST
- : BROWSER_VIEWER_TRANSPORT_SNAPSHOT;
+ return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;
},
normalizeViewerTransport(value = "") {
const normalized = String(value || "").trim().toLowerCase().replace("-", "_");
+ if (normalized === BROWSER_VIEWER_TRANSPORT_INTERACTIVE) {
+ return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;
+ }
if (normalized === BROWSER_VIEWER_TRANSPORT_SCREENCAST) {
return BROWSER_VIEWER_TRANSPORT_SCREENCAST;
}
@@ -974,6 +985,101 @@ const model = {
return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
},
+ usesInteractiveTransport() {
+ return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_INTERACTIVE
+ && Boolean(this.interactiveViewUrl);
+ },
+
+ isInteractiveSurface(stage = null) {
+ return this.usesInteractiveTransport() && stage === this._stageElement;
+ },
+
+ prepareInteractiveViewFrame(frame = null) {
+ const target = frame || this._stageElement?.querySelector?.(".browser-interactive-frame");
+ const remoteWindow = target?.contentWindow;
+ if (!remoteWindow) return false;
+ try {
+ const remoteDocument = target.contentDocument || remoteWindow.document;
+ if (!remoteDocument) return false;
+ if (!remoteDocument.getElementById("a0-xpra-browser-frame-css")) {
+ const style = remoteDocument.createElement("style");
+ style.id = "a0-xpra-browser-frame-css";
+ style.textContent = `
+ #shadow_pointer {
+ display: none !important;
+ visibility: hidden !important;
+ opacity: 0 !important;
+ }
+ .window canvas,
+ .undecorated canvas {
+ display: block !important;
+ margin: 0 !important;
+ }
+ `;
+ remoteDocument.head?.appendChild(style);
+ }
+
+ const normalizeWindows = () => {
+ const windows = Object.values(remoteWindow.client?.id_to_window || {});
+ for (const xpraWindow of windows) {
+ xpraWindow.resizable = false;
+ xpraWindow.decorations = false;
+ xpraWindow.decorated = false;
+ xpraWindow.metadata = { ...(xpraWindow.metadata || {}), decorations: false };
+ xpraWindow._set_decorated?.(false);
+ xpraWindow.configure_border_class?.();
+ xpraWindow.leftoffset = 0;
+ xpraWindow.rightoffset = 0;
+ xpraWindow.topoffset = 0;
+ xpraWindow.bottomoffset = 0;
+ xpraWindow.updateCSSGeometry?.();
+ }
+ return windows.length > 0;
+ };
+
+ const screen = remoteDocument.querySelector?.("#screen");
+ if (screen && !remoteWindow.__a0BrowserFrameObserver && remoteWindow.MutationObserver) {
+ const observer = new remoteWindow.MutationObserver(normalizeWindows);
+ observer.observe(screen, { childList: true });
+ remoteWindow.__a0BrowserFrameObserver = observer;
+ }
+ return normalizeWindows();
+ } catch {
+ return false;
+ }
+ },
+
+ syncInteractiveViewSize() {
+ if (!this.usesInteractiveTransport()) return;
+ const frame = this._stageElement?.querySelector?.(".browser-interactive-frame");
+ try {
+ this.prepareInteractiveViewFrame(frame);
+ frame?.contentWindow?.client?._screen_resized?.();
+ } catch {}
+ },
+
+ applyViewer(data = {}) {
+ if (data?.viewer_transport) {
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
+ }
+ if (Object.prototype.hasOwnProperty.call(data || {}, "interactive_view")) {
+ const viewer = data.interactive_view;
+ this.interactiveViewUrl = viewer?.available && viewer?.url ? String(viewer.url) : "";
+ this.viewerFallbackReason = String(data.viewer_fallback_reason || viewer?.error || "");
+ }
+ if (this.viewerTransport !== BROWSER_VIEWER_TRANSPORT_INTERACTIVE) {
+ this.interactiveViewUrl = "";
+ }
+ },
+
+ onInteractiveViewLoad() {
+ if (!this.usesInteractiveTransport()) return;
+ this.prepareInteractiveViewFrame();
+ this.switchingBrowserId = null;
+ this._surfaceSwitching = false;
+ this.queueViewportSync(true);
+ },
+
supportsBinaryFrames() {
return BROWSER_BINARY_FRAME_REQUESTS_ENABLED && BROWSER_BINARY_PAYLOADS_SUPPORTED;
},
@@ -1003,9 +1109,9 @@ const model = {
return { width, height };
},
- scheduleViewportSyncForSurface(sequence, delayMs = 0) {
+ scheduleViewportSyncForSurface(sequence, delayMs = 0, mode = this._mode) {
globalThis.setTimeout?.(() => {
- if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") {
+ if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== mode) {
return;
}
this.queueViewportSync(true);
@@ -1093,7 +1199,8 @@ const model = {
replaceAll: Boolean(data.all_browsers),
replaceContext: !data.all_browsers,
});
- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
+ this.applyViewer(data);
+ this._subscribedViewerTransport = this.viewerTransport;
this.setActiveBrowserId(
data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,
data.active_browser_context_id || contextId,
@@ -1108,9 +1215,7 @@ const model = {
const frameHandler = ({ data }) => {
if (data?.context_id !== this.contextId) return;
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
- if (data?.viewer_transport) {
- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
- }
+ this.applyViewer(data);
this.applyTabScope(data);
const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
@@ -1180,9 +1285,7 @@ const model = {
const stateHandler = ({ data }) => {
if (data?.context_id !== this.contextId) return;
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
- if (data?.viewer_transport) {
- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
- }
+ this.applyViewer(data);
this.applyTabScope(data);
const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
if (Array.isArray(data.browsers)) {
@@ -1384,7 +1487,7 @@ const model = {
},
hasFrame() {
- return Boolean(this.frameSrc || this.frameCanvasReady);
+ return Boolean(this.interactiveViewUrl || this.frameSrc || this.frameCanvasReady);
},
paintFrameBitmap(bitmap) {
@@ -1419,6 +1522,10 @@ const model = {
},
frameElement() {
+ if (this.usesInteractiveTransport()) {
+ const iframe = this._stageElement?.querySelector?.(".browser-interactive-frame");
+ if (iframe) return iframe;
+ }
if (this.frameCanvasReady) {
const canvas = this.currentFrameCanvas();
if (canvas) return canvas;
@@ -1467,7 +1574,7 @@ const model = {
replaceAll: Boolean(data.all_browsers),
replaceContext: !data.all_browsers,
});
- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
+ this.applyViewer(data);
const result = data.result || {};
const resultContextId = this.normalizeContextId(
result.context_id
@@ -1510,7 +1617,12 @@ const model = {
previousActiveBrowserId,
previousActiveContextId,
);
- if ((commandName === "open" || commandName === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
+ const viewerTransportChanged = this._subscribedViewerTransport !== this.viewerTransport;
+ if (
+ (commandName === "open" || commandName === "close" || activeChanged || viewerTransportChanged)
+ && this.contextId
+ && this.activeBrowserId
+ ) {
await this.connectViewer({
browserId: this.activeBrowserId,
contextId: this.activeBrowserContextId,
@@ -1670,6 +1782,10 @@ const model = {
return this.sameBrowserTab(browser?.id, browser?.context_id, this.activeBrowserId, this.activeBrowserContextId);
},
+ isBrowserLoading(browser) {
+ return Boolean(browser?.loading || (this.isActiveBrowser(browser) && this.isBusy()));
+ },
+
browserTabTitle(browser) {
const title = String(browser?.title || "").trim();
const url = String(browser?.currentUrl || "").trim();
@@ -1831,6 +1947,7 @@ const model = {
if (snapshot.state) {
this.applyActiveFrameState(snapshot.state);
}
+ if (this.usesInteractiveTransport()) return;
const frameBrowserId = snapshotId || this.activeBrowserId;
this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
browserId: frameBrowserId,
@@ -2521,13 +2638,21 @@ const model = {
queueViewportSync(force = false) {
this.clearRenderedFrameIfViewportChanged();
+ if (force) {
+ if (this._viewportSyncTimer) {
+ globalThis.clearTimeout(this._viewportSyncTimer);
+ this._viewportSyncTimer = null;
+ }
+ void this.syncViewport(true);
+ return;
+ }
if (this._viewportSyncTimer) {
- globalThis.clearTimeout(this._viewportSyncTimer);
+ return;
}
this._viewportSyncTimer = globalThis.setTimeout(() => {
this._viewportSyncTimer = null;
- void this.syncViewport(force);
- }, force ? 0 : VIEWPORT_SYNC_DEBOUNCE_MS);
+ void this.syncViewport(false);
+ }, VIEWPORT_SYNC_INTERVAL_MS);
},
async syncViewport(force = false, options = {}) {
@@ -2542,7 +2667,7 @@ const model = {
}
const key = `${contextId}:${this.activeBrowserId}:${viewport.width}x${viewport.height}`;
if (
- (!restartStream && this._lastViewportKey === key)
+ (!force && !restartStream && this._lastViewportKey === key)
|| (
!force
&& !restartStream
@@ -2555,11 +2680,13 @@ const model = {
return;
}
try {
+ this.syncInteractiveViewSize();
await websocket.emit("browser_viewer_input", {
context_id: contextId,
browser_id: this.activeBrowserId,
viewer_id: this._viewerToken,
input_type: "viewport",
+ viewer_transport: this.viewerTransport,
width: viewport.width,
height: viewport.height,
restart_stream: restartStream && this.usesScreencastTransport(),
@@ -2739,6 +2866,9 @@ const model = {
this._viewerToken = "";
this.switchingBrowserId = null;
this.viewerTransport = this.requestedViewerTransport();
+ this._subscribedViewerTransport = this.viewerTransport;
+ this.interactiveViewUrl = "";
+ this.viewerFallbackReason = "";
this.tabScope = "per_context";
this._surfaceMounted = false;
this._surfaceSwitching = false;
@@ -3010,13 +3140,6 @@ const model = {
return this.frameState?.currentUrl || this.address || "about:blank";
},
- loadingMessage() {
- if (this.browserInstallExpected) {
- const cacheDir = this.status?.playwright?.cache_dir || "/a0/tmp/playwright";
- return `Installing Chromium for the first Browser run. This can take a few minutes; future starts reuse ${cacheDir}.`;
- }
- return "Loading";
- },
};
export const store = createStore("browserPage", model);
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 2c546b421..edf4cd374 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -82,7 +82,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_conf
def anyio_backend():
return "asyncio"
-from helpers import ephemeral_images
+from helpers import ephemeral_images, virtual_desktop
from helpers.errors import RepairableException
from plugins._browser.helpers.config import (
build_browser_launch_config,
@@ -109,6 +109,8 @@ from plugins._browser.helpers.runtime import (
normalize_url,
)
import plugins._browser.helpers.runtime as browser_runtime_module
+from plugins._browser.helpers.interactive_view import BrowserInteractiveView
+import plugins._browser.helpers.interactive_view as browser_interactive_view_module
from plugins._browser.helpers.playwright import (
ensure_playwright_binary,
get_playwright_binary,
@@ -833,7 +835,8 @@ def test_browser_viewer_allows_slow_extension_startup():
assert "const BROWSER_COMMAND_TIMEOUT_MS = 45000;" in js
assert "? BROWSER_FIRST_INSTALL_TIMEOUT_MS" in js
assert ": BROWSER_SUBSCRIBE_TIMEOUT_MS" in js
- assert "Installing Chromium for the first Browser run" in js
+ assert "browserInstallExpected" in js
+ assert "Installing Chromium for the first Browser run" not in js
def test_browser_viewer_creates_chat_when_no_context_is_selected():
@@ -880,6 +883,20 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
assert "this.resetRenderedFrame();" in js
+def test_browser_interactive_modal_handoff_reconciles_after_xpra_resize():
+ js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
+ encoding="utf-8"
+ )
+
+ assert "const INTERACTIVE_VIEWPORT_SETTLE_MS = 320;" in js
+ assert "const surfaceMode = this._mode;" in js
+ assert 'surfaceMode === "modal" && this.usesInteractiveTransport()' in js
+ assert "this.scheduleViewportSyncForSurface(sequence, INTERACTIVE_VIEWPORT_SETTLE_MS, surfaceMode);" in js
+ assert "scheduleViewportSyncForSurface(sequence, delayMs = 0, mode = this._mode)" in js
+ assert "this._mode !== mode" in js
+ assert "(!force && !restartStream && this._lastViewportKey === key)" in js
+
+
def test_browser_surface_handoffs_keep_existing_frame_until_replacement_arrives():
js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
encoding="utf-8"
@@ -1393,6 +1410,12 @@ def test_browser_viewer_uses_tabs_for_session_switching():
assert ':key="$store.browserPage.browserTabKey(browser)"' in main_html
assert "browser.context_id" in main_html
assert ':title="$store.browserPage.browserTabTooltip(browser)"' in main_html
+ assert ':aria-busy="$store.browserPage.isBrowserLoading(browser).toString()"' in main_html
+ assert 'class="browser-tab-loading spinning"' in main_html
+ assert 'x-show="$store.browserPage.isBrowserLoading(browser)"' in main_html
+ assert "<span x-text=\"$store.browserPage.loadingMessage()\">Loading</span>" not in main_html
+ assert "isBrowserLoading(browser)" in browser_store
+ assert "loadingMessage()" not in browser_store
assert "browser-tab-context" not in main_html
assert 'handleSelectedContextChange($store.chats?.selected)' in main_html
assert "activeBrowserContextId" in browser_store
@@ -1445,7 +1468,7 @@ def test_browser_tabs_close_without_confirmation_or_busy_lock():
assert "_commandInFlightCount" in browser_store
-def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
+def test_browser_viewer_defaults_to_interactive_with_screencast_and_snapshot_fallbacks():
ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
encoding="utf-8"
)
@@ -1465,6 +1488,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert 'runtime.call("screenshot"' in ws_browser
assert 'VIEWER_TRANSPORT_SNAPSHOT = "snapshot"' in ws_browser
assert 'VIEWER_TRANSPORT_SCREENCAST = "screencast"' in ws_browser
+ assert 'VIEWER_TRANSPORT_INTERACTIVE = "interactive"' in ws_browser
+ assert "async def _effective_viewer(" in ws_browser
+ assert "return VIEWER_TRANSPORT_SCREENCAST, viewer" in ws_browser
assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert "self._stream_state" in ws_browser
@@ -1510,8 +1536,20 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "return await globalThis.createImageBitmap(blob);" in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
- assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
- assert "liveScreencastEnabled: true" in browser_store
+ assert 'const BROWSER_VIEWER_TRANSPORT_INTERACTIVE = "interactive";' in browser_store
+ assert "const VIEWPORT_SYNC_INTERVAL_MS = 50;" in browser_store
+ assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE" in browser_store
+ assert "return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;" in browser_store
+ assert "usesInteractiveTransport()" in browser_store
+ assert "isInteractiveSurface(stage = null)" in browser_store
+ assert "prepareInteractiveViewFrame(frame = null)" in browser_store
+ assert 'style.id = "a0-xpra-browser-frame-css";' in browser_store
+ assert "#shadow_pointer" in browser_store
+ assert "xpraWindow._set_decorated?.(false);" in browser_store
+ assert "xpraWindow.updateCSSGeometry?.();" in browser_store
+ assert "syncInteractiveViewSize()" in browser_store
+ assert "frame?.contentWindow?.client?._screen_resized?.();" in browser_store
+ assert "applyViewer(data = {})" in browser_store
assert "requestedViewerTransport()" in browser_store
assert "normalizeViewerTransport(value = \"\")" in browser_store
assert "usesScreencastTransport()" in browser_store
@@ -1583,6 +1621,11 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "canvas_wheel_screenshot" not in ws_browser
assert "surface_mode: this._mode" not in browser_store
assert '<canvas class="browser-frame browser-frame-canvas"' in main_html
+ assert '<iframe class="browser-interactive-frame"' in main_html
+ assert 'x-if="$store.browserPage.isInteractiveSurface($el.parentElement)"' in main_html
+ assert 'aria-label="Browser viewport"' in main_html
+ assert 'title="Interactive Browser viewport"' not in main_html
+ assert 'allow="clipboard-read; clipboard-write"' in main_html
assert 'x-init="$store.browserPage.attachFrameCanvas($el)"' in main_html
assert '<img class="browser-frame browser-frame-image"' in main_html
assert "$store.browserPage.hasFrame()" in main_html
@@ -2207,7 +2250,12 @@ def test_browser_docker_installs_full_chromium_to_tmp_cache():
).read_text(encoding="utf-8")
assert '"headless": not bool(browser_display)' in runtime
assert 'launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display}' in runtime
+ assert 'launch_kwargs["no_viewport"] = True' in runtime
+ assert '"windowState": "fullscreen"' not in runtime
+ assert "self.interactive_view.ensure_display()" in runtime
assert " xvfb \\" in install_additional
+ assert "XPRA_PACKAGES=(xpra xpra-x11 xpra-html5)" in install_additional
+ assert " xdotool \\" in install_additional
def test_browser_startup_migration_prepares_current_playwright_binary():
@@ -2222,10 +2270,248 @@ def test_browser_startup_migration_prepares_current_playwright_binary():
).read_text(encoding="utf-8")
assert "class BrowserPlaywrightCacheMigration(Extension)" in extension
+ assert "virtual_desktop_routes.install_route_hooks()" in extension
assert "hooks.prepare_playwright_cache()" in extension
assert "PrintStyle.warning" in extension
+def test_browser_interactive_views_use_isolated_loopback_sessions(monkeypatch, tmp_path):
+ class FakeProcess:
+ def __init__(self):
+ self.returncode = None
+
+ def poll(self):
+ return self.returncode
+
+ def terminate(self):
+ self.returncode = 0
+
+ def wait(self, timeout=None):
+ return self.returncode
+
+ next_display = iter((71, 72))
+ next_port = iter((44001, 44002))
+
+ def fake_ensure_display(view):
+ view.display = next(next_display)
+ view._xvfb = FakeProcess()
+ return view.display_name
+
+ def fake_resize(view, width, height):
+ view.width, view.height = virtual_desktop.normalize_size(width, height)
+ return {"ok": True, "width": view.width, "height": view.height}
+
+ def fake_start_xpra(view, xpra):
+ assert xpra == "/usr/bin/xpra"
+ view.port = next(next_port)
+ view._xpra = FakeProcess()
+
+ monkeypatch.setattr(
+ browser_interactive_view_module.files,
+ "get_abs_path",
+ lambda *parts: str(tmp_path.joinpath(*parts)),
+ )
+ monkeypatch.setattr(BrowserInteractiveView, "ensure_display", fake_ensure_display)
+ monkeypatch.setattr(BrowserInteractiveView, "resize", fake_resize)
+ monkeypatch.setattr(BrowserInteractiveView, "_start_xpra", fake_start_xpra)
+ monkeypatch.setattr(
+ browser_interactive_view_module,
+ "collect_status",
+ lambda: {
+ "available": True,
+ "missing": [],
+ "binaries": {"xpra": "/usr/bin/xpra"},
+ "xpra_html_root": "/usr/share/xpra/www",
+ },
+ )
+
+ views = [BrowserInteractiveView("ctx-a"), BrowserInteractiveView("ctx-b")]
+ try:
+ viewers = [view.ensure_viewer(1200, 700) for view in views]
+
+ assert views[0].token != views[1].token
+ assert views[0].display != views[1].display
+ assert all(viewer["available"] for viewer in viewers)
+ for view, viewer in zip(views, viewers, strict=True):
+ endpoint = virtual_desktop.proxy_for_token(view.token)
+ assert endpoint is not None
+ assert endpoint.host == "127.0.0.1"
+ assert endpoint.owner == "browser"
+ assert endpoint.port == view.port
+ assert viewer["token"] == view.token
+ assert "file_transfer=false" in viewer["url"]
+ assert "printing=false" in viewer["url"]
+ finally:
+ for view in views:
+ view.close()
+
+ assert all(virtual_desktop.proxy_for_token(view.token) is None for view in views)
+
+
+async def test_browser_interactive_viewer_reuses_the_automated_page():
+ class FakePage:
+ viewport_size = {"width": 1024, "height": 768}
+
+ class FakeInteractiveView:
+ def __init__(self):
+ self.ensure_calls = []
+
+ def ensure_viewer(self, width, height):
+ self.ensure_calls.append((width, height))
+ return {
+ "available": True,
+ "token": "browser-token",
+ "url": "/desktop/session/browser-token/",
+ "width": width,
+ "height": height,
+ }
+
+ page = FakePage()
+ interactive_view = FakeInteractiveView()
+ viewport_calls = []
+ stopped = []
+ core = _BrowserRuntimeCore("ctx")
+ core.context = object()
+ core.pages[7] = BrowserPage(id=7, page=page)
+ core.interactive_view = interactive_view
+
+ async def ensure_started():
+ return None
+
+ async def stop_screencasts(browser_id):
+ stopped.append(browser_id)
+
+ async def set_viewport(browser_id, width, height, **kwargs):
+ viewport_calls.append((browser_id, width, height, kwargs))
+ return {"state": {"id": browser_id}, "viewport": {"width": width, "height": height}}
+
+ core.ensure_started = ensure_started
+ core._stop_screencasts_for_browser = stop_screencasts
+ core.set_viewport = set_viewport
+
+ result = await core.interactive_viewer(7, width=1200, height=700)
+
+ assert core.pages[7].page is page
+ assert interactive_view.ensure_calls == [(1200, 700)]
+ assert stopped == [7]
+ assert viewport_calls == [(7, 1200, 700, {"resize_interactive": True})]
+ assert core.last_interacted_browser_id == 7
+ assert result["url"] == "/desktop/session/browser-token/"
+
+
+async def test_browser_interactive_window_clips_chrome_without_fullscreen():
+ commands = []
+
+ class FakeSession:
+ async def send(self, method, params=None):
+ commands.append((method, params))
+ if method == "Browser.getWindowForTarget":
+ return {"windowId": 7}
+ if method == "Browser.getWindowBounds":
+ return {
+ "bounds": {
+ "windowState": "normal",
+ "left": 0,
+ "top": 0,
+ "width": 1024,
+ "height": 768,
+ }
+ }
+ return {}
+
+ async def detach(self):
+ commands.append(("detach", None))
+
+ class FakeContext:
+ async def new_cdp_session(self, page):
+ assert page is fake_page
+ return FakeSession()
+
+ class FakeInteractiveView:
+ display = 0
+ width = 1200
+ height = 700
+
+ class FakePage:
+ async def evaluate(self, script, **kwargs):
+ commands.append(("evaluate", script))
+ return 87
+
+ fake_page = FakePage()
+ core = _BrowserRuntimeCore("ctx")
+ core.context = FakeContext()
+ core.interactive_view = FakeInteractiveView()
+
+ await core._fit_browser_window(fake_page)
+ core.interactive_view.width = 1280
+ core.interactive_view.height = 720
+ await core._fit_browser_window(fake_page)
+ await core._reset_browser_window_session()
+
+ assert commands == [
+ ("Browser.getWindowForTarget", None),
+ ("Browser.getWindowBounds", {"windowId": 7}),
+ (
+ "evaluate",
+ "() => Math.max(0, globalThis.outerHeight - globalThis.innerHeight)",
+ ),
+ (
+ "Browser.setWindowBounds",
+ {
+ "windowId": 7,
+ "bounds": {
+ "windowState": "normal",
+ "left": 0,
+ "top": -87,
+ "width": 1200,
+ "height": 787,
+ },
+ },
+ ),
+ (
+ "Browser.setWindowBounds",
+ {
+ "windowId": 7,
+ "bounds": {
+ "windowState": "normal",
+ "left": 0,
+ "top": -87,
+ "width": 1280,
+ "height": 807,
+ },
+ },
+ ),
+ ("detach", None),
+ ]
+
+
+async def test_browser_viewer_falls_back_when_interactive_runtime_is_unavailable():
+ calls = []
+
+ class FakeRuntime:
+ async def call(self, method, *args, **kwargs):
+ calls.append((method, args, kwargs))
+ return {"available": False, "error": "xpra unavailable"}
+
+ handler = ws_browser_module.WsBrowser(SimpleNamespace(), threading.RLock(), manager=None)
+ transport, viewer = await handler._effective_viewer(
+ FakeRuntime(),
+ 7,
+ {
+ "viewer_transport": "interactive",
+ "viewport_width": 1200,
+ "viewport_height": 700,
+ },
+ )
+
+ assert transport == ws_browser_module.VIEWER_TRANSPORT_SCREENCAST
+ assert viewer == {"available": False, "error": "xpra unavailable"}
+ assert calls == [("interactive_viewer", (7,), {"width": 1200, "height": 700})]
+
+
def test_browser_runtime_removes_stale_profile_singletons(monkeypatch, tmp_path):
monkeypatch.setattr(
browser_runtime_module.files,
@@ -3245,6 +3531,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
"width": 1280,
"height": 720,
"restart_stream": True,
+ "viewer_transport": "interactive",
},
"sid-1",
)
@@ -3254,10 +3541,68 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
"snapshot": None,
}
assert calls == [
- ("set_viewport", (7, 1280, 720), {"restart_screencast": True})
+ (
+ "set_viewport",
+ (7, 1280, 720),
+ {
+ "restart_screencast": True,
+ "resize_interactive": True,
+ "include_state": False,
+ },
+ )
]
+async def test_browser_interactive_resize_uses_native_window_viewport():
+ fitted = []
+
+ class FakePage:
+ viewport_size = None
+
+ async def set_viewport_size(self, viewport):
+ raise AssertionError("Interactive native viewport must not enable emulation")
+
+ class FakeInteractiveView:
+ display = 0
+ width = 1024
+ height = 768
+
+ def resize(self, width, height):
+ self.width = width
+ self.height = height
+ return {"ok": True, "width": width, "height": height}
+
+ page = FakePage()
+ core = _BrowserRuntimeCore("ctx")
+ core.context = object()
+ core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=page)
+ core.interactive_view = FakeInteractiveView()
+
+ async def fake_fit(fitted_page):
+ fitted.append(fitted_page)
+
+ async def fake_state(browser_id):
+ return {"id": browser_id}
+
+ core._fit_browser_window = fake_fit
+ core._state = fake_state
+
+ result = await core.set_viewport(
+ 7,
+ 1280,
+ 720,
+ resize_interactive=True,
+ include_state=False,
+ )
+
+ assert result == {
+ "state": None,
+ "viewport": {"width": 1280, "height": 720},
+ }
+ assert fitted == [page]
+
+
@pytest.mark.anyio
async def test_browser_runtime_restarts_screencast_without_resizing_same_viewport():
viewport_calls = []