Skip to content

Self-improving AI Employees for everyone

fix(browser): keep live streaming responsive during commands

trohitg87322eeb1d48
diff --git a/server/nodes/browser/_live_control.py b/server/nodes/browser/_live_control.py
new file mode 100644
index 00000000..52958b13
--- /dev/null
+++ b/server/nodes/browser/_live_control.py
@@ -0,0 +1,373 @@
+"""Bounded, ordered user commands independent of the live socket reader.
+
+One queue belongs to one profile hub. Release invalidates queued work at
+admission time, then waits for the dispatched command before releasing any
+pressed inputs. Chrome commands already sent are never assumed cancelled.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Any
+
+from services.plugin.base import NodeUserError
+from ._cdp import CDPDisconnected
+
+MAX_COMMANDS = 64
+MAX_BYTES = 64 * 1024
+INPUT_KINDS = frozenset({"mouse", "wheel", "key", "insert_text", "navigate", "tab", "copy", "dialog_reply"})
+
+
+@dataclass
+class Command:
+ viewer: Any
+ message: dict
+ session: Any
+ epoch: int
+ target: Any
+ size: int
+ queued_at: float
+
+
+class _GuardedSession:
+ def __init__(self, queue: "LiveControlQueue", command: Command, session: Any):
+ self.queue, self.command, self.session = queue, command, session
+
+ async def send(self, method, params=None, **kwargs):
+ if not self.queue.valid(self.command):
+ raise NodeUserError("Browser control or page changed; the queued command was discarded.")
+ params = params or {}
+ identity = None
+ if method == "Input.dispatchKeyEvent":
+ identity = (self.command.viewer.id, "key", params.get("code") or params.get("key"))
+ elif method == "Input.dispatchMouseEvent" and params.get("type") in {"mousePressed", "mouseReleased"}:
+ identity = (self.command.viewer.id, "mouse", params.get("button"))
+ # Record before dispatch: even a response timeout can leave a key held.
+ if identity and params.get("type") not in {"keyUp", "mouseReleased"}:
+ if identity not in self.queue._pressed and len(self.queue._pressed) >= MAX_COMMANDS:
+ raise NodeUserError("Too many held browser inputs. Hand control back before continuing.")
+ self.queue._pressed[identity] = (self.session, dict(params))
+ started = time.monotonic()
+ try:
+ result = await self.session.send(method, params, **kwargs)
+ except (TimeoutError, CDPDisconnected):
+ self.queue._unsafe = True
+ self.queue.invalidate(self.command.viewer.id)
+ self.queue._error(self.command.viewer, "control_uncertain", "Chrome did not confirm the action. Hand back control to safely restart the browser.")
+ raise
+ finally:
+ self.queue._observe("command_cdp", time.monotonic() - started)
+ if identity and params.get("type") in {"keyUp", "mouseReleased"}:
+ self.queue._pressed.pop(identity, None)
+ return result
+
+
+class LiveControlQueue:
+ def __init__(self, hub: Any):
+ self.hub = hub
+ self.controller = hub.controller
+ self._queue: deque[Command] = deque()
+ self._bytes = 0
+ self._epochs: dict[str, int] = {}
+ self._blocked: set[str] = set()
+ self._worker: asyncio.Task | None = None
+ self._takeover: asyncio.Task | None = None
+ self._takeover_viewer: str | None = None
+ self._takeover_pending: set[str] = set()
+ self._lock = asyncio.Lock()
+ self._current: Command | None = None
+ self._pressed: dict[tuple, tuple[Any, dict]] = {}
+ self._closed = False
+ self._unsafe = False
+ self._retiring = False
+ self._input_session = None
+ self._input_target = None
+ self._input_epoch = None
+ self._target_epoch = 0
+ self._last_target = None
+ self.controller.live_control_barrier = self.barrier
+
+ def _observe(self, name: str, value: float) -> None:
+ metrics = getattr(self.hub, "metrics", None)
+ if metrics is not None:
+ metrics.observe(name, value)
+
+ def _error(self, viewer, code, message):
+ viewer.send_json({"type": "error", "code": code, "message": message})
+
+ def _target(self):
+ target = self.controller.active_target_id
+ current = target, (self.controller.tabs.get(target or "") or {}).get("url")
+ if current != self._last_target:
+ self._last_target = current
+ self._target_epoch += 1
+ return self._target_epoch, current
+
+ def target_changed(self):
+ self._target()
+
+ def valid(self, command: Command, *, owner: bool = True) -> bool:
+ viewer = command.viewer
+ return (
+ not self._closed and not viewer.closed and viewer.visible
+ and (not owner or viewer.id not in self._blocked)
+ and command.epoch == self._epochs.get(viewer.id, 0)
+ and command.target == self._target()
+ and (not owner or self.controller.can_inject_input(viewer.id))
+ )
+
+ def guarded_session(self, viewer, session):
+ command = self._current
+ if command is not None and command.viewer.id == viewer.id and session is not None:
+ return _GuardedSession(self, command, session)
+ return session
+
+ def input_session(self, viewer):
+ # Dedicated to input: the screencast session may be detached whenever
+ # a viewer resizes or becomes hidden.
+ return self.guarded_session(viewer, self._input_session)
+
+ async def _ensure_input_session(self):
+ target = self.controller.active_target_id
+ epoch = self._target()
+ if self._input_session is not None and self._input_target == target and self._input_epoch == epoch:
+ return
+ if self._input_session is not None:
+ for viewer_id in {identity[0] for identity in self._pressed}:
+ await self._release_pressed(viewer_id)
+ if self._input_target == target:
+ self._input_epoch = epoch
+ return
+ await self._input_session.detach()
+ self._input_session = await self.hub.runtime.page_session(target)
+ self._input_target = target
+ self._input_epoch = epoch
+
+ def enqueue(self, viewer, message: dict, session) -> bool:
+ if self._closed or viewer.closed or not viewer.visible:
+ return False
+ kind = message.get("type")
+ if kind == "control_request" and viewer.id in self._takeover_pending:
+ return False
+ if kind in INPUT_KINDS and (viewer.id in self._blocked or self._unsafe or not self.controller.can_inject_input(viewer.id)):
+ return False
+ size = len(json.dumps(message, ensure_ascii=False).encode("utf-8"))
+ command = Command(viewer, dict(message), session, self._epochs.get(viewer.id, 0), self._target(), size, time.monotonic())
+ previous = self._queue[-1] if self._queue else None
+ if previous and self._coalescible(previous, command):
+ if kind == "wheel":
+ command.message["delta_x"] = float(previous.message.get("delta_x") or 0) + float(message.get("delta_x") or 0)
+ command.message["delta_y"] = float(previous.message.get("delta_y") or 0) + float(message.get("delta_y") or 0)
+ command.size = len(json.dumps(command.message).encode("utf-8"))
+ if self._bytes - previous.size + command.size <= MAX_BYTES:
+ self._queue[-1] = command
+ self._bytes += command.size - previous.size
+ return True
+ if len(self._queue) >= MAX_COMMANDS or self._bytes + command.size > MAX_BYTES:
+ self._error(viewer, "control_overloaded", "Browser input is busy. Wait for the current action, then try again.")
+ # Never silently lose a key-up on overflow: stop admission and
+ # release everything after the current command settles.
+ self.release(viewer, note="input queue overflow")
+ return False
+ if kind == "control_request":
+ self._takeover_pending.add(viewer.id)
+ self._queue.append(command)
+ self._bytes += command.size
+ self._ensure_worker()
+ return True
+
+ @staticmethod
+ def _coalescible(a: Command, b: Command) -> bool:
+ if (a.viewer.id, a.epoch, a.target) != (b.viewer.id, b.epoch, b.target):
+ return False
+ x, y = a.message, b.message
+ if x.get("modifiers", 0) != y.get("modifiers", 0) or x.get("type") != y.get("type"):
+ return False
+ wheel = y.get("type") == "wheel" and all(x.get(k, 0) == y.get(k, 0) for k in ("x", "y", "buttons"))
+ return (wheel or (
+ y.get("type") == "mouse" and x.get("action") == y.get("action") == "move"
+ and x.get("buttons", 0) == y.get("buttons", 0)
+ ))
+
+ def invalidate(self, viewer_id: str) -> None:
+ self._blocked.add(viewer_id)
+ self._epochs[viewer_id] = self._epochs.get(viewer_id, 0) + 1
+ for command in self._queue:
+ if command.viewer.id == viewer_id and command.message.get("type") == "control_request":
+ command.viewer.send_json({"type": "control", "granted": False, "reason": "viewer_unavailable"})
+ self._queue = deque(c for c in self._queue if c.viewer.id != viewer_id)
+ self._bytes = sum(c.size for c in self._queue)
+ self._takeover_pending.discard(viewer_id)
+ if self._takeover_viewer == viewer_id and self._takeover is not None:
+ self._takeover.cancel()
+
+ def release(self, viewer, *, outcome="handed_back", note="") -> None:
+ self.invalidate(viewer.id)
+ if getattr(self.controller, "controller_viewer", None) != viewer.id and getattr(self.controller, "state", None) != "awaiting_user":
+ return
+ # A zero-byte priority barrier is always admissible. At most one per
+ # viewer remains because invalidate removes its previous barrier.
+ if len(self._queue) >= MAX_COMMANDS:
+ removed = self._queue.pop()
+ self._bytes -= removed.size
+ self._takeover_pending.discard(removed.viewer.id)
+ self._error(removed.viewer, "control_overloaded", "Browser control changed. Try your request again.")
+ self._queue.appendleft(Command(viewer, {"type": "control_release", "outcome": outcome, "note": note}, None, 0, None, 0, time.monotonic()))
+ self._ensure_worker()
+
+ async def barrier(self, viewer_id: str | None) -> None:
+ if viewer_id is None:
+ return
+ self.invalidate(viewer_id)
+ if asyncio.current_task() in (self._worker, self._takeover):
+ await self._release_pressed(viewer_id)
+ else:
+ async with self._lock:
+ await self._release_pressed(viewer_id)
+
+ async def _release_pressed(self, viewer_id):
+ if self._unsafe:
+ await self._retire_uncertain_browser()
+ return
+ for identity, (session, params) in list(self._pressed.items()):
+ if identity[0] != viewer_id:
+ continue
+ release = dict(params)
+ if identity[1] == "key":
+ release.update(type="keyUp", modifiers=0)
+ release.pop("text", None)
+ release.pop("unmodifiedText", None)
+ method = "Input.dispatchKeyEvent"
+ else:
+ release.update(type="mouseReleased", buttons=0, modifiers=0)
+ method = "Input.dispatchMouseEvent"
+ try:
+ await session.send(method, release, timeout=5)
+ except Exception as exc:
+ self._unsafe = True
+ try:
+ await self._retire_uncertain_browser()
+ return
+ except Exception:
+ raise NodeUserError("Could not release browser input safely. Restart the browser before handing control back.") from exc
+ self._pressed.pop(identity, None)
+
+ async def _retire_uncertain_browser(self):
+ # A local timeout only abandons the reply; Chrome may still execute
+ # the command. Closing this managed process is the safety boundary.
+ try:
+ self._retiring = True
+ await asyncio.wait_for(self.hub.runtime.chrome.shutdown(), timeout=10)
+ except Exception as exc:
+ raise NodeUserError("Chrome has an unconfirmed action and could not be stopped. Control remains with you.") from exc
+ finally:
+ self._retiring = False
+ self._unsafe = False
+ self._pressed.clear()
+ self._input_session = None
+ self._input_target = None
+ self._input_epoch = None
+
+ def _ensure_worker(self):
+ if self._worker is None or self._worker.done():
+ self._worker = asyncio.create_task(self._run(), name="browser-live-control")
+
+ async def _run(self):
+ while self._queue:
+ command = self._queue.popleft()
+ self._bytes -= command.size
+ viewer, message = command.viewer, command.message
+ kind = message.get("type")
+ async with self._lock:
+ self._current = command
+ try:
+ self._observe("command_queue_wait", time.monotonic() - command.queued_at)
+ if kind == "control_release":
+ await self.controller.hand_back(viewer.id, outcome=message.get("outcome", "handed_back"), note=message.get("note", ""))
+ elif kind == "control_request":
+ if not self.valid(command, owner=False):
+ continue
+ self._takeover_viewer = viewer.id
+ self._takeover = asyncio.create_task(self._claim(command))
+ try:
+ await self._takeover
+ except asyncio.CancelledError:
+ viewer.send_json({"type": "control", "granted": False, "reason": "viewer_unavailable"})
+ if self._closed:
+ return
+ finally:
+ self._takeover = None
+ self._takeover_viewer = None
+ elif self.valid(command):
+ if hasattr(self.hub, "runtime") and hasattr(self.hub.runtime, "page_session"):
+ await self._ensure_input_session()
+ self.controller.touch_user_input()
+ if kind in {"mouse", "wheel", "key", "insert_text"}:
+ await self.hub.input(viewer, message)
+ elif kind in {"navigate", "tab"}:
+ await self._release_pressed(viewer.id)
+ await getattr(self.hub, kind)(viewer, message, command.session.policy)
+ elif kind == "copy":
+ await self.hub.copy(viewer)
+ elif kind == "dialog_reply":
+ await self.hub.dialog_reply(viewer, message)
+ except Exception as exc:
+ self._error(viewer, "control", str(exc)[:300])
+ if kind == "control_request":
+ viewer.send_json({"type": "control", "granted": False, "reason": "control_failed", "message": str(exc)[:300]})
+ finally:
+ if kind == "control_request":
+ self._takeover_pending.discard(viewer.id)
+ self._current = None
+
+ async def _claim(self, command):
+ viewer = command.viewer
+ await self.controller.acquire_lease(command.session, wait=2.0)
+ if not self.valid(command, owner=False):
+ return
+ granted, reason = await self.controller.take_over(
+ viewer.id, force=bool(command.message.get("force")),
+ valid=lambda: self.valid(command, owner=False),
+ )
+ viewer.send_json({"type": "control", "granted": granted, "reason": reason})
+ if granted:
+ self._blocked.discard(viewer.id)
+
+ async def close(self):
+ self._closed = True
+ if self._takeover is not None:
+ self._takeover.cancel()
+ self._queue.clear()
+ self._bytes = 0
+ if self._retiring:
+ # Chrome's close notification can run inside shutdown(). Waiting
+ # for the worker here would wait on our own shutdown barrier.
+ return
+ # Let already-dispatched CDP operations finish; their normal protocol
+ # timeout still bounds this wait. Do not tell agents input has settled
+ # merely because a Python task was cancelled.
+ if self._worker is not None and self._worker is not asyncio.current_task():
+ await asyncio.gather(self._worker, return_exceptions=True)
+ if self._input_session is not None:
+ await self._input_session.detach()
+ self._input_session = None
+
+ async def idle_without_viewers(self):
+ """Retain the reusable hub, but no command task/session after disposal."""
+ if self._worker is not None and self._worker is not asyncio.current_task():
+ await asyncio.gather(self._worker, return_exceptions=True)
+ async with self._lock:
+ if getattr(self.hub, "viewers", None):
+ return
+ for viewer_id in {identity[0] for identity in self._pressed}:
+ await self._release_pressed(viewer_id)
+ if self._input_session is not None:
+ await self._input_session.detach()
+ self._input_session = None
+ self._input_target = self._input_epoch = None
+ self._epochs.clear()
+ self._blocked.clear()
diff --git a/server/nodes/browser/_session.py b/server/nodes/browser/_session.py
index a5563999..f1ce6d27 100644
--- a/server/nodes/browser/_session.py
+++ b/server/nodes/browser/_session.py
@@ -268,10 +268,14 @@ class ProfileController:
async def release_lease(self, session_id: Optional[str] = None) -> None:
if session_id is None or (self.lease_session and self.lease_session.session_id == session_id):
+ barrier = getattr(self, "live_control_barrier", None)
+ if barrier is not None:
+ await barrier(self.controller_viewer)
if self.pending is not None:
self._resolve_pending("cancelled", "")
self.lease_session = None
if self.state != ControlState.IDLE:
+ self._cancel_viewer_drop(self.controller_viewer)
self.controller_viewer = None
await self._set_state(ControlState.IDLE, reason="released")
async with self._changed:
@@ -343,10 +347,10 @@ class ProfileController:
if request.future.done():
return request.future.result()
if time.monotonic() >= request.deadline:
- self._resolve_pending("timeout", "")
if self.state in (ControlState.AWAITING_USER, ControlState.USER):
- self.controller_viewer = None
- await self._set_state(ControlState.IDLE, reason="request timed out")
+ await self.hand_back(self.controller_viewer, outcome="timeout", note="request timed out")
+ else:
+ self._resolve_pending("timeout", "")
return {"status": "timeout", "note": ""}
return {"status": "still_waiting", "note": ""}
@@ -360,11 +364,17 @@ class ProfileController:
def _user_blocks_agent(self) -> bool:
return self._user_claim or self.state in (ControlState.USER, ControlState.AWAITING_USER)
- async def take_over(self, viewer_id: str, *, force: bool = False) -> tuple[bool, str]:
+ async def take_over(self, viewer_id: str, *, force: bool = False, valid: Optional[Callable[[], bool]] = None) -> tuple[bool, str]:
+ if valid is not None and not valid():
+ return False, "viewer_unavailable"
if self.controller_viewer and self.controller_viewer != viewer_id and not force:
return False, "held_by_other"
if self.lease_session is None:
return False, "no_session"
+ if self.controller_viewer and self.controller_viewer != viewer_id:
+ barrier = getattr(self, "live_control_barrier", None)
+ if barrier is not None:
+ await barrier(self.controller_viewer)
# Claim first so no new agent step starts while we wait for this one.
self._user_claim = True
try:
@@ -380,6 +390,8 @@ class ProfileController:
except asyncio.TimeoutError:
return False, "agent_busy"
try:
+ if valid is not None and not valid():
+ return False, "viewer_unavailable"
previous_controller = self.controller_viewer
self.controller_viewer = viewer_id
self.user_input_deadline = time.monotonic() + USER_IDLE_HANDBACK_SECONDS
@@ -398,9 +410,17 @@ class ProfileController:
async def hand_back(self, viewer_id: Optional[str], *, outcome: str = "handed_back", note: str = "") -> bool:
if self.state not in (ControlState.USER, ControlState.AWAITING_USER):
return False
+ if self.state == ControlState.USER and viewer_id is not None and viewer_id != self.controller_viewer:
+ return False
+ barrier = getattr(self, "live_control_barrier", None)
+ if barrier is not None:
+ await barrier(self.controller_viewer)
+ # Another transition may have completed while the dispatched command
+ # settled. Never release a newer viewer's control.
if self.state == ControlState.USER and viewer_id is not None and viewer_id != self.controller_viewer:
return False
self._resolve_pending(outcome, note)
+ self._cancel_viewer_drop(self.controller_viewer)
self.controller_viewer = None
self.lease_renewed = self.last_activity = time.monotonic()
await self._set_state(ControlState.IDLE, reason=outcome)
@@ -433,6 +453,11 @@ class ProfileController:
self._viewer_drop_tasks[viewer_id] = asyncio.create_task(_drop())
+ def _cancel_viewer_drop(self, viewer_id: Optional[str]) -> None:
+ task = self._viewer_drop_tasks.pop(viewer_id, None)
+ if task is not None and task is not asyncio.current_task():
+ task.cancel()
+
async def tick(self) -> None:
"""Periodic housekeeping (the fleet reaper calls this)."""
now = time.monotonic()
@@ -440,10 +465,10 @@ class ProfileController:
await self.hand_back(self.controller_viewer, outcome="handed_back", note="auto: no input for 10 minutes")
request = self.pending
if request is not None and now >= request.deadline and not request.future.done():
- self._resolve_pending("timeout", "")
if self.state in (ControlState.AWAITING_USER, ControlState.USER):
- self.controller_viewer = None
- await self._set_state(ControlState.IDLE, reason="request timed out")
+ await self.hand_back(self.controller_viewer, outcome="timeout", note="request timed out")
+ else:
+ self._resolve_pending("timeout", "")
def idle_for(self) -> float:
busy = self.state != ControlState.IDLE or self.viewers or self.pending is not None
diff --git a/server/nodes/browser/_stream.py b/server/nodes/browser/_stream.py
index 0ea90021..4ca40f8c 100644
--- a/server/nodes/browser/_stream.py
+++ b/server/nodes/browser/_stream.py
@@ -18,7 +18,7 @@ the page the agent is on and runs ``Page.startScreencast`` only while some
viewer is visible. Each viewer has a window of two unacknowledged frames and
one "latest frame" slot that newer frames overwrite, so a slow viewer drops
frames instead of stalling Chrome or the others; Chrome's own acknowledgement
-goes out after the first viewer's send completes (or 250 ms). Input reaches
+goes out as soon as the frame enters those bounded slots. Input reaches
the page only from the viewer that holds control, in the USER state.
"""
@@ -27,6 +27,7 @@ from __future__ import annotations
import asyncio
import base64
import json
+import itertools
import struct
import time
import uuid
@@ -38,13 +39,13 @@ from core.logging import get_logger
from ._cdp import CDPDisconnected, CDPError, CDPSession
from ._chrome import VIEWPORT_HEIGHT, VIEWPORT_WIDTH
+from ._stream_metrics import StreamMetrics
logger = get_logger(__name__)
FRAME_VERSION = 1
FRAME_KIND_JPEG = 1
_WINDOW = 2
-_ACK_AFTER = 0.25
_MAX_FRAME_EDGE = 1920
_CONTROL_QUEUE = 256
_INPUT_RATE = 200.0 # messages per second
@@ -53,6 +54,10 @@ _TEXT_LIMIT = 10_000
_CLIPBOARD_LIMIT = 100_000
_ATTACH_TIMEOUT = 5.0
_IDLE_POLL = 2.0
+_SCREENCAST_RETRY_DELAYS = (1.0, 2.0, 4.0)
+# Viewers can outlive a stopped profile and attach to a replacement hub on the
+# same socket. Never reuse sequence IDs while old frame ACKs are in transit.
+_FRAME_SEQUENCES = itertools.count(1)
CLOSE_UNAUTHENTICATED = 4001
CLOSE_ATTACH = 4002
@@ -101,6 +106,11 @@ class Viewer:
self.wakeup = asyncio.Event()
self.bucket = _Bucket(_INPUT_RATE, _INPUT_BURST)
self.closed = False
+ self.metrics = StreamMetrics()
+ self._pending_at = 0.0
+ self._outstanding: Dict[int, float] = {}
+ self._started_at = time.monotonic()
+ self._first_frame = True
def send_json(self, message: Dict[str, Any]) -> None:
if self.closed:
@@ -112,27 +122,65 @@ class Viewer:
self.wakeup.set()
def offer_frame(self, frame: bytes) -> bool:
- """Queue a frame; ``False`` when it was dropped (throttled)."""
+ """Keep the latest frame, including the final update inside an FPS interval."""
if self.closed or not self.visible:
return False
- if self.max_fps and time.monotonic() - self.last_frame_at < 1.0 / self.max_fps:
- return False
+ if self.pending is not None:
+ self.metrics.count("replaced")
self.pending = frame
+ self._pending_at = time.monotonic()
self.wakeup.set()
return True
+ def acknowledge(self, seq: Any = None) -> None:
+ # A corrupt envelope cannot yield a sequence. Its ACK consumes the oldest
+ # outstanding frame; well-formed duplicate/stale ACKs never grant credit.
+ if seq is None:
+ seq = next(iter(self._outstanding), None)
+ if not isinstance(seq, int):
+ return
+ sent = self._outstanding.pop(seq, None)
+ if sent is not None:
+ self.inflight = len(self._outstanding)
+ self.metrics.observe("viewer_ack", time.monotonic() - sent)
+ self.wakeup.set()
+
async def writer(self, on_sent: Any) -> None:
while not self.closed:
- await self.wakeup.wait()
self.wakeup.clear()
while not self.control.empty():
await self.websocket.send_text(json.dumps(self.control.get_nowait(), default=str))
- if self.pending is not None and self.inflight < _WINDOW:
- frame, self.pending = self.pending, None
- self.inflight += 1
- self.last_frame_at = time.monotonic()
- await self.websocket.send_bytes(frame)
- on_sent(self)
+ delay = None
+ if self.visible and self.pending is not None and self.inflight < _WINDOW:
+ delay = max(0.0, self.last_frame_at + (1.0 / self.max_fps if self.max_fps else 0) - time.monotonic())
+ if delay == 0:
+ frame, self.pending = self.pending, None
+ head_end = 4 + struct.unpack_from(">H", frame, 2)[0]
+ header = json.loads(frame[4:head_end])
+ seq = header["seq"]
+ self.last_frame_at = time.monotonic()
+ self._outstanding[seq] = self.last_frame_at
+ self.inflight = len(self._outstanding)
+ self.metrics.observe("frame_queue", self.last_frame_at - self._pending_at)
+ captured_at = header.get("ts")
+ if isinstance(captured_at, (int, float)):
+ self.metrics.observe("capture_to_send", time.time() - captured_at)
+ await self.websocket.send_bytes(frame)
+ self.metrics.observe("socket_send", time.monotonic() - self.last_frame_at)
+ if self._first_frame:
+ self._first_frame = False
+ self.metrics.observe("first_frame", time.monotonic() - self._started_at)
+ on_sent(self)
+ continue
+ if not self.visible:
+ self.pending = None
+ if delay is None:
+ await self.wakeup.wait()
+ else:
+ try:
+ await asyncio.wait_for(self.wakeup.wait(), timeout=delay)
+ except asyncio.TimeoutError:
+ pass
class ScreencastHub:
@@ -148,12 +196,18 @@ class ScreencastHub:
self._active = False
self._size = (0, 0)
self._resize_task: Optional[asyncio.Task] = None
- self._pending_ack: Optional[int] = None
- self._ack_timer: Optional[asyncio.TimerHandle] = None
+ self._retry_task: Optional[asyncio.Task] = None
+ self._pending_acks: List[tuple[CDPSession, int]] = []
+ self._ack_tasks: set[asyncio.Task] = set()
+ self._capture_generation = 0
+ self.metrics = StreamMetrics()
self._unsubscribers: List[Any] = []
self._lock = asyncio.Lock()
self.dead = False
self._unlisten = self.controller.add_listener(self._on_controller)
+ from ._live_control import LiveControlQueue
+
+ self.commands = LiveControlQueue(self)
# -- viewers -------------------------------------------------------------------
@@ -163,8 +217,15 @@ class ScreencastHub:
await self._refresh()
async def remove(self, viewer: Viewer) -> None:
+ self.commands.release(viewer, note="the owner closed the live view")
self.viewers.pop(viewer.id, None)
self.controller.viewer_detached(viewer.id)
+ if not self.viewers:
+ await self.commands.idle_without_viewers()
+ pending = [task for task in (self._resize_task, self._retry_task) if task is not None and task is not asyncio.current_task()]
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
await self._refresh()
def visible_viewers(self) -> List[Viewer]:
@@ -192,6 +253,8 @@ class ScreencastHub:
return {"type": "tabs", "tabs": [dict(t, active=t.get("target_id") == active) for t in self.controller.tabs.values()]}
async def _on_controller(self, kind: str, payload: Dict[str, Any]) -> None:
+ if kind in ("tabs", "page"):
+ self.commands.target_changed()
if kind in ("state", "request", "control_moved"):
for viewer in list(self.viewers.values()):
viewer.send_json(self.state_message(viewer))
@@ -207,6 +270,7 @@ class ScreencastHub:
await self._follow_target()
elif kind == "closed":
self.dead = True
+ await self.commands.close()
self._unlisten()
self.broadcast({"type": "idle", "reason": payload.get("reason")})
await self._stop_screencast()
@@ -224,11 +288,38 @@ class ScreencastHub:
async def _refresh(self) -> None:
async with self._lock:
size = self._wanted_size()
- if size == (0, 0):
+ if self.dead or size == (0, 0):
+ self._cancel_retry()
await self._stop_screencast_locked()
return
if not self._active or size != self._size or self.target_id != self.controller.active_target_id:
await self._start_screencast_locked(size)
+ if not self._active and self.runtime.running:
+ self._schedule_retry()
+
+ def _cancel_retry(self) -> None:
+ if self._retry_task is not None and self._retry_task is not asyncio.current_task():
+ self._retry_task.cancel()
+ self._retry_task = None
+
+ def _schedule_retry(self) -> None:
+ if self._retry_task is not None and not self._retry_task.done():
+ return
+
+ async def retry() -> None:
+ for delay in _SCREENCAST_RETRY_DELAYS:
+ await asyncio.sleep(delay)
+ if self.dead or not self.visible_viewers() or not self.runtime.running or self._active:
+ return
+ await self._refresh()
+ if self._active:
+ return
+ self.broadcast({"type": "error", "code": "screencast", "message": "The browser live view could not start. Reconnect the view to try again.", "retrying": False})
+
+ self._retry_task = asyncio.create_task(retry(), name="browser-screencast-retry")
+
+ def _stream_error(self, exc: Exception) -> None:
+ self.broadcast({"type": "error", "code": "screencast", "message": f"The browser live view could not start: {str(exc)[:200]}", "retrying": True})
def schedule_resize(self) -> None:
if self._resize_task is not None and not self._resize_task.done():
@@ -252,10 +343,12 @@ class ScreencastHub:
session = await self.runtime.page_session(self.controller.active_target_id)
except (CDPError, CDPDisconnected, TimeoutError) as exc:
logger.debug("[browser] live view could not attach: %s", exc)
+ self._stream_error(exc)
return
self.session, self.target_id = session, session.target_id
+ generation = self._capture_generation
self._unsubscribers = [
- session.on("Page.screencastFrame", self._on_frame),
+ session.on("Page.screencastFrame", lambda params: self._on_frame(params, session, generation)),
session.on("Page.javascriptDialogOpening", self._on_dialog),
]
try:
@@ -263,12 +356,13 @@ class ScreencastHub:
await session.send("Emulation.setFocusEmulationEnabled", {"enabled": True}, timeout=10)
await session.send(
"Page.startScreencast",
- {"format": "jpeg", "quality": 60, "maxWidth": size[0], "maxHeight": size[1], "everyNthFrame": 1, "maxFramesInFlight": _WINDOW},
+ {"format": "jpeg", "quality": 60, "maxWidth": size[0], "maxHeight": size[1], "everyNthFrame": 1, "maxFramesInFlight": _WINDOW, "sendLastFrame": True},
timeout=10,
)
except (CDPError, CDPDisconnected, TimeoutError) as exc:
logger.debug("[browser] startScreencast failed: %s", exc)
await self._stop_screencast_locked()
+ self._stream_error(exc)
return
self._active, self._size = True, size
self.broadcast({"type": "page", "target_id": self.target_id, **self._tab_facts(self.target_id)})
@@ -279,9 +373,19 @@ class ScreencastHub:
async def _stop_screencast(self) -> None:
async with self._lock:
+ self._cancel_retry()
await self._stop_screencast_locked()
async def _stop_screencast_locked(self) -> None:
+ self._capture_generation += 1
+ self._pending_acks.clear()
+ ack_tasks = list(self._ack_tasks)
+ for task in ack_tasks:
+ task.cancel()
+ if ack_tasks:
+ await asyncio.gather(*ack_tasks, return_exceptions=True)
+ for viewer in self.viewers.values():
+ viewer.pending = None
for unsubscribe in self._unsubscribers:
unsubscribe()
self._unsubscribers = []
@@ -294,12 +398,24 @@ class ScreencastHub:
pass
await session.detach()
- def _on_frame(self, params: Dict[str, Any]) -> None:
- self.seq += 1
+ def _on_frame(self, params: Dict[str, Any], source: Optional[CDPSession] = None, generation: Optional[int] = None) -> None:
+ if generation is not None and generation != self._capture_generation:
+ return
+ source = source or self.session
+ ack_id = params.get("sessionId")
+ if source is not None and isinstance(ack_id, int):
+ self._pending_acks.append((source, ack_id))
+ started = time.monotonic()
+ self.metrics.count("captured")
+ if not self.visible_viewers():
+ self._ack_now()
+ return
+ self.seq = next(_FRAME_SEQUENCES)
meta = params.get("metadata") or {}
try:
- jpeg = base64.b64decode(params.get("data") or "")
- except ValueError:
+ jpeg = base64.b64decode(params.get("data") or "", validate=True)
+ except (ValueError, TypeError):
+ self._ack_now()
return
header = {
"seq": self.seq,
@@ -313,28 +429,38 @@ class ScreencastHub:
"ts": meta.get("timestamp"),
}
frame = encode_frame(header, jpeg)
- offered = False
+ self.metrics.observe("frame_pack", time.monotonic() - started)
for viewer in self.visible_viewers():
- offered = viewer.offer_frame(frame) or offered
- self._pending_ack = params.get("sessionId")
- if not offered:
- self._ack_now()
- elif self._ack_timer is None:
- self._ack_timer = asyncio.get_running_loop().call_later(_ACK_AFTER, self._ack_now)
+ viewer.offer_frame(frame)
+ # Viewer backpressure is already bounded independently. Holding Chrome's
+ # capture credit for a slow decoder causes it to skip the last update.
+ self._ack_now()
def frame_sent(self, viewer: Viewer) -> None:
- self._ack_now()
+ self.metrics.count("sent")
def _ack_now(self) -> None:
- if self._ack_timer is not None:
- self._ack_timer.cancel()
- self._ack_timer = None
- session_id, self._pending_ack = self._pending_ack, None
- session = self.session
- if session_id is None or session is None:
+ debt, self._pending_acks = self._pending_acks, []
+ if not debt:
return
- task = asyncio.ensure_future(session.send("Page.screencastFrameAck", {"sessionId": session_id}, timeout=5))
- task.add_done_callback(lambda t: t.cancelled() or t.exception())
+ generation = self._capture_generation
+
+ async def flush() -> None:
+ try:
+ for session, session_id in debt:
+ if generation != self._capture_generation:
+ return
+ await session.send("Page.screencastFrameAck", {"sessionId": session_id}, timeout=5)
+ except (CDPError, CDPDisconnected, TimeoutError):
+ # Delivery may have succeeded. Never retry an ambiguous ACK and
+ # over-credit Chrome; recover with a fresh capture generation.
+ if generation == self._capture_generation and not self.dead:
+ self._active = False
+ self._schedule_retry()
+
+ task = asyncio.create_task(flush(), name="browser-frame-ack")
+ self._ack_tasks.add(task)
+ task.add_done_callback(self._ack_tasks.discard)
def _on_dialog(self, params: Dict[str, Any]) -> None:
self.broadcast({"type": "dialog", "kind": params.get("type"), "message": str(params.get("message") or "")[:2000]})
@@ -349,10 +475,11 @@ class ScreencastHub:
return max(0.0, min(fx, float(VIEWPORT_WIDTH * 4))), max(0.0, min(fy, float(VIEWPORT_HEIGHT * 20)))
async def input(self, viewer: Viewer, message: Dict[str, Any]) -> None:
- if not self.controller.can_inject_input(viewer.id) or self.session is None:
+ session = self.commands.input_session(viewer)
+ if not self.controller.can_inject_input(viewer.id) or session is None:
return
self.controller.touch_user_input()
- session, kind = self.session, message.get("type")
+ kind = message.get("type")
modifiers = int(message.get("modifiers") or 0) & 15
try:
if kind == "mouse":
@@ -408,7 +535,8 @@ class ScreencastHub:
async def navigate(self, viewer: Viewer, message: Dict[str, Any], policy: Any) -> None:
from ._netpolicy import url_block_reason
- if not self.controller.can_inject_input(viewer.id) or self.session is None:
+ session = self.commands.input_session(viewer)
+ if not self.controller.can_inject_input(viewer.id) or session is None:
return
action = str(message.get("action") or "")
try:
@@ -420,15 +548,15 @@ class ScreencastHub:
if reason:
viewer.send_json({"type": "error", "code": "blocked", "message": f"Cannot open {url}: {reason}."})
return
- await self.session.send("Page.navigate", {"url": url}, timeout=30)
+ await session.send("Page.navigate", {"url": url}, timeout=30)
elif action == "reload":
- await self.session.send("Page.reload", timeout=30)
+ await session.send("Page.reload", timeout=30)
elif action in ("back", "forward"):
- history = await self.session.send("Page.getNavigationHistory", timeout=10)
+ history = await session.send("Page.getNavigationHistory", timeout=10)
index = history["currentIndex"] + (-1 if action == "back" else 1)
entries = history.get("entries") or []
if 0 <= index < len(entries):
- await self.session.send("Page.navigateToHistoryEntry", {"entryId": entries[index]["id"]}, timeout=30)
+ await session.send("Page.navigateToHistoryEntry", {"entryId": entries[index]["id"]}, timeout=30)
except (CDPError, CDPDisconnected, TimeoutError) as exc:
viewer.send_json({"type": "error", "code": "navigation", "message": str(exc)[:300]})
@@ -436,11 +564,12 @@ class ScreencastHub:
if not self.controller.can_inject_input(viewer.id):
return
action, target = str(message.get("action") or ""), str(message.get("target_id") or "")
- cdp = self.runtime.cdp
+ cdp = self.commands.guarded_session(viewer, self.runtime.cdp)
try:
if action == "activate" and target in self.controller.tabs:
await cdp.send("Target.activateTarget", {"targetId": target}, timeout=10)
self.controller.active_target_id = target
+ self.commands.target_changed()
await self._refresh()
self.broadcast(self.tabs_message())
elif action == "close" and target in self.controller.tabs and len(self.controller.tabs) > 1:
@@ -448,26 +577,29 @@ class ScreencastHub:
elif action == "new":
created = await cdp.send("Target.createTarget", {"url": "about:blank"}, timeout=10)
self.controller.active_target_id = created.get("targetId")
+ self.commands.target_changed()
await self._refresh()
except (CDPError, CDPDisconnected, TimeoutError) as exc:
viewer.send_json({"type": "error", "code": "tab", "message": str(exc)[:300]})
async def dialog_reply(self, viewer: Viewer, message: Dict[str, Any]) -> None:
- if not self.controller.can_inject_input(viewer.id) or self.session is None:
+ session = self.commands.input_session(viewer)
+ if not self.controller.can_inject_input(viewer.id) or session is None:
return
params: Dict[str, Any] = {"accept": bool(message.get("accept"))}
if message.get("prompt_text") is not None:
params["promptText"] = str(message["prompt_text"])[:_TEXT_LIMIT]
try:
- await self.session.send("Page.handleJavaScriptDialog", params, timeout=10)
+ await session.send("Page.handleJavaScriptDialog", params, timeout=10)
except (CDPError, CDPDisconnected, TimeoutError):
pass
async def copy(self, viewer: Viewer) -> None:
- if not self.controller.can_inject_input(viewer.id) or self.session is None:
+ session = self.commands.input_session(viewer)
+ if not self.controller.can_inject_input(viewer.id) or session is None:
return
try:
- result = await self.session.send(
+ result = await session.send(
"Runtime.evaluate", {"expression": "String(window.getSelection ? window.getSelection() : '')", "returnByValue": True}, timeout=5
)
except (CDPError, CDPDisconnected, TimeoutError):
@@ -591,8 +723,7 @@ async def browser_live_view(websocket: WebSocket) -> None:
if kind == "ping":
viewer.send_json({"type": "pong"})
elif kind == "ack":
- viewer.inflight = max(0, viewer.inflight - 1)
- viewer.wakeup.set()
+ viewer.acknowledge(message.get("seq"))
elif kind == "viewport":
_apply_viewport(viewer, message)
if hub is not None:
@@ -600,30 +731,18 @@ async def browser_live_view(websocket: WebSocket) -> None:
elif kind == "visibility":
viewer.visible = bool(message.get("visible"))
if hub is not None:
- await hub._refresh()
+ if not viewer.visible:
+ hub.commands.release(viewer, note="the live view was hidden")
+ hub.schedule_resize()
elif hub is None:
viewer.send_json({"type": "idle", "reason": "not running"})
elif kind == "control_request":
- try:
- await hub.controller.acquire_lease(session, wait=2.0)
- except NodeUserError as exc:
- viewer.send_json({"type": "control", "granted": False, "reason": "held_by_other", "message": str(exc)[:300]})
- continue
- granted, reason = await hub.controller.take_over(viewer.id, force=bool(message.get("force")))
- viewer.send_json({"type": "control", "granted": granted, "reason": reason})
+ hub.commands.enqueue(viewer, message, session)
elif kind == "control_release":
outcome = "declined" if message.get("outcome") == "declined" else "handed_back"
- await hub.controller.hand_back(viewer.id, outcome=outcome, note=str(message.get("note") or "")[:1000])
- elif kind in ("mouse", "wheel", "key", "insert_text"):
- await hub.input(viewer, message)
- elif kind == "navigate":
- await hub.navigate(viewer, message, session.policy)
- elif kind == "tab":
- await hub.tab(viewer, message, session.policy)
- elif kind == "dialog_reply":
- await hub.dialog_reply(viewer, message)
- elif kind == "copy":
- await hub.copy(viewer)
+ hub.commands.release(viewer, outcome=outcome, note=str(message.get("note") or "")[:1000])
+ elif kind in ("mouse", "wheel", "key", "insert_text", "navigate", "tab", "dialog_reply", "copy"):
+ hub.commands.enqueue(viewer, message, session)
try:
await reader()
@@ -635,6 +754,7 @@ async def browser_live_view(websocket: WebSocket) -> None:
viewer.closed = True
viewer.wakeup.set()
writer.cancel()
+ await asyncio.gather(writer, return_exceptions=True)
if hub is not None:
await hub.remove(viewer)
elif controller is not None:
diff --git a/server/nodes/browser/_stream_metrics.py b/server/nodes/browser/_stream_metrics.py
new file mode 100644
index 00000000..ef29fa34
--- /dev/null
+++ b/server/nodes/browser/_stream_metrics.py
@@ -0,0 +1,43 @@
+"""Opt-in, bounded live-view diagnostics. Never accepts page or input payloads."""
+
+from __future__ import annotations
+
+import os
+import time
+from collections import Counter, defaultdict, deque
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class StreamMetrics:
+ def __init__(self) -> None:
+ self.enabled = os.environ.get("OPENCOMPANY_BROWSER_DIAGNOSTICS") == "1"
+ self.samples: dict[str, deque[float]] = defaultdict(lambda: deque(maxlen=256))
+ self.counts: Counter[str] = Counter()
+ self._reported = time.monotonic()
+
+ def observe(self, name: str, seconds: float) -> None:
+ if self.enabled:
+ self.samples[name].append(max(0.0, seconds) * 1000)
+ self._report()
+
+ def count(self, name: str) -> None:
+ if self.enabled:
+ self.counts[name] += 1
+ self._report()
+
+ def snapshot(self) -> dict:
+ timings = {}
+ for name, samples in self.samples.items():
+ values = sorted(samples)
+ if values:
+ timings[name] = {"n": len(values), "p50_ms": round(values[(len(values) - 1) // 2], 2), "p95_ms": round(values[min(len(values) - 1, int(len(values) * .95))], 2)}
+ return {"timings": timings, "counts": dict(self.counts)}
+
+ def _report(self) -> None:
+ now = time.monotonic()
+ if now - self._reported >= 5:
+ self._reported = now
+ logger.info("[browser] live-view timings %s", self.snapshot())
diff --git a/server/scripts/benchmark_browser_stream.py b/server/scripts/benchmark_browser_stream.py
new file mode 100644
index 00000000..7b10b2e9
--- /dev/null
+++ b/server/scripts/benchmark_browser_stream.py
@@ -0,0 +1,378 @@
+"""Real local Chrome → production stream → reference canvas latency benchmark.
+
+No download, real user profile, application database, or external website is used.
+Authentication/session lookup are replaced inside this benchmark process only.
+The measured viewer is a minimal reference decoder, NOT the React application.
+Run from server/: .venv/Scripts/python scripts/benchmark_browser_stream.py --help
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import contextlib
+import hashlib
+import importlib.util
+import json
+import math
+import os
+from pathlib import Path
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from types import ModuleType, SimpleNamespace
+
+SERVER = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SERVER))
+
+PAGE = """<!doctype html><meta charset=utf-8><style>
+html,body{margin:0;width:100%;height:100%;overflow:hidden;background:#888}canvas{display:block}
+</style><canvas id=p width=1280 height=800></canvas><script>
+const ctx=p.getContext('2d');let value=0;
+function paint(){ctx.fillStyle='#888';ctx.fillRect(0,0,1280,800);
+for(let i=0;i<16;i++){ctx.fillStyle=(value>>i)&1?'#fff':'#000';ctx.fillRect(i*32,0,32,32)}
+ctx.fillStyle='#111';ctx.font='40px sans-serif';ctx.fillText('Local stream benchmark '+value,80,150)}
+addEventListener('mousedown',()=>{value++;paint()});paint();
+</script>"""
+
+VIEWER = """<!doctype html><meta charset=utf-8><canvas id=p width=1280 height=800></canvas><script>
+const cfg=__CONFIG__,ctx=p.getContext('2d'),metrics={latencies:[],frameAge:[],decodeDraw:[],paintWait:[],misses:[],observed:[],errors:[],visibility:document.visibilityState};
+window.benchmark={ready:false,done:false,metrics};let count=0,pending=new Map(),chain=Promise.resolve();
+const ws=new WebSocket('ws://'+location.host+'/ws/browser');ws.binaryType='arraybuffer';
+const send=m=>ws.send(JSON.stringify(m));
+const pause=ms=>new Promise(r=>setTimeout(r,ms));
+const paintTurn=()=>new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));
+ws.onopen=()=>send({type:'attach',target:{kind:'node',workflow_id:'benchmark',node_id:'browser'},
+viewport:{width:1280,height:800,dpr:1},visible:true,max_fps:cfg.fps});
+ws.onmessage=e=>{if(typeof e.data==='string'){const m=JSON.parse(e.data);
+if(m.type==='attached')send({type:'control_request'});
+if(m.type==='state'&&m.controller==='you')window.benchmark.ready=true;
+if(m.type==='error')metrics.errors.push(m);return}
+chain=chain.then(async()=>{let seq;try{const decodeStarted=performance.now(),v=new DataView(e.data),end=4+v.getUint16(2,false);
+const h=JSON.parse(new TextDecoder().decode(new Uint8Array(e.data,4,end-4)));seq=h.seq;
+const image=await createImageBitmap(new Blob([e.data.slice(end)],{type:'image/jpeg'}));
+ctx.drawImage(image,0,0,1280,800);image.close();let code=0;
+for(let i=0;i<16;i++)if(ctx.getImageData(i*32+16,16,1,1).data[0]>128)code|=1<<i;
+const drawn=performance.now();await paintTurn();const now=performance.now();
+metrics.decodeDraw.push(drawn-decodeStarted);metrics.paintWait.push(now-drawn);
+metrics.observed.push(code);
+if(h.ts)metrics.frameAge.push(Date.now()-h.ts*1000);
+const sample=pending.get(code);if(sample){metrics.latencies.push(now-sample.started);pending.delete(code);sample.resolve()}
+}catch(error){metrics.errors.push(String(error))}finally{if(ws.readyState===1)send({type:'ack',seq})}})};
+ws.onerror=()=>metrics.errors.push('socket error');
+window.runBenchmark=async()=>{for(let i=0;i<cfg.samples+cfg.warmup;i++){
+await pause(cfg.interval_ms+(i%7));count++;
+let resolve;const seen=new Promise(r=>resolve=r);pending.set(count,{started:performance.now(),resolve});
+send({type:'mouse',action:'down',x:600,y:400,button:'left',buttons:1,click_count:1});
+send({type:'mouse',action:'up',x:600,y:400,button:'left',buttons:0,click_count:1});
+await Promise.race([seen,pause(cfg.deadline_ms)]);
+if(pending.has(count)){metrics.misses.push(count);pending.delete(count)}
+if(i===cfg.warmup-1){metrics.latencies=[];metrics.frameAge=[];metrics.decodeDraw=[];metrics.paintWait=[];metrics.misses=[];metrics.observed=[]}}
+send({type:'control_release'});window.benchmark.done=true;return metrics};
+</script>"""
+
+
+class BenchController:
+ """Identity/control fixture; production stream reader and CDP input are unchanged."""
+
+ def __init__(self, target: str, url: str):
+ self.active_target_id = target
+ self.tabs = {target: {"target_id": target, "url": url, "title": "Local benchmark"}}
+ self.controller_viewer = None
+ self.listeners = []
+
+ def add_listener(self, listener):
+ self.listeners.append(listener)
+ return lambda: self.listeners.remove(listener) if listener in self.listeners else None
+
+ def viewer_attached(self, _viewer):
+ pass
+
+ def viewer_detached(self, viewer):
+ if viewer == self.controller_viewer:
+ self.controller_viewer = None
+
+ def snapshot(self, viewer=None):
+ return {
+ "state": "user" if self.controller_viewer else "idle",
+ "controller": "you" if viewer == self.controller_viewer and viewer else None,
+ "request": None,
+ "profile": {"id": "benchmark", "name": "Benchmark"},
+ }
+
+ async def acquire_lease(self, _session, **_kwargs):
+ pass
+
+ async def take_over(self, viewer, **_kwargs):
+ self.controller_viewer = viewer
+ for listener in self.listeners:
+ await listener("state", {})
+ return True, "granted"
+
+ async def hand_back(self, _viewer, **_kwargs):
+ self.controller_viewer = None
+
+ def can_inject_input(self, viewer):
+ return viewer == self.controller_viewer
+
+ def touch_user_input(self):
+ pass
+
+
+def chrome_path(value: str | None) -> Path:
+ candidates = (
+ [value]
+ if value
+ else [
+ shutil.which("google-chrome"),
+ shutil.which("chromium"),
+ shutil.which("chromium-browser"),
+ r"C:\Program Files\Google\Chrome\Application\chrome.exe",
+ str(Path(os.environ.get("LOCALAPPDATA", "")) / "Google/Chrome/Application/chrome.exe"),
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+ ]
+ )
+ for candidate in candidates:
+ if candidate and Path(candidate).is_file():
+ return Path(candidate).resolve()
+ raise SystemExit("No installed Chrome found. Supply --chrome; this benchmark never downloads a browser.")
+
+
+def summarize(values):
+ if not values:
+ return {"count": 0, "p50_ms": None, "p95_ms": None}
+ ordered = sorted(values)
+ return {
+ "count": len(values),
+ "p50_ms": round(ordered[math.ceil(len(values) * 0.5) - 1], 2),
+ "p95_ms": round(ordered[math.ceil(len(values) * 0.95) - 1], 2),
+ "max_ms": round(ordered[-1], 2),
+ }
+
+
+async def run(args, scratch: Path):
+ os.environ["DATA_DIR"] = str(scratch / "data")
+ from fastapi import FastAPI
+ from fastapi.responses import HTMLResponse
+ import uvicorn
+
+ # Load only the two measured modules, avoiding the app's eager plugin
+ # discovery and all user settings/database initialization. These fixtures
+ # exist solely in this standalone process and never replace project files.
+ def fixture(name, **values):
+ module = ModuleType(name)
+ module.__dict__.update(values)
+ sys.modules[name] = module
+ return module
+
+ fixture("nodes", __path__=[str(SERVER / "nodes")])
+ fixture("nodes.browser", __path__=[str(SERVER / "nodes/browser")])
+ fixture("nodes.browser._chrome", VIEWPORT_WIDTH=1280, VIEWPORT_HEIGHT=800)
+ runtime_module = fixture("nodes.browser._runtime")
+ auth_module = fixture("services.authz")
+ fixture("services.plugin", __path__=[str(SERVER / "services/plugin")])
+ fixture("services.plugin.base", NodeUserError=type("NodeUserError", (Exception,), {}))
+ fixture("core.container", container=SimpleNamespace(settings=lambda: None, user_auth_service=None))
+ from nodes.browser._cdp import CDPConnection, read_devtools_active_port
+
+ source = (args.stream_source or SERVER / "nodes/browser/_stream.py").resolve()
+ source_bytes = source.read_bytes()
+ spec = importlib.util.spec_from_file_location("nodes.browser._benchmark_stream", source)
+ stream = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = stream
+ exec(compile(source_bytes, str(source), "exec"), stream.__dict__)
+ config = {
+ "samples": args.samples,
+ "warmup": args.warmup,
+ "fps": args.fps,
+ "interval_ms": args.interval_ms,
+ "deadline_ms": args.deadline_ms,
+ }
+ app = FastAPI()
+ app.add_api_route("/page", lambda: HTMLResponse(PAGE))
+ app.add_api_route("/viewer", lambda: HTMLResponse(VIEWER.replace("__CONFIG__", json.dumps(config))))
+ app.add_api_websocket_route("/ws/browser", stream.browser_live_view)
+ listener = socket.socket()
+ listener.bind(("127.0.0.1", 0))
+ port = listener.getsockname()[1]
+ http = uvicorn.Server(uvicorn.Config(app, log_level="error", lifespan="off", timeout_graceful_shutdown=3))
+ server_task = asyncio.create_task(http.serve(sockets=[listener]))
+ profile = scratch / "chrome-profile"
+ profile.mkdir()
+ chrome = chrome_path(args.chrome)
+ proc = None
+ cdp = None
+ try:
+ while not http.started:
+ if server_task.done():
+ await server_task
+ raise RuntimeError("Benchmark HTTP server did not start")
+ await asyncio.sleep(0.02)
+ command = [
+ str(chrome),
+ f"--user-data-dir={profile}",
+ "--remote-debugging-address=127.0.0.1",
+ "--remote-debugging-port=0",
+ "--headless=new",
+ "--no-first-run",
+ "--no-default-browser-check",
+ "--disable-background-networking",
+ "--disable-sync",
+ "--disable-component-update",
+ "--disable-background-timer-throttling",
+ "--disable-renderer-backgrounding",
+ "--disable-backgrounding-occluded-windows",
+ "--window-size=1280,800",
+ "--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE localhost, EXCLUDE 127.0.0.1",
+ "about:blank",
+ ]
+ allowed = {"PATH", "SYSTEMROOT", "WINDIR", "TEMP", "TMP", "TMPDIR", "HOME", "USERPROFILE", "DISPLAY", "XDG_RUNTIME_DIR"}
+ environment = {key: value for key, value in os.environ.items() if key.upper() in allowed}
+ proc = subprocess.Popen(
+ command,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ env=environment,
+ creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
+ )
+ deadline = time.monotonic() + 20
+ endpoint = None
+ while time.monotonic() < deadline:
+ endpoint = read_devtools_active_port(profile)
+ if endpoint:
+ break
+ if proc.poll() is not None:
+ raise RuntimeError(f"Isolated Chrome exited with code {proc.returncode}")
+ await asyncio.sleep(0.05)
+ if not endpoint:
+ raise RuntimeError("Isolated Chrome did not expose CDP within 20s")
+ cdp = await CDPConnection.connect(f"ws://127.0.0.1:{endpoint[0]}{endpoint[1]}")
+ version = await cdp.send("Browser.getVersion")
+ target = (await cdp.send("Target.createTarget", {"url": f"http://127.0.0.1:{port}/page"}))["targetId"]
+ target_session = await cdp.attach(target)
+ await target_session.send(
+ "Emulation.setDeviceMetricsOverride", {"width": 1280, "height": 800, "deviceScaleFactor": 1, "mobile": False}
+ )
+ controller = BenchController(target, f"http://127.0.0.1:{port}/page")
+
+ async def page_session(_target=None):
+ return await cdp.attach(_target or target)
+
+ runtime = SimpleNamespace(controller=controller, running=True, page_session=page_session, cdp=cdp)
+ facade = SimpleNamespace(running=lambda _profile: runtime, controller=lambda _profile: controller)
+ runtime_module.get_browser_runtime = lambda: facade
+
+ async def auth(_websocket, **_kwargs):
+ return "benchmark-owner"
+
+ async def attach(_owner, _target):
+ return SimpleNamespace(profile_id="benchmark", session_id="benchmark-session", policy=None)
+
+ auth_module.authenticate_ws = auth
+ stream._resolve_attach = attach
+ viewer_target = (await cdp.send("Target.createTarget", {"url": f"http://127.0.0.1:{port}/viewer"}))["targetId"]
+ viewer = await cdp.attach(viewer_target)
+ await viewer.send("Emulation.setFocusEmulationEnabled", {"enabled": True})
+ deadline = time.monotonic() + 20
+ while time.monotonic() < deadline:
+ state = await viewer.send("Runtime.evaluate", {"expression": "Boolean(window.benchmark?.ready)", "returnByValue": True})
+ if state.get("result", {}).get("value"):
+ break
+ await asyncio.sleep(0.05)
+ else:
+ debug = await viewer.send(
+ "Runtime.evaluate",
+ {
+ "expression": "JSON.stringify({url:location.href,benchmark:window.benchmark,body:document.body.innerText})",
+ "returnByValue": True,
+ },
+ )
+ raise RuntimeError(f"Viewer did not acquire benchmark control within 20s: {debug}")
+ await asyncio.sleep(0.3)
+ result = await viewer.send(
+ "Runtime.evaluate",
+ {"expression": "window.runBenchmark()", "awaitPromise": True, "returnByValue": True},
+ timeout=(args.samples + args.warmup) * (args.deadline_ms + args.interval_ms + 20) / 1000 + 20,
+ )
+ if "exceptionDetails" in result:
+ raise RuntimeError(json.dumps(result["exceptionDetails"]))
+ raw = result["result"]["value"]
+ target_count = await target_session.send("Runtime.evaluate", {"expression": "value", "returnByValue": True})
+ latency = summarize(raw["latencies"])
+ report = {
+ "label": args.label,
+ "measurement": "real Chrome CDP + production stream + reference canvas double-rAF paint proxy",
+ "limitations": [
+ "Not the production React renderer",
+ "Authentication and profile/control lookup are fixtures",
+ "Local static page; not WAN or third-party site latency",
+ "Missed updates are censored and reported separately; percentiles cover observed updates only",
+ ],
+ "stream_source": str(source),
+ "stream_sha256": hashlib.sha256(source_bytes).hexdigest(),
+ "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
+ "chrome": version.get("product"),
+ "config": config,
+ "input_to_paint": latency,
+ "frame_age": summarize(raw["frameAge"]),
+ "reference_decode_draw": summarize(raw["decodeDraw"]),
+ "reference_draw_to_double_raf": summarize(raw["paintWait"]),
+ "reference_visibility": raw["visibility"],
+ "missed_updates": len(raw["misses"]),
+ "target_clicks": target_count.get("result", {}).get("value"),
+ "expected_clicks": args.samples + args.warmup,
+ "errors": raw["errors"],
+ "target_200ms": bool(latency["p95_ms"] is not None and latency["p95_ms"] <= 200 and not raw["misses"]),
+ "raw": raw,
+ }
+ if args.output:
+ args.output.write_text(json.dumps(report, indent=2), encoding="utf-8")
+ print(json.dumps({key: value for key, value in report.items() if key != "raw"}, indent=2), flush=True)
+ finally:
+ if cdp:
+ with contextlib.suppress(Exception):
+ await cdp.send("Browser.close", timeout=5)
+ with contextlib.suppress(asyncio.TimeoutError):
+ await asyncio.wait_for(cdp.close(), timeout=10)
+ if proc and proc.poll() is None:
+ try:
+ await asyncio.to_thread(proc.wait, timeout=8)
+ except subprocess.TimeoutExpired:
+ if os.name == "nt":
+ await asyncio.to_thread(
+ subprocess.run,
+ ["taskkill", "/PID", str(proc.pid), "/T", "/F"],
+ capture_output=True,
+ creationflags=subprocess.CREATE_NO_WINDOW,
+ )
+ else:
+ proc.kill()
+ http.should_exit = True
+ with contextlib.suppress(asyncio.TimeoutError):
+ await asyncio.wait_for(server_task, timeout=10)
+ listener.close()
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--chrome", help="Existing Chrome executable; never downloaded")
+ parser.add_argument("--stream-source", type=Path, help="Preserved baseline _stream.py; default is current source")
+ parser.add_argument("--label", default="current")
+ parser.add_argument("--samples", type=int, default=50)
+ parser.add_argument("--warmup", type=int, default=5)
+ parser.add_argument("--fps", type=float, default=30)
+ parser.add_argument("--interval-ms", type=int, default=25)
+ parser.add_argument("--deadline-ms", type=int, default=1000)
+ parser.add_argument("--output", type=Path, help="JSON report path, preferably in your temporary directory")
+ args = parser.parse_args()
+ if args.samples < 1 or args.warmup < 0 or args.fps <= 0 or args.deadline_ms < 1 or args.interval_ms < 0:
+ parser.error("samples/fps/deadline must be positive; warmup/interval must be nonnegative")
+ with tempfile.TemporaryDirectory(prefix="opencompany-stream-benchmark-") as temp:
+ asyncio.run(run(args, Path(temp)))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server/tests/nodes/browser/test_browser_frame_delivery.py b/server/tests/nodes/browser/test_browser_frame_delivery.py
new file mode 100644
index 00000000..b4fdf45a
--- /dev/null
+++ b/server/tests/nodes/browser/test_browser_frame_delivery.py
@@ -0,0 +1,198 @@
+"""Frame-credit and trailing-delivery regressions without a Chrome dependency."""
+
+import asyncio
+import base64
+import json
+import time
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from nodes.browser import _stream
+from nodes.browser._cdp import CDPDisconnected
+from nodes.browser._stream_metrics import StreamMetrics
+
+
+def frame(seq):
+ return _stream.encode_frame({"seq": seq}, b"jpeg")
+
+
+def sequence(data):
+ return json.loads(data[4:4 + int.from_bytes(data[2:4], "big")])["seq"]
+
+
+def make_hub():
+ session = SimpleNamespace(target_id="page", send=AsyncMock(), detach=AsyncMock(), on=Mock(return_value=lambda: None))
+ controller = SimpleNamespace(active_target_id="page", add_listener=Mock(return_value=lambda: None), tabs={})
+ runtime = SimpleNamespace(controller=controller, running=True, page_session=AsyncMock(return_value=session))
+ hub = _stream.ScreencastHub(runtime)
+ hub.session = session
+ viewer = _stream.Viewer(None, "owner")
+ hub.viewers[viewer.id] = viewer
+ return hub, viewer, session
+
+
+async def stop_writer(viewer, task):
+ viewer.closed = True
+ viewer.wakeup.set()
+ await asyncio.wait_for(task, 1)
+
+
+async def test_final_frame_inside_fps_interval_is_delivered_without_another_event():
+ sent = asyncio.Queue()
+ socket = SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=sent.put_nowait))
+ viewer = _stream.Viewer(socket, "owner")
+ viewer.max_fps = 30
+ viewer.last_frame_at = time.monotonic()
+ viewer.offer_frame(frame(1))
+ task = asyncio.create_task(viewer.writer(lambda _: None))
+ try:
+ assert sequence(await asyncio.wait_for(sent.get(), 0.5)) == 1
+ assert viewer.pending is None
+ finally:
+ await stop_writer(viewer, task)
+
+
+async def test_latest_frame_replaces_pending_and_duplicate_ack_never_grants_credit():
+ sent = asyncio.Queue()
+ viewer = _stream.Viewer(SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=sent.put_nowait)), "owner")
+ task = asyncio.create_task(viewer.writer(lambda _: None))
+ try:
+ for seq in (1, 2):
+ viewer.offer_frame(frame(seq))
+ await asyncio.wait_for(sent.get(), 1)
+ viewer.offer_frame(frame(3))
+ viewer.offer_frame(frame(4))
+ await asyncio.sleep(0)
+ assert viewer.inflight == 2 and sequence(viewer.pending) == 4
+ viewer.acknowledge(1)
+ assert sequence(await asyncio.wait_for(sent.get(), 1)) == 4
+ viewer.acknowledge(1)
+ viewer.acknowledge(999)
+ assert viewer.inflight == 2
+ viewer.acknowledge() # malformed envelope cannot provide seq
+ assert viewer.inflight == 1
+ finally:
+ await stop_writer(viewer, task)
+
+
+async def test_control_messages_bypass_frame_throttle():
+ control = asyncio.Event()
+ viewer = _stream.Viewer(SimpleNamespace(send_text=AsyncMock(side_effect=lambda _: control.set()), send_bytes=AsyncMock()), "owner")
+ viewer.max_fps = 1
+ viewer.last_frame_at = time.monotonic()
+ viewer.offer_frame(frame(1))
+ task = asyncio.create_task(viewer.writer(lambda _: None))
+ try:
+ viewer.send_json({"type": "state"})
+ await asyncio.wait_for(control.wait(), .2)
+ viewer.websocket.send_bytes.assert_not_awaited()
+ finally:
+ await stop_writer(viewer, task)
+
+
+async def test_hidden_viewer_cannot_send_queued_picture():
+ viewer = _stream.Viewer(SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock()), "owner")
+ viewer.offer_frame(frame(1))
+ viewer.visible = False
+ task = asyncio.create_task(viewer.writer(lambda _: None))
+ await asyncio.sleep(0)
+ assert viewer.pending is None
+ viewer.websocket.send_bytes.assert_not_awaited()
+ await stop_writer(viewer, task)
+
+
+async def test_slow_viewer_does_not_stop_fast_viewer():
+ slow_sent, fast_sent = asyncio.Queue(), asyncio.Queue()
+ viewers = [_stream.Viewer(SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=queue.put_nowait)), "owner") for queue in (slow_sent, fast_sent)]
+ tasks = [asyncio.create_task(viewer.writer(lambda _: None)) for viewer in viewers]
+ try:
+ for seq in range(1, 5):
+ for viewer in viewers:
+ viewer.offer_frame(frame(seq))
+ assert sequence(await asyncio.wait_for(fast_sent.get(), 1)) == seq
+ viewers[1].acknowledge(seq)
+ assert viewers[0].inflight == 2
+ assert sequence(viewers[0].pending) == 4
+ assert slow_sent.qsize() == 2
+ finally:
+ for viewer, task in zip(viewers, tasks):
+ await stop_writer(viewer, task)
+
+
[email protected]("payload", [base64.b64encode(b"jpeg").decode(), "invalid!"])
+async def test_each_chrome_frame_occurrence_is_acked_even_with_same_id(payload):
+ hub, _, session = make_hub()
+ for _ in range(2):
+ hub._on_frame({"sessionId": 7, "data": payload})
+ hub._ack_now()
+ await asyncio.gather(*hub._ack_tasks)
+ assert session.send.await_count == 2
+ for call in session.send.await_args_list:
+ assert call.args == ("Page.screencastFrameAck", {"sessionId": 7})
+ await hub._stop_screencast()
+ assert not hub._ack_tasks
+
+
+async def test_old_capture_debt_and_callback_do_not_ack_replacement_session():
+ hub, _, old = make_hub()
+ generation = hub._capture_generation
+ params = {"sessionId": 1, "data": base64.b64encode(b"jpeg").decode()}
+ hub._on_frame(params, old, generation)
+ await hub._stop_screencast()
+ replacement = SimpleNamespace(send=AsyncMock())
+ hub.session = replacement
+ hub._on_frame(params, old, generation)
+ hub._ack_now()
+ await asyncio.sleep(0)
+ replacement.send.assert_not_awaited()
+ assert not hub._pending_acks and not hub._ack_tasks
+
+
+async def test_same_viewer_on_replacement_hub_does_not_reuse_old_frame_sequence():
+ old_hub, viewer, _ = make_hub()
+ sent = asyncio.Queue()
+ viewer.websocket = SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=sent.put_nowait))
+ task = asyncio.create_task(viewer.writer(lambda _: None))
+ params = {"sessionId": 1, "data": base64.b64encode(b"jpeg").decode()}
+ new_hub, _, _ = make_hub()
+ try:
+ old_hub._on_frame(params)
+ old_seq = sequence(await asyncio.wait_for(sent.get(), 1))
+ await old_hub._stop_screencast()
+ new_hub.viewers = {viewer.id: viewer}
+ new_hub._on_frame(params)
+ new_seq = sequence(await asyncio.wait_for(sent.get(), 1))
+ assert new_seq > old_seq
+ assert viewer.inflight == 2
+ viewer.acknowledge(old_seq)
+ viewer.acknowledge(old_seq)
+ assert viewer.inflight == 1 and new_seq in viewer._outstanding
+ finally:
+ await stop_writer(viewer, task)
+ await new_hub._stop_screencast()
+
+
+async def test_failed_ack_starts_fresh_capture_without_retrying_ambiguous_ack():
+ hub, _, session = make_hub()
+ session.send.side_effect = CDPDisconnected("lost")
+ hub._schedule_retry = Mock()
+ hub._on_frame({"sessionId": 9, "data": base64.b64encode(b"jpeg").decode()})
+ hub._ack_now()
+ await asyncio.gather(*hub._ack_tasks)
+ session.send.assert_awaited_once()
+ hub._schedule_retry.assert_called_once()
+ assert not hub._active
+
+
+def test_diagnostics_are_opt_in_and_bounded(monkeypatch):
+ monkeypatch.delenv("OPENCOMPANY_BROWSER_DIAGNOSTICS", raising=False)
+ disabled = StreamMetrics()
+ disabled.observe("frame_pack", 1)
+ assert disabled.snapshot() == {"timings": {}, "counts": {}}
+ monkeypatch.setenv("OPENCOMPANY_BROWSER_DIAGNOSTICS", "1")
+ enabled = StreamMetrics()
+ for _ in range(1000):
+ enabled.observe("frame_pack", .001)
+ assert enabled.snapshot()["timings"]["frame_pack"] == {"n": 256, "p50_ms": 1, "p95_ms": 1}
diff --git a/server/tests/nodes/browser/test_browser_live_control.py b/server/tests/nodes/browser/test_browser_live_control.py
new file mode 100644
index 00000000..2c6bb931
--- /dev/null
+++ b/server/tests/nodes/browser/test_browser_live_control.py
@@ -0,0 +1,342 @@
+"""Ordering, bounded admission and release barriers without a real browser."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from nodes.browser._live_control import LiveControlQueue, MAX_BYTES, MAX_COMMANDS
+from nodes.browser._session import BrowserSession, ControlState, ProfileController, SessionKey
+
+
+def setup_queue():
+ controller = ProfileController("profile", "Browser")
+ controller._broadcast = AsyncMock()
+ controller.active_target_id = "page"
+ controller.tabs = {"page": {"url": "https://example.test"}}
+ controller.state = ControlState.USER
+ controller.controller_viewer = "viewer"
+ session = BrowserSession(SessionKey("owner", "workflow", "node"), "profile")
+ controller.lease_session = session
+ messages = []
+ viewer = SimpleNamespace(id="viewer", visible=True, closed=False, send_json=messages.append)
+ page = SimpleNamespace(send=AsyncMock(return_value={}))
+ hub = SimpleNamespace(controller=controller, session=page, input=AsyncMock(), copy=AsyncMock(), navigate=AsyncMock(), tab=AsyncMock(), dialog_reply=AsyncMock())
+ queue = LiveControlQueue(hub)
+ return queue, hub, viewer, session, messages
+
+
+async def drain(queue):
+ if queue._worker:
+ await asyncio.wait_for(queue._worker, 1)
+
+
+async def test_only_adjacent_moves_and_wheels_coalesce_without_reordering_keys():
+ queue, hub, viewer, session, _ = setup_queue()
+ for message in [
+ {"type": "mouse", "action": "move", "x": 1},
+ {"type": "mouse", "action": "move", "x": 2},
+ {"type": "key", "action": "down", "key": "a"},
+ {"type": "wheel", "delta_y": 4},
+ {"type": "wheel", "delta_y": 6},
+ ]:
+ assert queue.enqueue(viewer, message, session)
+ assert len(queue._queue) == 3
+ await drain(queue)
+ actual = [call.args[1] for call in hub.input.await_args_list]
+ assert [m["type"] for m in actual] == ["mouse", "key", "wheel"]
+ assert actual[0]["x"] == 2 and actual[-1]["delta_y"] == 10
+
+
+async def test_wheels_over_different_scroll_containers_do_not_coalesce():
+ queue, hub, viewer, session, _ = setup_queue()
+ queue.enqueue(viewer, {"type": "wheel", "x": 10, "y": 10, "delta_y": 5}, session)
+ queue.enqueue(viewer, {"type": "wheel", "x": 100, "y": 10, "delta_y": 5}, session)
+ assert len(queue._queue) == 2
+ await drain(queue)
+ assert hub.input.await_count == 2
+
+
[email protected]("limit", ["count", "bytes"])
+async def test_overload_is_explicit_bounded_and_safely_releases_control(limit):
+ queue, _, viewer, session, messages = setup_queue()
+ if limit == "count":
+ for _ in range(MAX_COMMANDS):
+ assert queue.enqueue(viewer, {"type": "key", "action": "down", "key": "x"}, session)
+ rejected = {"type": "key", "action": "up", "key": "x"}
+ else:
+ rejected = {"type": "insert_text", "text": "x" * MAX_BYTES}
+ assert not queue.enqueue(viewer, rejected, session)
+ assert len(queue._queue) <= MAX_COMMANDS and queue._bytes <= MAX_BYTES
+ assert messages[-1]["code"] == "control_overloaded"
+ await drain(queue)
+ assert queue.controller.state == ControlState.IDLE
+
+
+async def test_target_epoch_discards_work_even_if_target_switches_back():
+ queue, hub, viewer, session, _ = setup_queue()
+ queue.enqueue(viewer, {"type": "key", "action": "down", "key": "a"}, session)
+ hub.controller.active_target_id = "other"
+ queue.target_changed()
+ hub.controller.active_target_id = "page"
+ queue.target_changed()
+ await drain(queue)
+ hub.input.assert_not_awaited()
+
+
+async def test_release_waits_dispatched_command_and_releases_keys_before_agent_admission():
+ queue, hub, viewer, session, _ = setup_queue()
+ entered, complete = asyncio.Event(), asyncio.Event()
+ events = []
+
+ async def send(method, params, **kwargs):
+ if method == "Runtime.evaluate":
+ events.append("read-start")
+ entered.set()
+ await complete.wait()
+ events.append("read-finished")
+ else:
+ events.append(params["type"])
+ return {}
+
+ hub.session.send.side_effect = send
+
+ async def key_input(v, message):
+ await queue.guarded_session(v, hub.session).send("Input.dispatchKeyEvent", {"type": "keyDown", "key": "Shift", "code": "ShiftLeft"})
+
+ async def copy(v):
+ await queue.guarded_session(v, hub.session).send("Runtime.evaluate", {})
+
+ hub.input.side_effect, hub.copy.side_effect = key_input, copy
+ queue.enqueue(viewer, {"type": "key", "action": "down", "key": "Shift"}, session)
+ queue.enqueue(viewer, {"type": "copy"}, session)
+ await entered.wait()
+ queue.enqueue(viewer, {"type": "key", "action": "down", "key": "z"}, session)
+ queue.release(viewer)
+ assert not queue.enqueue(viewer, {"type": "key", "action": "down", "key": "late"}, session)
+ await asyncio.sleep(0)
+ assert hub.controller.state == ControlState.USER
+ complete.set()
+ await drain(queue)
+ assert events == ["keyDown", "read-start", "read-finished", "keyUp"]
+ assert hub.controller.state == ControlState.IDLE and not queue._pressed
+
+
+async def test_automatic_handback_uses_same_dispatched_command_barrier():
+ queue, hub, viewer, session, _ = setup_queue()
+ entered, complete = asyncio.Event(), asyncio.Event()
+
+ async def copy(v):
+ entered.set()
+ await complete.wait()
+
+ hub.copy.side_effect = copy
+ queue.enqueue(viewer, {"type": "copy"}, session)
+ await entered.wait()
+ release = asyncio.create_task(hub.controller.hand_back(viewer.id))
+ await asyncio.sleep(0)
+ assert not release.done() and hub.controller.state == ControlState.USER
+ complete.set()
+ await release
+ await drain(queue)
+ assert hub.controller.state == ControlState.IDLE
+
+
+async def test_hidden_viewer_cannot_receive_pending_takeover_and_requests_are_single_flight():
+ queue, hub, viewer, session, _ = setup_queue()
+ hub.controller.state = ControlState.AGENT
+ hub.controller.controller_viewer = None
+ assert queue.enqueue(viewer, {"type": "control_request"}, session)
+ assert not queue.enqueue(viewer, {"type": "control_request"}, session)
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+ viewer.visible = False
+ queue.release(viewer)
+ await drain(queue)
+ assert hub.controller.controller_viewer is None
+ assert hub.controller.state == ControlState.AGENT
+
+
+async def test_guard_rechecks_ownership_between_two_awaited_cdp_commands():
+ queue, hub, viewer, session, messages = setup_queue()
+ entered, complete = asyncio.Event(), asyncio.Event()
+
+ async def send(method, params, **kwargs):
+ entered.set()
+ await complete.wait()
+ return {}
+
+ hub.session.send.side_effect = send
+
+ async def copy(v):
+ page = queue.guarded_session(v, hub.session)
+ await page.send("Page.getNavigationHistory", {})
+ await page.send("Page.navigateToHistoryEntry", {})
+
+ hub.copy.side_effect = copy
+ queue.enqueue(viewer, {"type": "copy"}, session)
+ await entered.wait()
+ queue.release(viewer)
+ complete.set()
+ await drain(queue)
+ assert hub.session.send.await_count == 1
+ assert any(m.get("code") == "control" for m in messages)
+
+
+async def test_timeout_retires_chrome_before_handback_and_failed_retirement_fails_closed():
+ queue, hub, viewer, session, _ = setup_queue()
+ hub.runtime = SimpleNamespace(chrome=SimpleNamespace(shutdown=AsyncMock(side_effect=RuntimeError("busy"))))
+
+ async def copy(v):
+ await queue.guarded_session(v, hub.session).send("Page.navigate", {})
+
+ hub.copy.side_effect = copy
+ hub.session.send.side_effect = TimeoutError("unknown outcome")
+ queue.enqueue(viewer, {"type": "copy"}, session)
+ await drain(queue)
+ assert queue._unsafe
+ queue.release(viewer)
+ await drain(queue)
+ assert hub.controller.state == ControlState.USER
+ hub.runtime.chrome.shutdown.side_effect = None
+ queue.release(viewer)
+ await drain(queue)
+ assert hub.controller.state == ControlState.IDLE
+ assert not queue._unsafe
+
+
+async def test_input_owns_a_session_independent_of_screencast_detach():
+ queue, hub, viewer, session, _ = setup_queue()
+ owned = SimpleNamespace(send=AsyncMock(return_value={}), detach=AsyncMock())
+ hub.runtime = SimpleNamespace(page_session=AsyncMock(return_value=owned))
+
+ async def copy(v):
+ hub.session = None # capture resize can retire the picture session
+ await queue.input_session(v).send("Runtime.evaluate", {})
+
+ hub.copy.side_effect = copy
+ queue.enqueue(viewer, {"type": "copy"}, session)
+ await drain(queue)
+ owned.send.assert_awaited_once()
+ owned.detach.assert_not_awaited()
+ await queue.close()
+ owned.detach.assert_awaited_once()
+
+
+async def test_retirement_close_notification_does_not_wait_on_its_own_worker():
+ queue, hub, viewer, _, _ = setup_queue()
+ queue._unsafe = True
+
+ async def shutdown():
+ await queue.close() # runtime close callback while release is active
+
+ hub.runtime = SimpleNamespace(chrome=SimpleNamespace(shutdown=shutdown))
+ queue.release(viewer)
+ await drain(queue)
+ assert queue._closed and not queue._unsafe
+ assert hub.controller.state == ControlState.IDLE
+
+
+async def test_forced_transfer_releases_original_session_keys_before_new_owner():
+ queue, hub, viewer, session, _ = setup_queue()
+ other = SimpleNamespace(id="other", visible=True, closed=False, send_json=lambda message: None)
+ queue._pressed[(viewer.id, "key", "ShiftLeft")] = (hub.session, {"type": "keyDown", "key": "Shift", "code": "ShiftLeft"})
+ queue.enqueue(other, {"type": "control_request", "force": True}, session)
+ await drain(queue)
+ hub.session.send.assert_awaited_once_with("Input.dispatchKeyEvent", {"type": "keyUp", "key": "Shift", "code": "ShiftLeft", "modifiers": 0}, timeout=5)
+ assert hub.controller.controller_viewer == "other"
+ assert viewer.id in queue._blocked and not queue._pressed
+
+
+async def test_navigation_in_same_target_releases_old_pressed_keys_before_next_input():
+ queue, hub, viewer, session, _ = setup_queue()
+ owned = SimpleNamespace(send=AsyncMock(return_value={}), detach=AsyncMock())
+ hub.runtime = SimpleNamespace(page_session=AsyncMock(return_value=owned))
+ await queue._ensure_input_session()
+ queue._pressed[(viewer.id, "key", "ShiftLeft")] = (owned, {"type": "keyDown", "key": "Shift", "code": "ShiftLeft"})
+ hub.controller.tabs["page"]["url"] = "https://example.test/next"
+ queue.target_changed()
+ queue.enqueue(viewer, {"type": "copy"}, session)
+ await drain(queue)
+ assert owned.send.await_args.args[1]["type"] == "keyUp"
+ assert not queue._pressed
+ hub.copy.assert_awaited_once()
+ owned.detach.assert_not_awaited()
+ await queue.idle_without_viewers()
+ owned.detach.assert_awaited_once()
+ assert queue._worker.done() and queue._input_session is None
+
+
+async def test_socket_reader_handles_ack_visibility_and_ping_while_navigation_waits(monkeypatch):
+ from fastapi import WebSocketDisconnect
+ import core.container
+ import services.authz
+ import nodes.browser._runtime as browser_runtime
+ from nodes.browser import _stream
+
+ queue, hub, _, session, _ = setup_queue()
+ entered, complete, acknowledged, hidden, pong = (asyncio.Event() for _ in range(5))
+ hub.commands = queue
+ hub.dead = False
+ hub.controller.controller_viewer = None
+
+ async def add(viewer):
+ hub.controller.controller_viewer = viewer.id
+
+ async def navigate(viewer, message, policy):
+ entered.set()
+ await complete.wait()
+
+ async def remove(viewer):
+ queue.release(viewer)
+
+ hub.add, hub.remove = add, remove
+ hub.navigate.side_effect = navigate
+ hub.state_message = lambda v: {"type": "state", "state": "user", "controller": "you"}
+ hub.tabs_message = lambda: {"type": "tabs", "tabs": []}
+ hub.schedule_resize = hidden.set
+ hub.frame_sent = lambda v: None
+
+ class Viewer(_stream.Viewer):
+ def acknowledge(self, seq=None):
+ acknowledged.set()
+
+ class Socket:
+ def __init__(self):
+ self.messages = asyncio.Queue()
+ async def accept(self):
+ pass
+ async def receive_json(self):
+ message = await self.messages.get()
+ if message is None:
+ raise WebSocketDisconnect()
+ return message
+ async def send_text(self, text):
+ if '"pong"' in text:
+ pong.set()
+ async def close(self, **kwargs):
+ pass
+
+ runtime = SimpleNamespace(running=lambda profile: object(), controller=lambda profile: hub.controller)
+ monkeypatch.setattr(core.container, "container", SimpleNamespace(settings=lambda: None, user_auth_service=None))
+ monkeypatch.setattr(services.authz, "authenticate_ws", AsyncMock(return_value="owner"))
+ monkeypatch.setattr(browser_runtime, "get_browser_runtime", lambda: runtime)
+ monkeypatch.setattr(_stream, "_resolve_attach", AsyncMock(return_value=session))
+ monkeypatch.setattr(_stream, "get_hub", lambda _: hub)
+ monkeypatch.setattr(_stream, "Viewer", Viewer)
+ socket = Socket()
+ socket.messages.put_nowait({"type": "attach", "target": {}})
+ socket.messages.put_nowait({"type": "navigate", "action": "reload"})
+ task = asyncio.create_task(_stream.browser_live_view(socket))
+ try:
+ await asyncio.wait_for(entered.wait(), 1)
+ for message in ({"type": "ack", "seq": 1}, {"type": "visibility", "visible": False}, {"type": "ping"}):
+ socket.messages.put_nowait(message)
+ await asyncio.wait_for(asyncio.gather(acknowledged.wait(), hidden.wait(), pong.wait()), 1)
+ assert hub.controller.state == ControlState.USER
+ finally:
+ complete.set()
+ socket.messages.put_nowait(None)
+ await asyncio.wait_for(task, 1)
+ await drain(queue)
diff --git a/server/tests/nodes/browser/test_browser_stream.py b/server/tests/nodes/browser/test_browser_stream.py
new file mode 100644
index 00000000..b3f6cc3d
--- /dev/null
+++ b/server/tests/nodes/browser/test_browser_stream.py
@@ -0,0 +1,70 @@
+"""Live-view recovery with fake CDP; never installs or launches Chrome."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from nodes.browser import _stream
+from nodes.browser._cdp import CDPDisconnected
+
+
+def make_hub(*, failures=0, fail_start=False):
+ session = SimpleNamespace(target_id="page", on=Mock(return_value=lambda: None), send=AsyncMock(), detach=AsyncMock())
+ if fail_start:
+ async def send(method, *args, **kwargs):
+ if method == "Page.startScreencast":
+ raise CDPDisconnected("lost")
+ session.send.side_effect = send
+ controller = SimpleNamespace(active_target_id="page", controller_viewer=None, state="idle", tabs={}, add_listener=Mock(return_value=lambda: None), viewer_attached=Mock(), viewer_detached=Mock())
+ page_session = AsyncMock(side_effect=[*[CDPDisconnected("lost") for _ in range(failures)], session]) if failures else AsyncMock(return_value=session)
+ runtime = SimpleNamespace(controller=controller, running=True, page_session=page_session)
+ hub = _stream.ScreencastHub(runtime)
+ viewer = _stream.Viewer(None, "owner")
+ return hub, viewer, runtime, session
+
+
+async def test_initial_attach_failure_recovers_without_viewer_resize(monkeypatch):
+ monkeypatch.setattr(_stream, "_SCREENCAST_RETRY_DELAYS", (0, 0, 0))
+ hub, viewer, runtime, session = make_hub(failures=1)
+ await hub.add(viewer)
+ message = viewer.control.get_nowait()
+ assert message["code"] == "screencast" and message["retrying"] is True
+ await hub._retry_task
+ assert hub._active and runtime.page_session.await_count == 2
+ await hub.remove(viewer)
+ session.detach.assert_awaited_once()
+
+
+async def test_start_failure_retries_are_bounded_and_reported(monkeypatch):
+ monkeypatch.setattr(_stream, "_SCREENCAST_RETRY_DELAYS", (0, 0, 0))
+ hub, viewer, runtime, session = make_hub(fail_start=True)
+ await hub.add(viewer)
+ await hub._retry_task
+ assert not hub._active and runtime.page_session.await_count == 4
+ messages = []
+ while not viewer.control.empty():
+ messages.append(viewer.control.get_nowait())
+ assert messages[-1]["retrying"] is False
+ assert session.detach.await_count == 4
+ await asyncio.sleep(0)
+ assert runtime.page_session.await_count == 4
+ await hub.remove(viewer)
+
+
[email protected]("end", ["hidden", "removed", "closed"])
+async def test_pending_retry_stops_when_no_visible_viewer_or_browser_closes(monkeypatch, end):
+ monkeypatch.setattr(_stream, "_SCREENCAST_RETRY_DELAYS", (60,))
+ hub, viewer, runtime, _ = make_hub(failures=1)
+ await hub.add(viewer)
+ retry = hub._retry_task
+ if end == "hidden":
+ viewer.visible = False
+ await hub._refresh()
+ elif end == "removed":
+ await hub.remove(viewer)
+ else:
+ await hub._on_controller("closed", {"reason": "stopped"})
+ await asyncio.gather(retry, return_exceptions=True)
+ assert retry.cancelled() and runtime.page_session.await_count == 1