Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(browser): keep live streaming responsive during commands
trohitg87322eeb1d48
diff --git a/server/nodes/browser/_live_control.py b/server/nodes/browser/_live_control.pynew file mode 100644index 00000000..52958b13--- /dev/null+++ b/server/nodes/browser/_live_control.py@@ -0,0 +1,373 @@+"""Bounded, ordered user commands independent of the live socket reader.++One queue belongs to one profile hub. Release invalidates queued work at+admission time, then waits for the dispatched command before releasing any+pressed inputs. Chrome commands already sent are never assumed cancelled.+"""++from __future__ import annotations++import asyncio+import json+import time+from collections import deque+from dataclasses import dataclass+from typing import Any++from services.plugin.base import NodeUserError+from ._cdp import CDPDisconnected++MAX_COMMANDS = 64+MAX_BYTES = 64 * 1024+INPUT_KINDS = frozenset({"mouse", "wheel", "key", "insert_text", "navigate", "tab", "copy", "dialog_reply"})+++@dataclass+class Command:+ viewer: Any+ message: dict+ session: Any+ epoch: int+ target: Any+ size: int+ queued_at: float+++class _GuardedSession:+ def __init__(self, queue: "LiveControlQueue", command: Command, session: Any):+ self.queue, self.command, self.session = queue, command, session++ async def send(self, method, params=None, **kwargs):+ if not self.queue.valid(self.command):+ raise NodeUserError("Browser control or page changed; the queued command was discarded.")+ params = params or {}+ identity = None+ if method == "Input.dispatchKeyEvent":+ identity = (self.command.viewer.id, "key", params.get("code") or params.get("key"))+ elif method == "Input.dispatchMouseEvent" and params.get("type") in {"mousePressed", "mouseReleased"}:+ identity = (self.command.viewer.id, "mouse", params.get("button"))+ # Record before dispatch: even a response timeout can leave a key held.+ if identity and params.get("type") not in {"keyUp", "mouseReleased"}:+ if identity not in self.queue._pressed and len(self.queue._pressed) >= MAX_COMMANDS:+ raise NodeUserError("Too many held browser inputs. Hand control back before continuing.")+ self.queue._pressed[identity] = (self.session, dict(params))+ started = time.monotonic()+ try:+ result = await self.session.send(method, params, **kwargs)+ except (TimeoutError, CDPDisconnected):+ self.queue._unsafe = True+ self.queue.invalidate(self.command.viewer.id)+ self.queue._error(self.command.viewer, "control_uncertain", "Chrome did not confirm the action. Hand back control to safely restart the browser.")+ raise+ finally:+ self.queue._observe("command_cdp", time.monotonic() - started)+ if identity and params.get("type") in {"keyUp", "mouseReleased"}:+ self.queue._pressed.pop(identity, None)+ return result+++class LiveControlQueue:+ def __init__(self, hub: Any):+ self.hub = hub+ self.controller = hub.controller+ self._queue: deque[Command] = deque()+ self._bytes = 0+ self._epochs: dict[str, int] = {}+ self._blocked: set[str] = set()+ self._worker: asyncio.Task | None = None+ self._takeover: asyncio.Task | None = None+ self._takeover_viewer: str | None = None+ self._takeover_pending: set[str] = set()+ self._lock = asyncio.Lock()+ self._current: Command | None = None+ self._pressed: dict[tuple, tuple[Any, dict]] = {}+ self._closed = False+ self._unsafe = False+ self._retiring = False+ self._input_session = None+ self._input_target = None+ self._input_epoch = None+ self._target_epoch = 0+ self._last_target = None+ self.controller.live_control_barrier = self.barrier++ def _observe(self, name: str, value: float) -> None:+ metrics = getattr(self.hub, "metrics", None)+ if metrics is not None:+ metrics.observe(name, value)++ def _error(self, viewer, code, message):+ viewer.send_json({"type": "error", "code": code, "message": message})++ def _target(self):+ target = self.controller.active_target_id+ current = target, (self.controller.tabs.get(target or "") or {}).get("url")+ if current != self._last_target:+ self._last_target = current+ self._target_epoch += 1+ return self._target_epoch, current++ def target_changed(self):+ self._target()++ def valid(self, command: Command, *, owner: bool = True) -> bool:+ viewer = command.viewer+ return (+ not self._closed and not viewer.closed and viewer.visible+ and (not owner or viewer.id not in self._blocked)+ and command.epoch == self._epochs.get(viewer.id, 0)+ and command.target == self._target()+ and (not owner or self.controller.can_inject_input(viewer.id))+ )++ def guarded_session(self, viewer, session):+ command = self._current+ if command is not None and command.viewer.id == viewer.id and session is not None:+ return _GuardedSession(self, command, session)+ return session++ def input_session(self, viewer):+ # Dedicated to input: the screencast session may be detached whenever+ # a viewer resizes or becomes hidden.+ return self.guarded_session(viewer, self._input_session)++ async def _ensure_input_session(self):+ target = self.controller.active_target_id+ epoch = self._target()+ if self._input_session is not None and self._input_target == target and self._input_epoch == epoch:+ return+ if self._input_session is not None:+ for viewer_id in {identity[0] for identity in self._pressed}:+ await self._release_pressed(viewer_id)+ if self._input_target == target:+ self._input_epoch = epoch+ return+ await self._input_session.detach()+ self._input_session = await self.hub.runtime.page_session(target)+ self._input_target = target+ self._input_epoch = epoch++ def enqueue(self, viewer, message: dict, session) -> bool:+ if self._closed or viewer.closed or not viewer.visible:+ return False+ kind = message.get("type")+ if kind == "control_request" and viewer.id in self._takeover_pending:+ return False+ if kind in INPUT_KINDS and (viewer.id in self._blocked or self._unsafe or not self.controller.can_inject_input(viewer.id)):+ return False+ size = len(json.dumps(message, ensure_ascii=False).encode("utf-8"))+ command = Command(viewer, dict(message), session, self._epochs.get(viewer.id, 0), self._target(), size, time.monotonic())+ previous = self._queue[-1] if self._queue else None+ if previous and self._coalescible(previous, command):+ if kind == "wheel":+ command.message["delta_x"] = float(previous.message.get("delta_x") or 0) + float(message.get("delta_x") or 0)+ command.message["delta_y"] = float(previous.message.get("delta_y") or 0) + float(message.get("delta_y") or 0)+ command.size = len(json.dumps(command.message).encode("utf-8"))+ if self._bytes - previous.size + command.size <= MAX_BYTES:+ self._queue[-1] = command+ self._bytes += command.size - previous.size+ return True+ if len(self._queue) >= MAX_COMMANDS or self._bytes + command.size > MAX_BYTES:+ self._error(viewer, "control_overloaded", "Browser input is busy. Wait for the current action, then try again.")+ # Never silently lose a key-up on overflow: stop admission and+ # release everything after the current command settles.+ self.release(viewer, note="input queue overflow")+ return False+ if kind == "control_request":+ self._takeover_pending.add(viewer.id)+ self._queue.append(command)+ self._bytes += command.size+ self._ensure_worker()+ return True++ @staticmethod+ def _coalescible(a: Command, b: Command) -> bool:+ if (a.viewer.id, a.epoch, a.target) != (b.viewer.id, b.epoch, b.target):+ return False+ x, y = a.message, b.message+ if x.get("modifiers", 0) != y.get("modifiers", 0) or x.get("type") != y.get("type"):+ return False+ wheel = y.get("type") == "wheel" and all(x.get(k, 0) == y.get(k, 0) for k in ("x", "y", "buttons"))+ return (wheel or (+ y.get("type") == "mouse" and x.get("action") == y.get("action") == "move"+ and x.get("buttons", 0) == y.get("buttons", 0)+ ))++ def invalidate(self, viewer_id: str) -> None:+ self._blocked.add(viewer_id)+ self._epochs[viewer_id] = self._epochs.get(viewer_id, 0) + 1+ for command in self._queue:+ if command.viewer.id == viewer_id and command.message.get("type") == "control_request":+ command.viewer.send_json({"type": "control", "granted": False, "reason": "viewer_unavailable"})+ self._queue = deque(c for c in self._queue if c.viewer.id != viewer_id)+ self._bytes = sum(c.size for c in self._queue)+ self._takeover_pending.discard(viewer_id)+ if self._takeover_viewer == viewer_id and self._takeover is not None:+ self._takeover.cancel()++ def release(self, viewer, *, outcome="handed_back", note="") -> None:+ self.invalidate(viewer.id)+ if getattr(self.controller, "controller_viewer", None) != viewer.id and getattr(self.controller, "state", None) != "awaiting_user":+ return+ # A zero-byte priority barrier is always admissible. At most one per+ # viewer remains because invalidate removes its previous barrier.+ if len(self._queue) >= MAX_COMMANDS:+ removed = self._queue.pop()+ self._bytes -= removed.size+ self._takeover_pending.discard(removed.viewer.id)+ self._error(removed.viewer, "control_overloaded", "Browser control changed. Try your request again.")+ self._queue.appendleft(Command(viewer, {"type": "control_release", "outcome": outcome, "note": note}, None, 0, None, 0, time.monotonic()))+ self._ensure_worker()++ async def barrier(self, viewer_id: str | None) -> None:+ if viewer_id is None:+ return+ self.invalidate(viewer_id)+ if asyncio.current_task() in (self._worker, self._takeover):+ await self._release_pressed(viewer_id)+ else:+ async with self._lock:+ await self._release_pressed(viewer_id)++ async def _release_pressed(self, viewer_id):+ if self._unsafe:+ await self._retire_uncertain_browser()+ return+ for identity, (session, params) in list(self._pressed.items()):+ if identity[0] != viewer_id:+ continue+ release = dict(params)+ if identity[1] == "key":+ release.update(type="keyUp", modifiers=0)+ release.pop("text", None)+ release.pop("unmodifiedText", None)+ method = "Input.dispatchKeyEvent"+ else:+ release.update(type="mouseReleased", buttons=0, modifiers=0)+ method = "Input.dispatchMouseEvent"+ try:+ await session.send(method, release, timeout=5)+ except Exception as exc:+ self._unsafe = True+ try:+ await self._retire_uncertain_browser()+ return+ except Exception:+ raise NodeUserError("Could not release browser input safely. Restart the browser before handing control back.") from exc+ self._pressed.pop(identity, None)++ async def _retire_uncertain_browser(self):+ # A local timeout only abandons the reply; Chrome may still execute+ # the command. Closing this managed process is the safety boundary.+ try:+ self._retiring = True+ await asyncio.wait_for(self.hub.runtime.chrome.shutdown(), timeout=10)+ except Exception as exc:+ raise NodeUserError("Chrome has an unconfirmed action and could not be stopped. Control remains with you.") from exc+ finally:+ self._retiring = False+ self._unsafe = False+ self._pressed.clear()+ self._input_session = None+ self._input_target = None+ self._input_epoch = None++ def _ensure_worker(self):+ if self._worker is None or self._worker.done():+ self._worker = asyncio.create_task(self._run(), name="browser-live-control")++ async def _run(self):+ while self._queue:+ command = self._queue.popleft()+ self._bytes -= command.size+ viewer, message = command.viewer, command.message+ kind = message.get("type")+ async with self._lock:+ self._current = command+ try:+ self._observe("command_queue_wait", time.monotonic() - command.queued_at)+ if kind == "control_release":+ await self.controller.hand_back(viewer.id, outcome=message.get("outcome", "handed_back"), note=message.get("note", ""))+ elif kind == "control_request":+ if not self.valid(command, owner=False):+ continue+ self._takeover_viewer = viewer.id+ self._takeover = asyncio.create_task(self._claim(command))+ try:+ await self._takeover+ except asyncio.CancelledError:+ viewer.send_json({"type": "control", "granted": False, "reason": "viewer_unavailable"})+ if self._closed:+ return+ finally:+ self._takeover = None+ self._takeover_viewer = None+ elif self.valid(command):+ if hasattr(self.hub, "runtime") and hasattr(self.hub.runtime, "page_session"):+ await self._ensure_input_session()+ self.controller.touch_user_input()+ if kind in {"mouse", "wheel", "key", "insert_text"}:+ await self.hub.input(viewer, message)+ elif kind in {"navigate", "tab"}:+ await self._release_pressed(viewer.id)+ await getattr(self.hub, kind)(viewer, message, command.session.policy)+ elif kind == "copy":+ await self.hub.copy(viewer)+ elif kind == "dialog_reply":+ await self.hub.dialog_reply(viewer, message)+ except Exception as exc:+ self._error(viewer, "control", str(exc)[:300])+ if kind == "control_request":+ viewer.send_json({"type": "control", "granted": False, "reason": "control_failed", "message": str(exc)[:300]})+ finally:+ if kind == "control_request":+ self._takeover_pending.discard(viewer.id)+ self._current = None++ async def _claim(self, command):+ viewer = command.viewer+ await self.controller.acquire_lease(command.session, wait=2.0)+ if not self.valid(command, owner=False):+ return+ granted, reason = await self.controller.take_over(+ viewer.id, force=bool(command.message.get("force")),+ valid=lambda: self.valid(command, owner=False),+ )+ viewer.send_json({"type": "control", "granted": granted, "reason": reason})+ if granted:+ self._blocked.discard(viewer.id)++ async def close(self):+ self._closed = True+ if self._takeover is not None:+ self._takeover.cancel()+ self._queue.clear()+ self._bytes = 0+ if self._retiring:+ # Chrome's close notification can run inside shutdown(). Waiting+ # for the worker here would wait on our own shutdown barrier.+ return+ # Let already-dispatched CDP operations finish; their normal protocol+ # timeout still bounds this wait. Do not tell agents input has settled+ # merely because a Python task was cancelled.+ if self._worker is not None and self._worker is not asyncio.current_task():+ await asyncio.gather(self._worker, return_exceptions=True)+ if self._input_session is not None:+ await self._input_session.detach()+ self._input_session = None++ async def idle_without_viewers(self):+ """Retain the reusable hub, but no command task/session after disposal."""+ if self._worker is not None and self._worker is not asyncio.current_task():+ await asyncio.gather(self._worker, return_exceptions=True)+ async with self._lock:+ if getattr(self.hub, "viewers", None):+ return+ for viewer_id in {identity[0] for identity in self._pressed}:+ await self._release_pressed(viewer_id)+ if self._input_session is not None:+ await self._input_session.detach()+ self._input_session = None+ self._input_target = self._input_epoch = None+ self._epochs.clear()+ self._blocked.clear()diff --git a/server/nodes/browser/_session.py b/server/nodes/browser/_session.pyindex a5563999..f1ce6d27 100644--- a/server/nodes/browser/_session.py+++ b/server/nodes/browser/_session.py@@ -268,10 +268,14 @@ class ProfileController:async def release_lease(self, session_id: Optional[str] = None) -> None:if session_id is None or (self.lease_session and self.lease_session.session_id == session_id):+ barrier = getattr(self, "live_control_barrier", None)+ if barrier is not None:+ await barrier(self.controller_viewer)if self.pending is not None:self._resolve_pending("cancelled", "")self.lease_session = Noneif self.state != ControlState.IDLE:+ self._cancel_viewer_drop(self.controller_viewer)self.controller_viewer = Noneawait self._set_state(ControlState.IDLE, reason="released")async with self._changed:@@ -343,10 +347,10 @@ class ProfileController:if request.future.done():return request.future.result()if time.monotonic() >= request.deadline:- self._resolve_pending("timeout", "")if self.state in (ControlState.AWAITING_USER, ControlState.USER):- self.controller_viewer = None- await self._set_state(ControlState.IDLE, reason="request timed out")+ await self.hand_back(self.controller_viewer, outcome="timeout", note="request timed out")+ else:+ self._resolve_pending("timeout", "")return {"status": "timeout", "note": ""}return {"status": "still_waiting", "note": ""}@@ -360,11 +364,17 @@ class ProfileController:def _user_blocks_agent(self) -> bool:return self._user_claim or self.state in (ControlState.USER, ControlState.AWAITING_USER)- async def take_over(self, viewer_id: str, *, force: bool = False) -> tuple[bool, str]:+ async def take_over(self, viewer_id: str, *, force: bool = False, valid: Optional[Callable[[], bool]] = None) -> tuple[bool, str]:+ if valid is not None and not valid():+ return False, "viewer_unavailable"if self.controller_viewer and self.controller_viewer != viewer_id and not force:return False, "held_by_other"if self.lease_session is None:return False, "no_session"+ if self.controller_viewer and self.controller_viewer != viewer_id:+ barrier = getattr(self, "live_control_barrier", None)+ if barrier is not None:+ await barrier(self.controller_viewer)# Claim first so no new agent step starts while we wait for this one.self._user_claim = Truetry:@@ -380,6 +390,8 @@ class ProfileController:except asyncio.TimeoutError:return False, "agent_busy"try:+ if valid is not None and not valid():+ return False, "viewer_unavailable"previous_controller = self.controller_viewerself.controller_viewer = viewer_idself.user_input_deadline = time.monotonic() + USER_IDLE_HANDBACK_SECONDS@@ -398,9 +410,17 @@ class ProfileController:async def hand_back(self, viewer_id: Optional[str], *, outcome: str = "handed_back", note: str = "") -> bool:if self.state not in (ControlState.USER, ControlState.AWAITING_USER):return False+ if self.state == ControlState.USER and viewer_id is not None and viewer_id != self.controller_viewer:+ return False+ barrier = getattr(self, "live_control_barrier", None)+ if barrier is not None:+ await barrier(self.controller_viewer)+ # Another transition may have completed while the dispatched command+ # settled. Never release a newer viewer's control.if self.state == ControlState.USER and viewer_id is not None and viewer_id != self.controller_viewer:return Falseself._resolve_pending(outcome, note)+ self._cancel_viewer_drop(self.controller_viewer)self.controller_viewer = Noneself.lease_renewed = self.last_activity = time.monotonic()await self._set_state(ControlState.IDLE, reason=outcome)@@ -433,6 +453,11 @@ class ProfileController:self._viewer_drop_tasks[viewer_id] = asyncio.create_task(_drop())+ def _cancel_viewer_drop(self, viewer_id: Optional[str]) -> None:+ task = self._viewer_drop_tasks.pop(viewer_id, None)+ if task is not None and task is not asyncio.current_task():+ task.cancel()+async def tick(self) -> None:"""Periodic housekeeping (the fleet reaper calls this)."""now = time.monotonic()@@ -440,10 +465,10 @@ class ProfileController:await self.hand_back(self.controller_viewer, outcome="handed_back", note="auto: no input for 10 minutes")request = self.pendingif request is not None and now >= request.deadline and not request.future.done():- self._resolve_pending("timeout", "")if self.state in (ControlState.AWAITING_USER, ControlState.USER):- self.controller_viewer = None- await self._set_state(ControlState.IDLE, reason="request timed out")+ await self.hand_back(self.controller_viewer, outcome="timeout", note="request timed out")+ else:+ self._resolve_pending("timeout", "")def idle_for(self) -> float:busy = self.state != ControlState.IDLE or self.viewers or self.pending is not Nonediff --git a/server/nodes/browser/_stream.py b/server/nodes/browser/_stream.pyindex 0ea90021..4ca40f8c 100644--- a/server/nodes/browser/_stream.py+++ b/server/nodes/browser/_stream.py@@ -18,7 +18,7 @@ the page the agent is on and runs ``Page.startScreencast`` only while someviewer is visible. Each viewer has a window of two unacknowledged frames andone "latest frame" slot that newer frames overwrite, so a slow viewer dropsframes instead of stalling Chrome or the others; Chrome's own acknowledgement-goes out after the first viewer's send completes (or 250 ms). Input reaches+goes out as soon as the frame enters those bounded slots. Input reachesthe page only from the viewer that holds control, in the USER state."""@@ -27,6 +27,7 @@ from __future__ import annotationsimport asyncioimport base64import json+import itertoolsimport structimport timeimport uuid@@ -38,13 +39,13 @@ from core.logging import get_loggerfrom ._cdp import CDPDisconnected, CDPError, CDPSessionfrom ._chrome import VIEWPORT_HEIGHT, VIEWPORT_WIDTH+from ._stream_metrics import StreamMetricslogger = get_logger(__name__)FRAME_VERSION = 1FRAME_KIND_JPEG = 1_WINDOW = 2-_ACK_AFTER = 0.25_MAX_FRAME_EDGE = 1920_CONTROL_QUEUE = 256_INPUT_RATE = 200.0 # messages per second@@ -53,6 +54,10 @@ _TEXT_LIMIT = 10_000_CLIPBOARD_LIMIT = 100_000_ATTACH_TIMEOUT = 5.0_IDLE_POLL = 2.0+_SCREENCAST_RETRY_DELAYS = (1.0, 2.0, 4.0)+# Viewers can outlive a stopped profile and attach to a replacement hub on the+# same socket. Never reuse sequence IDs while old frame ACKs are in transit.+_FRAME_SEQUENCES = itertools.count(1)CLOSE_UNAUTHENTICATED = 4001CLOSE_ATTACH = 4002@@ -101,6 +106,11 @@ class Viewer:self.wakeup = asyncio.Event()self.bucket = _Bucket(_INPUT_RATE, _INPUT_BURST)self.closed = False+ self.metrics = StreamMetrics()+ self._pending_at = 0.0+ self._outstanding: Dict[int, float] = {}+ self._started_at = time.monotonic()+ self._first_frame = Truedef send_json(self, message: Dict[str, Any]) -> None:if self.closed:@@ -112,27 +122,65 @@ class Viewer:self.wakeup.set()def offer_frame(self, frame: bytes) -> bool:- """Queue a frame; ``False`` when it was dropped (throttled)."""+ """Keep the latest frame, including the final update inside an FPS interval."""if self.closed or not self.visible:return False- if self.max_fps and time.monotonic() - self.last_frame_at < 1.0 / self.max_fps:- return False+ if self.pending is not None:+ self.metrics.count("replaced")self.pending = frame+ self._pending_at = time.monotonic()self.wakeup.set()return True+ def acknowledge(self, seq: Any = None) -> None:+ # A corrupt envelope cannot yield a sequence. Its ACK consumes the oldest+ # outstanding frame; well-formed duplicate/stale ACKs never grant credit.+ if seq is None:+ seq = next(iter(self._outstanding), None)+ if not isinstance(seq, int):+ return+ sent = self._outstanding.pop(seq, None)+ if sent is not None:+ self.inflight = len(self._outstanding)+ self.metrics.observe("viewer_ack", time.monotonic() - sent)+ self.wakeup.set()+async def writer(self, on_sent: Any) -> None:while not self.closed:- await self.wakeup.wait()self.wakeup.clear()while not self.control.empty():await self.websocket.send_text(json.dumps(self.control.get_nowait(), default=str))- if self.pending is not None and self.inflight < _WINDOW:- frame, self.pending = self.pending, None- self.inflight += 1- self.last_frame_at = time.monotonic()- await self.websocket.send_bytes(frame)- on_sent(self)+ delay = None+ if self.visible and self.pending is not None and self.inflight < _WINDOW:+ delay = max(0.0, self.last_frame_at + (1.0 / self.max_fps if self.max_fps else 0) - time.monotonic())+ if delay == 0:+ frame, self.pending = self.pending, None+ head_end = 4 + struct.unpack_from(">H", frame, 2)[0]+ header = json.loads(frame[4:head_end])+ seq = header["seq"]+ self.last_frame_at = time.monotonic()+ self._outstanding[seq] = self.last_frame_at+ self.inflight = len(self._outstanding)+ self.metrics.observe("frame_queue", self.last_frame_at - self._pending_at)+ captured_at = header.get("ts")+ if isinstance(captured_at, (int, float)):+ self.metrics.observe("capture_to_send", time.time() - captured_at)+ await self.websocket.send_bytes(frame)+ self.metrics.observe("socket_send", time.monotonic() - self.last_frame_at)+ if self._first_frame:+ self._first_frame = False+ self.metrics.observe("first_frame", time.monotonic() - self._started_at)+ on_sent(self)+ continue+ if not self.visible:+ self.pending = None+ if delay is None:+ await self.wakeup.wait()+ else:+ try:+ await asyncio.wait_for(self.wakeup.wait(), timeout=delay)+ except asyncio.TimeoutError:+ passclass ScreencastHub:@@ -148,12 +196,18 @@ class ScreencastHub:self._active = Falseself._size = (0, 0)self._resize_task: Optional[asyncio.Task] = None- self._pending_ack: Optional[int] = None- self._ack_timer: Optional[asyncio.TimerHandle] = None+ self._retry_task: Optional[asyncio.Task] = None+ self._pending_acks: List[tuple[CDPSession, int]] = []+ self._ack_tasks: set[asyncio.Task] = set()+ self._capture_generation = 0+ self.metrics = StreamMetrics()self._unsubscribers: List[Any] = []self._lock = asyncio.Lock()self.dead = Falseself._unlisten = self.controller.add_listener(self._on_controller)+ from ._live_control import LiveControlQueue++ self.commands = LiveControlQueue(self)# -- viewers -------------------------------------------------------------------@@ -163,8 +217,15 @@ class ScreencastHub:await self._refresh()async def remove(self, viewer: Viewer) -> None:+ self.commands.release(viewer, note="the owner closed the live view")self.viewers.pop(viewer.id, None)self.controller.viewer_detached(viewer.id)+ if not self.viewers:+ await self.commands.idle_without_viewers()+ pending = [task for task in (self._resize_task, self._retry_task) if task is not None and task is not asyncio.current_task()]+ for task in pending:+ task.cancel()+ await asyncio.gather(*pending, return_exceptions=True)await self._refresh()def visible_viewers(self) -> List[Viewer]:@@ -192,6 +253,8 @@ class ScreencastHub:return {"type": "tabs", "tabs": [dict(t, active=t.get("target_id") == active) for t in self.controller.tabs.values()]}async def _on_controller(self, kind: str, payload: Dict[str, Any]) -> None:+ if kind in ("tabs", "page"):+ self.commands.target_changed()if kind in ("state", "request", "control_moved"):for viewer in list(self.viewers.values()):viewer.send_json(self.state_message(viewer))@@ -207,6 +270,7 @@ class ScreencastHub:await self._follow_target()elif kind == "closed":self.dead = True+ await self.commands.close()self._unlisten()self.broadcast({"type": "idle", "reason": payload.get("reason")})await self._stop_screencast()@@ -224,11 +288,38 @@ class ScreencastHub:async def _refresh(self) -> None:async with self._lock:size = self._wanted_size()- if size == (0, 0):+ if self.dead or size == (0, 0):+ self._cancel_retry()await self._stop_screencast_locked()returnif not self._active or size != self._size or self.target_id != self.controller.active_target_id:await self._start_screencast_locked(size)+ if not self._active and self.runtime.running:+ self._schedule_retry()++ def _cancel_retry(self) -> None:+ if self._retry_task is not None and self._retry_task is not asyncio.current_task():+ self._retry_task.cancel()+ self._retry_task = None++ def _schedule_retry(self) -> None:+ if self._retry_task is not None and not self._retry_task.done():+ return++ async def retry() -> None:+ for delay in _SCREENCAST_RETRY_DELAYS:+ await asyncio.sleep(delay)+ if self.dead or not self.visible_viewers() or not self.runtime.running or self._active:+ return+ await self._refresh()+ if self._active:+ return+ self.broadcast({"type": "error", "code": "screencast", "message": "The browser live view could not start. Reconnect the view to try again.", "retrying": False})++ self._retry_task = asyncio.create_task(retry(), name="browser-screencast-retry")++ def _stream_error(self, exc: Exception) -> None:+ self.broadcast({"type": "error", "code": "screencast", "message": f"The browser live view could not start: {str(exc)[:200]}", "retrying": True})def schedule_resize(self) -> None:if self._resize_task is not None and not self._resize_task.done():@@ -252,10 +343,12 @@ class ScreencastHub:session = await self.runtime.page_session(self.controller.active_target_id)except (CDPError, CDPDisconnected, TimeoutError) as exc:logger.debug("[browser] live view could not attach: %s", exc)+ self._stream_error(exc)returnself.session, self.target_id = session, session.target_id+ generation = self._capture_generationself._unsubscribers = [- session.on("Page.screencastFrame", self._on_frame),+ session.on("Page.screencastFrame", lambda params: self._on_frame(params, session, generation)),session.on("Page.javascriptDialogOpening", self._on_dialog),]try:@@ -263,12 +356,13 @@ class ScreencastHub:await session.send("Emulation.setFocusEmulationEnabled", {"enabled": True}, timeout=10)await session.send("Page.startScreencast",- {"format": "jpeg", "quality": 60, "maxWidth": size[0], "maxHeight": size[1], "everyNthFrame": 1, "maxFramesInFlight": _WINDOW},+ {"format": "jpeg", "quality": 60, "maxWidth": size[0], "maxHeight": size[1], "everyNthFrame": 1, "maxFramesInFlight": _WINDOW, "sendLastFrame": True},timeout=10,)except (CDPError, CDPDisconnected, TimeoutError) as exc:logger.debug("[browser] startScreencast failed: %s", exc)await self._stop_screencast_locked()+ self._stream_error(exc)returnself._active, self._size = True, sizeself.broadcast({"type": "page", "target_id": self.target_id, **self._tab_facts(self.target_id)})@@ -279,9 +373,19 @@ class ScreencastHub:async def _stop_screencast(self) -> None:async with self._lock:+ self._cancel_retry()await self._stop_screencast_locked()async def _stop_screencast_locked(self) -> None:+ self._capture_generation += 1+ self._pending_acks.clear()+ ack_tasks = list(self._ack_tasks)+ for task in ack_tasks:+ task.cancel()+ if ack_tasks:+ await asyncio.gather(*ack_tasks, return_exceptions=True)+ for viewer in self.viewers.values():+ viewer.pending = Nonefor unsubscribe in self._unsubscribers:unsubscribe()self._unsubscribers = []@@ -294,12 +398,24 @@ class ScreencastHub:passawait session.detach()- def _on_frame(self, params: Dict[str, Any]) -> None:- self.seq += 1+ def _on_frame(self, params: Dict[str, Any], source: Optional[CDPSession] = None, generation: Optional[int] = None) -> None:+ if generation is not None and generation != self._capture_generation:+ return+ source = source or self.session+ ack_id = params.get("sessionId")+ if source is not None and isinstance(ack_id, int):+ self._pending_acks.append((source, ack_id))+ started = time.monotonic()+ self.metrics.count("captured")+ if not self.visible_viewers():+ self._ack_now()+ return+ self.seq = next(_FRAME_SEQUENCES)meta = params.get("metadata") or {}try:- jpeg = base64.b64decode(params.get("data") or "")- except ValueError:+ jpeg = base64.b64decode(params.get("data") or "", validate=True)+ except (ValueError, TypeError):+ self._ack_now()returnheader = {"seq": self.seq,@@ -313,28 +429,38 @@ class ScreencastHub:"ts": meta.get("timestamp"),}frame = encode_frame(header, jpeg)- offered = False+ self.metrics.observe("frame_pack", time.monotonic() - started)for viewer in self.visible_viewers():- offered = viewer.offer_frame(frame) or offered- self._pending_ack = params.get("sessionId")- if not offered:- self._ack_now()- elif self._ack_timer is None:- self._ack_timer = asyncio.get_running_loop().call_later(_ACK_AFTER, self._ack_now)+ viewer.offer_frame(frame)+ # Viewer backpressure is already bounded independently. Holding Chrome's+ # capture credit for a slow decoder causes it to skip the last update.+ self._ack_now()def frame_sent(self, viewer: Viewer) -> None:- self._ack_now()+ self.metrics.count("sent")def _ack_now(self) -> None:- if self._ack_timer is not None:- self._ack_timer.cancel()- self._ack_timer = None- session_id, self._pending_ack = self._pending_ack, None- session = self.session- if session_id is None or session is None:+ debt, self._pending_acks = self._pending_acks, []+ if not debt:return- task = asyncio.ensure_future(session.send("Page.screencastFrameAck", {"sessionId": session_id}, timeout=5))- task.add_done_callback(lambda t: t.cancelled() or t.exception())+ generation = self._capture_generation++ async def flush() -> None:+ try:+ for session, session_id in debt:+ if generation != self._capture_generation:+ return+ await session.send("Page.screencastFrameAck", {"sessionId": session_id}, timeout=5)+ except (CDPError, CDPDisconnected, TimeoutError):+ # Delivery may have succeeded. Never retry an ambiguous ACK and+ # over-credit Chrome; recover with a fresh capture generation.+ if generation == self._capture_generation and not self.dead:+ self._active = False+ self._schedule_retry()++ task = asyncio.create_task(flush(), name="browser-frame-ack")+ self._ack_tasks.add(task)+ task.add_done_callback(self._ack_tasks.discard)def _on_dialog(self, params: Dict[str, Any]) -> None:self.broadcast({"type": "dialog", "kind": params.get("type"), "message": str(params.get("message") or "")[:2000]})@@ -349,10 +475,11 @@ class ScreencastHub:return max(0.0, min(fx, float(VIEWPORT_WIDTH * 4))), max(0.0, min(fy, float(VIEWPORT_HEIGHT * 20)))async def input(self, viewer: Viewer, message: Dict[str, Any]) -> None:- if not self.controller.can_inject_input(viewer.id) or self.session is None:+ session = self.commands.input_session(viewer)+ if not self.controller.can_inject_input(viewer.id) or session is None:returnself.controller.touch_user_input()- session, kind = self.session, message.get("type")+ kind = message.get("type")modifiers = int(message.get("modifiers") or 0) & 15try:if kind == "mouse":@@ -408,7 +535,8 @@ class ScreencastHub:async def navigate(self, viewer: Viewer, message: Dict[str, Any], policy: Any) -> None:from ._netpolicy import url_block_reason- if not self.controller.can_inject_input(viewer.id) or self.session is None:+ session = self.commands.input_session(viewer)+ if not self.controller.can_inject_input(viewer.id) or session is None:returnaction = str(message.get("action") or "")try:@@ -420,15 +548,15 @@ class ScreencastHub:if reason:viewer.send_json({"type": "error", "code": "blocked", "message": f"Cannot open {url}: {reason}."})return- await self.session.send("Page.navigate", {"url": url}, timeout=30)+ await session.send("Page.navigate", {"url": url}, timeout=30)elif action == "reload":- await self.session.send("Page.reload", timeout=30)+ await session.send("Page.reload", timeout=30)elif action in ("back", "forward"):- history = await self.session.send("Page.getNavigationHistory", timeout=10)+ history = await session.send("Page.getNavigationHistory", timeout=10)index = history["currentIndex"] + (-1 if action == "back" else 1)entries = history.get("entries") or []if 0 <= index < len(entries):- await self.session.send("Page.navigateToHistoryEntry", {"entryId": entries[index]["id"]}, timeout=30)+ await session.send("Page.navigateToHistoryEntry", {"entryId": entries[index]["id"]}, timeout=30)except (CDPError, CDPDisconnected, TimeoutError) as exc:viewer.send_json({"type": "error", "code": "navigation", "message": str(exc)[:300]})@@ -436,11 +564,12 @@ class ScreencastHub:if not self.controller.can_inject_input(viewer.id):returnaction, target = str(message.get("action") or ""), str(message.get("target_id") or "")- cdp = self.runtime.cdp+ cdp = self.commands.guarded_session(viewer, self.runtime.cdp)try:if action == "activate" and target in self.controller.tabs:await cdp.send("Target.activateTarget", {"targetId": target}, timeout=10)self.controller.active_target_id = target+ self.commands.target_changed()await self._refresh()self.broadcast(self.tabs_message())elif action == "close" and target in self.controller.tabs and len(self.controller.tabs) > 1:@@ -448,26 +577,29 @@ class ScreencastHub:elif action == "new":created = await cdp.send("Target.createTarget", {"url": "about:blank"}, timeout=10)self.controller.active_target_id = created.get("targetId")+ self.commands.target_changed()await self._refresh()except (CDPError, CDPDisconnected, TimeoutError) as exc:viewer.send_json({"type": "error", "code": "tab", "message": str(exc)[:300]})async def dialog_reply(self, viewer: Viewer, message: Dict[str, Any]) -> None:- if not self.controller.can_inject_input(viewer.id) or self.session is None:+ session = self.commands.input_session(viewer)+ if not self.controller.can_inject_input(viewer.id) or session is None:returnparams: Dict[str, Any] = {"accept": bool(message.get("accept"))}if message.get("prompt_text") is not None:params["promptText"] = str(message["prompt_text"])[:_TEXT_LIMIT]try:- await self.session.send("Page.handleJavaScriptDialog", params, timeout=10)+ await session.send("Page.handleJavaScriptDialog", params, timeout=10)except (CDPError, CDPDisconnected, TimeoutError):passasync def copy(self, viewer: Viewer) -> None:- if not self.controller.can_inject_input(viewer.id) or self.session is None:+ session = self.commands.input_session(viewer)+ if not self.controller.can_inject_input(viewer.id) or session is None:returntry:- result = await self.session.send(+ result = await session.send("Runtime.evaluate", {"expression": "String(window.getSelection ? window.getSelection() : '')", "returnByValue": True}, timeout=5)except (CDPError, CDPDisconnected, TimeoutError):@@ -591,8 +723,7 @@ async def browser_live_view(websocket: WebSocket) -> None:if kind == "ping":viewer.send_json({"type": "pong"})elif kind == "ack":- viewer.inflight = max(0, viewer.inflight - 1)- viewer.wakeup.set()+ viewer.acknowledge(message.get("seq"))elif kind == "viewport":_apply_viewport(viewer, message)if hub is not None:@@ -600,30 +731,18 @@ async def browser_live_view(websocket: WebSocket) -> None:elif kind == "visibility":viewer.visible = bool(message.get("visible"))if hub is not None:- await hub._refresh()+ if not viewer.visible:+ hub.commands.release(viewer, note="the live view was hidden")+ hub.schedule_resize()elif hub is None:viewer.send_json({"type": "idle", "reason": "not running"})elif kind == "control_request":- try:- await hub.controller.acquire_lease(session, wait=2.0)- except NodeUserError as exc:- viewer.send_json({"type": "control", "granted": False, "reason": "held_by_other", "message": str(exc)[:300]})- continue- granted, reason = await hub.controller.take_over(viewer.id, force=bool(message.get("force")))- viewer.send_json({"type": "control", "granted": granted, "reason": reason})+ hub.commands.enqueue(viewer, message, session)elif kind == "control_release":outcome = "declined" if message.get("outcome") == "declined" else "handed_back"- await hub.controller.hand_back(viewer.id, outcome=outcome, note=str(message.get("note") or "")[:1000])- elif kind in ("mouse", "wheel", "key", "insert_text"):- await hub.input(viewer, message)- elif kind == "navigate":- await hub.navigate(viewer, message, session.policy)- elif kind == "tab":- await hub.tab(viewer, message, session.policy)- elif kind == "dialog_reply":- await hub.dialog_reply(viewer, message)- elif kind == "copy":- await hub.copy(viewer)+ hub.commands.release(viewer, outcome=outcome, note=str(message.get("note") or "")[:1000])+ elif kind in ("mouse", "wheel", "key", "insert_text", "navigate", "tab", "dialog_reply", "copy"):+ hub.commands.enqueue(viewer, message, session)try:await reader()@@ -635,6 +754,7 @@ async def browser_live_view(websocket: WebSocket) -> None:viewer.closed = Trueviewer.wakeup.set()writer.cancel()+ await asyncio.gather(writer, return_exceptions=True)if hub is not None:await hub.remove(viewer)elif controller is not None:diff --git a/server/nodes/browser/_stream_metrics.py b/server/nodes/browser/_stream_metrics.pynew file mode 100644index 00000000..ef29fa34--- /dev/null+++ b/server/nodes/browser/_stream_metrics.py@@ -0,0 +1,43 @@+"""Opt-in, bounded live-view diagnostics. Never accepts page or input payloads."""++from __future__ import annotations++import os+import time+from collections import Counter, defaultdict, deque++from core.logging import get_logger++logger = get_logger(__name__)+++class StreamMetrics:+ def __init__(self) -> None:+ self.enabled = os.environ.get("OPENCOMPANY_BROWSER_DIAGNOSTICS") == "1"+ self.samples: dict[str, deque[float]] = defaultdict(lambda: deque(maxlen=256))+ self.counts: Counter[str] = Counter()+ self._reported = time.monotonic()++ def observe(self, name: str, seconds: float) -> None:+ if self.enabled:+ self.samples[name].append(max(0.0, seconds) * 1000)+ self._report()++ def count(self, name: str) -> None:+ if self.enabled:+ self.counts[name] += 1+ self._report()++ def snapshot(self) -> dict:+ timings = {}+ for name, samples in self.samples.items():+ values = sorted(samples)+ if values:+ timings[name] = {"n": len(values), "p50_ms": round(values[(len(values) - 1) // 2], 2), "p95_ms": round(values[min(len(values) - 1, int(len(values) * .95))], 2)}+ return {"timings": timings, "counts": dict(self.counts)}++ def _report(self) -> None:+ now = time.monotonic()+ if now - self._reported >= 5:+ self._reported = now+ logger.info("[browser] live-view timings %s", self.snapshot())diff --git a/server/scripts/benchmark_browser_stream.py b/server/scripts/benchmark_browser_stream.pynew file mode 100644index 00000000..7b10b2e9--- /dev/null+++ b/server/scripts/benchmark_browser_stream.py@@ -0,0 +1,378 @@+"""Real local Chrome → production stream → reference canvas latency benchmark.++No download, real user profile, application database, or external website is used.+Authentication/session lookup are replaced inside this benchmark process only.+The measured viewer is a minimal reference decoder, NOT the React application.+Run from server/: .venv/Scripts/python scripts/benchmark_browser_stream.py --help+"""++from __future__ import annotations++import argparse+import asyncio+import contextlib+import hashlib+import importlib.util+import json+import math+import os+from pathlib import Path+import shutil+import socket+import subprocess+import sys+import tempfile+import time+from types import ModuleType, SimpleNamespace++SERVER = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(SERVER))++PAGE = """<!doctype html><meta charset=utf-8><style>+html,body{margin:0;width:100%;height:100%;overflow:hidden;background:#888}canvas{display:block}+</style><canvas id=p width=1280 height=800></canvas><script>+const ctx=p.getContext('2d');let value=0;+function paint(){ctx.fillStyle='#888';ctx.fillRect(0,0,1280,800);+for(let i=0;i<16;i++){ctx.fillStyle=(value>>i)&1?'#fff':'#000';ctx.fillRect(i*32,0,32,32)}+ctx.fillStyle='#111';ctx.font='40px sans-serif';ctx.fillText('Local stream benchmark '+value,80,150)}+addEventListener('mousedown',()=>{value++;paint()});paint();+</script>"""++VIEWER = """<!doctype html><meta charset=utf-8><canvas id=p width=1280 height=800></canvas><script>+const cfg=__CONFIG__,ctx=p.getContext('2d'),metrics={latencies:[],frameAge:[],decodeDraw:[],paintWait:[],misses:[],observed:[],errors:[],visibility:document.visibilityState};+window.benchmark={ready:false,done:false,metrics};let count=0,pending=new Map(),chain=Promise.resolve();+const ws=new WebSocket('ws://'+location.host+'/ws/browser');ws.binaryType='arraybuffer';+const send=m=>ws.send(JSON.stringify(m));+const pause=ms=>new Promise(r=>setTimeout(r,ms));+const paintTurn=()=>new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));+ws.onopen=()=>send({type:'attach',target:{kind:'node',workflow_id:'benchmark',node_id:'browser'},+viewport:{width:1280,height:800,dpr:1},visible:true,max_fps:cfg.fps});+ws.onmessage=e=>{if(typeof e.data==='string'){const m=JSON.parse(e.data);+if(m.type==='attached')send({type:'control_request'});+if(m.type==='state'&&m.controller==='you')window.benchmark.ready=true;+if(m.type==='error')metrics.errors.push(m);return}+chain=chain.then(async()=>{let seq;try{const decodeStarted=performance.now(),v=new DataView(e.data),end=4+v.getUint16(2,false);+const h=JSON.parse(new TextDecoder().decode(new Uint8Array(e.data,4,end-4)));seq=h.seq;+const image=await createImageBitmap(new Blob([e.data.slice(end)],{type:'image/jpeg'}));+ctx.drawImage(image,0,0,1280,800);image.close();let code=0;+for(let i=0;i<16;i++)if(ctx.getImageData(i*32+16,16,1,1).data[0]>128)code|=1<<i;+const drawn=performance.now();await paintTurn();const now=performance.now();+metrics.decodeDraw.push(drawn-decodeStarted);metrics.paintWait.push(now-drawn);+metrics.observed.push(code);+if(h.ts)metrics.frameAge.push(Date.now()-h.ts*1000);+const sample=pending.get(code);if(sample){metrics.latencies.push(now-sample.started);pending.delete(code);sample.resolve()}+}catch(error){metrics.errors.push(String(error))}finally{if(ws.readyState===1)send({type:'ack',seq})}})};+ws.onerror=()=>metrics.errors.push('socket error');+window.runBenchmark=async()=>{for(let i=0;i<cfg.samples+cfg.warmup;i++){+await pause(cfg.interval_ms+(i%7));count++;+let resolve;const seen=new Promise(r=>resolve=r);pending.set(count,{started:performance.now(),resolve});+send({type:'mouse',action:'down',x:600,y:400,button:'left',buttons:1,click_count:1});+send({type:'mouse',action:'up',x:600,y:400,button:'left',buttons:0,click_count:1});+await Promise.race([seen,pause(cfg.deadline_ms)]);+if(pending.has(count)){metrics.misses.push(count);pending.delete(count)}+if(i===cfg.warmup-1){metrics.latencies=[];metrics.frameAge=[];metrics.decodeDraw=[];metrics.paintWait=[];metrics.misses=[];metrics.observed=[]}}+send({type:'control_release'});window.benchmark.done=true;return metrics};+</script>"""+++class BenchController:+ """Identity/control fixture; production stream reader and CDP input are unchanged."""++ def __init__(self, target: str, url: str):+ self.active_target_id = target+ self.tabs = {target: {"target_id": target, "url": url, "title": "Local benchmark"}}+ self.controller_viewer = None+ self.listeners = []++ def add_listener(self, listener):+ self.listeners.append(listener)+ return lambda: self.listeners.remove(listener) if listener in self.listeners else None++ def viewer_attached(self, _viewer):+ pass++ def viewer_detached(self, viewer):+ if viewer == self.controller_viewer:+ self.controller_viewer = None++ def snapshot(self, viewer=None):+ return {+ "state": "user" if self.controller_viewer else "idle",+ "controller": "you" if viewer == self.controller_viewer and viewer else None,+ "request": None,+ "profile": {"id": "benchmark", "name": "Benchmark"},+ }++ async def acquire_lease(self, _session, **_kwargs):+ pass++ async def take_over(self, viewer, **_kwargs):+ self.controller_viewer = viewer+ for listener in self.listeners:+ await listener("state", {})+ return True, "granted"++ async def hand_back(self, _viewer, **_kwargs):+ self.controller_viewer = None++ def can_inject_input(self, viewer):+ return viewer == self.controller_viewer++ def touch_user_input(self):+ pass+++def chrome_path(value: str | None) -> Path:+ candidates = (+ [value]+ if value+ else [+ shutil.which("google-chrome"),+ shutil.which("chromium"),+ shutil.which("chromium-browser"),+ r"C:\Program Files\Google\Chrome\Application\chrome.exe",+ str(Path(os.environ.get("LOCALAPPDATA", "")) / "Google/Chrome/Application/chrome.exe"),+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",+ ]+ )+ for candidate in candidates:+ if candidate and Path(candidate).is_file():+ return Path(candidate).resolve()+ raise SystemExit("No installed Chrome found. Supply --chrome; this benchmark never downloads a browser.")+++def summarize(values):+ if not values:+ return {"count": 0, "p50_ms": None, "p95_ms": None}+ ordered = sorted(values)+ return {+ "count": len(values),+ "p50_ms": round(ordered[math.ceil(len(values) * 0.5) - 1], 2),+ "p95_ms": round(ordered[math.ceil(len(values) * 0.95) - 1], 2),+ "max_ms": round(ordered[-1], 2),+ }+++async def run(args, scratch: Path):+ os.environ["DATA_DIR"] = str(scratch / "data")+ from fastapi import FastAPI+ from fastapi.responses import HTMLResponse+ import uvicorn++ # Load only the two measured modules, avoiding the app's eager plugin+ # discovery and all user settings/database initialization. These fixtures+ # exist solely in this standalone process and never replace project files.+ def fixture(name, **values):+ module = ModuleType(name)+ module.__dict__.update(values)+ sys.modules[name] = module+ return module++ fixture("nodes", __path__=[str(SERVER / "nodes")])+ fixture("nodes.browser", __path__=[str(SERVER / "nodes/browser")])+ fixture("nodes.browser._chrome", VIEWPORT_WIDTH=1280, VIEWPORT_HEIGHT=800)+ runtime_module = fixture("nodes.browser._runtime")+ auth_module = fixture("services.authz")+ fixture("services.plugin", __path__=[str(SERVER / "services/plugin")])+ fixture("services.plugin.base", NodeUserError=type("NodeUserError", (Exception,), {}))+ fixture("core.container", container=SimpleNamespace(settings=lambda: None, user_auth_service=None))+ from nodes.browser._cdp import CDPConnection, read_devtools_active_port++ source = (args.stream_source or SERVER / "nodes/browser/_stream.py").resolve()+ source_bytes = source.read_bytes()+ spec = importlib.util.spec_from_file_location("nodes.browser._benchmark_stream", source)+ stream = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = stream+ exec(compile(source_bytes, str(source), "exec"), stream.__dict__)+ config = {+ "samples": args.samples,+ "warmup": args.warmup,+ "fps": args.fps,+ "interval_ms": args.interval_ms,+ "deadline_ms": args.deadline_ms,+ }+ app = FastAPI()+ app.add_api_route("/page", lambda: HTMLResponse(PAGE))+ app.add_api_route("/viewer", lambda: HTMLResponse(VIEWER.replace("__CONFIG__", json.dumps(config))))+ app.add_api_websocket_route("/ws/browser", stream.browser_live_view)+ listener = socket.socket()+ listener.bind(("127.0.0.1", 0))+ port = listener.getsockname()[1]+ http = uvicorn.Server(uvicorn.Config(app, log_level="error", lifespan="off", timeout_graceful_shutdown=3))+ server_task = asyncio.create_task(http.serve(sockets=[listener]))+ profile = scratch / "chrome-profile"+ profile.mkdir()+ chrome = chrome_path(args.chrome)+ proc = None+ cdp = None+ try:+ while not http.started:+ if server_task.done():+ await server_task+ raise RuntimeError("Benchmark HTTP server did not start")+ await asyncio.sleep(0.02)+ command = [+ str(chrome),+ f"--user-data-dir={profile}",+ "--remote-debugging-address=127.0.0.1",+ "--remote-debugging-port=0",+ "--headless=new",+ "--no-first-run",+ "--no-default-browser-check",+ "--disable-background-networking",+ "--disable-sync",+ "--disable-component-update",+ "--disable-background-timer-throttling",+ "--disable-renderer-backgrounding",+ "--disable-backgrounding-occluded-windows",+ "--window-size=1280,800",+ "--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE localhost, EXCLUDE 127.0.0.1",+ "about:blank",+ ]+ allowed = {"PATH", "SYSTEMROOT", "WINDIR", "TEMP", "TMP", "TMPDIR", "HOME", "USERPROFILE", "DISPLAY", "XDG_RUNTIME_DIR"}+ environment = {key: value for key, value in os.environ.items() if key.upper() in allowed}+ proc = subprocess.Popen(+ command,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ env=environment,+ creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,+ )+ deadline = time.monotonic() + 20+ endpoint = None+ while time.monotonic() < deadline:+ endpoint = read_devtools_active_port(profile)+ if endpoint:+ break+ if proc.poll() is not None:+ raise RuntimeError(f"Isolated Chrome exited with code {proc.returncode}")+ await asyncio.sleep(0.05)+ if not endpoint:+ raise RuntimeError("Isolated Chrome did not expose CDP within 20s")+ cdp = await CDPConnection.connect(f"ws://127.0.0.1:{endpoint[0]}{endpoint[1]}")+ version = await cdp.send("Browser.getVersion")+ target = (await cdp.send("Target.createTarget", {"url": f"http://127.0.0.1:{port}/page"}))["targetId"]+ target_session = await cdp.attach(target)+ await target_session.send(+ "Emulation.setDeviceMetricsOverride", {"width": 1280, "height": 800, "deviceScaleFactor": 1, "mobile": False}+ )+ controller = BenchController(target, f"http://127.0.0.1:{port}/page")++ async def page_session(_target=None):+ return await cdp.attach(_target or target)++ runtime = SimpleNamespace(controller=controller, running=True, page_session=page_session, cdp=cdp)+ facade = SimpleNamespace(running=lambda _profile: runtime, controller=lambda _profile: controller)+ runtime_module.get_browser_runtime = lambda: facade++ async def auth(_websocket, **_kwargs):+ return "benchmark-owner"++ async def attach(_owner, _target):+ return SimpleNamespace(profile_id="benchmark", session_id="benchmark-session", policy=None)++ auth_module.authenticate_ws = auth+ stream._resolve_attach = attach+ viewer_target = (await cdp.send("Target.createTarget", {"url": f"http://127.0.0.1:{port}/viewer"}))["targetId"]+ viewer = await cdp.attach(viewer_target)+ await viewer.send("Emulation.setFocusEmulationEnabled", {"enabled": True})+ deadline = time.monotonic() + 20+ while time.monotonic() < deadline:+ state = await viewer.send("Runtime.evaluate", {"expression": "Boolean(window.benchmark?.ready)", "returnByValue": True})+ if state.get("result", {}).get("value"):+ break+ await asyncio.sleep(0.05)+ else:+ debug = await viewer.send(+ "Runtime.evaluate",+ {+ "expression": "JSON.stringify({url:location.href,benchmark:window.benchmark,body:document.body.innerText})",+ "returnByValue": True,+ },+ )+ raise RuntimeError(f"Viewer did not acquire benchmark control within 20s: {debug}")+ await asyncio.sleep(0.3)+ result = await viewer.send(+ "Runtime.evaluate",+ {"expression": "window.runBenchmark()", "awaitPromise": True, "returnByValue": True},+ timeout=(args.samples + args.warmup) * (args.deadline_ms + args.interval_ms + 20) / 1000 + 20,+ )+ if "exceptionDetails" in result:+ raise RuntimeError(json.dumps(result["exceptionDetails"]))+ raw = result["result"]["value"]+ target_count = await target_session.send("Runtime.evaluate", {"expression": "value", "returnByValue": True})+ latency = summarize(raw["latencies"])+ report = {+ "label": args.label,+ "measurement": "real Chrome CDP + production stream + reference canvas double-rAF paint proxy",+ "limitations": [+ "Not the production React renderer",+ "Authentication and profile/control lookup are fixtures",+ "Local static page; not WAN or third-party site latency",+ "Missed updates are censored and reported separately; percentiles cover observed updates only",+ ],+ "stream_source": str(source),+ "stream_sha256": hashlib.sha256(source_bytes).hexdigest(),+ "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),+ "chrome": version.get("product"),+ "config": config,+ "input_to_paint": latency,+ "frame_age": summarize(raw["frameAge"]),+ "reference_decode_draw": summarize(raw["decodeDraw"]),+ "reference_draw_to_double_raf": summarize(raw["paintWait"]),+ "reference_visibility": raw["visibility"],+ "missed_updates": len(raw["misses"]),+ "target_clicks": target_count.get("result", {}).get("value"),+ "expected_clicks": args.samples + args.warmup,+ "errors": raw["errors"],+ "target_200ms": bool(latency["p95_ms"] is not None and latency["p95_ms"] <= 200 and not raw["misses"]),+ "raw": raw,+ }+ if args.output:+ args.output.write_text(json.dumps(report, indent=2), encoding="utf-8")+ print(json.dumps({key: value for key, value in report.items() if key != "raw"}, indent=2), flush=True)+ finally:+ if cdp:+ with contextlib.suppress(Exception):+ await cdp.send("Browser.close", timeout=5)+ with contextlib.suppress(asyncio.TimeoutError):+ await asyncio.wait_for(cdp.close(), timeout=10)+ if proc and proc.poll() is None:+ try:+ await asyncio.to_thread(proc.wait, timeout=8)+ except subprocess.TimeoutExpired:+ if os.name == "nt":+ await asyncio.to_thread(+ subprocess.run,+ ["taskkill", "/PID", str(proc.pid), "/T", "/F"],+ capture_output=True,+ creationflags=subprocess.CREATE_NO_WINDOW,+ )+ else:+ proc.kill()+ http.should_exit = True+ with contextlib.suppress(asyncio.TimeoutError):+ await asyncio.wait_for(server_task, timeout=10)+ listener.close()+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--chrome", help="Existing Chrome executable; never downloaded")+ parser.add_argument("--stream-source", type=Path, help="Preserved baseline _stream.py; default is current source")+ parser.add_argument("--label", default="current")+ parser.add_argument("--samples", type=int, default=50)+ parser.add_argument("--warmup", type=int, default=5)+ parser.add_argument("--fps", type=float, default=30)+ parser.add_argument("--interval-ms", type=int, default=25)+ parser.add_argument("--deadline-ms", type=int, default=1000)+ parser.add_argument("--output", type=Path, help="JSON report path, preferably in your temporary directory")+ args = parser.parse_args()+ if args.samples < 1 or args.warmup < 0 or args.fps <= 0 or args.deadline_ms < 1 or args.interval_ms < 0:+ parser.error("samples/fps/deadline must be positive; warmup/interval must be nonnegative")+ with tempfile.TemporaryDirectory(prefix="opencompany-stream-benchmark-") as temp:+ asyncio.run(run(args, Path(temp)))+++if __name__ == "__main__":+ main()diff --git a/server/tests/nodes/browser/test_browser_frame_delivery.py b/server/tests/nodes/browser/test_browser_frame_delivery.pynew file mode 100644index 00000000..b4fdf45a--- /dev/null+++ b/server/tests/nodes/browser/test_browser_frame_delivery.py@@ -0,0 +1,198 @@+"""Frame-credit and trailing-delivery regressions without a Chrome dependency."""++import asyncio+import base64+import json+import time+from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock++import pytest++from nodes.browser import _stream+from nodes.browser._cdp import CDPDisconnected+from nodes.browser._stream_metrics import StreamMetrics+++def frame(seq):+ return _stream.encode_frame({"seq": seq}, b"jpeg")+++def sequence(data):+ return json.loads(data[4:4 + int.from_bytes(data[2:4], "big")])["seq"]+++def make_hub():+ session = SimpleNamespace(target_id="page", send=AsyncMock(), detach=AsyncMock(), on=Mock(return_value=lambda: None))+ controller = SimpleNamespace(active_target_id="page", add_listener=Mock(return_value=lambda: None), tabs={})+ runtime = SimpleNamespace(controller=controller, running=True, page_session=AsyncMock(return_value=session))+ hub = _stream.ScreencastHub(runtime)+ hub.session = session+ viewer = _stream.Viewer(None, "owner")+ hub.viewers[viewer.id] = viewer+ return hub, viewer, session+++async def stop_writer(viewer, task):+ viewer.closed = True+ viewer.wakeup.set()+ await asyncio.wait_for(task, 1)+++async def test_final_frame_inside_fps_interval_is_delivered_without_another_event():+ sent = asyncio.Queue()+ socket = SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=sent.put_nowait))+ viewer = _stream.Viewer(socket, "owner")+ viewer.max_fps = 30+ viewer.last_frame_at = time.monotonic()+ viewer.offer_frame(frame(1))+ task = asyncio.create_task(viewer.writer(lambda _: None))+ try:+ assert sequence(await asyncio.wait_for(sent.get(), 0.5)) == 1+ assert viewer.pending is None+ finally:+ await stop_writer(viewer, task)+++async def test_latest_frame_replaces_pending_and_duplicate_ack_never_grants_credit():+ sent = asyncio.Queue()+ viewer = _stream.Viewer(SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=sent.put_nowait)), "owner")+ task = asyncio.create_task(viewer.writer(lambda _: None))+ try:+ for seq in (1, 2):+ viewer.offer_frame(frame(seq))+ await asyncio.wait_for(sent.get(), 1)+ viewer.offer_frame(frame(3))+ viewer.offer_frame(frame(4))+ await asyncio.sleep(0)+ assert viewer.inflight == 2 and sequence(viewer.pending) == 4+ viewer.acknowledge(1)+ assert sequence(await asyncio.wait_for(sent.get(), 1)) == 4+ viewer.acknowledge(1)+ viewer.acknowledge(999)+ assert viewer.inflight == 2+ viewer.acknowledge() # malformed envelope cannot provide seq+ assert viewer.inflight == 1+ finally:+ await stop_writer(viewer, task)+++async def test_control_messages_bypass_frame_throttle():+ control = asyncio.Event()+ viewer = _stream.Viewer(SimpleNamespace(send_text=AsyncMock(side_effect=lambda _: control.set()), send_bytes=AsyncMock()), "owner")+ viewer.max_fps = 1+ viewer.last_frame_at = time.monotonic()+ viewer.offer_frame(frame(1))+ task = asyncio.create_task(viewer.writer(lambda _: None))+ try:+ viewer.send_json({"type": "state"})+ await asyncio.wait_for(control.wait(), .2)+ viewer.websocket.send_bytes.assert_not_awaited()+ finally:+ await stop_writer(viewer, task)+++async def test_hidden_viewer_cannot_send_queued_picture():+ viewer = _stream.Viewer(SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock()), "owner")+ viewer.offer_frame(frame(1))+ viewer.visible = False+ task = asyncio.create_task(viewer.writer(lambda _: None))+ await asyncio.sleep(0)+ assert viewer.pending is None+ viewer.websocket.send_bytes.assert_not_awaited()+ await stop_writer(viewer, task)+++async def test_slow_viewer_does_not_stop_fast_viewer():+ slow_sent, fast_sent = asyncio.Queue(), asyncio.Queue()+ viewers = [_stream.Viewer(SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=queue.put_nowait)), "owner") for queue in (slow_sent, fast_sent)]+ tasks = [asyncio.create_task(viewer.writer(lambda _: None)) for viewer in viewers]+ try:+ for seq in range(1, 5):+ for viewer in viewers:+ viewer.offer_frame(frame(seq))+ assert sequence(await asyncio.wait_for(fast_sent.get(), 1)) == seq+ viewers[1].acknowledge(seq)+ assert viewers[0].inflight == 2+ assert sequence(viewers[0].pending) == 4+ assert slow_sent.qsize() == 2+ finally:+ for viewer, task in zip(viewers, tasks):+ await stop_writer(viewer, task)++[email protected]("payload", [base64.b64encode(b"jpeg").decode(), "invalid!"])+async def test_each_chrome_frame_occurrence_is_acked_even_with_same_id(payload):+ hub, _, session = make_hub()+ for _ in range(2):+ hub._on_frame({"sessionId": 7, "data": payload})+ hub._ack_now()+ await asyncio.gather(*hub._ack_tasks)+ assert session.send.await_count == 2+ for call in session.send.await_args_list:+ assert call.args == ("Page.screencastFrameAck", {"sessionId": 7})+ await hub._stop_screencast()+ assert not hub._ack_tasks+++async def test_old_capture_debt_and_callback_do_not_ack_replacement_session():+ hub, _, old = make_hub()+ generation = hub._capture_generation+ params = {"sessionId": 1, "data": base64.b64encode(b"jpeg").decode()}+ hub._on_frame(params, old, generation)+ await hub._stop_screencast()+ replacement = SimpleNamespace(send=AsyncMock())+ hub.session = replacement+ hub._on_frame(params, old, generation)+ hub._ack_now()+ await asyncio.sleep(0)+ replacement.send.assert_not_awaited()+ assert not hub._pending_acks and not hub._ack_tasks+++async def test_same_viewer_on_replacement_hub_does_not_reuse_old_frame_sequence():+ old_hub, viewer, _ = make_hub()+ sent = asyncio.Queue()+ viewer.websocket = SimpleNamespace(send_text=AsyncMock(), send_bytes=AsyncMock(side_effect=sent.put_nowait))+ task = asyncio.create_task(viewer.writer(lambda _: None))+ params = {"sessionId": 1, "data": base64.b64encode(b"jpeg").decode()}+ new_hub, _, _ = make_hub()+ try:+ old_hub._on_frame(params)+ old_seq = sequence(await asyncio.wait_for(sent.get(), 1))+ await old_hub._stop_screencast()+ new_hub.viewers = {viewer.id: viewer}+ new_hub._on_frame(params)+ new_seq = sequence(await asyncio.wait_for(sent.get(), 1))+ assert new_seq > old_seq+ assert viewer.inflight == 2+ viewer.acknowledge(old_seq)+ viewer.acknowledge(old_seq)+ assert viewer.inflight == 1 and new_seq in viewer._outstanding+ finally:+ await stop_writer(viewer, task)+ await new_hub._stop_screencast()+++async def test_failed_ack_starts_fresh_capture_without_retrying_ambiguous_ack():+ hub, _, session = make_hub()+ session.send.side_effect = CDPDisconnected("lost")+ hub._schedule_retry = Mock()+ hub._on_frame({"sessionId": 9, "data": base64.b64encode(b"jpeg").decode()})+ hub._ack_now()+ await asyncio.gather(*hub._ack_tasks)+ session.send.assert_awaited_once()+ hub._schedule_retry.assert_called_once()+ assert not hub._active+++def test_diagnostics_are_opt_in_and_bounded(monkeypatch):+ monkeypatch.delenv("OPENCOMPANY_BROWSER_DIAGNOSTICS", raising=False)+ disabled = StreamMetrics()+ disabled.observe("frame_pack", 1)+ assert disabled.snapshot() == {"timings": {}, "counts": {}}+ monkeypatch.setenv("OPENCOMPANY_BROWSER_DIAGNOSTICS", "1")+ enabled = StreamMetrics()+ for _ in range(1000):+ enabled.observe("frame_pack", .001)+ assert enabled.snapshot()["timings"]["frame_pack"] == {"n": 256, "p50_ms": 1, "p95_ms": 1}diff --git a/server/tests/nodes/browser/test_browser_live_control.py b/server/tests/nodes/browser/test_browser_live_control.pynew file mode 100644index 00000000..2c6bb931--- /dev/null+++ b/server/tests/nodes/browser/test_browser_live_control.py@@ -0,0 +1,342 @@+"""Ordering, bounded admission and release barriers without a real browser."""++import asyncio+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from nodes.browser._live_control import LiveControlQueue, MAX_BYTES, MAX_COMMANDS+from nodes.browser._session import BrowserSession, ControlState, ProfileController, SessionKey+++def setup_queue():+ controller = ProfileController("profile", "Browser")+ controller._broadcast = AsyncMock()+ controller.active_target_id = "page"+ controller.tabs = {"page": {"url": "https://example.test"}}+ controller.state = ControlState.USER+ controller.controller_viewer = "viewer"+ session = BrowserSession(SessionKey("owner", "workflow", "node"), "profile")+ controller.lease_session = session+ messages = []+ viewer = SimpleNamespace(id="viewer", visible=True, closed=False, send_json=messages.append)+ page = SimpleNamespace(send=AsyncMock(return_value={}))+ hub = SimpleNamespace(controller=controller, session=page, input=AsyncMock(), copy=AsyncMock(), navigate=AsyncMock(), tab=AsyncMock(), dialog_reply=AsyncMock())+ queue = LiveControlQueue(hub)+ return queue, hub, viewer, session, messages+++async def drain(queue):+ if queue._worker:+ await asyncio.wait_for(queue._worker, 1)+++async def test_only_adjacent_moves_and_wheels_coalesce_without_reordering_keys():+ queue, hub, viewer, session, _ = setup_queue()+ for message in [+ {"type": "mouse", "action": "move", "x": 1},+ {"type": "mouse", "action": "move", "x": 2},+ {"type": "key", "action": "down", "key": "a"},+ {"type": "wheel", "delta_y": 4},+ {"type": "wheel", "delta_y": 6},+ ]:+ assert queue.enqueue(viewer, message, session)+ assert len(queue._queue) == 3+ await drain(queue)+ actual = [call.args[1] for call in hub.input.await_args_list]+ assert [m["type"] for m in actual] == ["mouse", "key", "wheel"]+ assert actual[0]["x"] == 2 and actual[-1]["delta_y"] == 10+++async def test_wheels_over_different_scroll_containers_do_not_coalesce():+ queue, hub, viewer, session, _ = setup_queue()+ queue.enqueue(viewer, {"type": "wheel", "x": 10, "y": 10, "delta_y": 5}, session)+ queue.enqueue(viewer, {"type": "wheel", "x": 100, "y": 10, "delta_y": 5}, session)+ assert len(queue._queue) == 2+ await drain(queue)+ assert hub.input.await_count == 2++[email protected]("limit", ["count", "bytes"])+async def test_overload_is_explicit_bounded_and_safely_releases_control(limit):+ queue, _, viewer, session, messages = setup_queue()+ if limit == "count":+ for _ in range(MAX_COMMANDS):+ assert queue.enqueue(viewer, {"type": "key", "action": "down", "key": "x"}, session)+ rejected = {"type": "key", "action": "up", "key": "x"}+ else:+ rejected = {"type": "insert_text", "text": "x" * MAX_BYTES}+ assert not queue.enqueue(viewer, rejected, session)+ assert len(queue._queue) <= MAX_COMMANDS and queue._bytes <= MAX_BYTES+ assert messages[-1]["code"] == "control_overloaded"+ await drain(queue)+ assert queue.controller.state == ControlState.IDLE+++async def test_target_epoch_discards_work_even_if_target_switches_back():+ queue, hub, viewer, session, _ = setup_queue()+ queue.enqueue(viewer, {"type": "key", "action": "down", "key": "a"}, session)+ hub.controller.active_target_id = "other"+ queue.target_changed()+ hub.controller.active_target_id = "page"+ queue.target_changed()+ await drain(queue)+ hub.input.assert_not_awaited()+++async def test_release_waits_dispatched_command_and_releases_keys_before_agent_admission():+ queue, hub, viewer, session, _ = setup_queue()+ entered, complete = asyncio.Event(), asyncio.Event()+ events = []++ async def send(method, params, **kwargs):+ if method == "Runtime.evaluate":+ events.append("read-start")+ entered.set()+ await complete.wait()+ events.append("read-finished")+ else:+ events.append(params["type"])+ return {}++ hub.session.send.side_effect = send++ async def key_input(v, message):+ await queue.guarded_session(v, hub.session).send("Input.dispatchKeyEvent", {"type": "keyDown", "key": "Shift", "code": "ShiftLeft"})++ async def copy(v):+ await queue.guarded_session(v, hub.session).send("Runtime.evaluate", {})++ hub.input.side_effect, hub.copy.side_effect = key_input, copy+ queue.enqueue(viewer, {"type": "key", "action": "down", "key": "Shift"}, session)+ queue.enqueue(viewer, {"type": "copy"}, session)+ await entered.wait()+ queue.enqueue(viewer, {"type": "key", "action": "down", "key": "z"}, session)+ queue.release(viewer)+ assert not queue.enqueue(viewer, {"type": "key", "action": "down", "key": "late"}, session)+ await asyncio.sleep(0)+ assert hub.controller.state == ControlState.USER+ complete.set()+ await drain(queue)+ assert events == ["keyDown", "read-start", "read-finished", "keyUp"]+ assert hub.controller.state == ControlState.IDLE and not queue._pressed+++async def test_automatic_handback_uses_same_dispatched_command_barrier():+ queue, hub, viewer, session, _ = setup_queue()+ entered, complete = asyncio.Event(), asyncio.Event()++ async def copy(v):+ entered.set()+ await complete.wait()++ hub.copy.side_effect = copy+ queue.enqueue(viewer, {"type": "copy"}, session)+ await entered.wait()+ release = asyncio.create_task(hub.controller.hand_back(viewer.id))+ await asyncio.sleep(0)+ assert not release.done() and hub.controller.state == ControlState.USER+ complete.set()+ await release+ await drain(queue)+ assert hub.controller.state == ControlState.IDLE+++async def test_hidden_viewer_cannot_receive_pending_takeover_and_requests_are_single_flight():+ queue, hub, viewer, session, _ = setup_queue()+ hub.controller.state = ControlState.AGENT+ hub.controller.controller_viewer = None+ assert queue.enqueue(viewer, {"type": "control_request"}, session)+ assert not queue.enqueue(viewer, {"type": "control_request"}, session)+ await asyncio.sleep(0)+ await asyncio.sleep(0)+ viewer.visible = False+ queue.release(viewer)+ await drain(queue)+ assert hub.controller.controller_viewer is None+ assert hub.controller.state == ControlState.AGENT+++async def test_guard_rechecks_ownership_between_two_awaited_cdp_commands():+ queue, hub, viewer, session, messages = setup_queue()+ entered, complete = asyncio.Event(), asyncio.Event()++ async def send(method, params, **kwargs):+ entered.set()+ await complete.wait()+ return {}++ hub.session.send.side_effect = send++ async def copy(v):+ page = queue.guarded_session(v, hub.session)+ await page.send("Page.getNavigationHistory", {})+ await page.send("Page.navigateToHistoryEntry", {})++ hub.copy.side_effect = copy+ queue.enqueue(viewer, {"type": "copy"}, session)+ await entered.wait()+ queue.release(viewer)+ complete.set()+ await drain(queue)+ assert hub.session.send.await_count == 1+ assert any(m.get("code") == "control" for m in messages)+++async def test_timeout_retires_chrome_before_handback_and_failed_retirement_fails_closed():+ queue, hub, viewer, session, _ = setup_queue()+ hub.runtime = SimpleNamespace(chrome=SimpleNamespace(shutdown=AsyncMock(side_effect=RuntimeError("busy"))))++ async def copy(v):+ await queue.guarded_session(v, hub.session).send("Page.navigate", {})++ hub.copy.side_effect = copy+ hub.session.send.side_effect = TimeoutError("unknown outcome")+ queue.enqueue(viewer, {"type": "copy"}, session)+ await drain(queue)+ assert queue._unsafe+ queue.release(viewer)+ await drain(queue)+ assert hub.controller.state == ControlState.USER+ hub.runtime.chrome.shutdown.side_effect = None+ queue.release(viewer)+ await drain(queue)+ assert hub.controller.state == ControlState.IDLE+ assert not queue._unsafe+++async def test_input_owns_a_session_independent_of_screencast_detach():+ queue, hub, viewer, session, _ = setup_queue()+ owned = SimpleNamespace(send=AsyncMock(return_value={}), detach=AsyncMock())+ hub.runtime = SimpleNamespace(page_session=AsyncMock(return_value=owned))++ async def copy(v):+ hub.session = None # capture resize can retire the picture session+ await queue.input_session(v).send("Runtime.evaluate", {})++ hub.copy.side_effect = copy+ queue.enqueue(viewer, {"type": "copy"}, session)+ await drain(queue)+ owned.send.assert_awaited_once()+ owned.detach.assert_not_awaited()+ await queue.close()+ owned.detach.assert_awaited_once()+++async def test_retirement_close_notification_does_not_wait_on_its_own_worker():+ queue, hub, viewer, _, _ = setup_queue()+ queue._unsafe = True++ async def shutdown():+ await queue.close() # runtime close callback while release is active++ hub.runtime = SimpleNamespace(chrome=SimpleNamespace(shutdown=shutdown))+ queue.release(viewer)+ await drain(queue)+ assert queue._closed and not queue._unsafe+ assert hub.controller.state == ControlState.IDLE+++async def test_forced_transfer_releases_original_session_keys_before_new_owner():+ queue, hub, viewer, session, _ = setup_queue()+ other = SimpleNamespace(id="other", visible=True, closed=False, send_json=lambda message: None)+ queue._pressed[(viewer.id, "key", "ShiftLeft")] = (hub.session, {"type": "keyDown", "key": "Shift", "code": "ShiftLeft"})+ queue.enqueue(other, {"type": "control_request", "force": True}, session)+ await drain(queue)+ hub.session.send.assert_awaited_once_with("Input.dispatchKeyEvent", {"type": "keyUp", "key": "Shift", "code": "ShiftLeft", "modifiers": 0}, timeout=5)+ assert hub.controller.controller_viewer == "other"+ assert viewer.id in queue._blocked and not queue._pressed+++async def test_navigation_in_same_target_releases_old_pressed_keys_before_next_input():+ queue, hub, viewer, session, _ = setup_queue()+ owned = SimpleNamespace(send=AsyncMock(return_value={}), detach=AsyncMock())+ hub.runtime = SimpleNamespace(page_session=AsyncMock(return_value=owned))+ await queue._ensure_input_session()+ queue._pressed[(viewer.id, "key", "ShiftLeft")] = (owned, {"type": "keyDown", "key": "Shift", "code": "ShiftLeft"})+ hub.controller.tabs["page"]["url"] = "https://example.test/next"+ queue.target_changed()+ queue.enqueue(viewer, {"type": "copy"}, session)+ await drain(queue)+ assert owned.send.await_args.args[1]["type"] == "keyUp"+ assert not queue._pressed+ hub.copy.assert_awaited_once()+ owned.detach.assert_not_awaited()+ await queue.idle_without_viewers()+ owned.detach.assert_awaited_once()+ assert queue._worker.done() and queue._input_session is None+++async def test_socket_reader_handles_ack_visibility_and_ping_while_navigation_waits(monkeypatch):+ from fastapi import WebSocketDisconnect+ import core.container+ import services.authz+ import nodes.browser._runtime as browser_runtime+ from nodes.browser import _stream++ queue, hub, _, session, _ = setup_queue()+ entered, complete, acknowledged, hidden, pong = (asyncio.Event() for _ in range(5))+ hub.commands = queue+ hub.dead = False+ hub.controller.controller_viewer = None++ async def add(viewer):+ hub.controller.controller_viewer = viewer.id++ async def navigate(viewer, message, policy):+ entered.set()+ await complete.wait()++ async def remove(viewer):+ queue.release(viewer)++ hub.add, hub.remove = add, remove+ hub.navigate.side_effect = navigate+ hub.state_message = lambda v: {"type": "state", "state": "user", "controller": "you"}+ hub.tabs_message = lambda: {"type": "tabs", "tabs": []}+ hub.schedule_resize = hidden.set+ hub.frame_sent = lambda v: None++ class Viewer(_stream.Viewer):+ def acknowledge(self, seq=None):+ acknowledged.set()++ class Socket:+ def __init__(self):+ self.messages = asyncio.Queue()+ async def accept(self):+ pass+ async def receive_json(self):+ message = await self.messages.get()+ if message is None:+ raise WebSocketDisconnect()+ return message+ async def send_text(self, text):+ if '"pong"' in text:+ pong.set()+ async def close(self, **kwargs):+ pass++ runtime = SimpleNamespace(running=lambda profile: object(), controller=lambda profile: hub.controller)+ monkeypatch.setattr(core.container, "container", SimpleNamespace(settings=lambda: None, user_auth_service=None))+ monkeypatch.setattr(services.authz, "authenticate_ws", AsyncMock(return_value="owner"))+ monkeypatch.setattr(browser_runtime, "get_browser_runtime", lambda: runtime)+ monkeypatch.setattr(_stream, "_resolve_attach", AsyncMock(return_value=session))+ monkeypatch.setattr(_stream, "get_hub", lambda _: hub)+ monkeypatch.setattr(_stream, "Viewer", Viewer)+ socket = Socket()+ socket.messages.put_nowait({"type": "attach", "target": {}})+ socket.messages.put_nowait({"type": "navigate", "action": "reload"})+ task = asyncio.create_task(_stream.browser_live_view(socket))+ try:+ await asyncio.wait_for(entered.wait(), 1)+ for message in ({"type": "ack", "seq": 1}, {"type": "visibility", "visible": False}, {"type": "ping"}):+ socket.messages.put_nowait(message)+ await asyncio.wait_for(asyncio.gather(acknowledged.wait(), hidden.wait(), pong.wait()), 1)+ assert hub.controller.state == ControlState.USER+ finally:+ complete.set()+ socket.messages.put_nowait(None)+ await asyncio.wait_for(task, 1)+ await drain(queue)diff --git a/server/tests/nodes/browser/test_browser_stream.py b/server/tests/nodes/browser/test_browser_stream.pynew file mode 100644index 00000000..b3f6cc3d--- /dev/null+++ b/server/tests/nodes/browser/test_browser_stream.py@@ -0,0 +1,70 @@+"""Live-view recovery with fake CDP; never installs or launches Chrome."""++import asyncio+from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock++import pytest++from nodes.browser import _stream+from nodes.browser._cdp import CDPDisconnected+++def make_hub(*, failures=0, fail_start=False):+ session = SimpleNamespace(target_id="page", on=Mock(return_value=lambda: None), send=AsyncMock(), detach=AsyncMock())+ if fail_start:+ async def send(method, *args, **kwargs):+ if method == "Page.startScreencast":+ raise CDPDisconnected("lost")+ session.send.side_effect = send+ controller = SimpleNamespace(active_target_id="page", controller_viewer=None, state="idle", tabs={}, add_listener=Mock(return_value=lambda: None), viewer_attached=Mock(), viewer_detached=Mock())+ page_session = AsyncMock(side_effect=[*[CDPDisconnected("lost") for _ in range(failures)], session]) if failures else AsyncMock(return_value=session)+ runtime = SimpleNamespace(controller=controller, running=True, page_session=page_session)+ hub = _stream.ScreencastHub(runtime)+ viewer = _stream.Viewer(None, "owner")+ return hub, viewer, runtime, session+++async def test_initial_attach_failure_recovers_without_viewer_resize(monkeypatch):+ monkeypatch.setattr(_stream, "_SCREENCAST_RETRY_DELAYS", (0, 0, 0))+ hub, viewer, runtime, session = make_hub(failures=1)+ await hub.add(viewer)+ message = viewer.control.get_nowait()+ assert message["code"] == "screencast" and message["retrying"] is True+ await hub._retry_task+ assert hub._active and runtime.page_session.await_count == 2+ await hub.remove(viewer)+ session.detach.assert_awaited_once()+++async def test_start_failure_retries_are_bounded_and_reported(monkeypatch):+ monkeypatch.setattr(_stream, "_SCREENCAST_RETRY_DELAYS", (0, 0, 0))+ hub, viewer, runtime, session = make_hub(fail_start=True)+ await hub.add(viewer)+ await hub._retry_task+ assert not hub._active and runtime.page_session.await_count == 4+ messages = []+ while not viewer.control.empty():+ messages.append(viewer.control.get_nowait())+ assert messages[-1]["retrying"] is False+ assert session.detach.await_count == 4+ await asyncio.sleep(0)+ assert runtime.page_session.await_count == 4+ await hub.remove(viewer)++[email protected]("end", ["hidden", "removed", "closed"])+async def test_pending_retry_stops_when_no_visible_viewer_or_browser_closes(monkeypatch, end):+ monkeypatch.setattr(_stream, "_SCREENCAST_RETRY_DELAYS", (60,))+ hub, viewer, runtime, _ = make_hub(failures=1)+ await hub.add(viewer)+ retry = hub._retry_task+ if end == "hidden":+ viewer.visible = False+ await hub._refresh()+ elif end == "removed":+ await hub.remove(viewer)+ else:+ await hub._on_controller("closed", {"reason": "stopped"})+ await asyncio.gather(retry, return_exceptions=True)+ assert retry.cancelled() and runtime.page_session.await_count == 1