Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Bounded, ordered user commands independent of the live socket reader.
One queue belongs to one profile hub. Release invalidates queued work at
admission time, then waits for the dispatched command before releasing any
pressed inputs. Chrome commands already sent are never assumed cancelled.
"""
from __future__ import annotations
import asyncio
import json
import time
from collections import deque
from dataclasses import dataclass
from typing import Any
from services.plugin.base import NodeUserError
from ._cdp import CDPDisconnected
MAX_COMMANDS = 64
MAX_BYTES = 64 * 1024
INPUT_KINDS = frozenset({"mouse", "wheel", "key", "insert_text", "navigate", "tab", "copy", "dialog_reply"})
@dataclass
class Command:
viewer: Any
message: dict
session: Any
epoch: int
target: Any
size: int
queued_at: float
class _GuardedSession:
def __init__(self, queue: "LiveControlQueue", command: Command, session: Any):
self.queue, self.command, self.session = queue, command, session
async def send(self, method, params=None, **kwargs):
if not self.queue.valid(self.command):
raise NodeUserError("Browser control or page changed; the queued command was discarded.")
params = params or {}
identity = None
if method == "Input.dispatchKeyEvent":
identity = (self.command.viewer.id, "key", params.get("code") or params.get("key"))
elif method == "Input.dispatchMouseEvent" and params.get("type") in {"mousePressed", "mouseReleased"}:
identity = (self.command.viewer.id, "mouse", params.get("button"))
# Record before dispatch: even a response timeout can leave a key held.
if identity and params.get("type") not in {"keyUp", "mouseReleased"}:
if identity not in self.queue._pressed and len(self.queue._pressed) >= MAX_COMMANDS:
raise NodeUserError("Too many held browser inputs. Hand control back before continuing.")
self.queue._pressed[identity] = (self.session, dict(params))
started = time.monotonic()
try:
result = await self.session.send(method, params, **kwargs)
except (TimeoutError, CDPDisconnected):
self.queue._unsafe = True
self.queue.invalidate(self.command.viewer.id)
self.queue._error(self.command.viewer, "control_uncertain", "Chrome did not confirm the action. Hand back control to safely restart the browser.")
raise
finally:
self.queue._observe("command_cdp", time.monotonic() - started)
if identity and params.get("type") in {"keyUp", "mouseReleased"}:
self.queue._pressed.pop(identity, None)
return result
class LiveControlQueue:
def __init__(self, hub: Any):
self.hub = hub
self.controller = hub.controller
self._queue: deque[Command] = deque()
self._bytes = 0
self._epochs: dict[str, int] = {}
self._blocked: set[str] = set()
self._worker: asyncio.Task | None = None
self._takeover: asyncio.Task | None = None
self._takeover_viewer: str | None = None
self._takeover_pending: set[str] = set()
self._lock = asyncio.Lock()
self._current: Command | None = None
self._pressed: dict[tuple, tuple[Any, dict]] = {}
self._closed = False
self._unsafe = False
self._retiring = False
self._input_session = None
self._input_target = None
self._input_epoch = None
self._target_epoch = 0
self._last_target = None
self.controller.live_control_barrier = self.barrier
def _observe(self, name: str, value: float) -> None:
metrics = getattr(self.hub, "metrics", None)
if metrics is not None:
metrics.observe(name, value)
def _error(self, viewer, code, message):
viewer.send_json({"type": "error", "code": code, "message": message})
def _refuse_claim(self, viewer) -> None:
"""Every control request is answered: the viewer waits for it (the
Workspace's Take over resolves on it)."""
reason = "protected_login" if getattr(self.controller, "sensitive_login", False) else "viewer_unavailable"
viewer.send_json({"type": "control", "granted": False, "reason": reason})
def _target(self):
target = self.controller.active_target_id
current = target, (self.controller.tabs.get(target or "") or {}).get("url")
if current != self._last_target:
self._last_target = current
self._target_epoch += 1
return self._target_epoch, current
def target_changed(self):
self._target()
def valid(self, command: Command, *, owner: bool = True) -> bool:
viewer = command.viewer
return (
not self._closed and not viewer.closed and viewer.visible
and not getattr(self.controller, "sensitive_login", False)
and (not owner or viewer.id not in self._blocked)
and command.epoch == self._epochs.get(viewer.id, 0)
and command.target == self._target()
and (not owner or self.controller.can_inject_input(viewer.id))
)
def guarded_session(self, viewer, session):
command = self._current
if command is not None and command.viewer.id == viewer.id and session is not None:
return _GuardedSession(self, command, session)
return session
def input_session(self, viewer):
# Dedicated to input: the screencast session may be detached whenever
# a viewer resizes or becomes hidden.
return self.guarded_session(viewer, self._input_session)
async def _ensure_input_session(self):
target = self.controller.active_target_id
epoch = self._target()
if self._input_session is not None and self._input_target == target and self._input_epoch == epoch:
return
if self._input_session is not None:
for viewer_id in {identity[0] for identity in self._pressed}:
await self._release_pressed(viewer_id)
if self._input_target == target:
self._input_epoch = epoch
return
await self._input_session.detach()
self._input_session = await self.hub.runtime.page_session(target)
self._input_target = target
self._input_epoch = epoch
def enqueue(self, viewer, message: dict, session) -> bool:
kind = message.get("type")
if self._closed or viewer.closed or not viewer.visible or getattr(self.controller, "sensitive_login", False):
if kind == "control_request":
self._refuse_claim(viewer)
return False
# A second request while one is pending is answered with the first.
if kind == "control_request" and viewer.id in self._takeover_pending:
return False
if kind in INPUT_KINDS and (viewer.id in self._blocked or self._unsafe or not self.controller.can_inject_input(viewer.id)):
return False
size = len(json.dumps(message, ensure_ascii=False).encode("utf-8"))
command = Command(viewer, dict(message), session, self._epochs.get(viewer.id, 0), self._target(), size, time.monotonic())
previous = self._queue[-1] if self._queue else None
if previous and self._coalescible(previous, command):
if kind == "wheel":
command.message["delta_x"] = float(previous.message.get("delta_x") or 0) + float(message.get("delta_x") or 0)
command.message["delta_y"] = float(previous.message.get("delta_y") or 0) + float(message.get("delta_y") or 0)
command.size = len(json.dumps(command.message).encode("utf-8"))
if self._bytes - previous.size + command.size <= MAX_BYTES:
self._queue[-1] = command
self._bytes += command.size - previous.size
return True
if len(self._queue) >= MAX_COMMANDS or self._bytes + command.size > MAX_BYTES:
self._error(viewer, "control_overloaded", "Browser input is busy. Wait for the current action, then try again.")
if kind == "control_request":
self._refuse_claim(viewer)
# Never silently lose a key-up on overflow: stop admission and
# release everything after the current command settles.
self.release(viewer, note="input queue overflow")
return False
if kind == "control_request":
self._takeover_pending.add(viewer.id)
self._queue.append(command)
self._bytes += command.size
self._ensure_worker()
return True
@staticmethod
def _coalescible(a: Command, b: Command) -> bool:
if (a.viewer.id, a.epoch, a.target) != (b.viewer.id, b.epoch, b.target):
return False
x, y = a.message, b.message
if x.get("modifiers", 0) != y.get("modifiers", 0) or x.get("type") != y.get("type"):
return False
wheel = y.get("type") == "wheel" and all(x.get(k, 0) == y.get(k, 0) for k in ("x", "y", "buttons"))
return (wheel or (
y.get("type") == "mouse" and x.get("action") == y.get("action") == "move"
and x.get("buttons", 0) == y.get("buttons", 0)
))
def invalidate(self, viewer_id: str) -> None:
self._blocked.add(viewer_id)
self._epochs[viewer_id] = self._epochs.get(viewer_id, 0) + 1
for command in self._queue:
if command.viewer.id == viewer_id and command.message.get("type") == "control_request":
command.viewer.send_json({"type": "control", "granted": False, "reason": "viewer_unavailable"})
self._queue = deque(c for c in self._queue if c.viewer.id != viewer_id)
self._bytes = sum(c.size for c in self._queue)
self._takeover_pending.discard(viewer_id)
if self._takeover_viewer == viewer_id and self._takeover is not None:
self._takeover.cancel()
def release(self, viewer, *, outcome="control_lost", note="") -> None:
self.invalidate(viewer.id)
# Only the viewer in control hands it back. While the employee waits
# for the owner (awaiting_user) no viewer has it yet, so a view that
# is hidden, blurred or closed leaves the request waiting.
if getattr(self.controller, "controller_viewer", None) != viewer.id:
return
# A zero-byte priority barrier is always admissible. At most one per
# viewer remains because invalidate removes its previous barrier.
if len(self._queue) >= MAX_COMMANDS:
removed = self._queue.pop()
self._bytes -= removed.size
self._takeover_pending.discard(removed.viewer.id)
self._error(removed.viewer, "control_overloaded", "Browser control changed. Try your request again.")
self._queue.appendleft(Command(viewer, {"type": "control_release", "outcome": outcome, "note": note}, None, 0, None, 0, time.monotonic()))
self._ensure_worker()
async def barrier(self, viewer_id: str | None) -> None:
if viewer_id is None:
return
self.invalidate(viewer_id)
if asyncio.current_task() in (self._worker, self._takeover):
await self._release_pressed(viewer_id)
else:
async with self._lock:
await self._release_pressed(viewer_id)
async def _release_pressed(self, viewer_id):
if self._unsafe:
await self._retire_uncertain_browser()
return
for identity, (session, params) in list(self._pressed.items()):
if identity[0] != viewer_id:
continue
release = dict(params)
if identity[1] == "key":
release.update(type="keyUp", modifiers=0)
release.pop("text", None)
release.pop("unmodifiedText", None)
method = "Input.dispatchKeyEvent"
else:
release.update(type="mouseReleased", buttons=0, modifiers=0)
method = "Input.dispatchMouseEvent"
try:
await session.send(method, release, timeout=5)
except Exception as exc:
self._unsafe = True
try:
await self._retire_uncertain_browser()
return
except Exception:
raise NodeUserError("Could not release browser input safely. Restart the browser before handing control back.") from exc
self._pressed.pop(identity, None)
async def _retire_uncertain_browser(self):
# A local timeout only abandons the reply; Chrome may still execute
# the command. Closing this managed process is the safety boundary.
try:
self._retiring = True
await asyncio.wait_for(self.hub.runtime.chrome.shutdown(), timeout=10)
except Exception as exc:
raise NodeUserError("Chrome has an unconfirmed action and could not be stopped. Control remains with you.") from exc
finally:
self._retiring = False
self._unsafe = False
self._pressed.clear()
self._input_session = None
self._input_target = None
self._input_epoch = None
def _ensure_worker(self):
if self._worker is None or self._worker.done():
self._worker = asyncio.create_task(self._run(), name="browser-live-control")
async def _run(self):
while self._queue:
command = self._queue.popleft()
self._bytes -= command.size
viewer, message = command.viewer, command.message
kind = message.get("type")
async with self._lock:
self._current = command
try:
self._observe("command_queue_wait", time.monotonic() - command.queued_at)
if kind == "control_release":
await self.controller.hand_back(viewer.id, outcome=message.get("outcome", "handed_back"), note=message.get("note", ""))
elif kind == "control_request":
if not self.valid(command, owner=False):
self._refuse_claim(viewer)
continue
self._takeover_viewer = viewer.id
self._takeover = asyncio.create_task(self._claim(command))
try:
await self._takeover
except asyncio.CancelledError:
viewer.send_json({"type": "control", "granted": False, "reason": "viewer_unavailable"})
if self._closed:
return
finally:
self._takeover = None
self._takeover_viewer = None
elif self.valid(command):
if hasattr(self.hub, "runtime") and hasattr(self.hub.runtime, "page_session"):
await self._ensure_input_session()
self.controller.touch_user_input()
if kind in {"mouse", "wheel", "key", "insert_text"}:
await self.hub.input(viewer, message)
elif kind in {"navigate", "tab"}:
await self._release_pressed(viewer.id)
await getattr(self.hub, kind)(viewer, message, command.session.policy)
elif kind == "copy":
await self.hub.copy(viewer)
elif kind == "dialog_reply":
await self.hub.dialog_reply(viewer, message)
except Exception as exc:
self._error(viewer, "control", str(exc)[:300])
if kind == "control_request":
viewer.send_json({"type": "control", "granted": False, "reason": "control_failed", "message": str(exc)[:300]})
finally:
if kind == "control_request":
self._takeover_pending.discard(viewer.id)
self._current = None
async def _claim(self, command):
viewer = command.viewer
await self.controller.acquire_lease(command.session, wait=2.0)
if not self.valid(command, owner=False):
self._refuse_claim(viewer)
return
granted, reason = await self.controller.take_over(
viewer.id, force=bool(command.message.get("force")),
valid=lambda: self.valid(command, owner=False),
)
viewer.send_json({"type": "control", "granted": granted, "reason": reason})
if granted:
self._blocked.discard(viewer.id)
async def close(self):
self._closed = True
if self._takeover is not None:
self._takeover.cancel()
self._queue.clear()
self._bytes = 0
if self._retiring:
# Chrome's close notification can run inside shutdown(). Waiting
# for the worker here would wait on our own shutdown barrier.
return
# Let already-dispatched CDP operations finish; their normal protocol
# timeout still bounds this wait. Do not tell agents input has settled
# merely because a Python task was cancelled.
if self._worker is not None and self._worker is not asyncio.current_task():
await asyncio.gather(self._worker, return_exceptions=True)
if self._input_session is not None:
await self._input_session.detach()
self._input_session = None
async def idle_without_viewers(self):
"""Retain the reusable hub, but no command task/session after disposal."""
if self._worker is not None and self._worker is not asyncio.current_task():
await asyncio.gather(self._worker, return_exceptions=True)
async with self._lock:
if getattr(self.hub, "viewers", None):
return
for viewer_id in {identity[0] for identity in self._pressed}:
await self._release_pressed(viewer_id)
if self._input_session is not None:
await self._input_session.detach()
self._input_session = None
self._input_target = self._input_epoch = None
self._epochs.clear()
self._blocked.clear()