Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(browser): keep clicks responsive while hovering over the sign-in window

rubenluetke10-beepedb666c2148a
diff --git a/jarvis/ui/web/society_browser_routes.py b/jarvis/ui/web/society_browser_routes.py
index 353c45e7a6..d4823224ec 100644
--- a/jarvis/ui/web/society_browser_routes.py
+++ b/jarvis/ui/web/society_browser_routes.py
@@ -225,17 +225,49 @@ async def agent_browser_live(websocket: WebSocket, agent_id: str) -> None:
sender = asyncio.create_task(frames())
commands: asyncio.Queue[tuple[str, dict, str]] = asyncio.Queue(maxsize=256)
+ # Hover motion arrives ~30 times a second. Queued one by one behind a
+ # slower worker round trip it delayed every click by seconds and then
+ # overflowed the queue, so only the newest motion waits, behind input.
+ hover: list[tuple[str, dict, str]] = []
+ hover_ready = asyncio.Event()
+
+ async def next_control() -> tuple[str, dict, str]:
+ while True:
+ if not commands.empty():
+ return commands.get_nowait()
+ if hover:
+ hover_ready.clear()
+ return hover.pop()
+ getter = asyncio.ensure_future(commands.get())
+ waiter = asyncio.ensure_future(hover_ready.wait())
+ try:
+ done, _ = await asyncio.wait(
+ {getter, waiter}, return_when=asyncio.FIRST_COMPLETED
+ )
+ finally:
+ waiter.cancel()
+ if getter not in done:
+ getter.cancel()
+ if getter in done:
+ return getter.result()
async def controls() -> None:
while True:
- op, args, generation = await commands.get()
+ op, args, generation = await next_control()
+ motion = op == "click" and args.get("move_only") is True
try:
current = session.state.get("generation", session.generation)
if op not in {"takeover", "cancel"} and generation != current:
raise ValueError("The browser changed; wait for its new image")
result = await live.control(session, owner, op, args)
- await send({"kind": "control", "op": op, "ok": True, **result})
+ if not motion:
+ await send({"kind": "control", "op": op, "ok": True, **result})
except (ValueError, RuntimeError) as exc:
+ if motion:
+ # The next motion replaces a stale one; a hover error
+ # must not overwrite the viewer's visible state.
+ log.debug("Browser hover motion skipped: %s", exc)
+ continue
# Control errors are returned to the requesting viewer.
await send({"kind": "control", "op": op, "ok": False, "error": str(exc)[:500]})
@@ -257,7 +289,22 @@ async def agent_browser_live(websocket: WebSocket, agent_id: str) -> None:
receive = asyncio.create_task(websocket.receive_json())
try:
op, args = validate_control(value)
- commands.put_nowait((op, args, session.state.get("generation", session.generation)))
+ item = (op, args, session.state.get("generation", session.generation))
+ if op == "click" and args.get("move_only") is True:
+ hover[:] = [item]
+ hover_ready.set()
+ else:
+ # Motion recorded before this input is older than it.
+ hover.clear()
+ commands.put_nowait(item)
+ except asyncio.QueueFull:
+ await send(
+ {
+ "kind": "control",
+ "ok": False,
+ "error": "The browser is still catching up; try that again in a moment.",
+ }
+ )
except (ValueError, RuntimeError) as exc:
# Invalid controls are visibly rejected without closing a healthy stream.
await send({"kind": "control", "ok": False, "error": str(exc)[:500]})
diff --git a/tests/contract/test_browser_contract.py b/tests/contract/test_browser_contract.py
index a19719bb23..14e8744e37 100644
--- a/tests/contract/test_browser_contract.py
+++ b/tests/contract/test_browser_contract.py
@@ -652,6 +652,74 @@ async def test_viewer_disconnect_cancels_pending_takeover(monkeypatch, burst):
assert released.is_set()
+async def test_hover_motion_coalesces_and_never_delays_or_drops_clicks(monkeypatch):
+ import asyncio
+
+ from starlette.websockets import WebSocketDisconnect
+
+ from jarvis.ui.web import society_browser_routes as routes
+
+ clicked = asyncio.Event()
+ seen: list[tuple[str, int | None]] = []
+ sent: list[dict] = []
+
+ class Live:
+ async def subscribe(self, agent):
+ return SimpleNamespace(state={}, generation="g", control_owner=None), asyncio.Queue()
+
+ async def control(self, _session, _owner, op, args):
+ seen.append(("hover" if args.get("move_only") else op, args.get("x")))
+ await asyncio.sleep(0.02) # A slow worker round trip.
+ if op == "click" and not args.get("move_only"):
+ clicked.set()
+ return {}
+
+ async def unsubscribe(self, *args):
+ pass # Nothing to release in this fake.
+
+ class Socket:
+ scope = {}
+ reads = 0
+
+ async def accept(self):
+ pass # Test socket has no transport to accept.
+
+ async def close(self, **kwargs):
+ pass # Test socket has no transport to close.
+
+ async def receive_json(self):
+ self.reads += 1
+ # 400 motions overflowed the old 256-entry queue and closed the view.
+ if self.reads <= 400:
+ return {"op": "click", "args": {"x": self.reads, "y": 1, "move_only": True}}
+ if self.reads == 401:
+ return {"op": "click", "args": {"x": 9999, "y": 1}}
+ await clicked.wait()
+ raise WebSocketDisconnect()
+
+ async def send_json(self, value):
+ sent.append(value)
+
+ async def resolve(_):
+ return SimpleNamespace(agent_id="test")
+
+ async def runtime(_):
+ return SimpleNamespace(
+ roster=SimpleNamespace(resolve=resolve), browser=SimpleNamespace(live=Live())
+ )
+
+ monkeypatch.setattr(routes, "_runtime", runtime)
+ monkeypatch.setattr(routes, "credentials_valid", lambda _: True)
+ await asyncio.wait_for(routes.agent_browser_live(Socket(), "test"), 2)
+ assert ("click", 9999) in seen
+ # The click waits for at most the one motion already in flight.
+ assert seen.index(("click", 9999)) <= 2
+ assert sum(1 for kind, _ in seen if kind == "hover") <= 2
+ assert [m for m in sent if m.get("kind") == "control"] == [
+ {"kind": "control", "op": "click", "ok": True}
+ ]
+
+
@pytest.mark.parametrize(
"detail",
[