Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(browser): visualize real agent clicks and restore native views

rubenluetke10-beep09092bc4c46f
diff --git a/jarvis.spec b/jarvis.spec
index c65981e465..14f158fb09 100644
--- a/jarvis.spec
+++ b/jarvis.spec
@@ -71,6 +71,7 @@ VERSION = _package_version()
datas = []
datas.append((str(PROJECT_ROOT / "jarvis/society/browser/live_runner.py"), "jarvis/society/browser"))
datas.append((str(PROJECT_ROOT / "jarvis/society/browser/native_window.py"), "jarvis/society/browser"))
+datas.append((str(PROJECT_ROOT / "jarvis/society/browser/pointer.py"), "jarvis/society/browser"))
# Include the frontend build when present. Preserve its package-relative layout
# so the FastAPI static-files mount can serve it from a frozen application.
diff --git a/jarvis/society/browser/live.py b/jarvis/society/browser/live.py
index 7502d1ec38..298a977f12 100644
--- a/jarvis/society/browser/live.py
+++ b/jarvis/society/browser/live.py
@@ -76,6 +76,7 @@ class LiveSession:
generation: str = ""
closed: bool = False
stderr_tail: str = ""
+ window_upgrade_pending: bool = False
async def send(self, value: dict[str, Any]) -> None:
data = (json.dumps(value, ensure_ascii=True) + "\n").encode()
@@ -166,7 +167,7 @@ class LiveSession:
elif kind == "state":
self.state = event
self.publish(event)
- elif kind in {"frame", "dialog", "download", "warning", "step"}:
+ elif kind in {"frame", "pointer", "dialog", "download", "warning", "step"}:
self.publish(event)
elif kind == "fatal":
raise RuntimeError(event.get("error", "Browser worker failed"))
@@ -259,7 +260,7 @@ class LiveSessions:
self.idle_tasks[session.agent_id] = asyncio.create_task(expire())
- async def ensure(self, agent: Any) -> LiveSession:
+ async def ensure(self, agent: Any, *, window_view: bool = False) -> LiveSession:
agent_id = agent.agent_id
idle = self.idle_tasks.pop(agent_id, None)
if idle:
@@ -267,7 +268,17 @@ class LiveSessions:
async with self.locks.setdefault(agent_id, asyncio.Lock()):
old = self.sessions.get(agent_id)
if old and not old.closed:
- return old
+ upgrade = (
+ window_view
+ and os.name == "nt"
+ and getattr(agent, "browser_mode", "own") == "own"
+ and not old.state.get("full_window", False)
+ )
+ if not upgrade:
+ return old
+ if old.run_lock.locked() or old.control_owner:
+ old.window_upgrade_pending = True
+ return old
if old:
await old.close()
if not install.is_installed(self.data_dir):
@@ -297,6 +308,7 @@ class LiveSessions:
"ensure",
{
"profile_dir": str(folder / "browser-profile"),
+ "window_view": window_view,
"workspace": str(folder / "workspace"),
"executable": str(install.browser_executable(self.data_dir)),
"icon_path": str(
@@ -310,6 +322,16 @@ class LiveSessions:
timeout=90,
)
session.generation = result["generation"]
+ session.state = {
+ "kind": "state",
+ "manual": False,
+ "running": False,
+ "url": "",
+ "target": "",
+ "tabs": [],
+ **session.state,
+ "full_window": bool(result.get("full_window")),
+ }
self.sessions[agent_id] = session
self.release_when_idle(session)
return session
@@ -318,7 +340,7 @@ class LiveSessions:
raise
async def subscribe(self, agent: Any) -> tuple[LiveSession, LiveUpdates]:
- session = await self.ensure(agent)
+ session = await self.ensure(agent, window_view=True)
queue = LiveUpdates()
session.subscribers.add(queue)
try:
@@ -338,6 +360,8 @@ class LiveSessions:
if session.control_owner == owner:
await session.command("takeover", {"enabled": False})
session.control_owner = None
+ if session.window_upgrade_pending:
+ session.publish({"kind": "disconnected"})
if not session.subscribers:
await session.command("subscribe", {"enabled": False})
self.release_when_idle(session)
@@ -361,6 +385,8 @@ class LiveSessions:
session.publish({"kind": "control", "ok": True, "manual": False})
raise
session.control_owner = owner if args.get("enabled") else None
+ if not session.control_owner and session.window_upgrade_pending:
+ session.publish({"kind": "disconnected"})
session.publish({"kind": "control", "ok": True, **result})
return result
if op != "cancel" and session.control_owner != owner:
@@ -417,6 +443,8 @@ class LiveSessions:
session.rpc_context = None
session.active_trace = ""
session.active_chat = ""
+ if session.window_upgrade_pending:
+ session.publish({"kind": "disconnected"})
if not session.subscribers:
self.release_when_idle(session)
diff --git a/jarvis/society/browser/live_runner.py b/jarvis/society/browser/live_runner.py
index 89a56276f5..b78c8421ed 100644
--- a/jarvis/society/browser/live_runner.py
+++ b/jarvis/society/browser/live_runner.py
@@ -101,6 +101,8 @@ class Worker:
self.native_frame_at = 0.0
self.native_replay = False
self.branding: asyncio.Task | None = None
+ self.pointer: Any = None
+ self.visual_action = False
async def rpc(self, kind: str, payload: dict) -> dict:
key = uuid.uuid4().hex
@@ -116,6 +118,10 @@ class Worker:
from native_window import NativeWindow, available # type: ignore[import-not-found]
native_enabled = available()
+ if args.get("window_view") and sys.platform == "win32" and not native_enabled:
+ raise RuntimeError(
+ "Chrome needs an unlocked Windows desktop and the managed capture runtime"
+ )
from browser_use import Browser # type: ignore[import-not-found]
from playwright.async_api import async_playwright
@@ -193,6 +199,15 @@ class Worker:
allowed_domains=args.get("allowed_domains") or None,
)
await self.browser.start()
+ from pointer import PointerTracker # type: ignore[import-not-found]
+
+ self.pointer = PointerTracker(
+ self.generation,
+ emit,
+ lambda: self.visual_action and not self.manual,
+ lambda x, y: self.native.viewport_point(x, y) if self.native else (x, y, 1280, 800),
+ )
+ self.browser.cdp_client.send_raw = self.pointer.wrap(self.browser.cdp_client.send_raw)
self.context.on("page", self.page_opened)
for page in self.context.pages:
self.page_opened(page)
@@ -228,7 +243,11 @@ class Worker:
)
self.state_task = asyncio.create_task(self.watch_state())
self.stream = asyncio.create_task(self.watch())
- return {"generation": self.generation, "protocol": PROTOCOL_VERSION}
+ return {
+ "generation": self.generation,
+ "protocol": PROTOCOL_VERSION,
+ "full_window": bool(self.native),
+ }
def page_opened(self, page: Any) -> None:
page.on("dialog", self.on_dialog)
@@ -458,11 +477,14 @@ class Worker:
)
if not answer.get("ok"):
raise asyncio.CancelledError(answer.get("error", "Browser action denied"))
+ worker.visual_action = True
try:
result = await super().act(action, browser_session, *pos, **kw)
except Exception as exc:
# The result is emitted below and becomes the executor's visible failure.
result = ActionResult(error=f"{type(exc).__name__}: browser action failed")
+ finally:
+ worker.visual_action = False
emit("action_result", id=answer["permit"], result=result.model_dump(mode="json"))
return result
@@ -513,6 +535,8 @@ class Worker:
}
finally:
self.agent = None
+ if self.pointer:
+ self.pointer.clear()
self.step_idle.set()
async def command(self, op: str, args: dict) -> dict:
@@ -554,6 +578,8 @@ class Worker:
if generation != self.takeover_generation:
return {"manual": self.manual}
self.manual = True
+ if self.pointer:
+ self.pointer.clear()
else:
if self.native and self.manual:
from browser_use.browser import events # type: ignore[import-not-found]
diff --git a/jarvis/society/browser/native_window.py b/jarvis/society/browser/native_window.py
index 13269ca4e7..c54c04c3a6 100644
--- a/jarvis/society/browser/native_window.py
+++ b/jarvis/society/browser/native_window.py
@@ -266,6 +266,46 @@ class NativeWindow:
target = int(child)
raise RuntimeError("Chrome input widget nesting is invalid")
+ def viewport_point(self, x: float, y: float) -> tuple[float, float, int, int]:
+ """Project CDP viewport coordinates into the complete captured Chrome frame."""
+ c, w, u = self.ctypes, self.wintypes, self.user32
+ self._check_owner()
+ u.EnumChildWindows.argtypes = [w.HWND, self.callback_type, w.LPARAM]
+ u.EnumChildWindows.restype = w.BOOL
+ children = []
+
+ @self.callback_type
+ def visit(hwnd: int, _unused: int) -> bool:
+ name = c.create_unicode_buffer(128)
+ u.GetClassNameW(hwnd, name, len(name))
+ if name.value == "Chrome_RenderWidgetHostHWND" and u.IsWindowVisible(hwnd):
+ rect = w.RECT()
+ if u.GetWindowRect(hwnd, c.byref(rect)):
+ children.append((rect.right - rect.left, rect.left, rect.top))
+ return True
+
+ with self.dpi():
+ u.EnumChildWindows(self.hwnd, visit, 0)
+ if not children:
+ raise RuntimeError("Chrome viewport is unavailable")
+ width, left, top = max(children)
+ bounds = w.RECT()
+ dwm = c.WinDLL("dwmapi")
+ dwm.DwmGetWindowAttribute.argtypes = [w.HWND, w.DWORD, c.c_void_p, w.DWORD]
+ dwm.DwmGetWindowAttribute.restype = c.c_long
+ if dwm.DwmGetWindowAttribute(self.hwnd, 9, c.byref(bounds), c.sizeof(bounds)):
+ raise RuntimeError("Chrome frame geometry is unavailable")
+ frame = self.frame()
+ if not frame:
+ raise RuntimeError("Chrome frame is unavailable")
+ scale = width / 1280
+ return (
+ left - bounds.left + x * scale,
+ top - bounds.top + y * scale,
+ frame["width"],
+ frame["height"],
+ )
+
def shortcut(self, key: int, modifiers: list[int]) -> None:
"""Apply modifiers to Chrome's input queue, never to the physical keyboard."""
c, w, u = self.ctypes, self.wintypes, self.user32
diff --git a/jarvis/society/browser/pointer.py b/jarvis/society/browser/pointer.py
new file mode 100644
index 0000000000..e2ad7315ee
--- /dev/null
+++ b/jarvis/society/browser/pointer.py
@@ -0,0 +1,58 @@
+"""Read-only visualization of mouse commands successfully sent to Chrome."""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import Any
+
+log = logging.getLogger(__name__)
+
+
+class PointerTracker:
+ def __init__(self, generation: str, emit: Any, enabled: Any, project: Any) -> None:
+ self.generation = generation
+ self.emit = emit
+ self.enabled = enabled
+ self.project = project
+ self.click_id = 0
+ self.click = (0.0, 0.0)
+
+ def wrap(self, send: Any) -> Any:
+ async def observed(method: str, params: Any = None, session_id: str | None = None) -> Any:
+ result = await send(method=method, params=params, session_id=session_id)
+ if self.enabled() and method == "Input.dispatchMouseEvent" and isinstance(params, dict):
+ try:
+ kind = params.get("type")
+ if kind not in {"mouseMoved", "mousePressed", "mouseWheel"}:
+ return result
+ x, y = float(params["x"]), float(params["y"])
+ if not math.isfinite(x) or not math.isfinite(y):
+ return result
+ x, y, width, height = self.project(x, y)
+ if kind == "mousePressed":
+ self.click_id += 1
+ self.click = (x, y)
+ self.emit(
+ "pointer",
+ visible=True,
+ generation=self.generation,
+ x=x,
+ y=y,
+ width=width,
+ height=height,
+ click_id=self.click_id,
+ click_x=self.click[0],
+ click_y=self.click[1],
+ )
+ except Exception:
+ # Visualization must never turn a successful browser action into failure.
+ log.debug("Browser pointer visualization unavailable", exc_info=True)
+ return result
+
+ return observed
+
+ def clear(self) -> None:
+ self.click_id = 0
+ self.click = (0.0, 0.0)
+ self.emit("pointer", visible=False, generation=self.generation)
diff --git a/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx b/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx
index 2118579b74..c8efe04fdd 100644
--- a/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx
+++ b/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx
@@ -10,6 +10,7 @@ import { BrandedSelect } from "@/components/ui/select";
import type { SocietyAgent } from "../data";
import { useBrowserInstallStatus } from "../cardData";
import { useBrowserView } from "./useBrowserView";
+import { BrowserPointer } from "./BrowserPointer";
import "./agentCard.css";
export function AgentBrowserPreview({ agent }: { agent: SocietyAgent }) {
@@ -91,6 +92,7 @@ export function AgentBrowserPreview({ agent }: { agent: SocietyAgent }) {
control("key", { key: [...modifiers, e.key].join("+") });
}
}} />
+ <BrowserPointer pointer={state.ready && state.connected && !state.manual ? state.pointer : undefined} />
{!state.ready && <div className="absolute inset-0 grid place-items-center bg-muted p-3 text-center text-xs text-muted-foreground">
{install.data?.detail || status}
{install.data?.running && <span>{install.data.percent}%</span>}
diff --git a/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.test.tsx b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.test.tsx
new file mode 100644
index 0000000000..7fa1f2333d
--- /dev/null
+++ b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.test.tsx
@@ -0,0 +1,19 @@
+import { afterEach, expect, test, vi } from "vitest";
+import { cleanup, render, screen } from "@testing-library/react";
+import { BrowserPointer } from "./BrowserPointer";
+
+afterEach(() => { cleanup(); vi.restoreAllMocks(); });
+
+test("cursor uses frame coordinates with letterboxing and preserves a single click pulse", () => {
+ vi.spyOn(HTMLElement.prototype, "clientWidth", "get").mockReturnValue(640);
+ vi.spyOn(HTMLElement.prototype, "clientHeight", "get").mockReturnValue(480);
+ const pointer = { x: 100, y: 200, width: 1280, height: 800, click_id: 1, click_x: 100, click_y: 200 };
+ const { rerender } = render(<BrowserPointer pointer={pointer} />);
+ expect(screen.getByTestId("browser-agent-pointer").style.left).toBe("50px");
+ expect(screen.getByTestId("browser-agent-pointer").style.top).toBe("140px");
+ const click = screen.getByTestId("browser-click-mark");
+ rerender(<BrowserPointer pointer={{ ...pointer, x: 200 }} />);
+ expect(screen.getByTestId("browser-click-mark")).toBe(click);
+ rerender(<BrowserPointer />);
+ expect(screen.queryByTestId("browser-agent-pointer")).toBeNull();
+});
diff --git a/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.tsx b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.tsx
new file mode 100644
index 0000000000..8d6d6a90fc
--- /dev/null
+++ b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.tsx
@@ -0,0 +1,47 @@
+import { useLayoutEffect, useRef, useState } from "react";
+import "./browserPointer.css";
+
+export interface BrowserPointerState {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+ click_id: number;
+ click_x: number;
+ click_y: number;
+}
+
+/** A new vector mark driven by actual agent mouse events, never the OS cursor. */
+export function BrowserPointer({ pointer }: { pointer?: BrowserPointerState }) {
+ const host = useRef<HTMLDivElement>(null);
+ const [size, setSize] = useState({ width: 0, height: 0 });
+ useLayoutEffect(() => {
+ const el = host.current;
+ if (!el) return;
+ const measure = () => setSize({ width: el.clientWidth, height: el.clientHeight });
+ measure();
+ if (typeof ResizeObserver === "undefined") return;
+ const observer = new ResizeObserver(measure);
+ observer.observe(el);
+ return () => observer.disconnect();
+ }, []);
+ const scale = pointer ? Math.min(size.width / pointer.width, size.height / pointer.height) : 0;
+ const point = (x: number, y: number) => ({
+ left: (size.width - pointer!.width * scale) / 2 + x * scale,
+ top: (size.height - pointer!.height * scale) / 2 + y * scale,
+ });
+ return <div ref={host} className="jarvis-browser-pointer-layer" aria-hidden="true" data-testid="browser-pointer-layer">
+ {pointer && scale > 0 && <>
+ {pointer.click_id > 0 && <span key={pointer.click_id} className="jarvis-browser-click"
+ style={point(pointer.click_x, pointer.click_y)} data-testid="browser-click-mark" />}
+ <span className="jarvis-browser-pointer" style={point(pointer.x, pointer.y)} data-testid="browser-agent-pointer">
+ <svg width="23" height="28" viewBox="0 0 23 28" fill="none">
+ <path d="M3 2.5C2.1 1.8 1.3 2.4 1.5 3.5L4.7 23.1C4.9 24.4 6.1 24.7 6.9 23.6L11.1 17.7L18.4 16.7C19.8 16.5 20.2 15.4 19.1 14.5L3 2.5Z"
+ fill="var(--browser-pointer-fill)" stroke="var(--browser-pointer-edge)" strokeWidth="1.6" strokeLinejoin="round" />
+ <path d="M5 6.5L8 17" stroke="white" strokeOpacity=".45" strokeWidth="1.2" strokeLinecap="round" />
+ </svg>
+ <span className="jarvis-browser-pointer-label">Jarvis</span>
+ </span>
+ </>}
+ </div>;
+}
diff --git a/jarvis/ui/web/frontend/src/components/society/card/browserPointer.css b/jarvis/ui/web/frontend/src/components/society/card/browserPointer.css
new file mode 100644
index 0000000000..3236821601
--- /dev/null
+++ b/jarvis/ui/web/frontend/src/components/society/card/browserPointer.css
@@ -0,0 +1,49 @@
+.jarvis-browser-pointer-layer {
+ --browser-pointer-fill: #208678;
+ --browser-pointer-edge: #f7fffc;
+ position: absolute;
+ inset: 0;
+ pointer-events: none;
+ overflow: hidden;
+}
+.dark .jarvis-browser-pointer-layer {
+ --browser-pointer-fill: #4bb9a4;
+ --browser-pointer-edge: #e6fff6;
+}
+.jarvis-browser-pointer {
+ position: absolute;
+ width: 23px;
+ height: 28px;
+ transform: translate(-2px, -3px);
+ filter: drop-shadow(0 2px 3px #0008);
+ transition: left 80ms ease-out, top 80ms ease-out;
+}
+.jarvis-browser-pointer-label {
+ position: absolute;
+ left: 20px;
+ top: 21px;
+ padding: 2px 6px;
+ border-radius: 6px;
+ color: var(--browser-pointer-edge);
+ background: var(--browser-pointer-fill);
+ font: 600 10px/14px system-ui, sans-serif;
+ white-space: nowrap;
+}
+.jarvis-browser-click {
+ position: absolute;
+ width: 24px;
+ height: 24px;
+ margin: -12px;
+ border: 2px solid var(--browser-pointer-fill);
+ border-radius: 50%;
+ box-shadow: 0 0 0 1px var(--browser-pointer-edge);
+ animation: jarvis-browser-click 500ms ease-out forwards;
+}
+@keyframes jarvis-browser-click {
+ from { transform: scale(.45); opacity: .95; }
+ to { transform: scale(1.8); opacity: 0; }
+}
+@media (prefers-reduced-motion: reduce) {
+ .jarvis-browser-pointer { transition: none; }
+ .jarvis-browser-click { animation-duration: 120ms; }
+}
diff --git a/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts b/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts
index 2578af5c01..4cf826bdc6 100644
--- a/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts
+++ b/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts
@@ -1,6 +1,7 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { mintWsTicket } from "@/lib/ws";
import { jitteredDelay, requestConnect } from "@/lib/connectBudget";
+import type { BrowserPointerState } from "./BrowserPointer";
export interface BrowserViewState {
connected: boolean;
@@ -13,6 +14,7 @@ export interface BrowserViewState {
tabs: Array<{ id: string; url: string }>;
target: string;
error: string;
+ pointer?: BrowserPointerState;
approval?: { id: string; action: string };
dialog?: { type: string; message: string };
}
@@ -103,6 +105,7 @@ export function useBrowserView(agentId: string) {
}
if (event.generation !== generation) {
generation = event.generation;
+ setState((s) => ({ ...s, pointer: undefined }));
lastSequence = -1;
epoch++;
}
@@ -112,6 +115,14 @@ export function useBrowserView(agentId: string) {
decodeFrame();
} else if (event.kind === "starting") {
lastLiveEvent = Date.now();
+ } else if (event.kind === "pointer") {
+ if (generation && event.generation !== generation) return;
+ if (!event.visible) {
+ setState((s) => ({ ...s, pointer: undefined }));
+ } else if ([event.x, event.y, event.width, event.height, event.click_id, event.click_x, event.click_y].every(Number.isFinite)
+ && event.width > 0 && event.height > 0) {
+ setState((s) => ({ ...s, pointer: event }));
+ }
} else if (event.kind === "error") {
setState((s) => ({ ...s, ready: false, error: event.error }));
} else if (event.kind === "state") {
@@ -152,7 +163,7 @@ export function useBrowserView(agentId: string) {
manual.current = false;
claiming.current = false;
inputs.current = [];
- setState((s) => ({ ...s, connected: false, manual: false, controlPending: false }));
+ setState((s) => ({ ...s, connected: false, manual: false, controlPending: false, pointer: undefined }));
cancelConnect = requestConnect(() => void connect(), jitteredDelay(attempt++));
};
ws.onerror = () => ws.close();
diff --git a/tests/contract/browser_pointer_probe.py b/tests/contract/browser_pointer_probe.py
new file mode 100644
index 0000000000..6f8ccbdb46
--- /dev/null
+++ b/tests/contract/browser_pointer_probe.py
@@ -0,0 +1,60 @@
+"""Exercise the real Browser-Use CDP hook on a disposable local page."""
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+
+async def main():
+ root = Path(__file__).resolve().parents[2]
+ sys.path.insert(0, str(root / "jarvis/society/browser"))
+ output = sys.stdout
+ import live_runner
+
+ events = []
+ live_runner.emit = lambda kind, **data: events.append({"kind": kind, **data})
+ worker = live_runner.Worker()
+ try:
+ profile = Path(sys.argv[2])
+ await worker.start({"profile_dir": str(profile), "workspace": str(profile / "workspace"),
+ "executable": sys.argv[1], "allowed_domains": []})
+ await worker.page.goto("about:blank")
+ await worker.page.set_content('<button style="width:200px;height:100px" onclick="window.clicked=true">Click probe</button>')
+ await worker.focused()
+ target = next(t for t, p in worker.tabs.items() if p is worker.page)
+ client = worker.browser.cdp_client
+ attached = await client.send.Target.attachToTarget({"targetId": target, "flatten": True})
+ worker.visual_action = True # The real gate enables this only after approval.
+ for kind in ("mouseMoved", "mousePressed", "mouseReleased"):
+ await client.send.Input.dispatchMouseEvent(
+ {"type": kind, "x": 40, "y": 40, "button": "left", "clickCount": 1},
+ session_id=attached["sessionId"],
+ )
+ assert await worker.page.evaluate("window.clicked") is True
+ pointers = [e for e in events if e["kind"] == "pointer"]
+ assert len(pointers) == 2
+ assert pointers[-1]["click_id"] == 1
+ if worker.native:
+ assert pointers[-1]["y"] > 40
+ worker.pointer.clear()
+ assert events[-1]["visible"] is False
+ print(json.dumps({"clicked": True, "pointer_events": len(pointers)}), file=output)
+ finally:
+ worker.closed = True
+ tasks = [t for t in (worker.stream, worker.state_task) if t]
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ if worker.native:
+ worker.native.close()
+ if worker.browser:
+ await worker.browser.stop()
+ if worker.context:
+ await worker.context.close()
+ if worker.playwright:
+ await worker.playwright.stop()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/tests/contract/test_browser_live.py b/tests/contract/test_browser_live.py
index 3428367d86..e0c284cb84 100644
--- a/tests/contract/test_browser_live.py
+++ b/tests/contract/test_browser_live.py
@@ -33,6 +33,21 @@ def test_native_mouse_routes_text_to_the_webpage_widget(tmp_path):
assert '"badge": true' in result.stdout
+def test_actual_browser_click_emits_pointer_telemetry(tmp_path):
+ import subprocess
+ from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+
+ result = subprocess.run(
+ [os.environ["JARVIS_BROWSER_TEST_PYTHON"],
+ str(Path(__file__).with_name("browser_pointer_probe.py")),
+ os.environ["JARVIS_BROWSER_TEST_EXECUTABLE"], str(tmp_path / "profile")],
+ env=install.worker_env(tmp_path), capture_output=True, encoding="utf-8",
+ timeout=40, creationflags=NO_WINDOW_CREATIONFLAGS,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert '"clicked": true' in result.stdout
+
+
class PageHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/download":
@@ -129,6 +144,21 @@ async def test_second_viewer_receives_static_page_without_new_browser(live):
await live.close()
[email protected](os.name != "nt", reason="native window upgrade is Windows-only")
+async def test_viewer_replaces_idle_legacy_page_session(live):
+ agent = SimpleNamespace(agent_id="upgrade", model="", browser_allowed_domains=[])
+ try:
+ old = await live.ensure(agent)
+ old.state["full_window"] = False # Model a pre-native/locked-desktop session.
+ current, _ = await live.subscribe(agent)
+ assert current is not old
+ assert old.closed
+ assert current.state["full_window"] is True
+ assert len(live.sessions) == 1
+ finally:
+ await live.close()
+
+
@pytest.mark.skipif(os.name != "nt", reason="requires the native Windows window transport")
async def test_native_chrome_toolbar_keyboard_and_agent_handoff(live, site):
agent = SimpleNamespace(agent_id="native", model="", browser_allowed_domains=["http*://127.0.0.1"])
diff --git a/tests/unit/society/test_browser_pointer.py b/tests/unit/society/test_browser_pointer.py
new file mode 100644
index 0000000000..4703006322
--- /dev/null
+++ b/tests/unit/society/test_browser_pointer.py
@@ -0,0 +1,76 @@
+"""Pointer telemetry is evidence of successful, approved browser mouse input."""
+
+import pytest
+
+from jarvis.society.browser.pointer import PointerTracker
+
+
+async def test_pointer_tracks_real_click_and_keeps_click_when_mouse_moves():
+ events = []
+
+ async def send(**kwargs):
+ return {"ok": True}
+
+ tracker = PointerTracker("generation", lambda kind, **data: events.append((kind, data)),
+ lambda: True, lambda x, y: (x * 2, y * 2 + 80, 2560, 1680))
+ observed = tracker.wrap(send)
+ await observed("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 20, "y": 30})
+ await observed("Input.dispatchMouseEvent", {"type": "mouseMoved", "x": 40, "y": 50})
+ assert events[-1][1]["x"] == 80
+ assert events[-1][1]["click_id"] == 1
+ assert events[-1][1]["click_x"] == 40
+ assert events[-1][1]["click_y"] == 140
+ tracker.clear()
+ assert events[-1][1]["visible"] is False
+ await observed("Input.dispatchMouseEvent", {"type": "mouseMoved", "x": 40, "y": 50})
+ assert events[-1][1]["click_id"] == 0
+
+
+async def test_pointer_never_exposes_typed_text_or_unapproved_activity():
+ events = []
+
+ async def send(**kwargs):
+ return {}
+
+ enabled = False
+ tracker = PointerTracker("g", lambda *args, **kwargs: events.append(kwargs),
+ lambda: enabled, lambda x, y: (x, y, 1280, 800))
+ observed = tracker.wrap(send)
+ await observed("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 10, "y": 10})
+ enabled = True
+ await observed("Input.insertText", {"text": "private login input"})
+ assert events == []
+
+
+async def test_failed_mouse_command_has_no_click_marker():
+ events = []
+
+ async def send(**kwargs):
+ raise RuntimeError("Disconnected")
+
+ tracker = PointerTracker("g", lambda *args, **kwargs: events.append(kwargs),
+ lambda: True, lambda x, y: (x, y, 1280, 800))
+ with pytest.raises(RuntimeError, match="Disconnected"):
+ await tracker.wrap(send)("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 1, "y": 2})
+ assert not events
+
+
+async def test_native_upgrade_waits_for_the_active_browser_task(tmp_path):
+ import asyncio
+ import os
+ from types import SimpleNamespace
+ from jarvis.society.browser.live import LiveSessions
+
+ if os.name != "nt":
+ pytest.skip("Windows native-session upgrade")
+ live = LiveSessions(tmp_path)
+ old = SimpleNamespace(closed=False, state={"full_window": False},
+ run_lock=asyncio.Lock(), control_owner=None, window_upgrade_pending=False)
+ live.sessions["busy"] = old
+ await old.run_lock.acquire()
+ try:
+ assert await live.ensure(SimpleNamespace(agent_id="busy", browser_mode="own"), window_view=True) is old
+ assert old.window_upgrade_pending
+ assert not old.closed
+ finally:
+ old.run_lock.release()