Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(browser): visualize real agent clicks and restore native views
rubenluetke10-beep09092bc4c46f
diff --git a/jarvis.spec b/jarvis.specindex c65981e465..14f158fb09 100644--- a/jarvis.spec+++ b/jarvis.spec@@ -71,6 +71,7 @@ VERSION = _package_version()datas = []datas.append((str(PROJECT_ROOT / "jarvis/society/browser/live_runner.py"), "jarvis/society/browser"))datas.append((str(PROJECT_ROOT / "jarvis/society/browser/native_window.py"), "jarvis/society/browser"))+datas.append((str(PROJECT_ROOT / "jarvis/society/browser/pointer.py"), "jarvis/society/browser"))# Include the frontend build when present. Preserve its package-relative layout# so the FastAPI static-files mount can serve it from a frozen application.diff --git a/jarvis/society/browser/live.py b/jarvis/society/browser/live.pyindex 7502d1ec38..298a977f12 100644--- a/jarvis/society/browser/live.py+++ b/jarvis/society/browser/live.py@@ -76,6 +76,7 @@ class LiveSession:generation: str = ""closed: bool = Falsestderr_tail: str = ""+ window_upgrade_pending: bool = Falseasync def send(self, value: dict[str, Any]) -> None:data = (json.dumps(value, ensure_ascii=True) + "\n").encode()@@ -166,7 +167,7 @@ class LiveSession:elif kind == "state":self.state = eventself.publish(event)- elif kind in {"frame", "dialog", "download", "warning", "step"}:+ elif kind in {"frame", "pointer", "dialog", "download", "warning", "step"}:self.publish(event)elif kind == "fatal":raise RuntimeError(event.get("error", "Browser worker failed"))@@ -259,7 +260,7 @@ class LiveSessions:self.idle_tasks[session.agent_id] = asyncio.create_task(expire())- async def ensure(self, agent: Any) -> LiveSession:+ async def ensure(self, agent: Any, *, window_view: bool = False) -> LiveSession:agent_id = agent.agent_ididle = self.idle_tasks.pop(agent_id, None)if idle:@@ -267,7 +268,17 @@ class LiveSessions:async with self.locks.setdefault(agent_id, asyncio.Lock()):old = self.sessions.get(agent_id)if old and not old.closed:- return old+ upgrade = (+ window_view+ and os.name == "nt"+ and getattr(agent, "browser_mode", "own") == "own"+ and not old.state.get("full_window", False)+ )+ if not upgrade:+ return old+ if old.run_lock.locked() or old.control_owner:+ old.window_upgrade_pending = True+ return oldif old:await old.close()if not install.is_installed(self.data_dir):@@ -297,6 +308,7 @@ class LiveSessions:"ensure",{"profile_dir": str(folder / "browser-profile"),+ "window_view": window_view,"workspace": str(folder / "workspace"),"executable": str(install.browser_executable(self.data_dir)),"icon_path": str(@@ -310,6 +322,16 @@ class LiveSessions:timeout=90,)session.generation = result["generation"]+ session.state = {+ "kind": "state",+ "manual": False,+ "running": False,+ "url": "",+ "target": "",+ "tabs": [],+ **session.state,+ "full_window": bool(result.get("full_window")),+ }self.sessions[agent_id] = sessionself.release_when_idle(session)return session@@ -318,7 +340,7 @@ class LiveSessions:raiseasync def subscribe(self, agent: Any) -> tuple[LiveSession, LiveUpdates]:- session = await self.ensure(agent)+ session = await self.ensure(agent, window_view=True)queue = LiveUpdates()session.subscribers.add(queue)try:@@ -338,6 +360,8 @@ class LiveSessions:if session.control_owner == owner:await session.command("takeover", {"enabled": False})session.control_owner = None+ if session.window_upgrade_pending:+ session.publish({"kind": "disconnected"})if not session.subscribers:await session.command("subscribe", {"enabled": False})self.release_when_idle(session)@@ -361,6 +385,8 @@ class LiveSessions:session.publish({"kind": "control", "ok": True, "manual": False})raisesession.control_owner = owner if args.get("enabled") else None+ if not session.control_owner and session.window_upgrade_pending:+ session.publish({"kind": "disconnected"})session.publish({"kind": "control", "ok": True, **result})return resultif op != "cancel" and session.control_owner != owner:@@ -417,6 +443,8 @@ class LiveSessions:session.rpc_context = Nonesession.active_trace = ""session.active_chat = ""+ if session.window_upgrade_pending:+ session.publish({"kind": "disconnected"})if not session.subscribers:self.release_when_idle(session)diff --git a/jarvis/society/browser/live_runner.py b/jarvis/society/browser/live_runner.pyindex 89a56276f5..b78c8421ed 100644--- a/jarvis/society/browser/live_runner.py+++ b/jarvis/society/browser/live_runner.py@@ -101,6 +101,8 @@ class Worker:self.native_frame_at = 0.0self.native_replay = Falseself.branding: asyncio.Task | None = None+ self.pointer: Any = None+ self.visual_action = Falseasync def rpc(self, kind: str, payload: dict) -> dict:key = uuid.uuid4().hex@@ -116,6 +118,10 @@ class Worker:from native_window import NativeWindow, available # type: ignore[import-not-found]native_enabled = available()+ if args.get("window_view") and sys.platform == "win32" and not native_enabled:+ raise RuntimeError(+ "Chrome needs an unlocked Windows desktop and the managed capture runtime"+ )from browser_use import Browser # type: ignore[import-not-found]from playwright.async_api import async_playwright@@ -193,6 +199,15 @@ class Worker:allowed_domains=args.get("allowed_domains") or None,)await self.browser.start()+ from pointer import PointerTracker # type: ignore[import-not-found]++ self.pointer = PointerTracker(+ self.generation,+ emit,+ lambda: self.visual_action and not self.manual,+ lambda x, y: self.native.viewport_point(x, y) if self.native else (x, y, 1280, 800),+ )+ self.browser.cdp_client.send_raw = self.pointer.wrap(self.browser.cdp_client.send_raw)self.context.on("page", self.page_opened)for page in self.context.pages:self.page_opened(page)@@ -228,7 +243,11 @@ class Worker:)self.state_task = asyncio.create_task(self.watch_state())self.stream = asyncio.create_task(self.watch())- return {"generation": self.generation, "protocol": PROTOCOL_VERSION}+ return {+ "generation": self.generation,+ "protocol": PROTOCOL_VERSION,+ "full_window": bool(self.native),+ }def page_opened(self, page: Any) -> None:page.on("dialog", self.on_dialog)@@ -458,11 +477,14 @@ class Worker:)if not answer.get("ok"):raise asyncio.CancelledError(answer.get("error", "Browser action denied"))+ worker.visual_action = Truetry:result = await super().act(action, browser_session, *pos, **kw)except Exception as exc:# The result is emitted below and becomes the executor's visible failure.result = ActionResult(error=f"{type(exc).__name__}: browser action failed")+ finally:+ worker.visual_action = Falseemit("action_result", id=answer["permit"], result=result.model_dump(mode="json"))return result@@ -513,6 +535,8 @@ class Worker:}finally:self.agent = None+ if self.pointer:+ self.pointer.clear()self.step_idle.set()async def command(self, op: str, args: dict) -> dict:@@ -554,6 +578,8 @@ class Worker:if generation != self.takeover_generation:return {"manual": self.manual}self.manual = True+ if self.pointer:+ self.pointer.clear()else:if self.native and self.manual:from browser_use.browser import events # type: ignore[import-not-found]diff --git a/jarvis/society/browser/native_window.py b/jarvis/society/browser/native_window.pyindex 13269ca4e7..c54c04c3a6 100644--- a/jarvis/society/browser/native_window.py+++ b/jarvis/society/browser/native_window.py@@ -266,6 +266,46 @@ class NativeWindow:target = int(child)raise RuntimeError("Chrome input widget nesting is invalid")+ def viewport_point(self, x: float, y: float) -> tuple[float, float, int, int]:+ """Project CDP viewport coordinates into the complete captured Chrome frame."""+ c, w, u = self.ctypes, self.wintypes, self.user32+ self._check_owner()+ u.EnumChildWindows.argtypes = [w.HWND, self.callback_type, w.LPARAM]+ u.EnumChildWindows.restype = w.BOOL+ children = []++ @self.callback_type+ def visit(hwnd: int, _unused: int) -> bool:+ name = c.create_unicode_buffer(128)+ u.GetClassNameW(hwnd, name, len(name))+ if name.value == "Chrome_RenderWidgetHostHWND" and u.IsWindowVisible(hwnd):+ rect = w.RECT()+ if u.GetWindowRect(hwnd, c.byref(rect)):+ children.append((rect.right - rect.left, rect.left, rect.top))+ return True++ with self.dpi():+ u.EnumChildWindows(self.hwnd, visit, 0)+ if not children:+ raise RuntimeError("Chrome viewport is unavailable")+ width, left, top = max(children)+ bounds = w.RECT()+ dwm = c.WinDLL("dwmapi")+ dwm.DwmGetWindowAttribute.argtypes = [w.HWND, w.DWORD, c.c_void_p, w.DWORD]+ dwm.DwmGetWindowAttribute.restype = c.c_long+ if dwm.DwmGetWindowAttribute(self.hwnd, 9, c.byref(bounds), c.sizeof(bounds)):+ raise RuntimeError("Chrome frame geometry is unavailable")+ frame = self.frame()+ if not frame:+ raise RuntimeError("Chrome frame is unavailable")+ scale = width / 1280+ return (+ left - bounds.left + x * scale,+ top - bounds.top + y * scale,+ frame["width"],+ frame["height"],+ )+def shortcut(self, key: int, modifiers: list[int]) -> None:"""Apply modifiers to Chrome's input queue, never to the physical keyboard."""c, w, u = self.ctypes, self.wintypes, self.user32diff --git a/jarvis/society/browser/pointer.py b/jarvis/society/browser/pointer.pynew file mode 100644index 0000000000..e2ad7315ee--- /dev/null+++ b/jarvis/society/browser/pointer.py@@ -0,0 +1,58 @@+"""Read-only visualization of mouse commands successfully sent to Chrome."""++from __future__ import annotations++import logging+import math+from typing import Any++log = logging.getLogger(__name__)+++class PointerTracker:+ def __init__(self, generation: str, emit: Any, enabled: Any, project: Any) -> None:+ self.generation = generation+ self.emit = emit+ self.enabled = enabled+ self.project = project+ self.click_id = 0+ self.click = (0.0, 0.0)++ def wrap(self, send: Any) -> Any:+ async def observed(method: str, params: Any = None, session_id: str | None = None) -> Any:+ result = await send(method=method, params=params, session_id=session_id)+ if self.enabled() and method == "Input.dispatchMouseEvent" and isinstance(params, dict):+ try:+ kind = params.get("type")+ if kind not in {"mouseMoved", "mousePressed", "mouseWheel"}:+ return result+ x, y = float(params["x"]), float(params["y"])+ if not math.isfinite(x) or not math.isfinite(y):+ return result+ x, y, width, height = self.project(x, y)+ if kind == "mousePressed":+ self.click_id += 1+ self.click = (x, y)+ self.emit(+ "pointer",+ visible=True,+ generation=self.generation,+ x=x,+ y=y,+ width=width,+ height=height,+ click_id=self.click_id,+ click_x=self.click[0],+ click_y=self.click[1],+ )+ except Exception:+ # Visualization must never turn a successful browser action into failure.+ log.debug("Browser pointer visualization unavailable", exc_info=True)+ return result++ return observed++ def clear(self) -> None:+ self.click_id = 0+ self.click = (0.0, 0.0)+ self.emit("pointer", visible=False, generation=self.generation)diff --git a/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx b/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsxindex 2118579b74..c8efe04fdd 100644--- a/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx+++ b/jarvis/ui/web/frontend/src/components/society/card/AgentBrowserPreview.tsx@@ -10,6 +10,7 @@ import { BrandedSelect } from "@/components/ui/select";import type { SocietyAgent } from "../data";import { useBrowserInstallStatus } from "../cardData";import { useBrowserView } from "./useBrowserView";+import { BrowserPointer } from "./BrowserPointer";import "./agentCard.css";export function AgentBrowserPreview({ agent }: { agent: SocietyAgent }) {@@ -91,6 +92,7 @@ export function AgentBrowserPreview({ agent }: { agent: SocietyAgent }) {control("key", { key: [...modifiers, e.key].join("+") });}}} />+ <BrowserPointer pointer={state.ready && state.connected && !state.manual ? state.pointer : undefined} />{!state.ready && <div className="absolute inset-0 grid place-items-center bg-muted p-3 text-center text-xs text-muted-foreground">{install.data?.detail || status}{install.data?.running && <span>{install.data.percent}%</span>}diff --git a/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.test.tsx b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.test.tsxnew file mode 100644index 0000000000..7fa1f2333d--- /dev/null+++ b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.test.tsx@@ -0,0 +1,19 @@+import { afterEach, expect, test, vi } from "vitest";+import { cleanup, render, screen } from "@testing-library/react";+import { BrowserPointer } from "./BrowserPointer";++afterEach(() => { cleanup(); vi.restoreAllMocks(); });++test("cursor uses frame coordinates with letterboxing and preserves a single click pulse", () => {+ vi.spyOn(HTMLElement.prototype, "clientWidth", "get").mockReturnValue(640);+ vi.spyOn(HTMLElement.prototype, "clientHeight", "get").mockReturnValue(480);+ const pointer = { x: 100, y: 200, width: 1280, height: 800, click_id: 1, click_x: 100, click_y: 200 };+ const { rerender } = render(<BrowserPointer pointer={pointer} />);+ expect(screen.getByTestId("browser-agent-pointer").style.left).toBe("50px");+ expect(screen.getByTestId("browser-agent-pointer").style.top).toBe("140px");+ const click = screen.getByTestId("browser-click-mark");+ rerender(<BrowserPointer pointer={{ ...pointer, x: 200 }} />);+ expect(screen.getByTestId("browser-click-mark")).toBe(click);+ rerender(<BrowserPointer />);+ expect(screen.queryByTestId("browser-agent-pointer")).toBeNull();+});diff --git a/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.tsx b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.tsxnew file mode 100644index 0000000000..8d6d6a90fc--- /dev/null+++ b/jarvis/ui/web/frontend/src/components/society/card/BrowserPointer.tsx@@ -0,0 +1,47 @@+import { useLayoutEffect, useRef, useState } from "react";+import "./browserPointer.css";++export interface BrowserPointerState {+ x: number;+ y: number;+ width: number;+ height: number;+ click_id: number;+ click_x: number;+ click_y: number;+}++/** A new vector mark driven by actual agent mouse events, never the OS cursor. */+export function BrowserPointer({ pointer }: { pointer?: BrowserPointerState }) {+ const host = useRef<HTMLDivElement>(null);+ const [size, setSize] = useState({ width: 0, height: 0 });+ useLayoutEffect(() => {+ const el = host.current;+ if (!el) return;+ const measure = () => setSize({ width: el.clientWidth, height: el.clientHeight });+ measure();+ if (typeof ResizeObserver === "undefined") return;+ const observer = new ResizeObserver(measure);+ observer.observe(el);+ return () => observer.disconnect();+ }, []);+ const scale = pointer ? Math.min(size.width / pointer.width, size.height / pointer.height) : 0;+ const point = (x: number, y: number) => ({+ left: (size.width - pointer!.width * scale) / 2 + x * scale,+ top: (size.height - pointer!.height * scale) / 2 + y * scale,+ });+ return <div ref={host} className="jarvis-browser-pointer-layer" aria-hidden="true" data-testid="browser-pointer-layer">+ {pointer && scale > 0 && <>+ {pointer.click_id > 0 && <span key={pointer.click_id} className="jarvis-browser-click"+ style={point(pointer.click_x, pointer.click_y)} data-testid="browser-click-mark" />}+ <span className="jarvis-browser-pointer" style={point(pointer.x, pointer.y)} data-testid="browser-agent-pointer">+ <svg width="23" height="28" viewBox="0 0 23 28" fill="none">+ <path d="M3 2.5C2.1 1.8 1.3 2.4 1.5 3.5L4.7 23.1C4.9 24.4 6.1 24.7 6.9 23.6L11.1 17.7L18.4 16.7C19.8 16.5 20.2 15.4 19.1 14.5L3 2.5Z"+ fill="var(--browser-pointer-fill)" stroke="var(--browser-pointer-edge)" strokeWidth="1.6" strokeLinejoin="round" />+ <path d="M5 6.5L8 17" stroke="white" strokeOpacity=".45" strokeWidth="1.2" strokeLinecap="round" />+ </svg>+ <span className="jarvis-browser-pointer-label">Jarvis</span>+ </span>+ </>}+ </div>;+}diff --git a/jarvis/ui/web/frontend/src/components/society/card/browserPointer.css b/jarvis/ui/web/frontend/src/components/society/card/browserPointer.cssnew file mode 100644index 0000000000..3236821601--- /dev/null+++ b/jarvis/ui/web/frontend/src/components/society/card/browserPointer.css@@ -0,0 +1,49 @@+.jarvis-browser-pointer-layer {+ --browser-pointer-fill: #208678;+ --browser-pointer-edge: #f7fffc;+ position: absolute;+ inset: 0;+ pointer-events: none;+ overflow: hidden;+}+.dark .jarvis-browser-pointer-layer {+ --browser-pointer-fill: #4bb9a4;+ --browser-pointer-edge: #e6fff6;+}+.jarvis-browser-pointer {+ position: absolute;+ width: 23px;+ height: 28px;+ transform: translate(-2px, -3px);+ filter: drop-shadow(0 2px 3px #0008);+ transition: left 80ms ease-out, top 80ms ease-out;+}+.jarvis-browser-pointer-label {+ position: absolute;+ left: 20px;+ top: 21px;+ padding: 2px 6px;+ border-radius: 6px;+ color: var(--browser-pointer-edge);+ background: var(--browser-pointer-fill);+ font: 600 10px/14px system-ui, sans-serif;+ white-space: nowrap;+}+.jarvis-browser-click {+ position: absolute;+ width: 24px;+ height: 24px;+ margin: -12px;+ border: 2px solid var(--browser-pointer-fill);+ border-radius: 50%;+ box-shadow: 0 0 0 1px var(--browser-pointer-edge);+ animation: jarvis-browser-click 500ms ease-out forwards;+}+@keyframes jarvis-browser-click {+ from { transform: scale(.45); opacity: .95; }+ to { transform: scale(1.8); opacity: 0; }+}+@media (prefers-reduced-motion: reduce) {+ .jarvis-browser-pointer { transition: none; }+ .jarvis-browser-click { animation-duration: 120ms; }+}diff --git a/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts b/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.tsindex 2578af5c01..4cf826bdc6 100644--- a/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts+++ b/jarvis/ui/web/frontend/src/components/society/card/useBrowserView.ts@@ -1,6 +1,7 @@import { useCallback, useEffect, useRef, useState } from "react";import { mintWsTicket } from "@/lib/ws";import { jitteredDelay, requestConnect } from "@/lib/connectBudget";+import type { BrowserPointerState } from "./BrowserPointer";export interface BrowserViewState {connected: boolean;@@ -13,6 +14,7 @@ export interface BrowserViewState {tabs: Array<{ id: string; url: string }>;target: string;error: string;+ pointer?: BrowserPointerState;approval?: { id: string; action: string };dialog?: { type: string; message: string };}@@ -103,6 +105,7 @@ export function useBrowserView(agentId: string) {}if (event.generation !== generation) {generation = event.generation;+ setState((s) => ({ ...s, pointer: undefined }));lastSequence = -1;epoch++;}@@ -112,6 +115,14 @@ export function useBrowserView(agentId: string) {decodeFrame();} else if (event.kind === "starting") {lastLiveEvent = Date.now();+ } else if (event.kind === "pointer") {+ if (generation && event.generation !== generation) return;+ if (!event.visible) {+ setState((s) => ({ ...s, pointer: undefined }));+ } else if ([event.x, event.y, event.width, event.height, event.click_id, event.click_x, event.click_y].every(Number.isFinite)+ && event.width > 0 && event.height > 0) {+ setState((s) => ({ ...s, pointer: event }));+ }} else if (event.kind === "error") {setState((s) => ({ ...s, ready: false, error: event.error }));} else if (event.kind === "state") {@@ -152,7 +163,7 @@ export function useBrowserView(agentId: string) {manual.current = false;claiming.current = false;inputs.current = [];- setState((s) => ({ ...s, connected: false, manual: false, controlPending: false }));+ setState((s) => ({ ...s, connected: false, manual: false, controlPending: false, pointer: undefined }));cancelConnect = requestConnect(() => void connect(), jitteredDelay(attempt++));};ws.onerror = () => ws.close();diff --git a/tests/contract/browser_pointer_probe.py b/tests/contract/browser_pointer_probe.pynew file mode 100644index 0000000000..6f8ccbdb46--- /dev/null+++ b/tests/contract/browser_pointer_probe.py@@ -0,0 +1,60 @@+"""Exercise the real Browser-Use CDP hook on a disposable local page."""++import asyncio+import json+import sys+from pathlib import Path+++async def main():+ root = Path(__file__).resolve().parents[2]+ sys.path.insert(0, str(root / "jarvis/society/browser"))+ output = sys.stdout+ import live_runner++ events = []+ live_runner.emit = lambda kind, **data: events.append({"kind": kind, **data})+ worker = live_runner.Worker()+ try:+ profile = Path(sys.argv[2])+ await worker.start({"profile_dir": str(profile), "workspace": str(profile / "workspace"),+ "executable": sys.argv[1], "allowed_domains": []})+ await worker.page.goto("about:blank")+ await worker.page.set_content('<button style="width:200px;height:100px" onclick="window.clicked=true">Click probe</button>')+ await worker.focused()+ target = next(t for t, p in worker.tabs.items() if p is worker.page)+ client = worker.browser.cdp_client+ attached = await client.send.Target.attachToTarget({"targetId": target, "flatten": True})+ worker.visual_action = True # The real gate enables this only after approval.+ for kind in ("mouseMoved", "mousePressed", "mouseReleased"):+ await client.send.Input.dispatchMouseEvent(+ {"type": kind, "x": 40, "y": 40, "button": "left", "clickCount": 1},+ session_id=attached["sessionId"],+ )+ assert await worker.page.evaluate("window.clicked") is True+ pointers = [e for e in events if e["kind"] == "pointer"]+ assert len(pointers) == 2+ assert pointers[-1]["click_id"] == 1+ if worker.native:+ assert pointers[-1]["y"] > 40+ worker.pointer.clear()+ assert events[-1]["visible"] is False+ print(json.dumps({"clicked": True, "pointer_events": len(pointers)}), file=output)+ finally:+ worker.closed = True+ tasks = [t for t in (worker.stream, worker.state_task) if t]+ for task in tasks:+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)+ if worker.native:+ worker.native.close()+ if worker.browser:+ await worker.browser.stop()+ if worker.context:+ await worker.context.close()+ if worker.playwright:+ await worker.playwright.stop()+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/tests/contract/test_browser_live.py b/tests/contract/test_browser_live.pyindex 3428367d86..e0c284cb84 100644--- a/tests/contract/test_browser_live.py+++ b/tests/contract/test_browser_live.py@@ -33,6 +33,21 @@ def test_native_mouse_routes_text_to_the_webpage_widget(tmp_path):assert '"badge": true' in result.stdout+def test_actual_browser_click_emits_pointer_telemetry(tmp_path):+ import subprocess+ from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS++ result = subprocess.run(+ [os.environ["JARVIS_BROWSER_TEST_PYTHON"],+ str(Path(__file__).with_name("browser_pointer_probe.py")),+ os.environ["JARVIS_BROWSER_TEST_EXECUTABLE"], str(tmp_path / "profile")],+ env=install.worker_env(tmp_path), capture_output=True, encoding="utf-8",+ timeout=40, creationflags=NO_WINDOW_CREATIONFLAGS,+ )+ assert result.returncode == 0, result.stdout + result.stderr+ assert '"clicked": true' in result.stdout++class PageHandler(BaseHTTPRequestHandler):def do_GET(self):if self.path == "/download":@@ -129,6 +144,21 @@ async def test_second_viewer_receives_static_page_without_new_browser(live):await live.close()[email protected](os.name != "nt", reason="native window upgrade is Windows-only")+async def test_viewer_replaces_idle_legacy_page_session(live):+ agent = SimpleNamespace(agent_id="upgrade", model="", browser_allowed_domains=[])+ try:+ old = await live.ensure(agent)+ old.state["full_window"] = False # Model a pre-native/locked-desktop session.+ current, _ = await live.subscribe(agent)+ assert current is not old+ assert old.closed+ assert current.state["full_window"] is True+ assert len(live.sessions) == 1+ finally:+ await live.close()++@pytest.mark.skipif(os.name != "nt", reason="requires the native Windows window transport")async def test_native_chrome_toolbar_keyboard_and_agent_handoff(live, site):agent = SimpleNamespace(agent_id="native", model="", browser_allowed_domains=["http*://127.0.0.1"])diff --git a/tests/unit/society/test_browser_pointer.py b/tests/unit/society/test_browser_pointer.pynew file mode 100644index 0000000000..4703006322--- /dev/null+++ b/tests/unit/society/test_browser_pointer.py@@ -0,0 +1,76 @@+"""Pointer telemetry is evidence of successful, approved browser mouse input."""++import pytest++from jarvis.society.browser.pointer import PointerTracker+++async def test_pointer_tracks_real_click_and_keeps_click_when_mouse_moves():+ events = []++ async def send(**kwargs):+ return {"ok": True}++ tracker = PointerTracker("generation", lambda kind, **data: events.append((kind, data)),+ lambda: True, lambda x, y: (x * 2, y * 2 + 80, 2560, 1680))+ observed = tracker.wrap(send)+ await observed("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 20, "y": 30})+ await observed("Input.dispatchMouseEvent", {"type": "mouseMoved", "x": 40, "y": 50})+ assert events[-1][1]["x"] == 80+ assert events[-1][1]["click_id"] == 1+ assert events[-1][1]["click_x"] == 40+ assert events[-1][1]["click_y"] == 140+ tracker.clear()+ assert events[-1][1]["visible"] is False+ await observed("Input.dispatchMouseEvent", {"type": "mouseMoved", "x": 40, "y": 50})+ assert events[-1][1]["click_id"] == 0+++async def test_pointer_never_exposes_typed_text_or_unapproved_activity():+ events = []++ async def send(**kwargs):+ return {}++ enabled = False+ tracker = PointerTracker("g", lambda *args, **kwargs: events.append(kwargs),+ lambda: enabled, lambda x, y: (x, y, 1280, 800))+ observed = tracker.wrap(send)+ await observed("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 10, "y": 10})+ enabled = True+ await observed("Input.insertText", {"text": "private login input"})+ assert events == []+++async def test_failed_mouse_command_has_no_click_marker():+ events = []++ async def send(**kwargs):+ raise RuntimeError("Disconnected")++ tracker = PointerTracker("g", lambda *args, **kwargs: events.append(kwargs),+ lambda: True, lambda x, y: (x, y, 1280, 800))+ with pytest.raises(RuntimeError, match="Disconnected"):+ await tracker.wrap(send)("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 1, "y": 2})+ assert not events+++async def test_native_upgrade_waits_for_the_active_browser_task(tmp_path):+ import asyncio+ import os+ from types import SimpleNamespace+ from jarvis.society.browser.live import LiveSessions++ if os.name != "nt":+ pytest.skip("Windows native-session upgrade")+ live = LiveSessions(tmp_path)+ old = SimpleNamespace(closed=False, state={"full_window": False},+ run_lock=asyncio.Lock(), control_owner=None, window_upgrade_pending=False)+ live.sessions["busy"] = old+ await old.run_lock.acquire()+ try:+ assert await live.ensure(SimpleNamespace(agent_id="busy", browser_mode="own"), window_view=True) is old+ assert old.window_upgrade_pending+ assert not old.closed+ finally:+ old.run_lock.release()