Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(browser): show first pixels before loading the agent engine
rubenluetke10-beep344be27309d9
diff --git a/jarvis/mcp/jarvis_tools_server.py b/jarvis/mcp/jarvis_tools_server.pyindex 7803ea670e..62ca4b299b 100644--- a/jarvis/mcp/jarvis_tools_server.py+++ b/jarvis/mcp/jarvis_tools_server.py@@ -220,7 +220,7 @@ def build_server() -> Any:return tools@server.call_tool() # type: ignore[misc, no-untyped-call]- async def _call_tool(name: str, arguments: dict[str, Any] | None) -> list[Any]:+ async def _call_tool(name: str, arguments: dict[str, Any] | None) -> Any:from jarvis.core.protocols import SupervisorToolRequest, current_chat_turngateway = _gateway()@@ -267,6 +267,26 @@ def build_server() -> Any:return [types.TextContent(type="text", text=f"Tool failed: {type(exc).__name__}: {exc}")]+ if name == "society_browser":+ success = bool(getattr(result, "success", False))+ text = _render(result)+ if not success:+ # A partial browser run may already have submitted a form.+ # Preserve its observed outcome so a planner can inspect the+ # current page instead of blindly repeating a side effect.+ text = json.dumps(+ {+ "ok": False,+ "error": getattr(result, "error", None),+ "result": getattr(result, "output", None),+ "retry": "Inspect current state before repeating any action.",+ },+ ensure_ascii=False,+ default=str,+ )+ return types.CallToolResult(+ content=[types.TextContent(type="text", text=text)], isError=not success+ )return [types.TextContent(type="text", text=_render(result))]return serverdiff --git a/jarvis/society/browser/live_runner.py b/jarvis/society/browser/live_runner.pyindex b78c8421ed..5e5ca02b56 100644--- a/jarvis/society/browser/live_runner.py+++ b/jarvis/society/browser/live_runner.py@@ -11,6 +11,7 @@ import ipaddressimport jsonimport loggingimport os+import randomimport socketimport sysimport time@@ -73,6 +74,8 @@ class Worker:self.pending: dict[str, asyncio.Future] = {}self.context: Any = Noneself.browser: Any = None+ self.browser_lock = asyncio.Lock()+ self.browser_args: dict[str, Any] = {}self.page: Any = Noneself.playwright: Any = Noneself.agent: Any = None@@ -123,7 +126,6 @@ class Worker:"Chrome needs an unlocked Windows desktop and the managed capture runtime")- from browser_use import Browser # type: ignore[import-not-found]from playwright.async_api import async_playwrightprofile = Path(args["profile_dir"])@@ -137,7 +139,7 @@ class Worker:connection = await self.playwright.chromium.connect_over_cdp(cdp_url)self.context = connection.contexts[0]else:- self.context = await self.playwright.chromium.launch_persistent_context(+ self.context = await self.launch_context(str(profile),executable_path=args["executable"],headless=not native_enabled,@@ -190,24 +192,7 @@ class Worker:# A missing or partially written file is normal during launch.await asyncio.sleep(0.05)cdp_url = f"http://127.0.0.1:{port}"- self.browser = Browser(- cdp_url=cdp_url,- keep_alive=True,- enable_default_extensions=False,- use_cloud=False,- downloads_path=str(self.workspace / "downloads"),- allowed_domains=args.get("allowed_domains") or None,- )- await self.browser.start()- from pointer import PointerTracker # type: ignore[import-not-found]-- self.pointer = PointerTracker(- self.generation,- emit,- lambda: self.visual_action and not self.manual,- lambda x, y: self.native.viewport_point(x, y) if self.native else (x, y, 1280, 800),- )- self.browser.cdp_client.send_raw = self.pointer.wrap(self.browser.cdp_client.send_raw)+ self.browser_args = {"cdp_url": cdp_url, "allowed_domains": args.get("allowed_domains")}self.context.on("page", self.page_opened)for page in self.context.pages:self.page_opened(page)@@ -249,6 +234,54 @@ class Worker:"full_window": bool(self.native),}+ async def launch_context(self, profile: str, **options: Any) -> Any:+ """Chrome may release its profile mutex just after its parent exits."""+ for attempt in range(4):+ try:+ return await self.playwright.chromium.launch_persistent_context(profile, **options)+ except Exception as exc:+ if "ProcessSingleton" not in str(exc) or attempt == 3:+ raise+ logging.getLogger(__name__).debug("Waiting for the managed Chrome profile to close")+ await asyncio.sleep(random.SystemRandom().uniform(0.05, 0.15) * (2**attempt))+ raise RuntimeError("Managed browser profile is still in use")++ async def ensure_browser(self) -> None:+ """Connect the agent engine on demand; idle pixels need only Chromium."""+ async with self.browser_lock:+ if self.browser is not None:+ return++ def load_browser() -> Any:+ from browser_use import Browser # type: ignore[import-not-found]++ return Browser++ browser_class = await asyncio.to_thread(load_browser)+ browser = browser_class(+ cdp_url=self.browser_args["cdp_url"],+ keep_alive=True,+ enable_default_extensions=False,+ use_cloud=False,+ downloads_path=str(self.workspace / "downloads"),+ allowed_domains=self.browser_args.get("allowed_domains") or None,+ )+ try:+ await browser.start()+ except BaseException:+ await browser.stop()+ raise+ from pointer import PointerTracker # type: ignore[import-not-found]++ self.pointer = PointerTracker(+ self.generation,+ emit,+ lambda: self.visual_action and not self.manual,+ lambda x, y: self.native.viewport_point(x, y) if self.native else (x, y, 1280, 800),+ )+ browser.cdp_client.send_raw = self.pointer.wrap(browser.cdp_client.send_raw)+ self.browser = browser+def page_opened(self, page: Any) -> None:page.on("dialog", self.on_dialog)@@ -267,7 +300,7 @@ class Worker:self.tabs[info["targetInfo"]["targetId"]] = pagefinally:await session.detach()- focused = self.browser.get_focused_target()+ focused = self.browser.get_focused_target() if self.browser is not None else Noneif self.native and self.manual:window_title = self.native.title()candidates = []@@ -423,6 +456,7 @@ class Worker:await asyncio.sleep(1 / 15)async def run(self, args: dict) -> dict:+ await self.ensure_browser()self.step_idle.clear()previous_downloads = set(self.browser.downloaded_files)from browser_use import Agent, Tools # type: ignore[import-not-found]@@ -582,6 +616,7 @@ class Worker:self.pointer.clear()else:if self.native and self.manual:+ await self.ensure_browser()from browser_use.browser import events # type: ignore[import-not-found]await self.focused()@@ -625,6 +660,7 @@ class Worker:elif op == "reload":await page.reload()elif op == "tab":+ await self.ensure_browser()from browser_use.browser.events import SwitchTabEvent # type: ignore[import-not-found]if args.get("target") == "new":diff --git a/tests/contract/browser_pointer_probe.py b/tests/contract/browser_pointer_probe.pyindex 6f8ccbdb46..d17755d5bc 100644--- a/tests/contract/browser_pointer_probe.py+++ b/tests/contract/browser_pointer_probe.py@@ -5,10 +5,11 @@ import jsonimport sysfrom pathlib import Path+ROOT = Path(__file__).resolve().parents[2]+async def main():- root = Path(__file__).resolve().parents[2]- sys.path.insert(0, str(root / "jarvis/society/browser"))+ sys.path.insert(0, str(ROOT / "jarvis/society/browser"))output = sys.stdoutimport live_runner@@ -17,10 +18,21 @@ async def main():worker = live_runner.Worker()try:profile = Path(sys.argv[2])- await worker.start({"profile_dir": str(profile), "workspace": str(profile / "workspace"),- "executable": sys.argv[1], "allowed_domains": []})+ await worker.start(+ {+ "profile_dir": str(profile),+ "workspace": str(profile / "workspace"),+ "executable": sys.argv[1],+ "allowed_domains": [],+ }+ )+ assert worker.browser is None # First pixels need no agent engine.+ await worker.ensure_browser()await worker.page.goto("about:blank")- await worker.page.set_content('<button style="width:200px;height:100px" onclick="window.clicked=true">Click probe</button>')+ await worker.page.set_content(+ '<button style="width:200px;height:100px" '+ 'onclick="window.clicked=true">Click probe</button>'+ )await worker.focused()target = next(t for t, p in worker.tabs.items() if p is worker.page)client = worker.browser.cdp_clientdiff --git a/tests/contract/test_browser_desktop_mcp.py b/tests/contract/test_browser_desktop_mcp.pyindex 1acf8c37df..5ff735cb4f 100644--- a/tests/contract/test_browser_desktop_mcp.py+++ b/tests/contract/test_browser_desktop_mcp.py@@ -205,3 +205,36 @@ def test_jarvis_codex_seat_uses_owned_browser_not_inherited_plugins(coding = runner_cli.plan_codex(**args)assert "--ignore-user-config" not in coding.argvassert "browser_use" not in coding.argv+++async def test_failed_browser_mcp_preserves_partial_outcome_and_error_flag(monkeypatch):+ import json++ import mcp.types as types++ from jarvis.mcp import jarvis_tools_server as server++ tool = SimpleNamespace(+ name="society_browser",+ description="Browser",+ risk_tier="monitor",+ is_action_tool=True,+ input_schema={"type": "object"},+ )+ partial = {"ok": False, "urls": ["https://example.org/receipt"], "steps": 3}++ async def execute(name, args, request):+ return ToolResult(False, partial, "Model timed out after navigation")++ gateway = SimpleNamespace(catalog=lambda: [tool], execute=execute)+ monkeypatch.setattr(server, "_gateway", lambda: gateway)+ instance = server.build_server()+ request = types.CallToolRequest(+ params=types.CallToolRequestParams(name="society_browser", arguments={"task": "Read"})+ )+ response = await instance.request_handlers[types.CallToolRequest](request)+ assert response.root.isError+ body = json.loads(response.root.content[0].text)+ assert body["result"] == partial+ assert body["error"] == "Model timed out after navigation"+ assert "Inspect current state" in body["retry"]diff --git a/tests/contract/test_browser_live.py b/tests/contract/test_browser_live.pyindex 03d1db06f5..e8bc96be8b 100644--- a/tests/contract/test_browser_live.py+++ b/tests/contract/test_browser_live.py@@ -65,7 +65,23 @@ def test_actual_browser_click_emits_pointer_telemetry(tmp_path):class PageHandler(BaseHTTPRequestHandler):+ profile_requests = []+def do_GET(self):+ if self.path.startswith(("/login", "/account")):+ signed_in = "fixture_session=verified" in self.headers.get("Cookie", "")+ self.profile_requests.append((self.path, signed_in))+ self.send_response(200)+ self.send_header("Content-Type", "text/html")+ if self.path == "/login":+ self.send_header(+ "Set-Cookie",+ "fixture_session=verified; Path=/; Max-Age=3600; HttpOnly; SameSite=Lax",+ )+ signed_in = True+ self.end_headers()+ self.wfile.write(b"<h1>Signed in</h1>" if signed_in else b"<h1>Sign in required</h1>")+ returnif self.path == "/download":self.send_response(200)self.send_header("Content-Type", "application/octet-stream")@@ -100,6 +116,7 @@ class PageHandler(BaseHTTPRequestHandler):@pytest.fixturedef site():+ PageHandler.profile_requests = []server = ThreadingHTTPServer(("127.0.0.1", 0), PageHandler)thread = threading.Thread(target=server.serve_forever, daemon=True)thread.start()@@ -144,17 +161,25 @@ async def test_live_pixels_change_between_tasks_and_sessions_stay_open(live, sitawait live.close()-async def test_second_viewer_receives_static_page_without_new_browser(live):+async def test_second_viewer_receives_static_page_without_new_browser(live, record_property):+ import time+agent = SimpleNamespace(agent_id="static", model="", browser_allowed_domains=[])try:+ started = time.monotonic()session, first = await live.subscribe(agent)while (await asyncio.wait_for(first.get(), 5))["kind"] != "frame":pass # Drain initial state before attaching the second viewer.+ record_property("cold_first_frame_ms", round((time.monotonic() - started) * 1000, 2))await asyncio.sleep(1)+ started = time.monotonic()same, second = await live.subscribe(agent)assert same is sessionwhile (await asyncio.wait_for(second.get(), 5))["kind"] != "frame":pass # State can precede the first image for a new viewer.+ warm_ms = (time.monotonic() - started) * 1000+ record_property("warm_first_frame_ms", round(warm_ms, 2))+ assert warm_ms <= 2000assert len(live.sessions) == 1assert len(session.subscribers) == 2finally:@@ -608,3 +633,30 @@ async def test_finished_chat_releases_real_browser_for_the_next_task(live, monkejob.cancel()await asyncio.gather(job, return_exceptions=True)await live.close()+++async def test_login_profile_survives_restart_and_stays_with_its_agent(live, site):+ first = SimpleNamespace(+ agent_id="signed-in", model="", browser_allowed_domains=["http*://127.0.0.1"]+ )+ other = SimpleNamespace(+ agent_id="isolated", model="", browser_allowed_domains=["http*://127.0.0.1"]+ )++ async def visit(agent, path):+ session, _ = await live.subscribe(agent)+ await live.control(session, "viewer", "takeover", {"enabled": True})+ await live.control(session, "viewer", "navigate", {"url": site + path})+ await live.control(session, "viewer", "takeover", {"enabled": False})+ return session++ try:+ await visit(first, "/login")+ await live.close()+ restored = await visit(first, "/account?restored")+ isolated = await visit(other, "/account?other")+ assert restored is not isolated+ assert ("/account?restored", True) in PageHandler.profile_requests+ assert ("/account?other", False) in PageHandler.profile_requests+ finally:+ await live.close()