Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(browser): show first pixels before loading the agent engine

rubenluetke10-beep344be27309d9
diff --git a/jarvis/mcp/jarvis_tools_server.py b/jarvis/mcp/jarvis_tools_server.py
index 7803ea670e..62ca4b299b 100644
--- a/jarvis/mcp/jarvis_tools_server.py
+++ b/jarvis/mcp/jarvis_tools_server.py
@@ -220,7 +220,7 @@ def build_server() -> Any:
return tools
@server.call_tool() # type: ignore[misc, no-untyped-call]
- async def _call_tool(name: str, arguments: dict[str, Any] | None) -> list[Any]:
+ async def _call_tool(name: str, arguments: dict[str, Any] | None) -> Any:
from jarvis.core.protocols import SupervisorToolRequest, current_chat_turn
gateway = _gateway()
@@ -267,6 +267,26 @@ def build_server() -> Any:
return [
types.TextContent(type="text", text=f"Tool failed: {type(exc).__name__}: {exc}")
]
+ if name == "society_browser":
+ success = bool(getattr(result, "success", False))
+ text = _render(result)
+ if not success:
+ # A partial browser run may already have submitted a form.
+ # Preserve its observed outcome so a planner can inspect the
+ # current page instead of blindly repeating a side effect.
+ text = json.dumps(
+ {
+ "ok": False,
+ "error": getattr(result, "error", None),
+ "result": getattr(result, "output", None),
+ "retry": "Inspect current state before repeating any action.",
+ },
+ ensure_ascii=False,
+ default=str,
+ )
+ return types.CallToolResult(
+ content=[types.TextContent(type="text", text=text)], isError=not success
+ )
return [types.TextContent(type="text", text=_render(result))]
return server
diff --git a/jarvis/society/browser/live_runner.py b/jarvis/society/browser/live_runner.py
index b78c8421ed..5e5ca02b56 100644
--- a/jarvis/society/browser/live_runner.py
+++ b/jarvis/society/browser/live_runner.py
@@ -11,6 +11,7 @@ import ipaddress
import json
import logging
import os
+import random
import socket
import sys
import time
@@ -73,6 +74,8 @@ class Worker:
self.pending: dict[str, asyncio.Future] = {}
self.context: Any = None
self.browser: Any = None
+ self.browser_lock = asyncio.Lock()
+ self.browser_args: dict[str, Any] = {}
self.page: Any = None
self.playwright: Any = None
self.agent: Any = None
@@ -123,7 +126,6 @@ class Worker:
"Chrome needs an unlocked Windows desktop and the managed capture runtime"
)
- from browser_use import Browser # type: ignore[import-not-found]
from playwright.async_api import async_playwright
profile = Path(args["profile_dir"])
@@ -137,7 +139,7 @@ class Worker:
connection = await self.playwright.chromium.connect_over_cdp(cdp_url)
self.context = connection.contexts[0]
else:
- self.context = await self.playwright.chromium.launch_persistent_context(
+ self.context = await self.launch_context(
str(profile),
executable_path=args["executable"],
headless=not native_enabled,
@@ -190,24 +192,7 @@ class Worker:
# A missing or partially written file is normal during launch.
await asyncio.sleep(0.05)
cdp_url = f"http://127.0.0.1:{port}"
- self.browser = Browser(
- cdp_url=cdp_url,
- keep_alive=True,
- enable_default_extensions=False,
- use_cloud=False,
- downloads_path=str(self.workspace / "downloads"),
- allowed_domains=args.get("allowed_domains") or None,
- )
- await self.browser.start()
- from pointer import PointerTracker # type: ignore[import-not-found]
-
- self.pointer = PointerTracker(
- self.generation,
- emit,
- lambda: self.visual_action and not self.manual,
- lambda x, y: self.native.viewport_point(x, y) if self.native else (x, y, 1280, 800),
- )
- self.browser.cdp_client.send_raw = self.pointer.wrap(self.browser.cdp_client.send_raw)
+ self.browser_args = {"cdp_url": cdp_url, "allowed_domains": args.get("allowed_domains")}
self.context.on("page", self.page_opened)
for page in self.context.pages:
self.page_opened(page)
@@ -249,6 +234,54 @@ class Worker:
"full_window": bool(self.native),
}
+ async def launch_context(self, profile: str, **options: Any) -> Any:
+ """Chrome may release its profile mutex just after its parent exits."""
+ for attempt in range(4):
+ try:
+ return await self.playwright.chromium.launch_persistent_context(profile, **options)
+ except Exception as exc:
+ if "ProcessSingleton" not in str(exc) or attempt == 3:
+ raise
+ logging.getLogger(__name__).debug("Waiting for the managed Chrome profile to close")
+ await asyncio.sleep(random.SystemRandom().uniform(0.05, 0.15) * (2**attempt))
+ raise RuntimeError("Managed browser profile is still in use")
+
+ async def ensure_browser(self) -> None:
+ """Connect the agent engine on demand; idle pixels need only Chromium."""
+ async with self.browser_lock:
+ if self.browser is not None:
+ return
+
+ def load_browser() -> Any:
+ from browser_use import Browser # type: ignore[import-not-found]
+
+ return Browser
+
+ browser_class = await asyncio.to_thread(load_browser)
+ browser = browser_class(
+ cdp_url=self.browser_args["cdp_url"],
+ keep_alive=True,
+ enable_default_extensions=False,
+ use_cloud=False,
+ downloads_path=str(self.workspace / "downloads"),
+ allowed_domains=self.browser_args.get("allowed_domains") or None,
+ )
+ try:
+ await browser.start()
+ except BaseException:
+ await browser.stop()
+ raise
+ from pointer import PointerTracker # type: ignore[import-not-found]
+
+ self.pointer = PointerTracker(
+ self.generation,
+ emit,
+ lambda: self.visual_action and not self.manual,
+ lambda x, y: self.native.viewport_point(x, y) if self.native else (x, y, 1280, 800),
+ )
+ browser.cdp_client.send_raw = self.pointer.wrap(browser.cdp_client.send_raw)
+ self.browser = browser
+
def page_opened(self, page: Any) -> None:
page.on("dialog", self.on_dialog)
@@ -267,7 +300,7 @@ class Worker:
self.tabs[info["targetInfo"]["targetId"]] = page
finally:
await session.detach()
- focused = self.browser.get_focused_target()
+ focused = self.browser.get_focused_target() if self.browser is not None else None
if self.native and self.manual:
window_title = self.native.title()
candidates = []
@@ -423,6 +456,7 @@ class Worker:
await asyncio.sleep(1 / 15)
async def run(self, args: dict) -> dict:
+ await self.ensure_browser()
self.step_idle.clear()
previous_downloads = set(self.browser.downloaded_files)
from browser_use import Agent, Tools # type: ignore[import-not-found]
@@ -582,6 +616,7 @@ class Worker:
self.pointer.clear()
else:
if self.native and self.manual:
+ await self.ensure_browser()
from browser_use.browser import events # type: ignore[import-not-found]
await self.focused()
@@ -625,6 +660,7 @@ class Worker:
elif op == "reload":
await page.reload()
elif op == "tab":
+ await self.ensure_browser()
from browser_use.browser.events import SwitchTabEvent # type: ignore[import-not-found]
if args.get("target") == "new":
diff --git a/tests/contract/browser_pointer_probe.py b/tests/contract/browser_pointer_probe.py
index 6f8ccbdb46..d17755d5bc 100644
--- a/tests/contract/browser_pointer_probe.py
+++ b/tests/contract/browser_pointer_probe.py
@@ -5,10 +5,11 @@ import json
import sys
from pathlib import Path
+ROOT = Path(__file__).resolve().parents[2]
+
async def main():
- root = Path(__file__).resolve().parents[2]
- sys.path.insert(0, str(root / "jarvis/society/browser"))
+ sys.path.insert(0, str(ROOT / "jarvis/society/browser"))
output = sys.stdout
import live_runner
@@ -17,10 +18,21 @@ async def main():
worker = live_runner.Worker()
try:
profile = Path(sys.argv[2])
- await worker.start({"profile_dir": str(profile), "workspace": str(profile / "workspace"),
- "executable": sys.argv[1], "allowed_domains": []})
+ await worker.start(
+ {
+ "profile_dir": str(profile),
+ "workspace": str(profile / "workspace"),
+ "executable": sys.argv[1],
+ "allowed_domains": [],
+ }
+ )
+ assert worker.browser is None # First pixels need no agent engine.
+ await worker.ensure_browser()
await worker.page.goto("about:blank")
- await worker.page.set_content('<button style="width:200px;height:100px" onclick="window.clicked=true">Click probe</button>')
+ await worker.page.set_content(
+ '<button style="width:200px;height:100px" '
+ 'onclick="window.clicked=true">Click probe</button>'
+ )
await worker.focused()
target = next(t for t, p in worker.tabs.items() if p is worker.page)
client = worker.browser.cdp_client
diff --git a/tests/contract/test_browser_desktop_mcp.py b/tests/contract/test_browser_desktop_mcp.py
index 1acf8c37df..5ff735cb4f 100644
--- a/tests/contract/test_browser_desktop_mcp.py
+++ b/tests/contract/test_browser_desktop_mcp.py
@@ -205,3 +205,36 @@ def test_jarvis_codex_seat_uses_owned_browser_not_inherited_plugins(
coding = runner_cli.plan_codex(**args)
assert "--ignore-user-config" not in coding.argv
assert "browser_use" not in coding.argv
+
+
+async def test_failed_browser_mcp_preserves_partial_outcome_and_error_flag(monkeypatch):
+ import json
+
+ import mcp.types as types
+
+ from jarvis.mcp import jarvis_tools_server as server
+
+ tool = SimpleNamespace(
+ name="society_browser",
+ description="Browser",
+ risk_tier="monitor",
+ is_action_tool=True,
+ input_schema={"type": "object"},
+ )
+ partial = {"ok": False, "urls": ["https://example.org/receipt"], "steps": 3}
+
+ async def execute(name, args, request):
+ return ToolResult(False, partial, "Model timed out after navigation")
+
+ gateway = SimpleNamespace(catalog=lambda: [tool], execute=execute)
+ monkeypatch.setattr(server, "_gateway", lambda: gateway)
+ instance = server.build_server()
+ request = types.CallToolRequest(
+ params=types.CallToolRequestParams(name="society_browser", arguments={"task": "Read"})
+ )
+ response = await instance.request_handlers[types.CallToolRequest](request)
+ assert response.root.isError
+ body = json.loads(response.root.content[0].text)
+ assert body["result"] == partial
+ assert body["error"] == "Model timed out after navigation"
+ assert "Inspect current state" in body["retry"]
diff --git a/tests/contract/test_browser_live.py b/tests/contract/test_browser_live.py
index 03d1db06f5..e8bc96be8b 100644
--- a/tests/contract/test_browser_live.py
+++ b/tests/contract/test_browser_live.py
@@ -65,7 +65,23 @@ def test_actual_browser_click_emits_pointer_telemetry(tmp_path):
class PageHandler(BaseHTTPRequestHandler):
+ profile_requests = []
+
def do_GET(self):
+ if self.path.startswith(("/login", "/account")):
+ signed_in = "fixture_session=verified" in self.headers.get("Cookie", "")
+ self.profile_requests.append((self.path, signed_in))
+ self.send_response(200)
+ self.send_header("Content-Type", "text/html")
+ if self.path == "/login":
+ self.send_header(
+ "Set-Cookie",
+ "fixture_session=verified; Path=/; Max-Age=3600; HttpOnly; SameSite=Lax",
+ )
+ signed_in = True
+ self.end_headers()
+ self.wfile.write(b"<h1>Signed in</h1>" if signed_in else b"<h1>Sign in required</h1>")
+ return
if self.path == "/download":
self.send_response(200)
self.send_header("Content-Type", "application/octet-stream")
@@ -100,6 +116,7 @@ class PageHandler(BaseHTTPRequestHandler):
@pytest.fixture
def site():
+ PageHandler.profile_requests = []
server = ThreadingHTTPServer(("127.0.0.1", 0), PageHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
@@ -144,17 +161,25 @@ async def test_live_pixels_change_between_tasks_and_sessions_stay_open(live, sit
await live.close()
-async def test_second_viewer_receives_static_page_without_new_browser(live):
+async def test_second_viewer_receives_static_page_without_new_browser(live, record_property):
+ import time
+
agent = SimpleNamespace(agent_id="static", model="", browser_allowed_domains=[])
try:
+ started = time.monotonic()
session, first = await live.subscribe(agent)
while (await asyncio.wait_for(first.get(), 5))["kind"] != "frame":
pass # Drain initial state before attaching the second viewer.
+ record_property("cold_first_frame_ms", round((time.monotonic() - started) * 1000, 2))
await asyncio.sleep(1)
+ started = time.monotonic()
same, second = await live.subscribe(agent)
assert same is session
while (await asyncio.wait_for(second.get(), 5))["kind"] != "frame":
pass # State can precede the first image for a new viewer.
+ warm_ms = (time.monotonic() - started) * 1000
+ record_property("warm_first_frame_ms", round(warm_ms, 2))
+ assert warm_ms <= 2000
assert len(live.sessions) == 1
assert len(session.subscribers) == 2
finally:
@@ -608,3 +633,30 @@ async def test_finished_chat_releases_real_browser_for_the_next_task(live, monke
job.cancel()
await asyncio.gather(job, return_exceptions=True)
await live.close()
+
+
+async def test_login_profile_survives_restart_and_stays_with_its_agent(live, site):
+ first = SimpleNamespace(
+ agent_id="signed-in", model="", browser_allowed_domains=["http*://127.0.0.1"]
+ )
+ other = SimpleNamespace(
+ agent_id="isolated", model="", browser_allowed_domains=["http*://127.0.0.1"]
+ )
+
+ async def visit(agent, path):
+ session, _ = await live.subscribe(agent)
+ await live.control(session, "viewer", "takeover", {"enabled": True})
+ await live.control(session, "viewer", "navigate", {"url": site + path})
+ await live.control(session, "viewer", "takeover", {"enabled": False})
+ return session
+
+ try:
+ await visit(first, "/login")
+ await live.close()
+ restored = await visit(first, "/account?restored")
+ isolated = await visit(other, "/account?other")
+ assert restored is not isolated
+ assert ("/account?restored", True) in PageHandler.profile_requests
+ assert ("/account?other", False) in PageHandler.profile_requests
+ finally:
+ await live.close()