Skip to content

The agent that grows with you

fix(browser): stop the shared sandbox Chromium only after its last session, and detect it dead

Teknium59dc0f76c3db
diff --git a/tests/tools/test_bot_desktop_placement.py b/tests/tools/test_bot_desktop_placement.py
index 3522a2eb899..8426ea7e523 100644
--- a/tests/tools/test_bot_desktop_placement.py
+++ b/tests/tools/test_bot_desktop_placement.py
@@ -365,7 +365,7 @@ def test_sandbox_cdp_endpoint_is_rewritten_to_the_forwarded_local_port(monkeypat
monkeypatch.setattr(sandbox_host, "browser_profile_dir", lambda e: "/home/pn/.hermes/bot-desktop/browser-profile")
def run_in(env, argv, **_kw):
- out = b"9223\n/devtools/browser/abc\n" if argv[0] == "cat" else b""
+ out = b"9223\n/devtools/browser/abc\n" if "DevToolsActivePort" in " ".join(argv) else b""
return subprocess.CompletedProcess(argv, 0, out, b"")
monkeypatch.setattr(streams, "run_in", run_in)
monkeypatch.setattr(streams, "forward_port", lambda env, port, *, remote_host="127.0.0.1", user=None: 45000 + port % 7)
diff --git a/tests/tools/test_browser_cleanup.py b/tests/tools/test_browser_cleanup.py
index 5879020f729..2f9dfaadf92 100644
--- a/tests/tools/test_browser_cleanup.py
+++ b/tests/tools/test_browser_cleanup.py
@@ -217,3 +217,71 @@ class TestSharedBrowserSessions:
assert bs.get_session_info("a")["cdp_url"] == live
assert launcher.get_browser(key) is not None, "b's live browser must survive a's rebuild"
+
+
+class _FakeSandbox:
+ """The terminal sandbox's exec stream: one Chromium on the shared profile, launched by ``setsid``."""
+
+ def __init__(self):
+ self.running = None
+ self.killed = []
+
+ def run_in(self, env, argv, **kw):
+ cmd = " ".join(argv)
+ if "setsid" in cmd:
+ self.running = 4242
+ return _Completed(0, b"4242\n")
+ if "/dev/tcp" in cmd or argv[0] == "cat":
+ return _Completed(0, b"9222\n/devtools/browser/abc\n") if self.running else _Completed(1)
+ if argv[0] == "kill":
+ self.killed.append(int(argv[-1]))
+ self.running = None
+ return _Completed(0)
+ return _Completed(1)
+
+
+class _Completed:
+ def __init__(self, returncode, stdout=b""):
+ self.returncode, self.stdout, self.stderr = returncode, stdout, b""
+
+
+def fake_sandbox(monkeypatch, isolated_sessions):
+ from tools import browser_session_sandbox
+ from tools.bot_desktop import runtime, sandbox_host
+ from tools.environments import streams
+ sandbox, env = _FakeSandbox(), object()
+ monkeypatch.setattr(browser_session_sandbox, "_launched", {}, raising=False)
+ monkeypatch.setattr(streams, "run_in", sandbox.run_in)
+ monkeypatch.setattr(streams, "forward_port", lambda env, port, user=None: 50000 + port)
+ monkeypatch.setattr(streams, "sandbox_temp_dir", lambda env: "/tmp") # no-tmp: ok — sandbox-side path
+ monkeypatch.setattr(runtime, "_sandbox_env", lambda create=False: env)
+ monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":5"})
+ monkeypatch.setattr(sandbox_host, "_user_for", lambda env: "hermes")
+ monkeypatch.setattr(sandbox_host, "browser_profile_dir", lambda env: "/home/hermes/.bot-browser")
+ monkeypatch.setattr(sandbox_host, "chromium_executable", lambda env: "/usr/bin/chromium")
+ return sandbox
+
+
+class TestSandboxSharedBrowser:
+ """Every session placed in the terminal sandbox shares the one in-sandbox Chromium on the dock's profile."""
+
+ def test_the_sandbox_browser_is_stopped_only_after_its_last_session_tears_down(self, fake_sandbox,
+ isolated_sessions):
+ from tools.browser_session_sandbox import create_sandbox_session
+ isolated_sessions["task-A"] = create_sandbox_session("task-A") # launches it
+ isolated_sessions["task-B"] = create_sandbox_session("task-B") # attaches to it
+
+ lifecycle.cleanup_single_session("task-A")
+ assert fake_sandbox.killed == [] and fake_sandbox.running, "B still uses the sandbox browser"
+
+ lifecycle.cleanup_single_session("task-B")
+ assert fake_sandbox.killed == [4242]
+
+ def test_a_sandbox_session_is_dead_once_its_chromium_is_gone(self, fake_sandbox):
+ from tools.browser_session_sandbox import create_sandbox_session
+ info = create_sandbox_session("task-A")
+ assert bs._backend_dead(info) is False
+
+ fake_sandbox.running = None # crashed inside the sandbox
+ assert bs._backend_dead(info) is True
diff --git a/tools/browser_session.py b/tools/browser_session.py
index 2980a1da3a0..99961f1c081 100644
--- a/tools/browser_session.py
+++ b/tools/browser_session.py
@@ -303,6 +303,9 @@ def _backend_dead(session_info: dict[str, Any]) -> bool:
from tools.browser_lightpanda import get_server
server = get_server(session_info.get("session_name", ""))
return server is None or not server.is_alive()
+ if features.get("sandbox"):
+ from tools.browser_session_sandbox import sandbox_browser_alive
+ return not sandbox_browser_alive(session_info)
launcher_key = session_info.get("launcher_key") or (
"hermes-real-profile" if features.get("real_profile") else None)
if launcher_key:
diff --git a/tools/browser_session_lifecycle.py b/tools/browser_session_lifecycle.py
index 583bc1b8ca9..12f5df61691 100644
--- a/tools/browser_session_lifecycle.py
+++ b/tools/browser_session_lifecycle.py
@@ -3,7 +3,8 @@ points and the atexit sweep.
Teardown per session kind (CDP supervisor stopped first, so our WebSocket closes before the endpoint):
cloud → ``provider.close_session``; Lightpanda → ``stop_lightpanda``; launcher Chromium → ``stop_browser``
-(a shared persistent profile only once no other session uses it); sandbox → SIGTERM inside the sandbox;
+(a shared persistent profile only once no other session uses it); sandbox → SIGTERM inside the sandbox
+once no other session uses it;
real profile → shared, terminated at exit / consent-off; CDP override → nothing (the user's browser).
"""
@@ -132,11 +133,13 @@ def release_session_backend(cache_key: str, session_info: dict[str, Any]) -> Non
if features.get("lightpanda"):
from tools.browser_lightpanda import stop_lightpanda
stop_lightpanda(session_info.get("session_name", ""))
+ launcher_key = session_info.get("launcher_key")
+ if not launcher_key or _launcher_key_in_use(launcher_key, session_info):
+ return # nothing launched for it, or a shared browser another live session still uses
if features.get("sandbox"):
from tools.browser_session_sandbox import stop_sandbox_browser
stop_sandbox_browser(session_info)
- launcher_key = session_info.get("launcher_key")
- if launcher_key and not _launcher_key_in_use(launcher_key, session_info):
+ else:
from tools.browser_launcher import stop_browser
stop_browser(launcher_key)
diff --git a/tools/browser_session_sandbox.py b/tools/browser_session_sandbox.py
index 3484aadccf5..49f5f456b85 100644
--- a/tools/browser_session_sandbox.py
+++ b/tools/browser_session_sandbox.py
@@ -10,12 +10,19 @@ from __future__ import annotations
import logging
import shlex
+import threading
import time
from typing import Any, Optional
logger = logging.getLogger(__name__)
_LAUNCH_TIMEOUT_S = 60.0
+# Sandbox Chromium this process launched, per shared key: ``(sandbox env, pid)``. Every session on the profile
+# attaches to it, so it is stopped only when the LAST of them tears down (release_session_backend's in-use
+# check on the shared ``launcher_key``); the env is kept so a teardown outside the owner's scope hits the
+# right sandbox.
+_launched: dict[str, tuple[Any, int]] = {}
+_launched_lock = threading.Lock()
def _sandbox():
@@ -40,6 +47,25 @@ def _read_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:
return None
+def _live_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:
+ """``(port, path)`` of the Chromium on ``udd`` while its DevTools port still answers, in one exec (a
+ crashed Chromium leaves a stale ``DevToolsActivePort`` behind)."""
+ from tools.environments import streams
+ active = shlex.quote(f"{udd}/DevToolsActivePort")
+ script = f'p=$(head -n1 {active}) && exec 3<>"/dev/tcp/127.0.0.1/$p" && cat {active}'
+ proc = streams.run_in(env, ["bash", "-c", script], user=user, timeout=10)
+ lines = proc.stdout.decode("utf-8", "replace").splitlines() if proc.returncode == 0 else []
+ if len(lines) >= 2 and lines[0].strip().isdigit() and lines[1].startswith("/devtools/browser/"):
+ return int(lines[0].strip()), lines[1].strip()
+ return None
+
+
+def _shared_key(udd: str) -> str:
+ """The launcher key every session on this profile's sandbox Chromium shares (the sandbox is per profile)."""
+ from hermes_constants import hermes_home_key
+ return f"sandbox:{hermes_home_key()}:{udd}"
+
+
def _start_chromium(env, user, exe: str, udd: str, child_env: dict) -> int:
"""Launch detached in the sandbox; returns the sandbox pid."""
from tools.bot_desktop.browser import SHARED_PROFILE_ARGS
@@ -77,40 +103,53 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:
env, published = _sandbox()
user = sandbox_host._user_for(env)
udd = sandbox_host.browser_profile_dir(env)
- owned_pid = None
- found = _read_devtools(env, user, udd)
- if found is not None and not _port_answers(env, user, found[0]):
- found = None
+ key = _shared_key(udd)
+ found = _live_devtools(env, user, udd)
if found is None:
exe = sandbox_host.chromium_executable(env)
if not exe:
raise RuntimeError("the sandbox image has no Chromium to run the browser")
child_env = {**published, "TMPDIR": streams.sandbox_temp_dir(env)}
- owned_pid = _start_chromium(env, user, exe, udd, child_env)
+ pid = _start_chromium(env, user, exe, udd, child_env)
found = _wait_for_port(env, user, udd)
+ with _launched_lock:
+ _launched[key] = (env, pid)
local_port = streams.forward_port(env, found[0], user=user)
info = _session_record("sb", f"ws://127.0.0.1:{local_port}{found[1]}", {"local": True, "sandbox": True})
- info.update(sandbox_pid=owned_pid, shared_profile=True, launcher_key=None)
+ info.update(shared_profile=True, launcher_key=key)
logger.info("Created sandbox browser session %s for %s", info["session_name"], cache_key)
return info
-def _port_answers(env, user, port: int) -> bool:
+def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:
+ """The sandbox Chromium still serves this session's endpoint through the current port forward: False once
+ it died, was relaunched (new DevTools path) or the sandbox was replaced (new forward)."""
+ from tools.bot_desktop import runtime as _bd_runtime, sandbox_host
from tools.environments import streams
- probe = f"exec 3<>/dev/tcp/127.0.0.1/{int(port)}"
- return streams.run_in(env, ["bash", "-c", probe], user=user, timeout=10).returncode == 0
+ env = _bd_runtime._sandbox_env(create=False)
+ if env is None:
+ return False
+ try:
+ user = sandbox_host._user_for(env)
+ found = _live_devtools(env, user, sandbox_host.browser_profile_dir(env))
+ except (OSError, RuntimeError) as exc:
+ logger.debug("sandbox browser liveness probe failed: %s", exc)
+ return False
+ if found is None:
+ return False
+ return session_info.get("cdp_url") == f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}"
def stop_sandbox_browser(session_info: dict[str, Any]) -> None:
- """SIGTERM the sandbox Chromium this process launched (best effort; the sandbox's lifetime bounds it)."""
- pid = session_info.get("sandbox_pid")
- if not pid:
+ """SIGTERM the sandbox Chromium this process launched on the session's shared key (best effort; the
+ sandbox's lifetime bounds it). The caller guarantees no other live session uses it."""
+ with _launched_lock:
+ launched = _launched.pop(session_info.get("launcher_key") or "", None)
+ if launched is None:
return
- from tools.bot_desktop import runtime as _bd_runtime, sandbox_host
+ from tools.bot_desktop import sandbox_host
from tools.environments import streams
- env = _bd_runtime._sandbox_env(create=False)
- if env is None:
- return
+ env, pid = launched
try:
streams.run_in(env, ["kill", "-TERM", str(int(pid))], user=sandbox_host._user_for(env), timeout=15)
except (OSError, RuntimeError) as exc: