Skip to content

The agent that grows with you

fix(browser): a shared-profile browser is stopped only by its last holder across processes

Tekniumb2527c1ce95c
diff --git a/tests/tools/test_browser_launcher.py b/tests/tools/test_browser_launcher.py
index 4c84e09fbb1..287b062c8b8 100644
--- a/tests/tools/test_browser_launcher.py
+++ b/tests/tools/test_browser_launcher.py
@@ -7,6 +7,7 @@ launcher runs for real (spawn, readiness poll, registry, records, parent-first s
from __future__ import annotations
+import contextlib
import json
import os
import signal
@@ -270,6 +271,93 @@ def test_records_of_a_live_owner_are_left_alone(tmp_path, fake_chromium):
assert list(_records().glob("*.json"))
+# A Hermes process on a persistent profile: launches (or attaches to) its Chromium, prints the pid, waits
+# for a line on stdin, then exits normally (the session layer's exit teardown stops its browsers).
+_SHARED_PROFILE_USER = textwrap.dedent("""\
+ import os, sys
+ import tools.browser_session # registers the exit teardown
+ from tools import browser_launcher as launcher
+ launcher.needs_chromium_sandbox_bypass = lambda: False
+ exe, udd, running = sys.argv[1], sys.argv[2], int(sys.argv[3])
+ if running: # Chromium's singleton would hand a second launch to the running instance
+ launcher.running_instance = lambda d: (45678, "/devtools/browser/fake-id", running) if d == udd else None
+ browser, err = launcher.launch_chromium("profile:" + udd, executable=exe, user_data_dir=udd, ephemeral=False,
+ headed=False, env=dict(os.environ), timeout=30)
+ print(f"{browser.pid} {browser.owned}" if browser else err, flush=True)
+ sys.stdin.readline()
+""")
+
+
+def _shared_profile_user(fake_chromium, udd, running: int = 0) -> tuple[subprocess.Popen, int]:
+ proc = subprocess.Popen([sys.executable, "-c", _SHARED_PROFILE_USER, str(fake_chromium), str(udd), str(running)],
+ cwd=Path(__file__).resolve().parents[2], env=dict(os.environ), stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ line = proc.stdout.readline().split()
+ assert line and line[0].isdigit(), proc.stderr.read()[-2000:] if proc.poll() is not None else line
+ assert (line[1] == "True") is not bool(running)
+ return proc, int(line[0])
+
+
+def _gone(pid: int, timeout: float = 15) -> bool:
+ deadline = time.monotonic() + timeout
+ while launcher._pid_alive(pid) and time.monotonic() < deadline:
+ time.sleep(0.1)
+ return not launcher._pid_alive(pid)
+
+
+def _kill_fake(pid: int, fake_chromium) -> None:
+ with contextlib.suppress(OSError):
+ if str(fake_chromium) in Path(f"/proc/{pid}/cmdline").read_text(errors="replace"):
+ os.kill(pid, signal.SIGKILL)
+
+
[email protected]_system_guard_bypass # the fake browser is ours to stop if a leg failed
+def test_a_shared_profile_browser_outlives_its_launcher_while_another_process_holds_it(fake_chromium, tmp_path):
+ """Two Hermes processes on one persistent profile: the one that launched its Chromium lets go (its normal
+ exit) while the other still drives it. The browser stays up for the other one, and that last holder's
+ own normal exit stops it, leaving no launcher state behind."""
+ udd, key = tmp_path / "shared-profile", f"profile:{tmp_path / 'shared-profile'}"
+ mine, err = launcher.launch_chromium(key, executable=str(fake_chromium), user_data_dir=str(udd), ephemeral=False,
+ headed=False, env=dict(os.environ), timeout=30)
+ assert err is None and mine.owned
+ holder, _ = _shared_profile_user(fake_chromium, udd, running=mine.pid)
+ try:
+ launcher.stop_browser(key)
+ assert launcher._pid_alive(mine.pid), "the launcher let go of a browser another process still drives"
+
+ holder.communicate("\n", timeout=60) # the last holder exits normally
+ assert _gone(mine.pid)
+ assert not [p.name for p in _records().iterdir() if not p.name.startswith(".")]
+ finally:
+ holder.kill()
+ _kill_fake(mine.pid, fake_chromium)
+
+
[email protected]_system_guard_bypass # the fake browser is ours to stop if a leg failed
+def test_a_crashed_launchers_shared_browser_is_reaped_only_after_its_last_holder_is_gone(fake_chromium, tmp_path):
+ """The launcher crashed while another live Hermes process holds its browser: no reaper kills it. Once
+ that holder crashed too (proven by pid + start time), the next reap stops it."""
+ udd = tmp_path / "shared-profile"
+ owner, pid = _shared_profile_user(fake_chromium, udd)
+ holder, _ = _shared_profile_user(fake_chromium, udd, running=pid)
+ try:
+ owner.kill()
+ owner.wait(timeout=10)
+ assert launcher.reap_orphaned_browsers() == 0
+ assert launcher._pid_alive(pid), "reaped while a live process still holds it"
+
+ holder.kill()
+ holder.wait(timeout=10)
+ assert launcher.reap_orphaned_browsers() == 1
+ assert _gone(pid)
+ finally:
+ owner.kill()
+ holder.kill()
+ _kill_fake(pid, fake_chromium)
+
+
@pytest.mark.parametrize("headed,display,expect_headless", [
(False, ":1", True),
(True, ":1", False),
diff --git a/tools/browser_launcher.py b/tools/browser_launcher.py
index 3b5ec2ebd4a..412b7dc1aab 100644
--- a/tools/browser_launcher.py
+++ b/tools/browser_launcher.py
@@ -5,7 +5,9 @@ profile shared by every key). Chromium picks its own DevTools port (``--remote-d
and writes it to ``<user-data-dir>/DevToolsActivePort``; the CDP URL comes from that file, so there
is no port race. Every owned launch leaves a record in its profile's private launcher state dir
(``<HERMES_HOME>/cache/browser/chromium-records``, 0700) so a later Hermes process serving that profile
-can reap the Chromium of an owner that crashed, after proving the pid is still that browser.
+can reap the Chromium of an owner that crashed, after proving the pid is still that browser. A shared-profile
+Chromium other Hermes processes attached to is stopped only by its last holder
+(``tools/browser_launcher_holders.py``).
Stop is always parent-first (``ProcessRegistry._terminate_host_pid``): Chromium reaps its own
zygotes and renderers on SIGTERM, and killing a zygote first turns a clean exit into a crash dump.
@@ -366,7 +368,12 @@ def _attach_running(key: str, user_data_dir: str, ephemeral: bool, home_key: str
logger.info("browser: attaching to the Chromium already running on %s (pid %s)", user_data_dir, pid)
browser = _make_browser(key, port, path, pid=pid, start_time=_start_time(pid), user_data_dir=user_data_dir,
owned=False, ephemeral=False, home_key=home_key)
- _adopt_record(browser)
+ record = _record_path(browser)
+ if record is not None:
+ from tools import browser_launcher_holders as holders
+ with holders.records_lock(record.parent):
+ _adopt_record(browser)
+ holders.write_lease(record, browser) # its launcher's exit must not stop it under us
return browser
@@ -382,6 +389,10 @@ def launch_chromium(key: str, *, executable: Optional[str] = None, user_data_dir
a live Chromium on it is ATTACHED (``owned=False``, never killed). ``sandbox_bypass=False`` (the
user's real browser) never gets the host's sandbox-bypass flags."""
from hermes_constants import hermes_home_key
+ # Loaded now, not first at teardown: stopping the browser from an atexit hook must not be this module's
+ # first import (Python refuses new atexit registrations during shutdown, orphaning the browser). An
+ # attacher needs it too: it is the one that stops a shared browser when it is the last holder.
+ import tools.process_registry
with _key_lock(key): # one launch per key; other keys launch concurrently
existing = get_browser(key)
if existing is not None:
@@ -417,9 +428,6 @@ def _launch_owned(key: str, executable: str, udd: str, *, ephemeral: bool, packa
if env is None:
from tools.browser_session import build_browser_env
env = build_browser_env()
- # Loaded now, not first at teardown: stopping the browser from an atexit hook must not be this module's
- # first import (Python refuses new atexit registrations during shutdown, orphaning the browser).
- import tools.process_registry
try:
_prepare_dir(udd, ephemeral)
except OSError as exc:
@@ -474,7 +482,7 @@ def get_browser(key: str) -> Optional[LaunchedBrowser]:
return None
-def _forget(key: str, browser: LaunchedBrowser) -> None:
+def _forget(key: str, browser: LaunchedBrowser, *, drop_record: bool = True) -> None:
_browsers.pop(key, None)
proc = _procs.pop(key, None)
if proc is not None:
@@ -482,7 +490,12 @@ def _forget(key: str, browser: LaunchedBrowser) -> None:
proc.wait(timeout=0) # reap the zombie
except subprocess.TimeoutExpired:
pass
- if browser.owned:
+ if not browser.ephemeral:
+ record = _record_path(browser)
+ if record is not None:
+ from tools.browser_launcher_holders import drop_lease
+ drop_lease(record)
+ if browser.owned and drop_record:
_remove_record(browser)
if browser.ephemeral:
shutil.rmtree(browser.user_data_dir, ignore_errors=True)
@@ -505,16 +518,46 @@ def _terminate(pid: Optional[int], start_time: Optional[int], proc: Optional[sub
def stop_browser(key: str) -> None:
- """Stop ``key``'s browser if this process launched it (an attached one is only forgotten)."""
+ """Let go of ``key``'s browser. A throwaway one this process launched is stopped. A shared-profile one is
+ stopped only by its LAST holder across Hermes processes (:func:`_last_holder_out`); one a human started is
+ never stopped."""
with _lock:
browser = _browsers.get(key)
proc = _procs.get(key)
if browser is None:
return
- if browser.owned:
- _terminate(browser.pid, browser.start_time, proc)
+ record = None if browser.ephemeral else _record_path(browser)
+ if record is None:
+ if browser.owned:
+ _terminate(browser.pid, browser.start_time, proc)
+ with _lock:
+ _forget(key, browser)
+ return
+ from tools.browser_launcher_holders import records_lock
+ with records_lock(record.parent):
+ if _last_holder_out(record, browser):
+ _terminate(browser.pid, browser.start_time, proc)
+ record.unlink(missing_ok=True)
with _lock:
- _forget(key, browser)
+ _forget(key, browser, drop_record=False)
+
+
+def _last_holder_out(record: Path, browser: LaunchedBrowser) -> bool:
+ """Drop this process's hold on shared-profile ``browser`` (caller holds the records lock). True when it
+ must stop it now: no other live Hermes process holds it, and Hermes launched it (a launch record names
+ it). Otherwise a record this process holds, or whose owner is gone, goes to a live holder."""
+ from tools import browser_launcher_holders as holders
+ holders.drop_lease(record)
+ others = holders.other_live_holders(record, browser.start_time)
+ data = _load_record(record) if record.exists() else None
+ if data is not None and data.get("start_time") != browser.start_time:
+ data = None # an earlier browser on a recycled pid
+ held_elsewhere = data is not None and data.get("owner_pid") != os.getpid() and _owner_alive(data)
+ if others and data is not None and not held_elsewhere:
+ holders.hand_over(record, data, others[0])
+ if others or held_elsewhere:
+ return False
+ return browser.owned or data is not None
def stop_all_browsers(home_key: Optional[str] = None) -> None:
@@ -664,6 +707,7 @@ def _load_record(path: Path) -> Optional[dict]:
def _reap_record(path: Path, home: str) -> bool:
+ """Caller holds the records lock."""
record = _load_record(path)
if record is None:
return False
@@ -673,6 +717,11 @@ def _reap_record(path: Path, home: str) -> bool:
return False
if record.get("owner_pid") != os.getpid() and _owner_alive(record):
return False
+ from tools import browser_launcher_holders as holders
+ others = holders.other_live_holders(path, record.get("start_time"))
+ if others: # another live Hermes process still drives it
+ holders.hand_over(path, record, others[0])
+ return False
reaped = _is_recorded_browser(record, home)
if reaped:
_terminate(record["pid"], record.get("start_time"), None)
@@ -691,12 +740,16 @@ def _reap_home(home: str) -> int:
if not _is_private_dir(records):
logger.debug("browser: not reaping from %s: not a private directory of this user", records)
return 0
- try:
- paths = sorted(records.glob("*.json"))
- except OSError as exc:
- logger.debug("browser: launch records unavailable in %s: %s", records, exc)
- return 0
- return sum(1 for path in paths if _reap_record(path, home))
+ from tools import browser_launcher_holders as holders
+ with holders.records_lock(records):
+ try:
+ paths = sorted(records.glob("*.json"))
+ except OSError as exc:
+ logger.debug("browser: launch records unavailable in %s: %s", records, exc)
+ return 0
+ reaped = sum(1 for path in paths if _reap_record(path, home))
+ holders.drop_dead_leases(records)
+ return reaped
def _served_home_keys(homes) -> set[str]:
diff --git a/tools/browser_launcher_holders.py b/tools/browser_launcher_holders.py
new file mode 100644
index 00000000000..4f8844d7537
--- /dev/null
+++ b/tools/browser_launcher_holders.py
@@ -0,0 +1,125 @@
+"""Cross-process holders of a shared-profile Chromium (``tools/browser_launcher.py``).
+
+A persistent profile (``browser.user_data_dir``, the Bot Desktop profile, the real-profile copy) has ONE
+Chromium, launched by whichever Hermes process got there first; every other process attaches to it. Each
+attacher leaves a lease beside the launch record in the profile's private records dir. The process holding
+the record (the launcher, or whoever it was handed to) stops the browser on exit only when no other live
+Hermes process holds a lease; otherwise it hands the record to one of them, so the LAST holder stops it. A
+holder that crashed is dropped once its pid and start time no longer match, never on the pid alone.
+
+Every decision runs under the records dir's lock: a lease written while another process decides cannot be
+missed. Siblings late-import the facade (its ``_pid_alive`` / ``_start_time`` are the patched seams).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Iterator, Optional
+
+logger = logging.getLogger(__name__)
+
+_LOCK_NAME = ".holders.lock"
+
+
+def records_lock(records: Path) -> Iterator[None]:
+ """Exclusive, cross-process lock on ``records`` (the profile's private launcher state dir). Never nested:
+ two locks on one file in one process are two open file descriptions, and the second would wait forever."""
+ flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
+ fd = os.open(records / _LOCK_NAME, flags, 0o600)
+ try:
+ if os.name == "nt":
+ import msvcrt
+ while True:
+ try:
+ msvcrt.locking(fd, msvcrt.LK_LOCK, 1) # retries for ~10 s, then raises
+ break
+ except OSError:
+ continue
+ else:
+ import fcntl # windows-footgun: ok — the nt branch above
+ fcntl.flock(fd, fcntl.LOCK_EX)
+ yield
+ finally:
+ if os.name == "nt":
+ import msvcrt
+ with contextlib.suppress(OSError):
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
+ os.close(fd)
+
+
+def _lease_path(record: Path, holder_pid: int) -> Path:
+ return record.with_name(f"{record.stem}@{holder_pid}.lease")
+
+
+def _load(path: Path) -> Optional[dict]:
+ try:
+ data = json.loads(path.read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError):
+ data = None
+ if isinstance(data, dict):
+ return data
+ path.unlink(missing_ok=True)
+ return None
+
+
+def process_alive(pid, start_time) -> bool:
+ """``pid`` is still the process that wrote a record or lease: alive, and with the recorded start time
+ (a recycled pid has another one)."""
+ from tools import browser_launcher as launcher
+ if not isinstance(pid, int) or not launcher._pid_alive(pid):
+ return False
+ return start_time is None or launcher._start_time(pid) == start_time
+
+
+def write_lease(record: Path, browser) -> None:
+ """This process holds ``browser`` (caller holds :func:`records_lock`)."""
+ from tools import browser_launcher as launcher
+ from utils import atomic_json_write
+ me = os.getpid()
+ atomic_json_write(_lease_path(record, me), {
+ "holder_pid": me, "holder_start_time": launcher._start_time(me), "pid": browser.pid,
+ "start_time": browser.start_time, "user_data_dir": browser.user_data_dir}, indent=None)
+
+
+def drop_lease(record: Path) -> None:
+ _lease_path(record, os.getpid()).unlink(missing_ok=True)
+
+
+def other_live_holders(record: Path, browser_start_time) -> list[dict]:
+ """Leases of OTHER live processes on the browser ``record`` names (caller holds :func:`records_lock`).
+ A lease of a dead holder, or one naming an earlier browser on the same pid, is removed."""
+ holders = []
+ for path in sorted(record.parent.glob(f"{record.stem}@*.lease")):
+ lease = _load(path)
+ if lease is None or lease.get("holder_pid") == os.getpid():
+ continue
+ if lease.get("start_time") != browser_start_time or not process_alive(
+ lease.get("holder_pid"), lease.get("holder_start_time")):
+ path.unlink(missing_ok=True)
+ continue
+ holders.append(lease)
+ return holders
+
+
+def drop_dead_leases(records: Path) -> None:
+ """Leases whose holder is gone (caller holds :func:`records_lock`): a lease outlives its launch record when
+ the browser died while its holder crashed."""
+ for path in records.glob("*.lease"):
+ lease = _load(path)
+ if lease is not None and not process_alive(lease.get("holder_pid"), lease.get("holder_start_time")):
+ path.unlink(missing_ok=True)
+
+
+def hand_over(record: Path, data: dict, holder: dict) -> None:
+ """Make ``holder`` the record's owner: every reaper now sees a live owner, and the holder stops the
+ browser when it is the last one out."""
+ from utils import atomic_json_write
+ atomic_json_write(record, {**data, "owner_pid": holder["holder_pid"],
+ "owner_start_time": holder.get("holder_start_time")}, indent=None)
+ logger.info("browser: Chromium pid %s is still used by Hermes pid %s; handed it over instead of stopping it",
+ data.get("pid"), holder["holder_pid"])