Skip to content

The agent that grows with you

fix(browser): two homes pinning one browser profile share its launch record, leases and lock

Teknium505afcd73bc4
diff --git a/tests/tools/test_browser_launcher_shared_profile.py b/tests/tools/test_browser_launcher_shared_profile.py
new file mode 100644
index 00000000000..e2495d4014b
--- /dev/null
+++ b/tests/tools/test_browser_launcher_shared_profile.py
@@ -0,0 +1,147 @@
+"""One persistent (shared) profile has ONE Chromium, whichever Hermes home or launcher key reaches it.
+
+Two homes (profiles) pinning the same ``browser.user_data_dir``, or one process reaching one dir under two keys
+(a symlinked path), drive one browser: they share its launch record, leases and lock, so the LAST holder stops
+it and a crashed owner's browser is reaped from any home (``tools/browser_launcher.py``,
+``tools/browser_launcher_holders.py``). Runs the real launcher against a fake Chromium that takes the profile
+singleton and serves ``/json/version`` like the real one.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from tests.tools.browser_launcher_fakes import kill_fake, pids, write_fake_chromium
+from tests.waiting import wait_until
+from tools import browser_launcher as launcher
+
+pytestmark = [pytest.mark.platforms("linux"),
+ pytest.mark.live_system_guard_bypass] # the fake browsers are ours to stop if a leg failed
+
+_REPO = Path(__file__).resolve().parents[2]
+
+# A Hermes process on a persistent profile: launches or attaches, prints "<pid> <owned>", waits for a line on
+# stdin, then exits normally (the session layer's exit teardown lets go of its browsers).
+_PROFILE_USER = textwrap.dedent("""\
+ import os, sys
+ import tools.browser_session # registers the exit teardown
+ from tools import browser_launcher as launcher
+ launcher.needs_chromium_sandbox_bypass = lambda: False
+ exe, udd = sys.argv[1], sys.argv[2]
+ browser, err = launcher.launch_chromium("profile:" + udd, executable=exe, user_data_dir=udd, ephemeral=False,
+ headed=False, env=dict(os.environ), timeout=30)
+ print(f"{browser.pid} {browser.owned}" if browser else f"ERR {err}", flush=True)
+ sys.stdin.readline()
+""")
+
+
+def _user(home: Path, script: Path, udd: Path) -> tuple[subprocess.Popen, int, bool]:
+ home.mkdir(parents=True, exist_ok=True)
+ proc = subprocess.Popen([sys.executable, "-c", _PROFILE_USER, str(script), str(udd)], cwd=_REPO,
+ env={**os.environ, "HERMES_HOME": str(home)}, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ line = proc.stdout.readline().split()
+ assert line and line[0].isdigit(), proc.stderr.read()[-2000:] if proc.poll() is not None else line
+ return proc, int(line[0]), line[1] == "True"
+
+
+def _gone(pid: int) -> bool:
+ return wait_until(lambda: not launcher._pid_alive(pid), timeout=15, what=f"pid {pid} to exit")
+
+
+def _launcher_state(*homes: Path) -> list[str]:
+ """Every launch record and lease left anywhere: the homes' records dirs and the shared-profile root."""
+ roots = [h / "cache" / "browser" / "chromium-records" for h in homes] + [launcher._shared_profiles_root()]
+ return sorted(p.name for root in roots if root.is_dir() for p in root.rglob("*")
+ if p.suffix in (".json", ".lease"))
+
+
+def test_two_homes_pinning_one_profile_share_its_browser_until_the_last_one_leaves(tmp_path):
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd, h1, h2 = tmp_path / "pinned-profile", tmp_path / "home-1", tmp_path / "home-2"
+ a, pid, a_owned = _user(h1, script, udd)
+ b = None
+ try:
+ b, b_pid, b_owned = _user(h2, script, udd)
+ assert (a_owned, b_owned, b_pid) == (True, False, pid)
+
+ a.communicate("\n", timeout=60) # the launcher (home 1) exits normally while home 2 still drives it
+ assert launcher._pid_alive(pid), "the other home's launcher stopped a browser this home still holds"
+
+ b.communicate("\n", timeout=60) # the last holder exits normally
+ assert _gone(pid)
+ assert _launcher_state(h1, h2) == []
+ assert pids(udd, "launches") == [pid]
+ finally:
+ for proc in (a, b):
+ if proc is not None:
+ proc.kill()
+ kill_fake(pid, script)
+
+
+def test_shared_profile_state_lives_in_a_private_per_user_dir_outside_every_home(tmp_path):
+ from hermes_constants import get_hermes_home
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd = tmp_path / "pinned-profile"
+ browser, err = launcher.launch_chromium(f"profile:{udd}", executable=str(script), user_data_dir=str(udd),
+ ephemeral=False, headed=False, env=dict(os.environ), timeout=30)
+ try:
+ assert err is None
+ record = launcher._record_path(browser)
+ root = launcher._shared_profiles_root()
+ assert record.exists() and record.parent.parent == root
+ assert Path(get_hermes_home()) not in record.parents
+ for state in (root, record.parent):
+ assert stat.S_IMODE(state.lstat().st_mode) == 0o700 and not state.is_symlink()
+ finally:
+ launcher.stop_browser(f"profile:{udd}")
+ if browser is not None:
+ kill_fake(browser.pid, script)
+
+
+def test_a_crashed_launchers_shared_browser_is_reaped_from_another_home(tmp_path):
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd = tmp_path / "pinned-profile"
+ owner, pid, _ = _user(tmp_path / "home-1", script, udd)
+ try:
+ owner.kill()
+ owner.wait(timeout=10)
+ assert launcher._pid_alive(pid)
+
+ assert launcher.reap_orphaned_browsers() == 1 # this process serves only its own (other) home
+ assert _gone(pid)
+ finally:
+ owner.kill()
+ kill_fake(pid, script)
+
+
+def test_one_process_holding_a_profile_under_two_keys_keeps_it_until_both_let_go(tmp_path):
+ """One dir reached by two paths (a symlink) gives two launcher keys on one Chromium: dropping the key that
+ launched it must not stop it, nor drop this process's lease, while the other key still drives it."""
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd = tmp_path / "pinned-profile"
+ alias = tmp_path / "alias"
+ alias.symlink_to(udd)
+ kw = dict(executable=str(script), ephemeral=False, headed=False, env=dict(os.environ), timeout=30)
+ first, err = launcher.launch_chromium(f"profile:{udd}", user_data_dir=str(udd), **kw)
+ assert err is None and first.owned
+ try:
+ second, err = launcher.launch_chromium(f"profile:{alias}", user_data_dir=str(alias), **kw)
+ assert err is None and second.pid == first.pid and not second.owned
+
+ launcher.stop_browser(f"profile:{udd}")
+ assert launcher._pid_alive(first.pid), "dropping one key stopped the browser the other key still drives"
+ assert launcher.get_browser(f"profile:{alias}") is second
+
+ launcher.stop_browser(f"profile:{alias}")
+ assert _gone(first.pid)
+ assert _launcher_state() == []
+ finally:
+ kill_fake(first.pid, script)
diff --git a/tools/AGENTS.md b/tools/AGENTS.md
index 334ed6eec8a..7aa9bd35cd6 100644
--- a/tools/AGENTS.md
+++ b/tools/AGENTS.md
@@ -85,9 +85,10 @@ hybrid `::local` sidecar > cloud provider (failure → local session marked `fal
real profile (consented) > Lightpanda > the packaged Chromium from `browser_launcher.launch_chromium`
(one browser per cache key, `--remote-debugging-port=0` + `DevToolsActivePort`). State lives in
`browser_session.py`; siblings late-import it inside functions. Every launched Chromium is stopped
-parent-first and recorded per profile (`<HERMES_HOME>/cache/browser/chromium-records`, a private 0700
-dir) so a later process serving that profile reaps a crashed owner's browser after checking pid + start
-time + its `--user-data-dir` argv token; the reaper walks every home the process serves.
+parent-first and recorded in a private 0700 dir (a throwaway profile's in
+`<HERMES_HOME>/cache/browser/chromium-records`; a persistent profile's per OS user under the host lock dir,
+one for every home pinning it) so a later process reaps a crashed owner's browser after checking pid + start
+time + its `--user-data-dir` argv token; the reaper walks every served home and every shared profile.
`browser_tool.py` and `browser_tool_install.py` are frozen old-updater stubs only (see their
docstrings); never grow them.
diff --git a/tools/browser_launcher.py b/tools/browser_launcher.py
index 2d220d7ff6a..3e3fd04485c 100644
--- a/tools/browser_launcher.py
+++ b/tools/browser_launcher.py
@@ -3,10 +3,11 @@
One browser per launcher key (the session layer's cache key, or ``profile:<dir>`` for a persistent
profile shared by every key). Chromium picks its own DevTools port (``--remote-debugging-port=0``)
and writes it to ``<user-data-dir>/DevToolsActivePort``; the CDP URL comes from that file, so there
-is no port race. Every owned launch leaves a record in its profile's private launcher state dir
-(``<HERMES_HOME>/cache/browser/chromium-records``, 0700) so a later Hermes process serving that profile
-can reap the Chromium of an owner that crashed, after proving the pid is still that browser. A shared-profile
-Chromium other Hermes processes attached to is stopped only by its last holder
+is no port race. Every owned launch leaves a record in a private (0700) launcher state dir so a later
+Hermes process can reap the Chromium of an owner that crashed, after proving the pid is still that browser:
+a throwaway profile's in its home (``<HERMES_HOME>/cache/browser/chromium-records``), a persistent profile's
+in that profile's own dir under :func:`_shared_profiles_root` (per OS user: every home pinning the dir shares
+it). A shared-profile Chromium other Hermes processes attached to is stopped only by its last holder
(``tools/browser_launcher_holders.py``).
Stop is always parent-first (``ProcessRegistry._terminate_host_pid``): Chromium reaps its own
@@ -512,17 +513,32 @@ def get_browser(key: str) -> Optional[LaunchedBrowser]:
return None
+def _heir_key(browser: LaunchedBrowser) -> Optional[str]:
+ """Another key of this process on the same shared-profile Chromium (one dir reached by two paths, e.g. a
+ symlink): it inherits this process's hold, so dropping ``browser``'s key must neither drop the lease nor
+ stop the browser."""
+ if browser.ephemeral:
+ return None
+ with _lock:
+ return next((k for k, b in _browsers.items() if b is not browser and not b.ephemeral
+ and b.pid == browser.pid and b.start_time == browser.start_time), None)
+
+
def _forget(key: str, browser: LaunchedBrowser, *, drop_record: bool = True) -> None:
proc = None
+ heir = _heir_key(browser)
if _browsers.get(key) is browser: # a replacement registered since is not this browser's to drop
_browsers.pop(key)
proc = _procs.pop(key, None)
+ if proc is not None and heir is not None:
+ _procs.setdefault(heir, proc) # the heir stops (and reaps) it when it is the last one out
+ proc = None
if proc is not None:
try:
proc.wait(timeout=0) # reap the zombie
except subprocess.TimeoutExpired:
pass
- if not browser.ephemeral:
+ if not browser.ephemeral and heir is None:
record = _record_path(browser)
if record is not None:
from tools.browser_launcher_holders import drop_lease
@@ -572,12 +588,12 @@ def _stop_locked(key: str) -> None:
_forget(key, browser)
return
from tools.browser_launcher_holders import records_lock
- with records_lock(record.parent):
- if _last_holder_out(record, browser):
+ with records_lock(record.parent): # the decision and the forget are one step: another key may be the heir
+ if _heir_key(browser) is None and _last_holder_out(record, browser):
_terminate(browser.pid, browser.start_time, proc)
record.unlink(missing_ok=True)
- with _lock:
- _forget(key, browser, drop_record=False)
+ with _lock:
+ _forget(key, browser, drop_record=False)
def _last_holder_out(record: Path, browser: LaunchedBrowser) -> bool:
@@ -614,8 +630,8 @@ def _ephemeral_root(home: str) -> Path:
def _records_dir(home: str) -> Path:
- """Per-profile (``home``'s, not the process's): a crashed owner's Chromium is reaped by the next Hermes
- process serving that profile. Read only after ``_private_dir`` vouches for it."""
+ """``home``'s throwaway-profile records (not the process's home): a crashed owner's Chromium is reaped by the
+ next Hermes process serving that profile. Read only after ``_private_dir`` vouches for it."""
return Path(home) / "cache" / "browser" / "chromium-records"
@@ -644,10 +660,31 @@ def _private_dir(path: Path) -> Optional[Path]:
return None
+def _shared_profiles_root() -> Path:
+ """Launcher state of persistent (shared) profiles, per OS user rather than per HERMES_HOME: two homes pinning
+ one ``browser.user_data_dir`` drive its one Chromium, so they must see one launch record, one lease set and
+ one lock. The host rendezvous root the gateway's own cross-profile locks use."""
+ from gateway.status import _get_lock_dir
+ return _get_lock_dir() / "browser-shared-profiles"
+
+
+def _profile_digest(user_data_dir: str) -> str:
+ """One profile dir, however it is spelled (a symlink, a trailing slash)."""
+ return hashlib.sha256(os.path.realpath(user_data_dir).encode()).hexdigest()
+
+
+def _state_dir(browser: LaunchedBrowser) -> Optional[Path]:
+ """The private dir holding ``browser``'s launch record, leases and lock: its home's records dir for a
+ throwaway profile, else the shared profile's own dir under :func:`_shared_profiles_root`."""
+ if browser.ephemeral:
+ return _private_dir(_records_dir(browser.home_key))
+ root = _private_dir(_shared_profiles_root())
+ return _private_dir(root / _profile_digest(browser.user_data_dir)[:32]) if root is not None else None
+
+
def _record_path(browser: LaunchedBrowser) -> Optional[Path]:
- records = _private_dir(_records_dir(browser.home_key))
- digest = hashlib.sha256(browser.user_data_dir.encode()).hexdigest()[:16]
- return records / f"{digest}-{browser.pid}.json" if records is not None else None
+ records = _state_dir(browser)
+ return records / f"{_profile_digest(browser.user_data_dir)[:16]}-{browser.pid}.json" if records is not None else None
def _write_record(browser: LaunchedBrowser) -> None:
@@ -692,7 +729,9 @@ def _cmdline_names_dir(pid: int, user_data_dir: str) -> bool:
for tok in argv)
-def _inside_ephemeral_root(user_data_dir: str, home: str) -> bool:
+def _inside_ephemeral_root(user_data_dir: str, home: Optional[str]) -> bool:
+ if home is None: # a shared-profile state dir holds no throwaway profiles
+ return False
try:
rel = Path(user_data_dir).resolve().relative_to(_ephemeral_root(home).resolve())
except (OSError, ValueError):
@@ -700,7 +739,7 @@ def _inside_ephemeral_root(user_data_dir: str, home: str) -> bool:
return bool(rel.parts)
-def _is_recorded_browser(record: dict, home: str) -> bool:
+def _is_recorded_browser(record: dict, home: Optional[str]) -> bool:
pid, udd = record.get("pid"), record.get("user_data_dir")
if not isinstance(pid, int) or not isinstance(udd, str) or not _pid_alive(pid):
return False
@@ -744,7 +783,7 @@ def _load_record(path: Path) -> Optional[dict]:
return None
-def _reap_record(path: Path, home: str) -> bool:
+def _reap_record(path: Path, home: Optional[str]) -> bool:
"""Caller holds the records lock."""
record = _load_record(path)
if record is None:
@@ -773,7 +812,24 @@ def _reap_record(path: Path, home: str) -> bool:
def _reap_home(home: str) -> int:
- records = _records_dir(home)
+ return _reap_state_dir(_records_dir(home), home)
+
+
+def _reap_shared_profiles() -> int:
+ """Every shared profile's records, whichever home launched its browser."""
+ root = _shared_profiles_root()
+ if not os.path.lexists(root) or not _is_private_dir(root):
+ return 0
+ try:
+ states = sorted(root.iterdir())
+ except OSError as exc:
+ logger.debug("browser: shared-profile launcher state unavailable in %s: %s", root, exc)
+ return 0
+ return sum(_reap_state_dir(state, None) for state in states)
+
+
+def _reap_state_dir(records: Path, home: Optional[str]) -> int:
+ """Reap one launcher state dir (``home``'s records, or a shared profile's dir when ``home`` is None)."""
if not os.path.lexists(records):
return 0
if not _is_private_dir(records):
@@ -835,7 +891,7 @@ def reap_orphaned_browsers(homes=()) -> int:
(``_served_home_keys``; the janitor and atexit run outside any profile scope, so each home's records
are read by path); returns the count. A pid is signalled only after its start time AND its
``--user-data-dir`` argv token match the record. Throwaway profiles of owners that are gone go too."""
- reaped = 0
+ reaped = _reap_shared_profiles()
for home in sorted(_served_home_keys(homes)):
reaped += _reap_home(home)
_reclaim_throwaway_dirs(home)
diff --git a/tools/browser_launcher_holders.py b/tools/browser_launcher_holders.py
index be91540c203..70f2418f69b 100644
--- a/tools/browser_launcher_holders.py
+++ b/tools/browser_launcher_holders.py
@@ -2,7 +2,8 @@
A persistent profile (``browser.user_data_dir``, the Bot Desktop profile, the real-profile copy) has ONE
Chromium, launched by whichever Hermes process got there first; every other process attaches to it. Each
-attacher leaves a lease beside the launch record in the profile's private records dir. The process holding
+attacher leaves a lease beside the launch record in the profile's private state dir (per OS user, so every
+Hermes home pinning the dir sees the same leases: ``browser_launcher._shared_profiles_root``). The process holding
the record (the launcher, or whoever it was handed to) stops the browser on exit only when no other live
Hermes process holds a lease; otherwise it hands the record to one of them, so the LAST holder stops it. A
holder that crashed is dropped once its pid and start time no longer match, never on the pid alone.