The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): a shared-profile browser is stopped only by its last holder across processes
Tekniumb2527c1ce95c
diff --git a/tests/tools/test_browser_launcher.py b/tests/tools/test_browser_launcher.pyindex 4c84e09fbb1..287b062c8b8 100644--- a/tests/tools/test_browser_launcher.py+++ b/tests/tools/test_browser_launcher.py@@ -7,6 +7,7 @@ launcher runs for real (spawn, readiness poll, registry, records, parent-first sfrom __future__ import annotations+import contextlibimport jsonimport osimport signal@@ -270,6 +271,93 @@ def test_records_of_a_live_owner_are_left_alone(tmp_path, fake_chromium):assert list(_records().glob("*.json"))+# A Hermes process on a persistent profile: launches (or attaches to) its Chromium, prints the pid, waits+# for a line on stdin, then exits normally (the session layer's exit teardown stops its browsers).+_SHARED_PROFILE_USER = textwrap.dedent("""\+ import os, sys+ import tools.browser_session # registers the exit teardown+ from tools import browser_launcher as launcher+ launcher.needs_chromium_sandbox_bypass = lambda: False+ exe, udd, running = sys.argv[1], sys.argv[2], int(sys.argv[3])+ if running: # Chromium's singleton would hand a second launch to the running instance+ launcher.running_instance = lambda d: (45678, "/devtools/browser/fake-id", running) if d == udd else None+ browser, err = launcher.launch_chromium("profile:" + udd, executable=exe, user_data_dir=udd, ephemeral=False,+ headed=False, env=dict(os.environ), timeout=30)+ print(f"{browser.pid} {browser.owned}" if browser else err, flush=True)+ sys.stdin.readline()+""")+++def _shared_profile_user(fake_chromium, udd, running: int = 0) -> tuple[subprocess.Popen, int]:+ proc = subprocess.Popen([sys.executable, "-c", _SHARED_PROFILE_USER, str(fake_chromium), str(udd), str(running)],+ cwd=Path(__file__).resolve().parents[2], env=dict(os.environ), stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)+ line = proc.stdout.readline().split()+ assert line and line[0].isdigit(), proc.stderr.read()[-2000:] if proc.poll() is not None else line+ assert (line[1] == "True") is not bool(running)+ return proc, int(line[0])+++def _gone(pid: int, timeout: float = 15) -> bool:+ deadline = time.monotonic() + timeout+ while launcher._pid_alive(pid) and time.monotonic() < deadline:+ time.sleep(0.1)+ return not launcher._pid_alive(pid)+++def _kill_fake(pid: int, fake_chromium) -> None:+ with contextlib.suppress(OSError):+ if str(fake_chromium) in Path(f"/proc/{pid}/cmdline").read_text(errors="replace"):+ os.kill(pid, signal.SIGKILL)++[email protected]("linux")[email protected]_system_guard_bypass # the fake browser is ours to stop if a leg failed+def test_a_shared_profile_browser_outlives_its_launcher_while_another_process_holds_it(fake_chromium, tmp_path):+ """Two Hermes processes on one persistent profile: the one that launched its Chromium lets go (its normal+ exit) while the other still drives it. The browser stays up for the other one, and that last holder's+ own normal exit stops it, leaving no launcher state behind."""+ udd, key = tmp_path / "shared-profile", f"profile:{tmp_path / 'shared-profile'}"+ mine, err = launcher.launch_chromium(key, executable=str(fake_chromium), user_data_dir=str(udd), ephemeral=False,+ headed=False, env=dict(os.environ), timeout=30)+ assert err is None and mine.owned+ holder, _ = _shared_profile_user(fake_chromium, udd, running=mine.pid)+ try:+ launcher.stop_browser(key)+ assert launcher._pid_alive(mine.pid), "the launcher let go of a browser another process still drives"++ holder.communicate("\n", timeout=60) # the last holder exits normally+ assert _gone(mine.pid)+ assert not [p.name for p in _records().iterdir() if not p.name.startswith(".")]+ finally:+ holder.kill()+ _kill_fake(mine.pid, fake_chromium)++[email protected]("linux")[email protected]_system_guard_bypass # the fake browser is ours to stop if a leg failed+def test_a_crashed_launchers_shared_browser_is_reaped_only_after_its_last_holder_is_gone(fake_chromium, tmp_path):+ """The launcher crashed while another live Hermes process holds its browser: no reaper kills it. Once+ that holder crashed too (proven by pid + start time), the next reap stops it."""+ udd = tmp_path / "shared-profile"+ owner, pid = _shared_profile_user(fake_chromium, udd)+ holder, _ = _shared_profile_user(fake_chromium, udd, running=pid)+ try:+ owner.kill()+ owner.wait(timeout=10)+ assert launcher.reap_orphaned_browsers() == 0+ assert launcher._pid_alive(pid), "reaped while a live process still holds it"++ holder.kill()+ holder.wait(timeout=10)+ assert launcher.reap_orphaned_browsers() == 1+ assert _gone(pid)+ finally:+ owner.kill()+ holder.kill()+ _kill_fake(pid, fake_chromium)++@pytest.mark.parametrize("headed,display,expect_headless", [(False, ":1", True),(True, ":1", False),diff --git a/tools/browser_launcher.py b/tools/browser_launcher.pyindex 3b5ec2ebd4a..412b7dc1aab 100644--- a/tools/browser_launcher.py+++ b/tools/browser_launcher.py@@ -5,7 +5,9 @@ profile shared by every key). Chromium picks its own DevTools port (``--remote-dand writes it to ``<user-data-dir>/DevToolsActivePort``; the CDP URL comes from that file, so thereis no port race. Every owned launch leaves a record in its profile's private launcher state dir(``<HERMES_HOME>/cache/browser/chromium-records``, 0700) so a later Hermes process serving that profile-can reap the Chromium of an owner that crashed, after proving the pid is still that browser.+can reap the Chromium of an owner that crashed, after proving the pid is still that browser. A shared-profile+Chromium other Hermes processes attached to is stopped only by its last holder+(``tools/browser_launcher_holders.py``).Stop is always parent-first (``ProcessRegistry._terminate_host_pid``): Chromium reaps its ownzygotes and renderers on SIGTERM, and killing a zygote first turns a clean exit into a crash dump.@@ -366,7 +368,12 @@ def _attach_running(key: str, user_data_dir: str, ephemeral: bool, home_key: strlogger.info("browser: attaching to the Chromium already running on %s (pid %s)", user_data_dir, pid)browser = _make_browser(key, port, path, pid=pid, start_time=_start_time(pid), user_data_dir=user_data_dir,owned=False, ephemeral=False, home_key=home_key)- _adopt_record(browser)+ record = _record_path(browser)+ if record is not None:+ from tools import browser_launcher_holders as holders+ with holders.records_lock(record.parent):+ _adopt_record(browser)+ holders.write_lease(record, browser) # its launcher's exit must not stop it under usreturn browser@@ -382,6 +389,10 @@ def launch_chromium(key: str, *, executable: Optional[str] = None, user_data_dira live Chromium on it is ATTACHED (``owned=False``, never killed). ``sandbox_bypass=False`` (theuser's real browser) never gets the host's sandbox-bypass flags."""from hermes_constants import hermes_home_key+ # Loaded now, not first at teardown: stopping the browser from an atexit hook must not be this module's+ # first import (Python refuses new atexit registrations during shutdown, orphaning the browser). An+ # attacher needs it too: it is the one that stops a shared browser when it is the last holder.+ import tools.process_registrywith _key_lock(key): # one launch per key; other keys launch concurrentlyexisting = get_browser(key)if existing is not None:@@ -417,9 +428,6 @@ def _launch_owned(key: str, executable: str, udd: str, *, ephemeral: bool, packaif env is None:from tools.browser_session import build_browser_envenv = build_browser_env()- # Loaded now, not first at teardown: stopping the browser from an atexit hook must not be this module's- # first import (Python refuses new atexit registrations during shutdown, orphaning the browser).- import tools.process_registrytry:_prepare_dir(udd, ephemeral)except OSError as exc:@@ -474,7 +482,7 @@ def get_browser(key: str) -> Optional[LaunchedBrowser]:return None-def _forget(key: str, browser: LaunchedBrowser) -> None:+def _forget(key: str, browser: LaunchedBrowser, *, drop_record: bool = True) -> None:_browsers.pop(key, None)proc = _procs.pop(key, None)if proc is not None:@@ -482,7 +490,12 @@ def _forget(key: str, browser: LaunchedBrowser) -> None:proc.wait(timeout=0) # reap the zombieexcept subprocess.TimeoutExpired:pass- if browser.owned:+ if not browser.ephemeral:+ record = _record_path(browser)+ if record is not None:+ from tools.browser_launcher_holders import drop_lease+ drop_lease(record)+ if browser.owned and drop_record:_remove_record(browser)if browser.ephemeral:shutil.rmtree(browser.user_data_dir, ignore_errors=True)@@ -505,16 +518,46 @@ def _terminate(pid: Optional[int], start_time: Optional[int], proc: Optional[subdef stop_browser(key: str) -> None:- """Stop ``key``'s browser if this process launched it (an attached one is only forgotten)."""+ """Let go of ``key``'s browser. A throwaway one this process launched is stopped. A shared-profile one is+ stopped only by its LAST holder across Hermes processes (:func:`_last_holder_out`); one a human started is+ never stopped."""with _lock:browser = _browsers.get(key)proc = _procs.get(key)if browser is None:return- if browser.owned:- _terminate(browser.pid, browser.start_time, proc)+ record = None if browser.ephemeral else _record_path(browser)+ if record is None:+ if browser.owned:+ _terminate(browser.pid, browser.start_time, proc)+ with _lock:+ _forget(key, browser)+ return+ from tools.browser_launcher_holders import records_lock+ with records_lock(record.parent):+ if _last_holder_out(record, browser):+ _terminate(browser.pid, browser.start_time, proc)+ record.unlink(missing_ok=True)with _lock:- _forget(key, browser)+ _forget(key, browser, drop_record=False)+++def _last_holder_out(record: Path, browser: LaunchedBrowser) -> bool:+ """Drop this process's hold on shared-profile ``browser`` (caller holds the records lock). True when it+ must stop it now: no other live Hermes process holds it, and Hermes launched it (a launch record names+ it). Otherwise a record this process holds, or whose owner is gone, goes to a live holder."""+ from tools import browser_launcher_holders as holders+ holders.drop_lease(record)+ others = holders.other_live_holders(record, browser.start_time)+ data = _load_record(record) if record.exists() else None+ if data is not None and data.get("start_time") != browser.start_time:+ data = None # an earlier browser on a recycled pid+ held_elsewhere = data is not None and data.get("owner_pid") != os.getpid() and _owner_alive(data)+ if others and data is not None and not held_elsewhere:+ holders.hand_over(record, data, others[0])+ if others or held_elsewhere:+ return False+ return browser.owned or data is not Nonedef stop_all_browsers(home_key: Optional[str] = None) -> None:@@ -664,6 +707,7 @@ def _load_record(path: Path) -> Optional[dict]:def _reap_record(path: Path, home: str) -> bool:+ """Caller holds the records lock."""record = _load_record(path)if record is None:return False@@ -673,6 +717,11 @@ def _reap_record(path: Path, home: str) -> bool:return Falseif record.get("owner_pid") != os.getpid() and _owner_alive(record):return False+ from tools import browser_launcher_holders as holders+ others = holders.other_live_holders(path, record.get("start_time"))+ if others: # another live Hermes process still drives it+ holders.hand_over(path, record, others[0])+ return Falsereaped = _is_recorded_browser(record, home)if reaped:_terminate(record["pid"], record.get("start_time"), None)@@ -691,12 +740,16 @@ def _reap_home(home: str) -> int:if not _is_private_dir(records):logger.debug("browser: not reaping from %s: not a private directory of this user", records)return 0- try:- paths = sorted(records.glob("*.json"))- except OSError as exc:- logger.debug("browser: launch records unavailable in %s: %s", records, exc)- return 0- return sum(1 for path in paths if _reap_record(path, home))+ from tools import browser_launcher_holders as holders+ with holders.records_lock(records):+ try:+ paths = sorted(records.glob("*.json"))+ except OSError as exc:+ logger.debug("browser: launch records unavailable in %s: %s", records, exc)+ return 0+ reaped = sum(1 for path in paths if _reap_record(path, home))+ holders.drop_dead_leases(records)+ return reapeddef _served_home_keys(homes) -> set[str]:diff --git a/tools/browser_launcher_holders.py b/tools/browser_launcher_holders.pynew file mode 100644index 00000000000..4f8844d7537--- /dev/null+++ b/tools/browser_launcher_holders.py@@ -0,0 +1,125 @@+"""Cross-process holders of a shared-profile Chromium (``tools/browser_launcher.py``).++A persistent profile (``browser.user_data_dir``, the Bot Desktop profile, the real-profile copy) has ONE+Chromium, launched by whichever Hermes process got there first; every other process attaches to it. Each+attacher leaves a lease beside the launch record in the profile's private records dir. The process holding+the record (the launcher, or whoever it was handed to) stops the browser on exit only when no other live+Hermes process holds a lease; otherwise it hands the record to one of them, so the LAST holder stops it. A+holder that crashed is dropped once its pid and start time no longer match, never on the pid alone.++Every decision runs under the records dir's lock: a lease written while another process decides cannot be+missed. Siblings late-import the facade (its ``_pid_alive`` / ``_start_time`` are the patched seams).+"""++from __future__ import annotations++import contextlib+import json+import logging+import os+from pathlib import Path+from typing import Iterator, Optional++logger = logging.getLogger(__name__)++_LOCK_NAME = ".holders.lock"+++def records_lock(records: Path) -> Iterator[None]:+ """Exclusive, cross-process lock on ``records`` (the profile's private launcher state dir). Never nested:+ two locks on one file in one process are two open file descriptions, and the second would wait forever."""+ flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)+ fd = os.open(records / _LOCK_NAME, flags, 0o600)+ try:+ if os.name == "nt":+ import msvcrt+ while True:+ try:+ msvcrt.locking(fd, msvcrt.LK_LOCK, 1) # retries for ~10 s, then raises+ break+ except OSError:+ continue+ else:+ import fcntl # windows-footgun: ok — the nt branch above+ fcntl.flock(fd, fcntl.LOCK_EX)+ yield+ finally:+ if os.name == "nt":+ import msvcrt+ with contextlib.suppress(OSError):+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)+ os.close(fd)+++def _lease_path(record: Path, holder_pid: int) -> Path:+ return record.with_name(f"{record.stem}@{holder_pid}.lease")+++def _load(path: Path) -> Optional[dict]:+ try:+ data = json.loads(path.read_text(encoding="utf-8-sig"))+ except (OSError, ValueError):+ data = None+ if isinstance(data, dict):+ return data+ path.unlink(missing_ok=True)+ return None+++def process_alive(pid, start_time) -> bool:+ """``pid`` is still the process that wrote a record or lease: alive, and with the recorded start time+ (a recycled pid has another one)."""+ from tools import browser_launcher as launcher+ if not isinstance(pid, int) or not launcher._pid_alive(pid):+ return False+ return start_time is None or launcher._start_time(pid) == start_time+++def write_lease(record: Path, browser) -> None:+ """This process holds ``browser`` (caller holds :func:`records_lock`)."""+ from tools import browser_launcher as launcher+ from utils import atomic_json_write+ me = os.getpid()+ atomic_json_write(_lease_path(record, me), {+ "holder_pid": me, "holder_start_time": launcher._start_time(me), "pid": browser.pid,+ "start_time": browser.start_time, "user_data_dir": browser.user_data_dir}, indent=None)+++def drop_lease(record: Path) -> None:+ _lease_path(record, os.getpid()).unlink(missing_ok=True)+++def other_live_holders(record: Path, browser_start_time) -> list[dict]:+ """Leases of OTHER live processes on the browser ``record`` names (caller holds :func:`records_lock`).+ A lease of a dead holder, or one naming an earlier browser on the same pid, is removed."""+ holders = []+ for path in sorted(record.parent.glob(f"{record.stem}@*.lease")):+ lease = _load(path)+ if lease is None or lease.get("holder_pid") == os.getpid():+ continue+ if lease.get("start_time") != browser_start_time or not process_alive(+ lease.get("holder_pid"), lease.get("holder_start_time")):+ path.unlink(missing_ok=True)+ continue+ holders.append(lease)+ return holders+++def drop_dead_leases(records: Path) -> None:+ """Leases whose holder is gone (caller holds :func:`records_lock`): a lease outlives its launch record when+ the browser died while its holder crashed."""+ for path in records.glob("*.lease"):+ lease = _load(path)+ if lease is not None and not process_alive(lease.get("holder_pid"), lease.get("holder_start_time")):+ path.unlink(missing_ok=True)+++def hand_over(record: Path, data: dict, holder: dict) -> None:+ """Make ``holder`` the record's owner: every reaper now sees a live owner, and the holder stops the+ browser when it is the last one out."""+ from utils import atomic_json_write+ atomic_json_write(record, {**data, "owner_pid": holder["holder_pid"],+ "owner_start_time": holder.get("holder_start_time")}, indent=None)+ logger.info("browser: Chromium pid %s is still used by Hermes pid %s; handed it over instead of stopping it",+ data.get("pid"), holder["holder_pid"])