Skip to content

The agent that grows with you

fix(browser): starting a shared profile's browser is one locked check-launch-or-attach step

Teknium5bb282251045
diff --git a/tests/tools/test_browser_launcher_shared_start.py b/tests/tools/test_browser_launcher_shared_start.py
new file mode 100644
index 00000000000..1bfdf20c8eb
--- /dev/null
+++ b/tests/tools/test_browser_launcher_shared_start.py
@@ -0,0 +1,142 @@
+"""Starting a persistent (shared) profile's browser is one step under the profile's cross-process lock.
+
+Two Hermes processes starting one profile at once must end with ONE Chromium both drive: the later one waits,
+then attaches (a second launch would wipe the first's ``DevToolsActivePort`` and lose Chromium's singleton). An
+attach probes and leases under the same lock, so the owner's stop cannot fall between the live probe and the
+lease and hand out a dead endpoint (``tools/browser_launcher.py``). Runs the real launcher against a fake Chromium
+that takes the profile singleton and serves ``/json/version`` like the real one.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from tests.tools.browser_launcher_fakes import kill_fake, pids, write_fake_chromium
+from tests.waiting import wait_until
+from tools import browser_launcher as launcher
+
+pytestmark = [pytest.mark.platforms("linux"),
+ pytest.mark.live_system_guard_bypass] # the fake browsers are ours to stop if a leg failed
+
+_REPO = Path(__file__).resolve().parents[2]
+
+# A Hermes process on a persistent profile: prints "ready", launches once it reads "go", prints
+# "<pid> <owned> <cdp_url>" (or "ERR ..."), then exits normally on the next line.
+_RACER = textwrap.dedent("""\
+ import os, sys
+ import tools.browser_session # registers the exit teardown
+ from tools import browser_launcher as launcher
+ launcher.needs_chromium_sandbox_bypass = lambda: False
+ exe, udd = sys.argv[1], sys.argv[2]
+ print("ready", flush=True)
+ sys.stdin.readline()
+ browser, err = launcher.launch_chromium("profile:" + udd, executable=exe, user_data_dir=udd, ephemeral=False,
+ headed=False, env=dict(os.environ), timeout=30)
+ print(f"{browser.pid} {browser.owned} {browser.cdp_url}" if browser else f"ERR {err!r}", flush=True)
+ sys.stdin.readline()
+""")
+
+
+def _racer(script: Path, udd: Path, env: dict) -> subprocess.Popen:
+ proc = subprocess.Popen([sys.executable, "-c", _RACER, str(script), str(udd)], cwd=_REPO, env=env,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ assert proc.stdout.readline().strip() == "ready", proc.stderr.read()[-2000:]
+ return proc
+
+
+def _gone(pid: int) -> bool:
+ return wait_until(lambda: not launcher._pid_alive(pid), timeout=15, what=f"pid {pid} to exit")
+
+
+def test_two_processes_racing_a_first_launch_get_one_browser(tmp_path):
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd = tmp_path / "pinned-profile"
+ env = {**os.environ, "FAKE_CHROMIUM_READY_DELAY": "2"} # both check before the first one is ready
+ racers = [_racer(script, udd, env), _racer(script, udd, env)]
+ try:
+ for proc in racers:
+ proc.stdin.write("go\n")
+ proc.stdin.flush()
+ results = [proc.stdout.readline().split(" ", 2) for proc in racers]
+ assert not [r for r in results if r[0] == "ERR"], results
+ assert len({r[0] for r in results}) == 1 and len({r[2] for r in results}) == 1, results
+ assert sorted(r[1] for r in results) == ["False", "True"]
+ assert pids(udd, "spawns") == [int(results[0][0])], "a second Chromium was started on the same profile"
+
+ for proc in racers:
+ proc.communicate("\n", timeout=60)
+ assert _gone(int(results[0][0]))
+ finally:
+ for proc in racers:
+ proc.kill()
+ for pid in pids(udd, "spawns"):
+ kill_fake(pid, script)
+
+
+def test_an_owners_stop_cannot_fall_between_the_attach_probe_and_the_lease(tmp_path, monkeypatch):
+ """The owner exits (its stop runs) right after this process found its browser live: the stop waits for the
+ attach's lease and hands the browser over, so the endpoint this process got stays up."""
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd = tmp_path / "pinned-profile"
+ owner = _racer(script, udd, dict(os.environ))
+ owner.stdin.write("go\n")
+ owner.stdin.flush()
+ pid = int(owner.stdout.readline().split()[0])
+ real_probe, fired = launcher._devtools_endpoint_live, []
+
+ def probe_then_owner_exits(port, path):
+ live = real_probe(port, path)
+ if live and not fired:
+ fired.append(True)
+ owner.stdin.write("\n") # the owner's normal exit runs its stop now
+ owner.stdin.flush()
+ try:
+ owner.wait(timeout=3) # a stop not serialized with this attach finishes within this window
+ except subprocess.TimeoutExpired:
+ pass
+ return live
+
+ monkeypatch.setattr(launcher, "_devtools_endpoint_live", probe_then_owner_exits)
+ key = f"profile:{udd}"
+ try:
+ attached, err = launcher.launch_chromium(key, executable=str(script), user_data_dir=str(udd),
+ ephemeral=False, headed=False, env=dict(os.environ), timeout=30)
+ assert fired and err is None and attached.pid == pid and not attached.owned
+ owner.wait(timeout=30)
+ assert launcher._pid_alive(pid), "the owner stopped the browser this process had just attached to"
+
+ launcher.stop_browser(key) # now the last holder
+ assert _gone(pid)
+ finally:
+ owner.kill()
+ kill_fake(pid, script)
+
+
+def test_the_reaper_skips_a_profile_whose_first_launch_is_still_starting(tmp_path):
+ """A first launch holds the profile's lock for its whole readiness wait (up to 120 s): the janitor's reap
+ passes over that profile instead of stalling behind it, and leaves the starting browser alone."""
+ import threading
+ script = write_fake_chromium(tmp_path / "chromium")
+ udd = tmp_path / "pinned-profile"
+ starter = _racer(script, udd, {**os.environ, "FAKE_CHROMIUM_READY_DELAY": "600"})
+ reap = threading.Thread(target=launcher.reap_orphaned_browsers, daemon=True)
+ try:
+ starter.stdin.write("go\n")
+ starter.stdin.flush()
+ [pid] = wait_until(lambda: pids(udd, "launches"), timeout=30, what="the first launch's Chromium to start")
+ reap.start()
+ reap.join(timeout=20)
+ assert not reap.is_alive(), "the reaper is blocked behind a launch still in its readiness wait"
+ assert launcher._pid_alive(pid)
+ finally:
+ starter.kill()
+ starter.wait(timeout=10)
+ for spawned in pids(udd, "spawns"):
+ kill_fake(spawned, script)
+ reap.join(timeout=30)
diff --git a/tools/browser_launcher.py b/tools/browser_launcher.py
index 3e3fd04485c..5985b661f25 100644
--- a/tools/browser_launcher.py
+++ b/tools/browser_launcher.py
@@ -16,6 +16,7 @@ zygotes and renderers on SIGTERM, and killing a zygote first turns a clean exit
from __future__ import annotations
+import contextlib
import hashlib
import json
import logging
@@ -370,9 +371,8 @@ def _spawn(argv: list[str], env: dict, user_data_dir: str) -> subprocess.Popen:
**_spawn_kwargs())
-def _attach_running(key: str, user_data_dir: str, ephemeral: bool, home_key: str) -> Optional[LaunchedBrowser]:
- if ephemeral:
- return None
+def _attach_running(key: str, user_data_dir: str, home_key: str) -> Optional[LaunchedBrowser]:
+ """The live Chromium already on shared profile ``user_data_dir``, leased (caller holds its lock)."""
running = running_instance(user_data_dir)
if running is None:
return None
@@ -383,12 +383,29 @@ def _attach_running(key: str, user_data_dir: str, ephemeral: bool, home_key: str
record = _record_path(browser)
if record is not None:
from tools import browser_launcher_holders as holders
- with holders.records_lock(record.parent):
- _adopt_record(browser)
- holders.write_lease(record, browser) # its launcher's exit must not stop it under us
+ _adopt_record(browser)
+ holders.write_lease(record, browser) # its launcher's exit must not stop it under us
return browser
+def _start_shared(key: str, executable: str, udd: str, home_key: str,
+ **launch) -> tuple[Optional[LaunchedBrowser], Optional[str]]:
+ """Attach to, or launch, shared profile ``udd``'s one Chromium as ONE step under the profile's cross-process
+ lock: of two processes starting it at once the later waits, then attaches (a second launch would wipe the
+ first's DevToolsActivePort and lose Chromium's singleton), and an owner's stop cannot fall between the live
+ probe and the lease. A first launch holds the lock for its whole readiness wait; the reaper skips a busy
+ profile instead of waiting (:func:`_reap_shared_profiles`)."""
+ from tools import browser_launcher_holders as holders
+ state = _shared_state_dir(udd)
+ with holders.records_lock(state) if state is not None else contextlib.nullcontext():
+ attached = _attach_running(key, udd, home_key)
+ if attached is None:
+ return _launch_owned(key, executable, udd, ephemeral=False, home_key=home_key, **launch)
+ with _lock: # registered under the lock too: another key's stop must see this key as its heir
+ _browsers[key] = attached
+ return attached, None
+
+
def launch_chromium(key: str, *, executable: Optional[str] = None, user_data_dir: Optional[str] = None,
ephemeral: bool = True, headed: Optional[bool] = None, extra_args=(),
env: Optional[dict] = None, timeout: Optional[float] = None, sandbox_bypass: bool = True,
@@ -419,14 +436,11 @@ def launch_chromium(key: str, *, executable: Optional[str] = None, user_data_dir
return None, hint
executable = _resolve_executable()
udd = user_data_dir or _default_user_data_dir(key, home_key)
- attached = _attach_running(key, udd, ephemeral, home_key)
- if attached is not None:
- with _lock:
- _browsers[key] = attached
- return attached, None
- return _launch_owned(key, executable, udd, ephemeral=ephemeral, packaged=packaged, headed=headed,
- extra_args=extra_args, env=env, timeout=timeout, home_key=home_key,
- sandbox_bypass=sandbox_bypass)
+ launch = dict(packaged=packaged, headed=headed, extra_args=extra_args, env=env, timeout=timeout,
+ sandbox_bypass=sandbox_bypass)
+ if ephemeral: # a per-process dir: nobody else starts or attaches to it
+ return _launch_owned(key, executable, udd, ephemeral=True, home_key=home_key, **launch)
+ return _start_shared(key, executable, udd, home_key, **launch)
def _key_lock(key: str) -> threading.Lock:
@@ -673,13 +687,18 @@ def _profile_digest(user_data_dir: str) -> str:
return hashlib.sha256(os.path.realpath(user_data_dir).encode()).hexdigest()
+def _shared_state_dir(user_data_dir: str) -> Optional[Path]:
+ """Shared profile ``user_data_dir``'s own private dir (its launch record, leases and lock)."""
+ root = _private_dir(_shared_profiles_root())
+ return _private_dir(root / _profile_digest(user_data_dir)[:32]) if root is not None else None
+
+
def _state_dir(browser: LaunchedBrowser) -> Optional[Path]:
"""The private dir holding ``browser``'s launch record, leases and lock: its home's records dir for a
throwaway profile, else the shared profile's own dir under :func:`_shared_profiles_root`."""
if browser.ephemeral:
return _private_dir(_records_dir(browser.home_key))
- root = _private_dir(_shared_profiles_root())
- return _private_dir(root / _profile_digest(browser.user_data_dir)[:32]) if root is not None else None
+ return _shared_state_dir(browser.user_data_dir)
def _record_path(browser: LaunchedBrowser) -> Optional[Path]:
@@ -829,14 +848,19 @@ def _reap_shared_profiles() -> int:
def _reap_state_dir(records: Path, home: Optional[str]) -> int:
- """Reap one launcher state dir (``home``'s records, or a shared profile's dir when ``home`` is None)."""
+ """Reap one launcher state dir (``home``'s records, or a shared profile's dir when ``home`` is None). A
+ shared profile whose lock is taken (a first launch holds it for up to its whole readiness wait) is left
+ to the next pass."""
if not os.path.lexists(records):
return 0
if not _is_private_dir(records):
logger.debug("browser: not reaping from %s: not a private directory of this user", records)
return 0
from tools import browser_launcher_holders as holders
- with holders.records_lock(records):
+ with holders.records_lock(records, blocking=home is not None) as locked:
+ if not locked:
+ logger.debug("browser: %s is busy (a launch or stop holds it); reaping it next pass", records)
+ return 0
try:
paths = sorted(records.glob("*.json"))
except OSError as exc:
diff --git a/tools/browser_launcher_holders.py b/tools/browser_launcher_holders.py
index 70f2418f69b..abb6f1da300 100644
--- a/tools/browser_launcher_holders.py
+++ b/tools/browser_launcher_holders.py
@@ -44,36 +44,45 @@ def nt_lock_failure(exc: OSError, records: Path, expired: bool) -> Optional[OSEr
return None
-def _lock_nt(fd: int, records: Path) -> None:
+def _lock_nt(fd: int, records: Path, blocking: bool) -> bool:
import msvcrt
deadline = time.monotonic() + _NT_LOCK_TIMEOUT_S
while True:
try:
- msvcrt.locking(fd, msvcrt.LK_LOCK, 1)
- return
+ msvcrt.locking(fd, msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1)
+ return True
except OSError as exc:
- failure = nt_lock_failure(exc, records, time.monotonic() >= deadline)
+ failure = nt_lock_failure(exc, records, not blocking or time.monotonic() >= deadline)
if failure is exc:
raise
+ if not blocking:
+ return False
if failure is not None:
raise failure from exc
+def _lock_posix(fd: int, blocking: bool) -> bool:
+ import fcntl # windows-footgun: ok — POSIX only (records_lock's nt branch takes _lock_nt)
+ try:
+ fcntl.flock(fd, fcntl.LOCK_EX if blocking else fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ return False
+ return True
+
+
@contextlib.contextmanager
-def records_lock(records: Path) -> Iterator[None]:
- """Exclusive, cross-process lock on ``records`` (the profile's private launcher state dir). Never nested:
- two locks on one file in one process are two open file descriptions, and the second would wait forever."""
+def records_lock(records: Path, *, blocking: bool = True) -> Iterator[bool]:
+ """Exclusive, cross-process lock on ``records`` (a private launcher state dir); yields whether it is held
+ (always True when ``blocking``). Never nested: two locks on one file in one process are two open file
+ descriptions, and the second would wait forever."""
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
fd = os.open(records / _LOCK_NAME, flags, 0o600)
+ held = False
try:
- if os.name == "nt":
- _lock_nt(fd, records)
- else:
- import fcntl # windows-footgun: ok — the nt branch above
- fcntl.flock(fd, fcntl.LOCK_EX)
- yield
+ held = _lock_nt(fd, records, blocking) if os.name == "nt" else _lock_posix(fd, blocking)
+ yield held
finally:
- if os.name == "nt":
+ if held and os.name == "nt":
import msvcrt
with contextlib.suppress(OSError):
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)