Skip to content

The agent that grows with you

fix(browser): the sandbox Chromium is signalled only while its pid is still that browser

Tekniumd47ec1012c93
diff --git a/tests/tools/test_browser_cleanup.py b/tests/tools/test_browser_cleanup.py
index 764e657de3f..80595aaafa4 100644
--- a/tests/tools/test_browser_cleanup.py
+++ b/tests/tools/test_browser_cleanup.py
@@ -240,12 +240,14 @@ class _FakeSandbox:
def run_in(self, env, argv, **kw):
cmd = " ".join(argv)
+ if _stop_target(argv) is not None and "kill" in self.timeouts:
+ raise subprocess.TimeoutExpired(argv, kw.get("timeout"))
if argv[0] in self.timeouts:
raise subprocess.TimeoutExpired(argv, kw.get("timeout"))
if "setsid" in cmd:
self.running = 4242
self.launches += 1
- return _Completed(0, b"4242\n")
+ return _Completed(0, b"4242 777\n") # pid, start time
if argv[0] == "cat" and not self.port_published:
return _Completed(1)
if "/dev/tcp" in cmd:
@@ -261,13 +263,20 @@ class _FakeSandbox:
with contextlib.suppress(threading.BrokenBarrierError):
self.probe_barrier.wait(timeout=1.0)
return answer
- if argv[0] == "kill":
- self.killed.append(int(argv[-1]))
+ if _stop_target(argv) is not None:
+ self.killed.append(_stop_target(argv))
self.running = None
return _Completed(0)
return _Completed(1)
+def _stop_target(argv):
+ """The pid a ``stop_sandbox_browser`` exec signals (its identity check passed), else None."""
+ if argv[:2] == ["bash", "-c"] and "kill -TERM" in argv[2] and len(argv) > 4:
+ return int(argv[4])
+ return None
+
+
class _Completed:
def __init__(self, returncode, stdout=b""):
self.returncode, self.stdout, self.stderr = returncode, stdout, b""
@@ -430,13 +439,14 @@ class _FakeContainers:
if "setsid" in cmd:
self._next_pid += 1
self.running.setdefault(env, self._next_pid)
- return _Completed(0, f"{self._next_pid}\n".encode())
+ return _Completed(0, f"{self._next_pid} 777\n".encode())
if "/dev/tcp" in cmd or argv[0] == "cat":
pid = self.running.get(env)
return _Completed(0, f"9222\n/devtools/browser/{pid}\n".encode()) if pid else _Completed(3)
- if argv[0] == "kill":
- self.killed.append((env, int(argv[-1])))
- if self.running.get(env) == int(argv[-1]):
+ if _stop_target(argv) is not None:
+ pid = _stop_target(argv)
+ self.killed.append((env, pid))
+ if self.running.get(env) == pid:
del self.running[env]
return _Completed(0)
return _Completed(1)
diff --git a/tests/tools/test_browser_sandbox_stop_identity.py b/tests/tools/test_browser_sandbox_stop_identity.py
new file mode 100644
index 00000000000..66931bcc072
--- /dev/null
+++ b/tests/tools/test_browser_sandbox_stop_identity.py
@@ -0,0 +1,117 @@
+"""The sandbox Chromium is signalled only while its pid still names the browser this process launched.
+
+The "sandbox" here is this host through a plugin environment's ``exec_prefix`` (``env``), so the real launch,
+liveness and stop scripts run in a real shell against real processes. The stand-in Chromium and the unrelated
+job both block SIGTERM, so a delivered signal stays pending in ``/proc/<pid>/status`` (no timing involved).
+"""
+
+import signal
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+pytestmark = pytest.mark.platforms("linux")
+
+_SIGTERM_BIT = 1 << (signal.SIGTERM - 1)
+
+_STUB = textwrap.dedent("""\
+ #!{python}
+ import os, signal, socket, sys, time
+ signal.pthread_sigmask(signal.SIG_BLOCK, {{signal.SIGTERM}})
+ udd = next(a.split("=", 1)[1] for a in sys.argv if a.startswith("--user-data-dir="))
+ s = socket.socket(); s.bind(("127.0.0.1", 0)); s.listen(8)
+ tmp = os.path.join(udd, "DevToolsActivePort.tmp")
+ with open(tmp, "w") as f:
+ f.write(f"{{s.getsockname()[1]}}\\n/devtools/browser/stub\\n")
+ os.replace(tmp, os.path.join(udd, "DevToolsActivePort"))
+ time.sleep(120)
+ """)
+
+
+class _LocalSandbox:
+ """A plugin terminal environment whose exec prefix runs argv on this host."""
+
+ def exec_prefix(self, *, user=None, interactive=True):
+ return ["env"]
+
+
+def _sigterm_pending(pid: int) -> bool:
+ with open(f"/proc/{pid}/status", encoding="ascii") as fh:
+ fields = dict(line.split(":", 1) for line in fh if ":" in line)
+ return bool(int(fields["ShdPnd"].strip(), 16) & _SIGTERM_BIT)
+
+
+def local_sandbox(tmp_path, monkeypatch):
+ from tools import browser_session_sandbox
+ from tools.bot_desktop import runtime, sandbox_host
+ from tools.environments import streams
+
+ stub = tmp_path / "chromium"
+ stub.write_text(_STUB.format(python=sys.executable), encoding="utf-8")
+ stub.chmod(0o755)
+ env = _LocalSandbox()
+ monkeypatch.setattr(browser_session_sandbox, "_launched", {})
+ monkeypatch.setattr(browser_session_sandbox, "_confirmed", {})
+ monkeypatch.setattr(streams, "forward_port", lambda env, port, user=None: port)
+ monkeypatch.setattr(streams, "forward_failures", lambda env, port: 0)
+ monkeypatch.setattr(streams, "sandbox_temp_dir", lambda env: str(tmp_path))
+ monkeypatch.setattr(runtime, "_sandbox_env", lambda create=False: env)
+ monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":5"})
+ monkeypatch.setattr(sandbox_host, "_user_for", lambda env: None)
+ monkeypatch.setattr(sandbox_host, "browser_profile_dir", lambda env: str(tmp_path / "profile"))
+ monkeypatch.setattr(sandbox_host, "chromium_executable", lambda env: str(stub))
+ yield browser_session_sandbox
+ for entry in list(browser_session_sandbox._launched.values()):
+ _kill_if_ours(entry[1], tmp_path)
+
+
+def _kill_if_ours(pid: int, tmp_path) -> None:
+ """SIGKILL the stand-in Chromium from inside the "sandbox" (its own ``kill``, as a crash or the sandbox's
+ end would): it was started detached there, outside this test's process tree."""
+ try:
+ with open(f"/proc/{pid}/cmdline", "rb") as fh:
+ ours = str(tmp_path).encode() in fh.read()
+ except OSError:
+ return
+ if ours:
+ subprocess.run(["kill", "-KILL", str(pid)], check=False)
+
+
+def _launched_pid(sandbox_module, info) -> int:
+ return sandbox_module._launched[info["launcher_key"]][1]
+
+
+def test_stop_signals_the_chromium_it_launched(local_sandbox, tmp_path):
+ info = local_sandbox.create_sandbox_session("task-A")
+ pid = _launched_pid(local_sandbox, info)
+ try:
+ local_sandbox.stop_sandbox_browser(info)
+ assert _sigterm_pending(pid)
+ finally:
+ _kill_if_ours(pid, tmp_path)
+
+
+def test_stop_leaves_alone_a_pid_that_now_names_another_process(local_sandbox, tmp_path, caplog):
+ """Chromium crashed and the sandbox reused its pid for an unrelated job: stale-session eviction or idle
+ cleanup must not SIGTERM that job."""
+ info = local_sandbox.create_sandbox_session("task-A")
+ key = info["launcher_key"]
+ pid = _launched_pid(local_sandbox, info)
+ _kill_if_ours(pid, tmp_path) # the crash
+ job = subprocess.Popen( # the unrelated job now holding the recorded pid number
+ [sys.executable, "-c", "import signal,sys,time; signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGTERM}); "
+ "print('ready', flush=True); time.sleep(120)"], stdout=subprocess.PIPE, text=True)
+ try:
+ assert job.stdout.readline().strip() == "ready"
+ env, _pid, *identity = local_sandbox._launched[key]
+ local_sandbox._launched[key] = (env, job.pid, *identity)
+ with caplog.at_level("INFO", logger="tools.browser_session_sandbox"):
+ local_sandbox.stop_sandbox_browser(info)
+ assert not _sigterm_pending(job.pid), "an unrelated process on a reused pid was signalled"
+ assert str(job.pid) in caplog.text
+ finally:
+ job.kill()
+ job.wait()
diff --git a/tools/browser_session_sandbox.py b/tools/browser_session_sandbox.py
index 2d8bc135ea2..7945604238e 100644
--- a/tools/browser_session_sandbox.py
+++ b/tools/browser_session_sandbox.py
@@ -18,11 +18,13 @@ from typing import Any, Optional
logger = logging.getLogger(__name__)
_LAUNCH_TIMEOUT_S = 60.0
-# Sandbox Chromium this process launched, per shared key: ``(sandbox env, pid)``. Every session in that
-# container attaches to it, whichever profile it serves, so it is stopped only when the LAST of them tears
-# down (release_session_backend's in-use check on the shared ``launcher_key``); the env is kept so a
-# teardown outside the owner's scope hits the right sandbox.
-_launched: dict[str, tuple[Any, int]] = {}
+# Sandbox Chromium this process launched, per shared key: ``(sandbox env, pid, start time, user-data-dir)``.
+# Every session in that container attaches to it, whichever profile it serves, so it is stopped only when the
+# LAST of them tears down (release_session_backend's in-use check on the shared ``launcher_key``); the env is
+# kept so a teardown outside the owner's scope hits the right sandbox. The start time (``/proc/<pid>/stat``
+# field 22, read inside the sandbox at launch) and the ``--user-data-dir`` token are the identity a stop
+# re-checks there before signalling: a crashed Chromium's pid may name another job by then.
+_launched: dict[str, tuple[Any, int, Optional[str], str]] = {}
_launched_lock = threading.Lock()
# Per shared key: held across "is a browser running?" + launch + claim, and across the stop, so two sessions
# first using one container never both see no browser and both launch.
@@ -91,8 +93,22 @@ def _shared_key(env, udd: str) -> str:
return f"sandbox:{id(env):x}:{udd}"
-def _start_chromium(env, user, exe: str, udd: str, child_env: dict) -> int:
- """Launch detached in the sandbox; returns the sandbox pid."""
+# ``$p``'s start time (clock ticks since boot): field 22 of /proc/<pid>/stat, counted after the ``(comm)``
+# field, which may itself hold spaces and parentheses. Empty when the sandbox has no /proc.
+_START_TIME_OF_P = 's=$(cat /proc/"$p"/stat 2>/dev/null) && s=${s##*")"} && set -- $s && printf %s "${20}"'
+# Exit code of the stop script when the pid is gone or no longer the browser this process launched.
+_NOT_OURS_RC = 4
+# Signal "$1" only while it is still the launched browser: same start time ("$2"), and its argv carries the
+# launch's exact --user-data-dir token ("$3"). Check and signal in one exec inside the sandbox.
+_STOP_SCRIPT = (
+ f'p=$1; t=$({_START_TIME_OF_P}); [ -n "$t" ] && [ "$t" = "$2" ] || exit {_NOT_OURS_RC}; '
+ f'ok=; while IFS= read -r -d "" a; do [ "$a" = "$3" ] && ok=1; done < /proc/"$p"/cmdline; '
+ f'[ -n "$ok" ] || exit {_NOT_OURS_RC}; kill -TERM "$p"'
+)
+
+
+def _start_chromium(env, user, exe: str, udd: str, child_env: dict) -> tuple[int, Optional[str]]:
+ """Launch detached in the sandbox; returns its sandbox pid and start time (None when unreadable)."""
from tools.bot_desktop.browser import SHARED_PROFILE_ARGS
from tools.browser_launcher import CHROMIUM_SANDBOX_BYPASS_ARGS, build_chromium_argv
from tools.environments import streams
@@ -101,13 +117,17 @@ def _start_chromium(env, user, exe: str, udd: str, child_env: dict) -> int:
argv = build_chromium_argv(exe, udd, packaged=False, headed=True, sandbox_bypass=False,
extra_args=(*SHARED_PROFILE_ARGS, *CHROMIUM_SANDBOX_BYPASS_ARGS))
argv = list(dict.fromkeys(argv))
+ # setsid execs in place (a background child of a non-interactive shell leads no process group), so $! is
+ # Chromium itself, and its start time is fixed from the fork on.
script = (f"mkdir -p {shlex.quote(udd)} && rm -f {shlex.quote(udd)}/DevToolsActivePort && "
- f"(setsid {shlex.join(argv)} >/dev/null 2>&1 & echo $!)")
+ f"(setsid {shlex.join(argv)} >/dev/null 2>&1 & p=$!; echo \"$p $({_START_TIME_OF_P})\")")
proc = streams.run_in(env, ["bash", "-c", script], child_env=child_env, user=user, timeout=30)
- pid_text = proc.stdout.decode("utf-8", "replace").strip().splitlines()[-1:] if proc.returncode == 0 else []
- if not pid_text or not pid_text[0].isdigit():
+ last = proc.stdout.decode("utf-8", "replace").strip().splitlines()[-1:] if proc.returncode == 0 else []
+ words = last[0].split() if last else []
+ if not words or not words[0].isdigit():
raise RuntimeError(f"Chromium could not be started in the sandbox: {proc.stderr.decode('utf-8', 'replace')[-500:]}")
- return int(pid_text[0])
+ start = words[1] if len(words) > 1 and words[1].isdigit() else None
+ return int(words[0]), start
def _wait_for_port(env, user, udd: str) -> tuple[int, str]:
@@ -137,9 +157,9 @@ def _spawn_owned(env, user, udd: str, key: str, published: dict) -> int:
if not exe:
raise RuntimeError("the sandbox image has no Chromium to run the browser")
child_env = {**published, "TMPDIR": streams.sandbox_temp_dir(env)}
- pid = _start_chromium(env, user, exe, udd, child_env)
+ pid, start = _start_chromium(env, user, exe, udd, child_env)
with _launched_lock:
- _launched[key] = (env, pid)
+ _launched[key] = (env, pid, start, udd)
return pid
@@ -227,8 +247,19 @@ def _stop_launched(key: str) -> None:
return
from tools.bot_desktop import sandbox_host
from tools.environments import streams
- env, pid = launched
+ env, pid, start, udd = launched
+ if start is None:
+ logger.warning("browser: leaving sandbox Chromium pid %s running: its start time could not be read at "
+ "launch, so the pid cannot be proven to still be that browser", pid)
+ return
+ argv = ["bash", "-c", _STOP_SCRIPT, "hermes-sandbox-browser-stop", str(int(pid)), start, f"--user-data-dir={udd}"]
try:
- streams.run_in(env, ["kill", "-TERM", str(int(pid))], user=sandbox_host._user_for(env), timeout=15)
+ proc = streams.run_in(env, argv, user=sandbox_host._user_for(env), timeout=15)
except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
logger.debug("sandbox browser stop for pid %s failed: %s", pid, exc)
+ return
+ if proc.returncode == _NOT_OURS_RC:
+ logger.info("browser: not signalling sandbox pid %s: it is gone or no longer the Chromium launched on %s",
+ pid, udd)
+ elif proc.returncode != 0:
+ logger.debug("sandbox browser stop for pid %s exited %s", pid, proc.returncode)