Skip to content

The agent that grows with you

fix(browser): a failed sandbox liveness exec no longer kills the live browser

Tekniume5aee7e8dc1d
diff --git a/tests/tools/test_browser_cleanup.py b/tests/tools/test_browser_cleanup.py
index 2f9dfaadf92..301cda41cfa 100644
--- a/tests/tools/test_browser_cleanup.py
+++ b/tests/tools/test_browser_cleanup.py
@@ -1,6 +1,7 @@
"""Regression tests for browser session cleanup (tools/browser_session_lifecycle.py)."""
import os
+import subprocess
import sys
from unittest.mock import patch
@@ -225,14 +226,23 @@ class _FakeSandbox:
def __init__(self):
self.running = None
self.killed = []
+ self.probes = 0 # liveness execs (the /dev/tcp check)
+ self.probe_failure = None # "exit-255" (docker/ssh transport error) or "timeout"
+ self.forward_failures = 0 # connections through the port forward that never reached Chromium
def run_in(self, env, argv, **kw):
cmd = " ".join(argv)
if "setsid" in cmd:
self.running = 4242
return _Completed(0, b"4242\n")
+ if "/dev/tcp" in cmd:
+ self.probes += 1
+ if self.probe_failure == "exit-255":
+ return _Completed(255)
+ if self.probe_failure == "timeout":
+ raise subprocess.TimeoutExpired(argv, kw.get("timeout"))
if "/dev/tcp" in cmd or argv[0] == "cat":
- return _Completed(0, b"9222\n/devtools/browser/abc\n") if self.running else _Completed(1)
+ return _Completed(0, b"9222\n/devtools/browser/abc\n") if self.running else _Completed(3)
if argv[0] == "kill":
self.killed.append(int(argv[-1]))
self.running = None
@@ -252,8 +262,10 @@ def fake_sandbox(monkeypatch, isolated_sessions):
from tools.environments import streams
sandbox, env = _FakeSandbox(), object()
monkeypatch.setattr(browser_session_sandbox, "_launched", {}, raising=False)
+ monkeypatch.setattr(browser_session_sandbox, "_confirmed", {}, raising=False)
monkeypatch.setattr(streams, "run_in", sandbox.run_in)
monkeypatch.setattr(streams, "forward_port", lambda env, port, user=None: 50000 + port)
+ monkeypatch.setattr(streams, "forward_failures", lambda env, port: sandbox.forward_failures, raising=False)
monkeypatch.setattr(streams, "sandbox_temp_dir", lambda env: "/tmp") # no-tmp: ok — sandbox-side path
monkeypatch.setattr(runtime, "_sandbox_env", lambda create=False: env)
monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":5"})
@@ -284,4 +296,28 @@ class TestSandboxSharedBrowser:
assert bs._backend_dead(info) is False
fake_sandbox.running = None # crashed inside the sandbox
+ fake_sandbox.forward_failures += 1 # the harness's next connection through the forward fails
assert bs._backend_dead(info) is True
+
+ def test_a_failed_or_timed_out_liveness_exec_keeps_the_live_browser(self, fake_sandbox, isolated_sessions,
+ monkeypatch):
+ """Only a definitive answer from inside the sandbox ("no DevTools port file", "port closed") means
+ Chromium is gone; a docker/ssh exec that fails or times out says nothing about it."""
+ from tools.browser_session_sandbox import create_sandbox_session
+ monkeypatch.setattr(bs, "_create_session_for_key", lambda key, force_local: create_sandbox_session(key))
+ first = bs.get_session_info("task-A")
+ for failure in ("exit-255", "timeout"):
+ fake_sandbox.forward_failures += 1 # a connection through the forward failed: liveness is re-checked
+ fake_sandbox.probe_failure = failure
+ assert bs.get_session_info("task-A") is first, failure
+ assert fake_sandbox.killed == [] and fake_sandbox.running == 4242
+
+ def test_a_healthy_sandbox_session_spends_no_exec_per_browser_call(self, fake_sandbox, isolated_sessions,
+ monkeypatch):
+ from tools.browser_session_sandbox import create_sandbox_session
+ monkeypatch.setattr(bs, "_create_session_for_key", lambda key, force_local: create_sandbox_session(key))
+ first = bs.get_session_info("task-A")
+ fake_sandbox.probes = 0
+ for _ in range(5):
+ assert bs.get_session_info("task-A") is first
+ assert fake_sandbox.probes == 0
diff --git a/tests/tools/test_sandbox_port_forward.py b/tests/tools/test_sandbox_port_forward.py
new file mode 100644
index 00000000000..5c6228d80b0
--- /dev/null
+++ b/tests/tools/test_sandbox_port_forward.py
@@ -0,0 +1,71 @@
+"""The sandbox port forward (tools/environments/streams.py) reports connections that never reached their
+target: the sandbox browser's liveness is re-checked only after one (tools/browser_session_sandbox.py)."""
+
+import socket
+import subprocess
+import sys
+import threading
+import time
+
+import pytest
+
+from tools.environments import streams
+
+
+def local_relay(monkeypatch):
+ """The relay runs on this host instead of inside a sandbox: same script, same pipes."""
+ def open_stream(env, argv, **kw):
+ return subprocess.Popen([sys.executable, *argv[1:]], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL)
+ monkeypatch.setattr(streams, "open_stream", open_stream)
+ opened = []
+ yield opened
+ for env, port in opened:
+ streams._forwards.pop((id(env), "127.0.0.1", port)).close()
+
+
+def _closed_port() -> int:
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _one_shot_server(reply: bytes) -> int:
+ srv = socket.socket()
+ srv.bind(("127.0.0.1", 0))
+ srv.listen(1)
+
+ def serve():
+ conn, _ = srv.accept()
+ with conn, srv:
+ conn.sendall(reply)
+ threading.Thread(target=serve, daemon=True).start()
+ return srv.getsockname()[1]
+
+
+def _relay(local_port: int) -> bytes:
+ with socket.create_connection(("127.0.0.1", local_port), timeout=10) as conn:
+ conn.settimeout(10)
+ conn.shutdown(socket.SHUT_WR) # the client is done sending; the relay ends with the target's reply
+ chunks = []
+ while chunk := conn.recv(65536):
+ chunks.append(chunk)
+ return b"".join(chunks)
+
+
+def test_only_a_connection_that_never_reaches_its_target_counts_as_a_forward_failure(local_relay):
+ env = object()
+ live = _one_shot_server(b"devtools")
+ local_relay.append((env, live))
+ assert _relay(streams.forward_port(env, live)) == b"devtools"
+
+ dead = _closed_port() # Chromium died: nothing listens on its DevTools port any more
+ local_relay.append((env, dead))
+ assert _relay(streams.forward_port(env, dead)) == b""
+
+ deadline = time.monotonic() + 10
+ while streams.forward_failures(env, dead) == 0 and time.monotonic() < deadline:
+ time.sleep(0.05)
+ assert streams.forward_failures(env, dead) == 1
+ assert streams.forward_failures(env, live) == 0
diff --git a/tools/browser_session_sandbox.py b/tools/browser_session_sandbox.py
index 49f5f456b85..303cd35442b 100644
--- a/tools/browser_session_sandbox.py
+++ b/tools/browser_session_sandbox.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import logging
import shlex
+import subprocess
import threading
import time
from typing import Any, Optional
@@ -23,6 +24,14 @@ _LAUNCH_TIMEOUT_S = 60.0
# right sandbox.
_launched: dict[str, tuple[Any, int]] = {}
_launched_lock = threading.Lock()
+# Endpoints a liveness probe confirmed: ``cdp_url -> (id(env), sandbox port, forward failures then)``. A browser
+# call re-probes only after a connection through its forward failed (or the sandbox was replaced), so a
+# healthy call spends no exec in the sandbox.
+_confirmed: dict[str, tuple[int, int, int]] = {}
+# Exit code of the liveness script when the sandbox answers "not running": no DevTools port file, or nothing
+# listens on its port. Any other failure (docker/ssh transport error, timeout) says nothing about Chromium.
+_GONE_RC = 3
+GONE = "gone"
def _sandbox():
@@ -47,16 +56,25 @@ def _read_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:
return None
-def _live_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:
- """``(port, path)`` of the Chromium on ``udd`` while its DevTools port still answers, in one exec (a
- crashed Chromium leaves a stale ``DevToolsActivePort`` behind)."""
+def _live_devtools(env, user, udd: str):
+ """One exec: ``(port, path)`` of the Chromium on ``udd`` while its DevTools port answers; ``GONE`` when the
+ sandbox says it is not running (a crashed Chromium leaves a stale ``DevToolsActivePort`` behind); None
+ when the exec itself failed or timed out, which proves nothing either way."""
from tools.environments import streams
active = shlex.quote(f"{udd}/DevToolsActivePort")
- script = f'p=$(head -n1 {active}) && exec 3<>"/dev/tcp/127.0.0.1/$p" && cat {active}'
- proc = streams.run_in(env, ["bash", "-c", script], user=user, timeout=10)
+ script = (f'[ -s {active} ] || exit {_GONE_RC}; p=$(head -n1 {active}); '
+ f'( exec 3<>"/dev/tcp/127.0.0.1/$p" ) 2>/dev/null || exit {_GONE_RC}; cat {active}')
+ try:
+ proc = streams.run_in(env, ["bash", "-c", script], user=user, timeout=10)
+ except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
+ logger.debug("sandbox browser liveness probe failed: %s", exc)
+ return None
+ if proc.returncode == _GONE_RC:
+ return GONE
lines = proc.stdout.decode("utf-8", "replace").splitlines() if proc.returncode == 0 else []
if len(lines) >= 2 and lines[0].strip().isdigit() and lines[1].startswith("/devtools/browser/"):
return int(lines[0].strip()), lines[1].strip()
+ logger.debug("sandbox browser liveness probe inconclusive (exit %s)", proc.returncode)
return None
@@ -106,6 +124,9 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:
key = _shared_key(udd)
found = _live_devtools(env, user, udd)
if found is None:
+ # Launching blind would wipe a live browser's DevToolsActivePort and lose to its singleton.
+ raise RuntimeError("the terminal backend's sandbox did not answer whether its browser is running; try again")
+ if found is GONE:
exe = sandbox_host.chromium_executable(env)
if not exe:
raise RuntimeError("the sandbox image has no Chromium to run the browser")
@@ -116,6 +137,7 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:
_launched[key] = (env, pid)
local_port = streams.forward_port(env, found[0], user=user)
info = _session_record("sb", f"ws://127.0.0.1:{local_port}{found[1]}", {"local": True, "sandbox": True})
+ _confirmed[info["cdp_url"]] = (id(env), found[0], streams.forward_failures(env, found[0]))
info.update(shared_profile=True, launcher_key=key)
logger.info("Created sandbox browser session %s for %s", info["session_name"], cache_key)
return info
@@ -123,26 +145,38 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:
def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:
"""The sandbox Chromium still serves this session's endpoint through the current port forward: False once
- it died, was relaunched (new DevTools path) or the sandbox was replaced (new forward)."""
+ the sandbox answers that it died, was relaunched (new DevTools path) or the sandbox was replaced (new
+ forward). Re-probed only after a connection through the forward failed; a probe that cannot reach the
+ sandbox keeps the session."""
from tools.bot_desktop import runtime as _bd_runtime, sandbox_host
from tools.environments import streams
env = _bd_runtime._sandbox_env(create=False)
if env is None:
return False
+ cdp_url = session_info.get("cdp_url") or ""
+ confirmed = _confirmed.get(cdp_url)
+ if confirmed and confirmed[0] == id(env) and streams.forward_failures(env, confirmed[1]) == confirmed[2]:
+ return True
try:
user = sandbox_host._user_for(env)
- found = _live_devtools(env, user, sandbox_host.browser_profile_dir(env))
- except (OSError, RuntimeError) as exc:
+ udd = sandbox_host.browser_profile_dir(env)
+ except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
logger.debug("sandbox browser liveness probe failed: %s", exc)
- return False
+ return True
+ found = _live_devtools(env, user, udd)
if found is None:
+ return True
+ if found is GONE or cdp_url != f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}":
+ _confirmed.pop(cdp_url, None)
return False
- return session_info.get("cdp_url") == f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}"
+ _confirmed[cdp_url] = (id(env), found[0], streams.forward_failures(env, found[0]))
+ return True
def stop_sandbox_browser(session_info: dict[str, Any]) -> None:
"""SIGTERM the sandbox Chromium this process launched on the session's shared key (best effort; the
sandbox's lifetime bounds it). The caller guarantees no other live session uses it."""
+ _confirmed.pop(session_info.get("cdp_url") or "", None)
with _launched_lock:
launched = _launched.pop(session_info.get("launcher_key") or "", None)
if launched is None:
diff --git a/tools/environments/streams.py b/tools/environments/streams.py
index 4d659657e80..3240655637a 100644
--- a/tools/environments/streams.py
+++ b/tools/environments/streams.py
@@ -170,6 +170,9 @@ class PortForward:
self._server.listen(16)
self.local_port: int = self._server.getsockname()[1]
self._closed = False
+ # Connections that never reached the target (relay not started, or it exited non-zero before
+ # relaying a byte back): the one cheap signal that whatever listened there may be gone.
+ self.failures = 0
threading.Thread(target=self._accept_loop, name=f"port-forward-{self.local_port}", daemon=True).start()
def _accept_loop(self) -> None:
@@ -186,6 +189,7 @@ class PortForward:
user=self.user)
except Exception:
conn.close()
+ self.failures += 1
return
def to_remote() -> None:
@@ -205,11 +209,13 @@ class PortForward:
pass
threading.Thread(target=to_remote, daemon=True).start()
+ relayed = False
try:
while True:
data = proc.stdout.read1(65536) if hasattr(proc.stdout, "read1") else proc.stdout.read(65536)
if not data:
break
+ relayed = True
conn.sendall(data)
except OSError:
pass
@@ -217,6 +223,8 @@ class PortForward:
try:
conn.close()
finally:
+ if not relayed and _exited_nonzero(proc):
+ self.failures += 1
proc.kill()
def close(self) -> None:
@@ -227,6 +235,21 @@ class PortForward:
pass
+def _exited_nonzero(proc: subprocess.Popen) -> bool:
+ try:
+ return proc.wait(timeout=2) != 0
+ except subprocess.TimeoutExpired:
+ return False
+
+
+def forward_failures(env: BaseEnvironment, remote_port: int, *, remote_host: str = "127.0.0.1") -> int:
+ """How many connections through ``forward_port(env, remote_port)`` never reached the target (0 while no
+ forward is open). A caller caching "the target is up" re-checks only once this changes."""
+ with _forwards_lock:
+ fwd = _forwards.get((id(env), remote_host, int(remote_port)))
+ return fwd.failures if fwd is not None else 0
+
+
def forward_port(env: BaseEnvironment, remote_port: int, *, remote_host: str = "127.0.0.1",
user: Optional[str] = None) -> int:
"""Local port on 127.0.0.1 that reaches ``remote_host:remote_port`` inside ``env``; one listener per