The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(browser): a failed sandbox liveness exec no longer kills the live browser
Tekniume5aee7e8dc1d
diff --git a/tests/tools/test_browser_cleanup.py b/tests/tools/test_browser_cleanup.pyindex 2f9dfaadf92..301cda41cfa 100644--- a/tests/tools/test_browser_cleanup.py+++ b/tests/tools/test_browser_cleanup.py@@ -1,6 +1,7 @@"""Regression tests for browser session cleanup (tools/browser_session_lifecycle.py)."""import os+import subprocessimport sysfrom unittest.mock import patch@@ -225,14 +226,23 @@ class _FakeSandbox:def __init__(self):self.running = Noneself.killed = []+ self.probes = 0 # liveness execs (the /dev/tcp check)+ self.probe_failure = None # "exit-255" (docker/ssh transport error) or "timeout"+ self.forward_failures = 0 # connections through the port forward that never reached Chromiumdef run_in(self, env, argv, **kw):cmd = " ".join(argv)if "setsid" in cmd:self.running = 4242return _Completed(0, b"4242\n")+ if "/dev/tcp" in cmd:+ self.probes += 1+ if self.probe_failure == "exit-255":+ return _Completed(255)+ if self.probe_failure == "timeout":+ raise subprocess.TimeoutExpired(argv, kw.get("timeout"))if "/dev/tcp" in cmd or argv[0] == "cat":- return _Completed(0, b"9222\n/devtools/browser/abc\n") if self.running else _Completed(1)+ return _Completed(0, b"9222\n/devtools/browser/abc\n") if self.running else _Completed(3)if argv[0] == "kill":self.killed.append(int(argv[-1]))self.running = None@@ -252,8 +262,10 @@ def fake_sandbox(monkeypatch, isolated_sessions):from tools.environments import streamssandbox, env = _FakeSandbox(), object()monkeypatch.setattr(browser_session_sandbox, "_launched", {}, raising=False)+ monkeypatch.setattr(browser_session_sandbox, "_confirmed", {}, raising=False)monkeypatch.setattr(streams, "run_in", sandbox.run_in)monkeypatch.setattr(streams, "forward_port", lambda env, port, user=None: 50000 + port)+ monkeypatch.setattr(streams, "forward_failures", lambda env, port: sandbox.forward_failures, raising=False)monkeypatch.setattr(streams, "sandbox_temp_dir", lambda env: "/tmp") # no-tmp: ok — sandbox-side pathmonkeypatch.setattr(runtime, "_sandbox_env", lambda create=False: env)monkeypatch.setattr(runtime, "published_env", lambda: {"DISPLAY": ":5"})@@ -284,4 +296,28 @@ class TestSandboxSharedBrowser:assert bs._backend_dead(info) is Falsefake_sandbox.running = None # crashed inside the sandbox+ fake_sandbox.forward_failures += 1 # the harness's next connection through the forward failsassert bs._backend_dead(info) is True++ def test_a_failed_or_timed_out_liveness_exec_keeps_the_live_browser(self, fake_sandbox, isolated_sessions,+ monkeypatch):+ """Only a definitive answer from inside the sandbox ("no DevTools port file", "port closed") means+ Chromium is gone; a docker/ssh exec that fails or times out says nothing about it."""+ from tools.browser_session_sandbox import create_sandbox_session+ monkeypatch.setattr(bs, "_create_session_for_key", lambda key, force_local: create_sandbox_session(key))+ first = bs.get_session_info("task-A")+ for failure in ("exit-255", "timeout"):+ fake_sandbox.forward_failures += 1 # a connection through the forward failed: liveness is re-checked+ fake_sandbox.probe_failure = failure+ assert bs.get_session_info("task-A") is first, failure+ assert fake_sandbox.killed == [] and fake_sandbox.running == 4242++ def test_a_healthy_sandbox_session_spends_no_exec_per_browser_call(self, fake_sandbox, isolated_sessions,+ monkeypatch):+ from tools.browser_session_sandbox import create_sandbox_session+ monkeypatch.setattr(bs, "_create_session_for_key", lambda key, force_local: create_sandbox_session(key))+ first = bs.get_session_info("task-A")+ fake_sandbox.probes = 0+ for _ in range(5):+ assert bs.get_session_info("task-A") is first+ assert fake_sandbox.probes == 0diff --git a/tests/tools/test_sandbox_port_forward.py b/tests/tools/test_sandbox_port_forward.pynew file mode 100644index 00000000000..5c6228d80b0--- /dev/null+++ b/tests/tools/test_sandbox_port_forward.py@@ -0,0 +1,71 @@+"""The sandbox port forward (tools/environments/streams.py) reports connections that never reached their+target: the sandbox browser's liveness is re-checked only after one (tools/browser_session_sandbox.py)."""++import socket+import subprocess+import sys+import threading+import time++import pytest++from tools.environments import streams+++def local_relay(monkeypatch):+ """The relay runs on this host instead of inside a sandbox: same script, same pipes."""+ def open_stream(env, argv, **kw):+ return subprocess.Popen([sys.executable, *argv[1:]], stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL)+ monkeypatch.setattr(streams, "open_stream", open_stream)+ opened = []+ yield opened+ for env, port in opened:+ streams._forwards.pop((id(env), "127.0.0.1", port)).close()+++def _closed_port() -> int:+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _one_shot_server(reply: bytes) -> int:+ srv = socket.socket()+ srv.bind(("127.0.0.1", 0))+ srv.listen(1)++ def serve():+ conn, _ = srv.accept()+ with conn, srv:+ conn.sendall(reply)+ threading.Thread(target=serve, daemon=True).start()+ return srv.getsockname()[1]+++def _relay(local_port: int) -> bytes:+ with socket.create_connection(("127.0.0.1", local_port), timeout=10) as conn:+ conn.settimeout(10)+ conn.shutdown(socket.SHUT_WR) # the client is done sending; the relay ends with the target's reply+ chunks = []+ while chunk := conn.recv(65536):+ chunks.append(chunk)+ return b"".join(chunks)+++def test_only_a_connection_that_never_reaches_its_target_counts_as_a_forward_failure(local_relay):+ env = object()+ live = _one_shot_server(b"devtools")+ local_relay.append((env, live))+ assert _relay(streams.forward_port(env, live)) == b"devtools"++ dead = _closed_port() # Chromium died: nothing listens on its DevTools port any more+ local_relay.append((env, dead))+ assert _relay(streams.forward_port(env, dead)) == b""++ deadline = time.monotonic() + 10+ while streams.forward_failures(env, dead) == 0 and time.monotonic() < deadline:+ time.sleep(0.05)+ assert streams.forward_failures(env, dead) == 1+ assert streams.forward_failures(env, live) == 0diff --git a/tools/browser_session_sandbox.py b/tools/browser_session_sandbox.pyindex 49f5f456b85..303cd35442b 100644--- a/tools/browser_session_sandbox.py+++ b/tools/browser_session_sandbox.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport loggingimport shlex+import subprocessimport threadingimport timefrom typing import Any, Optional@@ -23,6 +24,14 @@ _LAUNCH_TIMEOUT_S = 60.0# right sandbox._launched: dict[str, tuple[Any, int]] = {}_launched_lock = threading.Lock()+# Endpoints a liveness probe confirmed: ``cdp_url -> (id(env), sandbox port, forward failures then)``. A browser+# call re-probes only after a connection through its forward failed (or the sandbox was replaced), so a+# healthy call spends no exec in the sandbox.+_confirmed: dict[str, tuple[int, int, int]] = {}+# Exit code of the liveness script when the sandbox answers "not running": no DevTools port file, or nothing+# listens on its port. Any other failure (docker/ssh transport error, timeout) says nothing about Chromium.+_GONE_RC = 3+GONE = "gone"def _sandbox():@@ -47,16 +56,25 @@ def _read_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:return None-def _live_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:- """``(port, path)`` of the Chromium on ``udd`` while its DevTools port still answers, in one exec (a- crashed Chromium leaves a stale ``DevToolsActivePort`` behind)."""+def _live_devtools(env, user, udd: str):+ """One exec: ``(port, path)`` of the Chromium on ``udd`` while its DevTools port answers; ``GONE`` when the+ sandbox says it is not running (a crashed Chromium leaves a stale ``DevToolsActivePort`` behind); None+ when the exec itself failed or timed out, which proves nothing either way."""from tools.environments import streamsactive = shlex.quote(f"{udd}/DevToolsActivePort")- script = f'p=$(head -n1 {active}) && exec 3<>"/dev/tcp/127.0.0.1/$p" && cat {active}'- proc = streams.run_in(env, ["bash", "-c", script], user=user, timeout=10)+ script = (f'[ -s {active} ] || exit {_GONE_RC}; p=$(head -n1 {active}); '+ f'( exec 3<>"/dev/tcp/127.0.0.1/$p" ) 2>/dev/null || exit {_GONE_RC}; cat {active}')+ try:+ proc = streams.run_in(env, ["bash", "-c", script], user=user, timeout=10)+ except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:+ logger.debug("sandbox browser liveness probe failed: %s", exc)+ return None+ if proc.returncode == _GONE_RC:+ return GONElines = proc.stdout.decode("utf-8", "replace").splitlines() if proc.returncode == 0 else []if len(lines) >= 2 and lines[0].strip().isdigit() and lines[1].startswith("/devtools/browser/"):return int(lines[0].strip()), lines[1].strip()+ logger.debug("sandbox browser liveness probe inconclusive (exit %s)", proc.returncode)return None@@ -106,6 +124,9 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:key = _shared_key(udd)found = _live_devtools(env, user, udd)if found is None:+ # Launching blind would wipe a live browser's DevToolsActivePort and lose to its singleton.+ raise RuntimeError("the terminal backend's sandbox did not answer whether its browser is running; try again")+ if found is GONE:exe = sandbox_host.chromium_executable(env)if not exe:raise RuntimeError("the sandbox image has no Chromium to run the browser")@@ -116,6 +137,7 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:_launched[key] = (env, pid)local_port = streams.forward_port(env, found[0], user=user)info = _session_record("sb", f"ws://127.0.0.1:{local_port}{found[1]}", {"local": True, "sandbox": True})+ _confirmed[info["cdp_url"]] = (id(env), found[0], streams.forward_failures(env, found[0]))info.update(shared_profile=True, launcher_key=key)logger.info("Created sandbox browser session %s for %s", info["session_name"], cache_key)return info@@ -123,26 +145,38 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:"""The sandbox Chromium still serves this session's endpoint through the current port forward: False once- it died, was relaunched (new DevTools path) or the sandbox was replaced (new forward)."""+ the sandbox answers that it died, was relaunched (new DevTools path) or the sandbox was replaced (new+ forward). Re-probed only after a connection through the forward failed; a probe that cannot reach the+ sandbox keeps the session."""from tools.bot_desktop import runtime as _bd_runtime, sandbox_hostfrom tools.environments import streamsenv = _bd_runtime._sandbox_env(create=False)if env is None:return False+ cdp_url = session_info.get("cdp_url") or ""+ confirmed = _confirmed.get(cdp_url)+ if confirmed and confirmed[0] == id(env) and streams.forward_failures(env, confirmed[1]) == confirmed[2]:+ return Truetry:user = sandbox_host._user_for(env)- found = _live_devtools(env, user, sandbox_host.browser_profile_dir(env))- except (OSError, RuntimeError) as exc:+ udd = sandbox_host.browser_profile_dir(env)+ except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:logger.debug("sandbox browser liveness probe failed: %s", exc)- return False+ return True+ found = _live_devtools(env, user, udd)if found is None:+ return True+ if found is GONE or cdp_url != f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}":+ _confirmed.pop(cdp_url, None)return False- return session_info.get("cdp_url") == f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}"+ _confirmed[cdp_url] = (id(env), found[0], streams.forward_failures(env, found[0]))+ return Truedef stop_sandbox_browser(session_info: dict[str, Any]) -> None:"""SIGTERM the sandbox Chromium this process launched on the session's shared key (best effort; thesandbox's lifetime bounds it). The caller guarantees no other live session uses it."""+ _confirmed.pop(session_info.get("cdp_url") or "", None)with _launched_lock:launched = _launched.pop(session_info.get("launcher_key") or "", None)if launched is None:diff --git a/tools/environments/streams.py b/tools/environments/streams.pyindex 4d659657e80..3240655637a 100644--- a/tools/environments/streams.py+++ b/tools/environments/streams.py@@ -170,6 +170,9 @@ class PortForward:self._server.listen(16)self.local_port: int = self._server.getsockname()[1]self._closed = False+ # Connections that never reached the target (relay not started, or it exited non-zero before+ # relaying a byte back): the one cheap signal that whatever listened there may be gone.+ self.failures = 0threading.Thread(target=self._accept_loop, name=f"port-forward-{self.local_port}", daemon=True).start()def _accept_loop(self) -> None:@@ -186,6 +189,7 @@ class PortForward:user=self.user)except Exception:conn.close()+ self.failures += 1returndef to_remote() -> None:@@ -205,11 +209,13 @@ class PortForward:passthreading.Thread(target=to_remote, daemon=True).start()+ relayed = Falsetry:while True:data = proc.stdout.read1(65536) if hasattr(proc.stdout, "read1") else proc.stdout.read(65536)if not data:break+ relayed = Trueconn.sendall(data)except OSError:pass@@ -217,6 +223,8 @@ class PortForward:try:conn.close()finally:+ if not relayed and _exited_nonzero(proc):+ self.failures += 1proc.kill()def close(self) -> None:@@ -227,6 +235,21 @@ class PortForward:pass+def _exited_nonzero(proc: subprocess.Popen) -> bool:+ try:+ return proc.wait(timeout=2) != 0+ except subprocess.TimeoutExpired:+ return False+++def forward_failures(env: BaseEnvironment, remote_port: int, *, remote_host: str = "127.0.0.1") -> int:+ """How many connections through ``forward_port(env, remote_port)`` never reached the target (0 while no+ forward is open). A caller caching "the target is up" re-checks only once this changes."""+ with _forwards_lock:+ fwd = _forwards.get((id(env), remote_host, int(remote_port)))+ return fwd.failures if fwd is not None else 0++def forward_port(env: BaseEnvironment, remote_port: int, *, remote_host: str = "127.0.0.1",user: Optional[str] = None) -> int:"""Local port on 127.0.0.1 that reaches ``remote_host:remote_port`` inside ``env``; one listener per