Skip to content

The agent that grows with you

fix(browser): a sandbox browser relaunched on the same port is re-checked within 30 s

Teknium127d782e6f07
diff --git a/tests/tools/test_browser_cleanup.py b/tests/tools/test_browser_cleanup.py
index 543a861d8f4..8e3ed3b4f2b 100644
--- a/tests/tools/test_browser_cleanup.py
+++ b/tests/tools/test_browser_cleanup.py
@@ -3,6 +3,7 @@
import os
import subprocess
import sys
+from types import SimpleNamespace
from unittest.mock import patch
import pytest
@@ -229,11 +230,14 @@ class _FakeSandbox:
self.probes = 0 # liveness execs (the /dev/tcp check)
self.probe_failure = None # "exit-255" (docker/ssh transport error) or "timeout"
self.forward_failures = 0 # connections through the port forward that never reached Chromium
+ self.browser_id = "abc" # a relaunch on the same port gets a new /devtools/browser/<id>
+ self.launches = 0
def run_in(self, env, argv, **kw):
cmd = " ".join(argv)
if "setsid" in cmd:
self.running = 4242
+ self.launches += 1
return _Completed(0, b"4242\n")
if "/dev/tcp" in cmd:
self.probes += 1
@@ -242,7 +246,8 @@ class _FakeSandbox:
if self.probe_failure == "timeout":
raise subprocess.TimeoutExpired(argv, kw.get("timeout"))
if "/dev/tcp" in cmd or argv[0] == "cat":
- return _Completed(0, b"9222\n/devtools/browser/abc\n") if self.running else _Completed(3)
+ found = f"9222\n/devtools/browser/{self.browser_id}\n".encode()
+ return _Completed(0, found) if self.running else _Completed(3)
if argv[0] == "kill":
self.killed.append(int(argv[-1]))
self.running = None
@@ -322,6 +327,23 @@ class TestSandboxSharedBrowser:
assert bs.get_session_info("task-A") is first
assert fake_sandbox.probes == 0
+ def test_a_browser_relaunched_on_the_same_port_is_rechecked_once_the_confirmation_expires(
+ self, fake_sandbox, monkeypatch):
+ """No forward failure surfaces for a relaunch the harness never connected through, so the confirmed
+ answer is only trusted for a short while: healthy calls inside it spend no exec."""
+ from tools import browser_session_sandbox
+ from tools.browser_session_sandbox import create_sandbox_session
+ clock = [1000.0]
+ monkeypatch.setattr(browser_session_sandbox, "time", SimpleNamespace(monotonic=lambda: clock[0]))
+ info = create_sandbox_session("task-A")
+ fake_sandbox.browser_id = "relaunched" # another process relaunched it on the same port
+ fake_sandbox.probes = 0
+ clock[0] += 5
+ assert bs._backend_dead(info) is False and fake_sandbox.probes == 0, "inside the TTL: no exec"
+ clock[0] += 60
+ assert bs._backend_dead(info) is True
+ assert fake_sandbox.probes == 1
+
class _FakeContainers:
"""Several terminal sandboxes (one ``env`` object per container), each with its own Chromium."""
diff --git a/tests/tools/test_sandbox_port_forward.py b/tests/tools/test_sandbox_port_forward.py
index 5c6228d80b0..d80dd3f4dde 100644
--- a/tests/tools/test_sandbox_port_forward.py
+++ b/tests/tools/test_sandbox_port_forward.py
@@ -9,6 +9,7 @@ import time
import pytest
+from tests.waiting import wait_until
from tools.environments import streams
@@ -31,7 +32,7 @@ def _closed_port() -> int:
return s.getsockname()[1]
-def _one_shot_server(reply: bytes) -> int:
+def _one_shot_server(reply: bytes, *, read_request: bool = False) -> int:
srv = socket.socket()
srv.bind(("127.0.0.1", 0))
srv.listen(1)
@@ -39,14 +40,17 @@ def _one_shot_server(reply: bytes) -> int:
def serve():
conn, _ = srv.accept()
with conn, srv:
+ if read_request:
+ conn.recv(65536)
conn.sendall(reply)
threading.Thread(target=serve, daemon=True).start()
return srv.getsockname()[1]
-def _relay(local_port: int) -> bytes:
+def _relay(local_port: int, request: bytes = b"") -> bytes:
with socket.create_connection(("127.0.0.1", local_port), timeout=10) as conn:
conn.settimeout(10)
+ conn.sendall(request)
conn.shutdown(socket.SHUT_WR) # the client is done sending; the relay ends with the target's reply
chunks = []
while chunk := conn.recv(65536):
@@ -69,3 +73,25 @@ def test_only_a_connection_that_never_reaches_its_target_counts_as_a_forward_fai
time.sleep(0.05)
assert streams.forward_failures(env, dead) == 1
assert streams.forward_failures(env, live) == 0
+
+
+_NOT_FOUND = b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n"
+
+
+def test_a_404_on_the_old_browser_path_counts_as_a_forward_failure(local_relay):
+ """Chromium relaunched on the same port answers the cached ``/devtools/browser/<id>`` with 404: the
+ endpoint the session holds is gone although the port relayed bytes."""
+ env = object()
+ stale = _one_shot_server(_NOT_FOUND, read_request=True)
+ local_relay.append((env, stale))
+ request = b"GET /devtools/browser/old-id HTTP/1.1\r\nHost: 127.0.0.1\r\nUpgrade: websocket\r\n\r\n"
+ assert _relay(streams.forward_port(env, stale), request).startswith(b"HTTP/1.1 404")
+ wait_until(lambda: streams.forward_failures(env, stale) == 1, what="the stale-path 404 counted as a failure")
+
+ other = _one_shot_server(_NOT_FOUND, read_request=True) # a dev server's own 404 is a relayed reply
+ local_relay.append((env, other))
+ assert _relay(streams.forward_port(env, other), b"GET /missing HTTP/1.1\r\n\r\n").startswith(b"HTTP/1.1 404")
+ live = _one_shot_server(b"HTTP/1.1 101 Switching Protocols\r\n\r\n", read_request=True)
+ local_relay.append((env, live))
+ assert _relay(streams.forward_port(env, live), request).startswith(b"HTTP/1.1 101")
+ assert streams.forward_failures(env, other) == 0 and streams.forward_failures(env, live) == 0
diff --git a/tools/browser_session_sandbox.py b/tools/browser_session_sandbox.py
index 1332aba9fd7..e760b4b19a5 100644
--- a/tools/browser_session_sandbox.py
+++ b/tools/browser_session_sandbox.py
@@ -24,10 +24,12 @@ _LAUNCH_TIMEOUT_S = 60.0
# teardown outside the owner's scope hits the right sandbox.
_launched: dict[str, tuple[Any, int]] = {}
_launched_lock = threading.Lock()
-# Endpoints a liveness probe confirmed: ``cdp_url -> (id(env), sandbox port, forward failures then)``. A browser
-# call re-probes only after a connection through its forward failed (or the sandbox was replaced), so a
-# healthy call spends no exec in the sandbox.
-_confirmed: dict[str, tuple[int, int, int]] = {}
+# Endpoints a liveness probe confirmed: ``cdp_url -> (id(env), sandbox port, forward failures then, when)``. A
+# browser call re-probes after a connection through its forward failed, the sandbox was replaced, or the
+# confirmation is older than _CONFIRMED_TTL_S (a relaunch on the same port that nothing connected to yet
+# leaves no failure behind), so a healthy call spends no exec in the sandbox.
+_confirmed: dict[str, tuple[int, int, int, float]] = {}
+_CONFIRMED_TTL_S = 30.0
# Exit code of the liveness script when the sandbox answers "not running": no DevTools port file, or nothing
# listens on its port. Any other failure (docker/ssh transport error, timeout) says nothing about Chromium.
_GONE_RC = 3
@@ -139,17 +141,22 @@ def create_sandbox_session(cache_key: str) -> dict[str, Any]:
_launched[key] = (env, pid)
local_port = streams.forward_port(env, found[0], user=user)
info = _session_record("sb", f"ws://127.0.0.1:{local_port}{found[1]}", {"local": True, "sandbox": True})
- _confirmed[info["cdp_url"]] = (id(env), found[0], streams.forward_failures(env, found[0]))
+ _confirm(info["cdp_url"], env, found[0])
info.update(shared_profile=True, launcher_key=key)
logger.info("Created sandbox browser session %s for %s", info["session_name"], cache_key)
return info
+def _confirm(cdp_url: str, env, port: int) -> None:
+ from tools.environments import streams
+ _confirmed[cdp_url] = (id(env), port, streams.forward_failures(env, port), time.monotonic())
+
+
def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:
"""The sandbox Chromium still serves this session's endpoint through the current port forward: False once
the sandbox answers that it died, was relaunched (new DevTools path) or the sandbox was replaced (new
- forward). Re-probed only after a connection through the forward failed; a probe that cannot reach the
- sandbox keeps the session."""
+ forward). Re-probed only after a connection through the forward failed or the last confirmation expired;
+ a probe that cannot reach the sandbox keeps the session."""
from tools.bot_desktop import runtime as _bd_runtime, sandbox_host
from tools.environments import streams
env = _bd_runtime._sandbox_env(create=False)
@@ -157,7 +164,8 @@ def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:
return False
cdp_url = session_info.get("cdp_url") or ""
confirmed = _confirmed.get(cdp_url)
- if confirmed and confirmed[0] == id(env) and streams.forward_failures(env, confirmed[1]) == confirmed[2]:
+ if (confirmed and confirmed[0] == id(env) and streams.forward_failures(env, confirmed[1]) == confirmed[2]
+ and time.monotonic() - confirmed[3] < _CONFIRMED_TTL_S):
return True
try:
user = sandbox_host._user_for(env)
@@ -171,7 +179,7 @@ def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:
if found is GONE or cdp_url != f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}":
_confirmed.pop(cdp_url, None)
return False
- _confirmed[cdp_url] = (id(env), found[0], streams.forward_failures(env, found[0]))
+ _confirm(cdp_url, env, found[0])
return True
diff --git a/tools/environments/streams.py b/tools/environments/streams.py
index 3240655637a..bf37d772012 100644
--- a/tools/environments/streams.py
+++ b/tools/environments/streams.py
@@ -170,8 +170,9 @@ class PortForward:
self._server.listen(16)
self.local_port: int = self._server.getsockname()[1]
self._closed = False
- # Connections that never reached the target (relay not started, or it exited non-zero before
- # relaying a byte back): the one cheap signal that whatever listened there may be gone.
+ # Connections that never reached the target (relay not started, it exited non-zero before
+ # relaying a byte back, or the browser path it asked for is gone): the one cheap signal that
+ # whatever listened there may be gone.
self.failures = 0
threading.Thread(target=self._accept_loop, name=f"port-forward-{self.local_port}", daemon=True).start()
@@ -192,12 +193,16 @@ class PortForward:
self.failures += 1
return
+ request: list[bytes] = []
+
def to_remote() -> None:
try:
while True:
data = conn.recv(65536)
if not data:
break
+ if not request:
+ request.append(data)
proc.stdin.write(data)
proc.stdin.flush()
except (OSError, ValueError):
@@ -209,13 +214,13 @@ class PortForward:
pass
threading.Thread(target=to_remote, daemon=True).start()
- relayed = False
+ reply = b""
try:
while True:
data = proc.stdout.read1(65536) if hasattr(proc.stdout, "read1") else proc.stdout.read(65536)
if not data:
break
- relayed = True
+ reply = reply or data
conn.sendall(data)
except OSError:
pass
@@ -223,7 +228,7 @@ class PortForward:
try:
conn.close()
finally:
- if not relayed and _exited_nonzero(proc):
+ if (not reply and _exited_nonzero(proc)) or _stale_browser_path(request, reply):
self.failures += 1
proc.kill()
@@ -235,6 +240,13 @@ class PortForward:
pass
+def _stale_browser_path(request: list[bytes], reply: bytes) -> bool:
+ """A Chromium relaunched on the same port answers the old ``/devtools/browser/<id>`` with 404: the
+ endpoint the caller cached is gone even though the port relayed a reply."""
+ return (bool(request) and request[0].startswith(b"GET /devtools/browser/")
+ and reply.startswith((b"HTTP/1.1 404 ", b"HTTP/1.0 404 ")))
+
+
def _exited_nonzero(proc: subprocess.Popen) -> bool:
try:
return proc.wait(timeout=2) != 0