Skip to content

Self-improving AI Employees for everyone

fix(browser): drain private proxy connections and fence runtime cleanup

trohitg15f4d4133521
diff --git a/server/nodes/browser/__init__.py b/server/nodes/browser/__init__.py
index 486dddfd..d0c8466d 100644
--- a/server/nodes/browser/__init__.py
+++ b/server/nodes/browser/__init__.py
@@ -78,7 +78,7 @@ async def _cleanup_deleted_workflow_profile(database: Any, principal: str, profi
from ._profiles import ProfileStore, remove_profile_files
from ._runtime import get_browser_runtime
profile = await ProfileStore(database).get(principal, profile_id)
- if profile.kind != "employee" or profile.workflow_id != workflow_id or await database.get_workflow(workflow_id) is not None:
+ if profile.kind not in {"employee", "agent"} or profile.workflow_id != workflow_id or await database.get_workflow(workflow_id) is not None:
raise NodeUserError("Browser cleanup is not associated with a deleted workflow.")
binding = await bind_profile(database, profile.id, principal)
if binding["owner_id"] != replica_id():
diff --git a/server/nodes/browser/_egress.py b/server/nodes/browser/_egress.py
index fd2b8a9a..d805038a 100644
--- a/server/nodes/browser/_egress.py
+++ b/server/nodes/browser/_egress.py
@@ -79,17 +79,29 @@ def _split_host_port(authority: str, default_port: int) -> Tuple[str, int]:
class EgressProxy:
"""One filtering HTTP/CONNECT proxy on ``127.0.0.1:<ephemeral>``."""
- def __init__(self, policy: PolicyProvider, *, name: str = "browser-egress") -> None:
+ def __init__(self, policy: PolicyProvider, *, name: str = "browser-egress", runtime_version: int = 0) -> None:
+ if runtime_version not in (0, 1):
+ raise ValueError("Unsupported browser proxy resource version")
+ self.runtime_version = runtime_version
self._policy = policy
self._name = name
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._thread: Optional[threading.Thread] = None
self._server: Optional[asyncio.base_events.Server] = None
self.port: Optional[int] = None
+ self._tasks: set[asyncio.Task] = set()
+ self._writers: set[asyncio.StreamWriter] = set()
+ self._lifecycle_lock = threading.RLock()
+ self._closing = False
+ self._drain_timeout = 5.0
+ self._halfclose_timeout = 10.0
+ self._close_timeout = 1.0
# -- lifecycle (called from the main loop) ------------------------------
def start(self) -> int:
+ if self.runtime_version == 1:
+ return self._start_v1()
if self.port is not None:
return self.port
ready = threading.Event()
@@ -124,6 +136,8 @@ class EgressProxy:
return self.port
def stop(self) -> None:
+ if self.runtime_version == 1:
+ return self._stop_v1()
loop, server = self._loop, self._server
if loop is None:
return
@@ -148,6 +162,129 @@ class EgressProxy:
self._loop = self._server = self._thread = None
self.port = None
+ def _start_v1(self) -> int:
+ with self._lifecycle_lock:
+ if self._thread is not None and self._thread.is_alive():
+ if self._closing or self.port is None:
+ raise RuntimeError("Browser proxy cleanup has not completed")
+ return self.port
+ self._closing = False
+ ready = threading.Event()
+ errors = []
+
+ def run():
+ loop = asyncio.new_event_loop()
+ self._loop = loop
+ asyncio.set_event_loop(loop)
+ try:
+ self._server = loop.run_until_complete(asyncio.start_server(
+ self._handle, host="127.0.0.1", port=0, limit=_HEADER_LIMIT))
+ self.port = self._server.sockets[0].getsockname()[1]
+ ready.set()
+ loop.run_forever()
+ except BaseException as exc:
+ errors.append(exc)
+ ready.set()
+ finally:
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ loop.close()
+
+ self._thread = threading.Thread(target=run, name=self._name, daemon=True)
+ self._thread.start()
+ if not ready.wait(timeout=10):
+ self._closing = True
+ raise RuntimeError("Browser proxy startup did not complete")
+ if errors:
+ raise errors[0]
+ return self.port
+
+ async def _close_writer(self, writer):
+ try:
+ writer.close()
+ await asyncio.wait_for(writer.wait_closed(), timeout=self._close_timeout)
+ except (OSError, RuntimeError, TimeoutError):
+ transport = getattr(writer, "transport", None)
+ if transport is not None:
+ transport.abort()
+ finally:
+ self._writers.discard(writer)
+
+ def _stop_v1(self) -> bool:
+ with self._lifecycle_lock:
+ thread, loop = self._thread, self._loop
+ if thread is None:
+ return True
+ if not thread.is_alive():
+ self._loop = self._server = self._thread = None
+ self.port = None
+ return True
+ self._closing = True
+
+ async def shutdown():
+ if self._server is not None:
+ self._server.close()
+ tasks = [t for t in self._tasks if t is not asyncio.current_task()]
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*(self._close_writer(w) for w in tuple(self._writers)), return_exceptions=True)
+ if tasks:
+ await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=3)
+ if self._server is not None:
+ await asyncio.wait_for(self._server.wait_closed(), timeout=1)
+ loop.call_soon(loop.stop)
+
+ try:
+ if loop is None or loop.is_closed():
+ return False
+ future = asyncio.run_coroutine_threadsafe(shutdown(), loop)
+ future.result(timeout=5)
+ thread.join(timeout=2)
+ except Exception:
+ # Keep every handle: a caller may retry cleanup but must not
+ # replace this still-live proxy with a second resource.
+ logger.warning("[browser] proxy cleanup is still pending")
+ return False
+ if thread.is_alive():
+ return False
+ self._loop = self._server = self._thread = None
+ self.port = None
+ self._tasks.clear()
+ self._writers.clear()
+ return True
+
+ async def _relay_v1(self, src, dst):
+ try:
+ while True:
+ data = await src.read(_RELAY_CHUNK)
+ if not data:
+ if dst.can_write_eof():
+ dst.write_eof()
+ await asyncio.wait_for(dst.drain(), timeout=self._drain_timeout)
+ return "eof"
+ dst.write(data)
+ await asyncio.wait_for(dst.drain(), timeout=self._drain_timeout)
+ except (ConnectionError, OSError, TimeoutError, RuntimeError):
+ return "error"
+
+ async def _relay_pair(self, client_reader, client_writer, upstream_reader, upstream_writer):
+ tasks = {asyncio.create_task(self._relay_v1(client_reader, upstream_writer)),
+ asyncio.create_task(self._relay_v1(upstream_reader, client_writer))}
+ self._tasks.update(tasks)
+ try:
+ done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+ if all(task.result() == "eof" for task in done) and pending:
+ # TCP half-close is valid: let the other direction finish its
+ # response, but never retain an idle tunnel indefinitely.
+ await asyncio.wait_for(asyncio.gather(*pending), timeout=self._halfclose_timeout)
+ except TimeoutError:
+ pass
+ finally:
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ self._tasks.difference_update(tasks)
+
# -- connection handling (proxy thread) ---------------------------------
async def _resolve_allowed(self, host: str, port: int, policy: NetPolicy) -> list[str]:
@@ -189,15 +326,23 @@ class EgressProxy:
# localhost may resolve to IPv6 first while the app
# listens only on IPv4. Try only addresses already
# checked above; never resolve the hostname again.
- return await asyncio.open_connection(address, port)
+ opened = await asyncio.open_connection(address, port)
+ if self.runtime_version == 1:
+ self._writers.add(opened[1])
+ return opened
except OSError as exc:
last_error = exc
raise last_error
except (OSError, asyncio.TimeoutError) as exc:
raise ConnectionError(f"could not connect to {host}:{port}: {exc}") from exc
- @staticmethod
- async def _refuse(writer: asyncio.StreamWriter, status: str, reason: str) -> None:
+ async def _drain(self, writer):
+ if self.runtime_version == 1:
+ await asyncio.wait_for(writer.drain(), timeout=self._drain_timeout)
+ else:
+ await writer.drain()
+
+ async def _refuse(self, writer: asyncio.StreamWriter, status: str, reason: str) -> None:
body = f"Blocked by OpenCompany: {reason}\n".encode("utf-8", errors="replace")
head = (
f"HTTP/1.1 {status}\r\nContent-Type: text/plain; charset=utf-8\r\n"
@@ -205,11 +350,15 @@ class EgressProxy:
).encode("ascii")
try:
writer.write(head + body)
- await writer.drain()
+ await self._drain(writer)
except (ConnectionError, OSError):
pass
async def _handle(self, client_reader: asyncio.StreamReader, client_writer: asyncio.StreamWriter) -> None:
+ current = asyncio.current_task()
+ if self.runtime_version == 1:
+ self._tasks.add(current)
+ self._writers.add(client_writer)
upstream_writer: Optional[asyncio.StreamWriter] = None
try:
head = await _read_head(client_reader)
@@ -225,7 +374,7 @@ class EgressProxy:
host, port = _split_host_port(target, 443)
upstream_reader, upstream_writer = await self._open(host, port, policy)
client_writer.write(b"HTTP/1.1 200 Connection Established\r\n\r\n")
- await client_writer.drain()
+ await self._drain(client_writer)
else:
url = urlsplit(target)
if url.scheme.lower() != "http" or not url.hostname:
@@ -254,12 +403,12 @@ class EgressProxy:
# host, down this already-checked socket.
headers.append(b"Connection: " + (b"Upgrade" if upgrade else b"close"))
upstream_writer.write(f"{method} {path} {version}\r\n".encode("latin-1") + b"\r\n".join(headers) + b"\r\n\r\n")
- await upstream_writer.drain()
+ await self._drain(upstream_writer)
- await asyncio.gather(
- _relay(client_reader, upstream_writer),
- _relay(upstream_reader, client_writer),
- )
+ if self.runtime_version == 1:
+ await self._relay_pair(client_reader, client_writer, upstream_reader, upstream_writer)
+ else:
+ await asyncio.gather(_relay(client_reader, upstream_writer), _relay(upstream_reader, client_writer))
except _Blocked as exc:
await self._refuse(client_writer, "403 Forbidden", str(exc))
except ConnectionError as exc:
@@ -271,7 +420,10 @@ class EgressProxy:
except Exception: # noqa: BLE001 - one bad request must not stop the proxy
logger.debug("[browser] egress proxy request failed", exc_info=True)
finally:
- for writer in (upstream_writer, client_writer):
+ if self.runtime_version == 1:
+ await asyncio.gather(*(self._close_writer(w) for w in (upstream_writer, client_writer) if w is not None), return_exceptions=True)
+ self._tasks.discard(current)
+ for writer in (() if self.runtime_version == 1 else (upstream_writer, client_writer)):
if writer is not None:
try:
writer.close()
diff --git a/server/nodes/browser/_runtime.py b/server/nodes/browser/_runtime.py
index 79114b54..ee5f6962 100644
--- a/server/nodes/browser/_runtime.py
+++ b/server/nodes/browser/_runtime.py
@@ -107,6 +107,12 @@ class BrowserRuntime:
self._controllers: Dict[str, ProfileController] = {}
self._chromes: Dict[str, ChromeProcess] = {}
self._proxies: Dict[str, EgressProxy] = {}
+ self._teardown_locks: Dict[str, asyncio.Lock] = {}
+ self._stop_locks: Dict[str, asyncio.Lock] = {}
+ self._stopping: set[str] = set()
+ self._cleanup_pending: set[str] = set()
+ self._shutting_down_v1 = False
+ self._resource_versions: Dict[str, int] = {}
self._starting: Dict[str, asyncio.Task] = {}
self._sessions: Dict[str, BrowserSession] = {}
self._reaper: Optional[asyncio.Task] = None
@@ -192,6 +198,14 @@ class BrowserRuntime:
async def open(self, profile: Profile, *, wait: float = 60.0) -> ProfileRuntime:
"""The profile's running browser, starting Chrome if needed."""
+ if profile.kind == "agent":
+ from services.plugin.deps import get_database
+ from ._profiles import ProfileStore
+ # Entry-point independent: a credentials/viewer open may be first.
+ profile = await ProfileStore(get_database()).get(profile.owner_id, profile.id)
+ self._resource_versions[profile.id] = 1
+ if self._shutting_down_v1 or profile.id in self._stopping or profile.id in self._cleanup_pending:
+ raise NodeUserError("Private Browser cleanup is still pending. Retry after owner recovery.")
from services.browser_owners import _persistent
from services.plugin.deps import get_database
if getattr(self._settings(), "distributed_mode", False) is True or _persistent(get_database()):
@@ -207,9 +221,14 @@ class BrowserRuntime:
controller.needs_observation |= bool(binding.get("needs_observation"))
controller.challenge_required = bool(binding.get("challenge_required"))
controller.recovered_assistance = binding.get("assistance")
+ if getattr(profile, "runtime_version", 0) == 1 and (
+ self._shutting_down_v1 or profile.id in self._stopping or profile.id in self._cleanup_pending):
+ raise NodeUserError("Private Browser cleanup is still pending. Retry after owner recovery.")
runtime = self._profiles.get(profile.id)
if runtime is not None and runtime.running:
return runtime
+ if runtime is not None and getattr(profile, "runtime_version", 0) == 1:
+ await self._teardown(runtime)
self._ensure_reaper()
task = self._starting.get(profile.id)
if task is None or task.done():
@@ -226,6 +245,23 @@ class BrowserRuntime:
raise NodeUserError("The browser is still starting; try again in a moment.") from None
async def _start(self, profile: Profile) -> ProfileRuntime:
+ try:
+ return await self._start_impl(profile)
+ except BaseException:
+ if getattr(profile, "runtime_version", 0) == 1:
+ runtime = self._profiles.get(profile.id)
+ if runtime is not None:
+ await self._teardown(runtime)
+ else:
+ self._cleanup_pending.add(profile.id)
+ chrome = self._chromes.get(profile.id)
+ if chrome is not None:
+ await self._stop_private_chrome(chrome)
+ await self._stop_private_proxy(profile.id)
+ self._cleanup_pending.discard(profile.id)
+ raise
+
+ async def _start_impl(self, profile: Profile) -> ProfileRuntime:
settings = self._settings()
pin = get_config().chrome
installer = get_chrome_installer()
@@ -244,7 +280,8 @@ class BrowserRuntime:
controller = self.controller_for(profile)
proxy = self._proxies.get(profile.id)
if proxy is None:
- proxy = EgressProxy(lambda c=controller: self._effective_policy(c), name=f"browser-egress-{profile.id}")
+ proxy = EgressProxy(lambda c=controller: self._effective_policy(c), name=f"browser-egress-{profile.id}",
+ **({"runtime_version": 1} if getattr(profile, "runtime_version", 0) == 1 else {}))
self._proxies[profile.id] = proxy
proxy_port = await asyncio.to_thread(proxy.start)
@@ -293,7 +330,8 @@ class BrowserRuntime:
raise NodeUserError("The launched browser is older than this profile or the minimum supported browser version. Update the selected browser or use another profile.")
await self._wire(runtime)
except BaseException:
- await chrome.shutdown()
+ if getattr(profile, "runtime_version", 0) != 1:
+ await chrome.shutdown()
raise
self._profiles[profile.id] = runtime
controller.last_activity = time.monotonic()
@@ -385,6 +423,9 @@ class BrowserRuntime:
if self._profiles.get(runtime.profile.id) is not runtime:
return
logger.warning("[browser] Chrome for profile %s stopped unexpectedly", runtime.profile.id)
+ if getattr(runtime.profile, "runtime_version", 0) == 1:
+ await self.stop_profile(runtime.profile.id, reason="Chrome stopped")
+ return
await self._teardown(runtime)
controller = runtime.controller
controller.tabs.clear()
@@ -394,6 +435,21 @@ class BrowserRuntime:
await controller.release_lease()
async def _teardown(self, runtime: ProfileRuntime) -> None:
+ if getattr(runtime.profile, "runtime_version", 0) == 1:
+ async with self._teardown_locks.setdefault(runtime.profile.id, asyncio.Lock()):
+ if self._profiles.get(runtime.profile.id) is not runtime:
+ return
+ self._cleanup_pending.add(runtime.profile.id)
+ try:
+ await asyncio.to_thread(runtime.cli.stop_daemon)
+ except Exception:
+ logger.debug("[browser] private daemon stop failed", exc_info=True)
+ await self._stop_private_chrome(runtime.chrome)
+ await self._stop_private_proxy(runtime.profile.id)
+ if self._profiles.get(runtime.profile.id) is runtime:
+ self._profiles.pop(runtime.profile.id, None)
+ self._cleanup_pending.discard(runtime.profile.id)
+ return
self._profiles.pop(runtime.profile.id, None)
try:
runtime.cli.stop_daemon()
@@ -407,9 +463,55 @@ class BrowserRuntime:
except Exception: # noqa: BLE001
pass
+ async def _stop_private_chrome(self, chrome: ChromeProcess) -> None:
+ try:
+ await asyncio.wait_for(chrome.shutdown(), timeout=_STOP_TIMEOUT)
+ except Exception:
+ await chrome.stop()
+ if chrome.is_running():
+ raise NodeUserError("Private Browser cleanup is still pending; restore its owner before deleting it.")
+
+ async def _stop_private_proxy(self, profile_id: str) -> None:
+ proxy = self._proxies.get(profile_id)
+ if proxy is None:
+ return
+ if await asyncio.to_thread(proxy.stop) is False:
+ raise NodeUserError("Private Browser proxy cleanup is still pending. Retry after owner recovery.")
+ if self._proxies.get(profile_id) is proxy:
+ self._proxies.pop(profile_id, None)
+
async def stop_profile(self, profile_id: str, *, reason: str = "") -> None:
runtime = self._profiles.get(profile_id)
+ private = (self._resource_versions.get(profile_id) == 1
+ or getattr(getattr(runtime, "profile", None), "runtime_version", 0) == 1
+ or getattr(self._proxies.get(profile_id), "runtime_version", 0) == 1)
+ if private:
+ async with self._stop_locks.setdefault(profile_id, asyncio.Lock()):
+ self._stopping.add(profile_id)
+ try:
+ await self._stop_profile_impl(profile_id, reason=reason, private=True)
+ finally:
+ self._stopping.discard(profile_id)
+ return
+ await self._stop_profile_impl(profile_id, reason=reason)
+
+ async def _stop_profile_impl(self, profile_id: str, *, reason: str, private: bool = False) -> None:
+ if private:
+ starting = self._starting.get(profile_id)
+ if starting is not None and not starting.done() and starting is not asyncio.current_task():
+ starting.cancel()
+ await asyncio.gather(starting, return_exceptions=True)
+ runtime = self._profiles.get(profile_id)
if runtime is None:
+ if private:
+ chrome = self._chromes.get(profile_id)
+ if chrome is not None:
+ await self._stop_private_chrome(chrome)
+ proxy = self._proxies.get(profile_id)
+ if proxy is not None and getattr(proxy, "runtime_version", 0) == 1:
+ await self._stop_private_proxy(profile_id)
+ if private:
+ self._cleanup_pending.discard(profile_id)
return
controller = runtime.controller
await controller.release_lease()
@@ -423,9 +525,10 @@ class BrowserRuntime:
controller.tabs.clear()
controller.active_target_id = None
await controller.emit("closed", {"reason": reason or "stopped"})
- proxy = self._proxies.pop(profile_id, None)
- if proxy is not None:
- await asyncio.to_thread(proxy.stop)
+ if not private:
+ proxy = self._proxies.pop(profile_id, None)
+ if proxy is not None:
+ await asyncio.to_thread(proxy.stop)
logger.info("[browser] profile %s stopped (%s)", profile_id, reason or "requested")
async def _make_room(self, *, exclude: str) -> None:
@@ -458,15 +561,33 @@ class BrowserRuntime:
async def shutdown(self) -> None:
"""Stop every profile's Chrome gracefully (the plugin shutdown hook)."""
+ self._shutting_down_v1 = True
if self._reaper is not None:
self._reaper.cancel()
- await asyncio.gather(*(self._teardown(r) for r in list(self._profiles.values())), return_exceptions=True)
- for proxy in list(self._proxies.values()):
+ starting = [task for profile_id, task in self._starting.items()
+ if self._resource_versions.get(profile_id) == 1 and not task.done()]
+ for task in starting:
+ task.cancel()
+ await asyncio.gather(*starting, return_exceptions=True)
+ private_ids = {profile_id for profile_id, version in self._resource_versions.items() if version == 1}
+ private_ids.update(profile_id for profile_id, runtime in self._profiles.items()
+ if getattr(runtime.profile, "runtime_version", 0) == 1)
+ private_ids.update(profile_id for profile_id, proxy in self._proxies.items()
+ if getattr(proxy, "runtime_version", 0) == 1)
+ await asyncio.gather(*(self.stop_profile(profile_id, reason="backend shutdown") for profile_id in private_ids),
+ *(self._teardown(r) for r in list(self._profiles.values()) if r.profile.id not in private_ids),
+ return_exceptions=True)
+ for profile_id, proxy in list(self._proxies.items()):
try:
- proxy.stop()
+ if getattr(proxy, "runtime_version", 0) == 1:
+ await self._stop_private_proxy(profile_id)
+ else:
+ proxy.stop()
+ self._proxies.pop(profile_id, None)
except Exception: # noqa: BLE001
pass
- self._proxies.clear()
+ if self._cleanup_pending or any(getattr(proxy, "runtime_version", 0) == 1 for proxy in self._proxies.values()):
+ raise NodeUserError("Private Browser cleanup is still pending; browser ownership must remain fenced.")
# -- status ------------------------------------------------------------------
diff --git a/server/tests/nodes/browser/test_egress_v1.py b/server/tests/nodes/browser/test_egress_v1.py
new file mode 100644
index 00000000..7d897986
--- /dev/null
+++ b/server/tests/nodes/browser/test_egress_v1.py
@@ -0,0 +1,181 @@
+"""Controlled local sockets only; no Chrome or external network."""
+import asyncio
+import threading
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from nodes.browser._egress import EgressProxy
+from services.netpolicy import NetPolicy
+
+
+async def test_private_proxy_preserves_tcp_half_close_and_drains_every_handle():
+ async def respond(reader, writer):
+ data = await reader.read()
+ writer.write(b"reply:" + data)
+ await writer.drain()
+ writer.close()
+ await writer.wait_closed()
+ upstream = await asyncio.start_server(respond, "127.0.0.1", 0)
+ upstream_port = upstream.sockets[0].getsockname()[1]
+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)
+ try:
+ port = await asyncio.to_thread(proxy.start)
+ reader, writer = await asyncio.open_connection("127.0.0.1", port)
+ writer.write(f"CONNECT 127.0.0.1:{upstream_port} HTTP/1.1\r\n\r\n".encode())
+ await writer.drain()
+ assert b"200" in await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 2)
+ writer.write(b"payload")
+ await writer.drain()
+ writer.write_eof()
+ assert await asyncio.wait_for(reader.read(), 2) == b"reply:payload"
+ writer.close()
+ await writer.wait_closed()
+ finally:
+ assert await asyncio.to_thread(proxy.stop) is True
+ upstream.close()
+ await upstream.wait_closed()
+ assert not proxy._tasks and not proxy._writers and proxy._thread is None
+ assert await asyncio.to_thread(proxy.stop) is True
+
+
+async def test_backpressured_writer_is_bounded_and_close_precedes_wait_closed():
+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)
+ proxy._drain_timeout = proxy._close_timeout = 0.01
+ reader = asyncio.StreamReader()
+ reader.feed_data(b"x")
+ events = []
+ class Writer:
+ transport = SimpleNamespace(abort=lambda: events.append("abort"))
+ def write(self, _): pass
+ async def drain(self): await asyncio.sleep(100)
+ def close(self): events.append("close")
+ async def wait_closed(self): await asyncio.sleep(100)
+ writer = Writer()
+ assert await proxy._relay_v1(reader, writer) == "error"
+ await proxy._close_writer(writer)
+ assert events == ["close", "abort"]
+
+
+async def test_stop_drains_an_active_idle_tunnel_before_waiting_for_server_close():
+ finished = asyncio.Event()
+ async def respond(reader, writer):
+ try:
+ await reader.read()
+ finally:
+ writer.close()
+ await writer.wait_closed()
+ finished.set()
+ upstream = await asyncio.start_server(respond, "127.0.0.1", 0)
+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)
+ try:
+ port = await asyncio.to_thread(proxy.start)
+ reader, writer = await asyncio.open_connection("127.0.0.1", port)
+ target = upstream.sockets[0].getsockname()[1]
+ writer.write(f"CONNECT 127.0.0.1:{target} HTTP/1.1\r\n\r\n".encode())
+ await writer.drain()
+ await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 2)
+ assert await asyncio.wait_for(asyncio.to_thread(proxy.stop), 4) is True
+ assert await asyncio.wait_for(reader.read(), 2) == b""
+ writer.close()
+ await writer.wait_closed()
+ await asyncio.wait_for(finished.wait(), 2)
+ assert not proxy._writers and not proxy._tasks
+ finally:
+ await asyncio.to_thread(proxy.stop)
+ upstream.close()
+ await upstream.wait_closed()
+
+
+def test_failed_stop_retains_live_resource_for_retry(monkeypatch):
+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)
+ thread = SimpleNamespace(is_alive=lambda: True)
+ loop = SimpleNamespace(is_closed=lambda: False)
+ proxy._thread, proxy._loop, proxy.port = thread, loop, 1234
+ def submit(coro, _loop):
+ coro.close()
+ return SimpleNamespace(result=lambda **_: (_ for _ in ()).throw(TimeoutError()))
+ monkeypatch.setattr(asyncio, "run_coroutine_threadsafe", submit)
+ assert proxy.stop() is False
+ assert proxy._thread is thread and proxy._loop is loop and proxy.port == 1234
+ with pytest.raises(RuntimeError, match="cleanup"):
+ proxy.start()
+
+
+async def test_private_runtime_teardown_runs_blocking_cleanup_off_loop_and_keeps_failed_proxy():
+ from nodes.browser._runtime import BrowserRuntime
+ from services.plugin import NodeUserError
+ runtime = BrowserRuntime()
+ threads = []
+ profile = SimpleNamespace(id="private", runtime_version=1)
+ proxy = SimpleNamespace(runtime_version=1, stop=lambda: threads.append(threading.get_ident()) or False)
+ running = SimpleNamespace(profile=profile, cli=SimpleNamespace(stop_daemon=lambda: threads.append(threading.get_ident())),
+ chrome=SimpleNamespace(shutdown=AsyncMock(), stop=AsyncMock(), is_running=lambda: False))
+ runtime._profiles[profile.id] = running
+ runtime._proxies[profile.id] = proxy
+ with pytest.raises(NodeUserError, match="cleanup"):
+ await runtime._teardown(running)
+ assert runtime._proxies[profile.id] is proxy
+ assert threads and all(t != threading.get_ident() for t in threads)
+ proxy.stop = lambda: True
+ await runtime._teardown(running)
+ assert profile.id not in runtime._proxies and profile.id not in runtime._profiles
+ assert profile.id not in runtime._cleanup_pending
+
+
+async def test_private_stop_blocks_reopen_until_events_finish(monkeypatch):
+ from nodes.browser._profiles import ProfileStore
+ from nodes.browser._runtime import BrowserRuntime
+ from services.plugin import NodeUserError
+ fleet = BrowserRuntime()
+ profile = SimpleNamespace(id="private", owner_id="owner", kind="agent", runtime_version=1)
+ monkeypatch.setattr("services.plugin.deps.get_database", lambda: object())
+ monkeypatch.setattr(ProfileStore, "get", AsyncMock(return_value=profile))
+ new_proxy = SimpleNamespace(runtime_version=1, stop=lambda: pytest.fail("new proxy was stopped"))
+ async def emit(*_):
+ with pytest.raises(NodeUserError, match="cleanup"):
+ await fleet.open(profile)
+ # A later, distinct resource must never be removed by an old stop tail.
+ fleet._proxies[profile.id] = new_proxy
+ controller = SimpleNamespace(release_lease=AsyncMock(), sensitive_login=False,
+ tabs={}, active_target_id=None, emit=emit)
+ running = SimpleNamespace(profile=profile, controller=controller,
+ cli=SimpleNamespace(stop_daemon=lambda: None),
+ chrome=SimpleNamespace(shutdown=AsyncMock(), is_running=lambda: False))
+ fleet._profiles[profile.id] = running
+ fleet._proxies[profile.id] = SimpleNamespace(runtime_version=1, stop=lambda: True)
+ await fleet.stop_profile(profile.id)
+ assert fleet._proxies[profile.id] is new_proxy
+ assert profile.id not in fleet._stopping
+
+
+async def test_shutdown_cancels_private_start_before_final_resource_cleanup():
+ from nodes.browser._runtime import BrowserRuntime
+ fleet = BrowserRuntime()
+ started = asyncio.Event()
+ stopped = []
+ async def starting():
+ started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ fleet._proxies["private"] = SimpleNamespace(runtime_version=1, stop=lambda: stopped.append(True) or True)
+ task = asyncio.create_task(starting())
+ fleet._starting["private"] = task
+ fleet._resource_versions["private"] = 1
+ await started.wait()
+ await fleet.shutdown()
+ assert task.cancelled() and stopped == [True]
+ assert not fleet._proxies
+
+
+async def test_shutdown_keeps_failed_private_proxy_and_reports_fencing_required():
+ from nodes.browser._runtime import BrowserRuntime
+ from services.plugin import NodeUserError
+ fleet = BrowserRuntime()
+ proxy = SimpleNamespace(runtime_version=1, stop=lambda: False)
+ fleet._proxies["private"] = proxy
+ with pytest.raises(NodeUserError, match="fenced"):
+ await fleet.shutdown()
+ assert fleet._proxies["private"] is proxy