Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(browser): drain private proxy connections and fence runtime cleanup
trohitg15f4d4133521
diff --git a/server/nodes/browser/__init__.py b/server/nodes/browser/__init__.pyindex 486dddfd..d0c8466d 100644--- a/server/nodes/browser/__init__.py+++ b/server/nodes/browser/__init__.py@@ -78,7 +78,7 @@ async def _cleanup_deleted_workflow_profile(database: Any, principal: str, profifrom ._profiles import ProfileStore, remove_profile_filesfrom ._runtime import get_browser_runtimeprofile = await ProfileStore(database).get(principal, profile_id)- if profile.kind != "employee" or profile.workflow_id != workflow_id or await database.get_workflow(workflow_id) is not None:+ if profile.kind not in {"employee", "agent"} or profile.workflow_id != workflow_id or await database.get_workflow(workflow_id) is not None:raise NodeUserError("Browser cleanup is not associated with a deleted workflow.")binding = await bind_profile(database, profile.id, principal)if binding["owner_id"] != replica_id():diff --git a/server/nodes/browser/_egress.py b/server/nodes/browser/_egress.pyindex fd2b8a9a..d805038a 100644--- a/server/nodes/browser/_egress.py+++ b/server/nodes/browser/_egress.py@@ -79,17 +79,29 @@ def _split_host_port(authority: str, default_port: int) -> Tuple[str, int]:class EgressProxy:"""One filtering HTTP/CONNECT proxy on ``127.0.0.1:<ephemeral>``."""- def __init__(self, policy: PolicyProvider, *, name: str = "browser-egress") -> None:+ def __init__(self, policy: PolicyProvider, *, name: str = "browser-egress", runtime_version: int = 0) -> None:+ if runtime_version not in (0, 1):+ raise ValueError("Unsupported browser proxy resource version")+ self.runtime_version = runtime_versionself._policy = policyself._name = nameself._loop: Optional[asyncio.AbstractEventLoop] = Noneself._thread: Optional[threading.Thread] = Noneself._server: Optional[asyncio.base_events.Server] = Noneself.port: Optional[int] = None+ self._tasks: set[asyncio.Task] = set()+ self._writers: set[asyncio.StreamWriter] = set()+ self._lifecycle_lock = threading.RLock()+ self._closing = False+ self._drain_timeout = 5.0+ self._halfclose_timeout = 10.0+ self._close_timeout = 1.0# -- lifecycle (called from the main loop) ------------------------------def start(self) -> int:+ if self.runtime_version == 1:+ return self._start_v1()if self.port is not None:return self.portready = threading.Event()@@ -124,6 +136,8 @@ class EgressProxy:return self.portdef stop(self) -> None:+ if self.runtime_version == 1:+ return self._stop_v1()loop, server = self._loop, self._serverif loop is None:return@@ -148,6 +162,129 @@ class EgressProxy:self._loop = self._server = self._thread = Noneself.port = None+ def _start_v1(self) -> int:+ with self._lifecycle_lock:+ if self._thread is not None and self._thread.is_alive():+ if self._closing or self.port is None:+ raise RuntimeError("Browser proxy cleanup has not completed")+ return self.port+ self._closing = False+ ready = threading.Event()+ errors = []++ def run():+ loop = asyncio.new_event_loop()+ self._loop = loop+ asyncio.set_event_loop(loop)+ try:+ self._server = loop.run_until_complete(asyncio.start_server(+ self._handle, host="127.0.0.1", port=0, limit=_HEADER_LIMIT))+ self.port = self._server.sockets[0].getsockname()[1]+ ready.set()+ loop.run_forever()+ except BaseException as exc:+ errors.append(exc)+ ready.set()+ finally:+ loop.run_until_complete(loop.shutdown_asyncgens())+ loop.close()++ self._thread = threading.Thread(target=run, name=self._name, daemon=True)+ self._thread.start()+ if not ready.wait(timeout=10):+ self._closing = True+ raise RuntimeError("Browser proxy startup did not complete")+ if errors:+ raise errors[0]+ return self.port++ async def _close_writer(self, writer):+ try:+ writer.close()+ await asyncio.wait_for(writer.wait_closed(), timeout=self._close_timeout)+ except (OSError, RuntimeError, TimeoutError):+ transport = getattr(writer, "transport", None)+ if transport is not None:+ transport.abort()+ finally:+ self._writers.discard(writer)++ def _stop_v1(self) -> bool:+ with self._lifecycle_lock:+ thread, loop = self._thread, self._loop+ if thread is None:+ return True+ if not thread.is_alive():+ self._loop = self._server = self._thread = None+ self.port = None+ return True+ self._closing = True++ async def shutdown():+ if self._server is not None:+ self._server.close()+ tasks = [t for t in self._tasks if t is not asyncio.current_task()]+ for task in tasks:+ task.cancel()+ await asyncio.gather(*(self._close_writer(w) for w in tuple(self._writers)), return_exceptions=True)+ if tasks:+ await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=3)+ if self._server is not None:+ await asyncio.wait_for(self._server.wait_closed(), timeout=1)+ loop.call_soon(loop.stop)++ try:+ if loop is None or loop.is_closed():+ return False+ future = asyncio.run_coroutine_threadsafe(shutdown(), loop)+ future.result(timeout=5)+ thread.join(timeout=2)+ except Exception:+ # Keep every handle: a caller may retry cleanup but must not+ # replace this still-live proxy with a second resource.+ logger.warning("[browser] proxy cleanup is still pending")+ return False+ if thread.is_alive():+ return False+ self._loop = self._server = self._thread = None+ self.port = None+ self._tasks.clear()+ self._writers.clear()+ return True++ async def _relay_v1(self, src, dst):+ try:+ while True:+ data = await src.read(_RELAY_CHUNK)+ if not data:+ if dst.can_write_eof():+ dst.write_eof()+ await asyncio.wait_for(dst.drain(), timeout=self._drain_timeout)+ return "eof"+ dst.write(data)+ await asyncio.wait_for(dst.drain(), timeout=self._drain_timeout)+ except (ConnectionError, OSError, TimeoutError, RuntimeError):+ return "error"++ async def _relay_pair(self, client_reader, client_writer, upstream_reader, upstream_writer):+ tasks = {asyncio.create_task(self._relay_v1(client_reader, upstream_writer)),+ asyncio.create_task(self._relay_v1(upstream_reader, client_writer))}+ self._tasks.update(tasks)+ try:+ done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)+ if all(task.result() == "eof" for task in done) and pending:+ # TCP half-close is valid: let the other direction finish its+ # response, but never retain an idle tunnel indefinitely.+ await asyncio.wait_for(asyncio.gather(*pending), timeout=self._halfclose_timeout)+ except TimeoutError:+ pass+ finally:+ for task in tasks:+ if not task.done():+ task.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)+ self._tasks.difference_update(tasks)+# -- connection handling (proxy thread) ---------------------------------async def _resolve_allowed(self, host: str, port: int, policy: NetPolicy) -> list[str]:@@ -189,15 +326,23 @@ class EgressProxy:# localhost may resolve to IPv6 first while the app# listens only on IPv4. Try only addresses already# checked above; never resolve the hostname again.- return await asyncio.open_connection(address, port)+ opened = await asyncio.open_connection(address, port)+ if self.runtime_version == 1:+ self._writers.add(opened[1])+ return openedexcept OSError as exc:last_error = excraise last_errorexcept (OSError, asyncio.TimeoutError) as exc:raise ConnectionError(f"could not connect to {host}:{port}: {exc}") from exc- @staticmethod- async def _refuse(writer: asyncio.StreamWriter, status: str, reason: str) -> None:+ async def _drain(self, writer):+ if self.runtime_version == 1:+ await asyncio.wait_for(writer.drain(), timeout=self._drain_timeout)+ else:+ await writer.drain()++ async def _refuse(self, writer: asyncio.StreamWriter, status: str, reason: str) -> None:body = f"Blocked by OpenCompany: {reason}\n".encode("utf-8", errors="replace")head = (f"HTTP/1.1 {status}\r\nContent-Type: text/plain; charset=utf-8\r\n"@@ -205,11 +350,15 @@ class EgressProxy:).encode("ascii")try:writer.write(head + body)- await writer.drain()+ await self._drain(writer)except (ConnectionError, OSError):passasync def _handle(self, client_reader: asyncio.StreamReader, client_writer: asyncio.StreamWriter) -> None:+ current = asyncio.current_task()+ if self.runtime_version == 1:+ self._tasks.add(current)+ self._writers.add(client_writer)upstream_writer: Optional[asyncio.StreamWriter] = Nonetry:head = await _read_head(client_reader)@@ -225,7 +374,7 @@ class EgressProxy:host, port = _split_host_port(target, 443)upstream_reader, upstream_writer = await self._open(host, port, policy)client_writer.write(b"HTTP/1.1 200 Connection Established\r\n\r\n")- await client_writer.drain()+ await self._drain(client_writer)else:url = urlsplit(target)if url.scheme.lower() != "http" or not url.hostname:@@ -254,12 +403,12 @@ class EgressProxy:# host, down this already-checked socket.headers.append(b"Connection: " + (b"Upgrade" if upgrade else b"close"))upstream_writer.write(f"{method} {path} {version}\r\n".encode("latin-1") + b"\r\n".join(headers) + b"\r\n\r\n")- await upstream_writer.drain()+ await self._drain(upstream_writer)- await asyncio.gather(- _relay(client_reader, upstream_writer),- _relay(upstream_reader, client_writer),- )+ if self.runtime_version == 1:+ await self._relay_pair(client_reader, client_writer, upstream_reader, upstream_writer)+ else:+ await asyncio.gather(_relay(client_reader, upstream_writer), _relay(upstream_reader, client_writer))except _Blocked as exc:await self._refuse(client_writer, "403 Forbidden", str(exc))except ConnectionError as exc:@@ -271,7 +420,10 @@ class EgressProxy:except Exception: # noqa: BLE001 - one bad request must not stop the proxylogger.debug("[browser] egress proxy request failed", exc_info=True)finally:- for writer in (upstream_writer, client_writer):+ if self.runtime_version == 1:+ await asyncio.gather(*(self._close_writer(w) for w in (upstream_writer, client_writer) if w is not None), return_exceptions=True)+ self._tasks.discard(current)+ for writer in (() if self.runtime_version == 1 else (upstream_writer, client_writer)):if writer is not None:try:writer.close()diff --git a/server/nodes/browser/_runtime.py b/server/nodes/browser/_runtime.pyindex 79114b54..ee5f6962 100644--- a/server/nodes/browser/_runtime.py+++ b/server/nodes/browser/_runtime.py@@ -107,6 +107,12 @@ class BrowserRuntime:self._controllers: Dict[str, ProfileController] = {}self._chromes: Dict[str, ChromeProcess] = {}self._proxies: Dict[str, EgressProxy] = {}+ self._teardown_locks: Dict[str, asyncio.Lock] = {}+ self._stop_locks: Dict[str, asyncio.Lock] = {}+ self._stopping: set[str] = set()+ self._cleanup_pending: set[str] = set()+ self._shutting_down_v1 = False+ self._resource_versions: Dict[str, int] = {}self._starting: Dict[str, asyncio.Task] = {}self._sessions: Dict[str, BrowserSession] = {}self._reaper: Optional[asyncio.Task] = None@@ -192,6 +198,14 @@ class BrowserRuntime:async def open(self, profile: Profile, *, wait: float = 60.0) -> ProfileRuntime:"""The profile's running browser, starting Chrome if needed."""+ if profile.kind == "agent":+ from services.plugin.deps import get_database+ from ._profiles import ProfileStore+ # Entry-point independent: a credentials/viewer open may be first.+ profile = await ProfileStore(get_database()).get(profile.owner_id, profile.id)+ self._resource_versions[profile.id] = 1+ if self._shutting_down_v1 or profile.id in self._stopping or profile.id in self._cleanup_pending:+ raise NodeUserError("Private Browser cleanup is still pending. Retry after owner recovery.")from services.browser_owners import _persistentfrom services.plugin.deps import get_databaseif getattr(self._settings(), "distributed_mode", False) is True or _persistent(get_database()):@@ -207,9 +221,14 @@ class BrowserRuntime:controller.needs_observation |= bool(binding.get("needs_observation"))controller.challenge_required = bool(binding.get("challenge_required"))controller.recovered_assistance = binding.get("assistance")+ if getattr(profile, "runtime_version", 0) == 1 and (+ self._shutting_down_v1 or profile.id in self._stopping or profile.id in self._cleanup_pending):+ raise NodeUserError("Private Browser cleanup is still pending. Retry after owner recovery.")runtime = self._profiles.get(profile.id)if runtime is not None and runtime.running:return runtime+ if runtime is not None and getattr(profile, "runtime_version", 0) == 1:+ await self._teardown(runtime)self._ensure_reaper()task = self._starting.get(profile.id)if task is None or task.done():@@ -226,6 +245,23 @@ class BrowserRuntime:raise NodeUserError("The browser is still starting; try again in a moment.") from Noneasync def _start(self, profile: Profile) -> ProfileRuntime:+ try:+ return await self._start_impl(profile)+ except BaseException:+ if getattr(profile, "runtime_version", 0) == 1:+ runtime = self._profiles.get(profile.id)+ if runtime is not None:+ await self._teardown(runtime)+ else:+ self._cleanup_pending.add(profile.id)+ chrome = self._chromes.get(profile.id)+ if chrome is not None:+ await self._stop_private_chrome(chrome)+ await self._stop_private_proxy(profile.id)+ self._cleanup_pending.discard(profile.id)+ raise++ async def _start_impl(self, profile: Profile) -> ProfileRuntime:settings = self._settings()pin = get_config().chromeinstaller = get_chrome_installer()@@ -244,7 +280,8 @@ class BrowserRuntime:controller = self.controller_for(profile)proxy = self._proxies.get(profile.id)if proxy is None:- proxy = EgressProxy(lambda c=controller: self._effective_policy(c), name=f"browser-egress-{profile.id}")+ proxy = EgressProxy(lambda c=controller: self._effective_policy(c), name=f"browser-egress-{profile.id}",+ **({"runtime_version": 1} if getattr(profile, "runtime_version", 0) == 1 else {}))self._proxies[profile.id] = proxyproxy_port = await asyncio.to_thread(proxy.start)@@ -293,7 +330,8 @@ class BrowserRuntime:raise NodeUserError("The launched browser is older than this profile or the minimum supported browser version. Update the selected browser or use another profile.")await self._wire(runtime)except BaseException:- await chrome.shutdown()+ if getattr(profile, "runtime_version", 0) != 1:+ await chrome.shutdown()raiseself._profiles[profile.id] = runtimecontroller.last_activity = time.monotonic()@@ -385,6 +423,9 @@ class BrowserRuntime:if self._profiles.get(runtime.profile.id) is not runtime:returnlogger.warning("[browser] Chrome for profile %s stopped unexpectedly", runtime.profile.id)+ if getattr(runtime.profile, "runtime_version", 0) == 1:+ await self.stop_profile(runtime.profile.id, reason="Chrome stopped")+ returnawait self._teardown(runtime)controller = runtime.controllercontroller.tabs.clear()@@ -394,6 +435,21 @@ class BrowserRuntime:await controller.release_lease()async def _teardown(self, runtime: ProfileRuntime) -> None:+ if getattr(runtime.profile, "runtime_version", 0) == 1:+ async with self._teardown_locks.setdefault(runtime.profile.id, asyncio.Lock()):+ if self._profiles.get(runtime.profile.id) is not runtime:+ return+ self._cleanup_pending.add(runtime.profile.id)+ try:+ await asyncio.to_thread(runtime.cli.stop_daemon)+ except Exception:+ logger.debug("[browser] private daemon stop failed", exc_info=True)+ await self._stop_private_chrome(runtime.chrome)+ await self._stop_private_proxy(runtime.profile.id)+ if self._profiles.get(runtime.profile.id) is runtime:+ self._profiles.pop(runtime.profile.id, None)+ self._cleanup_pending.discard(runtime.profile.id)+ returnself._profiles.pop(runtime.profile.id, None)try:runtime.cli.stop_daemon()@@ -407,9 +463,55 @@ class BrowserRuntime:except Exception: # noqa: BLE001pass+ async def _stop_private_chrome(self, chrome: ChromeProcess) -> None:+ try:+ await asyncio.wait_for(chrome.shutdown(), timeout=_STOP_TIMEOUT)+ except Exception:+ await chrome.stop()+ if chrome.is_running():+ raise NodeUserError("Private Browser cleanup is still pending; restore its owner before deleting it.")++ async def _stop_private_proxy(self, profile_id: str) -> None:+ proxy = self._proxies.get(profile_id)+ if proxy is None:+ return+ if await asyncio.to_thread(proxy.stop) is False:+ raise NodeUserError("Private Browser proxy cleanup is still pending. Retry after owner recovery.")+ if self._proxies.get(profile_id) is proxy:+ self._proxies.pop(profile_id, None)+async def stop_profile(self, profile_id: str, *, reason: str = "") -> None:runtime = self._profiles.get(profile_id)+ private = (self._resource_versions.get(profile_id) == 1+ or getattr(getattr(runtime, "profile", None), "runtime_version", 0) == 1+ or getattr(self._proxies.get(profile_id), "runtime_version", 0) == 1)+ if private:+ async with self._stop_locks.setdefault(profile_id, asyncio.Lock()):+ self._stopping.add(profile_id)+ try:+ await self._stop_profile_impl(profile_id, reason=reason, private=True)+ finally:+ self._stopping.discard(profile_id)+ return+ await self._stop_profile_impl(profile_id, reason=reason)++ async def _stop_profile_impl(self, profile_id: str, *, reason: str, private: bool = False) -> None:+ if private:+ starting = self._starting.get(profile_id)+ if starting is not None and not starting.done() and starting is not asyncio.current_task():+ starting.cancel()+ await asyncio.gather(starting, return_exceptions=True)+ runtime = self._profiles.get(profile_id)if runtime is None:+ if private:+ chrome = self._chromes.get(profile_id)+ if chrome is not None:+ await self._stop_private_chrome(chrome)+ proxy = self._proxies.get(profile_id)+ if proxy is not None and getattr(proxy, "runtime_version", 0) == 1:+ await self._stop_private_proxy(profile_id)+ if private:+ self._cleanup_pending.discard(profile_id)returncontroller = runtime.controllerawait controller.release_lease()@@ -423,9 +525,10 @@ class BrowserRuntime:controller.tabs.clear()controller.active_target_id = Noneawait controller.emit("closed", {"reason": reason or "stopped"})- proxy = self._proxies.pop(profile_id, None)- if proxy is not None:- await asyncio.to_thread(proxy.stop)+ if not private:+ proxy = self._proxies.pop(profile_id, None)+ if proxy is not None:+ await asyncio.to_thread(proxy.stop)logger.info("[browser] profile %s stopped (%s)", profile_id, reason or "requested")async def _make_room(self, *, exclude: str) -> None:@@ -458,15 +561,33 @@ class BrowserRuntime:async def shutdown(self) -> None:"""Stop every profile's Chrome gracefully (the plugin shutdown hook)."""+ self._shutting_down_v1 = Trueif self._reaper is not None:self._reaper.cancel()- await asyncio.gather(*(self._teardown(r) for r in list(self._profiles.values())), return_exceptions=True)- for proxy in list(self._proxies.values()):+ starting = [task for profile_id, task in self._starting.items()+ if self._resource_versions.get(profile_id) == 1 and not task.done()]+ for task in starting:+ task.cancel()+ await asyncio.gather(*starting, return_exceptions=True)+ private_ids = {profile_id for profile_id, version in self._resource_versions.items() if version == 1}+ private_ids.update(profile_id for profile_id, runtime in self._profiles.items()+ if getattr(runtime.profile, "runtime_version", 0) == 1)+ private_ids.update(profile_id for profile_id, proxy in self._proxies.items()+ if getattr(proxy, "runtime_version", 0) == 1)+ await asyncio.gather(*(self.stop_profile(profile_id, reason="backend shutdown") for profile_id in private_ids),+ *(self._teardown(r) for r in list(self._profiles.values()) if r.profile.id not in private_ids),+ return_exceptions=True)+ for profile_id, proxy in list(self._proxies.items()):try:- proxy.stop()+ if getattr(proxy, "runtime_version", 0) == 1:+ await self._stop_private_proxy(profile_id)+ else:+ proxy.stop()+ self._proxies.pop(profile_id, None)except Exception: # noqa: BLE001pass- self._proxies.clear()+ if self._cleanup_pending or any(getattr(proxy, "runtime_version", 0) == 1 for proxy in self._proxies.values()):+ raise NodeUserError("Private Browser cleanup is still pending; browser ownership must remain fenced.")# -- status ------------------------------------------------------------------diff --git a/server/tests/nodes/browser/test_egress_v1.py b/server/tests/nodes/browser/test_egress_v1.pynew file mode 100644index 00000000..7d897986--- /dev/null+++ b/server/tests/nodes/browser/test_egress_v1.py@@ -0,0 +1,181 @@+"""Controlled local sockets only; no Chrome or external network."""+import asyncio+import threading+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from nodes.browser._egress import EgressProxy+from services.netpolicy import NetPolicy+++async def test_private_proxy_preserves_tcp_half_close_and_drains_every_handle():+ async def respond(reader, writer):+ data = await reader.read()+ writer.write(b"reply:" + data)+ await writer.drain()+ writer.close()+ await writer.wait_closed()+ upstream = await asyncio.start_server(respond, "127.0.0.1", 0)+ upstream_port = upstream.sockets[0].getsockname()[1]+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)+ try:+ port = await asyncio.to_thread(proxy.start)+ reader, writer = await asyncio.open_connection("127.0.0.1", port)+ writer.write(f"CONNECT 127.0.0.1:{upstream_port} HTTP/1.1\r\n\r\n".encode())+ await writer.drain()+ assert b"200" in await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 2)+ writer.write(b"payload")+ await writer.drain()+ writer.write_eof()+ assert await asyncio.wait_for(reader.read(), 2) == b"reply:payload"+ writer.close()+ await writer.wait_closed()+ finally:+ assert await asyncio.to_thread(proxy.stop) is True+ upstream.close()+ await upstream.wait_closed()+ assert not proxy._tasks and not proxy._writers and proxy._thread is None+ assert await asyncio.to_thread(proxy.stop) is True+++async def test_backpressured_writer_is_bounded_and_close_precedes_wait_closed():+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)+ proxy._drain_timeout = proxy._close_timeout = 0.01+ reader = asyncio.StreamReader()+ reader.feed_data(b"x")+ events = []+ class Writer:+ transport = SimpleNamespace(abort=lambda: events.append("abort"))+ def write(self, _): pass+ async def drain(self): await asyncio.sleep(100)+ def close(self): events.append("close")+ async def wait_closed(self): await asyncio.sleep(100)+ writer = Writer()+ assert await proxy._relay_v1(reader, writer) == "error"+ await proxy._close_writer(writer)+ assert events == ["close", "abort"]+++async def test_stop_drains_an_active_idle_tunnel_before_waiting_for_server_close():+ finished = asyncio.Event()+ async def respond(reader, writer):+ try:+ await reader.read()+ finally:+ writer.close()+ await writer.wait_closed()+ finished.set()+ upstream = await asyncio.start_server(respond, "127.0.0.1", 0)+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)+ try:+ port = await asyncio.to_thread(proxy.start)+ reader, writer = await asyncio.open_connection("127.0.0.1", port)+ target = upstream.sockets[0].getsockname()[1]+ writer.write(f"CONNECT 127.0.0.1:{target} HTTP/1.1\r\n\r\n".encode())+ await writer.drain()+ await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), 2)+ assert await asyncio.wait_for(asyncio.to_thread(proxy.stop), 4) is True+ assert await asyncio.wait_for(reader.read(), 2) == b""+ writer.close()+ await writer.wait_closed()+ await asyncio.wait_for(finished.wait(), 2)+ assert not proxy._writers and not proxy._tasks+ finally:+ await asyncio.to_thread(proxy.stop)+ upstream.close()+ await upstream.wait_closed()+++def test_failed_stop_retains_live_resource_for_retry(monkeypatch):+ proxy = EgressProxy(lambda: NetPolicy(), runtime_version=1)+ thread = SimpleNamespace(is_alive=lambda: True)+ loop = SimpleNamespace(is_closed=lambda: False)+ proxy._thread, proxy._loop, proxy.port = thread, loop, 1234+ def submit(coro, _loop):+ coro.close()+ return SimpleNamespace(result=lambda **_: (_ for _ in ()).throw(TimeoutError()))+ monkeypatch.setattr(asyncio, "run_coroutine_threadsafe", submit)+ assert proxy.stop() is False+ assert proxy._thread is thread and proxy._loop is loop and proxy.port == 1234+ with pytest.raises(RuntimeError, match="cleanup"):+ proxy.start()+++async def test_private_runtime_teardown_runs_blocking_cleanup_off_loop_and_keeps_failed_proxy():+ from nodes.browser._runtime import BrowserRuntime+ from services.plugin import NodeUserError+ runtime = BrowserRuntime()+ threads = []+ profile = SimpleNamespace(id="private", runtime_version=1)+ proxy = SimpleNamespace(runtime_version=1, stop=lambda: threads.append(threading.get_ident()) or False)+ running = SimpleNamespace(profile=profile, cli=SimpleNamespace(stop_daemon=lambda: threads.append(threading.get_ident())),+ chrome=SimpleNamespace(shutdown=AsyncMock(), stop=AsyncMock(), is_running=lambda: False))+ runtime._profiles[profile.id] = running+ runtime._proxies[profile.id] = proxy+ with pytest.raises(NodeUserError, match="cleanup"):+ await runtime._teardown(running)+ assert runtime._proxies[profile.id] is proxy+ assert threads and all(t != threading.get_ident() for t in threads)+ proxy.stop = lambda: True+ await runtime._teardown(running)+ assert profile.id not in runtime._proxies and profile.id not in runtime._profiles+ assert profile.id not in runtime._cleanup_pending+++async def test_private_stop_blocks_reopen_until_events_finish(monkeypatch):+ from nodes.browser._profiles import ProfileStore+ from nodes.browser._runtime import BrowserRuntime+ from services.plugin import NodeUserError+ fleet = BrowserRuntime()+ profile = SimpleNamespace(id="private", owner_id="owner", kind="agent", runtime_version=1)+ monkeypatch.setattr("services.plugin.deps.get_database", lambda: object())+ monkeypatch.setattr(ProfileStore, "get", AsyncMock(return_value=profile))+ new_proxy = SimpleNamespace(runtime_version=1, stop=lambda: pytest.fail("new proxy was stopped"))+ async def emit(*_):+ with pytest.raises(NodeUserError, match="cleanup"):+ await fleet.open(profile)+ # A later, distinct resource must never be removed by an old stop tail.+ fleet._proxies[profile.id] = new_proxy+ controller = SimpleNamespace(release_lease=AsyncMock(), sensitive_login=False,+ tabs={}, active_target_id=None, emit=emit)+ running = SimpleNamespace(profile=profile, controller=controller,+ cli=SimpleNamespace(stop_daemon=lambda: None),+ chrome=SimpleNamespace(shutdown=AsyncMock(), is_running=lambda: False))+ fleet._profiles[profile.id] = running+ fleet._proxies[profile.id] = SimpleNamespace(runtime_version=1, stop=lambda: True)+ await fleet.stop_profile(profile.id)+ assert fleet._proxies[profile.id] is new_proxy+ assert profile.id not in fleet._stopping+++async def test_shutdown_cancels_private_start_before_final_resource_cleanup():+ from nodes.browser._runtime import BrowserRuntime+ fleet = BrowserRuntime()+ started = asyncio.Event()+ stopped = []+ async def starting():+ started.set()+ try:+ await asyncio.Event().wait()+ finally:+ fleet._proxies["private"] = SimpleNamespace(runtime_version=1, stop=lambda: stopped.append(True) or True)+ task = asyncio.create_task(starting())+ fleet._starting["private"] = task+ fleet._resource_versions["private"] = 1+ await started.wait()+ await fleet.shutdown()+ assert task.cancelled() and stopped == [True]+ assert not fleet._proxies+++async def test_shutdown_keeps_failed_private_proxy_and_reports_fencing_required():+ from nodes.browser._runtime import BrowserRuntime+ from services.plugin import NodeUserError+ fleet = BrowserRuntime()+ proxy = SimpleNamespace(runtime_version=1, stop=lambda: False)+ fleet._proxies["private"] = proxy+ with pytest.raises(NodeUserError, match="fenced"):+ await fleet.shutdown()+ assert fleet._proxies["private"] is proxy