Python
fix(server): a websocket client that stopped reading no longer hangs shutdown
Devika Vermab50283ed36be
diff --git a/coworker/server/app.py b/coworker/server/app.pyindex e2bb9a99..67fa6e21 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -298,6 +298,11 @@ def create_app(manager: SessionManager) -> FastAPI:allow_methods=["*"],allow_headers=["*"],)+ # A GUI websocket whose client stopped reading is dropped, and every open websocket can be+ # aborted at shutdown, so neither holds the server up (ws_guard.py, T3-3).+ from .ws_guard import WebsocketGuard++ app.add_middleware(WebsocketGuard)app.state.manager = manager@app.get("/v1/capabilities")diff --git a/coworker/server/run.py b/coworker/server/run.pyindex 16ad182a..4725bdc6 100644--- a/coworker/server/run.py+++ b/coworker/server/run.py@@ -222,10 +222,19 @@ def main(argv=None) -> None:_exit_when_orphaned()_warn_if_state_shared()+ from .ws_guard import GuardedServer+app = build_app(args.cwd, args.model, args.mode)- uvicorn.run(+ # GuardedServer: uvicorn's server, but it aborts the websockets still open before it+ # waits for connections at shutdown, so Ctrl+C never hangs on a tab that stopped+ # reading (ws_guard.py, T3-3).+ config = uvicorn.Config(app, host=args.host, port=args.port, ws_max_size=_WS_MAX_FRAME_BYTES)+ try:+ GuardedServer(config).run()+ except KeyboardInterrupt:+ pass # as uvicorn.run: the signal is raised again once the server has stoppedfinally:if generated_token_path is not None:generated_token_path.unlink(missing_ok=True)diff --git a/coworker/server/ws_guard.py b/coworker/server/ws_guard.pynew file mode 100644index 00000000..319e9909--- /dev/null+++ b/coworker/server/ws_guard.py@@ -0,0 +1,126 @@+"""Websockets that cannot hold the server up (T3-3).++The server hung at "Shutting down" after Ctrl+C when a browser tab had stopped reading its+websocket (a frozen or discarded tab, or a laptop that slept) while the server kept pushing live+events. The unread data filled the connection's send buffer, so its close frame could never go+out and the connection never closed. uvicorn waits for every connection to close before it stops,+and since Python 3.12 `Server.wait_closed()` also waits for every transport, so even a second+Ctrl+C could not get past it.++Two guards, both by aborting the connection's transport (an abort drops what is still buffered;+a close waits to send it):++- `WebsocketGuard` (ASGI middleware): on the GUI's own websockets, a send that cannot finish+ within `send_timeout` seconds means the client stopped reading, and the connection is+ aborted. The machine links (`/ws/machines/…`) are left alone: they carry large frames over+ slow networks.+- `GuardedServer` (uvicorn): at shutdown, every websocket still open is aborted before uvicorn+ waits for its connections.+"""++from __future__ import annotations++import asyncio+import logging+import os+import weakref+from typing import Any, Awaitable, Callable, Optional++import uvicorn+from starlette.websockets import WebSocketDisconnect++log = logging.getLogger(__name__)++SEND_TIMEOUT_SECONDS = 15.0+# The GUI's own websockets: the session stream and the app-wide event stream.+GUARDED_PATHS = ("/ws/session/", "/ws/events")++_open: "weakref.WeakSet[Any]" = weakref.WeakSet()+++def _send_timeout() -> float:+ try:+ value = float(os.environ.get("COWORKER_WS_SEND_TIMEOUT", "") or SEND_TIMEOUT_SECONDS)+ except ValueError:+ return SEND_TIMEOUT_SECONDS+ return value if value > 0 else SEND_TIMEOUT_SECONDS+++def _transport_of(send: Callable[..., Any]) -> Optional[asyncio.BaseTransport]:+ """The connection's transport, from uvicorn's websocket protocol (its `send` is a bound+ method of the protocol, for both the websockets and the wsproto implementation). None under+ a test client or another server."""+ owner = getattr(send, "__self__", None)+ transport = getattr(owner, "transport", None)+ return transport if isinstance(transport, asyncio.BaseTransport) else None+++def _abort(transport: Optional[asyncio.BaseTransport]) -> None:+ if transport is None:+ return+ try:+ transport.abort() # type: ignore[attr-defined]+ except Exception: # noqa: BLE001 - already gone+ pass+++def open_count() -> int:+ """How many websocket transports are open now (for tests and logs)."""+ return sum(1 for t in list(_open) if not t.is_closing())+++def abort_open_websockets() -> int:+ """Abort every websocket still open. Returns how many were aborted."""+ n = 0+ for transport in list(_open):+ if not transport.is_closing():+ n += 1+ _abort(transport)+ _open.clear()+ return n+++class WebsocketGuard:+ """ASGI middleware: track every websocket's transport, and abort a GUI websocket whose+ client stopped reading. Plain HTTP passes straight through."""++ def __init__(self, app: Callable[..., Awaitable[None]], *, send_timeout: Optional[float] = None) -> None:+ self.app = app+ self.send_timeout = send_timeout++ async def __call__(self, scope: dict[str, Any], receive: Callable[..., Any], send: Callable[..., Any]) -> None:+ if scope.get("type") != "websocket":+ await self.app(scope, receive, send)+ return+ transport = _transport_of(send)+ if transport is not None:+ _open.add(transport)+ path = str(scope.get("path") or "")+ timeout = self.send_timeout if self.send_timeout is not None else _send_timeout()++ async def guarded_send(message: dict[str, Any]) -> None:+ try:+ await asyncio.wait_for(send(message), timeout)+ except asyncio.TimeoutError:+ log.warning("websocket %s: the client stopped reading for %.0f s; dropping it", path, timeout)+ _abort(transport)+ # what a handler already expects when its client goes away+ raise WebSocketDisconnect(code=1011, reason="client stopped reading")++ try:+ # only the GUI's own websockets get the send timeout; every one is tracked+ await self.app(scope, receive, guarded_send if path.startswith(GUARDED_PATHS) else send)+ finally:+ if transport is not None:+ _open.discard(transport)+++class GuardedServer(uvicorn.Server):+ """uvicorn's server, but at shutdown the websockets still open are aborted first, so it+ never waits on a client that stopped reading."""++ async def shutdown(self, sockets: Any = None) -> None:+ n = abort_open_websockets()+ if n:+ log.info("aborted %d open websocket(s) before shutting down", n)+ await super().shutdown(sockets=sockets)diff --git a/tests/test_server.py b/tests/test_server.pyindex 10862b94..7a18fd12 100644--- a/tests/test_server.py+++ b/tests/test_server.py@@ -483,9 +483,6 @@ def test_ws_rate_limits_inbound_frames(tmp_path):def test_server_sets_explicit_websocket_frame_limit(tmp_path, monkeypatch):- import sys- from types import SimpleNamespace-from coworker.server import run as server_runseen = {}@@ -494,15 +491,21 @@ def test_server_sets_explicit_websocket_frame_limit(tmp_path, monkeypatch):monkeypatch.setattr(server_run, "_ensure_ca_bundle", lambda: None)monkeypatch.setattr(server_run, "_exit_when_orphaned", lambda: None)monkeypatch.setattr(server_run, "build_app", lambda *args: fake_app)- monkeypatch.setitem(- sys.modules,- "uvicorn",- SimpleNamespace(run=lambda app, **kwargs: seen.update(app=app, **kwargs)),- )+ from coworker.server import ws_guard++ class _Server:+ # T3-3: the launcher runs uvicorn through GuardedServer (ws_guard.py)+ def __init__(self, config):+ seen.update(app=config.app, ws_max_size=config.ws_max_size, port=config.port)++ def run(self):+ seen["ran"] = True++ monkeypatch.setattr(ws_guard, "GuardedServer", _Server)server_run.main(["--cwd", str(tmp_path), "--port", "8766"])- assert seen["app"] is fake_app+ assert seen["app"] is fake_app and seen["ran"] and seen["port"] == 8766assert seen["ws_max_size"] == server_run._WS_MAX_FRAME_BYTESdiff --git a/tests/test_ws_guard.py b/tests/test_ws_guard.pynew file mode 100644index 00000000..d0a7b2f7--- /dev/null+++ b/tests/test_ws_guard.py@@ -0,0 +1,140 @@+"""T3-3: a websocket whose client stopped reading must not hold the server up. A GUI websocket+send that cannot finish in time drops the connection, and at shutdown every open websocket is+aborted before uvicorn waits for its connections."""++from __future__ import annotations++import asyncio++import pytest+from fastapi.testclient import TestClient+from starlette.websockets import WebSocketDisconnect++from coworker.server import ws_guard+++class _Transport(asyncio.BaseTransport):+ def __init__(self) -> None:+ super().__init__()+ self.aborted = False++ def abort(self) -> None:+ self.aborted = True++ def is_closing(self) -> bool:+ return self.aborted+++class _Protocol:+ """Stands in for uvicorn's websocket protocol: its `send` is a bound method, and it holds+ the connection's transport."""++ def __init__(self, stuck: bool) -> None:+ self.transport = _Transport()+ self.stuck = stuck+ self.sent: list[dict] = []++ async def send(self, message: dict) -> None:+ if self.stuck and message.get("type") == "websocket.send":+ await asyncio.sleep(3600) # the client stopped reading: the buffer never drains+ self.sent.append(message)+++async def _receive() -> dict:+ return {"type": "websocket.connect"}+++def _app(log: list):+ async def app(scope, receive, send):+ assert ws_guard.open_count() == 1 # tracked while it is open+ await send({"type": "websocket.accept"})+ try:+ await send({"type": "websocket.send", "text": "x"})+ except WebSocketDisconnect as exc:+ log.append(exc.code)+ return+ log.append("sent")++ return app+++def test_a_gui_websocket_whose_client_stopped_reading_is_dropped():+ log: list = []+ proto = _Protocol(stuck=True)+ guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)+ asyncio.run(guard({"type": "websocket", "path": "/ws/session/s1"}, _receive, proto.send))+ assert proto.transport.aborted and log == [1011]+ assert ws_guard.open_count() == 0 # no longer tracked+++def test_a_reading_client_is_untouched():+ log: list = []+ proto = _Protocol(stuck=False)+ guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)+ asyncio.run(guard({"type": "websocket", "path": "/ws/events"}, _receive, proto.send))+ assert not proto.transport.aborted and log == ["sent"]+++def test_every_websocket_is_tracked_only_while_open():+ log: list = []+ proto = _Protocol(stuck=False)+ guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)+ asyncio.run(guard({"type": "websocket", "path": "/ws/machines/m1/p"}, _receive, proto.send))+ assert log == ["sent"] and ws_guard.open_count() == 0 and proto.transport not in ws_guard._open+++def test_machine_links_have_no_send_timeout():+ """/ws/machines carries large frames over slow links: no timeout there, only the+ shutdown abort."""++ async def go():+ proto = _Protocol(stuck=True)+ guard = ws_guard.WebsocketGuard(_app([]), send_timeout=0.05)+ task = asyncio.create_task(guard({"type": "websocket", "path": "/ws/machines/m1/p"}, _receive, proto.send))+ await asyncio.sleep(0.3)+ assert not task.done() and not proto.transport.aborted # still waiting, not dropped+ assert ws_guard.abort_open_websockets() == 1 # what shutdown does+ assert proto.transport.aborted+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ asyncio.run(go())+++def test_the_server_aborts_open_websockets_before_it_waits(monkeypatch):+ order: list[str] = []+ transport = _Transport()+ ws_guard._open.add(transport)++ async def base_shutdown(self, sockets=None):+ order.append("uvicorn waits" + (" (none open)" if ws_guard.open_count() == 0 else " (still open!)"))++ monkeypatch.setattr(ws_guard.uvicorn.Server, "shutdown", base_shutdown)+ server = ws_guard.GuardedServer.__new__(ws_guard.GuardedServer)+ asyncio.run(server.shutdown())+ assert transport.aborted and order == ["uvicorn waits (none open)"]+++def test_the_send_timeout_can_be_set_and_falls_back(monkeypatch):+ monkeypatch.setenv("COWORKER_WS_SEND_TIMEOUT", "2.5")+ assert ws_guard._send_timeout() == 2.5+ for bad in ("0", "-1", "soon"):+ monkeypatch.setenv("COWORKER_WS_SEND_TIMEOUT", bad)+ assert ws_guard._send_timeout() == ws_guard.SEND_TIMEOUT_SECONDS+++def test_the_app_still_serves_its_websockets_through_the_guard(tmp_path, monkeypatch):+ """Under the test client there is no transport to track; the session websocket works as+ before."""+ from coworker.server import SessionManager, create_app++ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))+ manager = SessionManager(workspace=tmp_path)+ client = TestClient(create_app(manager))+ with client.websocket_connect("/ws/events"):+ pass+ with client.websocket_connect("/ws/session/g1") as ws:+ assert ws.receive_json()["type"] in ("ready", "export_notice")+ if hasattr(manager, "observability"):+ manager.observability.close()