Skip to content
Python

fix(server): a websocket client that stopped reading no longer hangs shutdown

Devika Vermab50283ed36be
diff --git a/coworker/server/app.py b/coworker/server/app.py
index e2bb9a99..67fa6e21 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -298,6 +298,11 @@ def create_app(manager: SessionManager) -> FastAPI:
allow_methods=["*"],
allow_headers=["*"],
)
+ # A GUI websocket whose client stopped reading is dropped, and every open websocket can be
+ # aborted at shutdown, so neither holds the server up (ws_guard.py, T3-3).
+ from .ws_guard import WebsocketGuard
+
+ app.add_middleware(WebsocketGuard)
app.state.manager = manager
@app.get("/v1/capabilities")
diff --git a/coworker/server/run.py b/coworker/server/run.py
index 16ad182a..4725bdc6 100644
--- a/coworker/server/run.py
+++ b/coworker/server/run.py
@@ -222,10 +222,19 @@ def main(argv=None) -> None:
_exit_when_orphaned()
_warn_if_state_shared()
+ from .ws_guard import GuardedServer
+
app = build_app(args.cwd, args.model, args.mode)
- uvicorn.run(
+ # GuardedServer: uvicorn's server, but it aborts the websockets still open before it
+ # waits for connections at shutdown, so Ctrl+C never hangs on a tab that stopped
+ # reading (ws_guard.py, T3-3).
+ config = uvicorn.Config(
app, host=args.host, port=args.port, ws_max_size=_WS_MAX_FRAME_BYTES
)
+ try:
+ GuardedServer(config).run()
+ except KeyboardInterrupt:
+ pass # as uvicorn.run: the signal is raised again once the server has stopped
finally:
if generated_token_path is not None:
generated_token_path.unlink(missing_ok=True)
diff --git a/coworker/server/ws_guard.py b/coworker/server/ws_guard.py
new file mode 100644
index 00000000..319e9909
--- /dev/null
+++ b/coworker/server/ws_guard.py
@@ -0,0 +1,126 @@
+"""Websockets that cannot hold the server up (T3-3).
+
+The server hung at "Shutting down" after Ctrl+C when a browser tab had stopped reading its
+websocket (a frozen or discarded tab, or a laptop that slept) while the server kept pushing live
+events. The unread data filled the connection's send buffer, so its close frame could never go
+out and the connection never closed. uvicorn waits for every connection to close before it stops,
+and since Python 3.12 `Server.wait_closed()` also waits for every transport, so even a second
+Ctrl+C could not get past it.
+
+Two guards, both by aborting the connection's transport (an abort drops what is still buffered;
+a close waits to send it):
+
+- `WebsocketGuard` (ASGI middleware): on the GUI's own websockets, a send that cannot finish
+ within `send_timeout` seconds means the client stopped reading, and the connection is
+ aborted. The machine links (`/ws/machines/…`) are left alone: they carry large frames over
+ slow networks.
+- `GuardedServer` (uvicorn): at shutdown, every websocket still open is aborted before uvicorn
+ waits for its connections.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import weakref
+from typing import Any, Awaitable, Callable, Optional
+
+import uvicorn
+from starlette.websockets import WebSocketDisconnect
+
+log = logging.getLogger(__name__)
+
+SEND_TIMEOUT_SECONDS = 15.0
+# The GUI's own websockets: the session stream and the app-wide event stream.
+GUARDED_PATHS = ("/ws/session/", "/ws/events")
+
+_open: "weakref.WeakSet[Any]" = weakref.WeakSet()
+
+
+def _send_timeout() -> float:
+ try:
+ value = float(os.environ.get("COWORKER_WS_SEND_TIMEOUT", "") or SEND_TIMEOUT_SECONDS)
+ except ValueError:
+ return SEND_TIMEOUT_SECONDS
+ return value if value > 0 else SEND_TIMEOUT_SECONDS
+
+
+def _transport_of(send: Callable[..., Any]) -> Optional[asyncio.BaseTransport]:
+ """The connection's transport, from uvicorn's websocket protocol (its `send` is a bound
+ method of the protocol, for both the websockets and the wsproto implementation). None under
+ a test client or another server."""
+ owner = getattr(send, "__self__", None)
+ transport = getattr(owner, "transport", None)
+ return transport if isinstance(transport, asyncio.BaseTransport) else None
+
+
+def _abort(transport: Optional[asyncio.BaseTransport]) -> None:
+ if transport is None:
+ return
+ try:
+ transport.abort() # type: ignore[attr-defined]
+ except Exception: # noqa: BLE001 - already gone
+ pass
+
+
+def open_count() -> int:
+ """How many websocket transports are open now (for tests and logs)."""
+ return sum(1 for t in list(_open) if not t.is_closing())
+
+
+def abort_open_websockets() -> int:
+ """Abort every websocket still open. Returns how many were aborted."""
+ n = 0
+ for transport in list(_open):
+ if not transport.is_closing():
+ n += 1
+ _abort(transport)
+ _open.clear()
+ return n
+
+
+class WebsocketGuard:
+ """ASGI middleware: track every websocket's transport, and abort a GUI websocket whose
+ client stopped reading. Plain HTTP passes straight through."""
+
+ def __init__(self, app: Callable[..., Awaitable[None]], *, send_timeout: Optional[float] = None) -> None:
+ self.app = app
+ self.send_timeout = send_timeout
+
+ async def __call__(self, scope: dict[str, Any], receive: Callable[..., Any], send: Callable[..., Any]) -> None:
+ if scope.get("type") != "websocket":
+ await self.app(scope, receive, send)
+ return
+ transport = _transport_of(send)
+ if transport is not None:
+ _open.add(transport)
+ path = str(scope.get("path") or "")
+ timeout = self.send_timeout if self.send_timeout is not None else _send_timeout()
+
+ async def guarded_send(message: dict[str, Any]) -> None:
+ try:
+ await asyncio.wait_for(send(message), timeout)
+ except asyncio.TimeoutError:
+ log.warning("websocket %s: the client stopped reading for %.0f s; dropping it", path, timeout)
+ _abort(transport)
+ # what a handler already expects when its client goes away
+ raise WebSocketDisconnect(code=1011, reason="client stopped reading")
+
+ try:
+ # only the GUI's own websockets get the send timeout; every one is tracked
+ await self.app(scope, receive, guarded_send if path.startswith(GUARDED_PATHS) else send)
+ finally:
+ if transport is not None:
+ _open.discard(transport)
+
+
+class GuardedServer(uvicorn.Server):
+ """uvicorn's server, but at shutdown the websockets still open are aborted first, so it
+ never waits on a client that stopped reading."""
+
+ async def shutdown(self, sockets: Any = None) -> None:
+ n = abort_open_websockets()
+ if n:
+ log.info("aborted %d open websocket(s) before shutting down", n)
+ await super().shutdown(sockets=sockets)
diff --git a/tests/test_server.py b/tests/test_server.py
index 10862b94..7a18fd12 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -483,9 +483,6 @@ def test_ws_rate_limits_inbound_frames(tmp_path):
def test_server_sets_explicit_websocket_frame_limit(tmp_path, monkeypatch):
- import sys
- from types import SimpleNamespace
-
from coworker.server import run as server_run
seen = {}
@@ -494,15 +491,21 @@ def test_server_sets_explicit_websocket_frame_limit(tmp_path, monkeypatch):
monkeypatch.setattr(server_run, "_ensure_ca_bundle", lambda: None)
monkeypatch.setattr(server_run, "_exit_when_orphaned", lambda: None)
monkeypatch.setattr(server_run, "build_app", lambda *args: fake_app)
- monkeypatch.setitem(
- sys.modules,
- "uvicorn",
- SimpleNamespace(run=lambda app, **kwargs: seen.update(app=app, **kwargs)),
- )
+ from coworker.server import ws_guard
+
+ class _Server:
+ # T3-3: the launcher runs uvicorn through GuardedServer (ws_guard.py)
+ def __init__(self, config):
+ seen.update(app=config.app, ws_max_size=config.ws_max_size, port=config.port)
+
+ def run(self):
+ seen["ran"] = True
+
+ monkeypatch.setattr(ws_guard, "GuardedServer", _Server)
server_run.main(["--cwd", str(tmp_path), "--port", "8766"])
- assert seen["app"] is fake_app
+ assert seen["app"] is fake_app and seen["ran"] and seen["port"] == 8766
assert seen["ws_max_size"] == server_run._WS_MAX_FRAME_BYTES
diff --git a/tests/test_ws_guard.py b/tests/test_ws_guard.py
new file mode 100644
index 00000000..d0a7b2f7
--- /dev/null
+++ b/tests/test_ws_guard.py
@@ -0,0 +1,140 @@
+"""T3-3: a websocket whose client stopped reading must not hold the server up. A GUI websocket
+send that cannot finish in time drops the connection, and at shutdown every open websocket is
+aborted before uvicorn waits for its connections."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from fastapi.testclient import TestClient
+from starlette.websockets import WebSocketDisconnect
+
+from coworker.server import ws_guard
+
+
+class _Transport(asyncio.BaseTransport):
+ def __init__(self) -> None:
+ super().__init__()
+ self.aborted = False
+
+ def abort(self) -> None:
+ self.aborted = True
+
+ def is_closing(self) -> bool:
+ return self.aborted
+
+
+class _Protocol:
+ """Stands in for uvicorn's websocket protocol: its `send` is a bound method, and it holds
+ the connection's transport."""
+
+ def __init__(self, stuck: bool) -> None:
+ self.transport = _Transport()
+ self.stuck = stuck
+ self.sent: list[dict] = []
+
+ async def send(self, message: dict) -> None:
+ if self.stuck and message.get("type") == "websocket.send":
+ await asyncio.sleep(3600) # the client stopped reading: the buffer never drains
+ self.sent.append(message)
+
+
+async def _receive() -> dict:
+ return {"type": "websocket.connect"}
+
+
+def _app(log: list):
+ async def app(scope, receive, send):
+ assert ws_guard.open_count() == 1 # tracked while it is open
+ await send({"type": "websocket.accept"})
+ try:
+ await send({"type": "websocket.send", "text": "x"})
+ except WebSocketDisconnect as exc:
+ log.append(exc.code)
+ return
+ log.append("sent")
+
+ return app
+
+
+def test_a_gui_websocket_whose_client_stopped_reading_is_dropped():
+ log: list = []
+ proto = _Protocol(stuck=True)
+ guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)
+ asyncio.run(guard({"type": "websocket", "path": "/ws/session/s1"}, _receive, proto.send))
+ assert proto.transport.aborted and log == [1011]
+ assert ws_guard.open_count() == 0 # no longer tracked
+
+
+def test_a_reading_client_is_untouched():
+ log: list = []
+ proto = _Protocol(stuck=False)
+ guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)
+ asyncio.run(guard({"type": "websocket", "path": "/ws/events"}, _receive, proto.send))
+ assert not proto.transport.aborted and log == ["sent"]
+
+
+def test_every_websocket_is_tracked_only_while_open():
+ log: list = []
+ proto = _Protocol(stuck=False)
+ guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)
+ asyncio.run(guard({"type": "websocket", "path": "/ws/machines/m1/p"}, _receive, proto.send))
+ assert log == ["sent"] and ws_guard.open_count() == 0 and proto.transport not in ws_guard._open
+
+
+def test_machine_links_have_no_send_timeout():
+ """/ws/machines carries large frames over slow links: no timeout there, only the
+ shutdown abort."""
+
+ async def go():
+ proto = _Protocol(stuck=True)
+ guard = ws_guard.WebsocketGuard(_app([]), send_timeout=0.05)
+ task = asyncio.create_task(guard({"type": "websocket", "path": "/ws/machines/m1/p"}, _receive, proto.send))
+ await asyncio.sleep(0.3)
+ assert not task.done() and not proto.transport.aborted # still waiting, not dropped
+ assert ws_guard.abort_open_websockets() == 1 # what shutdown does
+ assert proto.transport.aborted
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ asyncio.run(go())
+
+
+def test_the_server_aborts_open_websockets_before_it_waits(monkeypatch):
+ order: list[str] = []
+ transport = _Transport()
+ ws_guard._open.add(transport)
+
+ async def base_shutdown(self, sockets=None):
+ order.append("uvicorn waits" + (" (none open)" if ws_guard.open_count() == 0 else " (still open!)"))
+
+ monkeypatch.setattr(ws_guard.uvicorn.Server, "shutdown", base_shutdown)
+ server = ws_guard.GuardedServer.__new__(ws_guard.GuardedServer)
+ asyncio.run(server.shutdown())
+ assert transport.aborted and order == ["uvicorn waits (none open)"]
+
+
+def test_the_send_timeout_can_be_set_and_falls_back(monkeypatch):
+ monkeypatch.setenv("COWORKER_WS_SEND_TIMEOUT", "2.5")
+ assert ws_guard._send_timeout() == 2.5
+ for bad in ("0", "-1", "soon"):
+ monkeypatch.setenv("COWORKER_WS_SEND_TIMEOUT", bad)
+ assert ws_guard._send_timeout() == ws_guard.SEND_TIMEOUT_SECONDS
+
+
+def test_the_app_still_serves_its_websockets_through_the_guard(tmp_path, monkeypatch):
+ """Under the test client there is no transport to track; the session websocket works as
+ before."""
+ from coworker.server import SessionManager, create_app
+
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
+ manager = SessionManager(workspace=tmp_path)
+ client = TestClient(create_app(manager))
+ with client.websocket_connect("/ws/events"):
+ pass
+ with client.websocket_connect("/ws/session/g1") as ws:
+ assert ws.receive_json()["type"] in ("ready", "export_notice")
+ if hasattr(manager, "observability"):
+ manager.observability.close()