Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
fix(websocket): isolate slow clients from outbound fanout (#5655)
chengyongrue2df7df9f4a7
diff --git a/nanobot/channels/websocket/runtime.py b/nanobot/channels/websocket/runtime.pyindex 6cca22162..4280a5865 100644--- a/nanobot/channels/websocket/runtime.py+++ b/nanobot/channels/websocket/runtime.py@@ -10,9 +10,11 @@ import socketimport sslimport uuidfrom contextlib import suppress+from dataclasses import dataclassfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Self, TypeGuard, castfrom urllib.parse import urlsplit, urlunsplit+from weakref import WeakSetfrom pydantic import Field, PrivateAttr, field_validator, model_validatorfrom websockets.asyncio.server import Server, ServerConnection, serve, unix_serve@@ -61,6 +63,13 @@ _LISTENER_CHECK_INTERVAL_S = 0.5_LISTENER_STABLE_AFTER_S = 30.0_LISTENER_RESTART_BACKOFF_S = (1.0, 2.0, 4.0, 8.0, 16.0, 30.0)+# Outbound delivery is isolated per connection. A bounded queue keeps a slow+# or suspended terminal from retaining an unbounded stream in server memory.+_OUTBOUND_QUEUE_MAX_FRAMES = 256+_OUTBOUND_QUEUE_MAX_BYTES = 8 * 1024 * 1024+_OUTBOUND_SEND_TIMEOUT_S = 10.0+_OUTBOUND_CLOSE_TIMEOUT_S = 1.0+# A bind conflict or invalid address needs operator action and must not be# retried forever. These errors can be caused by a transient local network# interruption and are safe to retry at the channel boundary.@@ -332,6 +341,21 @@ class _ListenerUnavailableError(OSError):"""Raised when a previously bound listener loses its serving socket."""+@dataclass(slots=True)+class _OutboundFrame:+ raw: str+ utf8_bytes: int+ label: str+++@dataclass(slots=True)+class _ConnectionOutbound:+ queue: asyncio.Queue[_OutboundFrame]+ buffered_bytes: int = 0+ writer: asyncio.Task[None] | None = None+ closing: bool = False++class WebSocketChannel(BaseChannel):"""Run a local WebSocket server; forward text/JSON messages to the message bus."""@@ -360,6 +384,9 @@ class WebSocketChannel(BaseChannel):self._stop_event: asyncio.Event | None = Noneself._server_task: asyncio.Task[None] | None = Noneself._server: Server | None = None+ self._connection_outbound: dict[ServerConnection, _ConnectionOutbound] = {}+ self._outbound_retire_tasks: set[asyncio.Task[None]] = set()+ self._retired_connections: WeakSet[ServerConnection] = WeakSet()self.gateway = gatewayself._media = gateway.media@@ -441,9 +468,20 @@ class WebSocketChannel(BaseChannel):def _attach(self, connection: ServerConnection, chat_id: str) -> None:"""Idempotently subscribe *connection* to *chat_id*."""+ if not self._register_connection_outbound(connection):+ returnself._subs.setdefault(chat_id, set()).add(connection)self._conn_chats.setdefault(connection, set()).add(chat_id)+ def _register_connection_outbound(self, connection: ServerConnection) -> bool:+ if connection in self._retired_connections:+ return False+ self._connection_outbound.setdefault(+ connection,+ _ConnectionOutbound(asyncio.Queue(maxsize=_OUTBOUND_QUEUE_MAX_FRAMES)),+ )+ return True+def _detach(self, connection: ServerConnection, chat_id: str) -> None:chats = self._conn_chats.get(connection)if chats is not None:@@ -466,7 +504,20 @@ class WebSocketChannel(BaseChannel):async def _cleanup_connection(self, connection: ServerConnection) -> None:"""Remove *connection* from every subscription set; safe to call multiple times."""- await self._commands.cleanup_connection(connection)+ self._retired_connections.add(connection)+ state = self._connection_outbound.get(connection)+ if state is not None:+ state.closing = True+ await self._stop_connection_writer(state)+ try:+ await self._commands.cleanup_connection(connection)+ finally:+ for chat_id in tuple(self._conn_chats.get(connection, ())):+ self._detach(connection, chat_id)+ self._conn_default.pop(connection, None)+ self.gateway.endpoint.discard_connection(connection)+ if self._connection_outbound.get(connection) is state:+ self._connection_outbound.pop(connection, None)async def _hydrate_after_subscribe(self, chat_id: str) -> None:"""Replay persisted or actively running per-chat state after subscribe."""@@ -482,12 +533,7 @@ class WebSocketChannel(BaseChannel):payload: dict[str, Any] = {"event": event}payload.update(fields)raw = json.dumps(payload, ensure_ascii=False)- try:- await connection.send(raw)- except ConnectionClosed:- await self._cleanup_connection(connection)- except Exception as e:- self.logger.warning("failed to send {} event: {}", event, e)+ await self._safe_send_to(connection, raw, label=f" {event} ")@classmethoddef default_config(cls) -> dict[str, Any]:@@ -748,6 +794,7 @@ class WebSocketChannel(BaseChannel):self._server_task = Noneasync def _connection_loop(self, connection: ServerConnection) -> None:+ self._retired_connections.discard(connection)request = connection.requestpath_part = request.path if request else "/"_, query = _parse_request_path(path_part)@@ -817,6 +864,16 @@ class WebSocketChannel(BaseChannel):envelope: dict[str, Any],) -> None:"""Delegate one typed envelope to the WebUI application router."""+ await self._dispatch_active_envelope(connection, client_id, envelope)++ async def _dispatch_active_envelope(+ self,+ connection: ServerConnection,+ client_id: str,+ envelope: dict[str, Any],+ ) -> None:+ if not self._register_connection_outbound(connection):+ returnawait self._commands.dispatch(connection, client_id, envelope)def _prune_webui_request_operations(self) -> None:@@ -827,11 +884,18 @@ class WebSocketChannel(BaseChannel):async def stop(self) -> None:server_task = self._server_task- if not self._running and server_task is None:+ if (+ not self._running+ and server_task is None+ and not self._connection_outbound+ and not self._outbound_retire_tasks+ ):returnself._running = Falseif self._stop_event:self._stop_event.set()+ for connection in tuple(self._connection_outbound):+ await self._cleanup_connection(connection)if server_task:try:await server_task@@ -844,11 +908,146 @@ class WebSocketChannel(BaseChannel):self.logger.warning("server task error during shutdown: {}", e)if self._server_task is server_task:self._server_task = None+ retire_tasks = tuple(self._outbound_retire_tasks)+ if retire_tasks:+ await asyncio.gather(*retire_tasks, return_exceptions=True)await self._commands.close()self._subs.clear()self._conn_chats.clear()self._conn_default.clear()+ @staticmethod+ async def _stop_connection_writer(state: _ConnectionOutbound) -> None:+ task = state.writer+ current = asyncio.current_task()+ if task is not None and task is not current and not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ while True:+ try:+ state.queue.get_nowait()+ except asyncio.QueueEmpty:+ break+ state.buffered_bytes = 0++ def _start_connection_writer(+ self,+ connection: ServerConnection,+ state: _ConnectionOutbound,+ ) -> None:+ if state.closing or (state.writer is not None and not state.writer.done()):+ return+ state.writer = asyncio.create_task(+ self._drain_connection_outbound(connection, state),+ name=f"websocket-outbound-{id(connection):x}",+ )++ async def _drain_connection_outbound(+ self,+ connection: ServerConnection,+ state: _ConnectionOutbound,+ ) -> None:+ current = asyncio.current_task()+ try:+ while not state.closing:+ try:+ frame = state.queue.get_nowait()+ except asyncio.QueueEmpty:+ return+ try:+ async with asyncio.timeout(_OUTBOUND_SEND_TIMEOUT_S):+ await connection.send(frame.raw)+ except asyncio.CancelledError:+ raise+ except TimeoutError:+ self.logger.warning("connection send timed out{}", frame.label)+ self._schedule_connection_retirement(+ connection,+ state,+ close_connection=True,+ close_code=1013,+ close_reason="outbound send timeout",+ )+ return+ except ConnectionClosed:+ self.logger.warning("connection gone{}", frame.label)+ self._schedule_connection_retirement(+ connection,+ state,+ close_connection=False,+ )+ return+ except Exception:+ self.logger.exception("send failed{}", frame.label)+ self._schedule_connection_retirement(+ connection,+ state,+ close_connection=True,+ close_code=1011,+ close_reason="outbound send failed",+ )+ return+ finally:+ state.buffered_bytes = max(0, state.buffered_bytes - frame.utf8_bytes)+ finally:+ if state.writer is current:+ state.writer = None+ if not state.closing and not state.queue.empty():+ self._start_connection_writer(connection, state)++ def _schedule_connection_retirement(+ self,+ connection: ServerConnection,+ state: _ConnectionOutbound,+ *,+ close_connection: bool,+ close_code: int = 1000,+ close_reason: str = "",+ ) -> None:+ if state.closing:+ return+ state.closing = True+ task = asyncio.create_task(+ self._retire_connection(+ connection,+ close_connection=close_connection,+ close_code=close_code,+ close_reason=close_reason,+ ),+ name=f"websocket-retire-{id(connection):x}",+ )+ self._outbound_retire_tasks.add(task)+ task.add_done_callback(self._outbound_retire_tasks.discard)++ async def _retire_connection(+ self,+ connection: ServerConnection,+ *,+ close_connection: bool,+ close_code: int,+ close_reason: str,+ ) -> None:+ try:+ if close_connection:+ try:+ async with asyncio.timeout(_OUTBOUND_CLOSE_TIMEOUT_S):+ await connection.close(code=close_code, reason=close_reason)+ except TimeoutError:+ self.logger.warning("timed out closing slow WebSocket connection")+ with suppress(Exception):+ connection.transport.abort()+ except ConnectionClosed:+ pass+ except Exception:+ self.logger.exception("failed to close WebSocket connection")+ with suppress(Exception):+ connection.transport.abort()+ finally:+ try:+ await self._cleanup_connection(connection)+ except Exception:+ self.logger.exception("failed to clean up WebSocket connection")+async def _safe_send_to(self,connection: ServerConnection,@@ -856,15 +1055,43 @@ class WebSocketChannel(BaseChannel):*,label: str = "",) -> None:- """Send a raw frame to one connection, cleaning up on ConnectionClosed."""+ """Queue one frame without letting this connection block other clients."""+ state = self._connection_outbound.get(connection)+ if state is None or state.closing:+ return+ utf8_bytes = len(raw.encode("utf-8"))+ if state.queue.full() or state.buffered_bytes + utf8_bytes > _OUTBOUND_QUEUE_MAX_BYTES:+ self.logger.warning(+ "disconnecting slow WebSocket connection: outbound queue full "+ "({} frames, {} bytes)",+ state.queue.qsize(),+ state.buffered_bytes,+ )+ self._schedule_connection_retirement(+ connection,+ state,+ close_connection=True,+ close_code=1013,+ close_reason="outbound queue full",+ )+ await asyncio.sleep(0)+ returntry:- await connection.send(raw)- except ConnectionClosed:- await self._cleanup_connection(connection)- self.logger.warning("connection gone{}", label)- except Exception:- self.logger.exception("send failed{}", label)- raise+ state.queue.put_nowait(_OutboundFrame(raw, utf8_bytes, label))+ except asyncio.QueueFull:+ self._schedule_connection_retirement(+ connection,+ state,+ close_connection=True,+ close_code=1013,+ close_reason="outbound queue full",+ )+ await asyncio.sleep(0)+ return+ state.buffered_bytes += utf8_bytes+ self._start_connection_writer(connection, state)+ # Give an idle writer a chance to start without waiting for physical I/O.+ await asyncio.sleep(0)def _persist_turn_transcript_event(self,diff --git a/nanobot/channels/websocket/tests/test_websocket_channel.py b/nanobot/channels/websocket/tests/test_websocket_channel.pyindex b325f0a04..b7250996c 100644--- a/nanobot/channels/websocket/tests/test_websocket_channel.py+++ b/nanobot/channels/websocket/tests/test_websocket_channel.py@@ -143,6 +143,14 @@ async def _connect_when_ready(url: str) -> Any:await asyncio.sleep(0.02)+async def _wait_for_connection_cleanup(+ channel: WebSocketChannel,+ connection: Any,+) -> None:+ while connection in channel._conn_chats:+ await asyncio.sleep(0)++async def _webui_mutate(client: Any,action: str,@@ -1392,6 +1400,7 @@ async def test_webui_sidebar_state_update_broadcasts_workbench_to_other_devices(source.request = SimpleNamespace(headers=Headers())other_device = AsyncMock()channel._webui_connections.update({source, other_device})+ channel._register_connection_outbound(other_device)request_id = "sidebar-workbench-state"await channel._dispatch_envelope(@@ -2243,6 +2252,7 @@ async def test_send_removes_connection_on_connection_closed() -> None:msg = OutboundMessage(channel="websocket", chat_id="chat-1", content="hello")await channel.send(msg)+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)assert "chat-1" not in channel._subsassert mock_ws not in channel._conn_chats@@ -2384,6 +2394,7 @@ async def test_send_delta_removes_connection_on_connection_closed() -> None:channel._attach(mock_ws, "chat-1")await channel.send_delta("chat-1", "chunk", stream_id="s1")+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)assert "chat-1" not in channel._subsassert mock_ws not in channel._conn_chats@@ -2872,7 +2883,7 @@ async def test_system_command_turn_end_only_refreshes_session_metadata() -> None("owner-new", "owner-old", False),],)-async def test_turn_end_persists_and_conditionally_clears_when_fanout_fails(+async def test_turn_end_persists_and_conditionally_clears_when_delivery_fails(active_owner: str,event_owner: str,expected_cleared: bool,@@ -2891,14 +2902,14 @@ async def test_turn_end_persists_and_conditionally_clears_when_fanout_fails(wth._WEBSOCKET_TURN_OWNERS[chat_id] = active_ownertry:- with pytest.raises(RuntimeError, match="fanout failed"):- await channel.send(OutboundMessage(- channel="websocket",- chat_id=chat_id,- content="",- metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: event_owner},- event=TurnEndEvent(),- ))+ await channel.send(OutboundMessage(+ channel="websocket",+ chat_id=chat_id,+ content="",+ metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: event_owner},+ event=TurnEndEvent(),+ ))+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)assert read_transcript_lines(f"websocket:{chat_id}")[-1]["event"] == "turn_end"assert (wth.websocket_turn_wall_started_at(chat_id) is None) is expected_cleared@@ -3264,7 +3275,7 @@ async def test_non_webui_transcript_failure_does_not_block_idle_cleanup(@pytest.mark.asyncio-async def test_idle_clears_matching_owner_when_fanout_fails() -> None:+async def test_idle_clears_matching_owner_when_delivery_fails() -> None:bus = MagicMock()channel = WebSocketChannel({"enabled": True, "allowFrom": ["*"]},@@ -3279,14 +3290,14 @@ async def test_idle_clears_matching_owner_when_fanout_fails() -> None:wth._WEBSOCKET_TURN_WALL_STARTED_AT[chat_id] = 1234.5wth._WEBSOCKET_TURN_OWNERS[chat_id] = owner- with pytest.raises(RuntimeError, match="fanout failed"):- await channel.send(OutboundMessage(- channel="websocket",- chat_id=chat_id,- content="",- metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: owner},- event=GoalStatusEvent(status="idle"),- ))+ await channel.send(OutboundMessage(+ channel="websocket",+ chat_id=chat_id,+ content="",+ metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: owner},+ event=GoalStatusEvent(status="idle"),+ ))+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)assert wth.websocket_turn_wall_started_at(chat_id) is Noneassert chat_id not in wth._WEBSOCKET_TURN_OWNERS@@ -3620,7 +3631,7 @@ async def test_send_session_updated_includes_scope_when_present() -> None:@pytest.mark.asyncio-async def test_send_non_connection_closed_exception_is_raised() -> None:+async def test_send_non_connection_closed_exception_is_isolated() -> None:bus = MagicMock()channel = WebSocketChannel({"enabled": True, "allowFrom": ["*"]}, bus, gateway=_basic_handler(bus))mock_ws = AsyncMock()@@ -3628,8 +3639,11 @@ async def test_send_non_connection_closed_exception_is_raised() -> None:channel._attach(mock_ws, "chat-1")msg = OutboundMessage(channel="websocket", chat_id="chat-1", content="hello")- with pytest.raises(RuntimeError, match="unexpected"):- await channel.send(msg)+ await channel.send(msg)+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)++ mock_ws.close.assert_awaited_once_with(code=1011, reason="outbound send failed")+ assert "chat-1" not in channel._subs@pytest.mark.asynciodiff --git a/nanobot/channels/websocket/tests/test_websocket_outbound_delivery.py b/nanobot/channels/websocket/tests/test_websocket_outbound_delivery.pynew file mode 100644index 000000000..6670fdca6--- /dev/null+++ b/nanobot/channels/websocket/tests/test_websocket_outbound_delivery.py@@ -0,0 +1,406 @@+"""Connection-local outbound delivery guarantees for the WebSocket channel."""++from __future__ import annotations++import asyncio+import json+from pathlib import Path+from typing import Any, cast+from unittest.mock import AsyncMock, MagicMock++import pytest++from nanobot.bus.events import OutboundMessage+from nanobot.bus.queue import MessageBus+from nanobot.channels.websocket import runtime+from nanobot.channels.websocket.runtime import WebSocketChannel, WebSocketConfig+from nanobot.webui.gateway_services import build_gateway_services+++class _RecordingConnection:+ def __init__(+ self,+ *,+ block_first_send: bool = False,+ block_close: bool = False,+ ) -> None:+ self.block_first_send = block_first_send+ self.block_close = block_close+ self.send_started = asyncio.Event()+ self.release_send = asyncio.Event()+ self.send_cancelled = asyncio.Event()+ self.closed = asyncio.Event()+ self.release_close = asyncio.Event()+ self.sent: asyncio.Queue[str] = asyncio.Queue()+ self.close_calls: list[tuple[int, str]] = []+ self.transport = MagicMock()+ self.send_calls = 0+ self.active_sends = 0+ self.max_active_sends = 0++ async def send(self, raw: str) -> None:+ self.send_calls += 1+ call_number = self.send_calls+ self.active_sends += 1+ self.max_active_sends = max(self.max_active_sends, self.active_sends)+ self.send_started.set()+ try:+ if self.block_first_send and call_number == 1:+ await self.release_send.wait()+ await self.sent.put(raw)+ except asyncio.CancelledError:+ self.send_cancelled.set()+ raise+ finally:+ self.active_sends -= 1++ async def close(self, code: int = 1000, reason: str = "") -> None:+ self.close_calls.append((code, reason))+ self.closed.set()+ if self.block_close:+ await self.release_close.wait()+++def _channel() -> WebSocketChannel:+ bus = MagicMock(spec=MessageBus)+ config = WebSocketConfig(+ enabled=True,+ allow_from=["*"],+ websocket_requires_token=False,+ )+ gateway = build_gateway_services(+ config=config,+ bus=bus,+ session_manager=None,+ static_dist_path=None,+ workspace_path=Path.cwd(),+ default_restrict_to_workspace=False,+ runtime_model_name=None,+ runtime_surface="tui",+ runtime_capabilities_overrides=None,+ )+ return WebSocketChannel(config, bus, gateway=gateway)+++def _message(chat_id: str, text: str) -> OutboundMessage:+ return OutboundMessage(channel="websocket", chat_id=chat_id, content=text)+++async def _next_text(connection: _RecordingConnection) -> str:+ raw = await asyncio.wait_for(connection.sent.get(), timeout=1)+ return str(json.loads(raw)["text"])+++async def _wait_for_connection_cleanup(+ channel: WebSocketChannel,+ connection: _RecordingConnection,+) -> None:+ while cast(Any, connection) in channel._connection_outbound:+ await asyncio.sleep(0)+++async def _wait_for_retire_tasks(channel: WebSocketChannel) -> None:+ while channel._outbound_retire_tasks:+ await asyncio.sleep(0)+++async def test_slow_tui_does_not_block_a_different_topic() -> None:+ channel = _channel()+ slow = _RecordingConnection(block_first_send=True)+ healthy = _RecordingConnection()+ channel._attach(cast(Any, slow), "chat-slow")+ channel._attach(cast(Any, healthy), "chat-healthy")++ try:+ await asyncio.wait_for(channel.send(_message("chat-slow", "slow")), timeout=1)+ await asyncio.wait_for(slow.send_started.wait(), timeout=1)++ await asyncio.wait_for(channel.send(_message("chat-healthy", "healthy")), timeout=1)+ assert await _next_text(healthy) == "healthy"+ assert slow.sent.empty()+ finally:+ slow.release_send.set()+ await channel._cleanup_connection(cast(Any, slow))+ await channel._cleanup_connection(cast(Any, healthy))+++async def test_slow_tui_does_not_block_same_topic_fanout() -> None:+ channel = _channel()+ slow = _RecordingConnection(block_first_send=True)+ healthy = _RecordingConnection()+ channel._attach(cast(Any, slow), "chat-shared")+ channel._attach(cast(Any, healthy), "chat-shared")++ try:+ await asyncio.wait_for(channel.send(_message("chat-shared", "hello")), timeout=1)+ await asyncio.wait_for(slow.send_started.wait(), timeout=1)+ assert await _next_text(healthy) == "hello"+ assert slow.sent.empty()+ finally:+ slow.release_send.set()+ await channel._cleanup_connection(cast(Any, slow))+ await channel._cleanup_connection(cast(Any, healthy))+++async def test_connection_writer_preserves_order_without_concurrent_sends() -> None:+ channel = _channel()+ connection = _RecordingConnection(block_first_send=True)+ channel._attach(cast(Any, connection), "chat-order")++ try:+ await channel.send(_message("chat-order", "one"))+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)+ await channel.send(_message("chat-order", "two"))+ await channel.send(_message("chat-order", "three"))++ connection.release_send.set()+ assert [await _next_text(connection) for _ in range(3)] == ["one", "two", "three"]+ assert connection.max_active_sends == 1+ finally:+ connection.release_send.set()+ await channel._cleanup_connection(cast(Any, connection))+++async def test_full_outbound_queue_disconnects_only_the_slow_tui(monkeypatch) -> None:+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_FRAMES", 2)+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 1024 * 1024)+ channel = _channel()+ slow = _RecordingConnection(block_first_send=True)+ healthy = _RecordingConnection()+ channel._attach(cast(Any, slow), "chat-shared")+ channel._attach(cast(Any, healthy), "chat-shared")++ try:+ for index in range(4):+ await channel.send(_message("chat-shared", str(index)))+ assert await _next_text(healthy) == str(index)+ if index == 0:+ await asyncio.wait_for(slow.send_started.wait(), timeout=1)++ await asyncio.wait_for(slow.closed.wait(), timeout=1)+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, slow), timeout=1)+ assert slow.close_calls == [(1013, "outbound queue full")]+ assert slow not in channel._conn_chats+ assert slow not in channel._subs["chat-shared"]+ assert healthy in channel._subs["chat-shared"]+ finally:+ slow.release_send.set()+ await channel._cleanup_connection(cast(Any, slow))+ await channel._cleanup_connection(cast(Any, healthy))+++async def test_outbound_byte_budget_disconnects_oversized_connection(monkeypatch) -> None:+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)+ channel = _channel()+ connection = _RecordingConnection()+ channel._attach(cast(Any, connection), "chat-large")++ try:+ await channel.send(_message("chat-large", "x" * 256))+ await asyncio.wait_for(connection.closed.wait(), timeout=1)+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, connection), timeout=1)+ assert connection.send_calls == 0+ assert connection.close_calls == [(1013, "outbound queue full")]+ finally:+ await channel._cleanup_connection(cast(Any, connection))+++async def test_cleanup_cancels_a_blocked_writer_and_discards_pending_frames() -> None:+ channel = _channel()+ connection = _RecordingConnection(block_first_send=True)+ channel._attach(cast(Any, connection), "chat-cleanup")+ channel._conn_default[cast(Any, connection)] = "chat-cleanup"++ await channel.send(_message("chat-cleanup", "one"))+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)+ await channel.send(_message("chat-cleanup", "two"))++ await asyncio.wait_for(channel._cleanup_connection(cast(Any, connection)), timeout=1)+ await channel._cleanup_connection(cast(Any, connection))++ assert connection.send_cancelled.is_set()+ assert connection not in channel._conn_chats+ assert connection not in channel._conn_default+ assert connection not in channel._connection_outbound+ assert channel._subs == {}+++async def test_send_timeout_retires_only_the_slow_connection(monkeypatch) -> None:+ monkeypatch.setattr(runtime, "_OUTBOUND_SEND_TIMEOUT_S", 0.01)+ channel = _channel()+ slow = _RecordingConnection(block_first_send=True)+ healthy = _RecordingConnection()+ channel._attach(cast(Any, slow), "chat-shared")+ channel._attach(cast(Any, healthy), "chat-shared")++ try:+ await channel.send(_message("chat-shared", "hello"))++ assert await _next_text(healthy) == "hello"+ await asyncio.wait_for(slow.closed.wait(), timeout=1)+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, slow), timeout=1)+ assert slow.close_calls == [(1013, "outbound send timeout")]+ assert slow not in channel._conn_chats+ assert healthy in channel._subs["chat-shared"]+ finally:+ slow.release_send.set()+ await channel._cleanup_connection(cast(Any, slow))+ await channel._cleanup_connection(cast(Any, healthy))+++async def test_stop_cancels_connection_writers() -> None:+ channel = _channel()+ connection = _RecordingConnection(block_first_send=True)+ channel._attach(cast(Any, connection), "chat-stop")++ await channel.send(_message("chat-stop", "one"))+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)+ await asyncio.wait_for(channel.stop(), timeout=1)++ assert connection.send_cancelled.is_set()+ assert channel._connection_outbound == {}+ assert channel._outbound_retire_tasks == set()+ assert channel._subs == {}+++async def test_stop_cancels_writers_before_waiting_for_the_server_task() -> None:+ channel = _channel()+ connection = _RecordingConnection(block_first_send=True)+ channel._attach(cast(Any, connection), "chat-stop-order")+ channel._running = True+ channel._stop_event = asyncio.Event()++ await channel.send(_message("chat-stop-order", "one"))+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)++ async def _server_shutdown() -> None:+ assert channel._stop_event is not None+ await channel._stop_event.wait()+ await connection.send_cancelled.wait()++ channel._server_task = asyncio.create_task(_server_shutdown())++ await asyncio.wait_for(channel.stop(), timeout=1)++ assert connection.send_cancelled.is_set()+ assert channel._server_task is None+++async def test_send_to_cleaned_connection_does_not_recreate_outbound_state() -> None:+ channel = _channel()+ connection = _RecordingConnection()+ channel._attach(cast(Any, connection), "chat-stale")+ await channel._cleanup_connection(cast(Any, connection))++ await channel._safe_send_to(cast(Any, connection), "stale")++ assert connection.send_calls == 0+ assert connection not in channel._connection_outbound+++async def test_attach_does_not_revive_a_cleaned_connection() -> None:+ channel = _channel()+ connection = _RecordingConnection()+ channel._attach(cast(Any, connection), "chat-before-cleanup")+ await channel._cleanup_connection(cast(Any, connection))++ channel._attach(cast(Any, connection), "chat-after-cleanup")++ assert connection not in channel._connection_outbound+ assert connection not in channel._conn_chats+ assert "chat-after-cleanup" not in channel._subs+++async def test_buffered_envelope_does_not_revive_a_cleaned_connection(monkeypatch) -> None:+ channel = _channel()+ connection = _RecordingConnection()+ channel._attach(cast(Any, connection), "chat-before-cleanup")+ await channel._cleanup_connection(cast(Any, connection))+ dispatch = AsyncMock()+ monkeypatch.setattr(channel._commands, "dispatch", dispatch)++ await channel._dispatch_envelope(cast(Any, connection), "client", {"type": "attach"})++ dispatch.assert_not_awaited()+ assert connection not in channel._connection_outbound+++async def test_control_broadcast_cannot_precede_ready() -> None:+ channel = _channel()+ connection = _RecordingConnection()+ channel._webui_connections.add(cast(Any, connection))++ await channel._send_event(cast(Any, connection), "session_updated", chat_id="chat")++ assert connection.send_calls == 0+ assert connection not in channel._connection_outbound+++async def test_close_timeout_aborts_transport_and_cleans_connection(monkeypatch) -> None:+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)+ monkeypatch.setattr(runtime, "_OUTBOUND_CLOSE_TIMEOUT_S", 0.01)+ channel = _channel()+ connection = _RecordingConnection(block_close=True)+ channel._attach(cast(Any, connection), "chat-close-timeout")++ await channel.send(_message("chat-close-timeout", "x" * 256))+ await asyncio.wait_for(connection.closed.wait(), timeout=1)+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, connection), timeout=1)++ connection.transport.abort.assert_called_once_with()+ assert connection not in channel._conn_chats+++async def test_retirement_contains_cleanup_failure(monkeypatch) -> None:+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)+ channel = _channel()+ connection = _RecordingConnection()+ channel._attach(cast(Any, connection), "chat-cleanup-failure")+ channel._webui_connections.add(cast(Any, connection))+ monkeypatch.setattr(+ channel._commands,+ "cleanup_connection",+ AsyncMock(side_effect=RuntimeError("cleanup failed")),+ )++ await channel.send(_message("chat-cleanup-failure", "x" * 256))+ await asyncio.wait_for(_wait_for_retire_tasks(channel), timeout=1)++ assert connection.close_calls == [(1013, "outbound queue full")]+ assert connection not in channel._connection_outbound+ assert connection not in channel._conn_chats+ assert connection not in channel._webui_connections+ assert channel._subs == {}+++async def test_normal_cleanup_can_overlap_connection_retirement(monkeypatch) -> None:+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)+ channel = _channel()+ connection = _RecordingConnection(block_close=True)+ channel._attach(cast(Any, connection), "chat-retire-race")++ await channel.send(_message("chat-retire-race", "x" * 256))+ await asyncio.wait_for(connection.closed.wait(), timeout=1)+ await asyncio.wait_for(channel._cleanup_connection(cast(Any, connection)), timeout=1)+ connection.release_close.set()+ await asyncio.wait_for(_wait_for_retire_tasks(channel), timeout=1)++ assert connection not in channel._connection_outbound+ assert connection not in channel._conn_chats+ assert channel._subs == {}