Skip to content

Your lightweight open-source personal agent.

fix(websocket): isolate slow clients from outbound fanout (#5655)

chengyongrue2df7df9f4a7
diff --git a/nanobot/channels/websocket/runtime.py b/nanobot/channels/websocket/runtime.py
index 6cca22162..4280a5865 100644
--- a/nanobot/channels/websocket/runtime.py
+++ b/nanobot/channels/websocket/runtime.py
@@ -10,9 +10,11 @@ import socket
import ssl
import uuid
from contextlib import suppress
+from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Self, TypeGuard, cast
from urllib.parse import urlsplit, urlunsplit
+from weakref import WeakSet
from pydantic import Field, PrivateAttr, field_validator, model_validator
from websockets.asyncio.server import Server, ServerConnection, serve, unix_serve
@@ -61,6 +63,13 @@ _LISTENER_CHECK_INTERVAL_S = 0.5
_LISTENER_STABLE_AFTER_S = 30.0
_LISTENER_RESTART_BACKOFF_S = (1.0, 2.0, 4.0, 8.0, 16.0, 30.0)
+# Outbound delivery is isolated per connection. A bounded queue keeps a slow
+# or suspended terminal from retaining an unbounded stream in server memory.
+_OUTBOUND_QUEUE_MAX_FRAMES = 256
+_OUTBOUND_QUEUE_MAX_BYTES = 8 * 1024 * 1024
+_OUTBOUND_SEND_TIMEOUT_S = 10.0
+_OUTBOUND_CLOSE_TIMEOUT_S = 1.0
+
# A bind conflict or invalid address needs operator action and must not be
# retried forever. These errors can be caused by a transient local network
# interruption and are safe to retry at the channel boundary.
@@ -332,6 +341,21 @@ class _ListenerUnavailableError(OSError):
"""Raised when a previously bound listener loses its serving socket."""
+@dataclass(slots=True)
+class _OutboundFrame:
+ raw: str
+ utf8_bytes: int
+ label: str
+
+
+@dataclass(slots=True)
+class _ConnectionOutbound:
+ queue: asyncio.Queue[_OutboundFrame]
+ buffered_bytes: int = 0
+ writer: asyncio.Task[None] | None = None
+ closing: bool = False
+
+
class WebSocketChannel(BaseChannel):
"""Run a local WebSocket server; forward text/JSON messages to the message bus."""
@@ -360,6 +384,9 @@ class WebSocketChannel(BaseChannel):
self._stop_event: asyncio.Event | None = None
self._server_task: asyncio.Task[None] | None = None
self._server: Server | None = None
+ self._connection_outbound: dict[ServerConnection, _ConnectionOutbound] = {}
+ self._outbound_retire_tasks: set[asyncio.Task[None]] = set()
+ self._retired_connections: WeakSet[ServerConnection] = WeakSet()
self.gateway = gateway
self._media = gateway.media
@@ -441,9 +468,20 @@ class WebSocketChannel(BaseChannel):
def _attach(self, connection: ServerConnection, chat_id: str) -> None:
"""Idempotently subscribe *connection* to *chat_id*."""
+ if not self._register_connection_outbound(connection):
+ return
self._subs.setdefault(chat_id, set()).add(connection)
self._conn_chats.setdefault(connection, set()).add(chat_id)
+ def _register_connection_outbound(self, connection: ServerConnection) -> bool:
+ if connection in self._retired_connections:
+ return False
+ self._connection_outbound.setdefault(
+ connection,
+ _ConnectionOutbound(asyncio.Queue(maxsize=_OUTBOUND_QUEUE_MAX_FRAMES)),
+ )
+ return True
+
def _detach(self, connection: ServerConnection, chat_id: str) -> None:
chats = self._conn_chats.get(connection)
if chats is not None:
@@ -466,7 +504,20 @@ class WebSocketChannel(BaseChannel):
async def _cleanup_connection(self, connection: ServerConnection) -> None:
"""Remove *connection* from every subscription set; safe to call multiple times."""
- await self._commands.cleanup_connection(connection)
+ self._retired_connections.add(connection)
+ state = self._connection_outbound.get(connection)
+ if state is not None:
+ state.closing = True
+ await self._stop_connection_writer(state)
+ try:
+ await self._commands.cleanup_connection(connection)
+ finally:
+ for chat_id in tuple(self._conn_chats.get(connection, ())):
+ self._detach(connection, chat_id)
+ self._conn_default.pop(connection, None)
+ self.gateway.endpoint.discard_connection(connection)
+ if self._connection_outbound.get(connection) is state:
+ self._connection_outbound.pop(connection, None)
async def _hydrate_after_subscribe(self, chat_id: str) -> None:
"""Replay persisted or actively running per-chat state after subscribe."""
@@ -482,12 +533,7 @@ class WebSocketChannel(BaseChannel):
payload: dict[str, Any] = {"event": event}
payload.update(fields)
raw = json.dumps(payload, ensure_ascii=False)
- try:
- await connection.send(raw)
- except ConnectionClosed:
- await self._cleanup_connection(connection)
- except Exception as e:
- self.logger.warning("failed to send {} event: {}", event, e)
+ await self._safe_send_to(connection, raw, label=f" {event} ")
@classmethod
def default_config(cls) -> dict[str, Any]:
@@ -748,6 +794,7 @@ class WebSocketChannel(BaseChannel):
self._server_task = None
async def _connection_loop(self, connection: ServerConnection) -> None:
+ self._retired_connections.discard(connection)
request = connection.request
path_part = request.path if request else "/"
_, query = _parse_request_path(path_part)
@@ -817,6 +864,16 @@ class WebSocketChannel(BaseChannel):
envelope: dict[str, Any],
) -> None:
"""Delegate one typed envelope to the WebUI application router."""
+ await self._dispatch_active_envelope(connection, client_id, envelope)
+
+ async def _dispatch_active_envelope(
+ self,
+ connection: ServerConnection,
+ client_id: str,
+ envelope: dict[str, Any],
+ ) -> None:
+ if not self._register_connection_outbound(connection):
+ return
await self._commands.dispatch(connection, client_id, envelope)
def _prune_webui_request_operations(self) -> None:
@@ -827,11 +884,18 @@ class WebSocketChannel(BaseChannel):
async def stop(self) -> None:
server_task = self._server_task
- if not self._running and server_task is None:
+ if (
+ not self._running
+ and server_task is None
+ and not self._connection_outbound
+ and not self._outbound_retire_tasks
+ ):
return
self._running = False
if self._stop_event:
self._stop_event.set()
+ for connection in tuple(self._connection_outbound):
+ await self._cleanup_connection(connection)
if server_task:
try:
await server_task
@@ -844,11 +908,146 @@ class WebSocketChannel(BaseChannel):
self.logger.warning("server task error during shutdown: {}", e)
if self._server_task is server_task:
self._server_task = None
+ retire_tasks = tuple(self._outbound_retire_tasks)
+ if retire_tasks:
+ await asyncio.gather(*retire_tasks, return_exceptions=True)
await self._commands.close()
self._subs.clear()
self._conn_chats.clear()
self._conn_default.clear()
+ @staticmethod
+ async def _stop_connection_writer(state: _ConnectionOutbound) -> None:
+ task = state.writer
+ current = asyncio.current_task()
+ if task is not None and task is not current and not task.done():
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ while True:
+ try:
+ state.queue.get_nowait()
+ except asyncio.QueueEmpty:
+ break
+ state.buffered_bytes = 0
+
+ def _start_connection_writer(
+ self,
+ connection: ServerConnection,
+ state: _ConnectionOutbound,
+ ) -> None:
+ if state.closing or (state.writer is not None and not state.writer.done()):
+ return
+ state.writer = asyncio.create_task(
+ self._drain_connection_outbound(connection, state),
+ name=f"websocket-outbound-{id(connection):x}",
+ )
+
+ async def _drain_connection_outbound(
+ self,
+ connection: ServerConnection,
+ state: _ConnectionOutbound,
+ ) -> None:
+ current = asyncio.current_task()
+ try:
+ while not state.closing:
+ try:
+ frame = state.queue.get_nowait()
+ except asyncio.QueueEmpty:
+ return
+ try:
+ async with asyncio.timeout(_OUTBOUND_SEND_TIMEOUT_S):
+ await connection.send(frame.raw)
+ except asyncio.CancelledError:
+ raise
+ except TimeoutError:
+ self.logger.warning("connection send timed out{}", frame.label)
+ self._schedule_connection_retirement(
+ connection,
+ state,
+ close_connection=True,
+ close_code=1013,
+ close_reason="outbound send timeout",
+ )
+ return
+ except ConnectionClosed:
+ self.logger.warning("connection gone{}", frame.label)
+ self._schedule_connection_retirement(
+ connection,
+ state,
+ close_connection=False,
+ )
+ return
+ except Exception:
+ self.logger.exception("send failed{}", frame.label)
+ self._schedule_connection_retirement(
+ connection,
+ state,
+ close_connection=True,
+ close_code=1011,
+ close_reason="outbound send failed",
+ )
+ return
+ finally:
+ state.buffered_bytes = max(0, state.buffered_bytes - frame.utf8_bytes)
+ finally:
+ if state.writer is current:
+ state.writer = None
+ if not state.closing and not state.queue.empty():
+ self._start_connection_writer(connection, state)
+
+ def _schedule_connection_retirement(
+ self,
+ connection: ServerConnection,
+ state: _ConnectionOutbound,
+ *,
+ close_connection: bool,
+ close_code: int = 1000,
+ close_reason: str = "",
+ ) -> None:
+ if state.closing:
+ return
+ state.closing = True
+ task = asyncio.create_task(
+ self._retire_connection(
+ connection,
+ close_connection=close_connection,
+ close_code=close_code,
+ close_reason=close_reason,
+ ),
+ name=f"websocket-retire-{id(connection):x}",
+ )
+ self._outbound_retire_tasks.add(task)
+ task.add_done_callback(self._outbound_retire_tasks.discard)
+
+ async def _retire_connection(
+ self,
+ connection: ServerConnection,
+ *,
+ close_connection: bool,
+ close_code: int,
+ close_reason: str,
+ ) -> None:
+ try:
+ if close_connection:
+ try:
+ async with asyncio.timeout(_OUTBOUND_CLOSE_TIMEOUT_S):
+ await connection.close(code=close_code, reason=close_reason)
+ except TimeoutError:
+ self.logger.warning("timed out closing slow WebSocket connection")
+ with suppress(Exception):
+ connection.transport.abort()
+ except ConnectionClosed:
+ pass
+ except Exception:
+ self.logger.exception("failed to close WebSocket connection")
+ with suppress(Exception):
+ connection.transport.abort()
+ finally:
+ try:
+ await self._cleanup_connection(connection)
+ except Exception:
+ self.logger.exception("failed to clean up WebSocket connection")
+
async def _safe_send_to(
self,
connection: ServerConnection,
@@ -856,15 +1055,43 @@ class WebSocketChannel(BaseChannel):
*,
label: str = "",
) -> None:
- """Send a raw frame to one connection, cleaning up on ConnectionClosed."""
+ """Queue one frame without letting this connection block other clients."""
+ state = self._connection_outbound.get(connection)
+ if state is None or state.closing:
+ return
+ utf8_bytes = len(raw.encode("utf-8"))
+ if state.queue.full() or state.buffered_bytes + utf8_bytes > _OUTBOUND_QUEUE_MAX_BYTES:
+ self.logger.warning(
+ "disconnecting slow WebSocket connection: outbound queue full "
+ "({} frames, {} bytes)",
+ state.queue.qsize(),
+ state.buffered_bytes,
+ )
+ self._schedule_connection_retirement(
+ connection,
+ state,
+ close_connection=True,
+ close_code=1013,
+ close_reason="outbound queue full",
+ )
+ await asyncio.sleep(0)
+ return
try:
- await connection.send(raw)
- except ConnectionClosed:
- await self._cleanup_connection(connection)
- self.logger.warning("connection gone{}", label)
- except Exception:
- self.logger.exception("send failed{}", label)
- raise
+ state.queue.put_nowait(_OutboundFrame(raw, utf8_bytes, label))
+ except asyncio.QueueFull:
+ self._schedule_connection_retirement(
+ connection,
+ state,
+ close_connection=True,
+ close_code=1013,
+ close_reason="outbound queue full",
+ )
+ await asyncio.sleep(0)
+ return
+ state.buffered_bytes += utf8_bytes
+ self._start_connection_writer(connection, state)
+ # Give an idle writer a chance to start without waiting for physical I/O.
+ await asyncio.sleep(0)
def _persist_turn_transcript_event(
self,
diff --git a/nanobot/channels/websocket/tests/test_websocket_channel.py b/nanobot/channels/websocket/tests/test_websocket_channel.py
index b325f0a04..b7250996c 100644
--- a/nanobot/channels/websocket/tests/test_websocket_channel.py
+++ b/nanobot/channels/websocket/tests/test_websocket_channel.py
@@ -143,6 +143,14 @@ async def _connect_when_ready(url: str) -> Any:
await asyncio.sleep(0.02)
+async def _wait_for_connection_cleanup(
+ channel: WebSocketChannel,
+ connection: Any,
+) -> None:
+ while connection in channel._conn_chats:
+ await asyncio.sleep(0)
+
+
async def _webui_mutate(
client: Any,
action: str,
@@ -1392,6 +1400,7 @@ async def test_webui_sidebar_state_update_broadcasts_workbench_to_other_devices(
source.request = SimpleNamespace(headers=Headers())
other_device = AsyncMock()
channel._webui_connections.update({source, other_device})
+ channel._register_connection_outbound(other_device)
request_id = "sidebar-workbench-state"
await channel._dispatch_envelope(
@@ -2243,6 +2252,7 @@ async def test_send_removes_connection_on_connection_closed() -> None:
msg = OutboundMessage(channel="websocket", chat_id="chat-1", content="hello")
await channel.send(msg)
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)
assert "chat-1" not in channel._subs
assert mock_ws not in channel._conn_chats
@@ -2384,6 +2394,7 @@ async def test_send_delta_removes_connection_on_connection_closed() -> None:
channel._attach(mock_ws, "chat-1")
await channel.send_delta("chat-1", "chunk", stream_id="s1")
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)
assert "chat-1" not in channel._subs
assert mock_ws not in channel._conn_chats
@@ -2872,7 +2883,7 @@ async def test_system_command_turn_end_only_refreshes_session_metadata() -> None
("owner-new", "owner-old", False),
],
)
-async def test_turn_end_persists_and_conditionally_clears_when_fanout_fails(
+async def test_turn_end_persists_and_conditionally_clears_when_delivery_fails(
active_owner: str,
event_owner: str,
expected_cleared: bool,
@@ -2891,14 +2902,14 @@ async def test_turn_end_persists_and_conditionally_clears_when_fanout_fails(
wth._WEBSOCKET_TURN_OWNERS[chat_id] = active_owner
try:
- with pytest.raises(RuntimeError, match="fanout failed"):
- await channel.send(OutboundMessage(
- channel="websocket",
- chat_id=chat_id,
- content="",
- metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: event_owner},
- event=TurnEndEvent(),
- ))
+ await channel.send(OutboundMessage(
+ channel="websocket",
+ chat_id=chat_id,
+ content="",
+ metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: event_owner},
+ event=TurnEndEvent(),
+ ))
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)
assert read_transcript_lines(f"websocket:{chat_id}")[-1]["event"] == "turn_end"
assert (wth.websocket_turn_wall_started_at(chat_id) is None) is expected_cleared
@@ -3264,7 +3275,7 @@ async def test_non_webui_transcript_failure_does_not_block_idle_cleanup(
@pytest.mark.asyncio
-async def test_idle_clears_matching_owner_when_fanout_fails() -> None:
+async def test_idle_clears_matching_owner_when_delivery_fails() -> None:
bus = MagicMock()
channel = WebSocketChannel(
{"enabled": True, "allowFrom": ["*"]},
@@ -3279,14 +3290,14 @@ async def test_idle_clears_matching_owner_when_fanout_fails() -> None:
wth._WEBSOCKET_TURN_WALL_STARTED_AT[chat_id] = 1234.5
wth._WEBSOCKET_TURN_OWNERS[chat_id] = owner
- with pytest.raises(RuntimeError, match="fanout failed"):
- await channel.send(OutboundMessage(
- channel="websocket",
- chat_id=chat_id,
- content="",
- metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: owner},
- event=GoalStatusEvent(status="idle"),
- ))
+ await channel.send(OutboundMessage(
+ channel="websocket",
+ chat_id=chat_id,
+ content="",
+ metadata={WEBSOCKET_TURN_OWNER_METADATA_KEY: owner},
+ event=GoalStatusEvent(status="idle"),
+ ))
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)
assert wth.websocket_turn_wall_started_at(chat_id) is None
assert chat_id not in wth._WEBSOCKET_TURN_OWNERS
@@ -3620,7 +3631,7 @@ async def test_send_session_updated_includes_scope_when_present() -> None:
@pytest.mark.asyncio
-async def test_send_non_connection_closed_exception_is_raised() -> None:
+async def test_send_non_connection_closed_exception_is_isolated() -> None:
bus = MagicMock()
channel = WebSocketChannel({"enabled": True, "allowFrom": ["*"]}, bus, gateway=_basic_handler(bus))
mock_ws = AsyncMock()
@@ -3628,8 +3639,11 @@ async def test_send_non_connection_closed_exception_is_raised() -> None:
channel._attach(mock_ws, "chat-1")
msg = OutboundMessage(channel="websocket", chat_id="chat-1", content="hello")
- with pytest.raises(RuntimeError, match="unexpected"):
- await channel.send(msg)
+ await channel.send(msg)
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, mock_ws), timeout=1)
+
+ mock_ws.close.assert_awaited_once_with(code=1011, reason="outbound send failed")
+ assert "chat-1" not in channel._subs
@pytest.mark.asyncio
diff --git a/nanobot/channels/websocket/tests/test_websocket_outbound_delivery.py b/nanobot/channels/websocket/tests/test_websocket_outbound_delivery.py
new file mode 100644
index 000000000..6670fdca6
--- /dev/null
+++ b/nanobot/channels/websocket/tests/test_websocket_outbound_delivery.py
@@ -0,0 +1,406 @@
+"""Connection-local outbound delivery guarantees for the WebSocket channel."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any, cast
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from nanobot.bus.events import OutboundMessage
+from nanobot.bus.queue import MessageBus
+from nanobot.channels.websocket import runtime
+from nanobot.channels.websocket.runtime import WebSocketChannel, WebSocketConfig
+from nanobot.webui.gateway_services import build_gateway_services
+
+
+class _RecordingConnection:
+ def __init__(
+ self,
+ *,
+ block_first_send: bool = False,
+ block_close: bool = False,
+ ) -> None:
+ self.block_first_send = block_first_send
+ self.block_close = block_close
+ self.send_started = asyncio.Event()
+ self.release_send = asyncio.Event()
+ self.send_cancelled = asyncio.Event()
+ self.closed = asyncio.Event()
+ self.release_close = asyncio.Event()
+ self.sent: asyncio.Queue[str] = asyncio.Queue()
+ self.close_calls: list[tuple[int, str]] = []
+ self.transport = MagicMock()
+ self.send_calls = 0
+ self.active_sends = 0
+ self.max_active_sends = 0
+
+ async def send(self, raw: str) -> None:
+ self.send_calls += 1
+ call_number = self.send_calls
+ self.active_sends += 1
+ self.max_active_sends = max(self.max_active_sends, self.active_sends)
+ self.send_started.set()
+ try:
+ if self.block_first_send and call_number == 1:
+ await self.release_send.wait()
+ await self.sent.put(raw)
+ except asyncio.CancelledError:
+ self.send_cancelled.set()
+ raise
+ finally:
+ self.active_sends -= 1
+
+ async def close(self, code: int = 1000, reason: str = "") -> None:
+ self.close_calls.append((code, reason))
+ self.closed.set()
+ if self.block_close:
+ await self.release_close.wait()
+
+
+def _channel() -> WebSocketChannel:
+ bus = MagicMock(spec=MessageBus)
+ config = WebSocketConfig(
+ enabled=True,
+ allow_from=["*"],
+ websocket_requires_token=False,
+ )
+ gateway = build_gateway_services(
+ config=config,
+ bus=bus,
+ session_manager=None,
+ static_dist_path=None,
+ workspace_path=Path.cwd(),
+ default_restrict_to_workspace=False,
+ runtime_model_name=None,
+ runtime_surface="tui",
+ runtime_capabilities_overrides=None,
+ )
+ return WebSocketChannel(config, bus, gateway=gateway)
+
+
+def _message(chat_id: str, text: str) -> OutboundMessage:
+ return OutboundMessage(channel="websocket", chat_id=chat_id, content=text)
+
+
+async def _next_text(connection: _RecordingConnection) -> str:
+ raw = await asyncio.wait_for(connection.sent.get(), timeout=1)
+ return str(json.loads(raw)["text"])
+
+
+async def _wait_for_connection_cleanup(
+ channel: WebSocketChannel,
+ connection: _RecordingConnection,
+) -> None:
+ while cast(Any, connection) in channel._connection_outbound:
+ await asyncio.sleep(0)
+
+
+async def _wait_for_retire_tasks(channel: WebSocketChannel) -> None:
+ while channel._outbound_retire_tasks:
+ await asyncio.sleep(0)
+
+
+async def test_slow_tui_does_not_block_a_different_topic() -> None:
+ channel = _channel()
+ slow = _RecordingConnection(block_first_send=True)
+ healthy = _RecordingConnection()
+ channel._attach(cast(Any, slow), "chat-slow")
+ channel._attach(cast(Any, healthy), "chat-healthy")
+
+ try:
+ await asyncio.wait_for(channel.send(_message("chat-slow", "slow")), timeout=1)
+ await asyncio.wait_for(slow.send_started.wait(), timeout=1)
+
+ await asyncio.wait_for(channel.send(_message("chat-healthy", "healthy")), timeout=1)
+ assert await _next_text(healthy) == "healthy"
+ assert slow.sent.empty()
+ finally:
+ slow.release_send.set()
+ await channel._cleanup_connection(cast(Any, slow))
+ await channel._cleanup_connection(cast(Any, healthy))
+
+
+async def test_slow_tui_does_not_block_same_topic_fanout() -> None:
+ channel = _channel()
+ slow = _RecordingConnection(block_first_send=True)
+ healthy = _RecordingConnection()
+ channel._attach(cast(Any, slow), "chat-shared")
+ channel._attach(cast(Any, healthy), "chat-shared")
+
+ try:
+ await asyncio.wait_for(channel.send(_message("chat-shared", "hello")), timeout=1)
+ await asyncio.wait_for(slow.send_started.wait(), timeout=1)
+ assert await _next_text(healthy) == "hello"
+ assert slow.sent.empty()
+ finally:
+ slow.release_send.set()
+ await channel._cleanup_connection(cast(Any, slow))
+ await channel._cleanup_connection(cast(Any, healthy))
+
+
+async def test_connection_writer_preserves_order_without_concurrent_sends() -> None:
+ channel = _channel()
+ connection = _RecordingConnection(block_first_send=True)
+ channel._attach(cast(Any, connection), "chat-order")
+
+ try:
+ await channel.send(_message("chat-order", "one"))
+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)
+ await channel.send(_message("chat-order", "two"))
+ await channel.send(_message("chat-order", "three"))
+
+ connection.release_send.set()
+ assert [await _next_text(connection) for _ in range(3)] == ["one", "two", "three"]
+ assert connection.max_active_sends == 1
+ finally:
+ connection.release_send.set()
+ await channel._cleanup_connection(cast(Any, connection))
+
+
+async def test_full_outbound_queue_disconnects_only_the_slow_tui(monkeypatch) -> None:
+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_FRAMES", 2)
+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 1024 * 1024)
+ channel = _channel()
+ slow = _RecordingConnection(block_first_send=True)
+ healthy = _RecordingConnection()
+ channel._attach(cast(Any, slow), "chat-shared")
+ channel._attach(cast(Any, healthy), "chat-shared")
+
+ try:
+ for index in range(4):
+ await channel.send(_message("chat-shared", str(index)))
+ assert await _next_text(healthy) == str(index)
+ if index == 0:
+ await asyncio.wait_for(slow.send_started.wait(), timeout=1)
+
+ await asyncio.wait_for(slow.closed.wait(), timeout=1)
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, slow), timeout=1)
+ assert slow.close_calls == [(1013, "outbound queue full")]
+ assert slow not in channel._conn_chats
+ assert slow not in channel._subs["chat-shared"]
+ assert healthy in channel._subs["chat-shared"]
+ finally:
+ slow.release_send.set()
+ await channel._cleanup_connection(cast(Any, slow))
+ await channel._cleanup_connection(cast(Any, healthy))
+
+
+async def test_outbound_byte_budget_disconnects_oversized_connection(monkeypatch) -> None:
+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)
+ channel = _channel()
+ connection = _RecordingConnection()
+ channel._attach(cast(Any, connection), "chat-large")
+
+ try:
+ await channel.send(_message("chat-large", "x" * 256))
+ await asyncio.wait_for(connection.closed.wait(), timeout=1)
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, connection), timeout=1)
+ assert connection.send_calls == 0
+ assert connection.close_calls == [(1013, "outbound queue full")]
+ finally:
+ await channel._cleanup_connection(cast(Any, connection))
+
+
+async def test_cleanup_cancels_a_blocked_writer_and_discards_pending_frames() -> None:
+ channel = _channel()
+ connection = _RecordingConnection(block_first_send=True)
+ channel._attach(cast(Any, connection), "chat-cleanup")
+ channel._conn_default[cast(Any, connection)] = "chat-cleanup"
+
+ await channel.send(_message("chat-cleanup", "one"))
+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)
+ await channel.send(_message("chat-cleanup", "two"))
+
+ await asyncio.wait_for(channel._cleanup_connection(cast(Any, connection)), timeout=1)
+ await channel._cleanup_connection(cast(Any, connection))
+
+ assert connection.send_cancelled.is_set()
+ assert connection not in channel._conn_chats
+ assert connection not in channel._conn_default
+ assert connection not in channel._connection_outbound
+ assert channel._subs == {}
+
+
+async def test_send_timeout_retires_only_the_slow_connection(monkeypatch) -> None:
+ monkeypatch.setattr(runtime, "_OUTBOUND_SEND_TIMEOUT_S", 0.01)
+ channel = _channel()
+ slow = _RecordingConnection(block_first_send=True)
+ healthy = _RecordingConnection()
+ channel._attach(cast(Any, slow), "chat-shared")
+ channel._attach(cast(Any, healthy), "chat-shared")
+
+ try:
+ await channel.send(_message("chat-shared", "hello"))
+
+ assert await _next_text(healthy) == "hello"
+ await asyncio.wait_for(slow.closed.wait(), timeout=1)
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, slow), timeout=1)
+ assert slow.close_calls == [(1013, "outbound send timeout")]
+ assert slow not in channel._conn_chats
+ assert healthy in channel._subs["chat-shared"]
+ finally:
+ slow.release_send.set()
+ await channel._cleanup_connection(cast(Any, slow))
+ await channel._cleanup_connection(cast(Any, healthy))
+
+
+async def test_stop_cancels_connection_writers() -> None:
+ channel = _channel()
+ connection = _RecordingConnection(block_first_send=True)
+ channel._attach(cast(Any, connection), "chat-stop")
+
+ await channel.send(_message("chat-stop", "one"))
+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)
+ await asyncio.wait_for(channel.stop(), timeout=1)
+
+ assert connection.send_cancelled.is_set()
+ assert channel._connection_outbound == {}
+ assert channel._outbound_retire_tasks == set()
+ assert channel._subs == {}
+
+
+async def test_stop_cancels_writers_before_waiting_for_the_server_task() -> None:
+ channel = _channel()
+ connection = _RecordingConnection(block_first_send=True)
+ channel._attach(cast(Any, connection), "chat-stop-order")
+ channel._running = True
+ channel._stop_event = asyncio.Event()
+
+ await channel.send(_message("chat-stop-order", "one"))
+ await asyncio.wait_for(connection.send_started.wait(), timeout=1)
+
+ async def _server_shutdown() -> None:
+ assert channel._stop_event is not None
+ await channel._stop_event.wait()
+ await connection.send_cancelled.wait()
+
+ channel._server_task = asyncio.create_task(_server_shutdown())
+
+ await asyncio.wait_for(channel.stop(), timeout=1)
+
+ assert connection.send_cancelled.is_set()
+ assert channel._server_task is None
+
+
+async def test_send_to_cleaned_connection_does_not_recreate_outbound_state() -> None:
+ channel = _channel()
+ connection = _RecordingConnection()
+ channel._attach(cast(Any, connection), "chat-stale")
+ await channel._cleanup_connection(cast(Any, connection))
+
+ await channel._safe_send_to(cast(Any, connection), "stale")
+
+ assert connection.send_calls == 0
+ assert connection not in channel._connection_outbound
+
+
+async def test_attach_does_not_revive_a_cleaned_connection() -> None:
+ channel = _channel()
+ connection = _RecordingConnection()
+ channel._attach(cast(Any, connection), "chat-before-cleanup")
+ await channel._cleanup_connection(cast(Any, connection))
+
+ channel._attach(cast(Any, connection), "chat-after-cleanup")
+
+ assert connection not in channel._connection_outbound
+ assert connection not in channel._conn_chats
+ assert "chat-after-cleanup" not in channel._subs
+
+
+async def test_buffered_envelope_does_not_revive_a_cleaned_connection(monkeypatch) -> None:
+ channel = _channel()
+ connection = _RecordingConnection()
+ channel._attach(cast(Any, connection), "chat-before-cleanup")
+ await channel._cleanup_connection(cast(Any, connection))
+ dispatch = AsyncMock()
+ monkeypatch.setattr(channel._commands, "dispatch", dispatch)
+
+ await channel._dispatch_envelope(cast(Any, connection), "client", {"type": "attach"})
+
+ dispatch.assert_not_awaited()
+ assert connection not in channel._connection_outbound
+
+
+async def test_control_broadcast_cannot_precede_ready() -> None:
+ channel = _channel()
+ connection = _RecordingConnection()
+ channel._webui_connections.add(cast(Any, connection))
+
+ await channel._send_event(cast(Any, connection), "session_updated", chat_id="chat")
+
+ assert connection.send_calls == 0
+ assert connection not in channel._connection_outbound
+
+
+async def test_close_timeout_aborts_transport_and_cleans_connection(monkeypatch) -> None:
+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)
+ monkeypatch.setattr(runtime, "_OUTBOUND_CLOSE_TIMEOUT_S", 0.01)
+ channel = _channel()
+ connection = _RecordingConnection(block_close=True)
+ channel._attach(cast(Any, connection), "chat-close-timeout")
+
+ await channel.send(_message("chat-close-timeout", "x" * 256))
+ await asyncio.wait_for(connection.closed.wait(), timeout=1)
+ await asyncio.wait_for(_wait_for_connection_cleanup(channel, connection), timeout=1)
+
+ connection.transport.abort.assert_called_once_with()
+ assert connection not in channel._conn_chats
+
+
+async def test_retirement_contains_cleanup_failure(monkeypatch) -> None:
+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)
+ channel = _channel()
+ connection = _RecordingConnection()
+ channel._attach(cast(Any, connection), "chat-cleanup-failure")
+ channel._webui_connections.add(cast(Any, connection))
+ monkeypatch.setattr(
+ channel._commands,
+ "cleanup_connection",
+ AsyncMock(side_effect=RuntimeError("cleanup failed")),
+ )
+
+ await channel.send(_message("chat-cleanup-failure", "x" * 256))
+ await asyncio.wait_for(_wait_for_retire_tasks(channel), timeout=1)
+
+ assert connection.close_calls == [(1013, "outbound queue full")]
+ assert connection not in channel._connection_outbound
+ assert connection not in channel._conn_chats
+ assert connection not in channel._webui_connections
+ assert channel._subs == {}
+
+
+async def test_normal_cleanup_can_overlap_connection_retirement(monkeypatch) -> None:
+ monkeypatch.setattr(runtime, "_OUTBOUND_QUEUE_MAX_BYTES", 64)
+ channel = _channel()
+ connection = _RecordingConnection(block_close=True)
+ channel._attach(cast(Any, connection), "chat-retire-race")
+
+ await channel.send(_message("chat-retire-race", "x" * 256))
+ await asyncio.wait_for(connection.closed.wait(), timeout=1)
+ await asyncio.wait_for(channel._cleanup_connection(cast(Any, connection)), timeout=1)
+ connection.release_close.set()
+ await asyncio.wait_for(_wait_for_retire_tasks(channel), timeout=1)
+
+ assert connection not in channel._connection_outbound
+ assert connection not in channel._conn_chats
+ assert channel._subs == {}