Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

13.9 KB
Raw Blame
"""Server heartbeat negotiation and lifecycle on the ``/ws/chat`` endpoint."""

from __future__ import annotations

import asyncio
import json
import logging
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, patch

import pytest
from anyio import BrokenResourceError, ClosedResourceError
from fastapi import WebSocketDisconnect

from tests.shared.async_waits import eventually
from xagent.web.api import websocket as websocket_api

TASK_ID = 41
CAPABILITIES_FRAME = {"type": "connection_capabilities", "capabilities": ["heartbeat"]}
HEARTBEAT_FRAME = {"type": "heartbeat"}
STATUS_MARKER = {"type": "status_request_handled"}
# Mirrors ``DEAD_AFTER_MS`` in the frontend ``use-websocket`` hook.
CLIENT_DEAD_AFTER_SECONDS = 30.0
HEARTBEAT_LOGGER = "xagent.web.api.websocket"


class _FakeSocket:
    """Records sent frames; ``confirmation_delay`` slows only the capabilities
    frame so a heartbeat task started too early would overtake it.
    ``heartbeat_error`` is raised by every heartbeat send, and
    ``heartbeat_senders`` names the task that attempted each one."""

    def __init__(
        self,
        liveness: list[str],
        *,
        confirmation_delay: float = 0.0,
        heartbeat_error: BaseException | None = None,
    ) -> None:
        self.query_params = SimpleNamespace(
            getlist=lambda key: list(liveness) if key == "liveness" else []
        )
        self.accept = AsyncMock()
        self.close = AsyncMock()
        self.frames: list[dict[str, Any]] = []
        self.heartbeat_senders: list[str] = []
        self._inbox: asyncio.Queue[Any] = asyncio.Queue()
        self._confirmation_delay = confirmation_delay
        self._heartbeat_error = heartbeat_error

    async def send_text(self, text: str) -> None:
        frame = json.loads(text)
        if frame == CAPABILITIES_FRAME and self._confirmation_delay:
            await asyncio.sleep(self._confirmation_delay)
        if frame == HEARTBEAT_FRAME:
            current = asyncio.current_task()
            assert current is not None
            self.heartbeat_senders.append(current.get_coro().__qualname__)
            if self._heartbeat_error is not None:
                raise self._heartbeat_error
        self.frames.append(frame)

    async def receive_text(self) -> str:
        item = await self._inbox.get()
        if isinstance(item, BaseException):
            raise item
        return item

    def client_sends(self, item: Any) -> None:
        self._inbox.put_nowait(item)


async def _status_marker(websocket: _FakeSocket, *_: Any) -> None:
    websocket.frames.append(STATUS_MARKER)


def _endpoint_env(manager: websocket_api.ConnectionManager, **overrides: Any) -> Any:
    """Patch the endpoint's collaborators so no database is touched."""
    return patch.multiple(
        websocket_api,
        manager=manager,
        get_authenticated_user=AsyncMock(
            return_value=SimpleNamespace(id=7, is_admin=False)
        ),
        _private_websocket_task_access_sync=lambda **_: "authorized",
        handle_status_request=_status_marker,
        **overrides,
    )


async def _finished(endpoint: asyncio.Task[None]) -> list[Any]:
    # Bounded, so an endpoint that never stops its heartbeat task fails the
    # test instead of hanging it.
    return await asyncio.wait_for(
        asyncio.gather(endpoint, return_exceptions=True), timeout=5
    )


def _run_endpoint(websocket: _FakeSocket) -> asyncio.Task[None]:
    return asyncio.ensure_future(
        websocket_api.websocket_chat_endpoint(websocket, TASK_ID, None)  # type: ignore[arg-type]
    )


def _running_heartbeats() -> list[asyncio.Task[Any]]:
    return [
        task
        for task in asyncio.all_tasks()
        if task.get_coro().__qualname__ == "_run_chat_heartbeat" and not task.done()
    ]


def _heartbeat_warnings(caplog: pytest.LogCaptureFixture) -> list[logging.LogRecord]:
    return [
        record
        for record in caplog.records
        if record.name == HEARTBEAT_LOGGER and "heartbeat" in record.getMessage()
    ]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("liveness", "negotiated"),
    [
        ([], False),
        (["heartbeat"], True),
        (["ping"], False),
        (["heartbeat", "heartbeat"], False),
    ],
)
async def test_ws_chat_liveness_negotiation(
    liveness: list[str], negotiated: bool
) -> None:
    manager = websocket_api.ConnectionManager()
    websocket = _FakeSocket(liveness, confirmation_delay=0.1)

    with _endpoint_env(manager, HEARTBEAT_INTERVAL_SECONDS=0.005):
        endpoint = _run_endpoint(websocket)
        await asyncio.sleep(0.3)  # many heartbeat intervals
        websocket.client_sends(WebSocketDisconnect())
        await _finished(endpoint)

    if negotiated:
        # The acknowledgement precedes the status snapshot and every heartbeat.
        assert websocket.frames[0] == CAPABILITIES_FRAME
        assert STATUS_MARKER in websocket.frames
        assert HEARTBEAT_FRAME in websocket.frames
    else:
        assert websocket.frames == [STATUS_MARKER]


@pytest.mark.asyncio
async def test_ws_chat_heartbeat_survives_blocked_loop() -> None:
    manager = websocket_api.ConnectionManager()
    handler_started = asyncio.Event()
    release_handler = asyncio.Event()

    async def blocked_execute_task(*_: Any) -> None:
        handler_started.set()
        await release_handler.wait()

    # Phase 1: the read loop is stuck inside a long handler, and the heartbeat
    # keeps flowing; after a normal disconnect the task is gone.
    first = _FakeSocket(["heartbeat"])
    with _endpoint_env(
        manager,
        HEARTBEAT_INTERVAL_SECONDS=0.05,
        handle_execute_task=blocked_execute_task,
    ):
        endpoint = _run_endpoint(first)
        try:
            first.client_sends(json.dumps({"type": "execute_task"}))
            await eventually(lambda: handler_started.is_set() or endpoint.done())
            assert handler_started.is_set()
            beats = first.frames.count(HEARTBEAT_FRAME)
            await eventually(
                lambda: first.frames.count(HEARTBEAT_FRAME) >= beats + 2
                or endpoint.done()
            )
            assert not endpoint.done()
        finally:
            release_handler.set()
            first.client_sends(WebSocketDisconnect())
            assert await _finished(endpoint) == [None]

        assert manager.connection_count() == 0
        assert _running_heartbeats() == []

    # Phase 2: a stand-in heartbeat task raises past the real task's own
    # handling; teardown still removes the connection from the manager.
    heartbeat_started = asyncio.Event()

    async def failing_heartbeat(*_: Any) -> None:
        heartbeat_started.set()
        raise ValueError("heartbeat fault")

    second = _FakeSocket(["heartbeat"])
    with _endpoint_env(
        manager,
        _run_chat_heartbeat=failing_heartbeat,
    ):
        endpoint = _run_endpoint(second)
        try:
            await eventually(lambda: heartbeat_started.is_set() or endpoint.done())
            assert heartbeat_started.is_set()
            assert manager.connection_count() == 1
        finally:
            second.client_sends(WebSocketDisconnect())
            assert await _finished(endpoint) == [None]
        assert manager.connection_count() == 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "access_delay", [0.0, 0.3], ids=["immediate", "delayed-access"]
)
@pytest.mark.parametrize(
    "send_error",
    [
        RuntimeError("socket closed"),
        ConnectionError("socket gone"),
        WebSocketDisconnect(code=1006),
        BrokenResourceError(),
        ClosedResourceError(),
    ],
    ids=lambda error: type(error).__name__,
)
async def test_ws_chat_heartbeat_send_failure_ends_quietly(
    send_error: BaseException, access_delay: float, caplog: pytest.LogCaptureFixture
) -> None:
    manager = websocket_api.ConnectionManager()
    websocket = _FakeSocket(["heartbeat"], heartbeat_error=send_error)
    run_db_io = websocket_api.run_db_io_cancellation_safe

    async def delayed_access(operation: Any) -> Any:
        # Model a busy executor: authorization can legitimately outlast 200 ms.
        await asyncio.sleep(access_delay)
        return await run_db_io(operation)

    with (
        caplog.at_level(logging.WARNING, logger=HEARTBEAT_LOGGER),
        _endpoint_env(
            manager,
            HEARTBEAT_INTERVAL_SECONDS=0.01,
            run_db_io_cancellation_safe=delayed_access,
        ),
    ):
        endpoint = _run_endpoint(websocket)
        try:
            # Wait for the send and task exit, not a wall-clock estimate of
            # how long authorization and event-loop scheduling should take.
            await eventually(
                lambda: websocket.heartbeat_senders and not _running_heartbeats(),
                timeout=5,
            )

            # One failed send ended the task; the read loop still answers.
            assert websocket.heartbeat_senders == ["_run_chat_heartbeat"]
            assert _running_heartbeats() == []
            websocket.client_sends(json.dumps({"type": "status_request"}))
            await eventually(
                lambda: websocket.frames.count(STATUS_MARKER) >= 2, timeout=5
            )
            assert websocket.frames.count(STATUS_MARKER) == 2
        finally:
            websocket.client_sends(WebSocketDisconnect())
            assert await _finished(endpoint) == [None]

    assert _heartbeat_warnings(caplog) == []
    assert manager.connection_count() == 0


@pytest.mark.asyncio
async def test_ws_chat_heartbeat_logs_other_exceptions_while_connected(
    caplog: pytest.LogCaptureFixture,
) -> None:
    manager = websocket_api.ConnectionManager()
    websocket = _FakeSocket(["heartbeat"], heartbeat_error=ValueError("bad frame"))

    with (
        caplog.at_level(logging.WARNING, logger=HEARTBEAT_LOGGER),
        _endpoint_env(manager, HEARTBEAT_INTERVAL_SECONDS=0.01),
    ):
        endpoint = _run_endpoint(websocket)
        await asyncio.sleep(0.2)

        # Logged by the task itself, with the traceback, before any teardown.
        assert not endpoint.done()
        assert _running_heartbeats() == []
        (record,) = _heartbeat_warnings(caplog)
        assert record.exc_info is not None
        assert isinstance(record.exc_info[1], ValueError)
        assert str(TASK_ID) in record.getMessage()

        websocket.client_sends(WebSocketDisconnect())
        assert await _finished(endpoint) == [None]

    # Teardown does not log the same fault a second time.
    assert len(_heartbeat_warnings(caplog)) == 1
    assert manager.connection_count() == 0


class _OrderRecordingManager(websocket_api.ConnectionManager):
    def __init__(self) -> None:
        super().__init__()
        self.heartbeats_running_at_disconnect: list[int] = []

    def disconnect(self, websocket: Any) -> None:
        self.heartbeats_running_at_disconnect.append(len(_running_heartbeats()))
        super().disconnect(websocket)


@pytest.mark.asyncio
async def test_ws_chat_heartbeat_stops_before_the_connection_is_unregistered(
    caplog: pytest.LogCaptureFixture,
) -> None:
    manager = _OrderRecordingManager()
    websocket = _FakeSocket(["heartbeat"])

    with (
        caplog.at_level(logging.WARNING, logger=HEARTBEAT_LOGGER),
        _endpoint_env(manager, HEARTBEAT_INTERVAL_SECONDS=0.01),
    ):
        endpoint = _run_endpoint(websocket)
        await asyncio.sleep(0.05)
        assert len(_running_heartbeats()) == 1
        websocket.client_sends(WebSocketDisconnect())
        assert await _finished(endpoint) == [None]

    assert manager.heartbeats_running_at_disconnect == [0]
    # Cancelling the heartbeat on a normal disconnect is not a fault.
    assert _heartbeat_warnings(caplog) == []


@pytest.mark.asyncio
async def test_ws_chat_heartbeat_in_shared_mode_goes_through_the_writer(
    caplog: pytest.LogCaptureFixture,
) -> None:
    manager = websocket_api.ConnectionManager()
    websocket = _FakeSocket(["heartbeat"])
    bridge = SimpleNamespace(discard_recipient=lambda recipient: None)
    queued_heartbeats: list[_FakeSocket] = []
    enqueue = websocket_api.TaskSocketWriter.enqueue

    def record_enqueue(
        writer: websocket_api.TaskSocketWriter, text: str, **kwargs: Any
    ) -> asyncio.Future[None] | None:
        delivery = enqueue(writer, text, **kwargs)
        if json.loads(text) == HEARTBEAT_FRAME:
            queued_heartbeats.append(writer.socket)
        return delivery

    with (
        caplog.at_level(logging.WARNING, logger=HEARTBEAT_LOGGER),
        patch(
            "xagent.web.services.task_event_bridge.get_task_event_bridge",
            return_value=bridge,
        ),
        patch.object(websocket_api.TaskSocketWriter, "enqueue", record_enqueue),
        _endpoint_env(
            manager,
            HEARTBEAT_INTERVAL_SECONDS=0.01,
            get_shared_task_execution_enabled=lambda: True,
        ),
    ):
        endpoint = _run_endpoint(websocket)
        try:
            await eventually(
                lambda: HEARTBEAT_FRAME in websocket.frames or endpoint.done()
            )
            assert HEARTBEAT_FRAME in websocket.frames
            assert websocket.frames[0] == CAPABILITIES_FRAME
            assert queued_heartbeats and all(
                socket is websocket for socket in queued_heartbeats
            )

            # A failed write closes the writer and ends the heartbeat quietly.
            websocket._heartbeat_error = OSError("write failed")
            await eventually(
                lambda: not _running_heartbeats()
                and manager.connection_count() == 0
                and websocket.close.await_count > 0
            )
            websocket.close.assert_awaited_with(code=1013)
        finally:
            websocket.client_sends(WebSocketDisconnect())
            assert await _finished(endpoint) == [None]

    assert _heartbeat_warnings(caplog) == []


def test_heartbeat_interval_stays_within_half_the_dead_threshold() -> None:
    assert websocket_api.HEARTBEAT_INTERVAL_SECONDS * 2 <= CLIENT_DEAD_AFTER_SECONDS