Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

19.8 KB
Raw Blame
"""A planned restart closes registered WebSockets with 1012 Service Restart."""

import asyncio
import json
import time

import pytest
from starlette.websockets import WebSocketState

from xagent.web.api import websocket as ws_module
from xagent.web.api.websocket import ConnectionManager, _command_origins


class FakeSocket:
    def __init__(
        self,
        *,
        fail_close: bool = False,
        hang_close: bool = False,
        log: list[str] | None = None,
        send_yields: bool = False,
    ) -> None:
        self.send_yields = send_yields
        self.closed: list[tuple[int, str | None]] = []
        self.fail_close = fail_close
        self.hang_close = hang_close
        self.log = log if log is not None else []

    async def send_text(self, text: str) -> None:
        if self.send_yields:
            await asyncio.sleep(0)
        self.log.append(f"sent:{text}")

    async def close(self, code: int = 1000, reason: str | None = None) -> None:
        if self.hang_close:
            await asyncio.Event().wait()
        if self.fail_close:
            raise RuntimeError("peer already gone")
        self.log.append(f"close:{code}")
        self.closed.append((code, reason))


class FakeSse(FakeSocket):
    is_broadcast_only = True


@pytest.fixture
def manager():
    mgr = ConnectionManager()
    yield mgr
    _command_origins._origins.clear()


@pytest.mark.asyncio
async def test_observation_only_sockets_close_with_1012_and_unregister(manager):
    a, b = FakeSocket(), FakeSocket()
    manager.register_connection(a, 1)
    manager.register_connection(b, 2, public_audience=True)

    assert await manager.close_for_restart() == 2

    assert a.closed == [(1012, "Service Restart")]
    assert b.closed == [(1012, "Service Restart")]
    assert manager.connection_count() == 0


@pytest.mark.asyncio
async def test_sse_sinks_are_left_open(manager):
    sse, ws = FakeSse(), FakeSocket()
    manager.register_connection(sse, 1)
    manager.register_connection(ws, 1)

    await manager.close_for_restart()

    assert sse.closed == []
    assert ws.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_socket_with_pending_command_closes_after_reply(manager):
    waiting, idle = FakeSocket(), FakeSocket()
    manager.register_connection(waiting, 1)
    manager.register_connection(idle, 1)
    _command_origins.register("cmd-1", waiting, 1)

    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.2)
    assert idle.closed == [(1012, "Service Restart")]
    assert waiting.closed == []

    _command_origins.discard_command("cmd-1", 1)
    await asyncio.wait_for(closing, timeout=2)
    assert waiting.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_pending_command_is_closed_at_the_deadline(manager):
    waiting = FakeSocket()
    manager.register_connection(waiting, 1)
    _command_origins.register("cmd-1", waiting, 1)

    await asyncio.wait_for(manager.close_for_restart(reply_deadline_seconds=0.1), 2)

    assert waiting.closed == [(1012, "Service Restart")]
    assert manager.connection_count() == 0


@pytest.mark.asyncio
async def test_close_failure_still_unregisters_and_is_idempotent(manager):
    gone = FakeSocket(fail_close=True)
    manager.register_connection(gone, 1)

    assert await manager.close_for_restart() == 1
    assert manager.connection_count() == 0
    assert await manager.close_for_restart() == 0


def test_close_code_constant():
    assert ws_module.WS_CLOSE_SERVICE_RESTART == 1012


class _FakeBridge:
    def __init__(self) -> None:
        self.pending: set[int] = set()

    def has_pending_for_recipient(self, recipient) -> bool:
        return id(recipient) in self.pending

    def discard_recipient(self, recipient) -> None:
        self.pending.discard(id(recipient))


@pytest.fixture
def shared_mode(monkeypatch):
    from xagent.web.services import task_event_bridge

    bridge = _FakeBridge()
    monkeypatch.setattr(ws_module, "get_shared_task_execution_enabled", lambda: True)
    monkeypatch.setattr(task_event_bridge, "get_task_event_bridge", lambda: bridge)
    return bridge


@pytest.mark.asyncio
async def test_shared_mode_pending_command_waits_for_its_reply(manager, shared_mode):
    waiting = FakeSocket()
    manager.register_connection(waiting, 1)
    shared_mode.pending.add(id(waiting))

    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.2)
    assert waiting.closed == []

    shared_mode.pending.clear()
    await asyncio.wait_for(closing, timeout=2)
    assert waiting.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_socket_registered_while_draining_is_closed_at_once(manager):
    await manager.close_for_restart()
    late, sse = FakeSocket(), FakeSse()

    manager.register_connection(late, 1)
    manager.register_connection(sse, 1)
    await asyncio.sleep(0.05)

    assert late.closed == [(1012, "Service Restart")]
    assert sse.closed == []
    assert manager.connection_count() == 1  # only the SSE sink remains


@pytest.mark.asyncio
async def test_end_restart_drain_admits_new_sockets_again(manager):
    await manager.close_for_restart()
    manager.end_restart_drain()
    late = FakeSocket()

    manager.register_connection(late, 1)
    await asyncio.sleep(0.05)

    assert late.closed == []
    assert manager.connection_count() == 1


@pytest.mark.asyncio
async def test_stalled_close_is_bounded_and_unregisters(manager):
    stuck = FakeSocket(hang_close=True)
    manager.register_connection(stuck, 1)

    handled = await asyncio.wait_for(
        manager.close_for_restart(close_timeout_seconds=0.1), timeout=2
    )

    assert handled == 1
    assert manager.connection_count() == 0


@pytest.mark.asyncio
async def test_ingress_holds_the_close_until_the_handler_finishes(manager):
    log: list[str] = []
    ws = FakeSocket(log=log)
    manager.register_connection(ws, 1)
    release = asyncio.Event()

    async def handler() -> None:
        async with manager.ingress(ws):
            await release.wait()
            await ws.send_text("accepted")

    work = asyncio.create_task(handler())
    await asyncio.sleep(0.05)
    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.2)
    assert log == []

    release.set()
    await asyncio.wait_for(asyncio.gather(work, closing), timeout=2)
    assert log == ["sent:accepted", "close:1012"]


@pytest.mark.asyncio
async def test_ingress_is_released_when_the_handler_raises(manager):
    ws = FakeSocket()
    manager.register_connection(ws, 1)

    with pytest.raises(RuntimeError):
        async with manager.ingress(ws):
            raise RuntimeError("handler failed")

    assert not manager._has_pending_command(ws)


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "name", ["handle_chat_message", "handle_pause_task", "handle_resume_task"]
)
async def test_command_handlers_hold_ingress_while_enqueueing(monkeypatch, name):
    log: list[str] = []
    ws = FakeSocket(log=log)
    gm = ws_module.manager
    gm.register_connection(ws, 1)
    release = asyncio.Event()

    async def blocked_enqueue(**_kwargs):
        await release.wait()
        raise ValueError("rejected")

    async def reply(message, websocket):
        await websocket.send_text("reply")

    monkeypatch.setattr(ws_module, "_enqueue_websocket_task_command", blocked_enqueue)
    monkeypatch.setattr(gm, "send_personal_message", reply)
    monkeypatch.setattr(ws_module, "send_message_delivery", _reply_delivery)
    handler = getattr(ws_module, name)
    work = asyncio.create_task(handler(ws, 1, {"command_id": "c1"}))
    await asyncio.sleep(0.05)
    try:
        closing = asyncio.create_task(gm.close_for_restart(reply_deadline_seconds=5))
        await asyncio.sleep(0.2)
        assert log == []
        release.set()
        await asyncio.wait_for(asyncio.gather(work, closing), timeout=2)
        assert log[-1] == "close:1012"
        assert log.index("sent:reply") < log.index("close:1012")
    finally:
        gm.end_restart_drain()
        gm.disconnect(ws)


async def _reply_delivery(websocket, **_kwargs):
    await websocket.send_text("reply")


@pytest.fixture
def global_manager(monkeypatch):
    """The module-level manager, which the real reply path is bound to."""

    async def passthrough(message):
        return message

    monkeypatch.setattr(ws_module, "_with_current_task_control_state", passthrough)
    gm = ws_module.manager
    yield gm
    gm.end_restart_drain()
    for ws in list(gm._connection_task_ids):
        gm.disconnect(ws)
    _command_origins._origins.clear()


@pytest.mark.asyncio
async def test_task_id_updated_reply_precedes_1012_through_the_real_reply_path(
    global_manager,
):
    # A legacy first chat: the reply re-registers the socket on the new task
    # while the manager drains, then sends ``task_id_updated``.
    log: list[str] = []
    ws = FakeSocket(log=log, send_yields=True)
    global_manager.register_connection(ws, 1)
    _command_origins.register("cmd-1", ws, 1)

    closing = asyncio.create_task(
        global_manager.close_for_restart(reply_deadline_seconds=5)
    )
    await asyncio.sleep(0.1)
    reply = _command_origins.reply_for("cmd-1", 1)
    await reply({"type": "task_id_updated", "new_task_id": 2})
    await asyncio.sleep(0.1)
    assert [e.split(":", 1)[0] for e in log] == ["sent"]
    assert json.loads(log[0].removeprefix("sent:"))["new_task_id"] == 2

    _command_origins.discard_command("cmd-1", 1)
    await asyncio.wait_for(closing, timeout=2)
    assert log[1:] == ["close:1012"]


@pytest.mark.asyncio
async def test_move_connection_while_holding_ingress_is_not_closed_at_once(manager):
    ws = FakeSocket()
    manager.register_connection(ws, 1)
    release = asyncio.Event()

    async def handler() -> None:
        async with manager.ingress(ws):
            await release.wait()

    work = asyncio.create_task(handler())
    await asyncio.sleep(0.05)
    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.1)

    manager.move_connection(ws, 2)
    await asyncio.sleep(0.2)
    assert ws.closed == []
    assert manager.connection_count() == 1

    release.set()
    await asyncio.wait_for(asyncio.gather(work, closing), timeout=2)
    assert ws.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_ingress_hands_over_to_the_origin_without_a_close_gap(manager):
    # The accepted path: the handler registers the origin, then leaves ingress.
    ws = FakeSocket()
    manager.register_connection(ws, 1)
    release = asyncio.Event()

    async def handler() -> None:
        async with manager.ingress(ws):
            _command_origins.register("cmd-1", ws, 1)
            await release.wait()

    work = asyncio.create_task(handler())
    await asyncio.sleep(0.05)
    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    release.set()
    await asyncio.wait_for(work, timeout=1)
    await asyncio.sleep(0.2)
    assert ws.closed == []

    _command_origins.discard_command("cmd-1", 1)
    await asyncio.wait_for(closing, timeout=2)
    assert ws.closed == [(1012, "Service Restart")]


@pytest.fixture
def real_bridge(monkeypatch):
    monkeypatch.setenv("XAGENT_REDIS_URL", "redis://127.0.0.1:1/0")
    from xagent.web.services import task_event_bridge
    from xagent.web.services.task_event_bridge import TaskEventBridge

    bridge = TaskEventBridge()
    bridge.ready.set()
    monkeypatch.setattr(ws_module, "get_shared_task_execution_enabled", lambda: True)
    monkeypatch.setattr(task_event_bridge, "_bridge", bridge)
    return bridge


@pytest.mark.asyncio
async def test_real_bridge_origin_holds_the_close_until_discarded(manager, real_bridge):
    async def reply(_message) -> None:
        return None

    ws = FakeSocket()
    manager.register_connection(ws, 1)
    token = real_bridge.register_origin(1, "cmd-1", reply, recipient=ws)

    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.2)
    assert ws.closed == []

    real_bridge.discard_origin(token)
    await asyncio.wait_for(closing, timeout=2)
    assert ws.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_unavailable_bridge_counts_as_no_pending_command(manager, monkeypatch):
    from xagent.web.services import task_event_bridge

    monkeypatch.setattr(ws_module, "get_shared_task_execution_enabled", lambda: True)
    monkeypatch.setattr(task_event_bridge, "_bridge", None)  # raises ConnectionError
    ws = FakeSocket()
    manager.register_connection(ws, 1)

    assert not manager._has_pending_command(ws)
    await asyncio.wait_for(manager.close_for_restart(reply_deadline_seconds=5), 1)
    assert ws.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_end_restart_drain_during_the_wait_sends_no_further_1012(manager):
    waiting = FakeSocket()
    manager.register_connection(waiting, 1)
    _command_origins.register("cmd-1", waiting, 1)

    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.2)
    manager.end_restart_drain()
    _command_origins.discard_command("cmd-1", 1)

    assert await asyncio.wait_for(closing, timeout=2) == 0
    assert waiting.closed == []
    assert manager.connection_count() == 1


@pytest.mark.asyncio
async def test_end_restart_drain_lets_a_started_close_finish(manager, monkeypatch):
    monkeypatch.setattr(ws_module, "RESTART_CLOSE_TIMEOUT_SECONDS", 0.1)
    await manager.close_for_restart()
    late = FakeSocket(hang_close=True)
    manager.register_connection(late, 1)
    await asyncio.sleep(0.02)  # the scheduled close is now inside close()
    manager.end_restart_drain()
    await asyncio.sleep(0.3)  # past its timeout: the started close unregisters

    assert manager._restart_closes == set()
    assert manager.connection_count() == 0

    fresh = FakeSocket()
    manager.register_connection(fresh, 1)
    await asyncio.sleep(0.05)
    assert fresh.closed == []
    assert manager.connection_count() == 1


@pytest.mark.asyncio
async def test_end_restart_drain_stops_a_scheduled_close_that_has_not_started(
    manager,
):
    await manager.close_for_restart()
    late = FakeSocket()
    manager.register_connection(late, 1)  # schedules the close, not yet run
    manager.end_restart_drain()
    await asyncio.sleep(0.05)

    assert late.closed == []
    assert manager._restart_closes == set()
    assert manager.connection_count() == 1


@pytest.mark.asyncio
async def test_close_is_never_invoked_after_the_drain_generation_moved(manager):
    seen: list[int] = []

    class GenerationSocket(FakeSocket):
        async def close(self, code: int = 1000, reason: str | None = None) -> None:
            seen.append(manager._restart_drain_generation)
            await super().close(code, reason)

    ws = GenerationSocket()
    manager.register_connection(ws, 1)
    generation = manager._restart_drain_generation
    closing = asyncio.create_task(manager._close_for_restart(ws, 1.0, generation))
    # Queued behind the close's first step: an abort that lands as soon as
    # the close yields for the first time.
    asyncio.get_running_loop().call_soon(manager.end_restart_drain)
    await closing
    await asyncio.sleep(0)

    # Whether or not the close got in first, it was never invoked after the
    # abort, and the socket is unregistered exactly when its close went out.
    assert all(gen == generation for gen in seen)
    assert (ws.closed == []) == (seen == [])
    assert (manager.connection_count() == 1) == (seen == [])


@pytest.mark.asyncio
async def test_zero_close_timeout_still_closes_and_unregisters(manager):
    ws = FakeSocket()
    manager.register_connection(ws, 1)

    handled = await manager.close_for_restart(close_timeout_seconds=0)

    assert handled == 1
    assert ws.closed == [(1012, "Service Restart")]
    assert manager.connection_count() == 0


@pytest.mark.asyncio
async def test_a_close_from_an_ended_drain_is_a_no_op(manager):
    ws = FakeSocket()
    manager.register_connection(ws, 1)
    stale = manager._restart_drain_generation
    manager.end_restart_drain()

    await manager._close_for_restart(ws, 1.0, stale)

    assert ws.closed == []
    assert manager.connection_count() == 1


@pytest.mark.asyncio
async def test_cancelled_close_for_restart_ends_the_drain(manager):
    waiting = FakeSocket()
    manager.register_connection(waiting, 1)
    _command_origins.register("cmd-1", waiting, 1)

    closing = asyncio.create_task(manager.close_for_restart(reply_deadline_seconds=5))
    await asyncio.sleep(0.1)
    closing.cancel()
    with pytest.raises(asyncio.CancelledError):
        await closing

    late = FakeSocket()
    manager.register_connection(late, 2)
    await asyncio.sleep(0.05)
    assert late.closed == []
    assert waiting.closed == []


@pytest.mark.asyncio
async def test_failed_close_for_restart_ends_the_drain(manager, monkeypatch):
    ws = FakeSocket()
    manager.register_connection(ws, 1)

    async def boom(*_args):
        raise ValueError("boom")

    monkeypatch.setattr(manager, "_close_for_restart", boom)
    with pytest.raises(ValueError):
        await manager.close_for_restart()
    monkeypatch.undo()

    late = FakeSocket()
    manager.register_connection(late, 2)
    await asyncio.sleep(0.05)
    assert late.closed == []


@pytest.mark.asyncio
async def test_deadline_counts_from_the_call_not_after_the_first_close(manager):
    stuck = FakeSocket(hang_close=True)
    waiting = FakeSocket()
    manager.register_connection(stuck, 1)
    manager.register_connection(waiting, 1)
    _command_origins.register("cmd-1", waiting, 1)

    started = time.monotonic()
    await asyncio.wait_for(
        manager.close_for_restart(
            reply_deadline_seconds=0.4, close_timeout_seconds=0.4
        ),
        timeout=3,
    )

    # close_timeout + one more close, not close_timeout + deadline + close.
    assert time.monotonic() - started < 0.4 + 0.4 - 0.1
    assert waiting.closed == [(1012, "Service Restart")]


@pytest.mark.asyncio
async def test_execute_task_holds_ingress_while_it_runs(global_manager, monkeypatch):
    ws = FakeSocket()
    release = asyncio.Event()

    async def acknowledge(_message, _websocket):
        return None

    async def blocked_load(*_args, **_kwargs):
        await release.wait()
        return None  # task not found: the handler answers with an error

    monkeypatch.setattr(global_manager, "send_personal_message", acknowledge)
    monkeypatch.setattr(ws_module, "run_db_io_cancellation_safe", blocked_load)
    user = type("User", (), {"id": 1, "is_admin": False})()
    work = asyncio.create_task(ws_module.handle_execute_task(ws, 1, {"user": user}))
    await asyncio.sleep(0.05)
    assert global_manager._has_pending_command(ws)

    release.set()
    await work
    assert not global_manager._has_pending_command(ws)


@pytest.mark.asyncio
async def test_scheduled_close_uses_the_shared_timeout_constant(manager, monkeypatch):
    monkeypatch.setattr(ws_module, "RESTART_CLOSE_TIMEOUT_SECONDS", 0.05)
    await manager.close_for_restart()
    stuck = FakeSocket(hang_close=True)

    manager.register_connection(stuck, 1)
    await asyncio.sleep(0.3)

    assert manager.connection_count() == 0


def test_close_timeout_default_is_the_shared_constant():
    import inspect

    default = inspect.signature(ConnectionManager.close_for_restart).parameters[
        "close_timeout_seconds"
    ]
    assert default.default == ws_module.RESTART_CLOSE_TIMEOUT_SECONDS


class _StateSocket:
    def __init__(self, state) -> None:
        self.application_state = state


def test_receive_after_our_own_close_is_not_an_error(manager):
    gone = _StateSocket(WebSocketState.DISCONNECTED)
    live = _StateSocket(WebSocketState.CONNECTED)

    assert manager.was_closed_by_server(gone, RuntimeError("receive after close"))
    assert not manager.was_closed_by_server(live, RuntimeError("real fault"))
    assert not manager.was_closed_by_server(gone, ConnectionError("reset"))