Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

25.0 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock

import pytest
from anyio import BrokenResourceError, ClosedResourceError

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from xagent.web.api import public_chat_access
from xagent.web.api import websocket as websocket_api
from xagent.web.api.websocket import (
    ConnectionManager,
    _with_current_task_control_state,
)
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.user import User


class _BlockingWebSocket:
    query_params = SimpleNamespace(getlist=lambda key: [])

    def __init__(self) -> None:
        self.receive_started = asyncio.Event()
        self.accepted = False
        self.closed: tuple[int, str] | None = None
        self.messages: list[str] = []

    async def accept(self) -> None:
        self.accepted = True

    async def close(self, *, code: int, reason: str) -> None:
        self.closed = (code, reason)

    async def send_text(self, message: str) -> None:
        self.messages.append(message)

    async def receive_text(self) -> str:
        self.receive_started.set()
        await asyncio.Future()
        raise AssertionError("unreachable")


class _ClosedWebSocket:
    def __init__(self, error_type: type[Exception]) -> None:
        self._error_type = error_type

    async def send_text(self, message: str) -> None:
        raise self._error_type


class _RejectIfReceivedWebSocket(_BlockingWebSocket):
    async def receive_text(self) -> str:
        self.receive_started.set()
        raise AssertionError("foreign task socket entered the receive loop")


class _RecordingWebSocket:
    def __init__(self) -> None:
        self.messages: list[str] = []

    async def send_text(self, message: str) -> None:
        self.messages.append(message)


class _BlockingSendWebSocket(_RecordingWebSocket):
    def __init__(self) -> None:
        super().__init__()
        self.send_started = asyncio.Event()
        self.release_send = asyncio.Event()

    async def send_text(self, message: str) -> None:
        self.send_started.set()
        await self.release_send.wait()
        await super().send_text(message)


@pytest.fixture()
def current_task(tmp_path) -> Task:
    init_db(db_url=f"sqlite:///{tmp_path / 'task-state-events.db'}")
    db = next(get_db())
    try:
        user = User(username="event-user", password_hash="hash", is_admin=False)
        db.add(user)
        db.commit()
        task = Task(
            user_id=user.id,
            title="Event state",
            description="Event state",
            status=TaskStatus.RUNNING,
            execution_mode="auto",
            run_id="run-current",
            state_version=7,
            control_state="running",
        )
        db.add(task)
        db.commit()
        db.refresh(task)
        db.expunge(task)
        yield task
    finally:
        db.close()
        Base.metadata.drop_all(bind=get_engine())


@pytest.mark.asyncio
async def test_late_state_event_is_rewritten_to_current_snapshot(current_task) -> None:
    event = await _with_current_task_control_state(
        {
            "type": "task_paused",
            "task_id": int(current_task.id),
            "status": "paused",
        }
    )

    assert event["type"] == "task_paused"
    assert event["run_id"] == "run-current"
    assert event["state_version"] == 7
    assert event["control_state"] == "running"
    assert event["status"] == "running"


@pytest.mark.asyncio
async def test_task_info_trace_gets_versioned_state_tuple(current_task) -> None:
    event = await _with_current_task_control_state(
        {
            "type": "trace_event",
            "event_type": "task_info",
            "task_id": int(current_task.id),
            "data": {"id": int(current_task.id), "status": "paused"},
        }
    )

    assert event["state_version"] == 7
    assert event["data"] == {
        "id": int(current_task.id),
        "status": "running",
        "run_id": "run-current",
        "state_version": 7,
        "control_state": "running",
    }


@pytest.mark.asyncio
async def test_producer_snapshot_is_not_relabelled_as_a_newer_run(current_task) -> None:
    event = await _with_current_task_control_state(
        {
            "type": "task_completed",
            "task_id": int(current_task.id),
            "run_id": "run-old",
            "state_version": 5,
            "control_state": "completed",
            "status": "completed",
            "result": "old result",
        }
    )

    assert event["run_id"] == "run-old"
    assert event["state_version"] == 5
    assert event["control_state"] == "completed"
    assert event["status"] == "completed"


@pytest.mark.asyncio
async def test_boolean_state_version_is_replaced_with_current_snapshot(
    current_task,
) -> None:
    event = await _with_current_task_control_state(
        {
            "type": "task_completed",
            "task_id": int(current_task.id),
            "run_id": "run-old",
            "state_version": True,
            "control_state": "completed",
            "status": "completed",
        }
    )

    assert event["run_id"] == "run-current"
    assert event["state_version"] == 7
    assert event["control_state"] == "running"
    assert event["status"] == "running"


@pytest.mark.asyncio
async def test_websocket_endpoint_disconnects_when_cancelled(
    current_task: Task,
    monkeypatch,
) -> None:
    db = next(get_db())
    task = db.query(Task).filter(Task.id == int(current_task.id)).one()
    task_id = int(current_task.id)
    owner_id = int(task.user_id)
    db.close()
    websocket = _BlockingWebSocket()
    connection_manager = ConnectionManager()
    monkeypatch.setattr(websocket_api, "manager", connection_manager)
    monkeypatch.setattr(
        websocket_api,
        "get_authenticated_user",
        AsyncMock(return_value=SimpleNamespace(id=owner_id, is_admin=False)),
    )
    monkeypatch.setattr(websocket_api, "handle_status_request", AsyncMock())

    endpoint = asyncio.create_task(
        websocket_api.websocket_chat_endpoint(websocket, task_id, "token")
    )
    await websocket.receive_started.wait()

    assert connection_manager.active_connections[task_id] == [websocket]

    endpoint.cancel()
    with pytest.raises(asyncio.CancelledError):
        await endpoint

    assert task_id not in connection_manager.active_connections


@pytest.mark.asyncio
@pytest.mark.parametrize("actor_kind", ["owner", "admin"])
async def test_private_websocket_registers_only_authorized_task_audiences(
    current_task: Task,
    monkeypatch,
    actor_kind: str,
) -> None:
    db = next(get_db())
    task = db.query(Task).filter(Task.id == int(current_task.id)).one()
    if actor_kind == "admin":
        actor = User(username="endpoint-admin", password_hash="hash", is_admin=True)
        db.add(actor)
        db.commit()
        actor_id = int(actor.id)
    else:
        actor_id = int(task.user_id)
    db.close()

    websocket = _BlockingWebSocket()
    connection_manager = ConnectionManager()
    monkeypatch.setattr(websocket_api, "manager", connection_manager)
    monkeypatch.setattr(
        websocket_api,
        "get_authenticated_user",
        AsyncMock(
            return_value=SimpleNamespace(
                id=actor_id,
                is_admin=actor_kind == "admin",
            )
        ),
    )
    monkeypatch.setattr(websocket_api, "handle_status_request", AsyncMock())

    endpoint = asyncio.create_task(
        websocket_api.websocket_chat_endpoint(websocket, int(current_task.id), "token")
    )
    await websocket.receive_started.wait()

    assert websocket.accepted is True
    assert websocket.closed is None
    assert connection_manager.connections_for_task(int(current_task.id)) == [websocket]
    await connection_manager.broadcast_to_task(
        {"type": "authorized-private-event"},
        int(current_task.id),
    )
    assert json.loads(websocket.messages[-1])["type"] == "authorized-private-event"

    endpoint.cancel()
    with pytest.raises(asyncio.CancelledError):
        await endpoint


@pytest.mark.asyncio
async def test_private_websocket_rejects_foreign_task_before_registration_or_receive(
    current_task: Task,
    monkeypatch,
) -> None:
    db = next(get_db())
    intruder = User(username="endpoint-intruder", password_hash="hash", is_admin=False)
    db.add(intruder)
    db.commit()
    intruder_id = int(intruder.id)
    db.close()

    websocket = _RejectIfReceivedWebSocket()
    connection_manager = ConnectionManager()
    monkeypatch.setattr(websocket_api, "manager", connection_manager)
    monkeypatch.setattr(
        websocket_api,
        "get_authenticated_user",
        AsyncMock(return_value=SimpleNamespace(id=intruder_id, is_admin=False)),
    )
    status = AsyncMock()
    monkeypatch.setattr(websocket_api, "handle_status_request", status)

    await websocket_api.websocket_chat_endpoint(
        websocket,
        int(current_task.id),
        "token",
    )

    assert websocket.accepted is True
    assert websocket.closed == (4003, "Task is no longer available.")
    assert websocket.receive_started.is_set() is False
    assert connection_manager.active_connections == {}
    status.assert_not_awaited()
    await connection_manager.broadcast_to_task(
        {"type": "must-not-reach-foreign"},
        int(current_task.id),
    )
    assert websocket.messages == []


@pytest.mark.asyncio
async def test_missing_private_task_stays_unregistered_until_recovery_moves_connection(
    current_task: Task,
    monkeypatch,
) -> None:
    missing_task_id = int(current_task.id) + 424242
    replacement_task_id = int(current_task.id)
    websocket = _BlockingWebSocket()
    connection_manager = ConnectionManager()

    async def recover_missing_task(_websocket, task_id, _message_data) -> None:
        assert task_id == missing_task_id
        assert connection_manager.active_connections == {}
        connection_manager.move_connection(websocket, replacement_task_id)
        await connection_manager.broadcast_to_task(
            {"type": "replacement-task-event"},
            replacement_task_id,
        )
        assert json.loads(websocket.messages[-1])["type"] == "replacement-task-event"

    websocket.receive_text = AsyncMock(
        side_effect=[json.dumps({"type": "chat"}), asyncio.CancelledError()]
    )
    monkeypatch.setattr(websocket_api, "manager", connection_manager)
    monkeypatch.setattr(
        websocket_api,
        "get_authenticated_user",
        AsyncMock(
            return_value=SimpleNamespace(id=int(current_task.user_id), is_admin=False)
        ),
    )
    monkeypatch.setattr(websocket_api, "handle_status_request", AsyncMock())
    monkeypatch.setattr(websocket_api, "handle_chat_message", recover_missing_task)

    with pytest.raises(asyncio.CancelledError):
        await websocket_api.websocket_chat_endpoint(websocket, missing_task_id, "token")

    assert websocket.accepted is True
    assert connection_manager.active_connections == {}


@pytest.mark.asyncio
async def test_websocket_endpoint_disconnects_moved_connection_when_cancelled(
    current_task: Task,
    monkeypatch,
) -> None:
    initial_task_id = int(current_task.id) + 424242
    moved_task_id = int(current_task.id)
    websocket = _BlockingWebSocket()
    connection_manager = ConnectionManager()

    async def move_connection_during_initial_status(*args) -> None:
        connection_manager.move_connection(websocket, moved_task_id)

    monkeypatch.setattr(websocket_api, "manager", connection_manager)
    monkeypatch.setattr(
        websocket_api,
        "get_authenticated_user",
        AsyncMock(
            return_value=SimpleNamespace(
                id=int(current_task.user_id),
                is_admin=False,
            )
        ),
    )
    monkeypatch.setattr(
        websocket_api,
        "handle_status_request",
        AsyncMock(side_effect=move_connection_during_initial_status),
    )

    endpoint = asyncio.create_task(
        websocket_api.websocket_chat_endpoint(websocket, initial_task_id, "token")
    )
    await websocket.receive_started.wait()

    assert connection_manager.active_connections == {moved_task_id: [websocket]}

    endpoint.cancel()
    with pytest.raises(asyncio.CancelledError):
        await endpoint

    assert connection_manager.active_connections == {}


@pytest.mark.asyncio
@pytest.mark.parametrize("endpoint_kind", ["public", "share"])
async def test_public_websocket_endpoint_disconnects_reassigned_connection_when_cancelled(
    monkeypatch,
    endpoint_kind: str,
) -> None:
    initial_task_id = 42
    moved_task_id = 99
    websocket = _BlockingWebSocket()
    connection_manager = ConnectionManager()
    principal = websocket_api.WebSocketPrincipal(id=7, is_admin=False)

    async def reassign_during_initial_status(*args) -> None:
        connection_manager.register_connection(websocket, moved_task_id)

    monkeypatch.setattr(public_chat_access, "manager", connection_manager)
    monkeypatch.setattr(
        public_chat_access,
        "_authorize_public_chat_websocket",
        AsyncMock(return_value=principal),
    )
    monkeypatch.setattr(
        public_chat_access,
        "_authorize_share_chat_websocket",
        AsyncMock(return_value=principal),
    )
    monkeypatch.setattr(
        public_chat_access,
        "handle_status_request",
        AsyncMock(side_effect=reassign_during_initial_status),
    )

    if endpoint_kind == "public":
        endpoint = asyncio.create_task(
            public_chat_access.public_chat_websocket_endpoint(
                websocket=websocket,
                task_id=initial_task_id,
                token="token",
                expected_auth_mode="widget",
            )
        )
    else:
        endpoint = asyncio.create_task(
            public_chat_access.share_chat_websocket_endpoint(
                websocket=websocket,
                task_id=initial_task_id,
                token="token",
            )
        )

    await websocket.receive_started.wait()

    assert connection_manager.active_connections == {moved_task_id: [websocket]}

    endpoint.cancel()
    with pytest.raises(asyncio.CancelledError):
        await endpoint

    assert connection_manager.active_connections == {}
    assert connection_manager._connection_task_ids == {}


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "error_type",
    [ClosedResourceError, BrokenResourceError],
)
async def test_broadcast_skips_closed_connection_and_reaches_live_connection(
    error_type: type[Exception],
) -> None:
    task_id = 42
    closed_websocket = _ClosedWebSocket(error_type)
    live_websocket = _RecordingWebSocket()
    connection_manager = ConnectionManager()
    connection_manager.register_connection(closed_websocket, task_id)
    connection_manager.register_connection(live_websocket, task_id)

    message = {"type": "diagnostic", "message": "still live"}
    await connection_manager.broadcast_to_task(message, task_id)

    assert live_websocket.messages == [json.dumps(message)]
    assert connection_manager.active_connections[task_id] == [live_websocket]


@pytest.mark.asyncio
async def test_broadcast_serializes_shared_payload_once(monkeypatch) -> None:
    task_id = 42
    first_websocket = _RecordingWebSocket()
    second_websocket = _RecordingWebSocket()
    connection_manager = ConnectionManager()
    connection_manager.register_connection(first_websocket, task_id)
    connection_manager.register_connection(second_websocket, task_id)
    original_dumps = json.dumps
    serialized_values: list[dict] = []

    def counted_dumps(value, *args, **kwargs):
        serialized_values.append(value)
        return original_dumps(value, *args, **kwargs)

    monkeypatch.setattr(websocket_api.json, "dumps", counted_dumps)

    message = {"type": "diagnostic", "message": "shared"}
    await connection_manager.broadcast_to_task(message, task_id)

    encoded_message = original_dumps(message)
    assert serialized_values == [message]
    assert first_websocket.messages == [encoded_message]
    assert second_websocket.messages == [encoded_message]


@pytest.mark.asyncio
async def test_broadcast_reraises_unexpected_connection_error() -> None:
    task_id = 42
    failed_websocket = _ClosedWebSocket(ValueError)
    connection_manager = ConnectionManager()
    connection_manager.register_connection(failed_websocket, task_id)

    with pytest.raises(ValueError):
        await connection_manager.broadcast_to_task({"type": "diagnostic"}, task_id)

    assert task_id not in connection_manager.active_connections


@pytest.mark.asyncio
async def test_broadcast_rechecks_membership_after_message_enrichment(
    monkeypatch,
) -> None:
    task_id = 42
    websocket = _RecordingWebSocket()
    connection_manager = ConnectionManager()
    connection_manager.register_connection(websocket, task_id)

    async def detach_during_enrichment(message, **kwargs):
        connection_manager.detach_task_connections(task_id)
        return message

    monkeypatch.setattr(
        websocket_api,
        "_with_current_task_control_state",
        detach_during_enrichment,
    )

    await connection_manager.broadcast_to_task({"type": "diagnostic"}, task_id)

    assert websocket.messages == []
    assert connection_manager.active_connections == {}


@pytest.mark.asyncio
async def test_broadcast_includes_connection_registered_during_enrichment(
    monkeypatch,
) -> None:
    task_id = 42
    existing_websocket = _RecordingWebSocket()
    new_websocket = _RecordingWebSocket()
    connection_manager = ConnectionManager()
    connection_manager.register_connection(existing_websocket, task_id)

    async def connect_during_enrichment(message, **kwargs):
        connection_manager.register_connection(new_websocket, task_id)
        return message

    monkeypatch.setattr(
        websocket_api,
        "_with_current_task_control_state",
        connect_during_enrichment,
    )

    message = {"type": "diagnostic"}
    await connection_manager.broadcast_to_task(message, task_id)

    encoded_message = json.dumps(message)
    assert existing_websocket.messages == [encoded_message]
    assert new_websocket.messages == [encoded_message]


@pytest.mark.asyncio
async def test_broadcast_skips_connection_moved_during_fanout() -> None:
    task_id = 42
    moved_task_id = 99
    blocking_websocket = _BlockingSendWebSocket()
    moved_websocket = _RecordingWebSocket()
    connection_manager = ConnectionManager()
    connection_manager.register_connection(blocking_websocket, task_id)
    connection_manager.register_connection(moved_websocket, task_id)

    broadcast = asyncio.create_task(
        connection_manager.broadcast_to_task({"type": "diagnostic"}, task_id)
    )
    await blocking_websocket.send_started.wait()
    connection_manager.move_connection(moved_websocket, moved_task_id)
    blocking_websocket.release_send.set()
    await broadcast

    assert moved_websocket.messages == []
    assert connection_manager.active_connections == {
        task_id: [blocking_websocket],
        moved_task_id: [moved_websocket],
    }


def test_detach_task_connections_removes_forward_and_reverse_membership() -> None:
    task_id = 42
    other_task_id = 99
    first_websocket = _RecordingWebSocket()
    second_websocket = _RecordingWebSocket()
    other_websocket = _RecordingWebSocket()
    connection_manager = ConnectionManager()
    connection_manager.register_connection(first_websocket, task_id)
    connection_manager.register_connection(second_websocket, task_id)
    connection_manager.register_connection(other_websocket, other_task_id)

    detached = connection_manager.detach_task_connections(task_id)

    assert detached == [first_websocket, second_websocket]
    assert connection_manager.active_connections == {other_task_id: [other_websocket]}
    assert connection_manager._connection_task_ids == {other_websocket: other_task_id}


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "status", [TaskStatus.RUNNING, TaskStatus.COMPLETED, TaskStatus.WAITING_FOR_USER]
)
async def test_shared_reconciliation_reads_persisted_state_and_stops(
    current_task, monkeypatch, status
):
    import threading

    from xagent.web.models.database import get_session_local
    from xagent.web.services import task_event_bridge, task_stream_snapshot

    with get_session_local()() as db:
        task = db.get(Task, current_task.id)
        task.status = status
        task.control_state = status.value
        task.output = "complete durable result"
        task.lease_attempt_id = "attempt-current"
        db.commit()
    monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
    monkeypatch.setattr(
        task_event_bridge, "_bridge", SimpleNamespace(discard_recipient=lambda ws: None)
    )
    loop_thread = threading.get_ident()
    read_threads = []
    load = task_stream_snapshot.load_task_stream_snapshots

    def load_off_loop(task_ids):
        read_threads.append(threading.get_ident())
        return load(task_ids)

    monkeypatch.setattr(
        task_stream_snapshot, "load_task_stream_snapshots", load_off_loop
    )
    received = asyncio.Event()

    class Socket(_RecordingWebSocket):
        async def send_text(self, message):
            await super().send_text(message)
            received.set()

    socket = Socket()
    manager = ConnectionManager()
    manager.register_connection(socket, current_task.id)
    writer = manager._writers[socket]
    manager.start_stream_reconciliation()
    reconciler = manager._stream_reconciler
    try:
        await asyncio.wait_for(received.wait(), timeout=DB_PROGRESS_TIMEOUT)
        frame = json.loads(socket.messages[0])
        assert frame["type"] == "task_stream_snapshot"
        assert frame["task_id"] == current_task.id
        assert frame["run_id"] == "run-current"
        assert frame["state_version"] == 7
        assert frame["status"] == status.value
        assert frame["lease_attempt_id"] == "attempt-current"
        assert frame["output"] == (
            "complete durable result" if status == TaskStatus.COMPLETED else None
        )
        if status == TaskStatus.WAITING_FOR_USER:
            assert "question" in frame and "interactions" in frame
        assert read_threads and loop_thread not in read_threads
    finally:
        await asyncio.wait_for(
            manager.stop_stream_reconciliation(), timeout=DB_PROGRESS_TIMEOUT
        )
        manager.disconnect(socket)
        await asyncio.wait_for(
            asyncio.gather(writer.task, return_exceptions=True), timeout=2
        )
    assert reconciler.done()
    assert manager._stream_reconciler is None
    assert not manager.active_connections


@pytest.mark.asyncio
async def test_shared_delivery_logs_connection_without_writer(monkeypatch, caplog):
    monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "false")
    manager = ConnectionManager()
    socket = _RecordingWebSocket()
    manager.register_connection(socket, 42)
    await manager.deliver_shared_event(
        {"type": "diagnostic", "content": "private content"}, 42
    )
    assert "connection has no writer task_id=42" in caplog.text
    assert "private content" not in caplog.text
    manager.disconnect(socket)


@pytest.mark.asyncio
@pytest.mark.parametrize("started", [False, True])
async def test_manager_disconnect_reentry_cleans_writer_and_both_origin_registries(
    monkeypatch, started
):
    from xagent.web.services import task_event_bridge

    monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
    monkeypatch.setenv("XAGENT_REDIS_URL", "redis://localhost:6379/0")
    bridge = task_event_bridge.TaskEventBridge()
    bridge.ready.set()
    monkeypatch.setattr(task_event_bridge, "_bridge", bridge)
    origins = websocket_api._CommandOriginRegistry()
    monkeypatch.setattr(websocket_api, "_command_origins", origins)
    manager = ConnectionManager()
    disconnect = Mock(wraps=manager.disconnect)
    monkeypatch.setattr(manager, "disconnect", disconnect)
    socket = _BlockingSendWebSocket()
    manager.register_connection(socket, 42)
    writer = manager._writers[socket]
    origins.register("command-1", socket, 42)
    origin = bridge.register_origin(42, "command-1", AsyncMock(), recipient=socket)
    acknowledgement = writer.enqueue("pending", acknowledge=True)
    try:
        if started:
            await asyncio.wait_for(socket.send_started.wait(), timeout=1)
        manager.disconnect(socket)
        await asyncio.wait_for(
            asyncio.gather(writer.task, return_exceptions=True), timeout=2
        )
        with pytest.raises(ConnectionError):
            await asyncio.wait_for(acknowledgement, timeout=1)
        # stop() re-enters the real manager once; the writer has already been
        # popped, so this must terminate and leave both origin registries clean.
        assert disconnect.call_count == 2
        assert writer.closed and writer.queue.empty()
        assert not manager._writers
        assert not manager._connection_task_ids
        assert not manager.active_connections
        assert not origins.has("command-1", 42)
        assert origin not in bridge._origins
        manager.disconnect(socket)
        assert disconnect.call_count == 3
    finally:
        manager.disconnect(socket)
        await asyncio.gather(writer.task, return_exceptions=True)
        await bridge.close()