Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

7.4 KB
Raw Blame
"""Standby activation keeps the event loop serving while blocking work runs."""

from __future__ import annotations

import asyncio
import logging
import threading
import time
from unittest.mock import Mock

import pytest
from fastapi import FastAPI

from xagent.web import app as app_module
from xagent.web.services.standby import StandbyController, StandbyPhase
from xagent.web.startup_admission import (
    register_host_startup_admission,
    run_host_startup_admissions,
)

_BLOCK_SECONDS = 5.0


async def _wait_for(event: threading.Event) -> None:
    deadline = time.monotonic() + _BLOCK_SECONDS
    while not event.is_set():
        assert time.monotonic() < deadline, "blocking work never started"
        await asyncio.sleep(0.005)


def _patch_rest_of_admission(monkeypatch: pytest.MonkeyPatch) -> None:
    async def no_admissions(_app) -> None:
        return None

    monkeypatch.setattr(app_module, "run_host_startup_admissions", no_admissions)
    monkeypatch.setattr(
        "xagent.web.services.trace_database.get_trace_database_runtime", lambda: None
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("bound", [True, False])
async def test_database_init_does_not_block_the_event_loop(
    monkeypatch: pytest.MonkeyPatch, bound: bool
) -> None:
    started = threading.Event()
    release = threading.Event()
    ran_on: list[int] = []

    def blocking_init() -> None:
        ran_on.append(threading.get_ident())
        started.set()
        # Only a loop that keeps running can ever set ``release``.
        assert release.wait(_BLOCK_SECONDS), "event loop was blocked by init"

    monkeypatch.setattr(app_module, "complete_standby_db_init", blocking_init)
    monkeypatch.setattr(app_module, "init_db", blocking_init)
    _patch_rest_of_admission(monkeypatch)

    task = asyncio.ensure_future(
        app_module._admit_database_and_host(FastAPI(), bound=bound)
    )
    await _wait_for(started)
    ticks = 0
    for _ in range(5):
        await asyncio.sleep(0.01)
        ticks += 1
    assert ticks == 5 and not task.done()
    release.set()
    await asyncio.wait_for(task, _BLOCK_SECONDS)
    assert ran_on[0] != threading.get_ident()


@pytest.mark.asyncio
async def test_cancelled_activation_waits_for_database_init_to_finish(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    started = threading.Event()
    release = threading.Event()
    finished = threading.Event()

    def init() -> None:
        started.set()
        release.wait(_BLOCK_SECONDS)
        finished.set()

    monkeypatch.setattr(app_module, "complete_standby_db_init", init)
    _patch_rest_of_admission(monkeypatch)

    task = asyncio.ensure_future(
        app_module._admit_database_and_host(FastAPI(), bound=True)
    )
    await _wait_for(started)
    task.cancel()
    await asyncio.sleep(0.05)
    # The cancel must not abandon a half-done migration/seed.
    assert not task.done() and not finished.is_set()
    release.set()
    with pytest.raises(asyncio.CancelledError):
        await asyncio.wait_for(task, _BLOCK_SECONDS)
    assert finished.is_set()


@pytest.mark.asyncio
async def test_default_path_invokes_every_callback_on_the_event_loop() -> None:
    app = FastAPI()
    events: list[tuple[str, bool]] = []
    main = threading.get_ident()

    def factory():
        # A decorator/factory style callable: it needs the running loop while
        # building the awaitable it returns.
        asyncio.get_running_loop()

        async def run() -> None:
            events.append(("factory result", threading.get_ident() == main))

        return run()

    def plain() -> None:
        events.append(("plain", threading.get_ident() == main))

    async def async_check() -> None:
        events.append(("async", threading.get_ident() == main))

    register_host_startup_admission(app, factory)
    register_host_startup_admission(app, plain)
    register_host_startup_admission(app, async_check)
    register_host_startup_admission(app, Mock())
    await run_host_startup_admissions(app)
    assert events == [("factory result", True), ("plain", True), ("async", True)]


@pytest.mark.asyncio
async def test_opt_in_callback_runs_off_the_loop_in_registration_order() -> None:
    app = FastAPI()
    events: list[tuple[str, bool]] = []
    main = threading.get_ident()

    def threaded() -> None:
        events.append(("threaded", threading.get_ident() != main))

    async def async_check() -> None:
        events.append(("async", threading.get_ident() == main))

    register_host_startup_admission(app, async_check)
    register_host_startup_admission(app, threaded, run_in_thread=True)
    register_host_startup_admission(app, async_check)
    await run_host_startup_admissions(app)
    assert events == [("async", True), ("threaded", True), ("async", True)]


@pytest.mark.asyncio
async def test_opt_in_callback_returning_an_awaitable_is_rejected_clearly() -> None:
    app = FastAPI()

    async def coroutine_function() -> None: ...

    register_host_startup_admission(app, coroutine_function, run_in_thread=True)
    with pytest.raises(TypeError, match="run_in_thread=True"):
        await run_host_startup_admissions(app)


@pytest.mark.asyncio
async def test_sync_admission_rejection_propagates_unchanged() -> None:
    app = FastAPI()

    def reject() -> None:
        raise RuntimeError("catalog refused")

    register_host_startup_admission(app, reject, run_in_thread=True)
    with pytest.raises(RuntimeError, match="catalog refused"):
        await run_host_startup_admissions(app)


@pytest.mark.asyncio
async def test_activation_logs_step_duration_and_loop_lag(
    caplog: pytest.LogCaptureFixture,
) -> None:
    controller = StandbyController()

    async def stalls_the_loop() -> None:
        await asyncio.sleep(0.12)  # let the probe take a baseline tick
        time.sleep(0.3)  # synchronous work on the loop

    controller.begin()
    controller.defer("stalling step", stalls_the_loop)
    with caplog.at_level(logging.INFO, logger="xagent.web.services.standby"):
        await controller.activate()

    line = next(
        r.getMessage() for r in caplog.records if "stalling step took" in r.getMessage()
    )
    lag = float(line.rsplit("lag ", 1)[1].rstrip("s)"))
    assert lag >= 0.2, line
    assert any("activation complete in" in r.getMessage() for r in caplog.records)


@pytest.mark.asyncio
async def test_cancel_at_the_end_of_the_last_step_still_closes_activation() -> None:
    """A cancel landing after the final step must not strand the phase.

    The last step has no suspension of its own, so a cancel requested inside it
    is delivered at the first suspension after it: whatever activation does
    next. That point must stay inside the failure boundary, or the phase stays
    ``ACTIVATING`` and the failure hooks (which reseal admission and stop the
    started paths) never run.
    """
    controller = StandbyController()
    hooks: list[str] = []
    holder: dict[str, asyncio.Task[None]] = {}

    async def last_step_requests_cancel() -> None:
        holder["task"].cancel()  # delivered at the next suspension point

    async def hook() -> None:
        hooks.append("failure hook")

    controller.begin()
    controller.on_failure(hook)
    controller.defer("last step", last_step_requests_cancel)
    holder["task"] = asyncio.ensure_future(controller.activate())
    with pytest.raises(asyncio.CancelledError):
        await holder["task"]

    assert controller.phase is StandbyPhase.FAILED
    assert hooks == ["failure hook"]