Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

15.0 KB
Raw Blame
"""Standby through the real startup and shutdown handlers of the web app.

Pins what ``test_standby.py`` cannot: that ``startup_event`` defers every write,
claim path and singleton (not just the database half), that activation runs
them in a normal start's order, that a late activation failure undoes what the
early steps opened, that a standby never cleans up shared sandboxes, and that
its database binding is read-only until activation.
"""

from __future__ import annotations

import asyncio
import importlib
import importlib.abc
import importlib.util
import sys
import threading
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock

import pytest
import sqlalchemy as sa

from tests.integration.test_auto_migration_startup import (  # noqa: F401
    _patch_channel_modules_disabled,
    unregister_startup_task_extensions,
)
from xagent.config import STANDBY as STANDBY_ENV
from xagent.web.models import database as database_module
from xagent.web.services import task_execution
from xagent.web.services.release_hold import release_hold
from xagent.web.services.standby import StandbyPhase, standby

app_module = importlib.import_module("xagent.web.app")


@pytest.fixture(autouse=True)
def clean_state():
    standby.reset()
    yield
    standby.reset()
    release_hold.reopen()


class _Manager:
    async def initialize(self) -> None:
        return None

    async def list_skills(self) -> list[str]:
        return []

    async def list_templates(self) -> list[str]:
        return []


class _Sandbox:
    def __init__(self, events: list[str]) -> None:
        self._events = events
        self.cleanup = AsyncMock(side_effect=lambda: events.append("sandbox_cleanup"))

    async def _resolve_backend_probe(self) -> bool:
        return False

    async def warmup(self) -> None:
        self._events.append("sandbox_warmup")


def _patch_startup(monkeypatch: pytest.MonkeyPatch, events: list[str]) -> _Sandbox:
    """Record every startup side effect that standby must postpone."""
    _patch_channel_modules_disabled(monkeypatch)
    sandbox = _Sandbox(events)
    monkeypatch.setenv(STANDBY_ENV, "1")
    monkeypatch.setenv("LANCEDB_AUTO_MIGRATE", "false")
    monkeypatch.setattr(app_module, "init_db", lambda: events.append("init_db"))
    monkeypatch.setattr(
        app_module, "init_db_standby", lambda: events.append("bind_readonly")
    )
    monkeypatch.setattr(
        app_module,
        "complete_standby_db_init",
        lambda: events.append("init_db_deferred"),
    )

    async def admissions(_app) -> None:
        events.append("admissions")

    monkeypatch.setattr(app_module, "run_host_startup_admissions", admissions)
    monkeypatch.setattr(
        "xagent.web.services.trace_database.get_trace_database_runtime",
        lambda: events.append("trace_db"),
    )
    monkeypatch.setattr(
        app_module, "register_local_browser_runtime", lambda: events.append("browser")
    )
    for name in (
        "start_file_storage_startup_sync_task",
        "start_trigger_dispatcher_task",
        "start_task_lease_recovery_task",
        "start_task_auto_resume_task",
        "start_uploaded_file_recovery_task",
        "start_orphan_upload_gc_task",
        "start_retention_purge_task",
        "start_task_cleanup_retry_task",
        "start_composio_reconcile_task",
        "start_composio_missing_account_task",
        "start_runtime_performance_monitor",
        "start_app_process_diagnostics",
    ):
        monkeypatch.setattr(
            app_module, name, lambda _a, n=name, **_kw: events.append(n)
        )
    monkeypatch.setattr(
        app_module,
        "start_temp_file_cleanup_task",
        lambda _a: events.append("start_temp_file_cleanup_task"),
    )
    monkeypatch.setattr(
        "xagent.web.services.task_command_transport.start_task_command_dispatcher",
        lambda _e: events.append("command_dispatcher"),
    )
    monkeypatch.setattr("xagent.skills.utils.create_skill_manager", lambda: _Manager())
    monkeypatch.setattr(
        "xagent.templates.utils.create_template_manager", lambda: _Manager()
    )
    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.admit_memory_storage_at_startup",
        lambda: events.append("memory_admission"),
    )
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager", lambda: sandbox
    )

    async def readiness(_mgr) -> None:
        events.append("sandbox_readiness")

    monkeypatch.setattr(
        "xagent.web.sandbox_manager.check_sandbox_static_readiness", readiness
    )
    monkeypatch.setattr(
        "xagent.providers.vector_store.lancedb.get_connection_from_env",
        lambda: SimpleNamespace(),
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.LanceDB.schema_manager.check_table_needs_migration",
        lambda _c, _t: False,
    )
    return sandbox


@pytest.mark.usefixtures("unregister_startup_task_extensions")
@pytest.mark.asyncio
async def test_startup_event_defers_every_write_and_singleton_until_activation(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[str] = []
    sandbox = _patch_startup(monkeypatch, events)
    monkeypatch.setattr(
        "xagent.web.app.get_shared_task_execution_enabled", lambda: False
    )

    await app_module.startup_event()

    # Reads and warmup only; closed, sealed hold.
    assert events == [
        "bind_readonly",
        "browser",
        "sandbox_readiness",
        "start_runtime_performance_monitor",
        "start_app_process_diagnostics",
    ]
    assert release_hold.closed and release_hold.sealed
    assert standby.phase is StandbyPhase.STANDBY
    sandbox.cleanup.assert_not_awaited()

    await standby.activate()
    after = events[5:]
    expected_order = [
        "init_db_deferred",
        "admissions",
        "trace_db",
        "start_task_lease_recovery_task",
        "memory_admission",
        "sandbox_cleanup",
        "sandbox_warmup",
        "command_dispatcher",
        "start_composio_reconcile_task",
    ]
    positions = [after.index(name) for name in expected_order]
    assert positions == sorted(positions), after
    assert "init_db" not in events
    assert not release_hold.closed and standby.phase is StandbyPhase.ACTIVE


_CHANNEL_MODULES = {
    "xagent.web.channels.feishu.bot": "get_feishu_channel",
    "xagent.web.channels.slack.bot": "get_slack_channel",
    "xagent.web.channels.telegram.bot": "get_telegram_channel",
}


class _RecordingChannelFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader):
    """Serve fake, enabled channel modules and record imports and starts."""

    def __init__(self, events: list[str]) -> None:
        self._events = events
        # The loop each import would bind, as lark_oapi.ws.client does at import.
        self.import_loops: list[asyncio.AbstractEventLoop | None] = []

    def find_spec(self, name, path=None, target=None):
        if name in _CHANNEL_MODULES:
            return importlib.util.spec_from_loader(name, self)
        return None

    def create_module(self, spec):
        return None

    def exec_module(self, module) -> None:
        name = module.__name__
        short = name.split(".")[-2]
        self._events.append(f"import_{short}")
        try:
            self.import_loops.append(asyncio.get_event_loop())
        except RuntimeError:
            self.import_loops.append(None)
        events = self._events

        class _Channel:
            enabled = True

            async def start(self) -> None:
                events.append(f"channel_start_{short}")

        channel = _Channel()
        setattr(module, _CHANNEL_MODULES[name], lambda: channel)


@pytest.mark.usefixtures("unregister_startup_task_extensions")
@pytest.mark.asyncio
async def test_standby_preloads_channel_modules_but_only_activation_starts_them(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[str] = []
    _patch_startup(monkeypatch, events)
    monkeypatch.setattr(
        "xagent.web.app.get_shared_task_execution_enabled", lambda: False
    )
    # Undo the stub modules _patch_startup installed; the finder serves fresh ones.
    for name in _CHANNEL_MODULES:
        package_name, _, attribute = name.rpartition(".")
        package = importlib.import_module(package_name)
        # The fake import rebinds the name on the parent package as well;
        # restoring sys.modules alone would leave that binding behind.
        previous = getattr(package, attribute, None)
        monkeypatch.setattr(package, attribute, previous, raising=False)
        monkeypatch.setitem(sys.modules, name, None)
        del sys.modules[name]
    for task_name in ("telegram_task", "feishu_task", "slack_task"):
        # Activation stores its channel tasks here; keep them off later tests.
        monkeypatch.setattr(app_module.app.state, task_name, None, raising=False)
    finder = _RecordingChannelFinder(events)
    monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])

    await app_module.startup_event()

    assert {"import_feishu", "import_slack", "import_telegram"} <= set(events)
    # lark_oapi binds its receive loop at import time: the preload must run on
    # the serving event loop, never in a worker thread with a loop of its own.
    running = asyncio.get_running_loop()
    assert finder.import_loops and all(loop is running for loop in finder.import_loops)
    assert not [e for e in events if e.startswith("channel_start_")]
    assert standby.phase is StandbyPhase.STANDBY

    await standby.activate()
    await asyncio.sleep(0)

    assert {
        "channel_start_feishu",
        "channel_start_slack",
        "channel_start_telegram",
    } <= set(events)
    # The modules were imported once, in standby, not again at activation.
    assert len([e for e in events if e.startswith("import_")]) == 3


@pytest.mark.usefixtures("unregister_startup_task_extensions")
@pytest.mark.asyncio
async def test_late_activation_failure_reseals_and_stops_started_loops(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    events: list[str] = []
    _patch_startup(monkeypatch, events)
    monkeypatch.setattr(
        "xagent.web.app.get_shared_task_execution_enabled", lambda: False
    )
    stopped: list[str] = []

    async def stop_loops() -> None:
        stopped.append("stopped")

    monkeypatch.setattr(app_module, "_stop_runtime_loops", stop_loops)

    monkeypatch.setattr(
        "xagent.web.dynamic_memory_store.admit_memory_storage_at_startup",
        lambda: (_ for _ in ()).throw(RuntimeError("memory admission failed")),
    )
    await app_module.startup_event()
    with pytest.raises(RuntimeError, match="memory admission failed"):
        await standby.activate()

    assert standby.phase is StandbyPhase.FAILED
    # "open admission" had already run: the failure hook undid it.
    assert "start_task_lease_recovery_task" in events
    assert release_hold.sealed and release_hold.closed
    assert stopped == ["stopped"]


@pytest.mark.asyncio
@pytest.mark.parametrize("started_as_standby", [True, False])
async def test_shutdown_cleans_sandboxes_only_when_it_owned_them(
    monkeypatch: pytest.MonkeyPatch, started_as_standby: bool
) -> None:
    events: list[str] = []
    sandbox = _Sandbox(events)
    monkeypatch.setattr(
        "xagent.web.sandbox_manager.get_sandbox_manager", lambda: sandbox
    )
    monkeypatch.setattr(app_module, "flush_langfuse", lambda: None)
    for name in (
        "stop_orphan_upload_gc_task",
        "stop_composio_reconcile_task",
        "stop_composio_missing_account_task",
        "stop_retention_purge_task",
        "stop_task_cleanup_retry_task",
        "stop_uploaded_file_recovery_task",
        "stop_task_auto_resume_task",
        "stop_task_lease_recovery_task",
        "stop_runtime_performance_monitor",
        "stop_app_process_diagnostics",
    ):
        monkeypatch.setattr(app_module, name, AsyncMock())
    # shutdown_event() leaves the process-wide background task manager refusing
    # work, and nothing in this process reopens it, so give it a fresh one:
    # later tests in this process still need the shared manager to accept work.
    monkeypatch.setattr(
        task_execution,
        "background_task_manager",
        task_execution.BackgroundTaskManager(),
    )
    if started_as_standby:
        standby.begin()

    await app_module.shutdown_event()

    if started_as_standby:
        sandbox.cleanup.assert_not_awaited()
    else:
        sandbox.cleanup.assert_awaited_once()


def _sqlite_at_older_revision(tmp_path: Path) -> str:
    from alembic.script import ScriptDirectory

    from xagent.db.migration import create_alembic_config

    url = f"sqlite:///{tmp_path / 'db.sqlite'}"
    engine = sa.create_engine(url)
    script = ScriptDirectory.from_config(create_alembic_config(engine))
    heads = set(script.get_heads())
    older = next(r.revision for r in script.walk_revisions() if r.revision not in heads)
    with engine.begin() as conn:
        conn.execute(sa.text("CREATE TABLE alembic_version (version_num VARCHAR(32))"))
        conn.execute(sa.text("INSERT INTO alembic_version VALUES (:r)"), {"r": older})
    engine.dispose()
    return url


def test_standby_database_is_read_only_until_activation(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    url = _sqlite_at_older_revision(tmp_path)
    monkeypatch.setattr(database_module, "_engine", None)
    monkeypatch.setattr(database_module, "_SessionLocal", None)
    monkeypatch.setattr(
        database_module, "_initialize_database_schema", lambda _engine: []
    )
    monkeypatch.setattr(
        "xagent.web.services.encryption_key_rotation.validate_stored_encryption_keys",
        lambda: None,
    )

    database_module.init_db_standby(url)
    try:
        with database_module.get_engine().begin() as conn:
            with pytest.raises(sa.exc.OperationalError, match="readonly"):
                conn.execute(sa.text("UPDATE alembic_version SET version_num = 'x'"))

        database_module.complete_standby_db_init()
        with database_module.get_engine().begin() as conn:
            conn.execute(sa.text("UPDATE alembic_version SET version_num = 'y'"))
    finally:
        database_module.get_engine().dispose()


@pytest.mark.asyncio
async def test_abort_stops_channels_and_the_shared_event_bridge(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    channel = asyncio.create_task(asyncio.sleep(60))
    monkeypatch.setattr(app_module.app.state, "slack_task", channel, raising=False)
    monkeypatch.setattr(app_module, "_stop_runtime_loops", AsyncMock())
    cleanup_stop = threading.Event()
    monkeypatch.setattr(
        app_module.app.state, "temp_file_cleanup_stop", cleanup_stop, raising=False
    )
    monkeypatch.setattr(app_module, "get_shared_task_execution_enabled", lambda: True)
    reconciliation = AsyncMock()
    bridge = AsyncMock()
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.stop_stream_reconciliation", reconciliation
    )
    monkeypatch.setattr(
        "xagent.web.services.task_event_bridge.stop_task_event_bridge", bridge
    )

    await app_module._abort_standby_activation()

    assert channel.cancelled()
    assert cleanup_stop.is_set()
    reconciliation.assert_awaited_once()
    bridge.assert_awaited_once()
    assert release_hold.sealed