Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

11.6 KB
Raw Blame
"""The backend, the agent worker and the Celery worker refuse a mismatched KB engine."""

from __future__ import annotations

import asyncio
import logging
import re
import subprocess
from pathlib import Path
from unittest.mock import AsyncMock, Mock

import lancedb
import pytest
from cryptography.fernet import Fernet

from xagent.core.tools.core.RAG_tools.core.exceptions import ConfigurationError
from xagent.core.tools.core.RAG_tools.storage.vector_backend import KB_ENGINE_RECORD


class _Started(Exception):
    pass


@pytest.fixture(autouse=True)
def milvus_record(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    monkeypatch.setenv("LANCEDB_DIR", str(tmp_path))
    monkeypatch.setenv("XAGENT_VECTOR_BACKEND", "lancedb")
    (tmp_path / KB_ENGINE_RECORD).write_text("milvus\n")


@pytest.mark.asyncio
async def test_backend_startup_refuses_before_touching_the_database(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from xagent.web import app

    initialize = AsyncMock()
    monkeypatch.setattr(app, "_initialize_database_and_admit_runtime", initialize)

    with pytest.raises(ConfigurationError, match="KB engine is milvus"):
        await app.startup_event()
    initialize.assert_not_called()


@pytest.mark.asyncio
async def test_agent_worker_refuses_before_accepting_tasks(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from xagent.web import worker

    monkeypatch.setenv("XAGENT_SHARED_TASK_EXECUTION_ENABLED", "true")
    monkeypatch.setenv("XAGENT_TASK_EXECUTION_ROLE", "worker")
    monkeypatch.setenv("XAGENT_REDIS_URL", "redis://localhost:6379/0")
    monkeypatch.setenv("ENCRYPTION_KEY", Fernet.generate_key().decode())
    for name in (
        "configure_db",
        "validate_worker_schema",
        "validate_interaction_rollout_at_startup",
        "register_local_browser_runtime",
    ):
        monkeypatch.setattr(worker, name, Mock())

    with pytest.raises(ConfigurationError, match="KB engine is milvus"):
        await worker.run_worker(stop=asyncio.Event())
    worker.register_local_browser_runtime.assert_not_called()


def test_celery_worker_exits_at_worker_init() -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    with pytest.raises(
        SystemExit,
        match="Refusing to start the Celery worker: This deployment's KB engine is milvus",
    ):
        worker_init.send(sender=None)


@pytest.mark.parametrize(
    ("returncode", "stdout", "stderr", "message"),
    [
        (3, "disk unavailable\n", "", "disk unavailable"),
        (
            3,
            "KB engine detected\nThis deployment's KB engine is milvus\n",
            "Cannot record the KB engine: [Errno 30] Read-only file system\n",
            "This deployment's KB engine is milvus",
        ),
        (
            -11,
            "",
            "Fatal Python error\nSegmentation fault\n",
            "the engine check was killed by signal 11: Segmentation fault",
        ),
        (
            -11,
            "",
            "WARNING\tCannot record the KB engine in /d/.kb-engine: [Errno 30] "
            "Read-only file system\n",
            "the engine check was killed by signal 11: Cannot record the KB engine "
            "in /d/.kb-engine: [Errno 30] Read-only file system",
        ),
        (-11, "", "", "the engine check was killed by signal 11"),
        (1, "", "", "the engine check exited with code 1"),
        (1, "stray output\n", "", "the engine check exited with code 1"),
    ],
)
def test_celery_worker_exits_on_any_engine_check_failure(
    monkeypatch: pytest.MonkeyPatch,
    returncode: int,
    stdout: str,
    stderr: str,
    message: str,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    monkeypatch.setattr(
        celery_app.subprocess,
        "run",
        lambda *_, **__: subprocess.CompletedProcess([], returncode, stdout, stderr),
    )

    with pytest.raises(
        SystemExit,
        match=f"Refusing to start the Celery worker: {re.escape(message)}$",
    ):
        worker_init.send(sender=None)


@pytest.mark.parametrize(
    ("empty_dir", "level", "message"),
    [
        (True, logging.WARNING, "LANCEDB_DIR is empty"),
        (False, logging.INFO, "Recorded KB engine lancedb in"),
    ],
)
def test_celery_worker_relays_what_a_passing_engine_check_logs(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    caplog: pytest.LogCaptureFixture,
    empty_dir: bool,
    level: int,
    message: str,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    if empty_dir:
        monkeypatch.setenv("LANCEDB_DIR", "")
    else:
        (tmp_path / KB_ENGINE_RECORD).unlink()
    caplog.set_level(logging.INFO)

    worker_init.send(sender=None)

    assert any(
        record.levelno == level and message in record.getMessage()
        for record in caplog.records
    )


def test_celery_worker_relays_the_engine_check_output_at_its_levels(
    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    script = (
        "import sys\n"
        "print('plain stdout')\n"
        "sys.stderr.write('ERROR\\tparsed error\\n')\n"
        "sys.stderr.write('raw stderr line\\n')\n"
    )
    monkeypatch.setattr(celery_app, "_LOCK_KB_ENGINE", script)
    caplog.set_level(logging.INFO)

    worker_init.send(sender=None)

    assert [(r.levelno, r.getMessage()) for r in caplog.records] == [
        (logging.INFO, "KB engine check: plain stdout"),
        (logging.ERROR, "KB engine check: parsed error"),
        (logging.WARNING, "KB engine check: raw stderr line"),
    ]


def test_celery_worker_relays_the_output_of_a_refusing_engine_check(
    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    script = (
        "import sys\n"
        "sys.stderr.write('WARNING\\tCannot record the KB engine\\n')\n"
        "print('refused')\n"
        "raise SystemExit(3)\n"
    )
    monkeypatch.setattr(celery_app, "_LOCK_KB_ENGINE", script)
    caplog.set_level(logging.INFO)

    with pytest.raises(SystemExit, match="Celery worker: refused$"):
        worker_init.send(sender=None)

    assert (logging.WARNING, "KB engine check: Cannot record the KB engine") in [
        (r.levelno, r.getMessage()) for r in caplog.records
    ]


def test_celery_worker_refuses_even_when_relaying_the_output_raises(
    caplog: pytest.LogCaptureFixture,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    def failing_filter(record: logging.LogRecord) -> bool:
        raise RuntimeError("the log filter failed")

    caplog.set_level(logging.INFO)
    celery_app.logger.addFilter(failing_filter)
    try:
        with pytest.raises(
            SystemExit,
            match="Refusing to start the Celery worker: This deployment's KB engine is milvus",
        ):
            worker_init.send(sender=None)
    finally:
        celery_app.logger.removeFilter(failing_filter)


def test_celery_worker_exits_when_the_engine_check_cannot_start(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    monkeypatch.setattr(celery_app.sys, "executable", str(tmp_path / "no-python"))

    with pytest.raises(SystemExit, match="cannot run the engine check: .*no-python"):
        worker_init.send(sender=None)


def test_celery_worker_reads_an_engine_check_that_prints_undecodable_bytes(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    script = (
        "import sys\n"
        "sys.stdout.buffer.write(b'\\xff the path is broken\\n')\n"
        "raise SystemExit(3)\n"
    )
    monkeypatch.setattr(celery_app, "_LOCK_KB_ENGINE", script)

    with pytest.raises(
        SystemExit, match="Refusing to start the Celery worker: . the path is broken$"
    ):
        worker_init.send(sender=None)


def test_celery_worker_exits_when_the_engine_check_output_cannot_be_read(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    def undecodable(*_: object, **__: object) -> None:
        raise UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte")

    monkeypatch.setattr(celery_app.subprocess, "run", undecodable)

    with pytest.raises(SystemExit, match="cannot run the engine check: .*0xff"):
        worker_init.send(sender=None)


def test_celery_parent_leaves_the_engine_check_to_a_child_process(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
    from celery.signals import worker_init

    from xagent.core.tools.core.RAG_tools.storage import vector_backend
    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    def in_the_parent() -> None:
        # Celery swallows an Exception raised by a signal handler, so exit instead.
        raise SystemExit("the engine check opened LanceDB in the Celery parent")

    monkeypatch.setattr(vector_backend, "lock_deployment_kb_engine", in_the_parent)
    (tmp_path / KB_ENGINE_RECORD).write_text("lancedb\n")

    worker_init.send(sender=None)


@pytest.fixture
def add_on(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
    """The Milvus add-on applied to a deployment that has no engine record yet."""
    (tmp_path / KB_ENGINE_RECORD).unlink()
    monkeypatch.setenv("XAGENT_VECTOR_BACKEND", "milvus")
    return tmp_path


def _seed_lancedb(directory: Path) -> None:
    lancedb.connect(str(directory)).create_table("documents", [{"id": "x"}])


@pytest.mark.asyncio
async def test_backend_starts_with_the_add_on_on_a_new_deployment(
    monkeypatch: pytest.MonkeyPatch, add_on: Path
) -> None:
    from xagent.web import app

    initialize = AsyncMock(side_effect=_Started)
    monkeypatch.setattr(app, "_initialize_database_and_admit_runtime", initialize)

    with pytest.raises(_Started):
        await app.startup_event()
    initialize.assert_awaited_once()
    assert (add_on / KB_ENGINE_RECORD).read_text() == "milvus\n"


@pytest.mark.asyncio
async def test_backend_refuses_the_add_on_over_existing_lancedb_data(
    monkeypatch: pytest.MonkeyPatch, add_on: Path
) -> None:
    from xagent.web import app

    _seed_lancedb(add_on)
    initialize = AsyncMock()
    monkeypatch.setattr(app, "_initialize_database_and_admit_runtime", initialize)

    with pytest.raises(ConfigurationError, match=r"is lancedb \(documents hold data\)"):
        await app.startup_event()
    initialize.assert_not_called()


def test_celery_worker_starts_with_the_add_on_on_a_new_deployment(
    add_on: Path,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    worker_init.send(sender=None)
    assert (add_on / KB_ENGINE_RECORD).read_text() == "milvus\n"


def test_celery_worker_exits_with_the_add_on_over_existing_lancedb_data(
    add_on: Path,
) -> None:
    from celery.signals import worker_init

    from xagent.web.jobs import celery_app  # noqa: F401 - connects the handler

    _seed_lancedb(add_on)

    with pytest.raises(SystemExit, match=r"is lancedb \(documents hold data\)"):
        worker_init.send(sender=None)