Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

11.2 KB
Raw Blame
"""Admission and lifetime contracts independent of PostgreSQL availability."""

import asyncio
import threading
from contextvars import ContextVar
from unittest.mock import AsyncMock, Mock

import pytest
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url
from sqlalchemy.pool import QueuePool

from xagent.web.services import trace_database, trace_handlers
from xagent.web.services.trace_database import TraceDatabaseRuntime

# A blocked worker that gives up returns its permit and lets queued work in,
# which on a stalled CI runner looks exactly like an admission bug. Tests set
# ``release`` in ``finally``, so workers wait far beyond any realistic stall;
# the bound (below pytest-timeout) only stops a missed release from keeping the
# default executor, and with it the pytest process, alive after a failure.
WORKER_RELEASE_BOUND = 240
# Loop-side waits, where expiry fails the test instead of changing what it
# measures.
HANG_TIMEOUT = 30


async def wait_until(predicate):
    async def wait():
        while not predicate():
            await asyncio.sleep(0)

    await asyncio.wait_for(wait(), HANG_TIMEOUT)


@pytest.mark.asyncio
async def test_admission_precedes_thread_submission_and_keeps_context(monkeypatch):
    runtime = TraceDatabaseRuntime(None, use_async=False, limit=2)
    release = threading.Event()
    started = []
    active_lock = threading.Lock()
    active = peak = 0
    context = ContextVar("test_trace_context", default="missing")
    token = context.set("lease-context")

    def write():
        nonlocal active, peak
        with active_lock:
            active += 1
            peak = max(peak, active)
        started.append(context.get())
        try:
            release.wait(WORKER_RELEASE_BOUND)
        finally:
            with active_lock:
                active -= 1

    tasks = [asyncio.create_task(runtime.run(write, lambda db: None)) for _ in range(6)]
    try:
        await wait_until(lambda: len(started) >= 2)
        await asyncio.sleep(0.01)
        assert len(started) == 2
        # API work on the default pool still has a worker available.
        assert await asyncio.wait_for(asyncio.to_thread(lambda: 42), HANG_TIMEOUT) == 42
    finally:
        release.set()
        await asyncio.gather(*tasks)
        context.reset(token)
        await runtime.close()
    assert started == ["lease-context"] * 6
    assert peak == 2


@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_queued", [False, True])
async def test_handler_cancellation_drains_before_releasing_admission(
    monkeypatch, cancel_queued
):
    from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent

    runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
    monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
    release = threading.Event()
    started = []
    handler = trace_handlers.DatabaseTraceHandler(42)

    def write(event, **_kwargs):
        started.append(event.id)
        release.wait(WORKER_RELEASE_BOUND)

    monkeypatch.setattr(handler, "_sync_save_to_database", write)
    events = [
        TraceEvent(TASK_START_GENERAL, task_id="42", require_persisted=True)
        for _ in range(2)
    ]
    callers = [
        asyncio.create_task(handler._save_to_database(event)) for event in events
    ]
    try:
        await wait_until(lambda: len(runtime._operations) == 2 and len(started) == 1)
        cancelled = callers[int(cancel_queued)]
        cancelled.cancel()
        await asyncio.sleep(0)
        cancelled.cancel()
        await asyncio.sleep(0.01)
        assert not cancelled.done()
        assert len(started) == 1
    finally:
        release.set()
        outcomes = await asyncio.gather(*callers, return_exceptions=True)
        await runtime.close()
    assert isinstance(outcomes[int(cancel_queued)], asyncio.CancelledError)
    assert started == [event.id for event in events]


@pytest.mark.asyncio
async def test_close_drains_accepted_writes_and_rejects_new_work():
    runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
    release = threading.Event()
    started = threading.Event()

    def write():
        started.set()
        release.wait(WORKER_RELEASE_BOUND)

    work = asyncio.create_task(runtime.run(write, lambda db: None))
    close = None
    try:
        await wait_until(started.is_set)
        close = asyncio.create_task(runtime.close())
        await wait_until(lambda: runtime._closing)
        with pytest.raises(RuntimeError, match="closing"):
            await runtime.run(lambda: None, lambda db: None)
        close.cancel()
        await asyncio.sleep(0.01)
        assert not close.done()
    finally:
        release.set()
        await work
        if close is not None:
            with pytest.raises(asyncio.CancelledError):
                await close
        await runtime.close()


@pytest.mark.asyncio
async def test_failed_write_returns_permit():
    runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)

    def fail():
        raise ValueError("write failed")

    with pytest.raises(ValueError, match="write failed"):
        await runtime.run(fail, lambda db: None)
    await runtime.run(lambda: None, lambda db: None)
    await runtime.close()


@pytest.mark.asyncio
async def test_preparation_is_bounded_drained_and_precedes_connection(
    tmp_path, monkeypatch
):
    source = create_engine(f"sqlite:///{tmp_path / 'prepare.db'}")
    runtime = TraceDatabaseRuntime(source, use_async=True, limit=4)
    monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
    from sqlalchemy import event

    from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent

    checkouts = []
    event.listen(
        runtime.engine.sync_engine, "checkout", lambda *args: checkouts.append(1)
    )
    started, release = threading.Event(), threading.Event()
    prepared = []
    context = ContextVar("preparation_context", default="missing")
    token = context.set("lease")
    handler = trace_handlers.DatabaseTraceHandler(1)

    def prepare(trace, **_kwargs):
        prepared.append(context.get())
        started.set()
        release.wait(WORKER_RELEASE_BOUND)
        return lambda db: None

    monkeypatch.setattr(handler, "_prepare_async_trace_transaction", prepare)
    callers = [
        asyncio.create_task(
            handler._save_to_database(TraceEvent(TASK_START_GENERAL, task_id="1"))
        )
        for _ in range(3)
    ]
    closing = None
    try:
        await wait_until(started.is_set)
        callers[0].cancel()
        await asyncio.sleep(0)
        callers[0].cancel()
        closing = asyncio.create_task(runtime.close())
        await asyncio.sleep(0.02)
        assert prepared == ["lease"]
        assert not checkouts
        assert not callers[0].done() and not closing.done()
        assert await asyncio.wait_for(asyncio.to_thread(lambda: 42), HANG_TIMEOUT) == 42
    finally:
        release.set()
        outcomes = await asyncio.gather(*callers, return_exceptions=True)
        await runtime.close()
        if closing is not None:
            await closing
        source.dispose()
        context.reset(token)
    assert isinstance(outcomes[0], asyncio.CancelledError)
    assert prepared == ["lease"] * 3


def test_sqlite_clamp_memory_fallback_and_positive_limit():
    engine = create_engine("sqlite://", poolclass=QueuePool, pool_size=3)
    try:
        assert TraceDatabaseRuntime(engine, use_async=False, limit=8).limit == 1
        memory_runtime = TraceDatabaseRuntime(engine, use_async=True, limit=2)
        assert memory_runtime.engine is None
        assert memory_runtime.limit == 1
        with pytest.raises(ValueError, match="positive"):
            TraceDatabaseRuntime(None, use_async=False, limit=0)
    finally:
        engine.dispose()


@pytest.mark.asyncio
@pytest.mark.parametrize("pool_size,expected", [(1, 1), (3, 2), (10, 8)])
async def test_sync_postgresql_pool_reserves_headroom(pool_size, expected):
    # Engine construction does not connect: exercise the real QueuePool and
    # PostgreSQL dialect without requiring a PostgreSQL server.
    source = create_engine("postgresql+psycopg2://", pool_size=pool_size)
    runtime = TraceDatabaseRuntime(source, use_async=False, limit=8)
    try:
        assert runtime.limit == expected
    finally:
        await runtime.close()
        source.dispose()


def test_runtime_is_owned_by_loop(monkeypatch):
    monkeypatch.setenv("XAGENT_ASYNC_TRACE_DB_ENABLED", "false")
    runtimes = []

    async def use():
        runtime = trace_database.get_trace_database_runtime()
        assert trace_database.get_trace_database_runtime() is runtime
        runtimes.append(runtime)
        await trace_database.close_trace_database_runtime()
        await trace_database.close_trace_database_runtime()

    asyncio.run(use())
    asyncio.run(use())
    assert runtimes[0] is not runtimes[1]


@pytest.mark.asyncio
async def test_async_config_preserves_url_options_and_bounds_pool(monkeypatch):
    source = Mock()
    source.dialect.name = "postgresql"
    source.url = make_url("postgresql://test@localhost/db?sslmode=require")
    source.get_execution_options.return_value = {"postgresql_readonly": True}
    engine = Mock(dispose=AsyncMock())
    create = Mock(return_value=engine)
    monkeypatch.setattr(trace_database, "create_async_engine", create)
    monkeypatch.setattr(trace_database, "get_engine", lambda: source)
    monkeypatch.setenv("XAGENT_ASYNC_TRACE_DB_ENABLED", "true")
    monkeypatch.setenv("XAGENT_TRACE_DB_MAX_INFLIGHT", "6")
    runtime = trace_database.get_trace_database_runtime()
    assert runtime.limit == 6
    url = create.call_args.args[0]
    assert url.drivername == "postgresql+psycopg"
    assert url.query == source.url.query
    kwargs = create.call_args.kwargs
    assert kwargs["pool_size"] == 6
    assert kwargs["max_overflow"] == 0
    assert kwargs["hide_parameters"] is True
    assert kwargs["execution_options"] == {"postgresql_readonly": True}
    await trace_database.close_trace_database_runtime()
    engine.dispose.assert_awaited_once()


@pytest.mark.parametrize(
    "failure",
    [
        ModuleNotFoundError("psycopg"),
        ImportError("driver unavailable"),
        TypeError("invalid engine option"),
        ValueError("invalid configuration"),
    ],
)
def test_async_engine_construction_failure_is_actionable(monkeypatch, failure):
    source = Mock()
    source.dialect.name = "postgresql"
    monkeypatch.setattr(
        trace_database,
        "create_async_engine",
        Mock(side_effect=failure),
    )
    with pytest.raises(
        RuntimeError, match="XAGENT_ASYNC_TRACE_DB_ENABLED=false"
    ) as raised:
        TraceDatabaseRuntime(source, use_async=True, limit=4)
    assert raised.value.__cause__ is failure
    assert "DATABASE_URL" in str(raised.value)
    assert "postgresql extra" in str(raised.value)


def test_unsupported_async_backend_fails_before_engine_construction(monkeypatch):
    source = Mock()
    source.dialect.name = "mysql"
    create = Mock()
    monkeypatch.setattr(trace_database, "create_async_engine", create)
    with pytest.raises(ValueError, match="requires PostgreSQL or SQLite") as raised:
        TraceDatabaseRuntime(source, use_async=True, limit=4)
    assert "XAGENT_ASYNC_TRACE_DB_ENABLED=false" in str(raised.value)
    create.assert_not_called()