Enterprise agent platform for building, deploying, and operating AI teammates.
"""Admission and lifetime contracts independent of PostgreSQL availability."""
import asyncio
import threading
from contextvars import ContextVar
from unittest.mock import AsyncMock, Mock
import pytest
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url
from sqlalchemy.pool import QueuePool
from xagent.web.services import trace_database, trace_handlers
from xagent.web.services.trace_database import TraceDatabaseRuntime
# A blocked worker that gives up returns its permit and lets queued work in,
# which on a stalled CI runner looks exactly like an admission bug. Tests set
# ``release`` in ``finally``, so workers wait far beyond any realistic stall;
# the bound (below pytest-timeout) only stops a missed release from keeping the
# default executor, and with it the pytest process, alive after a failure.
WORKER_RELEASE_BOUND = 240
# Loop-side waits, where expiry fails the test instead of changing what it
# measures.
HANG_TIMEOUT = 30
async def wait_until(predicate):
async def wait():
while not predicate():
await asyncio.sleep(0)
await asyncio.wait_for(wait(), HANG_TIMEOUT)
@pytest.mark.asyncio
async def test_admission_precedes_thread_submission_and_keeps_context(monkeypatch):
runtime = TraceDatabaseRuntime(None, use_async=False, limit=2)
release = threading.Event()
started = []
active_lock = threading.Lock()
active = peak = 0
context = ContextVar("test_trace_context", default="missing")
token = context.set("lease-context")
def write():
nonlocal active, peak
with active_lock:
active += 1
peak = max(peak, active)
started.append(context.get())
try:
release.wait(WORKER_RELEASE_BOUND)
finally:
with active_lock:
active -= 1
tasks = [asyncio.create_task(runtime.run(write, lambda db: None)) for _ in range(6)]
try:
await wait_until(lambda: len(started) >= 2)
await asyncio.sleep(0.01)
assert len(started) == 2
# API work on the default pool still has a worker available.
assert await asyncio.wait_for(asyncio.to_thread(lambda: 42), HANG_TIMEOUT) == 42
finally:
release.set()
await asyncio.gather(*tasks)
context.reset(token)
await runtime.close()
assert started == ["lease-context"] * 6
assert peak == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_queued", [False, True])
async def test_handler_cancellation_drains_before_releasing_admission(
monkeypatch, cancel_queued
):
from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
release = threading.Event()
started = []
handler = trace_handlers.DatabaseTraceHandler(42)
def write(event, **_kwargs):
started.append(event.id)
release.wait(WORKER_RELEASE_BOUND)
monkeypatch.setattr(handler, "_sync_save_to_database", write)
events = [
TraceEvent(TASK_START_GENERAL, task_id="42", require_persisted=True)
for _ in range(2)
]
callers = [
asyncio.create_task(handler._save_to_database(event)) for event in events
]
try:
await wait_until(lambda: len(runtime._operations) == 2 and len(started) == 1)
cancelled = callers[int(cancel_queued)]
cancelled.cancel()
await asyncio.sleep(0)
cancelled.cancel()
await asyncio.sleep(0.01)
assert not cancelled.done()
assert len(started) == 1
finally:
release.set()
outcomes = await asyncio.gather(*callers, return_exceptions=True)
await runtime.close()
assert isinstance(outcomes[int(cancel_queued)], asyncio.CancelledError)
assert started == [event.id for event in events]
@pytest.mark.asyncio
async def test_close_drains_accepted_writes_and_rejects_new_work():
runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
release = threading.Event()
started = threading.Event()
def write():
started.set()
release.wait(WORKER_RELEASE_BOUND)
work = asyncio.create_task(runtime.run(write, lambda db: None))
close = None
try:
await wait_until(started.is_set)
close = asyncio.create_task(runtime.close())
await wait_until(lambda: runtime._closing)
with pytest.raises(RuntimeError, match="closing"):
await runtime.run(lambda: None, lambda db: None)
close.cancel()
await asyncio.sleep(0.01)
assert not close.done()
finally:
release.set()
await work
if close is not None:
with pytest.raises(asyncio.CancelledError):
await close
await runtime.close()
@pytest.mark.asyncio
async def test_failed_write_returns_permit():
runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
def fail():
raise ValueError("write failed")
with pytest.raises(ValueError, match="write failed"):
await runtime.run(fail, lambda db: None)
await runtime.run(lambda: None, lambda db: None)
await runtime.close()
@pytest.mark.asyncio
async def test_preparation_is_bounded_drained_and_precedes_connection(
tmp_path, monkeypatch
):
source = create_engine(f"sqlite:///{tmp_path / 'prepare.db'}")
runtime = TraceDatabaseRuntime(source, use_async=True, limit=4)
monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
from sqlalchemy import event
from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
checkouts = []
event.listen(
runtime.engine.sync_engine, "checkout", lambda *args: checkouts.append(1)
)
started, release = threading.Event(), threading.Event()
prepared = []
context = ContextVar("preparation_context", default="missing")
token = context.set("lease")
handler = trace_handlers.DatabaseTraceHandler(1)
def prepare(trace, **_kwargs):
prepared.append(context.get())
started.set()
release.wait(WORKER_RELEASE_BOUND)
return lambda db: None
monkeypatch.setattr(handler, "_prepare_async_trace_transaction", prepare)
callers = [
asyncio.create_task(
handler._save_to_database(TraceEvent(TASK_START_GENERAL, task_id="1"))
)
for _ in range(3)
]
closing = None
try:
await wait_until(started.is_set)
callers[0].cancel()
await asyncio.sleep(0)
callers[0].cancel()
closing = asyncio.create_task(runtime.close())
await asyncio.sleep(0.02)
assert prepared == ["lease"]
assert not checkouts
assert not callers[0].done() and not closing.done()
assert await asyncio.wait_for(asyncio.to_thread(lambda: 42), HANG_TIMEOUT) == 42
finally:
release.set()
outcomes = await asyncio.gather(*callers, return_exceptions=True)
await runtime.close()
if closing is not None:
await closing
source.dispose()
context.reset(token)
assert isinstance(outcomes[0], asyncio.CancelledError)
assert prepared == ["lease"] * 3
def test_sqlite_clamp_memory_fallback_and_positive_limit():
engine = create_engine("sqlite://", poolclass=QueuePool, pool_size=3)
try:
assert TraceDatabaseRuntime(engine, use_async=False, limit=8).limit == 1
memory_runtime = TraceDatabaseRuntime(engine, use_async=True, limit=2)
assert memory_runtime.engine is None
assert memory_runtime.limit == 1
with pytest.raises(ValueError, match="positive"):
TraceDatabaseRuntime(None, use_async=False, limit=0)
finally:
engine.dispose()
@pytest.mark.asyncio
@pytest.mark.parametrize("pool_size,expected", [(1, 1), (3, 2), (10, 8)])
async def test_sync_postgresql_pool_reserves_headroom(pool_size, expected):
# Engine construction does not connect: exercise the real QueuePool and
# PostgreSQL dialect without requiring a PostgreSQL server.
source = create_engine("postgresql+psycopg2://", pool_size=pool_size)
runtime = TraceDatabaseRuntime(source, use_async=False, limit=8)
try:
assert runtime.limit == expected
finally:
await runtime.close()
source.dispose()
def test_runtime_is_owned_by_loop(monkeypatch):
monkeypatch.setenv("XAGENT_ASYNC_TRACE_DB_ENABLED", "false")
runtimes = []
async def use():
runtime = trace_database.get_trace_database_runtime()
assert trace_database.get_trace_database_runtime() is runtime
runtimes.append(runtime)
await trace_database.close_trace_database_runtime()
await trace_database.close_trace_database_runtime()
asyncio.run(use())
asyncio.run(use())
assert runtimes[0] is not runtimes[1]
@pytest.mark.asyncio
async def test_async_config_preserves_url_options_and_bounds_pool(monkeypatch):
source = Mock()
source.dialect.name = "postgresql"
source.url = make_url("postgresql://test@localhost/db?sslmode=require")
source.get_execution_options.return_value = {"postgresql_readonly": True}
engine = Mock(dispose=AsyncMock())
create = Mock(return_value=engine)
monkeypatch.setattr(trace_database, "create_async_engine", create)
monkeypatch.setattr(trace_database, "get_engine", lambda: source)
monkeypatch.setenv("XAGENT_ASYNC_TRACE_DB_ENABLED", "true")
monkeypatch.setenv("XAGENT_TRACE_DB_MAX_INFLIGHT", "6")
runtime = trace_database.get_trace_database_runtime()
assert runtime.limit == 6
url = create.call_args.args[0]
assert url.drivername == "postgresql+psycopg"
assert url.query == source.url.query
kwargs = create.call_args.kwargs
assert kwargs["pool_size"] == 6
assert kwargs["max_overflow"] == 0
assert kwargs["hide_parameters"] is True
assert kwargs["execution_options"] == {"postgresql_readonly": True}
await trace_database.close_trace_database_runtime()
engine.dispose.assert_awaited_once()
@pytest.mark.parametrize(
"failure",
[
ModuleNotFoundError("psycopg"),
ImportError("driver unavailable"),
TypeError("invalid engine option"),
ValueError("invalid configuration"),
],
)
def test_async_engine_construction_failure_is_actionable(monkeypatch, failure):
source = Mock()
source.dialect.name = "postgresql"
monkeypatch.setattr(
trace_database,
"create_async_engine",
Mock(side_effect=failure),
)
with pytest.raises(
RuntimeError, match="XAGENT_ASYNC_TRACE_DB_ENABLED=false"
) as raised:
TraceDatabaseRuntime(source, use_async=True, limit=4)
assert raised.value.__cause__ is failure
assert "DATABASE_URL" in str(raised.value)
assert "postgresql extra" in str(raised.value)
def test_unsupported_async_backend_fails_before_engine_construction(monkeypatch):
source = Mock()
source.dialect.name = "mysql"
create = Mock()
monkeypatch.setattr(trace_database, "create_async_engine", create)
with pytest.raises(ValueError, match="requires PostgreSQL or SQLite") as raised:
TraceDatabaseRuntime(source, use_async=True, limit=4)
assert "XAGENT_ASYNC_TRACE_DB_ENABLED=false" in str(raised.value)
create.assert_not_called()