Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

12.9 KB
Raw Blame
from __future__ import annotations

import asyncio
import random
import sqlite3
from pathlib import Path
from typing import Any

import pytest
import sqlalchemy
from sqlalchemy import exc as sa_exc

from xagent.core.agent.checkpoint import ExecutionEventPersistenceError
from xagent.web.services import db_write_retry
from xagent.web.services.db_write_retry import (
    DbWriteRetryPolicy,
    is_retryable_database_error,
    retry_db_write_async,
)

_COUNTER = "xagent.execution_event.write.retries"


class _PgOrig(Exception):
    def __init__(self, message: str, *, pgcode: str | None = None) -> None:
        super().__init__(message)
        self.pgcode = pgcode


class _SqlstateOrig(Exception):
    def __init__(self, message: str, *, sqlstate: str) -> None:
        super().__init__(message)
        self.sqlstate = sqlstate


def _operational(pgcode: str | None = None) -> sa_exc.OperationalError:
    return sa_exc.OperationalError("stmt", {}, _PgOrig("boom", pgcode=pgcode))


def _sqlite_locked() -> sa_exc.OperationalError:
    return sa_exc.OperationalError(
        "stmt", {}, sqlite3.OperationalError("database is locked")
    )


def _persistence_error_from(cause: BaseException) -> ExecutionEventPersistenceError:
    try:
        raise ExecutionEventPersistenceError("fact failed to commit") from cause
    except ExecutionEventPersistenceError as exc:
        return exc


def _invalidated() -> sa_exc.DBAPIError:
    return sa_exc.DBAPIError("stmt", {}, Exception("gone"), connection_invalidated=True)


@pytest.mark.parametrize(
    "build",
    [
        lambda: _operational("40P01"),
        lambda: _operational("40001"),
        lambda: _operational("55P03"),
        lambda: _operational("57P01"),
        lambda: _operational(None),
        _invalidated,
        lambda: sa_exc.InterfaceError("stmt", {}, Exception("closed")),
        _sqlite_locked,
        lambda: _persistence_error_from(_operational("40P01")),
    ],
    ids=[
        "deadlock",
        "serialization-failure",
        "lock-not-available",
        "admin-shutdown",
        "lost-connection",
        "connection-invalidated",
        "interface-error",
        "sqlite-locked",
        "persistence-error-from-retryable",
    ],
)
def test_retryable_errors(build) -> None:
    assert is_retryable_database_error(build()) is True


@pytest.mark.parametrize(
    "build",
    [
        lambda: _operational("57014"),
        lambda: sa_exc.OperationalError(
            "stmt", {}, _SqlstateOrig("canceled", sqlstate="57014")
        ),
        lambda: sa_exc.TimeoutError("QueuePool limit reached"),
        lambda: sa_exc.IntegrityError("stmt", {}, Exception("dup")),
        lambda: sa_exc.DataError("stmt", {}, Exception("bad value")),
        lambda: sa_exc.ProgrammingError("stmt", {}, Exception("no table")),
        lambda: _persistence_error_from(
            sa_exc.IntegrityError("stmt", {}, Exception("dup"))
        ),
        lambda: _persistence_error_from(_operational("57014")),
        lambda: RuntimeError("Execution event producer lost its task lease"),
        lambda: ValueError("nope"),
    ],
    ids=[
        "statement-timeout-pgcode",
        "statement-timeout-sqlstate",
        "pool-timeout",
        "integrity-error",
        "data-error",
        "programming-error",
        "persistence-error-from-integrity",
        "persistence-error-from-canceled",
        "lease-lost",
        "value-error",
    ],
)
def test_non_retryable_errors(build) -> None:
    assert is_retryable_database_error(build()) is False


def test_attempts_for_caps_sqlite_at_one_retry() -> None:
    policy = DbWriteRetryPolicy(attempts=5, base_delay_seconds=0, max_delay_seconds=0)

    assert policy.attempts_for(_sqlite_locked()) == 2
    assert policy.attempts_for(sqlite3.OperationalError("database is locked")) == 2
    assert policy.attempts_for(_persistence_error_from(_sqlite_locked())) == 2


def test_attempts_for_keeps_postgres_attempts() -> None:
    policy = DbWriteRetryPolicy(attempts=5, base_delay_seconds=0, max_delay_seconds=0)

    assert policy.attempts_for(_operational("40P01")) == 5


def test_attempts_for_never_raises_the_configured_attempts() -> None:
    policy = DbWriteRetryPolicy(attempts=1, base_delay_seconds=0, max_delay_seconds=0)

    assert policy.attempts_for(_sqlite_locked()) == 1
    assert policy.attempts_for(_operational("40P01")) == 1


def test_delay_is_full_jitter_below_the_capped_exponential_ceiling() -> None:
    policy = DbWriteRetryPolicy(
        attempts=6, base_delay_seconds=0.05, max_delay_seconds=0.3
    )
    rng = random.Random(1234)

    for retry in range(1, 6):
        ceiling = min(0.3, 0.05 * 2 ** (retry - 1))
        samples = [policy.delay(retry, rng=rng) for _ in range(50)]
        assert all(0 <= sample <= ceiling for sample in samples)
        assert max(samples) > 0


def test_delay_is_zero_without_a_base_delay() -> None:
    policy = DbWriteRetryPolicy(attempts=3, base_delay_seconds=0, max_delay_seconds=1)

    assert policy.delay(1, rng=random.Random(0)) == 0
    assert policy.delay(4, rng=random.Random(0)) == 0


def test_from_config_converts_milliseconds_to_seconds(monkeypatch) -> None:
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_ATTEMPTS", "4")
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_BASE_DELAY_MS", "20")
    monkeypatch.setenv("XAGENT_DB_WRITE_RETRY_MAX_DELAY_MS", "500")

    assert DbWriteRetryPolicy.from_config() == DbWriteRetryPolicy(
        attempts=4, base_delay_seconds=0.02, max_delay_seconds=0.5
    )


@pytest.fixture
def counters(monkeypatch) -> list[tuple[str, dict[str, str]]]:
    recorded: list[tuple[str, dict[str, str]]] = []

    def record(metric: str, *, amount: int = 1, attributes=None) -> None:
        recorded.append((metric, dict(attributes or {})))

    monkeypatch.setattr(db_write_retry, "increment_counter", record)
    return recorded


def _outcomes(counters: list[tuple[str, dict[str, str]]]) -> list[str]:
    assert all(metric == _COUNTER for metric, _ in counters)
    return [attributes["outcome"] for _, attributes in counters]


_FAST = DbWriteRetryPolicy(attempts=3, base_delay_seconds=0, max_delay_seconds=0)


class _FlakyOperation:
    def __init__(self, failures: list[BaseException], result: Any = "ok") -> None:
        self._failures = list(failures)
        self._result = result
        self.calls = 0

    async def __call__(self) -> Any:
        self.calls += 1
        if self._failures:
            raise self._failures.pop(0)
        return self._result


@pytest.mark.asyncio
async def test_retryable_failure_then_success(counters) -> None:
    operation = _FlakyOperation([_operational("40P01")], result="committed")

    result = await retry_db_write_async(operation, site="test", policy=_FAST)

    assert result == "committed"
    assert operation.calls == 2
    assert counters == [
        (_COUNTER, {"site": "test", "outcome": "retried"}),
        (_COUNTER, {"site": "test", "outcome": "recovered"}),
    ]


@pytest.mark.asyncio
async def test_retry_counter_exports_its_site(monkeypatch) -> None:
    """The site must survive the exporter's attribute filter, not just the call."""
    from opentelemetry.sdk.metrics import MeterProvider
    from opentelemetry.sdk.metrics.export import InMemoryMetricReader

    from xagent.core import runtime_performance as metrics

    reader = InMemoryMetricReader()
    provider = MeterProvider(metric_readers=[reader], shutdown_on_exit=False)
    monkeypatch.setattr(
        metrics,
        "runtime_performance",
        metrics.RuntimePerformanceTelemetry(provider.get_meter("test")),
    )
    try:
        operation = _FlakyOperation([_operational("40P01")])
        await retry_db_write_async(operation, site="trace_fact", policy=_FAST)
        data = reader.get_metrics_data()
    finally:
        provider.shutdown()
    assert data is not None
    (metric,) = [
        m
        for r in data.resource_metrics
        for scope in r.scope_metrics
        for m in scope.metrics
        if m.name == _COUNTER
    ]
    assert {
        (point.attributes["site"], point.attributes["outcome"]): point.value
        for point in metric.data.data_points
    } == {("trace_fact", "retried"): 1, ("trace_fact", "recovered"): 1}


@pytest.mark.asyncio
async def test_first_attempt_success_is_not_counted(counters) -> None:
    operation = _FlakyOperation([])

    assert await retry_db_write_async(operation, site="test", policy=_FAST) == "ok"

    assert operation.calls == 1
    assert counters == []


@pytest.mark.asyncio
async def test_non_retryable_error_is_raised_unchanged(counters) -> None:
    error = sa_exc.IntegrityError("stmt", {}, Exception("dup"))
    operation = _FlakyOperation([error])

    with pytest.raises(sa_exc.IntegrityError) as caught:
        await retry_db_write_async(operation, site="test", policy=_FAST)

    assert caught.value is error
    assert operation.calls == 1
    assert counters == []


@pytest.mark.asyncio
async def test_exhaustion_raises_the_last_error_with_its_cause(counters) -> None:
    cause = _operational("40P01")
    last = _persistence_error_from(cause)
    operation = _FlakyOperation([_operational("40P01"), _operational("40P01"), last])

    with pytest.raises(ExecutionEventPersistenceError) as caught:
        await retry_db_write_async(operation, site="test", policy=_FAST)

    assert caught.value is last
    assert caught.value.__cause__ is cause
    assert operation.calls == 3
    assert _outcomes(counters) == ["retried", "retried", "failed"]


@pytest.mark.asyncio
async def test_sqlite_error_is_retried_once(counters) -> None:
    operation = _FlakyOperation([_sqlite_locked() for _ in range(5)])

    with pytest.raises(sa_exc.OperationalError):
        await retry_db_write_async(operation, site="test", policy=_FAST)

    assert operation.calls == 2
    assert _outcomes(counters) == ["retried", "failed"]


@pytest.mark.asyncio
async def test_single_attempt_policy_disables_retries(counters) -> None:
    policy = DbWriteRetryPolicy(attempts=1, base_delay_seconds=0, max_delay_seconds=0)
    operation = _FlakyOperation([_operational("40P01")])

    with pytest.raises(sa_exc.OperationalError):
        await retry_db_write_async(operation, site="test", policy=policy)

    assert operation.calls == 1
    assert counters == []


@pytest.mark.asyncio
async def test_cancellation_during_backoff_stops_retrying(
    counters, monkeypatch
) -> None:
    # Jitter may draw near zero; pin the wait so the backoff is really long.
    monkeypatch.setattr(
        DbWriteRetryPolicy, "delay", lambda self, retry, *, rng=None: 10.0
    )
    policy = DbWriteRetryPolicy(attempts=3, base_delay_seconds=10, max_delay_seconds=10)
    failed = asyncio.Event()
    calls = 0

    async def operation() -> None:
        nonlocal calls
        calls += 1
        failed.set()
        raise _operational("40P01")

    task = asyncio.create_task(
        retry_db_write_async(operation, site="test", policy=policy)
    )
    await failed.wait()
    await asyncio.sleep(0)
    task.cancel()
    with pytest.raises(asyncio.CancelledError):
        await task

    assert calls == 1
    assert _outcomes(counters) == ["retried", "failed"]


@pytest.mark.asyncio
async def test_already_cancelling_caller_is_not_retried(counters) -> None:
    operation = _FlakyOperation([_operational("40P01")])
    error = operation._failures[0]

    async def body() -> None:
        task = asyncio.current_task()
        assert task is not None
        task.cancel()
        try:
            await asyncio.sleep(0)
        except asyncio.CancelledError:
            assert task.cancelling() > 0
            with pytest.raises(sa_exc.OperationalError) as caught:
                await retry_db_write_async(
                    operation,
                    site="test",
                    policy=DbWriteRetryPolicy(
                        attempts=3, base_delay_seconds=10, max_delay_seconds=10
                    ),
                )
            assert caught.value is error

    await asyncio.wait_for(asyncio.create_task(body()), timeout=5)

    assert operation.calls == 1
    assert counters == []


def test_real_sqlite_busy_error_is_retryable_once(tmp_path: Path) -> None:
    url = f"sqlite:///{tmp_path / 'busy.db'}"
    holder = sqlalchemy.create_engine(url, connect_args={"timeout": 0.05})
    writer = sqlalchemy.create_engine(url, connect_args={"timeout": 0.05})
    try:
        with holder.begin() as connection:
            connection.execute(sqlalchemy.text("CREATE TABLE facts (id INTEGER)"))

        raw = holder.raw_connection()
        try:
            raw.cursor().execute("BEGIN IMMEDIATE")
            with pytest.raises(sa_exc.OperationalError) as caught:
                with writer.begin() as connection:
                    connection.execute(sqlalchemy.text("INSERT INTO facts VALUES (1)"))
        finally:
            raw.rollback()
            raw.close()

        policy = DbWriteRetryPolicy(
            attempts=3, base_delay_seconds=0, max_delay_seconds=0
        )
        assert is_retryable_database_error(caught.value) is True
        assert policy.attempts_for(caught.value) == 2
    finally:
        holder.dispose()
        writer.dispose()