Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

20.5 KB
Raw Blame
import asyncio
from dataclasses import replace
from datetime import timedelta

import pytest
from sqlalchemy import select, text, update
from sqlalchemy.exc import DataError, OperationalError
from sqlalchemy.orm import sessionmaker

from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.pool_contention_shared import GUARD_TIMEOUT
from tests.web.services.task_database_shared import engine as engine_fixture
from tests.web.services.task_database_shared import task_id as task_id_fixture
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task, TaskStatus
from xagent.web.services import task_lease_service as ls

engine = engine_fixture
task_id = task_id_fixture
DEFERRED = ls.TaskLeaseRefreshState.DEFERRED


@pytest.fixture
def scenario(engine, task_id, monkeypatch):
    if engine.dialect.name != "postgresql":
        pytest.skip("PostgreSQL row locking")
    factory = sessionmaker(engine)
    monkeypatch.setattr("xagent.web.models.database._SessionLocal", factory)
    with factory() as db:
        uid = db.get(Task, task_id).user_id
        ids = [task_id]
        for title in ["B", "C"]:
            row = Task(user_id=uid, title=title)
            db.add(row)
            db.flush()
            ids.append(row.id)
        db.commit()
        batch = tuple(
            ls.acquire_task_lease(db, tid, runner_id="probe", new_run=True)
            for tid in ids
        )
        db.execute(
            update(Task)
            .where(Task.id.in_(ids))
            .values(last_heartbeat_at=ls.utc_now() - timedelta(seconds=20))
        )
        db.commit()
    return factory, batch, uid


def heartbeats(factory, batch):
    with factory() as db:
        return dict(
            db.execute(
                select(Task.id, Task.last_heartbeat_at).where(
                    Task.id.in_([lease.task_id for lease in batch])
                )
            ).all()
        )


def test_blocked_middle_row_healthy_rows_commit_then_retry(scenario):
    factory, batch, _ = scenario
    before = heartbeats(factory, batch)
    with factory() as blocker:
        blocker.execute(
            update(Task)
            .where(Task.id == batch[1].task_id)
            .values(updated_at=Task.updated_at)
        )
        states = ls.refresh_task_leases_isolated(batch)
        after = heartbeats(factory, batch)
        assert [states[ls._task_lease_key(lease)] for lease in batch] == [
            "refreshed",
            DEFERRED,
            "refreshed",
        ]
        assert after[batch[0].task_id] > before[batch[0].task_id]
        assert after[batch[1].task_id] == before[batch[1].task_id]
        assert after[batch[2].task_id] > before[batch[2].task_id]
        # Three retries must not write B while the conflicting lock remains.
        for _ in range(3):
            assert (
                ls.refresh_task_leases_isolated((batch[1],))[
                    ls._task_lease_key(batch[1])
                ]
                == DEFERRED
            )
        assert heartbeats(factory, batch) == after
        blocker.rollback()
    assert (
        ls.refresh_task_leases_isolated((batch[1],))[ls._task_lease_key(batch[1])]
        == "refreshed"
    )
    final = heartbeats(factory, batch)
    assert final[batch[1].task_id] > before[batch[1].task_id]
    assert final[batch[0].task_id] == after[batch[0].task_id]


def test_no_key_update_preserves_fk_key_share_compatibility(scenario):
    factory, batch, uid = scenario
    with factory() as child:
        child.add(
            TaskChatMessage(
                task_id=batch[1].task_id,
                user_id=uid,
                role="assistant",
                message_type="assistant_response",
                content="uncommitted",
            )
        )
        child.flush()
        assert set(ls.refresh_task_leases_isolated(batch).values()) == {"refreshed"}
        child.rollback()


@pytest.mark.parametrize("change", ["attempt", "runner", "run", "terminal", "delete"])
def test_uncommitted_changes_defer_then_classify_committed_state(scenario, change):
    factory, batch, _ = scenario
    lease = batch[1]
    with factory() as writer:
        row = writer.get(Task, lease.task_id)
        if change == "attempt":
            row.lease_attempt_id = "new-attempt"
        elif change == "runner":
            row.runner_id = "new-runner"
        elif change == "run":
            row.run_id = "new-run"
        elif change == "terminal":
            row.status = TaskStatus.COMPLETED
        else:
            writer.delete(row)
        writer.flush()
        result = ls.refresh_task_leases_isolated(batch)
        assert result[ls._task_lease_key(lease)] == DEFERRED
        writer.commit()
    result = ls.refresh_task_leases_isolated((lease,))
    assert result[ls._task_lease_key(lease)] == (
        "settlement_ready" if change == "terminal" else "lost"
    )


def test_sql_failure_rolls_back_healthy_updates(scenario, monkeypatch):
    factory, batch, _ = scenario
    before = heartbeats(factory, batch)
    original_commit = factory.class_.commit

    def fail_commit(db):
        db.execute(text("select 1/0"))
        original_commit(db)

    monkeypatch.setattr(factory.class_, "commit", fail_commit)
    with pytest.raises(DataError, match="division by zero"):
        ls.refresh_task_leases_isolated(batch)
    assert heartbeats(factory, batch) == before


def test_table_lock_is_not_skipped_and_hits_database_timeout(scenario):
    factory, batch, _ = scenario
    with factory() as blocker:
        blocker.execute(text("LOCK TABLE tasks IN ACCESS EXCLUSIVE MODE"))
        with pytest.raises(OperationalError, match="lock timeout"):
            ls.refresh_task_leases_isolated(batch)
        blocker.rollback()
    assert set(ls.refresh_task_leases_isolated(batch).values()) == {"refreshed"}


def test_long_locked_task_can_expire_while_healthy_tasks_renew(scenario):
    factory, batch, _ = scenario
    with factory() as db:
        db.execute(
            update(Task)
            .where(Task.id == batch[1].task_id)
            .values(lease_expires_at=ls.utc_now() - timedelta(seconds=1))
        )
        db.commit()
    with factory() as blocker:
        blocker.execute(
            update(Task)
            .where(Task.id == batch[1].task_id)
            .values(updated_at=Task.updated_at)
        )
        result = ls.refresh_task_leases_isolated(batch)
        assert result[ls._task_lease_key(batch[1])] == DEFERRED
        with factory() as observer:
            expired = observer.scalar(
                select(Task.lease_expires_at).where(Task.id == batch[1].task_id)
            )
            assert expired < ls.utc_now()
        blocker.rollback()


def test_old_and_new_attempt_in_same_batch_must_not_share_success(scenario):
    factory, batch, _ = scenario
    old = batch[1]
    with factory() as db:
        current = ls.acquire_task_lease(
            db, old.task_id, runner_id=old.runner_id, expected_run_id=old.run_id
        )
    states = ls.refresh_task_leases_isolated((old, current))
    assert states[ls._task_lease_key(old)] == "lost"
    assert states[ls._task_lease_key(current)] == "refreshed"


@pytest.mark.parametrize("field", ["run_id", "attempt_id"])
def test_missing_identity_must_never_renew(scenario, field):
    factory, batch, _ = scenario
    lease = replace(batch[1], **{field: None})
    with factory() as db:
        db.execute(
            update(Task)
            .where(Task.id == lease.task_id)
            .values(**{"run_id" if field == "run_id" else "lease_attempt_id": None})
        )
        db.commit()
    assert (
        ls.refresh_task_leases_isolated((lease,))[ls._task_lease_key(lease)] == "lost"
    )


@pytest.fixture
def inline_db_io(monkeypatch):
    """Keep fake-clock scheduling tests independent of thread-pool latency.

    Only for stubs that return immediately; blocking stubs must keep the
    worker boundary to avoid stalling the event loop.

    Tests of real locks, in-flight replacement and cancellation deliberately
    keep the production database worker boundary instead.
    """

    async def run_inline(operation):
        return operation()

    monkeypatch.setattr(ls, "run_db_io_cancellation_safe", run_inline)


def _manager_with_clock(monkeypatch):
    """Advance heartbeat deadlines without wall-clock sleeps or short SQL budgets."""
    from unittest.mock import Mock

    clock = Mock(wraps=asyncio.get_running_loop())
    clock.time.return_value = 0.0
    monkeypatch.setattr(ls, "get_task_lease_heartbeat_seconds", lambda: 20)
    manager = ls._TaskLeaseHeartbeatManager(clock)
    settled = asyncio.Event()
    settle = manager._settle_refresh_waiter

    def record(*args, **kwargs):
        settle(*args, **kwargs)
        settled.set()

    monkeypatch.setattr(manager, "_settle_refresh_waiter", record)
    return manager, clock, settled


async def _advance_heartbeat(manager, clock, settled, at):
    settled.clear()
    clock.time.return_value = at
    manager._wake_event.set()
    await asyncio.wait_for(settled.wait(), timeout=GUARD_TIMEOUT)


@pytest.mark.asyncio
async def test_deferred_subset_retries_without_speeding_up_healthy_leases(
    monkeypatch, inline_db_io
):
    manager, clock, settled = _manager_with_clock(monkeypatch)
    batch = tuple(
        ls.TaskLease(task_id=i, runner_id="r", run_id="run", attempt_id="a")
        for i in range(3)
    )
    calls = []

    def refresh(items):
        calls.append(items)
        return {
            ls._task_lease_key(lease): DEFERRED
            if lease == batch[1] and len(calls) <= 2
            else ls.TaskLeaseRefreshState.REFRESHED
            for lease in items
        }

    monkeypatch.setattr(ls, "refresh_task_leases_isolated", refresh)
    registrations = [manager.register(lease) for lease in batch]
    previous_error = RuntimeError("previous checkout timeout")
    registrations[1]._entry.outcome = ls.TaskLeaseHeartbeatOutcome(
        pool_timeout=previous_error
    )
    try:
        await asyncio.sleep(0)
        await _advance_heartbeat(manager, clock, settled, 20)
        assert registrations[1]._entry.outcome.pool_timeout is previous_error
        assert not registrations[1].terminal_event.is_set()
        await _advance_heartbeat(manager, clock, settled, 21)
        assert registrations[1]._entry.outcome.pool_timeout is previous_error
        assert registrations[1]._entry.deferred_since == 20
        await _advance_heartbeat(manager, clock, settled, 22)
        assert registrations[1]._entry.outcome.pool_timeout is None
        assert registrations[1]._entry.retry_at is None
        assert registrations[1]._entry.deferred_since is None
        await _advance_heartbeat(manager, clock, settled, 40)
        assert calls == [batch, (batch[1],), (batch[1],), batch]
    finally:
        for registration in registrations:
            await registration.close()
        await manager.wait_until_idle()


@pytest.mark.asyncio
async def test_retry_database_error_waits_for_normal_tick(monkeypatch, inline_db_io):
    manager, clock, settled = _manager_with_clock(monkeypatch)
    lease = ls.TaskLease(task_id=1, runner_id="r", run_id="run", attempt_id="a")
    calls = []

    def refresh(items):
        calls.append(items)
        if len(calls) == 2:
            raise RuntimeError("database error during retry")
        return {ls._task_lease_key(lease): DEFERRED}

    monkeypatch.setattr(ls, "refresh_task_leases_isolated", refresh)
    registration = manager.register(lease)
    try:
        await asyncio.sleep(0)
        await _advance_heartbeat(manager, clock, settled, 20)
        await _advance_heartbeat(manager, clock, settled, 21)
        assert registration._entry.retry_at is None
        assert not registration.terminal_event.is_set()
        await _advance_heartbeat(manager, clock, settled, 40)
        assert len(calls) == 3
        assert registration._entry.retry_at == 41
        await asyncio.wait_for(registration.close(), GUARD_TIMEOUT)
        await asyncio.wait_for(manager.wait_until_idle(), GUARD_TIMEOUT)
        assert len(calls) == 3  # Closing does not wait for a future retry.
    finally:
        await registration.close()
        await manager.wait_until_idle()


@pytest.mark.asyncio
async def test_late_deferred_result_cannot_schedule_replacement_registration(
    monkeypatch,
):
    from threading import Event

    manager, clock, settled = _manager_with_clock(monkeypatch)
    lease = ls.TaskLease(task_id=1, runner_id="r", run_id="run", attempt_id="a")
    ready, release = Event(), Event()
    calls = []

    def refresh(items):
        calls.append(items)
        if len(calls) == 1:
            ready.set()
            assert release.wait(GUARD_TIMEOUT)
            return {ls._task_lease_key(lease): DEFERRED}
        return {
            ls._task_lease_key(item): ls.TaskLeaseRefreshState.REFRESHED
            for item in items
        }

    monkeypatch.setattr(ls, "refresh_task_leases_isolated", refresh)
    old = manager.register(lease)
    new = None
    closing = None
    try:
        await asyncio.sleep(0)
        clock.time.return_value = 20
        manager._wake_event.set()
        assert await asyncio.to_thread(ready.wait, GUARD_TIMEOUT)
        closing = asyncio.create_task(old.close())
        await asyncio.sleep(0)
        assert not closing.done()
        # Same-key replacement is stricter than merely changing attempt: the
        # old in-flight entry must not attach a retry to the new entry object.
        new = manager.register(lease)
        assert old._entry is not new._entry
        release.set()
        await asyncio.wait_for(closing, GUARD_TIMEOUT)
        assert new._entry.retry_at is None
        await _advance_heartbeat(manager, clock, settled, 40)
        assert calls == [(lease,), (lease,)]
        assert not new.terminal_event.is_set()
    finally:
        release.set()
        if closing is not None:
            await closing
        await old.close()
        if new is not None:
            await new.close()
        await manager.wait_until_idle()


@pytest.mark.asyncio
async def test_manager_retries_real_locked_lease_after_healthy_commit(
    scenario, monkeypatch
):
    factory, batch, _ = scenario
    manager, clock, settled = _manager_with_clock(monkeypatch)
    original_refresh = ls.refresh_task_leases_isolated
    calls = []

    def refresh(items):
        calls.append(items)
        return original_refresh(items)

    monkeypatch.setattr(ls, "refresh_task_leases_isolated", refresh)
    registrations = [manager.register(lease) for lease in batch]
    before = heartbeats(factory, batch)
    try:
        await asyncio.sleep(0)
        with factory() as blocker:
            blocker.execute(
                update(Task)
                .where(Task.id == batch[1].task_id)
                .values(updated_at=Task.updated_at)
            )
            await _advance_heartbeat(manager, clock, settled, 20)
            after = heartbeats(factory, batch)
            assert after[batch[0].task_id] > before[batch[0].task_id]
            assert after[batch[1].task_id] == before[batch[1].task_id]
            assert after[batch[2].task_id] > before[batch[2].task_id]
            assert not registrations[1].terminal_event.is_set()
            blocker.rollback()
        await _advance_heartbeat(manager, clock, settled, 21)
        final = heartbeats(factory, batch)
        assert final[batch[1].task_id] > before[batch[1].task_id]
        assert final[batch[0].task_id] == after[batch[0].task_id]
        assert final[batch[2].task_id] == after[batch[2].task_id]
        assert calls == [batch, (batch[1],)]
    finally:
        for registration in registrations:
            await registration.close()
        await manager.wait_until_idle()


def test_preexecution_validation_waits_for_definitive_ownership(scenario, monkeypatch):
    import time
    from concurrent.futures import ThreadPoolExecutor
    from threading import Event

    from sqlalchemy.orm import Session

    factory, batch, _ = scenario
    lease = batch[1]
    ready = Event()
    pids = []

    class ValidationSession(Session):
        def __enter__(self):
            super().__enter__()
            self.execute(text("SET LOCAL statement_timeout='5s'"))
            pids.append(self.scalar(text("select pg_backend_pid()")))
            ready.set()
            return self

    validation_factory = sessionmaker(factory.kw["bind"], class_=ValidationSession)
    monkeypatch.setattr("xagent.web.models.database._SessionLocal", validation_factory)
    with factory() as blocker, ThreadPoolExecutor(1) as pool:
        blocker.execute(
            update(Task)
            .where(Task.id == lease.task_id)
            .values(lease_attempt_id="successor")
        )
        future = pool.submit(ls.validate_preacquired_task_lease_isolated, lease)
        try:
            assert ready.wait(DB_PROGRESS_TIMEOUT)
            with factory() as observer:
                deadline = time.monotonic() + DB_PROGRESS_TIMEOUT
                while not observer.scalar(
                    text("select pg_blocking_pids(:pid)"), {"pid": pids[0]}
                ):
                    assert not future.done(), (
                        "Admission must not return DEFERRED as successful validation"
                    )
                    assert time.monotonic() < deadline
                    time.sleep(0.005)
        finally:
            blocker.commit()
        assert (
            future.result(timeout=DB_PROGRESS_TIMEOUT) == ls.TaskLeaseRefreshState.LOST
        )


@pytest.mark.asyncio
async def test_slow_subset_retry_does_not_skip_normal_batch_deadline(
    monkeypatch, inline_db_io
):
    manager, clock, settled = _manager_with_clock(monkeypatch)
    batch = tuple(
        ls.TaskLease(task_id=i, runner_id="r", run_id="run", attempt_id="a")
        for i in range(2)
    )
    calls = []
    normal_batch = asyncio.Event()
    loop = asyncio.get_running_loop()

    def refresh(items):
        calls.append(items)
        if len(calls) == 2:
            clock.time.return_value = 42  # Retry crossed the normal tick at 40.
        if len(calls) == 3:
            loop.call_soon_threadsafe(normal_batch.set)
        return {
            ls._task_lease_key(lease): DEFERRED
            if lease == batch[1] and len(calls) == 1
            else ls.TaskLeaseRefreshState.REFRESHED
            for lease in items
        }

    monkeypatch.setattr(ls, "refresh_task_leases_isolated", refresh)
    registrations = [manager.register(lease) for lease in batch]
    try:
        await asyncio.sleep(0)
        await _advance_heartbeat(manager, clock, settled, 20)
        await _advance_heartbeat(manager, clock, settled, 21)
        await asyncio.wait_for(normal_batch.wait(), GUARD_TIMEOUT)
        assert calls == [batch, (batch[1],), batch]
    finally:
        for registration in registrations:
            await registration.close()
        await manager.wait_until_idle()


@pytest.mark.asyncio
async def test_repeated_cancellation_drains_inflight_retry(monkeypatch):
    from threading import Event

    manager, clock, settled = _manager_with_clock(monkeypatch)
    monkeypatch.setattr(ls, "_get_task_lease_heartbeat_manager", lambda: manager)
    lease = ls.TaskLease(task_id=1, runner_id="r", run_id="run", attempt_id="a")
    ready, release, finished = Event(), Event(), Event()
    calls = []

    def refresh(items):
        calls.append(items)
        if len(calls) == 1:
            return {ls._task_lease_key(lease): DEFERRED}
        try:
            ready.set()
            assert release.wait(GUARD_TIMEOUT)
            return {ls._task_lease_key(lease): ls.TaskLeaseRefreshState.REFRESHED}
        finally:
            finished.set()

    monkeypatch.setattr(ls, "refresh_task_leases_isolated", refresh)
    heartbeat = asyncio.create_task(ls.run_task_lease_heartbeat(lease, asyncio.Event()))
    try:
        await asyncio.sleep(0)
        await asyncio.sleep(0)
        await _advance_heartbeat(manager, clock, settled, 20)
        clock.time.return_value = 21
        manager._wake_event.set()
        assert await asyncio.to_thread(ready.wait, GUARD_TIMEOUT)
        for _ in range(2):
            heartbeat.cancel()
            await asyncio.sleep(0)
            await asyncio.sleep(0)
            assert not heartbeat.done()
            assert not finished.is_set()
        release.set()
        with pytest.raises(asyncio.CancelledError):
            await asyncio.wait_for(heartbeat, GUARD_TIMEOUT)
        assert finished.is_set()
        await asyncio.wait_for(manager.wait_until_idle(), GUARD_TIMEOUT)
        assert len(calls) == 2
        assert not manager._entries
    finally:
        release.set()
        if not heartbeat.done():
            heartbeat.cancel()
        await asyncio.gather(heartbeat, return_exceptions=True)
        await manager.wait_until_idle()