Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

12.7 KB
Raw Blame
from __future__ import annotations

import asyncio
from concurrent.futures import ThreadPoolExecutor

import pytest

from xagent.web.models.database import (
    Base,
    get_db,
    get_engine,
    get_session_local,
    init_db,
)
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.user import User
from xagent.web.services.task_execution_controller import (
    StaleTaskRunError,
    StaleTaskStateVersionError,
    TaskControlState,
    TaskExecutionController,
    apply_task_control_transition,
    control_state_for_status,
    task_control_snapshot,
    transition_task_control_state_sync,
)


@pytest.fixture()
def db_session(tmp_path):
    init_db(db_url=f"sqlite:///{tmp_path / 'task-controller.db'}")
    db = next(get_db())
    try:
        yield db
    finally:
        db.close()
        Base.metadata.drop_all(bind=get_engine())


def _create_task(db) -> Task:
    user = User(username="controller-user", password_hash="hash", is_admin=False)
    db.add(user)
    db.commit()
    task = Task(
        user_id=user.id,
        title="Controller test",
        description="Controller test",
        status=TaskStatus.PENDING,
        execution_mode="auto",
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    return task


def test_snapshot_requires_a_persisted_task_id() -> None:
    task = Task(
        user_id=1,
        title="Transient task",
        description="Transient task",
        status=TaskStatus.PENDING,
        execution_mode="auto",
    )

    with pytest.raises(ValueError, match="task with no ID"):
        task_control_snapshot(task)


def test_unknown_task_status_has_a_clear_error() -> None:
    with pytest.raises(ValueError, match="Unsupported task status"):
        control_state_for_status("cancelled")  # type: ignore[arg-type]


def test_transition_does_not_flush_unrelated_pending_objects(db_session) -> None:
    task = _create_task(db_session)
    unrelated = Task(
        user_id=task.user_id,
        title="Unrelated task",
        description="Unrelated task",
        status=TaskStatus.PENDING,
        execution_mode="auto",
    )
    db_session.add(unrelated)
    db_session.commit()
    unrelated.title = "Still pending"

    snapshot = apply_task_control_transition(
        task,
        TaskControlState.RUNNING,
        status=TaskStatus.RUNNING,
        new_run=True,
    )

    assert snapshot.control_state == TaskControlState.RUNNING
    assert unrelated in db_session.dirty


@pytest.mark.asyncio
async def test_same_task_commands_are_fifo_serialized() -> None:
    controller = TaskExecutionController()
    first_entered = asyncio.Event()
    release_first = asyncio.Event()
    order: list[str] = []

    async def first() -> None:
        async with controller.command(7):
            order.append("first-enter")
            first_entered.set()
            await release_first.wait()
            order.append("first-exit")

    async def queued(name: str) -> None:
        await first_entered.wait()
        async with controller.command(7):
            order.append(name)

    tasks = [
        asyncio.create_task(first()),
        asyncio.create_task(queued("second")),
        asyncio.create_task(queued("third")),
    ]
    await first_entered.wait()
    await asyncio.sleep(0)
    release_first.set()
    await asyncio.gather(*tasks)

    assert order == ["first-enter", "first-exit", "second", "third"]
    assert controller._gates == {}


@pytest.mark.asyncio
async def test_gate_is_reentrant_for_nested_transport_and_turn_claim() -> None:
    controller = TaskExecutionController()

    async with controller.command(11):
        async with controller.command(11):
            assert controller._gates[11].depth == 2

    assert controller._gates == {}


@pytest.mark.asyncio
async def test_child_task_reentry_fails_instead_of_deadlocking() -> None:
    controller = TaskExecutionController()

    async def reenter_from_child() -> None:
        async with controller.command(11):
            raise AssertionError("child task must not acquire its parent's gate")

    async with controller.command(11):
        child = asyncio.create_task(reenter_from_child())
        with pytest.raises(RuntimeError, match="reentered from a child asyncio task"):
            await asyncio.wait_for(child, timeout=1)

    assert controller._gates == {}


@pytest.mark.asyncio
async def test_cancelled_waiter_does_not_split_or_leak_the_gate() -> None:
    controller = TaskExecutionController()
    owner_entered = asyncio.Event()
    release_owner = asyncio.Event()
    third_entered = asyncio.Event()

    async def owner() -> None:
        async with controller.command(19):
            owner_entered.set()
            await release_owner.wait()

    async def waiter() -> None:
        async with controller.command(19):
            raise AssertionError("cancelled waiter must not enter")

    async def third() -> None:
        async with controller.command(19):
            third_entered.set()

    owner_task = asyncio.create_task(owner())
    await owner_entered.wait()
    waiter_task = asyncio.create_task(waiter())
    await asyncio.sleep(0)
    waiter_task.cancel()
    with pytest.raises(asyncio.CancelledError):
        await waiter_task

    third_task = asyncio.create_task(third())
    await asyncio.sleep(0)
    assert not third_entered.is_set()
    release_owner.set()
    await asyncio.gather(owner_task, third_task)
    assert controller._gates == {}


@pytest.mark.asyncio
async def test_different_tasks_can_progress_concurrently() -> None:
    controller = TaskExecutionController()
    both_entered = asyncio.Event()
    entered: set[int] = set()

    async def run(task_id: int) -> None:
        async with controller.command(task_id):
            entered.add(task_id)
            if len(entered) == 2:
                both_entered.set()
            await asyncio.wait_for(both_entered.wait(), timeout=1)

    await asyncio.gather(run(1), run(2))
    assert entered == {1, 2}


def test_transitions_keep_run_identity_and_advance_version(db_session) -> None:
    task = _create_task(db_session)

    running = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.RUNNING,
        status=TaskStatus.RUNNING,
        new_run=True,
    )
    pause_requested = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.PAUSE_REQUESTED,
        expected_run_id=running.run_id,
    )
    paused = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.PAUSED,
        status=TaskStatus.PAUSED,
        expected_run_id=running.run_id,
    )

    assert running.run_id
    assert pause_requested.run_id == running.run_id == paused.run_id
    assert [
        running.state_version,
        pause_requested.state_version,
        paused.state_version,
    ] == [
        1,
        2,
        3,
    ]
    assert pause_requested.status == TaskStatus.RUNNING
    assert pause_requested.control_state == TaskControlState.PAUSE_REQUESTED

    with pytest.raises(StaleTaskRunError):
        transition_task_control_state_sync(
            int(task.id),
            TaskControlState.COMPLETED,
            status=TaskStatus.COMPLETED,
            expected_run_id="superseded-run",
        )


def test_transition_rejects_a_stale_state_version(db_session) -> None:
    task = _create_task(db_session)
    running = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.RUNNING,
        status=TaskStatus.RUNNING,
        new_run=True,
    )

    with pytest.raises(StaleTaskRunError, match="state changed"):
        transition_task_control_state_sync(
            int(task.id),
            TaskControlState.FAILED,
            status=TaskStatus.FAILED,
            expected_run_id=running.run_id,
            expected_state_version=running.state_version - 1,
        )

    db_session.expire_all()
    stored = db_session.get(Task, int(task.id))
    assert stored is not None
    assert stored.status == TaskStatus.RUNNING
    assert stored.state_version == running.state_version


def test_run_and_version_fences_raise_distinguishable_errors(db_session) -> None:
    """A rotated run and a moved version must not look alike to a caller.

    They mean opposite things: a rotated run targets an execution that no
    longer exists and is terminal, while a moved version is still this run's
    row and is worth re-reading. Callers cannot tell them apart by message,
    so the version fence raises its own subclass.
    """

    task = _create_task(db_session)
    running = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.RUNNING,
        status=TaskStatus.RUNNING,
        new_run=True,
    )

    with pytest.raises(StaleTaskRunError) as rotated:
        transition_task_control_state_sync(
            int(task.id),
            TaskControlState.PAUSED,
            expected_run_id="some-other-run",
        )
    assert not isinstance(rotated.value, StaleTaskStateVersionError)

    with pytest.raises(StaleTaskStateVersionError):
        transition_task_control_state_sync(
            int(task.id),
            TaskControlState.PAUSED,
            expected_run_id=running.run_id,
            expected_state_version=running.state_version - 1,
        )


def test_transition_rejects_a_version_that_moved_under_a_stale_orm_row(
    db_session,
) -> None:
    """The SQL fence, not the Python pre-check, is what stops a lost update.

    ``apply_task_control_transition`` compares ``expected_state_version``
    against the ORM row it was handed. When that row was loaded before a
    concurrent writer committed, the comparison passes and only the
    ``WHERE coalesce(state_version, 0) = :expected`` predicate on the UPDATE
    is left to catch it. This is the branch that raises from the rowcount
    check rather than the pre-check, and its message has to name both fences:
    an operator reading "no longer belongs to run X" while the run never
    changed is sent the wrong way.
    """

    task = _create_task(db_session)
    running = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.RUNNING,
        status=TaskStatus.RUNNING,
        new_run=True,
    )

    SessionLocal = get_session_local()
    reader = SessionLocal()
    try:
        # Loaded before the concurrent write, so its in-memory version is the
        # one the pre-check will (wrongly) accept.
        stale_row = reader.query(Task).filter(Task.id == int(task.id)).first()
        assert stale_row is not None
        stale_version = int(stale_row.state_version)

        with SessionLocal() as writer:
            winner = writer.query(Task).filter(Task.id == int(task.id)).first()
            assert winner is not None
            apply_task_control_transition(
                winner,
                TaskControlState.PAUSED,
                status=TaskStatus.PAUSED,
                expected_run_id=running.run_id,
            )
            writer.commit()

        # The rowcount miss cannot tell which fence rejected it, so when a
        # version was supplied it reports the deferrable type -- the safe half
        # of that ambiguity, since the retry reads a fresh row and can then
        # tell precisely. This is the only path that reaches that selection;
        # the pre-check above covers the unambiguous cases.
        with pytest.raises(StaleTaskStateVersionError, match="at state version"):
            apply_task_control_transition(
                stale_row,
                TaskControlState.RESUME_REQUESTED,
                expected_run_id=running.run_id,
                expected_state_version=stale_version,
            )
    finally:
        reader.rollback()
        reader.close()

    # The concurrent writer's state survives; nothing was clobbered.
    db_session.expire_all()
    stored = db_session.get(Task, int(task.id))
    assert stored is not None
    assert stored.status == TaskStatus.PAUSED
    assert stored.control_state == TaskControlState.PAUSED.value


def test_concurrent_transitions_get_distinct_atomic_versions(db_session) -> None:
    task = _create_task(db_session)
    running = transition_task_control_state_sync(
        int(task.id),
        TaskControlState.RUNNING,
        status=TaskStatus.RUNNING,
        new_run=True,
    )

    def transition(state: TaskControlState):
        return transition_task_control_state_sync(
            int(task.id),
            state,
            expected_run_id=running.run_id,
        )

    with ThreadPoolExecutor(max_workers=2) as executor:
        pause_future = executor.submit(transition, TaskControlState.PAUSE_REQUESTED)
        resume_future = executor.submit(transition, TaskControlState.RESUME_REQUESTED)
        snapshots = [pause_future.result(), resume_future.result()]

    assert {snapshot.state_version for snapshot in snapshots} == {2, 3}
    assert {snapshot.run_id for snapshot in snapshots} == {running.run_id}