Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

15.5 KB
Raw Blame
"""Durable form responses defer while the live-control resume slot is busy."""

from __future__ import annotations

import asyncio
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from typing import Iterator
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from tests.web.services.task_lease_shared import (
    live_task_lease as live_task_lease_fixture,
)
from xagent.core.agent.runner import UserMessageInjectionOutcome
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_command_terminal_event import TaskCommandTerminalEvent
from xagent.web.models.user import User
from xagent.web.services import task_command_execution as command_execution_service
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services.chat_history_service import DELIVERY_PENDING
from xagent.web.services.task_command_execution import execute_durable_task_command
from xagent.web.services.task_command_transport import (
    COMMAND_COMPLETED,
    COMMAND_FAILED,
    COMMAND_PENDING,
    COMMAND_PROCESSING,
    ClaimedTaskCommand,
    TaskCommandDeferred,
    TaskCommandKind,
    dispatch_one_task_command,
    enqueue_task_command,
    get_runner_id,
    max_command_defers,
)
from xagent.web.services.task_dispatch_cap import DispatchCapSnapshot, DispatchPolicy
from xagent.web.services.task_execution import ResumeReservationOutcome

live_task_lease = live_task_lease_fixture


@pytest.fixture()
def db_session(tmp_path):
    init_db(db_url=f"sqlite:///{tmp_path / 'resume_contention.db'}")
    db = next(get_db())
    try:
        yield db
    finally:
        db.close()
        Base.metadata.drop_all(bind=get_engine())


def _user(db, username: str) -> User:
    user = User(username=username, password_hash="x", is_admin=False)
    db.add(user)
    db.commit()
    db.refresh(user)
    return user


def _live_task(db, owner_id: int) -> Task:
    task = Task(
        user_id=owner_id,
        title="t",
        description="d",
        status=TaskStatus.RUNNING,
        execution_mode="balanced",
        source="sdk",
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    task.runner_id = "live-runner"
    task.run_id = "live-run"
    db.commit()
    return task


def _message_command(
    task: Task,
    owner: User,
    command_id: str,
    *,
    attempt_count: int = 1,
) -> ClaimedTaskCommand:
    return ClaimedTaskCommand(
        id=1,
        task_id=int(task.id),
        actor_user_id=int(owner.id),
        command_id=command_id,
        kind=TaskCommandKind.MESSAGE,
        payload={
            "type": "chat_message",
            "message": "apply this form response once",
            "client_message_id": command_id,
            "files": [],
        },
        target_run_id="live-run",
        attempt_count=attempt_count,
    )


@contextmanager
def _live_control_environment(
    *,
    outcome: ResumeReservationOutcome | None = None,
    background_manager=None,
) -> Iterator[tuple[MagicMock, object]]:
    agent = MagicMock()
    agent.supports_live_control.return_value = True
    agent.get_dag_pattern.return_value = None
    # The injection outcome is read two ways downstream: truthiness (did an
    # exact checkpoint accept the message at all) and identity against
    # ``POSTED_FRESH`` (did this call write a new turn, as opposed to a
    # replayed one). A bare ``True`` satisfies the first read but is never
    # ``is POSTED_FRESH``, so it silently skips the branch that fires only
    # on identity -- the stub has to be a real enum member to exercise that
    # branch at all.
    agent.post_user_message = AsyncMock(
        return_value=UserMessageInjectionOutcome.POSTED_FRESH
    )
    if background_manager is None:
        background_manager = MagicMock()
        background_manager.try_reserve_resume.return_value = outcome
        background_manager.resume_holder_age_seconds.return_value = None
        background_manager.running_tasks.get.return_value = None
        # An open cap: the dispatcher reads the snapshot and takes a slot.
        background_manager.dispatch_cap_snapshot.return_value = DispatchCapSnapshot(
            policy=DispatchPolicy(limit=80, interactive_reserve=0),
            active_ids=frozenset(),
        )

    with (
        patch(
            "xagent.web.services.agent_service_manager.get_agent_manager",
            return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
        ),
        patch(
            "xagent.web.api.websocket.manager",
            MagicMock(
                broadcast_to_task=AsyncMock(),
                send_personal_message=AsyncMock(),
            ),
        ),
        patch(
            "xagent.web.services.task_execution.execute_resume_background", AsyncMock()
        ),
        patch(
            "xagent.web.services.task_execution.background_task_manager",
            background_manager,
        ),
        # A message that opens a new turn schedules a real background run.
        # Against the stub agent that run fails the task, racing the test's
        # assertions on the turn it just started, so never start it here.
        patch(
            "xagent.web.services.task_orchestrator._schedule_bg",
            side_effect=_unstarted_background_turn,
        ),
    ):
        yield agent, background_manager


def _unstarted_background_turn(**_kwargs: object) -> "asyncio.Task[None]":
    async def unstarted() -> None:
        return None

    return asyncio.create_task(unstarted())


@pytest.mark.asyncio
async def test_cancel_clears_pre_registration_resume_reservation() -> None:
    background_manager = task_execution_service.BackgroundTaskManager()
    assert (
        background_manager.try_reserve_resume(7, expected_run_id="live-run")
        is ResumeReservationOutcome.RESERVED
    )
    assert background_manager.resume_holder_age_seconds(7) is not None

    outcome = await background_manager.cancel_task(7)

    assert outcome.requested is False
    assert background_manager.resume_holder_age_seconds(7) is None
    assert (
        background_manager.try_reserve_resume(7, expected_run_id="live-run")
        is ResumeReservationOutcome.RESERVED
    )


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "outcome",
    [
        ResumeReservationOutcome.RESERVATION_HELD,
        ResumeReservationOutcome.COORDINATOR_RUNNING,
        ResumeReservationOutcome.SHUTTING_DOWN,
    ],
)
async def test_contended_durable_message_defers_without_injection(
    db_session,
    outcome: ResumeReservationOutcome,
) -> None:
    owner = _user(db_session, f"contention-owner-{outcome.value}")
    task = _live_task(db_session, int(owner.id))
    command = _message_command(task, owner, "form-turn")

    with (
        _live_control_environment(outcome=outcome) as (agent, _),
        pytest.raises(TaskCommandDeferred) as exc_info,
    ):
        await execute_durable_task_command(command)

    assert "resume slot" in str(exc_info.value)
    assert exc_info.value.resend_safe is True
    agent.post_user_message.assert_not_awaited()


@pytest.mark.asyncio
async def test_recovered_delivery_makes_contention_unsafe_to_resend(
    db_session,
) -> None:
    owner = _user(db_session, "recovered-delivery-owner")
    task = _live_task(db_session, int(owner.id))
    command_id = "recovered-form-turn"
    db_session.add(
        TaskChatMessage(
            task_id=int(task.id),
            user_id=int(owner.id),
            role="user",
            message_type="user_message",
            content="apply this form response once",
            turn_id=command_id,
            delivery_status=DELIVERY_PENDING,
        )
    )
    db_session.commit()
    command = _message_command(
        task,
        owner,
        command_id,
        attempt_count=2,
    )

    with (
        _live_control_environment(
            outcome=ResumeReservationOutcome.RESERVATION_HELD
        ) as (agent, _),
        pytest.raises(TaskCommandDeferred) as exc_info,
    ):
        await execute_durable_task_command(command)

    assert exc_info.value.resend_safe is False
    agent.post_user_message.assert_not_awaited()


@pytest.mark.asyncio
async def test_dispatcher_reclaims_and_applies_message_after_contention_clears(
    live_task_lease,
    db_session,
    caplog: pytest.LogCaptureFixture,
) -> None:
    caplog.set_level("INFO", logger=command_execution_service.__name__)
    owner = _user(db_session, "eventual-delivery-owner")
    task = _live_task(db_session, int(owner.id))
    task.runner_id = get_runner_id()
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=1)
    db_session.commit()
    live_task_lease(db_session, task)

    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(owner.id),
        command_id="eventual-form-turn",
        kind=TaskCommandKind.MESSAGE,
        payload={
            "type": "chat_message",
            "message": "apply this form response once",
            "client_message_id": "eventual-form-turn",
            "files": [],
        },
    )
    db_session.commit()

    background_manager = task_execution_service.BackgroundTaskManager()
    assert (
        background_manager.try_reserve_resume(
            int(task.id),
            expected_run_id="live-run",
        )
        is ResumeReservationOutcome.RESERVED
    )
    with _live_control_environment(background_manager=background_manager) as (
        agent,
        _,
    ):
        assert await dispatch_one_task_command(
            execute_durable_task_command,
            command_db_id=enqueued.command_id,
        )

        db_session.expire_all()
        stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
        assert stored is not None
        assert stored.status == COMMAND_PENDING
        assert stored.failure_count == 0
        assert stored.defer_count == 1
        # A defer that still has budget must not stage a terminal event.
        assert (
            db_session.query(TaskCommandTerminalEvent)
            .filter(TaskCommandTerminalEvent.task_command_id == enqueued.command_id)
            .count()
            == 0
        )
        assert "holder_age_seconds=" in caplog.text
        assert "holder_age_seconds=unknown" not in caplog.text

        background_manager.release_resume_reservation(int(task.id))
        stored.claim_expires_at = None
        db_session.commit()
        # A stub that is truthy but not `is POSTED_FRESH` would let the
        # dispatch above still report success while silently skipping the
        # legacy-interaction close below it -- truthiness alone cannot tell
        # the two apart. Patching with a spy lets us assert
        # `close_spy.call_count == 1`, which fails if that regression
        # silently skips the close; `wraps=` is added separately so the
        # real close still executes for its own side effects (the DB
        # write), keeping the rest of the test's assertions valid.
        with patch.object(
            command_execution_service,
            "close_legacy_resume_interaction_sync",
            wraps=command_execution_service.close_legacy_resume_interaction_sync,
        ) as close_spy:
            assert await dispatch_one_task_command(
                execute_durable_task_command,
                command_db_id=enqueued.command_id,
            )
            await asyncio.sleep(0)
        assert close_spy.call_count == 1
        # The call site swallows any internal failure into a warning
        # (`except Exception: logger.warning(...)`), so call_count alone
        # cannot tell a successful close from one that raised and was
        # silently caught. Assert the warning is absent to catch that case.
        assert "legacy resume interaction close failed" not in caplog.text

    db_session.expire_all()
    stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
    assert stored is not None
    assert stored.status == COMMAND_COMPLETED
    assert stored.failure_count == 0
    assert stored.defer_count == 1
    agent.post_user_message.assert_awaited_once()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("unsettled_prior_claim", "recovered_delivery", "expected_resend_safe"),
    [
        (False, False, True),
        # Models worker B reclaiming an expired attempt from worker A before
        # worker A notices claim loss. The missing delivery row is not durable
        # proof that worker A cannot resume and inject after this read.
        (True, False, False),
        # A recovered delivery row alone must force unsafe even when the
        # claim arithmetic is clean: this pins the marker term in isolation.
        (False, True, False),
        (True, True, False),
    ],
)
async def test_exhausted_contention_persists_evidence_based_resend_safety(
    db_session,
    unsettled_prior_claim: bool,
    recovered_delivery: bool,
    expected_resend_safe: bool,
) -> None:
    owner = _user(
        db_session,
        f"terminal-owner-{unsettled_prior_claim}-{recovered_delivery}",
    )
    task = _live_task(db_session, int(owner.id))
    task.runner_id = get_runner_id()
    task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=1)
    db_session.commit()
    command_id = f"terminal-form-turn-{unsettled_prior_claim}-{recovered_delivery}"
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(owner.id),
        command_id=command_id,
        kind=TaskCommandKind.MESSAGE,
        payload={
            "type": "chat_message",
            "message": "apply this form response once",
            "client_message_id": command_id,
            "files": [],
        },
    )
    stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
    assert stored is not None
    stored.defer_count = max_command_defers() - 1
    # Each prior clean contention both claimed and deferred once. A stale
    # worker adds one unmatched claim that the reclaiming worker must treat as
    # possible in-flight injection evidence even when no delivery row exists.
    stored.attempt_count = max_command_defers() - 1
    if unsettled_prior_claim:
        stored.status = COMMAND_PROCESSING
        stored.claimed_by = "stale-worker"
        stored.claim_expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
        stored.attempt_count += 1
    if recovered_delivery:
        db_session.add(
            TaskChatMessage(
                task_id=int(task.id),
                user_id=int(owner.id),
                role="user",
                message_type="user_message",
                content="apply this form response once",
                turn_id=command_id,
                delivery_status=DELIVERY_PENDING,
            )
        )
    db_session.commit()

    with _live_control_environment(
        outcome=ResumeReservationOutcome.RESERVATION_HELD
    ) as (agent, _):
        assert await dispatch_one_task_command(
            execute_durable_task_command,
            command_db_id=enqueued.command_id,
        )

    db_session.expire_all()
    stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
    assert stored is not None
    assert stored.status == COMMAND_FAILED
    assert stored.failure_count == 0
    assert stored.defer_count == max_command_defers()
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == enqueued.command_id)
        .one()
    )
    assert event.resend_safe is expected_resend_safe
    agent.post_user_message.assert_not_awaited()