Enterprise agent platform for building, deploying, and operating AI teammates.
"""Durable form responses defer while the live-control resume slot is busy."""
from __future__ import annotations
import asyncio
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from typing import Iterator
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.web.services.task_lease_shared import (
live_task_lease as live_task_lease_fixture,
)
from xagent.core.agent.runner import UserMessageInjectionOutcome
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.database import Base, get_db, get_engine, init_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_command_terminal_event import TaskCommandTerminalEvent
from xagent.web.models.user import User
from xagent.web.services import task_command_execution as command_execution_service
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services.chat_history_service import DELIVERY_PENDING
from xagent.web.services.task_command_execution import execute_durable_task_command
from xagent.web.services.task_command_transport import (
COMMAND_COMPLETED,
COMMAND_FAILED,
COMMAND_PENDING,
COMMAND_PROCESSING,
ClaimedTaskCommand,
TaskCommandDeferred,
TaskCommandKind,
dispatch_one_task_command,
enqueue_task_command,
get_runner_id,
max_command_defers,
)
from xagent.web.services.task_dispatch_cap import DispatchCapSnapshot, DispatchPolicy
from xagent.web.services.task_execution import ResumeReservationOutcome
live_task_lease = live_task_lease_fixture
@pytest.fixture()
def db_session(tmp_path):
init_db(db_url=f"sqlite:///{tmp_path / 'resume_contention.db'}")
db = next(get_db())
try:
yield db
finally:
db.close()
Base.metadata.drop_all(bind=get_engine())
def _user(db, username: str) -> User:
user = User(username=username, password_hash="x", is_admin=False)
db.add(user)
db.commit()
db.refresh(user)
return user
def _live_task(db, owner_id: int) -> Task:
task = Task(
user_id=owner_id,
title="t",
description="d",
status=TaskStatus.RUNNING,
execution_mode="balanced",
source="sdk",
)
db.add(task)
db.commit()
db.refresh(task)
task.runner_id = "live-runner"
task.run_id = "live-run"
db.commit()
return task
def _message_command(
task: Task,
owner: User,
command_id: str,
*,
attempt_count: int = 1,
) -> ClaimedTaskCommand:
return ClaimedTaskCommand(
id=1,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id=command_id,
kind=TaskCommandKind.MESSAGE,
payload={
"type": "chat_message",
"message": "apply this form response once",
"client_message_id": command_id,
"files": [],
},
target_run_id="live-run",
attempt_count=attempt_count,
)
@contextmanager
def _live_control_environment(
*,
outcome: ResumeReservationOutcome | None = None,
background_manager=None,
) -> Iterator[tuple[MagicMock, object]]:
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
# The injection outcome is read two ways downstream: truthiness (did an
# exact checkpoint accept the message at all) and identity against
# ``POSTED_FRESH`` (did this call write a new turn, as opposed to a
# replayed one). A bare ``True`` satisfies the first read but is never
# ``is POSTED_FRESH``, so it silently skips the branch that fires only
# on identity -- the stub has to be a real enum member to exercise that
# branch at all.
agent.post_user_message = AsyncMock(
return_value=UserMessageInjectionOutcome.POSTED_FRESH
)
if background_manager is None:
background_manager = MagicMock()
background_manager.try_reserve_resume.return_value = outcome
background_manager.resume_holder_age_seconds.return_value = None
background_manager.running_tasks.get.return_value = None
# An open cap: the dispatcher reads the snapshot and takes a slot.
background_manager.dispatch_cap_snapshot.return_value = DispatchCapSnapshot(
policy=DispatchPolicy(limit=80, interactive_reserve=0),
active_ids=frozenset(),
)
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch(
"xagent.web.api.websocket.manager",
MagicMock(
broadcast_to_task=AsyncMock(),
send_personal_message=AsyncMock(),
),
),
patch(
"xagent.web.services.task_execution.execute_resume_background", AsyncMock()
),
patch(
"xagent.web.services.task_execution.background_task_manager",
background_manager,
),
# A message that opens a new turn schedules a real background run.
# Against the stub agent that run fails the task, racing the test's
# assertions on the turn it just started, so never start it here.
patch(
"xagent.web.services.task_orchestrator._schedule_bg",
side_effect=_unstarted_background_turn,
),
):
yield agent, background_manager
def _unstarted_background_turn(**_kwargs: object) -> "asyncio.Task[None]":
async def unstarted() -> None:
return None
return asyncio.create_task(unstarted())
@pytest.mark.asyncio
async def test_cancel_clears_pre_registration_resume_reservation() -> None:
background_manager = task_execution_service.BackgroundTaskManager()
assert (
background_manager.try_reserve_resume(7, expected_run_id="live-run")
is ResumeReservationOutcome.RESERVED
)
assert background_manager.resume_holder_age_seconds(7) is not None
outcome = await background_manager.cancel_task(7)
assert outcome.requested is False
assert background_manager.resume_holder_age_seconds(7) is None
assert (
background_manager.try_reserve_resume(7, expected_run_id="live-run")
is ResumeReservationOutcome.RESERVED
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"outcome",
[
ResumeReservationOutcome.RESERVATION_HELD,
ResumeReservationOutcome.COORDINATOR_RUNNING,
ResumeReservationOutcome.SHUTTING_DOWN,
],
)
async def test_contended_durable_message_defers_without_injection(
db_session,
outcome: ResumeReservationOutcome,
) -> None:
owner = _user(db_session, f"contention-owner-{outcome.value}")
task = _live_task(db_session, int(owner.id))
command = _message_command(task, owner, "form-turn")
with (
_live_control_environment(outcome=outcome) as (agent, _),
pytest.raises(TaskCommandDeferred) as exc_info,
):
await execute_durable_task_command(command)
assert "resume slot" in str(exc_info.value)
assert exc_info.value.resend_safe is True
agent.post_user_message.assert_not_awaited()
@pytest.mark.asyncio
async def test_recovered_delivery_makes_contention_unsafe_to_resend(
db_session,
) -> None:
owner = _user(db_session, "recovered-delivery-owner")
task = _live_task(db_session, int(owner.id))
command_id = "recovered-form-turn"
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
message_type="user_message",
content="apply this form response once",
turn_id=command_id,
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
command = _message_command(
task,
owner,
command_id,
attempt_count=2,
)
with (
_live_control_environment(
outcome=ResumeReservationOutcome.RESERVATION_HELD
) as (agent, _),
pytest.raises(TaskCommandDeferred) as exc_info,
):
await execute_durable_task_command(command)
assert exc_info.value.resend_safe is False
agent.post_user_message.assert_not_awaited()
@pytest.mark.asyncio
async def test_dispatcher_reclaims_and_applies_message_after_contention_clears(
live_task_lease,
db_session,
caplog: pytest.LogCaptureFixture,
) -> None:
caplog.set_level("INFO", logger=command_execution_service.__name__)
owner = _user(db_session, "eventual-delivery-owner")
task = _live_task(db_session, int(owner.id))
task.runner_id = get_runner_id()
task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=1)
db_session.commit()
live_task_lease(db_session, task)
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id="eventual-form-turn",
kind=TaskCommandKind.MESSAGE,
payload={
"type": "chat_message",
"message": "apply this form response once",
"client_message_id": "eventual-form-turn",
"files": [],
},
)
db_session.commit()
background_manager = task_execution_service.BackgroundTaskManager()
assert (
background_manager.try_reserve_resume(
int(task.id),
expected_run_id="live-run",
)
is ResumeReservationOutcome.RESERVED
)
with _live_control_environment(background_manager=background_manager) as (
agent,
_,
):
assert await dispatch_one_task_command(
execute_durable_task_command,
command_db_id=enqueued.command_id,
)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_PENDING
assert stored.failure_count == 0
assert stored.defer_count == 1
# A defer that still has budget must not stage a terminal event.
assert (
db_session.query(TaskCommandTerminalEvent)
.filter(TaskCommandTerminalEvent.task_command_id == enqueued.command_id)
.count()
== 0
)
assert "holder_age_seconds=" in caplog.text
assert "holder_age_seconds=unknown" not in caplog.text
background_manager.release_resume_reservation(int(task.id))
stored.claim_expires_at = None
db_session.commit()
# A stub that is truthy but not `is POSTED_FRESH` would let the
# dispatch above still report success while silently skipping the
# legacy-interaction close below it -- truthiness alone cannot tell
# the two apart. Patching with a spy lets us assert
# `close_spy.call_count == 1`, which fails if that regression
# silently skips the close; `wraps=` is added separately so the
# real close still executes for its own side effects (the DB
# write), keeping the rest of the test's assertions valid.
with patch.object(
command_execution_service,
"close_legacy_resume_interaction_sync",
wraps=command_execution_service.close_legacy_resume_interaction_sync,
) as close_spy:
assert await dispatch_one_task_command(
execute_durable_task_command,
command_db_id=enqueued.command_id,
)
await asyncio.sleep(0)
assert close_spy.call_count == 1
# The call site swallows any internal failure into a warning
# (`except Exception: logger.warning(...)`), so call_count alone
# cannot tell a successful close from one that raised and was
# silently caught. Assert the warning is absent to catch that case.
assert "legacy resume interaction close failed" not in caplog.text
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_COMPLETED
assert stored.failure_count == 0
assert stored.defer_count == 1
agent.post_user_message.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("unsettled_prior_claim", "recovered_delivery", "expected_resend_safe"),
[
(False, False, True),
# Models worker B reclaiming an expired attempt from worker A before
# worker A notices claim loss. The missing delivery row is not durable
# proof that worker A cannot resume and inject after this read.
(True, False, False),
# A recovered delivery row alone must force unsafe even when the
# claim arithmetic is clean: this pins the marker term in isolation.
(False, True, False),
(True, True, False),
],
)
async def test_exhausted_contention_persists_evidence_based_resend_safety(
db_session,
unsettled_prior_claim: bool,
recovered_delivery: bool,
expected_resend_safe: bool,
) -> None:
owner = _user(
db_session,
f"terminal-owner-{unsettled_prior_claim}-{recovered_delivery}",
)
task = _live_task(db_session, int(owner.id))
task.runner_id = get_runner_id()
task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=1)
db_session.commit()
command_id = f"terminal-form-turn-{unsettled_prior_claim}-{recovered_delivery}"
enqueued = enqueue_task_command(
db_session,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id=command_id,
kind=TaskCommandKind.MESSAGE,
payload={
"type": "chat_message",
"message": "apply this form response once",
"client_message_id": command_id,
"files": [],
},
)
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
stored.defer_count = max_command_defers() - 1
# Each prior clean contention both claimed and deferred once. A stale
# worker adds one unmatched claim that the reclaiming worker must treat as
# possible in-flight injection evidence even when no delivery row exists.
stored.attempt_count = max_command_defers() - 1
if unsettled_prior_claim:
stored.status = COMMAND_PROCESSING
stored.claimed_by = "stale-worker"
stored.claim_expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
stored.attempt_count += 1
if recovered_delivery:
db_session.add(
TaskChatMessage(
task_id=int(task.id),
user_id=int(owner.id),
role="user",
message_type="user_message",
content="apply this form response once",
turn_id=command_id,
delivery_status=DELIVERY_PENDING,
)
)
db_session.commit()
with _live_control_environment(
outcome=ResumeReservationOutcome.RESERVATION_HELD
) as (agent, _):
assert await dispatch_one_task_command(
execute_durable_task_command,
command_db_id=enqueued.command_id,
)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored is not None
assert stored.status == COMMAND_FAILED
assert stored.failure_count == 0
assert stored.defer_count == max_command_defers()
event = (
db_session.query(TaskCommandTerminalEvent)
.filter(TaskCommandTerminalEvent.task_command_id == enqueued.command_id)
.one()
)
assert event.resend_safe is expected_resend_safe
agent.post_user_message.assert_not_awaited()