Enterprise agent platform for building, deploying, and operating AI teammates.
"""A recovered delivery never resumes a run that has ended.
A durable MESSAGE command claimed its delivery row on the task's own run and
crashed before settling it. The run then ended: lease recovery found it not
recoverable and settled the task FAILED, or its runner finished it COMPLETED.
The retry finds the pending row as a recovered claim on the command's own
run, which is normally redriven through a resume so the run replays the turn
id against its checkpoint. An ended run must not be resumed: the retry
settles the command as accepted with an unknown outcome, advances the row out
of ``pending`` without running the turn, and leaves the task in its terminal
status with its control state, run, diagnostic and result untouched.
The routing snapshot can be stale, so the refusal is also enforced by the
RESUME_REQUESTED transition and by the resume lease claim, each as part of
its own conditional UPDATE.
"""
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from typing import Any, Iterator
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from sqlalchemy.exc import OperationalError
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from tests.web.api.test_durable_message_resume_contention import (
_live_control_environment,
_live_task,
_message_command,
_user,
live_task_lease,
)
from tests.web.api.test_recovered_delivery_outcome_unknown import (
MESSAGE,
TURN_ID,
_assert_outcome_unknown_frames,
_enqueue_settled_unknown_command,
_outcome_unknown_result,
_pending_row,
_RecordingReply,
_row_status,
_settled_task,
_user_rows,
)
from xagent.core.agent import Agent, ExecutionContext
from xagent.core.agent.runner import AgentRunner
from xagent.core.guardrails import input as input_review
from xagent.web.api import websocket as websocket_api
from xagent.web.models.database import (
Base,
get_db,
get_engine,
get_session_local,
init_db,
)
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.services import task_command_execution as command_execution_service
from xagent.web.services import task_execution as task_execution_service
from xagent.web.services import task_execution_controller as controller_module
from xagent.web.services.chat_history_service import (
DELIVERY_DISPATCHED,
DELIVERY_FAILED,
DELIVERY_OUTCOME_UNKNOWN,
DELIVERY_PENDING,
)
from xagent.web.services.client_error_messages import ClientErrorCode
from xagent.web.services.task_command_execution import execute_durable_task_command
from xagent.web.services.task_command_transport import (
COMMAND_COMPLETED,
COMMAND_PENDING,
COMMAND_PROCESSING,
TaskCommandDeferred,
TaskCommandKind,
TaskCommandRejected,
dispatch_one_task_command,
enqueue_task_command,
get_runner_id,
)
from xagent.web.services.task_execution import ResumeReservationOutcome
from xagent.web.services.task_execution_controller import (
StaleTaskRunError,
TaskControlState,
TaskStatusRefusedError,
apply_task_control_transition,
)
from xagent.web.services.task_lease_service import (
TaskLease,
get_expired_task_lease_candidates,
recover_expired_task_lease_no_commit,
release_task_lease_no_commit,
)
from xagent.web.services.task_orchestrator import TaskTurnOrchestrator, TurnKind
# Re-exported fixture from the contention suite.
live_task_lease = live_task_lease
RECOVERY_ERROR = "not recoverable: checkpoint missing"
FINAL_OUTPUT = "the run's final answer"
ENDED_STATUSES = pytest.mark.parametrize(
"ended_status", [TaskStatus.FAILED, TaskStatus.COMPLETED]
)
@pytest.fixture()
def db_session(tmp_path):
init_db(db_url=f"sqlite:///{tmp_path / 'recovered_failed_run.db'}")
db = next(get_db())
try:
yield db
finally:
db.close()
Base.metadata.drop_all(bind=get_engine())
@pytest.fixture()
def recording_reply() -> Iterator[_RecordingReply]:
reply = _RecordingReply()
with patch.object(command_execution_service, "command_reply", return_value=reply):
yield reply
def _expired_running_task(db, owner_id: int) -> Task:
"""A RUNNING task on the command's own run whose lease has expired."""
task = _live_task(db, owner_id)
task.runner_id = "dead-runner"
task.lease_attempt_id = "dead-attempt"
task.lease_expires_at = datetime.now(timezone.utc) - timedelta(minutes=5)
task.control_state = TaskControlState.RUNNING.value
db.commit()
db.refresh(task)
return task
def _recover_to_failed(task_id: int) -> None:
"""Run lease recovery's NOT_RECOVERABLE settlement in its own session."""
with get_session_local()() as db:
now = datetime.now(timezone.utc)
candidate = next(
candidate
for candidate in get_expired_task_lease_candidates(db, cutoff=now, limit=10)
if candidate.task_id == task_id
)
assert recover_expired_task_lease_no_commit(
db,
candidate,
status=TaskStatus.FAILED,
recovered_at=now,
error_message=RECOVERY_ERROR,
)
db.commit()
def _complete_run(task_id: int) -> None:
"""The run's own runner finishes it COMPLETED, in its own session."""
with get_session_local()() as db:
db.query(Task).filter(Task.id == task_id).update(
{Task.output: FINAL_OUTPUT}, synchronize_session=False
)
assert release_task_lease_no_commit(
db,
TaskLease(
task_id=task_id,
runner_id="dead-runner",
run_id="live-run",
attempt_id="dead-attempt",
),
status=TaskStatus.COMPLETED,
)
db.commit()
def _end_run(task_id: int, ended_status: TaskStatus) -> None:
if ended_status == TaskStatus.FAILED:
_recover_to_failed(task_id)
else:
assert ended_status == TaskStatus.COMPLETED
_complete_run(task_id)
def _ended_task(db, owner_id: int, ended_status: TaskStatus) -> Task:
task = _expired_running_task(db, owner_id)
_end_run(int(task.id), ended_status)
db.expire_all()
task = db.get(Task, int(task.id))
assert task.status == ended_status
return task
def _assert_still_ended(
db, task_id: int, ended_status: TaskStatus, *, state_version: int
) -> None:
db.expire_all()
stored = db.get(Task, task_id)
assert stored.status == ended_status
assert stored.control_state == ended_status.value.lower()
if ended_status == TaskStatus.FAILED:
assert stored.error_message == RECOVERY_ERROR
else:
assert stored.output == FINAL_OUTPUT
assert stored.error_message is None
assert stored.run_id == "live-run"
assert stored.runner_id is None
assert int(stored.state_version or 0) == state_version
@pytest.mark.asyncio
@ENDED_STATUSES
async def test_recovered_delivery_on_ended_run_settles_outcome_unknown(
db_session,
recording_reply: _RecordingReply,
ended_status: TaskStatus,
) -> None:
owner = _user(db_session, f"ended-run-owner-{ended_status.value}")
task = _ended_task(db_session, int(owner.id), ended_status)
task_id = int(task.id)
state_version = int(task.state_version or 0)
_pending_row(db_session, task, int(owner.id))
publish = AsyncMock()
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED) as (
agent,
background_manager,
),
patch.object(task_execution_service, "publish_task_event", publish),
):
result = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
task_execution_service.execute_resume_background.assert_not_called()
assert result == _outcome_unknown_result(task)
agent.post_user_message.assert_not_awaited()
background_manager.try_reserve_resume.assert_not_called()
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
assert len(_user_rows(db_session, task_id)) == 1
_assert_still_ended(db_session, task_id, ended_status, state_version=state_version)
_assert_outcome_unknown_frames(recording_reply)
publish.assert_not_awaited()
@pytest.mark.asyncio
@ENDED_STATUSES
@pytest.mark.parametrize("fence", ["pre_check", "conditional_update"])
async def test_run_ended_after_routing_snapshot_is_caught_by_the_transition(
db_session,
recording_reply: _RecordingReply,
fence: str,
ended_status: TaskStatus,
) -> None:
"""The run ends after the handler routed on a RUNNING row.
``pre_check`` lands it before the transition loads the row;
``conditional_update`` lands it after the load, so only the UPDATE's own
status predicate can refuse it.
"""
owner = _user(db_session, f"transition-race-{fence}-{ended_status.value}")
task = _expired_running_task(db_session, int(owner.id))
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
if fence == "pre_check":
real_sync = controller_module.transition_task_control_state_sync
def recover_then_transition(*args: Any, **kwargs: Any):
_end_run(task_id, ended_status)
return real_sync(*args, **kwargs)
race = patch.object(
controller_module,
"transition_task_control_state_sync",
side_effect=recover_then_transition,
)
else:
real_apply = controller_module.apply_task_control_transition
def recover_then_apply(task_row: Task, *args: Any, **kwargs: Any):
assert task_row.status == TaskStatus.RUNNING
_end_run(task_id, ended_status)
return real_apply(task_row, *args, **kwargs)
race = patch.object(
controller_module,
"apply_task_control_transition",
side_effect=recover_then_apply,
)
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED) as (
agent,
background_manager,
),
race as raced,
):
result = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
task_execution_service.execute_resume_background.assert_not_called()
assert raced.call_count == 1
assert raced.call_args.kwargs["refuse_terminal_status"] is True
assert result == _outcome_unknown_result(task)
background_manager.release_resume_reservation.assert_called_with(task_id)
background_manager.register_reserved_resume.assert_not_called()
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
db_session.expire_all()
ended_version = int(db_session.get(Task, task_id).state_version or 0)
_assert_still_ended(db_session, task_id, ended_status, state_version=ended_version)
_assert_outcome_unknown_frames(recording_reply)
agent.post_user_message.assert_not_awaited()
@pytest.mark.asyncio
@ENDED_STATUSES
async def test_run_ended_after_the_transition_is_caught_by_the_lease_claim(
db_session,
recording_reply: _RecordingReply,
ended_status: TaskStatus,
) -> None:
"""The run ends after RESUME_REQUESTED, before the resume claims it."""
owner = _user(db_session, f"claim-race-owner-{ended_status.value}")
task = _expired_running_task(db_session, int(owner.id))
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
real_sync = controller_module.transition_task_control_state_sync
def transition_then_recover(*args: Any, **kwargs: Any):
snapshot = real_sync(*args, **kwargs)
_end_run(task_id, ended_status)
return snapshot
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.post_user_message = AsyncMock()
background_manager = task_execution_service.BackgroundTaskManager()
resume_spy = AsyncMock(wraps=task_execution_service.execute_resume_background)
with (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch.object(
task_execution_service, "background_task_manager", background_manager
),
patch.object(task_execution_service, "execute_resume_background", resume_spy),
patch.object(task_execution_service, "publish_task_event", AsyncMock()),
patch.object(
controller_module,
"transition_task_control_state_sync",
side_effect=transition_then_recover,
),
):
try:
first = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
except TaskCommandDeferred:
# The handoff returned before the resume claimed; the row was
# still pending when the command read it.
first = None
assert resume_spy.await_count == 1
assert resume_spy.await_args.kwargs["refuse_terminal_status"] is True
assert resume_spy.await_args.kwargs["delivery_claimed_fresh"] is False
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if task_id not in background_manager.running_tasks:
break
await asyncio.sleep(0.01)
assert task_id not in background_manager.running_tasks
assert _row_status(db_session, task_id) == DELIVERY_OUTCOME_UNKNOWN
recording_reply.frames.clear()
result = (
first
if first is not None
else await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=3)
)
)
assert result == _outcome_unknown_result(task)
agent.post_user_message.assert_not_awaited()
assert _row_status(db_session, task_id) == DELIVERY_OUTCOME_UNKNOWN
db_session.expire_all()
ended_version = int(db_session.get(Task, task_id).state_version or 0)
_assert_still_ended(db_session, task_id, ended_status, state_version=ended_version)
if first is None:
_assert_outcome_unknown_frames(recording_reply)
@pytest.mark.asyncio
@ENDED_STATUSES
async def test_fresh_message_on_ended_task_still_appends_a_new_run(
db_session,
ended_status: TaskStatus,
) -> None:
owner = _user(db_session, f"fresh-append-owner-{ended_status.value}")
task = _ended_task(db_session, int(owner.id), ended_status)
task_id = int(task.id)
begin_turn = AsyncMock(wraps=TaskTurnOrchestrator.begin_turn)
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED) as (
agent,
_,
),
patch.object(TaskTurnOrchestrator, "begin_turn", begin_turn),
):
result = await execute_durable_task_command(
_message_command(task, owner, "fresh-turn", attempt_count=1)
)
task_execution_service.execute_resume_background.assert_not_called()
assert result == {
"task_id": task_id,
"command_id": "fresh-turn",
"kind": TaskCommandKind.MESSAGE.value,
}
begin_turn.assert_awaited_once()
assert begin_turn.await_args.kwargs["kind"] == TurnKind.APPEND
agent.post_user_message.assert_not_awaited()
db_session.expire_all()
stored = db_session.get(Task, task_id)
assert stored.status == TaskStatus.RUNNING
assert stored.run_id not in {None, "live-run"}
@pytest.mark.asyncio
@ENDED_STATUSES
async def test_same_id_resend_after_ended_run_refusal_reports_outcome_unknown(
db_session,
recording_reply: _RecordingReply,
ended_status: TaskStatus,
) -> None:
owner = _user(db_session, f"ended-resend-owner-{ended_status.value}")
task = _ended_task(db_session, int(owner.id), ended_status)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
command = _message_command(task, owner, TURN_ID, attempt_count=2)
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
result = await execute_durable_task_command(command)
assert result == _outcome_unknown_result(task)
_enqueue_settled_unknown_command(
db_session, task, owner, dict(command.payload), result=result
)
enqueued = websocket_api._enqueue_websocket_task_command_sync(
task_id=task_id,
actor_user_id=int(owner.id),
actor_is_admin=False,
command_id=TURN_ID,
kind=TaskCommandKind.MESSAGE,
payload=dict(command.payload),
allow_missing_task=True,
)
assert enqueued is not None
assert enqueued.created is False
assert enqueued.status == DELIVERY_OUTCOME_UNKNOWN
assert enqueued.payload_matches is True
assert result["kind"] == TaskCommandKind.MESSAGE.value
assert ClientErrorCode.MESSAGE_OUTCOME_UNKNOWN.value in {
frame.get("error_code") for frame in recording_reply.frames
}
@ENDED_STATUSES
def test_refused_status_transition_leaves_the_row_untouched(
db_session, ended_status: TaskStatus
) -> None:
owner = _user(db_session, f"transition-unit-owner-{ended_status.value}")
task = _ended_task(db_session, int(owner.id), ended_status)
task_id = int(task.id)
state_version = int(task.state_version or 0)
with pytest.raises(TaskStatusRefusedError) as exc_info:
apply_task_control_transition(
task,
TaskControlState.RESUME_REQUESTED,
expected_run_id="live-run",
refuse_terminal_status=True,
)
assert ended_status.value in str(exc_info.value)
db_session.rollback()
_assert_still_ended(db_session, task_id, ended_status, state_version=state_version)
# A stale run is still reported as a stale run, not as a status refusal.
task = db_session.get(Task, task_id)
task.status = TaskStatus.PAUSED
task.control_state = TaskControlState.PAUSED.value
db_session.commit()
with pytest.raises(StaleTaskRunError) as stale_info:
apply_task_control_transition(
task,
TaskControlState.RESUME_REQUESTED,
expected_run_id="another-run",
refuse_terminal_status=True,
)
assert not isinstance(stale_info.value, TaskStatusRefusedError)
@ENDED_STATUSES
@pytest.mark.parametrize("refuse", [False, True])
def test_resume_lease_claim_status_fence_is_opt_in(
db_session, refuse: bool, ended_status: TaskStatus
) -> None:
owner = _user(db_session, f"claim-unit-owner-{refuse}-{ended_status.value}")
task = _ended_task(db_session, int(owner.id), ended_status)
task_id = int(task.id)
state_version = int(task.state_version or 0)
refused: list[bool] = []
lease = task_execution_service._acquire_resume_task_lease(
task_id,
int(owner.id),
"live-run",
refuse_terminal_status=refuse,
run_not_resumable_out=refused,
)
if refuse:
assert lease is None
assert refused == [True]
_assert_still_ended(
db_session, task_id, ended_status, state_version=state_version
)
else:
# Every other resume caller keeps the claim it had.
assert lease is not None
assert refused == []
db_session.expire_all()
assert db_session.get(Task, task_id).status == TaskStatus.RUNNING
def test_resume_lease_claim_refused_by_a_live_owner_is_not_a_status_refusal(
db_session,
) -> None:
owner = _user(db_session, "claim-live-owner")
task = _live_task(db_session, int(owner.id))
task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=5)
db_session.commit()
refused: list[bool] = []
lease = task_execution_service._acquire_resume_task_lease(
int(task.id),
int(owner.id),
"live-run",
refuse_terminal_status=True,
run_not_resumable_out=refused,
)
assert lease is None
assert refused == []
@pytest.mark.asyncio
async def test_recovered_delivery_on_completed_run_is_never_resumed(
db_session,
recording_reply: _RecordingReply,
) -> None:
"""Regression for the probe that reproduced the COMPLETED hole.
Before the fix the retry deferred as "waiting for runtime injection",
handed the turn to ``execute_resume_background`` under the finished run,
left the task ``completed / resume_requested``, and the resume claim then
flipped COMPLETED back to RUNNING.
"""
owner = _user(db_session, "completed-probe-owner")
task = _settled_task(
db_session, int(owner.id), status=TaskStatus.COMPLETED, run_id="live-run"
)
task.output = FINAL_OUTPUT
db_session.commit()
task_id = int(task.id)
state_version = int(task.state_version or 0)
_pending_row(db_session, task, int(owner.id))
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
result = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
task_execution_service.execute_resume_background.assert_not_called()
assert result == _outcome_unknown_result(task)
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
_assert_outcome_unknown_frames(recording_reply)
db_session.expire_all()
stored = db_session.get(Task, task_id)
assert stored.status == TaskStatus.COMPLETED
assert stored.control_state == TaskControlState.COMPLETED.value
assert stored.output == FINAL_OUTPUT
assert stored.run_id == "live-run"
assert int(stored.state_version or 0) == state_version
# Even a resume handed this run anyway cannot claim it.
refused: list[bool] = []
assert (
task_execution_service._acquire_resume_task_lease(
task_id,
int(owner.id),
"live-run",
refuse_terminal_status=True,
run_not_resumable_out=refused,
)
is None
)
assert refused == [True]
db_session.expire_all()
assert db_session.get(Task, task_id).status == TaskStatus.COMPLETED
def _update_task(task_id: int, **values: Any) -> None:
with get_session_local()() as db:
db.query(Task).filter(Task.id == task_id).update(
values, synchronize_session=False
)
db.commit()
def _end_run_directly(task_id: int, ended_status: TaskStatus) -> None:
"""End the run whatever lease holds it (the run's own terminal write)."""
values: dict[Any, Any] = {
Task.status: ended_status,
Task.control_state: ended_status.value.lower(),
Task.runner_id: None,
Task.lease_expires_at: None,
Task.lease_attempt_id: None,
Task.state_version: Task.state_version + 1,
}
if ended_status == TaskStatus.FAILED:
values[Task.error_message] = RECOVERY_ERROR
else:
values[Task.output] = FINAL_OUTPUT
with get_session_local()() as db:
db.query(Task).filter(Task.id == task_id).update(
values, synchronize_session=False
)
db.commit()
def _real_resume_environment(agent: MagicMock, background_manager: Any):
"""Run the real resume coordinator; only the agent is a stand-in."""
return (
patch(
"xagent.web.services.agent_service_manager.get_agent_manager",
return_value=MagicMock(get_agent_for_task=AsyncMock(return_value=agent)),
),
patch.object(
task_execution_service, "background_task_manager", background_manager
),
)
async def _wait_for_resume_to_finish(background_manager: Any, task_id: int) -> None:
for _ in range(int(DB_PROGRESS_TIMEOUT / 0.01)):
if task_id not in background_manager.running_tasks:
return
await asyncio.sleep(0.01)
raise AssertionError("the resume coordinator never finished")
@pytest.mark.asyncio
@pytest.mark.parametrize("live", [False, True])
@pytest.mark.parametrize("unavailable", [False, True])
async def test_input_review_rejects_websocket_delivery_without_resuming(
db_session, live_task_lease, recording_reply, monkeypatch, live, unavailable
) -> None:
owner = _user(db_session, "review-owner")
task = _live_task(db_session, int(owner.id))
task_id = int(task.id)
prior_status = TaskStatus.RUNNING if live else TaskStatus.WAITING_FOR_USER
task.status = prior_status
task.control_state = prior_status.value
task.runner_id = get_runner_id() if live else None
task.lease_expires_at = (
datetime.now(timezone.utc) + timedelta(minutes=1) if live else None
)
db_session.commit()
if live:
live_task_lease(db_session, task)
runner = AgentRunner(agent=Agent(name="reviewed", patterns=[], llm=None))
context = ExecutionContext(execution_id=str(task_id))
runner.context_manager.set_context(context)
hook = AsyncMock(
return_value=input_review.InputReviewResult(
refusal="Reply refused.", unavailable=unavailable
)
)
monkeypatch.setattr(input_review, "_review", hook)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.get_dag_pattern.return_value = None
agent.post_user_message = AsyncMock(wraps=runner.post_user_message)
background_manager = task_execution_service.BackgroundTaskManager()
agent_patch, manager_patch = _real_resume_environment(agent, background_manager)
with (
agent_patch,
manager_patch,
patch.object(task_execution_service, "publish_task_event", AsyncMock()),
):
try:
await execute_durable_task_command(_message_command(task, owner, TURN_ID))
except TaskCommandDeferred:
assert not live
except TaskCommandRejected:
# A delivery that has already failed is rejected by the command
# dispatcher. A still-pending deferred delivery settles below.
pass
await _wait_for_resume_to_finish(background_manager, task_id)
hook.assert_awaited_once()
request = hook.await_args.args[0]
assert request.user_id == int(owner.id)
assert request.source == "reply"
assert agent.post_user_message.await_args.kwargs["request_interrupt"] is live
assert context.messages == []
agent.resume_task.assert_not_called()
assert _row_status(db_session, task_id) == DELIVERY_FAILED
notices = [
frame
for frame in recording_reply.frames
if frame.get("type") == "message_rejected"
]
assert len(notices) == 1
assert notices[0]["error_code"] == (
ClientErrorCode.INPUT_REVIEW_UNAVAILABLE.value
if unavailable
else ClientErrorCode.INPUT_REVIEW_BLOCKED.value
)
assert notices[0].get("retry_with_new_id", False) is unavailable
db_session.expire_all()
assert db_session.get(Task, task_id).status == prior_status
@pytest.mark.asyncio
@ENDED_STATUSES
async def test_posted_recovered_claim_whose_run_ends_before_the_claim_is_announced(
live_task_lease,
db_session,
ended_status: TaskStatus,
) -> None:
"""The live injection was accepted, then the run ended before resuming.
The ended run is still not resumed. The sender was already told the turn
was accepted and no resume will answer it, so the task's audience gets a
task-wide outcome-unknown notice; the row stays ``dispatched``.
"""
owner = _user(db_session, f"posted-owner-{ended_status.value}")
task = _live_task(db_session, int(owner.id))
task.runner_id = get_runner_id()
task.lease_expires_at = datetime.now(timezone.utc) + timedelta(minutes=1)
db_session.commit()
live_task_lease(db_session, task)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
real_sync = controller_module.transition_task_control_state_sync
def transition_then_end(*args: Any, **kwargs: Any):
snapshot = real_sync(*args, **kwargs)
_end_run_directly(task_id, ended_status)
return snapshot
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.post_user_message = AsyncMock(
return_value=task_execution_service.UserMessageInjectionOutcome.POSTED_FRESH
)
background_manager = task_execution_service.BackgroundTaskManager()
resume_spy = AsyncMock(wraps=task_execution_service.execute_resume_background)
publish = AsyncMock()
agent_patch, manager_patch = _real_resume_environment(agent, background_manager)
with (
agent_patch,
manager_patch,
patch.object(task_execution_service, "execute_resume_background", resume_spy),
patch.object(task_execution_service, "publish_task_event", publish),
patch.object(
controller_module,
"transition_task_control_state_sync",
side_effect=transition_then_end,
),
):
result = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
assert resume_spy.await_count == 1
assert resume_spy.await_args.kwargs["refuse_terminal_status"] is True
assert resume_spy.await_args.kwargs["delivery_already_dispatched"] is True
await _wait_for_resume_to_finish(background_manager, task_id)
# The live injection was accepted and answered as such.
assert result is not None and "delivery_outcome" not in result
agent.post_user_message.assert_awaited_once()
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
db_session.expire_all()
stored = db_session.get(Task, task_id)
assert stored.status == ended_status
assert stored.control_state == ended_status.value.lower()
assert stored.run_id == "live-run"
assert stored.runner_id is None
notices = [
call.args[0]
for call in publish.await_args_list
if call.args[0].get("error_code")
== ClientErrorCode.MESSAGE_OUTCOME_UNKNOWN.value
]
assert len(notices) == 1
notice = notices[0]
assert notice["type"] == "error"
assert notice["task_id"] == task_id
assert notice["turn_id"] == TURN_ID
assert notice["client_message_id"] == TURN_ID
assert MESSAGE not in str(notice)
assert not any(
call.args[0].get("error_code") == ClientErrorCode.TASK_BUSY.value
for call in publish.await_args_list
)
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["run_rotated", "foreign_live_owner"])
async def test_recovered_claim_refused_for_a_live_run_records_outcome_unknown(
db_session,
recording_reply: _RecordingReply,
change: str,
) -> None:
"""The claim is refused although the run did not end.
``run_rotated``: the task moved to another run (non-terminal) between the
transition and the claim -- the classifier's run-changed arm.
``foreign_live_owner``: a live foreign runner holds the same run. An
ordinary resume records that as a failed delivery; a recovered claim may
already have been applied, so it records outcome unknown instead.
"""
owner = _user(db_session, f"live-refusal-{change}")
task = _expired_running_task(db_session, int(owner.id))
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
real_sync = controller_module.transition_task_control_state_sync
def transition_then_change(*args: Any, **kwargs: Any):
snapshot = real_sync(*args, **kwargs)
if change == "run_rotated":
_update_task(
task_id,
status=TaskStatus.PAUSED,
control_state=TaskControlState.PAUSED.value,
run_id="rotated-run",
runner_id=None,
lease_expires_at=None,
lease_attempt_id=None,
)
else:
_update_task(
task_id,
runner_id="foreign-runner",
lease_attempt_id="foreign-attempt",
lease_expires_at=datetime.now(timezone.utc) + timedelta(minutes=5),
)
return snapshot
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.post_user_message = AsyncMock()
background_manager = task_execution_service.BackgroundTaskManager()
publish = AsyncMock()
agent_patch, manager_patch = _real_resume_environment(agent, background_manager)
with (
agent_patch,
manager_patch,
patch.object(task_execution_service, "publish_task_event", publish),
patch.object(
controller_module,
"transition_task_control_state_sync",
side_effect=transition_then_change,
),
):
try:
first = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
except TaskCommandDeferred:
first = None
await _wait_for_resume_to_finish(background_manager, task_id)
assert _row_status(db_session, task_id) == DELIVERY_OUTCOME_UNKNOWN
recording_reply.frames.clear()
result = (
first
if first is not None
else await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=3)
)
)
assert result == _outcome_unknown_result(task)
agent.post_user_message.assert_not_awaited()
assert not any(
call.args[0].get("error_code") == ClientErrorCode.TASK_BUSY.value
for call in publish.await_args_list
)
db_session.expire_all()
stored = db_session.get(Task, task_id)
if change == "run_rotated":
assert stored.status == TaskStatus.PAUSED
assert stored.run_id == "rotated-run"
else:
assert stored.status == TaskStatus.RUNNING
assert stored.runner_id == "foreign-runner"
assert stored.run_id == "live-run"
def test_resume_lease_claim_classifies_a_rotated_run_as_not_resumable(
db_session,
) -> None:
owner = _user(db_session, "claim-rotated-owner")
task = _settled_task(
db_session, int(owner.id), status=TaskStatus.PAUSED, run_id="rotated-run"
)
not_resumable: list[bool] = []
lease = task_execution_service._acquire_resume_task_lease(
int(task.id),
int(owner.id),
"live-run",
refuse_terminal_status=True,
run_not_resumable_out=not_resumable,
)
assert lease is None
assert not_resumable == [True]
db_session.expire_all()
stored = db_session.get(Task, int(task.id))
assert stored.status == TaskStatus.PAUSED
assert stored.run_id == "rotated-run"
@pytest.mark.asyncio
async def test_run_rotated_at_the_transition_settles_a_recovered_claim_unknown(
db_session,
recording_reply: _RecordingReply,
) -> None:
"""A stale run at RESUME_REQUESTED never fails a recovered claim."""
owner = _user(db_session, "transition-rotated-owner")
task = _expired_running_task(db_session, int(owner.id))
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
real_sync = controller_module.transition_task_control_state_sync
def rotate_then_transition(*args: Any, **kwargs: Any):
_update_task(task_id, run_id="rotated-run")
return real_sync(*args, **kwargs)
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED) as (
agent,
background_manager,
),
patch.object(
controller_module,
"transition_task_control_state_sync",
side_effect=rotate_then_transition,
),
):
result = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
task_execution_service.execute_resume_background.assert_not_called()
assert result == _outcome_unknown_result(task)
background_manager.release_resume_reservation.assert_called_with(task_id)
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
_assert_outcome_unknown_frames(recording_reply)
db_session.expire_all()
stored = db_session.get(Task, task_id)
assert stored.run_id == "rotated-run"
assert stored.status == TaskStatus.RUNNING
agent.post_user_message.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [TaskStatus.RUNNING, TaskStatus.WAITING_FOR_USER])
async def test_fresh_live_message_is_fenced_and_still_handed_off(
db_session,
status: TaskStatus,
) -> None:
"""A fresh message on a live run is fenced too, and hands off as before.
The fences only refuse an ended run; this one is not, so the message is
handed to the resume as usual, marked as a claim this attempt made.
"""
owner = _user(db_session, f"fresh-live-owner-{status.value}")
task = _expired_running_task(db_session, int(owner.id))
task.status = status
task.control_state = status.value.lower()
db_session.commit()
real_sync = controller_module.transition_task_control_state_sync
transition_spy = MagicMock(side_effect=real_sync)
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED),
patch.object(
controller_module, "transition_task_control_state_sync", transition_spy
),
):
with pytest.raises(TaskCommandDeferred):
# A deferred delivery waits for the (mocked) resume to inject it.
await execute_durable_task_command(
_message_command(task, owner, "fresh-live-turn", attempt_count=1)
)
resume = task_execution_service.execute_resume_background
assert resume.call_count == 1
assert resume.call_args.kwargs["refuse_terminal_status"] is True
assert resume.call_args.kwargs["delivery_claimed_fresh"] is True
assert transition_spy.call_count == 1
assert transition_spy.call_args.kwargs["refuse_terminal_status"] is True
db_session.expire_all()
assert db_session.get(Task, int(task.id)).status == status
def _processing_command_row(db, task: Task, owner, command) -> None:
enqueued = enqueue_task_command(
db,
task_id=int(task.id),
actor_user_id=int(owner.id),
command_id=command.command_id,
kind=TaskCommandKind.MESSAGE,
payload=dict(command.payload),
)
stored = db.get(TaskExecutionCommand, enqueued.command_id)
stored.status = COMMAND_PROCESSING
# The attempt that is executing it.
stored.attempt_count = command.attempt_count
db.commit()
def _stored_command_result(db, task_id: int, command_id: str) -> Any:
db.expire_all()
row = (
db.query(TaskExecutionCommand)
.filter(
TaskExecutionCommand.task_id == task_id,
TaskExecutionCommand.command_id == command_id,
)
.one()
)
return row.result
@pytest.mark.asyncio
async def test_retry_after_an_unrecorded_unknown_settlement_still_answers_unknown(
db_session,
recording_reply: _RecordingReply,
) -> None:
"""The attempt that settled the turn crashed before storing its result.
The row is ``dispatched`` and the in-memory marker is gone; the durable
record on the command row keeps the retry from reading it as accepted.
"""
owner = _user(db_session, "unrecorded-unknown-owner")
task = _ended_task(db_session, int(owner.id), TaskStatus.COMPLETED)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
command = _message_command(task, owner, TURN_ID, attempt_count=2)
_processing_command_row(db_session, task, owner, command)
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
first = await execute_durable_task_command(command)
assert first == _outcome_unknown_result(task)
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
# Recorded before the row write, so a crash after it loses nothing.
assert _stored_command_result(db_session, task_id, TURN_ID) == first
# The transport never stored the result; a later attempt runs again.
recording_reply.frames.clear()
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
retried = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=3)
)
assert retried == _outcome_unknown_result(task)
_assert_outcome_unknown_frames(recording_reply)
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
@pytest.mark.asyncio
async def test_retry_of_an_ordinary_dispatched_turn_is_still_accepted(
db_session,
recording_reply: _RecordingReply,
) -> None:
owner = _user(db_session, "ordinary-dispatched-owner")
task = _ended_task(db_session, int(owner.id), TaskStatus.COMPLETED)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
command = _message_command(task, owner, TURN_ID, attempt_count=3)
_processing_command_row(db_session, task, owner, command)
with get_session_local()() as db:
db.query(command_execution_service.TaskChatMessage).filter(
command_execution_service.TaskChatMessage.task_id == task_id
).update({"delivery_status": DELIVERY_DISPATCHED}, synchronize_session=False)
db.commit()
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
result = await execute_durable_task_command(command)
assert result == {
"task_id": task_id,
"command_id": TURN_ID,
"kind": TaskCommandKind.MESSAGE.value,
}
assert not any(
frame.get("error_code") == ClientErrorCode.MESSAGE_OUTCOME_UNKNOWN.value
for frame in recording_reply.frames
)
@pytest.mark.asyncio
async def test_refused_resume_whose_unknown_record_fails_does_not_report_a_failure(
db_session,
) -> None:
"""A DB error recording the refusal is logged; the run is not failed."""
owner = _user(db_session, "record-fails-owner")
task = _expired_running_task(db_session, int(owner.id))
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
real_sync = controller_module.transition_task_control_state_sync
def transition_then_complete(*args: Any, **kwargs: Any):
snapshot = real_sync(*args, **kwargs)
_end_run_directly(task_id, TaskStatus.COMPLETED)
return snapshot
real_mark = task_execution_service.mark_user_message_delivery_sync
marks: list[str] = []
def failing_mark(task_id_arg: int, turn_id: str, status: str):
# Only the first write fails, so a fall-through into the generic
# failure handler would get its own delivery write through and then
# announce a task failure.
marks.append(status)
if len(marks) == 1:
raise OperationalError("UPDATE", {}, Exception("database went away"))
return real_mark(task_id_arg, turn_id, status)
agent = MagicMock()
agent.supports_live_control.return_value = True
agent.post_user_message = AsyncMock()
background_manager = task_execution_service.BackgroundTaskManager()
publish = AsyncMock()
agent_patch, manager_patch = _real_resume_environment(agent, background_manager)
with (
agent_patch,
manager_patch,
patch.object(task_execution_service, "publish_task_event", publish),
patch.object(
task_execution_service,
"mark_user_message_delivery_sync",
side_effect=failing_mark,
),
patch.object(
controller_module,
"transition_task_control_state_sync",
side_effect=transition_then_complete,
),
):
with pytest.raises(TaskCommandDeferred):
await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
await _wait_for_resume_to_finish(background_manager, task_id)
assert marks == [DELIVERY_OUTCOME_UNKNOWN]
publish.assert_not_awaited()
agent.post_user_message.assert_not_awaited()
# Still pending: the retried command settles it through the early refusal.
assert _row_status(db_session, task_id) == DELIVERY_PENDING
db_session.expire_all()
stored = db_session.get(Task, task_id)
assert stored.status == TaskStatus.COMPLETED
assert stored.output == FINAL_OUTPUT
@pytest.mark.asyncio
async def test_unknown_record_is_written_before_the_row_write(
db_session,
recording_reply: _RecordingReply,
) -> None:
owner = _user(db_session, "record-order-owner")
task = _ended_task(db_session, int(owner.id), TaskStatus.COMPLETED)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
command = _message_command(task, owner, TURN_ID, attempt_count=2)
_processing_command_row(db_session, task, owner, command)
real_mark = command_execution_service.mark_user_message_delivery_sync
seen_at_row_write: list[Any] = []
def mark_checking_record(task_id_arg: int, turn_id: str, status: str):
if status == DELIVERY_DISPATCHED:
with get_session_local()() as db:
seen_at_row_write.append(
db.query(TaskExecutionCommand.result)
.filter(TaskExecutionCommand.task_id == task_id_arg)
.scalar()
)
return real_mark(task_id_arg, turn_id, status)
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED),
patch.object(
command_execution_service,
"mark_user_message_delivery_sync",
side_effect=mark_checking_record,
),
):
result = await execute_durable_task_command(command)
assert result == _outcome_unknown_result(task)
assert seen_at_row_write == [_outcome_unknown_result(task)]
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
def test_unknown_record_from_a_reclaimed_attempt_is_not_written(db_session) -> None:
owner = _user(db_session, "stale-record-owner")
task = _ended_task(db_session, int(owner.id), TaskStatus.COMPLETED)
task_id = int(task.id)
# The command was reclaimed: attempt 3 now owns it.
_processing_command_row(
db_session,
task,
owner,
_message_command(task, owner, TURN_ID, attempt_count=3),
)
command_execution_service._record_command_outcome_unknown_sync(
task_id, TURN_ID, attempt_count=2
)
assert _stored_command_result(db_session, task_id, TURN_ID) is None
command_execution_service._record_command_outcome_unknown_sync(
task_id, TURN_ID, attempt_count=3
)
assert _stored_command_result(db_session, task_id, TURN_ID) == (
_outcome_unknown_result(task)
)
@pytest.mark.asyncio
async def test_lost_row_write_ack_keeps_the_unknown_record_across_the_retry(
db_session,
recording_reply: _RecordingReply,
) -> None:
"""The ``dispatched`` write commits but its acknowledgement is lost.
The attempt fails; ``dispatched`` cannot become ``outcome_unknown``, so
the command row's record is the only trace. The transport's failure
handling must keep it, and the retry answers outcome unknown.
"""
owner = _user(db_session, "lost-ack-record-owner")
task = _ended_task(db_session, int(owner.id), TaskStatus.COMPLETED)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
enqueued = enqueue_task_command(
db_session,
task_id=task_id,
actor_user_id=int(owner.id),
command_id=TURN_ID,
kind=TaskCommandKind.MESSAGE,
payload=dict(_message_command(task, owner, TURN_ID).payload),
)
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
# An earlier attempt claimed the row and crashed; the next claim is #2.
stored.attempt_count = 1
db_session.commit()
real_mark = command_execution_service.mark_user_message_delivery_sync
def lose_ack(task_id_arg: int, turn_id: str, status: str):
transition = real_mark(task_id_arg, turn_id, status)
if status == DELIVERY_DISPATCHED:
raise OperationalError("COMMIT", {}, Exception("connection reset"))
return transition
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED),
patch.object(
command_execution_service,
"mark_user_message_delivery_sync",
side_effect=lose_ack,
),
):
assert await dispatch_one_task_command(
execute_durable_task_command, command_db_id=enqueued.command_id
)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored.status == COMMAND_PENDING
assert stored.attempt_count == 2
assert stored.result == _outcome_unknown_result(task)
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
stored.claim_expires_at = None
stored.retry_available_at = None
db_session.commit()
recording_reply.frames.clear()
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
assert await dispatch_one_task_command(
execute_durable_task_command, command_db_id=enqueued.command_id
)
db_session.expire_all()
stored = db_session.get(TaskExecutionCommand, enqueued.command_id)
assert stored.status == COMMAND_COMPLETED
assert stored.result == _outcome_unknown_result(task)
_assert_outcome_unknown_frames(recording_reply)
db_session.expire_all()
assert db_session.get(Task, task_id).status == TaskStatus.COMPLETED
@pytest.mark.asyncio
async def test_settlement_by_a_reclaimed_attempt_leaves_the_row_to_the_owner(
db_session,
recording_reply: _RecordingReply,
) -> None:
"""Attempt 2 lost its claim while its handler ran; attempt 3 owns it.
Attempt 2 may not write the record, so it must not advance the row
either: a ``dispatched`` row without the record reads as accepted.
"""
owner = _user(db_session, "reclaimed-settle-owner")
task = _ended_task(db_session, int(owner.id), TaskStatus.COMPLETED)
task_id = int(task.id)
_pending_row(db_session, task, int(owner.id))
_processing_command_row(
db_session,
task,
owner,
_message_command(task, owner, TURN_ID, attempt_count=3),
)
with (
_live_control_environment(outcome=ResumeReservationOutcome.RESERVED),
pytest.raises(TaskCommandDeferred),
):
# Answered from the still-pending row; the transport's deferral of
# this stale attempt is fenced out.
await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=2)
)
assert _row_status(db_session, task_id) == DELIVERY_PENDING
assert _stored_command_result(db_session, task_id, TURN_ID) is None
assert not any(
frame.get("error_code") == ClientErrorCode.MESSAGE_OUTCOME_UNKNOWN.value
for frame in recording_reply.frames
)
with _live_control_environment(outcome=ResumeReservationOutcome.RESERVED):
result = await execute_durable_task_command(
_message_command(task, owner, TURN_ID, attempt_count=3)
)
assert result == _outcome_unknown_result(task)
assert _row_status(db_session, task_id) == DELIVERY_DISPATCHED
assert _stored_command_result(db_session, task_id, TURN_ID) == result
_assert_outcome_unknown_frames(recording_reply)