Enterprise agent platform for building, deploying, and operating AI teammates.
"""A run interrupted by a persistence failure rests PAUSED instead of FAILED.
Covers every settlement point that acts on the interruption: the exception
path of a new run (``_schedule_bg`` -> ``settle_task_lease_isolated``), its
result path (``_finalize_task_execution_result_isolated``), and both paths of
a resumed run (``_finalize_resumed_task`` and the resume exception branch).
Runs on SQLite and, when ``XAGENT_TEST_POSTGRES_URL`` is set, on PostgreSQL
through the shared ``canonical`` fixture.
"""
from __future__ import annotations
import asyncio
import uuid
from datetime import timedelta
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
from tests.core.agent.test_react import FakeLLM, FakeTool
from tests.core.agent.test_runner import FakeWorkspaceManager
from tests.web.services.test_execution_event_persistence_settlement import (
_clear_bg_manager as _clear_bg_manager_fixture,
)
from tests.web.services.test_execution_event_persistence_settlement import (
_fail_once,
_react_execution,
_run_scheduled_turn,
_tool_turn_llm,
)
from tests.web.services.test_execution_event_recovery import tracer_for
from tests.web.services.test_task_execution_event_writer import (
canonical as canonical_fixture,
)
from tests.web.services.test_task_execution_event_writer import engine as engine_fixture
from tests.web.services.test_task_execution_event_writer import (
facts,
)
from tests.web.services.test_task_execution_event_writer import (
task_id as task_id_fixture,
)
from tests.web.services.test_task_lease_expiry_interruption import (
_fail_recording,
_snapshot,
)
from xagent.core.agent import (
Agent,
AgentRunner,
ExecutionContext,
PatternRuntime,
ReActPattern,
)
from xagent.core.agent.checkpoint import (
CHECKPOINT_SCHEMA_VERSION,
CHECKPOINT_TYPE,
CheckpointUnavailableError,
ExecutionEventPersistenceError,
TraceCheckpointStore,
)
from xagent.core.agent.interruption import InterruptionReason
from xagent.web.models.agent import Agent as AgentModel
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_auto_recovery import TaskAutoRecovery, TaskRecoveryEvent
from xagent.web.models.trigger import (
AgentTrigger,
TriggerRun,
TriggerRunStatus,
TriggerType,
)
from xagent.web.models.user import User
from xagent.web.models.workforce import Workforce, WorkforceRun
from xagent.web.services import task_auto_recovery
from xagent.web.services import task_execution_event_recovery as event_recovery
from xagent.web.services import task_orchestrator as orchestrator
from xagent.web.services.chat_history_service import (
DELIVERY_PENDING,
persist_user_message_no_commit,
)
from xagent.web.services.client_error_messages import CLIENT_SAFE_TASK_FAILURE
from xagent.web.services.task_auto_recovery import (
TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR,
InterruptionSettlementDeferred,
settlement_interruption_for_failure,
settlement_interruption_for_result,
)
from xagent.web.services.task_execution import (
_acquire_resume_task_lease,
_finalize_resumed_task,
_finalize_task_execution_result_isolated,
_PreparedTaskFileOutputs,
execute_resume_background,
)
from xagent.web.services.task_execution_event_writer import append_fact_no_commit
from xagent.web.services.task_lease_recovery import (
TASK_LEASE_PAUSED_TRIGGER_ERROR,
recover_task_lease_candidate_no_commit,
)
from xagent.web.services.task_lease_service import (
TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR,
CheckpointRecoveryResolution,
CheckpointRecoveryVerdict,
TaskLease,
acquire_task_lease,
bind_task_lease_context,
get_expired_task_lease_candidates,
utc_now,
)
from xagent.web.services.task_orchestrator import (
TASK_INTERRUPTION_PAUSED_MESSAGE,
TASK_USER_PAUSED_MESSAGE,
TaskTurnPayload,
_schedule_bg,
publish_settlement_pause,
settle_task_lease_isolated,
)
from xagent.web.services.task_settlement_report import SettlementReport
canonical = canonical_fixture
engine = engine_fixture
task_id = task_id_fixture
_clear_bg_manager = _clear_bg_manager_fixture
PAUSE_SWITCH = "XAGENT_TASK_INFRA_FAILURE_PAUSE_ENABLED"
RUN_ERROR = "setup/run error: ExecutionEventPersistenceError: event write failed"
NO_OUTPUTS = _PreparedTaskFileOutputs((), (), ())
@pytest.fixture(autouse=True)
def _pause_switch_on(monkeypatch):
# Unset, the switch follows automatic resume; these tests exercise it on.
monkeypatch.setenv(PAUSE_SWITCH, "true")
AUTO_RESUME_SWITCH = "XAGENT_TASK_AUTO_RESUME_ENABLED"
@pytest.fixture(autouse=True)
def _auto_resume_off(monkeypatch):
# These suites compare settlement with lease recovery and with settling
# as before, on the outcome automatic resume leaves to the user
# (``manual``, a FAILED TriggerRun). Tests of what a scheduled resume
# changes turn the switch on.
monkeypatch.setenv(AUTO_RESUME_SWITCH, "false")
@pytest.fixture(autouse=True)
def _late_bound_sessions(monkeypatch):
"""``task_execution`` binds the session factory at import; follow the
one the ``canonical`` fixture installs instead."""
from xagent.web.models import database
monkeypatch.setattr(
"xagent.web.services.task_execution.get_session_local",
lambda: database.get_session_local(),
)
def _start_run(factory, tid: int, **fields: Any) -> TaskLease:
with factory() as db:
task = db.get(Task, tid)
for name, value in fields.items():
setattr(task, name, value)
db.commit()
lease = acquire_task_lease(db, tid, new_run=True)
assert lease is not None
return lease
def _checkpoint(factory, tid: int, run_id: str, *, messages: int = 2) -> None:
"""A recoverable V2 checkpoint of ``run_id``."""
with factory() as db:
append_fact_no_commit(
db,
task_id=tid,
kind="recovery_state",
key=f"runtime:{uuid.uuid4()}",
payload={
"data": {
"checkpoint_type": CHECKPOINT_TYPE,
"snapshot_schema_version": CHECKPOINT_SCHEMA_VERSION,
"execution_id": str(tid),
"snapshot": _snapshot(messages=messages, iteration=2),
}
},
run_id=run_id,
)
db.commit()
def _state(factory, tid: int) -> tuple[Task, Any, list[Any]]:
with factory() as db:
task = db.get(Task, tid)
row = db.get(TaskAutoRecovery, tid)
events = list(
db.scalars(
sa.select(TaskRecoveryEvent)
.where(TaskRecoveryEvent.task_id == tid)
.order_by(TaskRecoveryEvent.id)
)
)
db.expunge_all()
return task, row, events
def _assistant_lines(factory, tid: int) -> list[str]:
with factory() as db:
return [
row.content
for row in db.query(TaskChatMessage)
.filter(TaskChatMessage.task_id == tid, TaskChatMessage.role == "assistant")
.order_by(TaskChatMessage.id)
]
def _settled_facts(factory, tid: int) -> list[dict[str, Any]]:
with factory() as db:
return [
row.payload for row in facts(db, tid) if row.kind == "execution_settled"
]
def _assert_no_failure_in_model_context(factory, tid: int) -> None:
"""The paused run must not read as a failed execution to the next turn."""
from xagent.web.services.task_event_context_service import (
load_task_event_context,
)
with factory() as db:
context = load_task_event_context(db, tid)
# summarize_execution_failure_event's line for a failed settlement.
assert "previous execution failed" not in str(context).lower()
def _assert_paused(factory, tid: int, lease: TaskLease, *, reason: str) -> Any:
task, row, events = _state(factory, tid)
assert task.status == TaskStatus.PAUSED
assert task.control_state == "paused"
assert task.run_id == lease.run_id
assert task.runner_id is None
assert task.error_message is None
assert row.run_id == lease.run_id
assert (row.reason, row.state) == (reason, "manual")
assert row.paused_state_version == task.state_version
assert row.progress_marker is not None
assert row.next_attempt_at is None
assert [(e.event, e.reason, e.run_id) for e in events] == [
("interrupted", reason, lease.run_id)
]
assert events[0].detail["task_status"] == "paused"
return row
def _settle(lease: TaskLease, **kwargs: Any) -> tuple[bool, SettlementReport]:
report = SettlementReport()
settled = settle_task_lease_isolated(
lease,
error_message=RUN_ERROR,
classify_unknown_tool_effect=True,
report=report,
**kwargs,
)
return settled, report
PAUSED_FACT = {"status": "paused", "result": {"error": None}}
def _persistence_interruption() -> InterruptionReason:
reason = settlement_interruption_for_failure(
ExecutionEventPersistenceError("event write failed")
)
assert reason is InterruptionReason.PERSISTENCE_FAILURE
return reason
# ---------------------------------------------------------------- classifiers
def test_settlement_acts_on_persistence_failures():
assert settlement_interruption_for_failure(
OperationalError("SELECT 1", {}, Exception("connection reset"))
) is (InterruptionReason.PERSISTENCE_FAILURE)
assert settlement_interruption_for_failure(RuntimeError("boom")) is None
persistence = {"success": False, "interruption_reason": "persistence_failure"}
assert (
settlement_interruption_for_result(persistence)
is InterruptionReason.PERSISTENCE_FAILURE
)
# Provider and model-output reasons: test_provider_failure_pause.py.
assert (
settlement_interruption_for_result(
{"success": False, "interruption_reason": "lease_expired"}
)
is None
)
# A quota stop keeps the runner's reason but wins over it.
assert (
settlement_interruption_for_result({**persistence, "status": "quota_exceeded"})
is None
)
# Only the top-level field counts, never the raw pattern result.
assert (
settlement_interruption_for_result(
{"success": False, "agent_result": persistence}
)
is None
)
assert settlement_interruption_for_result({**persistence, "success": True}) is None
# ------------------------------------------------- S2: new run, exception path
@pytest.mark.asyncio
async def test_checkpoint_write_failure_pauses_and_the_run_resumes(
canonical, monkeypatch, tmp_path
):
"""The checkpoint after a tool fails; settlement (DB fine again) pauses.
The paused task is resumable: a resume lease claims the same run and the
run continues from its checkpoint without re-running the tool.
"""
factory, tid = canonical
lease = _start_run(factory, tid)
tool = FakeTool()
async def execute(**kwargs):
with bind_task_lease_context(kwargs["task_lease"]):
runtime = PatternRuntime(tracer=TraceCheckpointStore(tracer_for(tid)))
original = runtime.checkpoint
async def fail_after_tool(label, **checkpoint_kwargs):
if label.endswith(("after_tool", "after_tool_batch")):
raise ExecutionEventPersistenceError("checkpoint write failed")
return await original(label, **checkpoint_kwargs)
monkeypatch.setattr(runtime, "checkpoint", fail_after_tool)
context = ExecutionContext(execution_id=str(tid), system_prompt="Calc")
context.add_user_message("2+2", metadata={"turn_id": "turn"})
await ReActPattern(max_iterations=3).run(
context=context, tools=[tool], runtime=runtime, llm=_tool_turn_llm()
)
events: list[dict[str, Any]] = []
async def publish(event, _task_id):
events.append(event)
with patch("xagent.web.services.task_events.publish_task_event", new=publish):
results = await _run_scheduled_turn(tid, lease, execute)
assert len(tool.calls) == 1
assert results == []
_assert_paused(factory, tid, lease, reason="persistence_failure")
_task_after, row, _events = _state(factory, tid)
assert "checkpoint write failed" in row.last_error
assert [event["type"] for event in events] == ["task_paused"]
assert events[0]["message"] == TASK_INTERRUPTION_PAUSED_MESSAGE
assert (events[0]["status"], events[0]["control_state"]) == ("paused", "paused")
assert events[0]["run_id"] == lease.run_id
assert "checkpoint write failed" not in str(events)
assert CLIENT_SAFE_TASK_FAILURE not in _assistant_lines(factory, tid)
assert _settled_facts(factory, tid) == [
{"status": "paused", "result": {"error": None}}
]
# Manual resume: same run, continues from the checkpoint.
with factory() as db:
user_id = int(db.get(Task, tid).user_id)
resume_lease = _acquire_resume_task_lease(
tid, user_id, lease.run_id, refuse_terminal_status=True
)
assert resume_lease is not None and resume_lease.run_id == lease.run_id
resume_llm = FakeLLM(responses=[{"content": "4", "done": True}])
with bind_task_lease_context(resume_lease):
runner = AgentRunner(
agent=Agent(
name="resumed",
patterns=[ReActPattern(max_iterations=3)],
tools=[tool],
llm=resume_llm,
),
tracer=TraceCheckpointStore(tracer_for(tid)),
workspace_manager=FakeWorkspaceManager(tmp_path),
)
result = await runner.resume(str(tid), task="2+2")
assert result["success"]
assert result["output"] == "4"
assert len(tool.calls) == 1
@pytest.mark.asyncio
async def test_unknown_tool_effect_still_fails_and_records_why(canonical, monkeypatch):
factory, tid = canonical
lease = _start_run(factory, tid)
_fail_once(monkeypatch, "tool_execution_end")
tool = FakeTool()
published = await _run_scheduled_turn(tid, lease, _react_execution(tid, tool))
assert len(tool.calls) == 1
task, row, events = _state(factory, tid)
assert task.status == TaskStatus.FAILED
assert task.error_message == TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR
assert [event["type"] for event in published] == ["task_error"]
assert (row.reason, row.state) == ("unknown_tool_effect", "manual")
assert [(e.event, e.reason) for e in events] == [
("interrupted", "unknown_tool_effect")
]
assert events[0].detail["task_status"] == "failed"
def test_run_without_checkpoint_fails_as_before(canonical):
factory, tid = canonical
lease = _start_run(factory, tid)
settled, report = _settle(lease, interruption=_persistence_interruption())
assert settled and not report.paused
task, row, _events = _state(factory, tid)
assert task.status == TaskStatus.FAILED
# The fail branch reports the committed FAILED control identity.
assert report.control_state["status"] == "failed"
assert report.control_state["run_id"] == lease.run_id
assert report.control_state["state_version"] == task.state_version
assert task.error_message == RUN_ERROR
assert (row.reason, row.state) == ("not_recoverable", "manual")
def test_report_stays_empty_when_the_lease_no_longer_owns_the_run(canonical):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
with factory() as db:
task = db.get(Task, tid)
task.control_state = "pause_requested"
task.run_id = "replacement-run"
db.commit()
settled, report = _settle(lease, interruption=_persistence_interruption())
assert not settled
assert report == SettlementReport()
@pytest.mark.parametrize("auto_resume", ["true", "false"])
def test_report_stays_empty_when_the_fenced_pause_write_misses(
canonical, monkeypatch, auto_resume
):
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
with factory() as db:
db.get(Task, tid).control_state = "pause_requested"
db.commit()
# The decision is to pause, but the fenced PAUSED write matches no row.
monkeypatch.setattr(
orchestrator,
"pause_and_release_task_lease_no_commit",
lambda _db, _lease: False,
)
_settled, report = _settle(lease, interruption=_persistence_interruption())
assert report == SettlementReport()
@pytest.mark.parametrize("auto_resume", ["true", "false"])
def test_report_stays_empty_when_the_pause_commit_fails(
canonical, monkeypatch, auto_resume
):
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
with factory() as db:
db.get(Task, tid).control_state = "pause_requested"
db.commit()
report = SettlementReport()
def failing_commit(self):
raise RuntimeError("commit failed")
monkeypatch.setattr("sqlalchemy.orm.Session.commit", failing_commit)
with pytest.raises(RuntimeError, match="commit failed"):
settle_task_lease_isolated(
lease,
error_message=RUN_ERROR,
classify_unknown_tool_effect=True,
interruption=_persistence_interruption(),
report=report,
)
assert report == SettlementReport()
def test_pause_requested_run_pauses_for_the_user(canonical):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
with factory() as db:
db.get(Task, tid).control_state = "pause_requested"
db.commit()
settled, report = _settle(lease, interruption=_persistence_interruption())
assert settled and report.paused_for is InterruptionReason.USER_PAUSE
_assert_paused(factory, tid, lease, reason="user_pause")
def test_failed_settlement_keeps_the_lease_for_ttl_recovery(canonical, monkeypatch):
"""Settlement itself fails (the database is still down): unchanged path.
The lease is retained, TTL recovery later pauses the run as
``lease_expired`` -- the reason it can still persist.
"""
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
def database_down(*_args, **_kwargs):
raise OperationalError("UPDATE tasks", {}, Exception("connection refused"))
monkeypatch.setattr(
orchestrator, "pause_and_release_task_lease_no_commit", database_down
)
with pytest.raises(OperationalError):
_settle(lease, interruption=_persistence_interruption())
task, row, events = _state(factory, tid)
assert task.status == TaskStatus.RUNNING
assert (task.runner_id, task.run_id) == (lease.runner_id, lease.run_id)
assert row is None and events == []
with factory() as db:
db.get(Task, tid).lease_expires_at = utc_now() - timedelta(seconds=5)
db.commit()
with factory() as db:
candidate = get_expired_task_lease_candidates(db, cutoff=utc_now(), limit=1)[0]
assert (
recover_task_lease_candidate_no_commit(
db, candidate, recovered_at=utc_now()
)
== TaskStatus.PAUSED
)
db.commit()
_assert_paused(factory, tid, lease, reason="lease_expired")
def test_unresolvable_checkpoint_keeps_the_lease(canonical, monkeypatch):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
def unavailable(*_args, **_kwargs):
raise CheckpointUnavailableError("recovery read failed")
monkeypatch.setattr(event_recovery, "read_event_checkpoint", unavailable)
with pytest.raises(InterruptionSettlementDeferred):
_settle(lease, interruption=_persistence_interruption())
task, row, _events = _state(factory, tid)
assert task.status == TaskStatus.RUNNING
assert task.runner_id == lease.runner_id
assert row is None
@pytest.mark.asyncio
async def test_deferred_result_settlement_retains_the_lease(canonical):
"""A result path that cannot resolve the checkpoint leaves it to TTL."""
factory, tid = canonical
lease = _start_run(factory, tid)
async def execute(**_kwargs):
raise InterruptionSettlementDeferred("checkpoint not resolvable")
published = await _run_scheduled_turn(tid, lease, execute)
assert published == []
task, row, _events = _state(factory, tid)
assert task.status == TaskStatus.RUNNING
assert task.runner_id == lease.runner_id
assert row is None
# --------------------------------------------------- S1: new run, result path
def _finalize(factory, tid: int, lease: TaskLease, result: dict[str, Any]):
with factory() as db:
user_id = int(db.get(Task, tid).user_id)
return _finalize_task_execution_result_isolated(
task_id=tid,
task_user_id=user_id,
pre_run_status=TaskStatus.RUNNING,
result=result,
expected_run_id=lease.run_id,
task_lease=lease,
resolved_scope_segments=(),
prepared_outputs=NO_OUTPUTS,
)
PERSISTENCE_RESULT = {
"success": False,
"status": "failed",
"output": "All patterns failed",
"error": "All 1 patterns failed: OperationalError: connection reset",
"interruption_reason": "persistence_failure",
}
def test_result_persistence_failure_pauses(canonical):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
finalized = _finalize(factory, tid, lease, PERSISTENCE_RESULT)
assert finalized.report.paused_for is InterruptionReason.PERSISTENCE_FAILURE
assert finalized.waiting_for_control and finalized.terminal_state_committed
assert finalized.final_task_status == "paused"
assert finalized.report.control_state["control_state"] == "paused"
# The scheduler's settlement releases the paused lease.
assert settle_task_lease_isolated(lease)
row = _assert_paused(factory, tid, lease, reason="persistence_failure")
assert row.last_error == PERSISTENCE_RESULT["error"]
# Settled like lease recovery, not as the run's unsuccessful result.
assert _settled_facts(factory, tid) == [PAUSED_FACT]
_assert_no_failure_in_model_context(factory, tid)
assert _assistant_lines(factory, tid) == []
@pytest.mark.parametrize(
"result",
[
{**PERSISTENCE_RESULT, "status": "quota_exceeded", "error_code": "quota"},
{**PERSISTENCE_RESULT, "interruption_reason": "lease_expired"},
{k: v for k, v in PERSISTENCE_RESULT.items() if k != "interruption_reason"},
],
ids=["quota_exceeded", "lease_expired", "no_reason"],
)
def test_other_results_fail_as_before(canonical, result):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
finalized = _finalize(factory, tid, lease, result)
assert finalized.report.paused_for is None
assert finalized.final_task_status == "failed"
task, row, events = _state(factory, tid)
assert task.status == TaskStatus.FAILED
assert task.error_message == result["error"]
assert row is None and events == []
def test_result_unresolvable_checkpoint_raises_without_writing(canonical, monkeypatch):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
def unavailable(*_args, **_kwargs):
raise CheckpointUnavailableError("recovery read failed")
monkeypatch.setattr(event_recovery, "read_event_checkpoint", unavailable)
with pytest.raises(InterruptionSettlementDeferred):
_finalize(factory, tid, lease, PERSISTENCE_RESULT)
task, row, _events = _state(factory, tid)
assert task.status == TaskStatus.RUNNING
assert task.runner_id == lease.runner_id
assert row is None
# ------------------------------------------------------- S3: resumed run
def test_resumed_result_persistence_failure_pauses(canonical):
factory, tid = canonical
lease = _start_run(factory, tid)
_checkpoint(factory, tid, lease.run_id)
with factory() as db:
user_id = int(db.get(Task, tid).user_id)
finalized = _finalize_resumed_task(
tid,
status="failed",
success=False,
output=PERSISTENCE_RESULT["output"],
task_owner_user_id=user_id,
result=dict(PERSISTENCE_RESULT),
task_lease=lease,
prepared_outputs=NO_OUTPUTS,
)
assert finalized["final_status"] == "paused"
assert finalized["lease_released"]
assert finalized["report"].paused_for is InterruptionReason.PERSISTENCE_FAILURE
_assert_paused(factory, tid, lease, reason="persistence_failure")
assert _assistant_lines(factory, tid) == []
assert _settled_facts(factory, tid) == [PAUSED_FACT]
_assert_no_failure_in_model_context(factory, tid)
class _FakeTracker:
quota_interrupt_reason = None
def __init__(self, *, task_id: int, **_kwargs: Any) -> None:
self.complete_tracking = AsyncMock()
self.stop_periodic_updates = AsyncMock()
async def start_tracking(self) -> None:
return None
async def interrupt_reason_for_quota(self) -> None:
return None
async def _resume_raising(
error: BaseException, settle: Any, *, explicit_cancel: bool = False
) -> tuple[AsyncMock, list[str]]:
"""Drive the resume entry point into its exception branch.
Returns the interruption-pause publisher and the broadcast frame types.
"""
lease = TaskLease(task_id=42, runner_id="runner-a", run_id="run-a")
heartbeat_task: asyncio.Task[Any] = asyncio.create_task(asyncio.sleep(0))
agent_service = MagicMock()
agent_service.resume_execution_by_id = AsyncMock(side_effect=error)
ws_manager = MagicMock(broadcast_to_task=AsyncMock())
publish_pause = AsyncMock()
patches = [
patch("xagent.web.services.task_execution._settle_resumed_task_lease", settle),
patch(
"xagent.web.services.task_execution.stop_task_lease_heartbeat",
new=AsyncMock(),
),
patch("xagent.web.api.websocket.manager", ws_manager),
patch(
"xagent.web.services.task_execution.background_task_manager.promote_resume_task"
),
patch("xagent.web.tracking.task_tracker.TaskTracker", _FakeTracker),
patch(
"xagent.web.services.task_orchestrator.publish_settlement_pause",
publish_pause,
),
patch(
"xagent.web.services.task_execution.background_task_manager"
".cancel_was_requested",
return_value=explicit_cancel,
),
patch(
"xagent.web.services.task_execution.load_routine_history_turn_id",
return_value=None,
),
]
for item in patches:
item.start()
try:
await execute_resume_background(
task_id=42,
agent_service=agent_service,
task_owner_user_id=1,
expected_run_id="run-a",
resolved_execution_scope=None,
preacquired_lease=lease,
preacquired_heartbeat_stop=asyncio.Event(),
preacquired_heartbeat_task=heartbeat_task,
preacquired_prior_status=TaskStatus.PAUSED,
)
finally:
for item in reversed(patches):
item.stop()
return publish_pause, [
call.args[0].get("type") for call in ws_manager.broadcast_to_task.call_args_list
]
@pytest.mark.asyncio
async def test_resume_exception_passes_the_interruption_and_announces_the_pause():
def settle(lease, **kwargs):
assert kwargs["interruption"] is InterruptionReason.PERSISTENCE_FAILURE
report = kwargs["report"]
report.paused_for = InterruptionReason.PERSISTENCE_FAILURE
report.control_state.update(status="paused", run_id=lease.run_id)
return True
settle_mock = MagicMock(side_effect=settle)
publish_pause, frames = await _resume_raising(
ExecutionEventPersistenceError("event write failed"), settle_mock
)
settle_mock.assert_called_once()
publish_pause.assert_awaited_once_with(
42,
SettlementReport(
control_state={"status": "paused", "run_id": "run-a"},
paused_for=InterruptionReason.PERSISTENCE_FAILURE,
),
)
assert "task_error" not in frames
@pytest.mark.asyncio
async def test_resume_exception_without_interruption_settles_unchanged():
settle_mock = MagicMock(return_value=True)
publish_pause, frames = await _resume_raising(
RuntimeError("tool crashed"), settle_mock
)
settle_mock.assert_called_once()
assert settle_mock.call_args.kwargs["interruption"] is None
publish_pause.assert_not_awaited()
assert "task_error" in frames
@pytest.mark.asyncio
async def test_resume_deferred_settlement_retains_the_lease():
settle_mock = MagicMock(return_value=True)
publish_pause, frames = await _resume_raising(
InterruptionSettlementDeferred("checkpoint not resolvable"), settle_mock
)
settle_mock.assert_not_called()
publish_pause.assert_not_awaited()
assert "task_error" not in frames
# ------------------------------------------- projections, compared differentially
#
# Each case settles two equivalent tasks in one database and compares every
# lifecycle projection: the task row, its lease fields, the settled fact and
# transcript, the TriggerRun, the WorkforceRun and the user's delivery row.
def _workforce_run(db, user: User) -> int:
"""A running, non-preview workforce run (names unique per call)."""
suffix = uuid.uuid4().hex[:8]
manager = AgentModel(user_id=user.id, name=f"pause manager {suffix}")
db.add(manager)
db.flush()
workforce = Workforce(
owner_user_id=user.id,
scope_type="user",
scope_id=str(user.id),
name=f"Pause workforce {suffix}",
manager_agent_id=manager.id,
status="published",
)
db.add(workforce)
db.flush()
run = WorkforceRun(
workforce_id=workforce.id,
user_id=user.id,
status="running",
snapshot={},
is_preview=False,
)
db.add(run)
db.flush()
return int(run.id)
def _projected_task(factory, *, source: str = "trigger") -> dict[str, Any]:
"""A V2 workforce task with a RUNNING TriggerRun, ready to start a run."""
with factory() as db:
user = db.query(User).order_by(User.id).first()
task = Task(
user_id=user.id,
title="Projected task",
description="persistence failure projections",
source=source,
conversation_storage_version=2,
)
db.add(task)
db.flush()
workforce_run_id = _workforce_run(db, user)
workforce_run = db.get(WorkforceRun, workforce_run_id)
workforce_run.task_id = task.id
task.agent_config = {"workforce_run_id": workforce_run_id}
trigger = AgentTrigger(
user_id=user.id,
workforce_id=workforce_run.workforce_id,
type=TriggerType.SCHEDULED.value,
name=f"Projection trigger {task.id}",
config={},
)
db.add(trigger)
db.flush()
trigger_run = TriggerRun(
trigger_id=trigger.id,
task_id=task.id,
status=TriggerRunStatus.RUNNING.value,
idempotency_key=f"projection-{task.id}",
)
db.add(trigger_run)
db.commit()
return {
"task": int(task.id),
"user": int(user.id),
"workforce_run": workforce_run_id,
"trigger_run": int(trigger_run.id),
"turn_id": f"turn-{uuid.uuid4().hex[:8]}",
}
def _prepare_run(
factory, ids: dict[str, Any], *, checkpoint: bool = True, **fields: Any
) -> TaskLease:
"""Start a run with prior output, a pending user turn and a checkpoint."""
lease = _start_run(factory, ids["task"], **fields)
with factory() as db:
task = db.get(Task, ids["task"])
task.output = "previous answer"
persist_user_message_no_commit(
db,
ids["task"],
ids["user"],
"2+2",
turn_id=ids["turn_id"],
delivery_status=DELIVERY_PENDING,
)
db.commit()
if checkpoint:
_checkpoint(factory, ids["task"], lease.run_id)
with factory() as db:
ids["version"] = int(db.get(Task, ids["task"]).state_version)
return lease
def _projection(factory, ids: dict[str, Any], lease: TaskLease) -> dict[str, Any]:
tid = ids["task"]
with factory() as db:
task = db.get(Task, tid)
trigger_run = db.get(TriggerRun, ids["trigger_run"])
workforce_run = db.get(WorkforceRun, ids["workforce_run"])
messages = (
db.query(TaskChatMessage)
.filter(TaskChatMessage.task_id == tid)
.order_by(TaskChatMessage.id)
.all()
)
return {
"status": task.status,
"control_state": task.control_state,
"error_message": task.error_message,
"output": task.output,
"completion_outcome": task.completion_outcome,
"runner_id": task.runner_id,
"lease_expires_at": task.lease_expires_at,
"lease_attempt_id": task.lease_attempt_id,
"same_run": task.run_id == lease.run_id,
"version_delta": int(task.state_version) - ids["version"],
"settled": [
row.payload for row in facts(db, tid) if row.kind == "execution_settled"
],
"assistant": [m.content for m in messages if m.role == "assistant"],
"delivery": [m.delivery_status for m in messages if m.role == "user"],
"trigger_run": (
trigger_run.status,
trigger_run.error_message,
trigger_run.finished_at is not None,
),
"workforce_run": (workforce_run.status, workforce_run.completed_at is None),
}
async def _schedule_failing_turn(
factory,
ids: dict[str, Any],
lease: TaskLease,
error: BaseException,
*,
events: list[dict[str, Any]] | None = None,
) -> list[str]:
"""Run one scheduled turn whose execution raises ``error``.
``events``, when given, receives every published frame.
"""
if events is None:
events = []
async def publish(event, _task_id):
events.append(event)
async def execute(**_kwargs):
raise error
with (
patch.object(orchestrator, "run_task_lease_heartbeat", new=AsyncMock()),
patch.object(
orchestrator, "load_task_setup_snapshot_sync", return_value=MagicMock()
),
patch.object(
orchestrator, "resolve_execution_scope", return_value=None, create=True
),
patch("xagent.web.services.task_execution.execute_task_background", execute),
patch("xagent.web.services.task_event_display.publish_task_result", publish),
patch("xagent.web.services.task_events.publish_task_event", publish),
patch.object(orchestrator, "_get_agent_manager", return_value=MagicMock()),
):
await _schedule_bg(
task_id=ids["task"],
task_owner_user_id=ids["user"],
task_source="trigger",
task_lease=lease,
payload=TaskTurnPayload("2+2", turn_id=ids["turn_id"]),
force_fresh=False,
context=None,
)
return [str(event.get("type")) for event in events]
def _without_interruption(monkeypatch) -> None:
"""Settle as before this change: no settling path sees an interruption."""
monkeypatch.setattr(
orchestrator, "settlement_interruption_for_failure", lambda _exc: None
)
def _persistence_error() -> ExecutionEventPersistenceError:
return ExecutionEventPersistenceError("event write failed")
def _expire_and_recover(factory, ids: dict[str, Any]) -> TaskStatus | None:
with factory() as db:
db.get(Task, ids["task"]).lease_expires_at = utc_now() - timedelta(seconds=5)
db.commit()
with factory() as db:
(candidate,) = [
c
for c in get_expired_task_lease_candidates(db, cutoff=utc_now(), limit=10)
if c.task_id == ids["task"]
]
status = recover_task_lease_candidate_no_commit(
db, candidate, recovered_at=utc_now()
)
db.commit()
return status
@pytest.mark.asyncio
@pytest.mark.parametrize("auto_resume", ["true", "false"])
async def test_pause_projects_like_lease_recovery(canonical, monkeypatch, auto_resume):
"""S2's pause leaves every projection as a lease-expiry pause does,
except the TriggerRun message, which names an interruption."""
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, _tid = canonical
paused, recovered = _projected_task(factory), _projected_task(factory)
paused_lease = _prepare_run(factory, paused)
recovered_lease = _prepare_run(factory, recovered)
frames = await _schedule_failing_turn(
factory, paused, paused_lease, _persistence_error()
)
assert _expire_and_recover(factory, recovered) == TaskStatus.PAUSED
left = _projection(factory, paused, paused_lease)
right = _projection(factory, recovered, recovered_lease)
_assert_recovers_like_lease_recovery(
factory,
paused,
recovered,
(left.pop("trigger_run"), right.pop("trigger_run")),
auto_resume=auto_resume,
)
assert TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR != TASK_LEASE_PAUSED_TRIGGER_ERROR
assert left == right
assert left["status"] == TaskStatus.PAUSED
assert left["output"] == "previous answer"
assert left["workforce_run"] == ("paused", True)
# The run may have consumed the message: "do not resend", never failed.
assert left["delivery"] == ["dispatched"]
assert frames == ["task_paused"] + (
["task_recovery_update"] if auto_resume == "true" else []
)
_task, row, _events = _state(factory, paused["task"])
assert row.reason == "persistence_failure"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("source", "state", "detail", "event"),
[
("trigger", "disabled", None, "interrupted"),
# Ineligibility wins over the switch.
("a2a", "ineligible", "ineligible:source_a2a", "ineligible"),
],
)
async def test_switch_off_settles_exactly_as_before(
canonical, monkeypatch, source, state, detail, event
):
"""Switch off: the same FAILED settlement as without any interruption,
plus a ``disabled`` row (``ineligible`` for an ineligible task)."""
monkeypatch.setenv(PAUSE_SWITCH, "false")
factory, _tid = canonical
gated = _projected_task(factory, source=source)
baseline = _projected_task(factory, source=source)
gated_lease = _prepare_run(factory, gated)
baseline_lease = _prepare_run(factory, baseline)
gated_frames = await _schedule_failing_turn(
factory, gated, gated_lease, _persistence_error()
)
with monkeypatch.context() as m:
_without_interruption(m)
baseline_frames = await _schedule_failing_turn(
factory, baseline, baseline_lease, _persistence_error()
)
gated_projection = _projection(factory, gated, gated_lease)
assert gated_projection == _projection(factory, baseline, baseline_lease)
assert gated_projection["status"] == TaskStatus.FAILED
assert gated_frames == baseline_frames == ["task_error"]
_task, row, events = _state(factory, gated["task"])
assert (row.reason, row.state, row.state_detail) == (
"persistence_failure",
state,
detail,
)
assert [(e.event, e.detail["task_status"]) for e in events] == [(event, "failed")]
assert _state(factory, baseline["task"])[1] is None
@pytest.mark.asyncio
async def test_ineligible_task_settles_exactly_as_before(canonical, monkeypatch):
factory, _tid = canonical
gated = _projected_task(factory, source="a2a")
baseline = _projected_task(factory, source="a2a")
gated_lease = _prepare_run(factory, gated)
baseline_lease = _prepare_run(factory, baseline)
await _schedule_failing_turn(factory, gated, gated_lease, _persistence_error())
with monkeypatch.context() as m:
_without_interruption(m)
await _schedule_failing_turn(
factory, baseline, baseline_lease, _persistence_error()
)
assert _projection(factory, gated, gated_lease) == _projection(
factory, baseline, baseline_lease
)
_task, row, events = _state(factory, gated["task"])
assert (row.state, row.state_detail) == ("ineligible", "ineligible:source_a2a")
assert [e.event for e in events] == ["ineligible"]
@pytest.mark.asyncio
@pytest.mark.parametrize("auto_resume", ["true", "false"])
@pytest.mark.parametrize("failure", ["python", "database", "marker"])
async def test_recording_failure_does_not_change_the_pause(
canonical, monkeypatch, failure, auto_resume
):
"""A failed recording settles the pause as a recorded ``manual`` one
would: the TriggerRun fails with the interruption message and nothing
announces a resume. Only a recorded, scheduled resume differs."""
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, _tid = canonical
recorded, unrecorded = _projected_task(factory), _projected_task(factory)
recorded_lease = _prepare_run(factory, recorded)
unrecorded_lease = _prepare_run(factory, unrecorded)
recorded_frames = await _schedule_failing_turn(
factory, recorded, recorded_lease, _persistence_error()
)
with monkeypatch.context() as m:
_fail_recording(m, failure)
unrecorded_frames = await _schedule_failing_turn(
factory, unrecorded, unrecorded_lease, _persistence_error()
)
left = _projection(factory, recorded, recorded_lease)
right = _projection(factory, unrecorded, unrecorded_lease)
assert right["trigger_run"] == (
TriggerRunStatus.FAILED.value,
TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR,
True,
)
assert unrecorded_frames == ["task_paused"]
if auto_resume == "true":
assert left.pop("trigger_run") == (TriggerRunStatus.RUNNING.value, None, False)
right.pop("trigger_run")
assert recorded_frames == ["task_paused", "task_recovery_update"]
else:
assert recorded_frames == ["task_paused"]
assert left == right
assert _state(factory, unrecorded["task"])[1:] == (None, [])
@pytest.mark.parametrize("auto_resume", ["true", "false"])
def test_resume_requested_run_pauses_like_lease_recovery(
canonical, monkeypatch, auto_resume
):
"""A RUNNING row already RESUME_REQUESTED is paused (control ``paused``)
by an interruption, exactly as lease-expiry recovery pauses it."""
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, _tid = canonical
settled, recovered = _projected_task(factory), _projected_task(factory)
settled_lease = _prepare_run(factory, settled)
recovered_lease = _prepare_run(factory, recovered)
with factory() as db:
for ids in (settled, recovered):
db.get(Task, ids["task"]).control_state = "resume_requested"
db.commit()
with factory() as db:
for ids in (settled, recovered):
ids["version"] = int(db.get(Task, ids["task"]).state_version)
assert _settle(settled_lease, interruption=_persistence_interruption())[0]
assert _expire_and_recover(factory, recovered) == TaskStatus.PAUSED
left = _projection(factory, settled, settled_lease)
right = _projection(factory, recovered, recovered_lease)
_assert_recovers_like_lease_recovery(
factory,
settled,
recovered,
(left.pop("trigger_run"), right.pop("trigger_run")),
auto_resume=auto_resume,
)
for side in (left, right):
side.pop("delivery") # Reconciled by the scheduler, not by settle.
assert left == right
assert (left["status"], left["control_state"]) == (TaskStatus.PAUSED, "paused")
_task, row, _events = _state(factory, settled["task"])
assert row.reason == "persistence_failure"
def _assert_recovers_like_lease_recovery(
factory,
settled: dict[str, Any],
recovered: dict[str, Any],
triggers: tuple[tuple[Any, ...], tuple[Any, ...]],
*,
auto_resume: str,
settled_error: str = TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR,
) -> None:
"""Settlement records and projects a pause as lease recovery does.
With automatic resume on both schedule it and keep the TriggerRun
RUNNING; with it off both leave it to the user and fail the TriggerRun,
each with its own message.
"""
left_trigger, right_trigger = triggers
if auto_resume == "true":
assert (
left_trigger
== right_trigger
== (
TriggerRunStatus.RUNNING.value,
None,
False,
)
)
expected = "scheduled"
else:
assert left_trigger == (TriggerRunStatus.FAILED.value, settled_error, True)
assert right_trigger == (
TriggerRunStatus.FAILED.value,
TASK_LEASE_PAUSED_TRIGGER_ERROR,
True,
)
expected = "manual"
_left_task, left_row, left_events = _state(factory, settled["task"])
_right_task, right_row, right_events = _state(factory, recovered["task"])
assert left_row.state == right_row.state == expected
assert [(e.event, e.attempt) for e in left_events] == [
(e.event, e.attempt) for e in right_events
]
def _finalize_and_release(factory, ids, lease, result) -> Any:
finalized = _finalize(factory, ids["task"], lease, result)
# The scheduler's settlement releases the lease after the result path.
assert settle_task_lease_isolated(lease)
return finalized
def _drop_reason(projection: dict[str, Any]) -> dict[str, Any]:
"""The settled fact carries the result itself, reason included."""
for fact in projection["settled"]:
assert fact["result"].pop("interruption_reason") == "persistence_failure"
return projection
@pytest.mark.parametrize("switch", ["on", "off"])
def test_quota_stop_wins_over_its_interruption_reason(canonical, monkeypatch, switch):
"""``quota_exceeded`` keeps the runner's reason but settles as a quota
stop: the same FAILED projections as a quota result without one."""
if switch == "off":
monkeypatch.setenv(PAUSE_SWITCH, "false")
factory, _tid = canonical
quota = {**PERSISTENCE_RESULT, "status": "quota_exceeded", "error_code": "q"}
plain_quota = {k: v for k, v in quota.items() if k != "interruption_reason"}
gated, baseline = _projected_task(factory), _projected_task(factory)
gated_lease = _prepare_run(factory, gated)
baseline_lease = _prepare_run(factory, baseline)
left = _finalize_and_release(factory, gated, gated_lease, quota)
right = _finalize_and_release(factory, baseline, baseline_lease, plain_quota)
assert left.report.paused_for is None
assert right.report.paused_for is None
gated_projection = _drop_reason(_projection(factory, gated, gated_lease))
assert gated_projection == _projection(factory, baseline, baseline_lease)
assert gated_projection["status"] == TaskStatus.FAILED
assert _state(factory, gated["task"])[1] is None
def test_result_switch_off_settles_exactly_as_before(canonical, monkeypatch):
monkeypatch.setenv(PAUSE_SWITCH, "0")
factory, _tid = canonical
plain = {k: v for k, v in PERSISTENCE_RESULT.items() if k != "interruption_reason"}
gated, baseline = _projected_task(factory), _projected_task(factory)
gated_lease = _prepare_run(factory, gated)
baseline_lease = _prepare_run(factory, baseline)
_finalize_and_release(factory, gated, gated_lease, dict(PERSISTENCE_RESULT))
_finalize_and_release(factory, baseline, baseline_lease, plain)
gated_projection = _drop_reason(_projection(factory, gated, gated_lease))
assert gated_projection == _projection(factory, baseline, baseline_lease)
assert gated_projection["status"] == TaskStatus.FAILED
_task, row, _events = _state(factory, gated["task"])
assert (row.reason, row.state) == ("persistence_failure", "disabled")
@pytest.mark.parametrize("auto_resume", ["true", "false"])
@pytest.mark.parametrize("failure", ["python", "database", "marker"])
def test_result_recording_failure_does_not_change_the_pause(
canonical, monkeypatch, failure, auto_resume
):
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, _tid = canonical
recorded, unrecorded = _projected_task(factory), _projected_task(factory)
recorded_lease = _prepare_run(factory, recorded)
unrecorded_lease = _prepare_run(factory, unrecorded)
left = _finalize_and_release(
factory, recorded, recorded_lease, dict(PERSISTENCE_RESULT)
)
with monkeypatch.context() as m:
_fail_recording(m, failure)
right = _finalize_and_release(
factory, unrecorded, unrecorded_lease, dict(PERSISTENCE_RESULT)
)
assert left.report.paused_for is not None
assert right.report.paused_for is not None
assert right.report.recovery is None
left_projection = _projection(factory, recorded, recorded_lease)
right_projection = _projection(factory, unrecorded, unrecorded_lease)
assert right_projection["trigger_run"] == (
TriggerRunStatus.FAILED.value,
TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR,
True,
)
if auto_resume == "true":
assert left_projection.pop("trigger_run") == (
TriggerRunStatus.RUNNING.value,
None,
False,
)
right_projection.pop("trigger_run")
assert left_projection == right_projection
assert left_projection["status"] == TaskStatus.PAUSED
assert _state(factory, unrecorded["task"])[1:] == (None, [])
def test_resumed_quota_result_settles_exactly_as_before(canonical):
factory, _tid = canonical
quota = {**PERSISTENCE_RESULT, "status": "quota_exceeded"}
plain_quota = {k: v for k, v in quota.items() if k != "interruption_reason"}
gated, baseline = _projected_task(factory), _projected_task(factory)
gated_lease = _prepare_run(factory, gated)
baseline_lease = _prepare_run(factory, baseline)
for ids, lease, result in (
(gated, gated_lease, quota),
(baseline, baseline_lease, plain_quota),
):
finalized = _finalize_resumed_task(
ids["task"],
status="quota_exceeded",
success=False,
output=result["output"],
task_owner_user_id=ids["user"],
result=dict(result),
task_lease=lease,
prepared_outputs=NO_OUTPUTS,
)
assert finalized["final_status"] == "failed"
assert finalized["report"].paused_for is None
gated_projection = _drop_reason(_projection(factory, gated, gated_lease))
assert gated_projection == _projection(factory, baseline, baseline_lease)
assert _state(factory, gated["task"])[1] is None
# --------------------------------------------- result paths, real database
def _finalize_resumed(ids: dict[str, Any], lease: TaskLease, result: dict[str, Any]):
return _finalize_resumed_task(
ids["task"],
status=str(result.get("status") or "failed"),
success=False,
output=result["output"],
task_owner_user_id=ids["user"],
result=dict(result),
task_lease=lease,
prepared_outputs=NO_OUTPUTS,
)
def _settle_result(path: str, factory, ids, lease, result) -> None:
"""Settle ``result`` through S1 (and the scheduler's release) or S3."""
if path == "result":
_finalize_and_release(factory, ids, lease, dict(result))
else:
assert _finalize_resumed(ids, lease, result)["lease_released"]
RESULT_PATHS = pytest.mark.parametrize("path", ["result", "resumed_result"])
@RESULT_PATHS
@pytest.mark.parametrize("auto_resume", ["true", "false"])
def test_result_pause_projects_like_lease_recovery(
canonical, monkeypatch, path, auto_resume
):
monkeypatch.setenv(AUTO_RESUME_SWITCH, auto_resume)
factory, _tid = canonical
paused, recovered = _projected_task(factory), _projected_task(factory)
paused_lease = _prepare_run(factory, paused)
recovered_lease = _prepare_run(factory, recovered)
_settle_result(path, factory, paused, paused_lease, PERSISTENCE_RESULT)
assert _expire_and_recover(factory, recovered) == TaskStatus.PAUSED
left = _projection(factory, paused, paused_lease)
right = _projection(factory, recovered, recovered_lease)
_assert_recovers_like_lease_recovery(
factory,
paused,
recovered,
(left.pop("trigger_run"), right.pop("trigger_run")),
auto_resume=auto_resume,
)
# Only lease recovery reconciles orphaned delivery rows.
left.pop("delivery")
right.pop("delivery")
assert left == right
assert left["settled"] == [PAUSED_FACT]
_task, row, _events = _state(factory, paused["task"])
assert row.reason == "persistence_failure"
@RESULT_PATHS
@pytest.mark.parametrize(
"case",
["unknown_tool_effect", "not_recoverable", "pause_requested", "resume_requested"],
)
def test_result_paths_decide_every_verdict(canonical, monkeypatch, path, case):
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids, checkpoint=case != "not_recoverable")
if case == "unknown_tool_effect":
monkeypatch.setattr(
task_auto_recovery,
"resolve_checkpoint_recovery_with_data",
lambda _db, _candidate: CheckpointRecoveryResolution(
CheckpointRecoveryVerdict.UNKNOWN_TOOL_EFFECT
),
)
if case in {"pause_requested", "resume_requested"}:
with factory() as db:
db.get(Task, ids["task"]).control_state = case
db.commit()
_settle_result(path, factory, ids, lease, PERSISTENCE_RESULT)
task, row, _events = _state(factory, ids["task"])
settled = _projection(factory, ids, lease)["settled"]
if case in {"unknown_tool_effect", "not_recoverable"}:
assert task.status == TaskStatus.FAILED
assert task.error_message == (
TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR
if case == "unknown_tool_effect"
else PERSISTENCE_RESULT["error"]
)
assert (row.reason, row.state) == (case, "manual")
assert [fact["status"] for fact in settled] == ["failed"]
else:
assert (task.status, task.control_state) == (TaskStatus.PAUSED, "paused")
assert task.runner_id is None
assert (row.reason, row.state) == (
"user_pause" if case == "pause_requested" else "persistence_failure",
"manual",
)
assert settled == [PAUSED_FACT]
@RESULT_PATHS
@pytest.mark.parametrize("case", ["switch_off", "ineligible"])
def test_result_paths_settle_exactly_as_before(canonical, monkeypatch, path, case):
if case == "switch_off":
monkeypatch.setenv(PAUSE_SWITCH, "false")
source = "a2a" if case == "ineligible" else "trigger"
factory, _tid = canonical
plain = {k: v for k, v in PERSISTENCE_RESULT.items() if k != "interruption_reason"}
gated = _projected_task(factory, source=source)
baseline = _projected_task(factory, source=source)
gated_lease = _prepare_run(factory, gated)
baseline_lease = _prepare_run(factory, baseline)
_settle_result(path, factory, gated, gated_lease, PERSISTENCE_RESULT)
_settle_result(path, factory, baseline, baseline_lease, plain)
gated_projection = _drop_reason(_projection(factory, gated, gated_lease))
assert gated_projection == _projection(factory, baseline, baseline_lease)
assert gated_projection["status"] == TaskStatus.FAILED
_task, row, _events = _state(factory, gated["task"])
assert (row.reason, row.state) == (
"persistence_failure",
"disabled" if case == "switch_off" else "ineligible",
)
assert _state(factory, baseline["task"])[1] is None
def test_resume_settlement_pauses_on_a_real_database(canonical):
"""The resume exception path's settlement against the real database."""
from xagent.web.services.task_execution import _settle_resumed_task_lease
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids)
report = SettlementReport()
assert _settle_resumed_task_lease(
lease,
error_message=RUN_ERROR,
interruption=_persistence_interruption(),
report=report,
)
assert report.paused_for is InterruptionReason.PERSISTENCE_FAILURE
projection = _projection(factory, ids, lease)
assert (projection["status"], projection["runner_id"]) == (TaskStatus.PAUSED, None)
assert projection["settled"] == [PAUSED_FACT]
assert projection["trigger_run"][1] == TASK_INTERRUPTION_PAUSED_TRIGGER_ERROR
# ----------------------------------------- undecidable interruptions defer to TTL
def _break_decision(m: pytest.MonkeyPatch, failure: str, error: Exception) -> None:
def boom(*_args: Any, **_kwargs: Any) -> Any:
raise error
if failure == "eligibility":
m.setattr(task_auto_recovery, "auto_recovery_eligibility", boom)
else:
m.setattr(event_recovery, "read_event_checkpoint", boom)
def _connection_reset() -> OperationalError:
return OperationalError("SELECT", {}, Exception("connection reset"))
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["exception", "result", "resumed_result"])
@pytest.mark.parametrize("failure", ["eligibility", "checkpoint_read"])
async def test_undecidable_interruption_keeps_the_lease_for_ttl(
canonical, monkeypatch, path, failure
):
"""A connectivity failure deciding the interruption defers it on every
settling path: the run stays RUNNING under its lease, and TTL recovery
pauses it once the database answers again."""
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids)
with monkeypatch.context() as m:
_break_decision(m, failure, _connection_reset())
if path == "exception":
frames = await _schedule_failing_turn(
factory, ids, lease, _persistence_error()
)
assert frames == []
else:
with pytest.raises(InterruptionSettlementDeferred) as deferred:
if path == "result":
_finalize(factory, ids["task"], lease, dict(PERSISTENCE_RESULT))
else:
_finalize_resumed(ids, lease, PERSISTENCE_RESULT)
assert deferred.value.__cause__ is not None
task, row, _events = _state(factory, ids["task"])
assert task.status == TaskStatus.RUNNING
assert (task.runner_id, task.run_id) == (lease.runner_id, lease.run_id)
assert row is None
assert _projection(factory, ids, lease)["settled"] == []
assert _expire_and_recover(factory, ids) == TaskStatus.PAUSED
_task, row, _events = _state(factory, ids["task"])
assert (row.reason, row.state) == ("lease_expired", "manual")
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["exception", "result", "resumed_result"])
@pytest.mark.parametrize("failure", ["eligibility", "checkpoint_read"])
async def test_reproducible_decision_fault_settles_as_before(
canonical, monkeypatch, path, failure
):
"""A fault that would recur on every attempt must not defer: TTL recovery
reads the checkpoint the same way, so the run would stay RUNNING forever.
It settles FAILED, as before this change, and releases the lease."""
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids)
with monkeypatch.context() as m:
_break_decision(m, failure, TypeError("resolver bug"))
if path == "exception":
await _schedule_failing_turn(factory, ids, lease, _persistence_error())
elif path == "result":
_finalize_and_release(factory, ids, lease, dict(PERSISTENCE_RESULT))
else:
_finalize_resumed(ids, lease, PERSISTENCE_RESULT)
task, _row, _events = _state(factory, ids["task"])
if path == "exception" and failure == "checkpoint_read":
# The legacy settlement itself classifies the run's tool effect from
# the same checkpoint (unknown_tool_effect_of_run) and has always
# kept the lease when that read raises; unchanged here.
assert task.status == TaskStatus.RUNNING
assert (task.runner_id, task.run_id) == (lease.runner_id, lease.run_id)
return
assert task.status == TaskStatus.FAILED
assert task.runner_id is None and task.lease_expires_at is None
# ----------------------------------------------------- broadcasts and metrics
@pytest.mark.asyncio
@pytest.mark.parametrize(
("reason", "message"),
[
(InterruptionReason.USER_PAUSE, TASK_USER_PAUSED_MESSAGE),
(InterruptionReason.PERSISTENCE_FAILURE, TASK_INTERRUPTION_PAUSED_MESSAGE),
],
)
async def test_pause_broadcast_says_who_paused(reason, message):
with patch(
"xagent.web.services.task_events.publish_task_event", new=AsyncMock()
) as published:
await publish_settlement_pause(
7,
SettlementReport(control_state={"status": "paused"}, paused_for=reason),
)
event = published.await_args.args[0]
assert (event["type"], event["message"], event["status"]) == (
"task_paused",
message,
"paused",
)
assert event["interruption_reason"] == reason.value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"report",
[SettlementReport(), SettlementReport(control_state={"status": "failed"})],
)
async def test_pause_broadcast_skips_a_report_without_a_pause(report):
with patch(
"xagent.web.services.task_events.publish_task_event", new=AsyncMock()
) as published:
await publish_settlement_pause(7, report)
published.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(("switch", "failed"), [("true", False), ("false", True)])
async def test_trigger_metrics_do_not_count_a_pause_as_a_failure(
canonical, monkeypatch, switch, failed
):
from xagent.core.utils import setup_metrics
monkeypatch.setenv(PAUSE_SWITCH, switch)
finished: list[bool] = []
monkeypatch.setattr(setup_metrics.trigger_execution, "start", lambda: 0.0)
monkeypatch.setattr(
setup_metrics.trigger_execution,
"finish",
lambda _started, *, cancelled, failed: finished.append(failed),
)
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids)
await _schedule_failing_turn(factory, ids, lease, _persistence_error())
await asyncio.sleep(0)
assert finished == [failed]
@pytest.mark.asyncio
async def test_explicit_cancel_is_not_settled_as_an_interruption():
settle_mock = MagicMock(return_value=True)
publish_pause, _frames = await _resume_raising(
ExecutionEventPersistenceError("event write failed"),
settle_mock,
explicit_cancel=True,
)
settle_mock.assert_called_once()
assert settle_mock.call_args.kwargs["interruption"] is None
publish_pause.assert_not_awaited()
# --------------------------------------------- scheduled automatic resume
def _recovery_updates(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [event for event in events if event.get("type") == "task_recovery_update"]
@pytest.mark.asyncio
async def test_exception_pause_schedules_a_resume_and_announces_it(
canonical, monkeypatch
):
"""S2: the pause records ``scheduled``, keeps the TriggerRun RUNNING and,
after the commit, announces the pause and then the scheduled resume."""
monkeypatch.setenv(AUTO_RESUME_SWITCH, "true")
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids)
events: list[dict[str, Any]] = []
frames = await _schedule_failing_turn(
factory, ids, lease, _persistence_error(), events=events
)
assert frames == ["task_paused", "task_recovery_update"]
projection = _projection(factory, ids, lease)
assert projection["status"] == TaskStatus.PAUSED
assert projection["trigger_run"] == (TriggerRunStatus.RUNNING.value, None, False)
assert projection["workforce_run"] == ("paused", True)
task, row, recorded = _state(factory, ids["task"])
assert (row.reason, row.state) == ("persistence_failure", "scheduled")
assert [event.event for event in recorded] == ["interrupted", "scheduled"]
(update,) = _recovery_updates(events)
assert (update["task_id"], update["run_id"], update["state_version"]) == (
ids["task"],
lease.run_id,
task.state_version,
)
assert update["auto_resume"]["status"] == "scheduled"
assert update["auto_resume"]["reason"] == "persistence_failure"
assert update["auto_resume"]["attempt"] == 1
@RESULT_PATHS
def test_result_pause_reports_the_scheduled_resume(canonical, monkeypatch, path):
"""S1 and S3 carry the recorded resume in the report they announce."""
monkeypatch.setenv(AUTO_RESUME_SWITCH, "true")
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids)
if path == "result":
report = _finalize_and_release(
factory, ids, lease, dict(PERSISTENCE_RESULT)
).report
else:
report = _finalize_resumed(ids, lease, PERSISTENCE_RESULT)["report"]
task, row, _events = _state(factory, ids["task"])
assert report.paused_for is InterruptionReason.PERSISTENCE_FAILURE
assert report.recovery is not None
assert (report.recovery.state, report.recovery.task_id) == (
"scheduled",
ids["task"],
)
assert report.recovery.paused_state_version == task.state_version
assert report.recovery.next_attempt_at is not None
assert row.state == "scheduled"
projection = _projection(factory, ids, lease)
assert projection["trigger_run"] == (TriggerRunStatus.RUNNING.value, None, False)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("state", "detail", "expected"),
[
(
"scheduled",
None,
[("task_paused", None), ("task_recovery_update", "scheduled")],
),
(
"exhausted",
None,
[("task_paused", None), ("task_recovery_update", "stopped")],
),
("manual", "auto_resume_disabled", [("task_paused", None)]),
("manual", "supersede_check_failed", [("task_paused", None)]),
("ineligible", "ineligible:source_a2a", [("task_paused", None)]),
],
)
async def test_pause_broadcast_announces_only_a_planned_resume(state, detail, expected):
from datetime import datetime, timezone
from xagent.web.services.task_settlement_report import RecordedInterruption
recovery = RecordedInterruption(
task_id=7,
run_id="run-7",
reason="persistence_failure",
state=state,
state_detail=detail,
paused_state_version=4,
next_attempt_at=(
datetime(2026, 10, 10, 8, 0, 12, tzinfo=timezone.utc)
if state == "scheduled"
else None
),
attempt=1,
)
with patch(
"xagent.web.services.task_events.publish_task_event", new=AsyncMock()
) as published:
await publish_settlement_pause(
7,
SettlementReport(
control_state={"status": "paused"},
paused_for=InterruptionReason.PERSISTENCE_FAILURE,
recovery=recovery,
),
)
events = [call.args[0] for call in published.await_args_list]
assert [
(event["type"], event.get("auto_resume", {}).get("status")) for event in events
] == expected
@pytest.mark.asyncio
async def test_pause_broadcast_failure_still_announces_the_resume():
from xagent.web.services.task_settlement_report import RecordedInterruption
sent: list[str] = []
async def flaky(event: dict[str, Any], _task_id: int) -> None:
sent.append(event["type"])
if event["type"] == "task_paused":
raise RuntimeError("socket gone")
recovery = RecordedInterruption(
task_id=7,
run_id="run-7",
reason="lease_expired",
state="scheduled",
state_detail=None,
paused_state_version=4,
next_attempt_at=None,
attempt=1,
)
with patch("xagent.web.services.task_events.publish_task_event", flaky):
await publish_settlement_pause(
7,
SettlementReport(
paused_for=InterruptionReason.PERSISTENCE_FAILURE, recovery=recovery
),
)
assert sent == ["task_paused", "task_recovery_update"]
@pytest.mark.asyncio
async def test_new_run_result_pause_announces_its_resume_through_the_runner(
canonical, monkeypatch
):
"""S1 end to end through ``execute_task_background``'s own publish
point: the committed pause, then the scheduled resume."""
from xagent.web.services import task_execution as task_execution_service
monkeypatch.setenv(AUTO_RESUME_SWITCH, "true")
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids, checkpoint=False)
monkeypatch.setattr(
"xagent.web.services.task_setup_snapshot.get_session_local", lambda: factory
)
from xagent.web.services.task_setup_snapshot import load_task_setup_snapshot_sync
# Loaded before the run writes its checkpoint, as the scheduler does.
snapshot = load_task_setup_snapshot_sync(ids["task"], ids["user"])
_checkpoint(factory, ids["task"], lease.run_id)
published: list[dict[str, Any]] = []
async def publish(event: dict[str, Any], _task_id: int) -> None:
published.append(event)
class Manager:
def __getattr__(self, _name: str) -> Any:
return MagicMock()
async def wait_for_previous(self, _task_id: int) -> None:
return None
class AgentService(MagicMock):
pass
class AgentManager:
async def get_agent_for_task(self, *_args: Any, **_kwargs: Any) -> Any:
return AgentService()
async def execute_task(self, **_kwargs: Any) -> dict[str, Any]:
return dict(PERSISTENCE_RESULT)
monkeypatch.setattr(task_execution_service, "background_task_manager", Manager())
monkeypatch.setattr("xagent.web.services.task_events.publish_task_event", publish)
monkeypatch.setattr(
"xagent.web.services.task_event_display.publish_task_result", publish
)
await task_execution_service.execute_task_background(
task_id=ids["task"],
user_message="2+2",
context={"turn_id": ids["turn_id"]},
agent_manager=AgentManager(),
task_owner_user_id=ids["user"],
llm_user_message="2+2",
task_setup_snapshot=snapshot,
expected_run_id=lease.run_id,
task_lease=lease,
resolved_execution_scope=None,
)
types = [event.get("type") for event in published]
assert "task_paused" in types and "task_recovery_update" in types, types
assert types.index("task_paused") < types.index("task_recovery_update")
update = published[types.index("task_recovery_update")]
task, row, _events = _state(factory, ids["task"])
assert row.state == "scheduled"
assert (update["task_id"], update["run_id"], update["state_version"]) == (
ids["task"],
lease.run_id,
task.state_version,
)
assert update["auto_resume"]["status"] == "scheduled"
@pytest.mark.asyncio
async def test_run_pool_timeout_leaves_the_unsettled_record_to_its_scheduler(
canonical, monkeypatch
):
"""``execute_task_background`` re-raises a pool timeout without recording.
The run lease belongs to ``_schedule_bg``, which defers that same timeout
to TTL recovery and records it once (``test_task_orchestrator``'s
``UNSETTLED_SCENARIOS``). A second record here would count one retained
lease twice.
"""
from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError
from xagent.web.services import release_hold as release_hold_module
from xagent.web.services import task_execution as task_execution_service
factory, _tid = canonical
ids = _projected_task(factory)
lease = _prepare_run(factory, ids, checkpoint=False)
monkeypatch.setattr(
"xagent.web.services.task_setup_snapshot.get_session_local", lambda: factory
)
from xagent.web.services.task_setup_snapshot import load_task_setup_snapshot_sync
snapshot = load_task_setup_snapshot_sync(ids["task"], ids["user"])
class Manager:
def __getattr__(self, _name: str) -> Any:
return MagicMock()
async def wait_for_previous(self, _task_id: int) -> None:
return None
class AgentManager:
async def get_agent_for_task(self, *_args: Any, **_kwargs: Any) -> Any:
return MagicMock()
async def execute_task(self, **_kwargs: Any) -> dict[str, Any]:
raise SQLAlchemyTimeoutError("QueuePool limit reached")
async def publish(_event: dict[str, Any], _task_id: int) -> None:
return None
monkeypatch.setattr(task_execution_service, "background_task_manager", Manager())
monkeypatch.setattr("xagent.web.services.task_events.publish_task_event", publish)
monkeypatch.setattr(
"xagent.web.services.task_event_display.publish_task_result", publish
)
before = release_hold_module.release_hold.unsettled_total
with pytest.raises(SQLAlchemyTimeoutError):
await task_execution_service.execute_task_background(
task_id=ids["task"],
user_message="2+2",
context={"turn_id": ids["turn_id"]},
agent_manager=AgentManager(),
task_owner_user_id=ids["user"],
llm_user_message="2+2",
task_setup_snapshot=snapshot,
expected_run_id=lease.run_id,
task_lease=lease,
resolved_execution_scope=None,
)
assert release_hold_module.release_hold.unsettled_total == before